From 5e3284c18cee2810d9170c6b13ae5df16a39e48c Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Wed, 4 Nov 2026 10:46:50 +0800 Subject: [PATCH] [feat] agent: unify queries and improve trace presentation Signed-off-by: Weiming Liao --- .github/workflows/ci.yml | 3 + apps/api/src/opendirector_api/agent.py | 6 +- apps/api/src/opendirector_api/creation.py | 18 +- docs/architecture/agent-runtime.md | 2 +- docs/architecture/unified-project.md | 2 + .../core/src/opendirector/agent/deepseek.py | 7 +- .../core/src/opendirector/agent/gateway.py | 5 +- .../core/src/opendirector/agent/native.py | 2 +- packages/core/src/opendirector/agent/tools.py | 98 +- .../application/creation_canvas.py | 29 +- .../src/opendirector/domain/creation/base.py | 4 + .../opendirector/domain/creation/canvas.py | 21 +- .../opendirector/domain/creation/commands.py | 61 +- .../domain/creation/generation.py | 28 +- .../opendirector/domain/creation/models.py | 4 + .../src/opendirector/domain/input_rules.py | 2 + src/backend/chat.ts | 6 + src/backend/rules.ts | 5 +- src/canvas/AssistantReply.tsx | 85 +- src/canvas/CanvasWorkspace.tsx | 31 +- src/canvas/assistant-experience.css | 11 +- src/canvas/assistantActivity.ts | 58 + src/canvas/selectionPresentation.ts | 9 + src/canvas/useAssistantChat.ts | 9 +- src/canvas/useSelectionPresentation.ts | 19 +- src/contracts/creation.schema.json | 1224 ++++++++++++++--- src/contracts/generated.ts | 148 +- tests/assistantActivity.test.ts | 54 + tests/browser/assistant_activity.html | 8 + tests/browser/assistant_activity.mjs | 79 ++ tests/browser/assistant_activity_fixture.tsx | 15 + tests/browser/canvas_generation_reference.mjs | 66 + tests/browser/created_node_focus.mjs | 88 ++ tests/canvasSelectionPresentation.test.ts | 15 + tests/creationParameters.test.ts | 16 + tests/test_agent_artifacts.py | 58 +- tests/test_agent_gateway.py | 2 +- tests/test_agent_metadata.py | 49 +- tests/test_creation_canvas_service.py | 45 + tests/test_creation_contracts.py | 30 + tests/test_deepseek_runtime.py | 9 +- tests/test_native_runtime.py | 4 +- 42 files changed, 2074 insertions(+), 361 deletions(-) create mode 100644 src/canvas/assistantActivity.ts create mode 100644 tests/assistantActivity.test.ts create mode 100644 tests/browser/assistant_activity.html create mode 100644 tests/browser/assistant_activity.mjs create mode 100644 tests/browser/assistant_activity_fixture.tsx create mode 100644 tests/browser/canvas_generation_reference.mjs create mode 100644 tests/browser/created_node_focus.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07ef5e46c..024a27c62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,9 @@ jobs: curl --fail -H 'X-OpenDirector-Request: 1' "$CREATION_API_URL/api/creation-test-health" curl --silent --fail "$CREATION_WEB_URL/" >/dev/null node scripts/verify_creation_browser.mjs + node tests/browser/assistant_activity.mjs + node tests/browser/canvas_generation_reference.mjs + node tests/browser/created_node_focus.mjs node tests/browser/reference_reconnect.mjs node tests/browser/context_menu_dismissal.mjs node tests/browser/mp4_export.mjs diff --git a/apps/api/src/opendirector_api/agent.py b/apps/api/src/opendirector_api/agent.py index ea9e173be..ec5a9d5f1 100644 --- a/apps/api/src/opendirector_api/agent.py +++ b/apps/api/src/opendirector_api/agent.py @@ -252,6 +252,7 @@ async def turn(project_id: UUID, value: Turn, x_opendirector_session: UUID = Hea async def produce() -> None: text, status, error, accepted = '', 'complete', None, False activities = [] + plan = None try: user = {'id': str(value.id), 'role': 'user', 'text': value.text, 'content': value.content, 'assets': [{'id': str(assets[key].id), 'name': assets[key].name, 'type': assets[key].metadata.mimeType, @@ -282,6 +283,8 @@ async def produce() -> None: async for event in stream: if event['type'] == 'delta': text += event['text'] + if event['type'] == 'plan': + plan = event['entries'] if event['type'] == 'tool': previous = next((item for item in activities if item['id'] == event['id']), None) activity = {**{key: item for key, item in event.items() if key != 'type'}, @@ -308,7 +311,8 @@ async def produce() -> None: saved = await asyncio.to_thread(conversations.update, project_id, finish={ 'id': str(uuid5(value.id, 'assistant')), 'role': 'assistant', 'text': text, 'status': status, 'model': model, - 'runtime': value.runtime, 'thinking': value.thinking, 'tools': activities}, conversation_id=value.conversationId) + 'runtime': value.runtime, 'thinking': value.thinking, 'tools': activities, + **({'plan': plan} if plan is not None else {})}, conversation_id=value.conversationId) else: saved = None if asyncio.current_task().cancelling(): diff --git a/apps/api/src/opendirector_api/creation.py b/apps/api/src/opendirector_api/creation.py index b793d977a..665ce3fdb 100644 --- a/apps/api/src/opendirector_api/creation.py +++ b/apps/api/src/opendirector_api/creation.py @@ -6,7 +6,7 @@ from urllib.parse import quote, unquote from fastapi import APIRouter, Header, Query, Request -from fastapi.responses import Response, StreamingResponse +from fastapi.responses import FileResponse, Response, StreamingResponse from starlette.background import BackgroundTask from starlette.concurrency import run_in_threadpool @@ -85,10 +85,22 @@ def projection(project_id: UUID): return canvas.projection(project_id) @router.post('/projects/{project_id}/queries') - def query(project_id: UUID, value: CreationQuery): + async def query(project_id: UUID, value: CreationQuery): if value.projectId != project_id: raise ProjectError('查询不属于当前工程。') - return canvas.query(value) + result = await canvas.query(value) + if value.type == 'asset.preview' and 'preview' in result: + result['preview'].pop('path') + result['preview']['url'] = f'/api/creation/projects/{project_id}/assets/{value.objectId}/preview' + return result + + @router.get('/projects/{project_id}/assets/{asset_id}/preview') + async def asset_preview(project_id: UUID, asset_id: UUID): + target = projects.local.state_dir / 'previews' / str(project_id) + result = await run_in_threadpool(assets.preview, project_id, asset_id, target) + if 'preview' not in result: + return result + return FileResponse(result['preview']['path'], media_type=result['preview']['mimeType']) @router.get('/projects/{project_id}/history') def operations(project_id: UUID): diff --git a/docs/architecture/agent-runtime.md b/docs/architecture/agent-runtime.md index 1d0a19d5d..1069b23ae 100644 --- a/docs/architecture/agent-runtime.md +++ b/docs/architecture/agent-runtime.md @@ -37,7 +37,7 @@ Default 与 OpenCode CLI 分别配置。外部 CLI 不读取 OpenDirector 保存 适配层为每轮创建 `agent-runs//work/`，启动 harness 并连接本轮 MCP 工具入口。原生工具以此目录为工作目录，通过继承的 PATH 查找 FFmpeg、ffprobe 等程序。 -业务工具分为工程查询 `read_project`、技能或文本素材读取 `read`、命令契约查询 `command_schema` 和命令执行 `execute_command`。Gateway 按调用身份保留任务和结果：相同身份、相同输入复用结果，不同输入拒绝执行。 +业务工具分为工程与模型概览 `read_project`、对象／状态／素材查询 `query`、技能或文本素材读取 `read`、命令契约查询 `command_schema` 和命令执行 `execute_command`。Gateway 按调用身份保留任务和结果：相同身份、相同输入复用结果，不同输入拒绝执行。 新文件通过 `artifact.publish` 登记，再根据需要调用 `canvas.show` 展示。发布回执、素材登记和画布展示分别表达，不以创建节点代替文件交付。 diff --git a/docs/architecture/unified-project.md b/docs/architecture/unified-project.md index 5c3821235..6960f9d4c 100644 --- a/docs/architecture/unified-project.md +++ b/docs/architecture/unified-project.md @@ -110,6 +110,8 @@ UI、Agent 和后台完成写入共享可恢复变更通道。通知提示对象 自动聚焦仅发生在节点切换或输入面板重新打开时。参数调整、保存回执、后台及 Agent 更新不抢走用户正在操作的控件焦点。输入框持有焦点时，方向键、删除与选择快捷键作用于文本；多选、拖动和无生成提示词的节点沿用画布交互。 +同步新增的画布节点进入可见区域时，视口可居中展示新节点，并避让侧栏与 AI 对话面板；批量新增按整体范围展示。跟随不修改节点位置、选择或输入焦点。用户拖动、缩放、多选或编辑时不自动移动视口，结束操作后不补做已跳过的跟随。已有节点的状态更新不触发重新定位。 + 焦点属于 ViewState，不改变业务对象版本、生成意图或采用关系。 ## 模型与渲染边界 diff --git a/packages/core/src/opendirector/agent/deepseek.py b/packages/core/src/opendirector/agent/deepseek.py index e99a47c80..c9a28b1ae 100644 --- a/packages/core/src/opendirector/agent/deepseek.py +++ b/packages/core/src/opendirector/agent/deepseek.py @@ -166,6 +166,11 @@ def events(record, calls): return [] update = record['params']['update'] kind = update['sessionUpdate'] + if kind == 'agent_thought_chunk': + return [{'type': 'thinking'}] + if kind == 'plan': + return [{'type': 'plan', 'entries': [ + {'content': entry['content'], 'status': entry['status']} for entry in update['entries']]}] if kind == 'agent_message_chunk' and update['content']['type'] == 'text': return [{'type': 'delta', 'text': update['content']['text']}] if kind not in {'tool_call', 'tool_call_update'}: @@ -181,4 +186,4 @@ def events(record, calls): 'result': detail[:65536], 'resultTruncated': len(detail) > 65536} calls[identity] = event # Shared business activity is reported from authoritative gateway receipts. - return [] if any(name in {f'opendirector.{tool}', f'mcp__opendirector__{tool}'} for tool in ('read_project', 'read_canvas_item', 'preview_asset', 'read', 'command_schema', 'execute_command')) else [event] + return [] if any(name in {f'opendirector.{tool}', f'mcp__opendirector__{tool}'} for tool in ('read_project', 'query', 'read', 'command_schema', 'execute_command')) else [event] diff --git a/packages/core/src/opendirector/agent/gateway.py b/packages/core/src/opendirector/agent/gateway.py index 2a0136f69..c603ef454 100644 --- a/packages/core/src/opendirector/agent/gateway.py +++ b/packages/core/src/opendirector/agent/gateway.py @@ -61,7 +61,7 @@ async def execute(self, call_id: str, name: str, arguments: dict[str, JsonValue] raise ProjectError('工具调用标识或名称无效。') arguments = TypeAdapter(dict[str, JsonValue]).validate_python(arguments, strict=True) schema = next(item['function']['parameters'] for item in tools() if item['function']['name'] == name) - if set(arguments) - set(schema['properties']) or set(schema.get('required', [])) - set(arguments): + if 'properties' in schema and (set(arguments) - set(schema['properties']) or set(schema.get('required', [])) - set(arguments)): raise ProjectError('工具参数缺失或包含未定义字段。') value = await self.business.execute(self.run.project_id, self.run.turn_id, self.run.session, call_id, name, arguments, self.roots, self.run.config, @@ -91,6 +91,7 @@ async def execute(self, call_id: str, name: str, arguments: dict[str, JsonValue] async def close(self) -> None: self.closed = True for identity, task in self.calls.values(): - if json.loads(identity)[0] == 'read_canvas_item': + name, arguments = json.loads(identity) + if name == 'query' and arguments.get('type') == 'canvas.status': task.cancel() await asyncio.gather(*(task for _, task in self.calls.values()), return_exceptions=True) diff --git a/packages/core/src/opendirector/agent/native.py b/packages/core/src/opendirector/agent/native.py index eb8f6fe4a..9c9d0fd9d 100644 --- a/packages/core/src/opendirector/agent/native.py +++ b/packages/core/src/opendirector/agent/native.py @@ -146,7 +146,7 @@ async def connection(reader, writer): outcome = await gateway.call('mcp:' + key, params['name'], params.get('arguments', {})) result = {'content': [{'type': 'text', 'text': json.dumps(outcome.value, ensure_ascii=False)}], 'isError': bool(outcome.error)} - if params['name'] == 'preview_asset' and not outcome.error: + if params['name'] == 'query' and params.get('arguments', {}).get('type') == 'asset.preview' and not outcome.error: preview = outcome.value.get('preview') if preview: content = base64.b64encode(await asyncio.to_thread(Path(preview['path']).read_bytes)).decode() diff --git a/packages/core/src/opendirector/agent/tools.py b/packages/core/src/opendirector/agent/tools.py index 00aa81716..a0eaf530c 100644 --- a/packages/core/src/opendirector/agent/tools.py +++ b/packages/core/src/opendirector/agent/tools.py @@ -1,5 +1,6 @@ """Business commands shared by every agent runtime.""" import asyncio +import shutil from pathlib import Path from uuid import UUID, uuid5 @@ -10,9 +11,11 @@ from opendirector.application.creation_canvas import CreationCanvas from opendirector.domain.creation.assets import AssetProvenance from opendirector.domain.creation.base import ObjectVersion -from opendirector.domain.creation.commands import AssetPreviewQuery, CanvasItemQuery, COMMAND_MODELS, parse_command +from opendirector.domain.creation.commands import Query, COMMAND_MODELS, parse_command from opendirector.domain.creation.models import ModelSnapshot from opendirector.domain.projects import ProjectError +from opendirector.infrastructure.creation_projects import directory +from opendirector.infrastructure.local_projects import child ROOT = Path(__file__).resolve().parents[5] EXCLUDED = {'project.create', 'project.trash', 'project.restore', 'asset.import', 'job.checkpoint', @@ -45,13 +48,16 @@ def tools() -> list[dict]: {'type': 'object', 'properties': {}, 'additionalProperties': False}), tool_schema('read', 'Read a discovered skill file/resource, or text asset via asset:. No scripts.', {'type': 'object', 'properties': {'path': {'type': 'string'}}, 'required': ['path'], 'additionalProperties': False})] - result += [tool_schema('read_canvas_item', 'Read a canvas item, generation status and current asset; optionally wait for changes.', - CanvasItemQuery.model_json_schema()), - tool_schema('preview_asset', 'Preview an asset by ID: image thumbnail, video first frame, audio first 30 seconds or text excerpt.', - AssetPreviewQuery.model_json_schema())] + query_schema = TypeAdapter(Query).json_schema() + query_schema['type'] = 'object' + for definition in query_schema['$defs'].values(): + for field in ('queryId', 'projectId'): + definition['properties'].pop(field) + definition['required'].remove(field) + result.append(tool_schema('query', 'Query objects, jobs, assets or versions. canvas.status can wait; asset.preview provides media for native viewing.', query_schema)) command_type = {'type': 'string', 'enum': list(COMMANDS)} - result.append(tool_schema('command_schema', 'Read the exact input schema before executing a project command.', - {'type': 'object', 'properties': {'type': command_type}, 'required': ['type'], 'additionalProperties': False})) + result.append(tool_schema('command_schema', 'Read command inputs; pass modelConfigId for a compact schema constrained to the chosen model.', + {'type': 'object', 'properties': {'type': command_type, 'modelConfigId': {'type': 'string', 'format': 'uuid', 'description': 'Chosen generation model; narrows media kind and supported parameters.'}}, 'required': ['type'], 'additionalProperties': False})) versions = TypeAdapter(list[ObjectVersion]).json_schema() definitions = versions.pop('$defs') result.append(tool_schema('execute_command', 'Execute a validated project command. Read command_schema first. IDs and project scope come from the host.', @@ -74,15 +80,22 @@ async def execute(self, project_id: UUID, turn_id: UUID, session: str, call_id: snapshot = await asyncio.to_thread(self.projects.snapshot, project_id) return {'project': snapshot.model_dump(mode='json'), 'models': self.configs.list(), 'commands': list(COMMANDS), 'commandMetadata': {kind: {'description': model.__doc__} for kind, model in COMMANDS.items()}} - if name == 'read_canvas_item': - query = CanvasItemQuery.model_validate(arguments) - return await CreationCanvas(self.projects).wait_item_status(project_id, query.itemId, query.waitSeconds) - if name == 'preview_asset': - query = AssetPreviewQuery.model_validate(arguments) - with self.projects.local.transaction() as db: - directory, _, _ = self.projects._load(db, project_id) - target = directory / 'agent-runs' / str(turn_id) / 'work' / 'previews' - return await asyncio.to_thread(self.assets.preview, project_id, query.assetId, target) + if name == 'query': + if {'projectId', 'queryId'} & arguments.keys(): + raise ProjectError('查询身份与工程范围由宿主提供。') + query = TypeAdapter(Query).validate_python({**arguments, 'projectId': project_id, + 'queryId': uuid5(turn_id, call_id)}) + result = await CreationCanvas(self.projects).query(query) + if query.type == 'asset.preview' and 'preview' in result: + with self.projects.local.transaction() as db: + project_directory, _, _ = self.projects._load(db, project_id) + target = directory(project_directory / 'agent-runs' / str(turn_id) / 'work' / 'previews') + target.mkdir(parents=True, exist_ok=True) + source = Path(result['preview']['path']) + destination = child(target, source.name) + await asyncio.to_thread(shutil.copyfile, source, destination) + result['preview']['path'] = str(destination.resolve()) + return result if name == 'read': path = TypeAdapter(str).validate_python(arguments['path'], strict=True) if path.startswith('asset:'): @@ -111,6 +124,55 @@ async def execute(self, project_id: UUID, turn_id: UUID, session: str, call_id: for field in ('commandId', 'projectId'): schema['properties'].pop(field) schema['required'] = [key for key in schema['required'] if key not in {'commandId', 'projectId'}] + if arguments.get('modelConfigId'): + config_id = str(UUID(arguments['modelConfigId'])) + config = next((value for value in self.configs.list() if value['id'] == config_id), None) + if config is None: + raise ProjectError('所选模型不可用。') + media_kind = 'text' if config['kind'] == 'agent' else config['kind'] + def narrow(value: JsonValue) -> None: + if isinstance(value, dict): + discriminator = value.get('discriminator', {}) + mapping = discriminator.get('mapping', {}) + if discriminator.get('propertyName') == 'kind' and media_kind in mapping: + value['$ref'] = mapping[media_kind] + value.pop('oneOf') + value.pop('discriminator') + for child in value.values(): + narrow(child) + elif isinstance(value, list): + for child in value: + narrow(child) + narrow(schema) + parameters = schema.get('$defs', {}).get('VideoParameters') + if parameters and media_kind == 'video': + for field, choices in [('videoMode', config['inputRules']['videoModes']), + ('generateAudio', config['inputRules']['generateAudioOptions'])]: + prop = parameters['properties'][field] + prop['enum'] = choices + if prop['default'] not in choices: + prop.pop('default') + parameters.setdefault('required', []).append(field) + spec = schema['$defs']['VideoGenerationSpec'] + if 'parameters' not in spec.get('required', []): + spec.setdefault('required', []).append('parameters') + definitions = schema.pop('$defs', {}) + used: set[str] = set() + def references(value: JsonValue) -> None: + if isinstance(value, dict): + if '$ref' in value: + key = value['$ref'].rsplit('/', 1)[1] + if key not in used: + used.add(key) + references(definitions[key]) + for child in value.values(): + references(child) + elif isinstance(value, list): + for child in value: + references(child) + references(schema) + if used: + schema['$defs'] = {key: value for key, value in definitions.items() if key in used} return schema if name != 'execute_command': raise ProjectError('未知业务工具。') @@ -129,9 +191,9 @@ async def execute(self, project_id: UUID, turn_id: UUID, session: str, call_id: provenance = AssetProvenance(source='agent_written', agentRunId=turn_id, agentRuntime=native_runtime) if command.type == 'artifact.publish': with self.projects.local.transaction() as db: - directory, _, _ = self.projects._load(db, project_id) + project_directory, _, _ = self.projects._load(db, project_id) receipt = await asyncio.to_thread(publish_artifact, self.assets, command, - execution_directory=directory / 'agent-runs' / str(turn_id) / 'work', session=session, provenance=provenance) + execution_directory=project_directory / 'agent-runs' / str(turn_id) / 'work', session=session, provenance=provenance) else: receipt = await asyncio.to_thread(self.assets.execute, command, session=session, source='agent', provenance=provenance) else: diff --git a/packages/core/src/opendirector/application/creation_canvas.py b/packages/core/src/opendirector/application/creation_canvas.py index 3c7597736..c92b25e06 100644 --- a/packages/core/src/opendirector/application/creation_canvas.py +++ b/packages/core/src/opendirector/application/creation_canvas.py @@ -6,11 +6,16 @@ import asyncio from uuid import UUID +from pydantic import JsonValue +from pydantic_core import to_jsonable_python + +from opendirector.application.creation_assets import CreationAssets + from opendirector.domain.creation.assets import AssetRef -from opendirector.domain.creation.base import ObjectVersion -from opendirector.domain.creation.commands import CanvasShow, ReferenceWrite, SpecWrite -from opendirector.domain.creation.generation import ResolvedReference -from opendirector.domain.creation.canvas import media_card_size, node_capabilities, VideoCanvasItem +from opendirector.domain.creation.base import CanvasItemVersion, ObjectVersion +from opendirector.domain.creation.commands import CanvasShow, Query, ReferenceWrite, SpecWrite +from opendirector.domain.creation.generation import FollowingSource, ResolvedReference +from opendirector.domain.creation.canvas import CanvasReferenceSources, FollowingReferenceSource, PinnedReferenceSource, media_card_size, node_capabilities, VideoCanvasItem from opendirector.domain.creation.models import ModelConfigPublic from opendirector.domain.creation.project import ProjectSnapshot, asset_references from opendirector.domain.creation.validation import ContractError @@ -618,7 +623,11 @@ def item_status(self, project_id: UUID, item_id: UUID) -> dict: active = [job for job in related if job.status in {'queued', 'running'}] jobs = active[:50] if active else related[:1] asset = current_asset(snapshot, item) - return {'item': item.model_dump(mode='json'), + reference_sources = CanvasReferenceSources( + followCurrent=FollowingReferenceSource(source=FollowingSource(id=item.id)), + pinned=PinnedReferenceSource(source=asset_ref(asset), sourceVersion=CanvasItemVersion( + objectType='canvas_item', id=item.id, revision=item.revision))) if asset else None + return {'referenceSources': reference_sources.model_dump(mode='json') if reference_sources else None, 'item': item.model_dump(mode='json'), 'generationSpec': {'id': str(spec.id), 'revision': spec.revision, 'modelConfigId': str(spec.modelConfigId) if spec.modelConfigId else None, 'currentResultId': str(spec.currentResultId) if spec.currentResultId else None} if spec else None, @@ -718,7 +727,15 @@ def check_asset(asset): return dict(snapshot=snapshot, visibleItemIds=[i.id for i in snapshot.canvasItems if i.id in visible], hiddenItemIds=[i.id for i in snapshot.canvasItems if i.id not in visible], edges=edges, capabilities=capabilities) - def query(self, query): + async def query(self, query: Query) -> JsonValue: + if query.type == 'canvas.status': + return await self.wait_item_status(query.projectId, query.objectId, query.waitSeconds) + if query.type == 'asset.preview': + target = self.projects.local.state_dir / 'previews' / str(query.projectId) + return await asyncio.to_thread(CreationAssets(self.projects).preview, query.projectId, query.objectId, target) + return to_jsonable_python(await asyncio.to_thread(self.query_snapshot, query)) + + def query_snapshot(self, query): from opendirector.application.creation_history import CreationHistory from opendirector.infrastructure.creation_projects import object_index snapshot = self.projects.snapshot(query.projectId) diff --git a/packages/core/src/opendirector/domain/creation/base.py b/packages/core/src/opendirector/domain/creation/base.py index 0a2d3ac56..74abd9879 100644 --- a/packages/core/src/opendirector/domain/creation/base.py +++ b/packages/core/src/opendirector/domain/creation/base.py @@ -43,6 +43,10 @@ class ObjectVersion(Contract): revision: Revision = Field(description='Current object revision for optimistic concurrency.') +class CanvasItemVersion(ObjectVersion): + objectType: Literal['canvas_item'] = Field(description='Source canvas item type.') + + class Point(Contract): x: Number = Field(default=0, description='Horizontal coordinate.') y: Number = Field(default=0, description='Vertical coordinate.') diff --git a/packages/core/src/opendirector/domain/creation/canvas.py b/packages/core/src/opendirector/domain/creation/canvas.py index 855337c12..9402ff53a 100644 --- a/packages/core/src/opendirector/domain/creation/canvas.py +++ b/packages/core/src/opendirector/domain/creation/canvas.py @@ -5,10 +5,27 @@ from pydantic import Field, model_validator from .assets import AssetRef -from .base import Contract, Identifier, Point, Positive, ProjectEntity, unique -from .generation import GenerationSpec, ResolvedReference, validate_generation +from .base import CanvasItemVersion, Contract, Identifier, Point, Positive, ProjectEntity, unique +from .generation import FollowingSource, GenerationSpec, ResolvedReference, validate_generation from .models import ModelConfigPublic + +class FollowingReferenceSource(Contract): + binding: Literal['follow_current'] = Field(default='follow_current', description='Follow the canvas source current asset.') + source: FollowingSource = Field(description='Source canvas_item identity.') + + +class PinnedReferenceSource(Contract): + binding: Literal['pinned'] = Field(default='pinned', description='Use the fixed asset version.') + source: AssetRef = Field(description='Fixed current asset version.') + sourceVersion: CanvasItemVersion = Field(description='Canvas source version retaining the connection.') + + +class CanvasReferenceSources(Contract): + followCurrent: FollowingReferenceSource = Field(description='Source fields for a follow_current reference.') + pinned: PinnedReferenceSource = Field(description='Source fields for a pinned reference retaining its canvas connection.') + + CanvasKind = Literal['text', 'image', 'video', 'audio', 'file', 'group', 'timeline', 'subject', 'stack', 'tool', 'external'] MediaRole = Literal['empty', 'material', 'generation'] diff --git a/packages/core/src/opendirector/domain/creation/commands.py b/packages/core/src/opendirector/domain/creation/commands.py index 95f51306c..a23fb8df3 100644 --- a/packages/core/src/opendirector/domain/creation/commands.py +++ b/packages/core/src/opendirector/domain/creation/commands.py @@ -186,7 +186,7 @@ class SpecPromptWrite(Contract): class SpecWrite(Contract): - spec: GenerationSpec = Field(description='Generation configuration for this operation.') + spec: GenerationSpec = Field(description='Configuration with its current revision; the server increments it on commit.') class ReferenceWrite(Contract): @@ -313,7 +313,7 @@ def scope_and_versions(self): ('AssetWriteTextCommand', 'asset.writeText', AssetWriteText, 'Write a text asset or a new version of an existing text asset.'), ('AssetMetadataCommand', 'asset.updateMetadata', AssetMetadataUpdate, 'Rename an asset.'), ('AssetDeleteCommand', 'asset.delete', AssetDelete, 'Delete assets after an impact preview.'), - ('CanvasCreateGenerationCommand', 'canvas.createGeneration', CanvasCreateGeneration, 'Create a visible generation node with configuration and references atomically; does not submit generation.'), + ('CanvasCreateGenerationCommand', 'canvas.createGeneration', CanvasCreateGeneration, 'Create a visible generation node and references atomically, without separate configuration or display commands.'), ('CanvasShowCommand', 'canvas.show', CanvasShow, 'Create or restore a canvas display item.'), ('CanvasMoveCommand', 'canvas.move', CanvasMove, 'Move one or more canvas items.'), ('CanvasUpdateCommand', 'canvas.update', CanvasPresentationUpdate, 'Update canvas presentation, grouping or lock state.'), @@ -413,30 +413,53 @@ def outcome(self): return self -class CanvasItemQuery(Contract): - itemId: UUID = Field(description='Canvas item ID from the current project.') - waitSeconds: float = Field(default=0, ge=0, le=30, description='Wait for status changes, up to 30 seconds; zero returns immediately.') +class QueryBase(Contract): + queryId: UUID = Field(description='Caller-supplied query ID.') + projectId: UUID = Field(description='Owning project ID.') -class AssetPreviewQuery(Contract): - assetId: UUID = Field(description='Asset ID from the current project.') +class ProjectQuery(QueryBase): + type: Literal['project.get'] = Field(description='Read the current project.') -class Query(Contract): - queryId: UUID = Field(description='Caller-supplied query ID.') - projectId: UUID = Field(description='Owning project ID.') - type: Literal['project.get', 'object.get', 'asset.search', 'asset.usage', 'generationSpec.versions', 'canvas.versions', 'job.get'] = Field(description='Operation or variant discriminator.') - objectId: UUID | None = Field(default=None, description='Target business object ID.') - revision: Revision | None = Field(default=None, description='Historical revision to read, when specified.') - text: str = Field(default='', max_length=1000, description='UTF-8 text content or search query.') +class ObjectQuery(QueryBase): + type: Literal['object.get', 'job.get'] = Field(description='Read an object or generation job.') + objectId: UUID = Field(description='Target business object ID.') + revision: Revision | None = Field(default=None, description='Historical object revision, or current when omitted.') + + +class VersionsQuery(QueryBase): + type: Literal['generationSpec.versions', 'canvas.versions'] = Field(description='Read object versions.') + objectId: UUID = Field(description='Target configuration or canvas item ID.') + + +class AssetSearchQuery(QueryBase): + type: Literal['asset.search'] = Field(description='Search project assets.') + text: str = Field(default='', max_length=1000, description='Asset search text.') offset: int = Field(default=0, strict=True, ge=0, description='Number of matching records to skip.') limit: int = Field(default=50, strict=True, ge=1, le=200, description='Maximum number of records to return.') - @model_validator(mode='after') - def target(self): - if self.type in {'object.get', 'generationSpec.versions', 'canvas.versions', 'job.get'} and self.objectId is None: - raise ValueError('Object query needs a stable object ID') - return self + +class AssetUsageQuery(QueryBase): + type: Literal['asset.usage'] = Field(description='Read asset dependencies.') + objectId: UUID | None = Field(default=None, description='Asset ID, or all assets when omitted.') + offset: int = Field(default=0, strict=True, ge=0, description='Number of matching records to skip.') + limit: int = Field(default=50, strict=True, ge=1, le=200, description='Maximum number of records to return.') + + +class CanvasStatusQuery(QueryBase): + type: Literal['canvas.status'] = Field(description='Read node status, current asset and reusable referenceSources.') + objectId: UUID = Field(description='Canvas item ID.') + waitSeconds: float = Field(default=0, ge=0, le=30, description='Wait for generation changes; zero returns immediately.') + + +class AssetPreviewQuery(QueryBase): + type: Literal['asset.preview'] = Field(description='Preview an asset; does not establish model observation.') + objectId: UUID = Field(description='Asset ID.') + + +Query = Annotated[ProjectQuery | ObjectQuery | VersionsQuery | AssetSearchQuery | AssetUsageQuery | + CanvasStatusQuery | AssetPreviewQuery, Field(discriminator='type')] class ChangedObject(Contract): diff --git a/packages/core/src/opendirector/domain/creation/generation.py b/packages/core/src/opendirector/domain/creation/generation.py index ba1c2baab..9ed4fc7c2 100644 --- a/packages/core/src/opendirector/domain/creation/generation.py +++ b/packages/core/src/opendirector/domain/creation/generation.py @@ -8,7 +8,7 @@ from opendirector.domain.generation import GenerationFailure, ResultCheck from .assets import AssetRef, MediaMetadata -from .base import Contract, Digest, GenerationKind, NonNegative, ObjectVersion, PositiveInt, ProjectEntity, Revision, digest, unique +from .base import CanvasItemVersion, Contract, Digest, GenerationKind, NonNegative, ObjectVersion, PositiveInt, ProjectEntity, Revision, digest, unique from .models import ModelConfigPublic, ModelSnapshot, VideoMode @@ -26,7 +26,7 @@ class ImageParameters(Contract): class VideoParameters(Contract): - generateAudio: bool = Field(default=True, description='Whether video generation includes audio.') + generateAudio: bool = Field(default=True, description='Include audio; value must be allowed by model inputRules.generateAudioOptions.') videoMode: VideoMode = Field(default='first_frame', description='Video generation input mode.') durationSeconds: Annotated[float, Field(strict=True, ge=1, le=60)] | None = Field(default=None, description='Duration in seconds.') aspectRatio: str | None = Field(default=None, pattern=r'^[1-9]\d?:[1-9]\d?$', description='Output width-to-height ratio.') @@ -127,24 +127,19 @@ class ReferenceBase(ProjectEntity): class PinnedReference(ReferenceBase): - binding: Literal['pinned'] = Field(description='Business object displayed by this canvas item.') - source: AssetRef = Field(description='Input source or recorded provenance.') + binding: Literal['pinned'] = Field(description='Use a fixed asset version, retaining canvas provenance when supplied.') + source: AssetRef = Field(description='Fixed asset version used by generation.') # Historical provenance only; execution always uses the pinned asset. - sourceVersion: ObjectVersion | None = Field(default=None, exclude_if=lambda value: value is None, description='Captured source object version.') + sourceVersion: CanvasItemVersion | None = Field(default=None, exclude_if=lambda value: value is None, + description='Source canvas_item version for its connection; omit for asset-only references.') subjectRole: Literal['main', 'auxiliary', 'voice'] | None = Field(default=None, exclude_if=lambda value: value is None, description='Subject media slot used by this reference.') - @model_validator(mode='after') - def source_identity(self): - if self.sourceVersion and self.sourceVersion.objectType != 'canvas_item': - raise ValueError('Pinned source provenance must identify a canvas item') - return self - class FollowingReference(ReferenceBase): # A stable logical source item survives hiding; its current Asset/Spec # binding may change. It never points to a transient React Flow instance. - binding: Literal['follow_current'] = Field(description='Business object displayed by this canvas item.') - source: FollowingSource = Field(description='Input source or recorded provenance.') + binding: Literal['follow_current'] = Field(description="Follow the source canvas item's current asset.") + source: FollowingSource = Field(description='Source canvas_item identity.') Reference = Annotated[PinnedReference | FollowingReference, Field(discriminator='binding')] @@ -376,12 +371,15 @@ def validate_generation(spec: GenerationSpec, model: ModelConfigPublic, referenc raise ValueError('首尾帧和续写参考只能用于视频生成。') return params = spec.parameters - if not params.generateAudio and model.protocol != 'volcengine': + audio_options = rules.generateAudioOptions + if audio_options is None: + audio_options = (True, False) if model.protocol == 'volcengine' else (True,) + if params.generateAudio not in audio_options: raise ValueError('当前视频适配器不支持关闭生成音频。') if params.videoMode == 'first_frame' and len({ref.referenceId for ref in references if ref.asset.kind == 'image' and ref.subjectRole is None}) > 2: raise ValueError('首尾帧模式最多连接 2 个图片节点，请先断开多余连线。') if params.videoMode not in rules.videoModes: - raise ValueError('当前模型不支持这种视频模式。') + raise ValueError(f"当前模型不支持视频模式 {params.videoMode}；允许值：{', '.join(rules.videoModes)}。") if params.videoMode == 'continuation': continuation = [ref for ref in references if ref.role == 'continuation'] if len(continuation) != 1 or continuation[0].asset.kind != 'video' or kinds.count('video') != 1: diff --git a/packages/core/src/opendirector/domain/creation/models.py b/packages/core/src/opendirector/domain/creation/models.py index 42e388fd9..3ae18a6a4 100644 --- a/packages/core/src/opendirector/domain/creation/models.py +++ b/packages/core/src/opendirector/domain/creation/models.py @@ -43,6 +43,10 @@ class InputRules(Contract): referenceImageResolutions: tuple[str, ...] | None = Field(default=None, description='Supported reference image resolution values.') videoCapabilities: VideoCapabilities | None = Field(default=None, description='Supported video generation capabilities.') + generateAudioOptions: tuple[Annotated[bool, Field(strict=True)], ...] | None = Field( + default=None, min_length=1, max_length=2, exclude_if=lambda value: value is None, + description='Allowed generateAudio values; omitted in legacy execution snapshots.') + @model_validator(mode='after') def consistent(self): from .base import unique diff --git a/packages/core/src/opendirector/domain/input_rules.py b/packages/core/src/opendirector/domain/input_rules.py index c8d7067d3..823bad325 100644 --- a/packages/core/src/opendirector/domain/input_rules.py +++ b/packages/core/src/opendirector/domain/input_rules.py @@ -84,6 +84,8 @@ def _adapter_rules(config): def input_rules(config): rules = _adapter_rules(config) + if config['kind'] == 'video': + rules['generateAudioOptions'] = [True, False] if config.get('protocol') == 'volcengine' else [True] if config['kind'] == 'image' and rules['profile'] != 'vllm-omni-segmentation': from .image_output import image_outputs rules['imageOutputs'] = image_outputs(config, rules['profile']) diff --git a/src/backend/chat.ts b/src/backend/chat.ts index f5f998343..3b53d28ba 100644 --- a/src/backend/chat.ts +++ b/src/backend/chat.ts @@ -48,7 +48,12 @@ export interface ConversationSummary { title: string; messageCount: number; } +export interface ChatPlanEntry { + content: string; + status: "pending" | "in_progress" | "completed"; +} export interface ChatMessage { + plan?: ChatPlanEntry[]; runtime?: AgentRuntime; tools?: ChatTool[]; id: string; @@ -94,6 +99,7 @@ export interface Conversation extends ChatDraft { export type ChatEvent = | { type: "start"; conversation: Conversation } | { type: "thinking" } + | { type: "plan"; entries: ChatPlanEntry[] } | ({ type: "tool" } & ChatTool) | { type: "delta"; text: string } | { type: "end"; conversation: Conversation | null; error: string | null }; diff --git a/src/backend/rules.ts b/src/backend/rules.ts index ee568d639..97dd7a390 100644 --- a/src/backend/rules.ts +++ b/src/backend/rules.ts @@ -31,6 +31,7 @@ export interface InputRules { maxVideos: number; maxAudios: number; videoModes: BackendVideoMode[]; + generateAudioOptions?: boolean[] | null; imageOutputs?: { aspectRatio: string; size: string }[]; outputResolutions?: string[]; referenceImageResolutions?: string[]; @@ -387,7 +388,7 @@ export function serviceGenerationSettings( next.imagePreset = "none"; if (!/^[A-Za-z0-9_.-]{1,64}$/.test(next.voiceName ?? "")) next.voiceName = undefined; - if (config.protocol !== "volcengine") next.audioEnabled = true; + if (!(config.inputRules.generateAudioOptions?.includes(false) ?? config.protocol === "volcengine")) next.audioEnabled = true; } return next; } @@ -492,7 +493,7 @@ export function backendBlocker( if ( settings.mode === "video" && settings.audioEnabled === false && - config.protocol !== "volcengine" + !(rules.generateAudioOptions?.includes(false) ?? config.protocol === "volcengine") ) return "当前视频适配器不支持关闭生成音频。"; const references = backendReferences(settings, incoming, config); diff --git a/src/canvas/AssistantReply.tsx b/src/canvas/AssistantReply.tsx index 8558ee597..cc39f6b54 100644 --- a/src/canvas/AssistantReply.tsx +++ b/src/canvas/AssistantReply.tsx @@ -2,19 +2,8 @@ import { memo, useState } from "react"; import { Streamdown } from "streamdown"; import type { ChatMessage, ChatTool } from "../backend/chat"; import { CanvasIcon } from "./CanvasIcon"; +import { activityProgress, replyParts, toolLabel } from "./assistantActivity.ts"; -const TOOL_NAMES: Record = { - query: "查看工程", - read: "读取素材或技能", - command_schema: "查看操作说明", - execute_command: "执行工程操作", - read_project: "读取工程", - read_text_asset: "读取文本", - save_text_asset: "保存文本", - generate_asset: "生成素材", - get_generation_task: "查看生成进度", - read_skill: "读取技能", -}; const STATUS = { running: "进行中", complete: "已完成", @@ -23,15 +12,14 @@ const STATUS = { }; function ToolActivity({ tool }: { tool: ChatTool }) { - const operation = tool.arguments?.type; return (
- {TOOL_NAMES[tool.name] ?? tool.name}{typeof operation === "string" && {operation}} + {toolLabel(tool)} {STATUS[tool.status]} @@ -56,6 +44,44 @@ function ToolActivity({ tool }: { tool: ChatTool }) { ); } +function ToolGroup({ tools }: { tools: ChatTool[] }) { + const [open, setOpen] = useState(false); + const [hiddenCount, setHiddenCount] = useState(null); + const hidden = hiddenCount ?? Math.max(0, tools.length - 50); + const completed = tools.filter((tool) => tool.status === "complete").length; + const failed = tools.filter((tool) => tool.status === "failed"); + const interrupted = tools.filter((tool) => tool.status === "interrupted"); + const running = tools.filter((tool) => tool.status === "running"); + return ( +
+
{ + const next = event.currentTarget.open; + if (next) setHiddenCount((count) => count ?? Math.max(0, tools.length - 50)); + setOpen(next); + }} + > + + + 执行记录 · {tools.length} 次调用 + {completed}/{tools.length} 已完成{failed.length > 0 && ` · ${failed.length} 失败`}{interrupted.length > 0 && ` · ${interrupted.length} 中断`} + + + {open &&
+ {hidden > 0 && } + {tools.slice(hidden).map((tool) => )} +
} +
+
+ ); +} + export const AssistantReply = memo(function AssistantReply({ message, thinking, @@ -64,8 +90,6 @@ export const AssistantReply = memo(function AssistantReply({ thinking: boolean; }) { const [copyState, setCopyState] = useState("复制回复"); - const parts = []; - let offset = 0; const markdown = (text: string, key: string) => text && ( ); - for (const tool of message.tools ?? []) { - const end = Math.max( - offset, - Math.min(tool.textOffset, message.text.length), - ); - parts.push(markdown(message.text.slice(offset, end), `text-${tool.id}`)); - parts.push(); - offset = end; - } - parts.push(markdown(message.text.slice(offset), "tail")); return ( <> - {parts} + {!!message.plan?.length &&
+ + + 执行计划 + {message.plan.filter((step) => step.status === "completed").length}/{message.plan.length} 已完成 + + +
    {message.plan.map((step, index) =>
  1. + {step.content} +
    2. )}
+
} + {replyParts(message).map((part) => part.type === "text" + ? markdown(part.text, part.id) + : )} {message.status === "streaming" && (
- {thinking ? "正在思考…" : "正在回复…"} + {activityProgress(message, thinking)}
)} {!message.text && diff --git a/src/canvas/CanvasWorkspace.tsx b/src/canvas/CanvasWorkspace.tsx index 06931af99..fb874fafc 100644 --- a/src/canvas/CanvasWorkspace.tsx +++ b/src/canvas/CanvasWorkspace.tsx @@ -160,7 +160,7 @@ import { } from "./generationReferenceContent.ts"; import { referenceContentCommand } from "./generationReferenceEdit.ts"; import type { GenerationReferenceControls } from "./GenerationReferenceEditor"; -import { absolutePosition, arrangeNodes } from "./geometry"; +import { absolutePosition, arrangeNodes, boundsOfNodes } from "./geometry"; import { lockedNodeIds } from "./hierarchy"; import { beginTransaction, @@ -527,6 +527,8 @@ function Workspace() { const [measurements, setMeasurements] = useState< Record >({}); + const revealCreatedNodes = useRef<(nodes: CanvasNode[], allNodes: CanvasNode[]) => void>(() => {}); + const viewportMoving = useRef(false); const selectedRef = useRef(selected); selectedRef.current = selected; const selectionAtPointerDown = useRef([]); @@ -1465,6 +1467,11 @@ function Workspace() { projection.baseline = merged; projection.basis = snapshot; setHistory((history) => ({ ...history, present: merged })); + const known = new Set(current.nodes.map((node) => node.id)); + const created = next.nodes.filter((node) => !known.has(node.id)); + if (created.length) requestAnimationFrame(() => { + if (active && historyRef.current.present.id === current.id) revealCreatedNodes.current(created, next.nodes); + }); }) .catch((error) => active && notify(errorMessage(error))); }; @@ -2072,8 +2079,18 @@ function Workspace() { spacePressed || pointerGesture.current.moved || !selectionControls, - controlsKey: `${hasComposer}:${hasToolbar}:${composerExpanded}:${activeAction?.token ?? ""}:${activeAction?.tool === "outpaint" ? JSON.stringify(presentationBounds) : ""}`, + controlsKey: `${assistantOpen}:${assistantLayout.width}:${hasComposer}:${hasToolbar}:${composerExpanded}:${activeAction?.token ?? ""}:${activeAction?.tool === "outpaint" ? JSON.stringify(presentationBounds) : ""}`, }); + revealCreatedNodes.current = (nodes, allNodes) => { + if (switchPending.current || viewportMoving.current || interacting || pointerGesture.current.active || + references.connecting || spacePressed || selectedRef.current.length > 1 || + editing || preview || activeAction || textEditingId || composerExpanded) return; + const focused = document.activeElement; + if (focused instanceof HTMLElement && root.current?.contains(focused) && + focused.matches("input,textarea,[contenteditable=true]") && !focused.closest(".cw-assistant-panel")) return; + const bounds = boundsOfNodes(nodes, allNodes); + if (bounds) selectionPresentation.reveal(bounds); + }; useCanvasWheelBoundary(root, selectionPresentation.userNavigation); const flowNodes: FlowNode[] = useMemo( () => @@ -4347,13 +4364,19 @@ function Workspace() { }); }} onMoveStart={(event) => { - if (event) selectionPresentation.userNavigation(); + if (event) { + selectionPresentation.userNavigation(); + viewportMoving.current = true; + } }} onMove={(event, view) => { setZoom(view.zoom); if (event) setMenu(null); }} - onMoveEnd={(_, viewport) => command({ type: "viewport", viewport })} + onMoveEnd={(_, viewport) => { + viewportMoving.current = false; + command({ type: "viewport", viewport }); + }} minZoom={MIN_ZOOM} maxZoom={MAX_ZOOM} nodeDragThreshold={0} diff --git a/src/canvas/assistant-experience.css b/src/canvas/assistant-experience.css index b440bab62..bd4110357 100644 --- a/src/canvas/assistant-experience.css +++ b/src/canvas/assistant-experience.css @@ -121,9 +121,6 @@ .cw-chat-activity.is-running > summary { color: var(--chat-text); } -.cw-chat-activity.is-failed > summary { - color: #e6a7a7; -} .cw-chat-activity-detail { margin: 0 0 8px 22px; padding: 2px 0 2px 12px; @@ -205,3 +202,11 @@ .cw-chat-tool-data > summary > svg { flex-shrink: 0; transition: transform 0.15s; } .cw-chat-tool-data[open] > summary > svg { transform: rotate(90deg); } .cw-chat-tool-data > pre { max-height: 240px; overflow: auto; margin: 4px 0 8px; padding: 8px 10px; border: 1px solid var(--chat-border); border-radius: 6px; background: var(--chat-input); white-space: pre-wrap; overflow-wrap: anywhere; font-size: 11px; line-height: 1.5; } + +.cw-chat-tool-group-items { max-height: 360px; overflow: auto; padding-left: 14px; border-left: 1px solid var(--chat-border); } +.cw-chat-tool-group-items > .cw-chat-activity { margin: 2px 0; } +.cw-chat-show-earlier { border: 0; background: transparent; color: var(--chat-muted); font: inherit; font-size: 12px; cursor: pointer; padding: 7px 0; } +.cw-chat-plan ol { margin: 0 0 8px 22px; padding-left: 18px; } +.cw-chat-plan li { padding: 3px 0; } +.cw-chat-plan li[data-status="in_progress"] { color: var(--chat-text); } +.cw-chat-plan li[data-status="completed"] { text-decoration: line-through; } diff --git a/src/canvas/assistantActivity.ts b/src/canvas/assistantActivity.ts new file mode 100644 index 000000000..531edc4dc --- /dev/null +++ b/src/canvas/assistantActivity.ts @@ -0,0 +1,58 @@ +import type { ChatMessage, ChatTool } from "../backend/chat.ts"; + +const TOOL_NAMES: Record = { + query: "查询工程", + read: "读取素材或技能", + command_schema: "查看操作说明", + execute_command: "执行工程操作", + read_project: "读取工程", + read_canvas_item: "查看画布状态", + preview_asset: "预览素材", + read_text_asset: "读取文本", + save_text_asset: "保存文本", + generate_asset: "生成素材", + get_generation_task: "查看生成进度", + read_skill: "读取技能", + bash: "运行命令", + web_search: "搜索网页", + web_fetch: "读取网页", +}; + +export function toolLabel(tool: ChatTool): string { + const operation = tool.arguments?.type; + const name = TOOL_NAMES[tool.name] ?? tool.name; + return typeof operation === "string" ? `${name} ${operation}` : name; +} + +export type ReplyPart = + | { type: "text"; id: string; text: string } + | { type: "tools"; id: string; tools: ChatTool[] }; + +export function replyParts(message: Pick): ReplyPart[] { + // The backend records textOffset in Unicode code points, as Python len does. + const text = Array.from(message.text); + const parts: ReplyPart[] = []; + let offset = 0; + for (const tool of message.tools ?? []) { + const end = Math.max(offset, Math.min(tool.textOffset, text.length)); + if (end > offset) { + parts.push({ type: "text", id: `text-${tool.id}`, text: text.slice(offset, end).join("") }); + } + const previous = parts.at(-1); + if (previous?.type === "tools") previous.tools.push(tool); + else parts.push({ type: "tools", id: tool.id, tools: [tool] }); + offset = end; + } + if (offset < text.length) parts.push({ type: "text", id: "tail", text: text.slice(offset).join("") }); + return parts; +} + +export function activityProgress(message: ChatMessage, thinking: boolean): string { + const running = message.tools?.filter((tool) => tool.status === "running") ?? []; + if (running.length) { + return `正在执行：${toolLabel(running.at(-1)!)}${running.length > 1 ? `（${running.length} 项进行中）` : ""}`; + } + if (thinking) return "正在思考…"; + const step = message.plan?.find((entry) => entry.status === "in_progress"); + return step ? `正在执行计划：${step.content}` : "正在回复…"; +} diff --git a/src/canvas/selectionPresentation.ts b/src/canvas/selectionPresentation.ts index 1993581da..53f9aa9cb 100644 --- a/src/canvas/selectionPresentation.ts +++ b/src/canvas/selectionPresentation.ts @@ -16,6 +16,15 @@ const COMPOSER_GAP = 16; const clamp = (value: number, min: number, max: number) => Math.max(min, Math.min(max, value)); +/** Center content in the usable canvas, preserving zoom unless it cannot fit. */ +export function centeredViewport(node: Rect, viewport: Viewport, chrome: SelectionChrome): Viewport { + return selectionViewport(node, { + ...viewport, + x: chrome.safe.x + chrome.safe.width / 2 - (node.x + node.width / 2) * viewport.zoom, + y: chrome.safe.y + chrome.safe.height / 2 - (node.y + node.height / 2) * viewport.zoom, + }, chrome); +} + /** Keep the selected node and its controls in one visible envelope. * Authored node geometry never changes when opening a generator. */ export function selectionViewport( diff --git a/src/canvas/useAssistantChat.ts b/src/canvas/useAssistantChat.ts index dc006f505..1e24ac989 100644 --- a/src/canvas/useAssistantChat.ts +++ b/src/canvas/useAssistantChat.ts @@ -401,7 +401,14 @@ export function useAssistantChat( ), })); } else if (event.type === "thinking") setThinkingNow(true); - else if (event.type === "tool") { + else if (event.type === "plan") { + setValue((previous) => ({ + ...previous, + messages: previous.messages.map((message) => message.id === assistantId + ? { ...message, plan: event.entries } : message), + })); + } else if (event.type === "tool") { + setThinkingNow(false); const { type, ...tool } = event; setValue((previous) => ({ ...previous, diff --git a/src/canvas/useSelectionPresentation.ts b/src/canvas/useSelectionPresentation.ts index 4ca77e0fb..f09067a62 100644 --- a/src/canvas/useSelectionPresentation.ts +++ b/src/canvas/useSelectionPresentation.ts @@ -8,6 +8,7 @@ import { import { useReactFlow, useViewport } from "@xyflow/react"; import type { Rect, Viewport } from "./model"; import { + centeredViewport, selectionPlacement, selectionViewport, type SelectionChrome, @@ -71,13 +72,12 @@ export function useSelectionPresentation({ if (!host) return; const measure = () => { const rect = host.getBoundingClientRect(); - const sidebar = host - .querySelector(".cw-side-panel") - ?.getBoundingClientRect(); + const panels = [...host.querySelectorAll(".cw-side-panel,.cw-assistant-panel:not([hidden])")]; let left = rect.width < 700 ? 64 : 80; let right = rect.width - 20; - if (sidebar) { - if (sidebar.left - rect.left < rect.width / 2) + for (const panel of panels) { + const sidebar = panel.getBoundingClientRect(); + if (!panel.classList.contains("cw-assistant-panel") && sidebar.left - rect.left < rect.width / 2) left = Math.max(left, sidebar.right - rect.left + 16); else right = Math.min(right, sidebar.left - rect.left - 16); } @@ -121,8 +121,7 @@ export function useSelectionPresentation({ observer.observe(host); if (toolbar.current) observer.observe(toolbar.current); if (composer.current) observer.observe(composer.current); - const sidebar = host.querySelector(".cw-side-panel"); - if (sidebar) observer.observe(sidebar); + for (const panel of host.querySelectorAll(".cw-side-panel,.cw-assistant-panel:not([hidden])")) observer.observe(panel); measure(); return () => observer.disconnect(); }, [root, toolbar, composer, measurementKey, scaleComposer]); @@ -229,6 +228,12 @@ export function useSelectionPresentation({ }; present(selectionViewport(bounds, view, chrome)); }, + reveal: (bounds: Rect) => { + if (obscured || !chrome.safe.width) return; + present(centeredViewport(bounds, flow.getViewport(), { + safe: chrome.safe, toolbar: { width: 0, height: 0 }, composer: { width: 0, height: 0 }, + })); + }, placement, mediaRect: mediaBounds ? { diff --git a/src/contracts/creation.schema.json b/src/contracts/creation.schema.json index 3991d4053..1b3d15779 100644 --- a/src/contracts/creation.schema.json +++ b/src/contracts/creation.schema.json @@ -461,15 +461,36 @@ "AssetPreviewQuery": { "additionalProperties": false, "properties": { - "assetId": { - "description": "Asset ID from the current project.", + "queryId": { + "description": "Caller-supplied query ID.", "format": "uuid", - "title": "Assetid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "asset.preview", + "description": "Preview an asset; does not establish model observation.", + "title": "Type", + "type": "string" + }, + "objectId": { + "description": "Asset ID.", + "format": "uuid", + "title": "Objectid", "type": "string" } }, "required": [ - "assetId" + "queryId", + "projectId", + "type", + "objectId" ], "title": "AssetPreviewQuery", "type": "object" @@ -693,6 +714,117 @@ "title": "AssetRef", "type": "object" }, + "AssetSearchQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "asset.search", + "description": "Search project assets.", + "title": "Type", + "type": "string" + }, + "text": { + "default": "", + "description": "Asset search text.", + "maxLength": 1000, + "title": "Text", + "type": "string" + }, + "offset": { + "default": 0, + "description": "Number of matching records to skip.", + "minimum": 0, + "title": "Offset", + "type": "integer" + }, + "limit": { + "default": 50, + "description": "Maximum number of records to return.", + "maximum": 200, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + "required": [ + "queryId", + "projectId", + "type" + ], + "title": "AssetSearchQuery", + "type": "object" + }, + "AssetUsageQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "asset.usage", + "description": "Read asset dependencies.", + "title": "Type", + "type": "string" + }, + "objectId": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Asset ID, or all assets when omitted.", + "title": "Objectid" + }, + "offset": { + "default": 0, + "description": "Number of matching records to skip.", + "minimum": 0, + "title": "Offset", + "type": "integer" + }, + "limit": { + "default": 50, + "description": "Maximum number of records to return.", + "maximum": 200, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + "required": [ + "queryId", + "projectId", + "type" + ], + "title": "AssetUsageQuery", + "type": "object" + }, "AssetWriteText": { "additionalProperties": false, "properties": { @@ -1821,7 +1953,7 @@ }, "CanvasCreateGenerationCommand": { "additionalProperties": false, - "description": "Create a visible generation node with configuration and references atomically; does not submit generation.", + "description": "Create a visible generation node and references atomically, without separate configuration or display commands.", "properties": { "commandId": { "description": "Stable command ID for idempotent replay.", @@ -1866,7 +1998,7 @@ }, "payload": { "$ref": "#/$defs/CanvasCreateGeneration", - "description": "Create a visible generation node with configuration and references atomically; does not submit generation." + "description": "Create a visible generation node and references atomically, without separate configuration or display commands." } }, "required": [ @@ -2278,28 +2410,34 @@ "title": "CanvasItemBase", "type": "object" }, - "CanvasItemQuery": { + "CanvasItemVersion": { "additionalProperties": false, "properties": { - "itemId": { - "description": "Canvas item ID from the current project.", + "objectType": { + "const": "canvas_item", + "description": "Source canvas item type.", + "title": "Objecttype", + "type": "string" + }, + "id": { + "description": "Stable object ID.", "format": "uuid", - "title": "Itemid", + "title": "Id", "type": "string" }, - "waitSeconds": { - "default": 0, - "description": "Wait for status changes, up to 30 seconds; zero returns immediately.", - "maximum": 30, - "minimum": 0, - "title": "Waitseconds", - "type": "number" + "revision": { + "description": "Current object revision for optimistic concurrency.", + "minimum": 1, + "title": "Revision", + "type": "integer" } }, "required": [ - "itemId" + "objectType", + "id", + "revision" ], - "title": "CanvasItemQuery", + "title": "CanvasItemVersion", "type": "object" }, "CanvasMaterialize": { @@ -2786,6 +2924,25 @@ "title": "CanvasPublishMediaCommand", "type": "object" }, + "CanvasReferenceSources": { + "additionalProperties": false, + "properties": { + "followCurrent": { + "$ref": "#/$defs/FollowingReferenceSource", + "description": "Source fields for a follow_current reference." + }, + "pinned": { + "$ref": "#/$defs/PinnedReferenceSource", + "description": "Source fields for a pinned reference retaining its canvas connection." + } + }, + "required": [ + "followCurrent", + "pinned" + ], + "title": "CanvasReferenceSources", + "type": "object" + }, "CanvasShow": { "additionalProperties": false, "properties": { @@ -2932,6 +3089,51 @@ "title": "CanvasShowCommandOperation", "type": "object" }, + "CanvasStatusQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "canvas.status", + "description": "Read node status, current asset and reusable referenceSources.", + "title": "Type", + "type": "string" + }, + "objectId": { + "description": "Canvas item ID.", + "format": "uuid", + "title": "Objectid", + "type": "string" + }, + "waitSeconds": { + "default": 0, + "description": "Wait for generation changes; zero returns immediately.", + "maximum": 30, + "minimum": 0, + "title": "Waitseconds", + "type": "number" + } + }, + "required": [ + "queryId", + "projectId", + "type", + "objectId" + ], + "title": "CanvasStatusQuery", + "type": "object" + }, "CanvasUpdateCommand": { "additionalProperties": false, "description": "Update canvas presentation, grouping or lock state.", @@ -3587,8 +3789,45 @@ "description": "Immutable command execution receipt." }, "query": { - "$ref": "#/$defs/Query", - "description": "Read-only project query." + "description": "Read-only project query.", + "discriminator": { + "mapping": { + "asset.preview": "#/$defs/AssetPreviewQuery", + "asset.search": "#/$defs/AssetSearchQuery", + "asset.usage": "#/$defs/AssetUsageQuery", + "canvas.status": "#/$defs/CanvasStatusQuery", + "canvas.versions": "#/$defs/VersionsQuery", + "generationSpec.versions": "#/$defs/VersionsQuery", + "job.get": "#/$defs/ObjectQuery", + "object.get": "#/$defs/ObjectQuery", + "project.get": "#/$defs/ProjectQuery" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/$defs/ProjectQuery" + }, + { + "$ref": "#/$defs/ObjectQuery" + }, + { + "$ref": "#/$defs/VersionsQuery" + }, + { + "$ref": "#/$defs/AssetSearchQuery" + }, + { + "$ref": "#/$defs/AssetUsageQuery" + }, + { + "$ref": "#/$defs/CanvasStatusQuery" + }, + { + "$ref": "#/$defs/AssetPreviewQuery" + } + ], + "title": "Query" }, "event": { "$ref": "#/$defs/ProjectEvent", @@ -4678,13 +4917,13 @@ }, "binding": { "const": "follow_current", - "description": "Business object displayed by this canvas item.", + "description": "Follow the source canvas item's current asset.", "title": "Binding", "type": "string" }, "source": { "$ref": "#/$defs/FollowingSource", - "description": "Input source or recorded provenance." + "description": "Source canvas_item identity." } }, "required": [ @@ -4697,6 +4936,27 @@ "title": "FollowingReference", "type": "object" }, + "FollowingReferenceSource": { + "additionalProperties": false, + "properties": { + "binding": { + "const": "follow_current", + "default": "follow_current", + "description": "Follow the canvas source current asset.", + "title": "Binding", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FollowingSource", + "description": "Source canvas_item identity." + } + }, + "required": [ + "source" + ], + "title": "FollowingReferenceSource", + "type": "object" + }, "FollowingSource": { "additionalProperties": false, "properties": { @@ -6256,6 +6516,24 @@ ], "default": null, "description": "Supported video generation capabilities." + }, + "generateAudioOptions": { + "anyOf": [ + { + "items": { + "type": "boolean" + }, + "maxItems": 2, + "minItems": 1, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Allowed generateAudio values; omitted in legacy execution snapshots.", + "title": "Generateaudiooptions" } }, "required": [ @@ -8208,14 +8486,68 @@ "title": "NodeVersionApplyCommand", "type": "object" }, - "ObjectVersion": { + "ObjectQuery": { "additionalProperties": false, "properties": { - "objectType": { - "description": "Referenced business object type.", - "enum": [ - "project", - "asset", + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "description": "Read an object or generation job.", + "enum": [ + "object.get", + "job.get" + ], + "title": "Type", + "type": "string" + }, + "objectId": { + "description": "Target business object ID.", + "format": "uuid", + "title": "Objectid", + "type": "string" + }, + "revision": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Historical object revision, or current when omitted.", + "title": "Revision" + } + }, + "required": [ + "queryId", + "projectId", + "type", + "objectId" + ], + "title": "ObjectQuery", + "type": "object" + }, + "ObjectVersion": { + "additionalProperties": false, + "properties": { + "objectType": { + "description": "Referenced business object type.", + "enum": [ + "project", + "asset", "generation_spec", "reference", "canvas_item", @@ -8359,25 +8691,25 @@ }, "binding": { "const": "pinned", - "description": "Business object displayed by this canvas item.", + "description": "Use a fixed asset version, retaining canvas provenance when supplied.", "title": "Binding", "type": "string" }, "source": { "$ref": "#/$defs/AssetRef", - "description": "Input source or recorded provenance." + "description": "Fixed asset version used by generation." }, "sourceVersion": { "anyOf": [ { - "$ref": "#/$defs/ObjectVersion" + "$ref": "#/$defs/CanvasItemVersion" }, { "type": "null" } ], "default": null, - "description": "Captured source object version." + "description": "Source canvas_item version for its connection; omit for asset-only references." }, "subjectRole": { "anyOf": [ @@ -8408,6 +8740,32 @@ "title": "PinnedReference", "type": "object" }, + "PinnedReferenceSource": { + "additionalProperties": false, + "properties": { + "binding": { + "const": "pinned", + "default": "pinned", + "description": "Use the fixed asset version.", + "title": "Binding", + "type": "string" + }, + "source": { + "$ref": "#/$defs/AssetRef", + "description": "Fixed current asset version." + }, + "sourceVersion": { + "$ref": "#/$defs/CanvasItemVersion", + "description": "Canvas source version retaining the connection." + } + }, + "required": [ + "source", + "sourceVersion" + ], + "title": "PinnedReferenceSource", + "type": "object" + }, "Playhead": { "additionalProperties": false, "properties": { @@ -8664,6 +9022,36 @@ "title": "ProjectEvent", "type": "object" }, + "ProjectQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "project.get", + "description": "Read the current project.", + "title": "Type", + "type": "string" + } + }, + "required": [ + "queryId", + "projectId", + "type" + ], + "title": "ProjectQuery", + "type": "object" + }, "ProjectRename": { "additionalProperties": false, "properties": { @@ -9134,7 +9522,7 @@ "title": "PromptText", "type": "object" }, - "Query": { + "QueryBase": { "additionalProperties": false, "properties": { "queryId": { @@ -9148,78 +9536,13 @@ "format": "uuid", "title": "Projectid", "type": "string" - }, - "type": { - "description": "Operation or variant discriminator.", - "enum": [ - "project.get", - "object.get", - "asset.search", - "asset.usage", - "generationSpec.versions", - "canvas.versions", - "job.get" - ], - "title": "Type", - "type": "string" - }, - "objectId": { - "anyOf": [ - { - "format": "uuid", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target business object ID.", - "title": "Objectid" - }, - "revision": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Historical revision to read, when specified.", - "title": "Revision" - }, - "text": { - "default": "", - "description": "UTF-8 text content or search query.", - "maxLength": 1000, - "title": "Text", - "type": "string" - }, - "offset": { - "default": 0, - "description": "Number of matching records to skip.", - "minimum": 0, - "title": "Offset", - "type": "integer" - }, - "limit": { - "default": 50, - "description": "Maximum number of records to return.", - "maximum": 200, - "minimum": 1, - "title": "Limit", - "type": "integer" } }, "required": [ "queryId", - "projectId", - "type" + "projectId" ], - "title": "Query", + "title": "QueryBase", "type": "object" }, "RectangleLayerContent": { @@ -10420,7 +10743,7 @@ "additionalProperties": false, "properties": { "spec": { - "description": "Generation configuration for this operation.", + "description": "Configuration with its current revision; the server increments it on commit.", "discriminator": { "mapping": { "audio": "#/$defs/AudioGenerationSpec", @@ -12462,6 +12785,46 @@ "title": "Vector3", "type": "object" }, + "VersionsQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "description": "Read object versions.", + "enum": [ + "generationSpec.versions", + "canvas.versions" + ], + "title": "Type", + "type": "string" + }, + "objectId": { + "description": "Target configuration or canvas item ID.", + "format": "uuid", + "title": "Objectid", + "type": "string" + } + }, + "required": [ + "queryId", + "projectId", + "type", + "objectId" + ], + "title": "VersionsQuery", + "type": "object" + }, "VideoCanvasItem": { "additionalProperties": false, "properties": { @@ -12983,7 +13346,7 @@ "properties": { "generateAudio": { "default": true, - "description": "Whether video generation includes audio.", + "description": "Include audio; value must be allowed by model inputRules.generateAudioOptions.", "title": "Generateaudio", "type": "boolean" }, @@ -13465,8 +13828,45 @@ "description": "Immutable command execution receipt." }, "query": { - "$ref": "#/$defs/Query", - "description": "Read-only project query." + "description": "Read-only project query.", + "discriminator": { + "mapping": { + "asset.preview": "#/$defs/AssetPreviewQuery", + "asset.search": "#/$defs/AssetSearchQuery", + "asset.usage": "#/$defs/AssetUsageQuery", + "canvas.status": "#/$defs/CanvasStatusQuery", + "canvas.versions": "#/$defs/VersionsQuery", + "generationSpec.versions": "#/$defs/VersionsQuery", + "job.get": "#/$defs/ObjectQuery", + "object.get": "#/$defs/ObjectQuery", + "project.get": "#/$defs/ProjectQuery" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/$defs/ProjectQuery" + }, + { + "$ref": "#/$defs/ObjectQuery" + }, + { + "$ref": "#/$defs/VersionsQuery" + }, + { + "$ref": "#/$defs/AssetSearchQuery" + }, + { + "$ref": "#/$defs/AssetUsageQuery" + }, + { + "$ref": "#/$defs/CanvasStatusQuery" + }, + { + "$ref": "#/$defs/AssetPreviewQuery" + } + ], + "title": "Query" }, "event": { "$ref": "#/$defs/ProjectEvent", @@ -14046,15 +14446,36 @@ "AssetPreviewQuery": { "additionalProperties": false, "properties": { - "assetId": { - "description": "Asset ID from the current project.", + "queryId": { + "description": "Caller-supplied query ID.", "format": "uuid", - "title": "Assetid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "asset.preview", + "description": "Preview an asset; does not establish model observation.", + "title": "Type", + "type": "string" + }, + "objectId": { + "description": "Asset ID.", + "format": "uuid", + "title": "Objectid", "type": "string" } }, "required": [ - "assetId" + "queryId", + "projectId", + "type", + "objectId" ], "title": "AssetPreviewQuery", "type": "object" @@ -14289,6 +14710,123 @@ "title": "AssetRef", "type": "object" }, + "AssetSearchQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "asset.search", + "description": "Search project assets.", + "title": "Type", + "type": "string" + }, + "text": { + "default": "", + "description": "Asset search text.", + "maxLength": 1000, + "title": "Text", + "type": "string" + }, + "offset": { + "default": 0, + "description": "Number of matching records to skip.", + "minimum": 0, + "title": "Offset", + "type": "integer" + }, + "limit": { + "default": 50, + "description": "Maximum number of records to return.", + "maximum": 200, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + "required": [ + "queryId", + "projectId", + "type", + "text", + "offset", + "limit" + ], + "title": "AssetSearchQuery", + "type": "object" + }, + "AssetUsageQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "asset.usage", + "description": "Read asset dependencies.", + "title": "Type", + "type": "string" + }, + "objectId": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Asset ID, or all assets when omitted.", + "title": "Objectid" + }, + "offset": { + "default": 0, + "description": "Number of matching records to skip.", + "minimum": 0, + "title": "Offset", + "type": "integer" + }, + "limit": { + "default": 50, + "description": "Maximum number of records to return.", + "maximum": 200, + "minimum": 1, + "title": "Limit", + "type": "integer" + } + }, + "required": [ + "queryId", + "projectId", + "type", + "objectId", + "offset", + "limit" + ], + "title": "AssetUsageQuery", + "type": "object" + }, "AssetWriteText": { "additionalProperties": false, "properties": { @@ -15455,7 +15993,7 @@ }, "CanvasCreateGenerationCommand": { "additionalProperties": false, - "description": "Create a visible generation node with configuration and references atomically; does not submit generation.", + "description": "Create a visible generation node and references atomically, without separate configuration or display commands.", "properties": { "commandId": { "description": "Stable command ID for idempotent replay.", @@ -15500,7 +16038,7 @@ }, "payload": { "$ref": "#/$defs/CanvasCreateGeneration", - "description": "Create a visible generation node with configuration and references atomically; does not submit generation." + "description": "Create a visible generation node and references atomically, without separate configuration or display commands." } }, "required": [ @@ -15930,29 +16468,34 @@ "title": "CanvasItemBase", "type": "object" }, - "CanvasItemQuery": { + "CanvasItemVersion": { "additionalProperties": false, "properties": { - "itemId": { - "description": "Canvas item ID from the current project.", + "objectType": { + "const": "canvas_item", + "description": "Source canvas item type.", + "title": "Objecttype", + "type": "string" + }, + "id": { + "description": "Stable object ID.", "format": "uuid", - "title": "Itemid", + "title": "Id", "type": "string" }, - "waitSeconds": { - "default": 0, - "description": "Wait for status changes, up to 30 seconds; zero returns immediately.", - "maximum": 30, - "minimum": 0, - "title": "Waitseconds", - "type": "number" + "revision": { + "description": "Current object revision for optimistic concurrency.", + "minimum": 1, + "title": "Revision", + "type": "integer" } }, "required": [ - "itemId", - "waitSeconds" + "objectType", + "id", + "revision" ], - "title": "CanvasItemQuery", + "title": "CanvasItemVersion", "type": "object" }, "CanvasMaterialize": { @@ -16454,6 +16997,25 @@ "title": "CanvasPublishMediaCommand", "type": "object" }, + "CanvasReferenceSources": { + "additionalProperties": false, + "properties": { + "followCurrent": { + "$ref": "#/$defs/FollowingReferenceSource", + "description": "Source fields for a follow_current reference." + }, + "pinned": { + "$ref": "#/$defs/PinnedReferenceSource", + "description": "Source fields for a pinned reference retaining its canvas connection." + } + }, + "required": [ + "followCurrent", + "pinned" + ], + "title": "CanvasReferenceSources", + "type": "object" + }, "CanvasShow": { "additionalProperties": false, "properties": { @@ -16602,6 +17164,52 @@ "title": "CanvasShowCommandOperation", "type": "object" }, + "CanvasStatusQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "canvas.status", + "description": "Read node status, current asset and reusable referenceSources.", + "title": "Type", + "type": "string" + }, + "objectId": { + "description": "Canvas item ID.", + "format": "uuid", + "title": "Objectid", + "type": "string" + }, + "waitSeconds": { + "default": 0, + "description": "Wait for generation changes; zero returns immediately.", + "maximum": 30, + "minimum": 0, + "title": "Waitseconds", + "type": "number" + } + }, + "required": [ + "queryId", + "projectId", + "type", + "objectId", + "waitSeconds" + ], + "title": "CanvasStatusQuery", + "type": "object" + }, "CanvasUpdateCommand": { "additionalProperties": false, "description": "Update canvas presentation, grouping or lock state.", @@ -17267,8 +17875,45 @@ "description": "Immutable command execution receipt." }, "query": { - "$ref": "#/$defs/Query", - "description": "Read-only project query." + "description": "Read-only project query.", + "discriminator": { + "mapping": { + "asset.preview": "#/$defs/AssetPreviewQuery", + "asset.search": "#/$defs/AssetSearchQuery", + "asset.usage": "#/$defs/AssetUsageQuery", + "canvas.status": "#/$defs/CanvasStatusQuery", + "canvas.versions": "#/$defs/VersionsQuery", + "generationSpec.versions": "#/$defs/VersionsQuery", + "job.get": "#/$defs/ObjectQuery", + "object.get": "#/$defs/ObjectQuery", + "project.get": "#/$defs/ProjectQuery" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/$defs/ProjectQuery" + }, + { + "$ref": "#/$defs/ObjectQuery" + }, + { + "$ref": "#/$defs/VersionsQuery" + }, + { + "$ref": "#/$defs/AssetSearchQuery" + }, + { + "$ref": "#/$defs/AssetUsageQuery" + }, + { + "$ref": "#/$defs/CanvasStatusQuery" + }, + { + "$ref": "#/$defs/AssetPreviewQuery" + } + ], + "title": "Query" }, "event": { "$ref": "#/$defs/ProjectEvent", @@ -18407,13 +19052,13 @@ }, "binding": { "const": "follow_current", - "description": "Business object displayed by this canvas item.", + "description": "Follow the source canvas item's current asset.", "title": "Binding", "type": "string" }, "source": { "$ref": "#/$defs/FollowingSource", - "description": "Input source or recorded provenance." + "description": "Source canvas_item identity." } }, "required": [ @@ -18430,6 +19075,28 @@ "title": "FollowingReference", "type": "object" }, + "FollowingReferenceSource": { + "additionalProperties": false, + "properties": { + "binding": { + "const": "follow_current", + "default": "follow_current", + "description": "Follow the canvas source current asset.", + "title": "Binding", + "type": "string" + }, + "source": { + "$ref": "#/$defs/FollowingSource", + "description": "Source canvas_item identity." + } + }, + "required": [ + "binding", + "source" + ], + "title": "FollowingReferenceSource", + "type": "object" + }, "FollowingSource": { "additionalProperties": false, "properties": { @@ -20074,6 +20741,24 @@ ], "default": null, "description": "Supported video generation capabilities." + }, + "generateAudioOptions": { + "anyOf": [ + { + "items": { + "type": "boolean" + }, + "maxItems": 2, + "minItems": 1, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Allowed generateAudio values; omitted in legacy execution snapshots.", + "title": "Generateaudiooptions" } }, "required": [ @@ -22093,6 +22778,61 @@ "title": "NodeVersionApplyCommand", "type": "object" }, + "ObjectQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "description": "Read an object or generation job.", + "enum": [ + "object.get", + "job.get" + ], + "title": "Type", + "type": "string" + }, + "objectId": { + "description": "Target business object ID.", + "format": "uuid", + "title": "Objectid", + "type": "string" + }, + "revision": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Historical object revision, or current when omitted.", + "title": "Revision" + } + }, + "required": [ + "queryId", + "projectId", + "type", + "objectId", + "revision" + ], + "title": "ObjectQuery", + "type": "object" + }, "ObjectVersion": { "additionalProperties": false, "properties": { @@ -22248,25 +22988,25 @@ }, "binding": { "const": "pinned", - "description": "Business object displayed by this canvas item.", + "description": "Use a fixed asset version, retaining canvas provenance when supplied.", "title": "Binding", "type": "string" }, "source": { "$ref": "#/$defs/AssetRef", - "description": "Input source or recorded provenance." + "description": "Fixed asset version used by generation." }, "sourceVersion": { "anyOf": [ { - "$ref": "#/$defs/ObjectVersion" + "$ref": "#/$defs/CanvasItemVersion" }, { "type": "null" } ], "default": null, - "description": "Captured source object version." + "description": "Source canvas_item version for its connection; omit for asset-only references." }, "subjectRole": { "anyOf": [ @@ -22301,6 +23041,33 @@ "title": "PinnedReference", "type": "object" }, + "PinnedReferenceSource": { + "additionalProperties": false, + "properties": { + "binding": { + "const": "pinned", + "default": "pinned", + "description": "Use the fixed asset version.", + "title": "Binding", + "type": "string" + }, + "source": { + "$ref": "#/$defs/AssetRef", + "description": "Fixed current asset version." + }, + "sourceVersion": { + "$ref": "#/$defs/CanvasItemVersion", + "description": "Canvas source version retaining the connection." + } + }, + "required": [ + "binding", + "source", + "sourceVersion" + ], + "title": "PinnedReferenceSource", + "type": "object" + }, "Playhead": { "additionalProperties": false, "properties": { @@ -22570,6 +23337,36 @@ "title": "ProjectEvent", "type": "object" }, + "ProjectQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "const": "project.get", + "description": "Read the current project.", + "title": "Type", + "type": "string" + } + }, + "required": [ + "queryId", + "projectId", + "type" + ], + "title": "ProjectQuery", + "type": "object" + }, "ProjectRename": { "additionalProperties": false, "properties": { @@ -23057,7 +23854,7 @@ "title": "PromptText", "type": "object" }, - "Query": { + "QueryBase": { "additionalProperties": false, "properties": { "queryId": { @@ -23071,83 +23868,13 @@ "format": "uuid", "title": "Projectid", "type": "string" - }, - "type": { - "description": "Operation or variant discriminator.", - "enum": [ - "project.get", - "object.get", - "asset.search", - "asset.usage", - "generationSpec.versions", - "canvas.versions", - "job.get" - ], - "title": "Type", - "type": "string" - }, - "objectId": { - "anyOf": [ - { - "format": "uuid", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Target business object ID.", - "title": "Objectid" - }, - "revision": { - "anyOf": [ - { - "minimum": 1, - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Historical revision to read, when specified.", - "title": "Revision" - }, - "text": { - "default": "", - "description": "UTF-8 text content or search query.", - "maxLength": 1000, - "title": "Text", - "type": "string" - }, - "offset": { - "default": 0, - "description": "Number of matching records to skip.", - "minimum": 0, - "title": "Offset", - "type": "integer" - }, - "limit": { - "default": 50, - "description": "Maximum number of records to return.", - "maximum": 200, - "minimum": 1, - "title": "Limit", - "type": "integer" } }, "required": [ "queryId", - "projectId", - "type", - "objectId", - "revision", - "text", - "offset", - "limit" + "projectId" ], - "title": "Query", + "title": "QueryBase", "type": "object" }, "RectangleLayerContent": { @@ -24379,7 +25106,7 @@ "additionalProperties": false, "properties": { "spec": { - "description": "Generation configuration for this operation.", + "description": "Configuration with its current revision; the server increments it on commit.", "discriminator": { "mapping": { "audio": "#/$defs/AudioGenerationSpec", @@ -26532,6 +27259,46 @@ "title": "Vector3", "type": "object" }, + "VersionsQuery": { + "additionalProperties": false, + "properties": { + "queryId": { + "description": "Caller-supplied query ID.", + "format": "uuid", + "title": "Queryid", + "type": "string" + }, + "projectId": { + "description": "Owning project ID.", + "format": "uuid", + "title": "Projectid", + "type": "string" + }, + "type": { + "description": "Read object versions.", + "enum": [ + "generationSpec.versions", + "canvas.versions" + ], + "title": "Type", + "type": "string" + }, + "objectId": { + "description": "Target configuration or canvas item ID.", + "format": "uuid", + "title": "Objectid", + "type": "string" + } + }, + "required": [ + "queryId", + "projectId", + "type", + "objectId" + ], + "title": "VersionsQuery", + "type": "object" + }, "VideoCanvasItem": { "additionalProperties": false, "properties": { @@ -27084,7 +27851,7 @@ "properties": { "generateAudio": { "default": true, - "description": "Whether video generation includes audio.", + "description": "Include audio; value must be allowed by model inputRules.generateAudioOptions.", "title": "Generateaudio", "type": "boolean" }, @@ -27583,8 +28350,45 @@ "description": "Immutable command execution receipt." }, "query": { - "$ref": "#/$defs/Query", - "description": "Read-only project query." + "description": "Read-only project query.", + "discriminator": { + "mapping": { + "asset.preview": "#/$defs/AssetPreviewQuery", + "asset.search": "#/$defs/AssetSearchQuery", + "asset.usage": "#/$defs/AssetUsageQuery", + "canvas.status": "#/$defs/CanvasStatusQuery", + "canvas.versions": "#/$defs/VersionsQuery", + "generationSpec.versions": "#/$defs/VersionsQuery", + "job.get": "#/$defs/ObjectQuery", + "object.get": "#/$defs/ObjectQuery", + "project.get": "#/$defs/ProjectQuery" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/$defs/ProjectQuery" + }, + { + "$ref": "#/$defs/ObjectQuery" + }, + { + "$ref": "#/$defs/VersionsQuery" + }, + { + "$ref": "#/$defs/AssetSearchQuery" + }, + { + "$ref": "#/$defs/AssetUsageQuery" + }, + { + "$ref": "#/$defs/CanvasStatusQuery" + }, + { + "$ref": "#/$defs/AssetPreviewQuery" + } + ], + "title": "Query" }, "event": { "$ref": "#/$defs/ProjectEvent", diff --git a/src/contracts/generated.ts b/src/contracts/generated.ts index b28b13d07..fa98b081b 100644 --- a/src/contracts/generated.ts +++ b/src/contracts/generated.ts @@ -54,8 +54,9 @@ export interface AssetMetadataUpdateInput extends ContractInput { readonly name: string; } -export interface AssetPreviewQueryInput extends ContractInput { - readonly assetId: string; +export interface AssetPreviewQueryInput extends QueryBaseInput { + readonly type: "asset.preview"; + readonly objectId: string; } export interface AssetProvenanceInput extends ContractInput { @@ -80,6 +81,20 @@ export interface AssetRefInput extends ContractInput { readonly kind: "text" | "image" | "video" | "audio" | "file"; } +export interface AssetSearchQueryInput extends QueryBaseInput { + readonly type: "asset.search"; + readonly text?: string; + readonly offset?: number; + readonly limit?: number; +} + +export interface AssetUsageQueryInput extends QueryBaseInput { + readonly type: "asset.usage"; + readonly objectId?: string | null; + readonly offset?: number; + readonly limit?: number; +} + export interface AssetWriteTextInput extends ContractInput { readonly name: string; readonly text: string; @@ -241,9 +256,8 @@ export interface CanvasItemBaseInput extends ProjectEntityInput { readonly tags?: ReadonlyArray; } -export interface CanvasItemQueryInput extends ContractInput { - readonly itemId: string; - readonly waitSeconds?: number; +export interface CanvasItemVersionInput extends ObjectVersionInput { + readonly objectType: "canvas_item"; } export interface CanvasMaterializeInput extends ContractInput { @@ -304,6 +318,11 @@ export interface CanvasPublishMediaCommandInput extends CommandBaseInput { readonly payload: CanvasPublishMediaInput; } +export interface CanvasReferenceSourcesInput extends ContractInput { + readonly followCurrent: FollowingReferenceSourceInput; + readonly pinned: PinnedReferenceSourceInput; +} + export interface CanvasShowInput extends ContractInput { readonly item: TextCanvasItemInput | ImageCanvasItemInput | VideoCanvasItemInput | AudioCanvasItemInput | FileCanvasItemInput | SubjectCanvasItemInput | TimelineCanvasItemInput | StackCanvasItemInput | ToolCanvasItemInput | DirectorCanvasItemInput | GroupCanvasItemInput; } @@ -318,6 +337,12 @@ export interface CanvasShowCommandOperationInput { readonly payload: CanvasShowInput; } +export interface CanvasStatusQueryInput extends QueryBaseInput { + readonly type: "canvas.status"; + readonly objectId: string; + readonly waitSeconds?: number; +} + export interface CanvasUpdateCommandInput extends CommandBaseInput { readonly type: "canvas.update"; readonly payload: CanvasPresentationUpdateInput; @@ -374,7 +399,7 @@ export interface CreationContractsInput extends ContractInput { readonly editorDocument: TimelineDocumentInput | SubjectDocumentInput | LayerDocumentInput | ImageCropDocumentInput | StackDocumentInput | ToolDocumentInput | DirectorDocumentInput | MediaEditDocumentInput; readonly command: ProjectCreateCommandInput | ProjectRenameCommandInput | ProjectTrashCommandInput | ProjectRestoreCommandInput | AssetImportCommandInput | ArtifactPublishCommandInput | AssetWriteTextCommandInput | AssetMetadataCommandInput | AssetDeleteCommandInput | CanvasCreateGenerationCommandInput | CanvasShowCommandInput | CanvasMoveCommandInput | CanvasUpdateCommandInput | CanvasHideCommandInput | CanvasMaterializeCommandInput | CanvasApplyEditCommandInput | CanvasPublishMediaCommandInput | TimelinePublishRenderCommandInput | NodeVersionApplyCommandInput | CanvasConfigureCommandInput | CanvasEditorAttachCommandInput | CanvasEditorDetachCommandInput | SpecCreateCommandInput | SpecUpdateCommandInput | SpecRestoreCommandInput | SpecPromptCommandInput | ReferenceAddCommandInput | ReferenceUpdateCommandInput | ReferenceRestoreCommandInput | ReferenceRemoveCommandInput | GenerationSubmitCommandInput | JobCancelCommandInput | JobRecoverCommandInput | JobCheckpointCommandInput | ResultSelectCommandInput | EditorCreateCommandInput | EditorSaveCommandInput | HistoryUndoCommandInput | HistoryRedoCommandInput | BatchCommandInput; readonly receipt: CommandReceiptInput; - readonly query: QueryInput; + readonly query: ProjectQueryInput | ObjectQueryInput | VersionsQueryInput | AssetSearchQueryInput | AssetUsageQueryInput | CanvasStatusQueryInput | AssetPreviewQueryInput; readonly event: ProjectEventInput; readonly job: JobInput; readonly result: ResultVersionInput; @@ -478,6 +503,11 @@ export interface FollowingReferenceInput extends ReferenceBaseInput { readonly source: FollowingSourceInput; } +export interface FollowingReferenceSourceInput extends ContractInput { + readonly binding?: "follow_current"; + readonly source: FollowingSourceInput; +} + export interface FollowingSourceInput extends ContractInput { readonly objectType?: "canvas_item"; readonly id: string; @@ -640,6 +670,7 @@ export interface InputRulesInput extends ContractInput { readonly outputResolutions?: ReadonlyArray; readonly referenceImageResolutions?: ReadonlyArray | null; readonly videoCapabilities?: VideoCapabilitiesInput | null; + readonly generateAudioOptions?: ReadonlyArray | null; } export interface InputSnapshotInput extends ContractInput { @@ -828,6 +859,12 @@ export interface NodeVersionApplyCommandInput extends CommandBaseInput { readonly payload: NodeVersionApplyInput; } +export interface ObjectQueryInput extends QueryBaseInput { + readonly type: "object.get" | "job.get"; + readonly objectId: string; + readonly revision?: number | null; +} + export interface ObjectVersionInput extends ContractInput { readonly objectType: "project" | "asset" | "generation_spec" | "reference" | "canvas_item" | "editor_document" | "job" | "result"; readonly id: string; @@ -847,10 +884,16 @@ export interface PaletteProvenanceInput extends ContractInput { export interface PinnedReferenceInput extends ReferenceBaseInput { readonly binding: "pinned"; readonly source: AssetRefInput; - readonly sourceVersion?: ObjectVersionInput | null; + readonly sourceVersion?: CanvasItemVersionInput | null; readonly subjectRole?: "main" | "auxiliary" | "voice" | null; } +export interface PinnedReferenceSourceInput extends ContractInput { + readonly binding?: "pinned"; + readonly source: AssetRefInput; + readonly sourceVersion: CanvasItemVersionInput; +} + export interface PlayheadInput extends ContractInput { readonly editorDocumentId: string; readonly seconds?: number; @@ -892,6 +935,10 @@ export interface ProjectEventInput extends ContractInput { readonly jobIds?: ReadonlyArray; } +export interface ProjectQueryInput extends QueryBaseInput { + readonly type: "project.get"; +} + export interface ProjectRenameInput extends ProjectCreateInput { } @@ -935,15 +982,9 @@ export interface PromptTextInput extends ContractInput { readonly text: string; } -export interface QueryInput extends ContractInput { +export interface QueryBaseInput extends ContractInput { readonly queryId: string; readonly projectId: string; - readonly type: "project.get" | "object.get" | "asset.search" | "asset.usage" | "generationSpec.versions" | "canvas.versions" | "job.get"; - readonly objectId?: string | null; - readonly revision?: number | null; - readonly text?: string; - readonly offset?: number; - readonly limit?: number; } export interface RectangleLayerContentInput extends LayerContentBaseInput { @@ -1318,6 +1359,11 @@ export interface Vector3Input extends ContractInput { readonly z: number; } +export interface VersionsQueryInput extends QueryBaseInput { + readonly type: "generationSpec.versions" | "canvas.versions"; + readonly objectId: string; +} + export interface VideoCanvasItemInput extends MediaCanvasItemInput { readonly kind: "video"; } @@ -1451,8 +1497,9 @@ export interface AssetMetadataUpdate extends Contract { readonly name: string; } -export interface AssetPreviewQuery extends Contract { - readonly assetId: string; +export interface AssetPreviewQuery extends QueryBase { + readonly type: "asset.preview"; + readonly objectId: string; } export interface AssetProvenance extends Contract { @@ -1477,6 +1524,20 @@ export interface AssetRef extends Contract { readonly kind: "text" | "image" | "video" | "audio" | "file"; } +export interface AssetSearchQuery extends QueryBase { + readonly type: "asset.search"; + readonly text: string; + readonly offset: number; + readonly limit: number; +} + +export interface AssetUsageQuery extends QueryBase { + readonly type: "asset.usage"; + readonly objectId: string | null; + readonly offset: number; + readonly limit: number; +} + export interface AssetWriteText extends Contract { readonly name: string; readonly text: string; @@ -1638,9 +1699,8 @@ export interface CanvasItemBase extends ProjectEntity { readonly tags: ReadonlyArray; } -export interface CanvasItemQuery extends Contract { - readonly itemId: string; - readonly waitSeconds: number; +export interface CanvasItemVersion extends ObjectVersion { + readonly objectType: "canvas_item"; } export interface CanvasMaterialize extends Contract { @@ -1701,6 +1761,11 @@ export interface CanvasPublishMediaCommand extends CommandBase { readonly payload: CanvasPublishMedia; } +export interface CanvasReferenceSources extends Contract { + readonly followCurrent: FollowingReferenceSource; + readonly pinned: PinnedReferenceSource; +} + export interface CanvasShow extends Contract { readonly item: TextCanvasItem | ImageCanvasItem | VideoCanvasItem | AudioCanvasItem | FileCanvasItem | SubjectCanvasItem | TimelineCanvasItem | StackCanvasItem | ToolCanvasItem | DirectorCanvasItem | GroupCanvasItem; } @@ -1715,6 +1780,12 @@ export interface CanvasShowCommandOperation { readonly payload: CanvasShow; } +export interface CanvasStatusQuery extends QueryBase { + readonly type: "canvas.status"; + readonly objectId: string; + readonly waitSeconds: number; +} + export interface CanvasUpdateCommand extends CommandBase { readonly type: "canvas.update"; readonly payload: CanvasPresentationUpdate; @@ -1771,7 +1842,7 @@ export interface CreationContracts extends Contract { readonly editorDocument: TimelineDocument | SubjectDocument | LayerDocument | ImageCropDocument | StackDocument | ToolDocument | DirectorDocument | MediaEditDocument; readonly command: ProjectCreateCommand | ProjectRenameCommand | ProjectTrashCommand | ProjectRestoreCommand | AssetImportCommand | ArtifactPublishCommand | AssetWriteTextCommand | AssetMetadataCommand | AssetDeleteCommand | CanvasCreateGenerationCommand | CanvasShowCommand | CanvasMoveCommand | CanvasUpdateCommand | CanvasHideCommand | CanvasMaterializeCommand | CanvasApplyEditCommand | CanvasPublishMediaCommand | TimelinePublishRenderCommand | NodeVersionApplyCommand | CanvasConfigureCommand | CanvasEditorAttachCommand | CanvasEditorDetachCommand | SpecCreateCommand | SpecUpdateCommand | SpecRestoreCommand | SpecPromptCommand | ReferenceAddCommand | ReferenceUpdateCommand | ReferenceRestoreCommand | ReferenceRemoveCommand | GenerationSubmitCommand | JobCancelCommand | JobRecoverCommand | JobCheckpointCommand | ResultSelectCommand | EditorCreateCommand | EditorSaveCommand | HistoryUndoCommand | HistoryRedoCommand | BatchCommand; readonly receipt: CommandReceipt; - readonly query: Query; + readonly query: ProjectQuery | ObjectQuery | VersionsQuery | AssetSearchQuery | AssetUsageQuery | CanvasStatusQuery | AssetPreviewQuery; readonly event: ProjectEvent; readonly job: Job; readonly result: ResultVersion; @@ -1875,6 +1946,11 @@ export interface FollowingReference extends ReferenceBase { readonly source: FollowingSource; } +export interface FollowingReferenceSource extends Contract { + readonly binding: "follow_current"; + readonly source: FollowingSource; +} + export interface FollowingSource extends Contract { readonly objectType: "canvas_item"; readonly id: string; @@ -2037,6 +2113,7 @@ export interface InputRules extends Contract { readonly outputResolutions: ReadonlyArray; readonly referenceImageResolutions: ReadonlyArray | null; readonly videoCapabilities: VideoCapabilities | null; + readonly generateAudioOptions?: ReadonlyArray | null; } export interface InputSnapshot extends Contract { @@ -2224,6 +2301,12 @@ export interface NodeVersionApplyCommand extends CommandBase { readonly payload: NodeVersionApply; } +export interface ObjectQuery extends QueryBase { + readonly type: "object.get" | "job.get"; + readonly objectId: string; + readonly revision: number | null; +} + export interface ObjectVersion extends Contract { readonly objectType: "project" | "asset" | "generation_spec" | "reference" | "canvas_item" | "editor_document" | "job" | "result"; readonly id: string; @@ -2243,10 +2326,16 @@ export interface PaletteProvenance extends Contract { export interface PinnedReference extends ReferenceBase { readonly binding: "pinned"; readonly source: AssetRef; - readonly sourceVersion?: ObjectVersion | null; + readonly sourceVersion?: CanvasItemVersion | null; readonly subjectRole?: "main" | "auxiliary" | "voice" | null; } +export interface PinnedReferenceSource extends Contract { + readonly binding: "pinned"; + readonly source: AssetRef; + readonly sourceVersion: CanvasItemVersion; +} + export interface Playhead extends Contract { readonly editorDocumentId: string; readonly seconds: number; @@ -2288,6 +2377,10 @@ export interface ProjectEvent extends Contract { readonly jobIds: ReadonlyArray; } +export interface ProjectQuery extends QueryBase { + readonly type: "project.get"; +} + export interface ProjectRename extends ProjectCreate { } @@ -2331,15 +2424,9 @@ export interface PromptText extends Contract { readonly text: string; } -export interface Query extends Contract { +export interface QueryBase extends Contract { readonly queryId: string; readonly projectId: string; - readonly type: "project.get" | "object.get" | "asset.search" | "asset.usage" | "generationSpec.versions" | "canvas.versions" | "job.get"; - readonly objectId: string | null; - readonly revision: number | null; - readonly text: string; - readonly offset: number; - readonly limit: number; } export interface RectangleLayerContent extends LayerContentBase { @@ -2714,6 +2801,11 @@ export interface Vector3 extends Contract { readonly z: number; } +export interface VersionsQuery extends QueryBase { + readonly type: "generationSpec.versions" | "canvas.versions"; + readonly objectId: string; +} + export interface VideoCanvasItem extends MediaCanvasItem { readonly kind: "video"; } diff --git a/tests/assistantActivity.test.ts b/tests/assistantActivity.test.ts new file mode 100644 index 000000000..a94bb8d4f --- /dev/null +++ b/tests/assistantActivity.test.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import type { ChatMessage, ChatTool } from "../src/backend/chat.ts"; +import { activityProgress, replyParts } from "../src/canvas/assistantActivity.ts"; + +const tool = (id: string, offset: number, status: ChatTool["status"] = "complete"): ChatTool => ({ + id, textOffset: offset, name: "web_fetch", status, +}); +const message = (tools: ChatTool[], text = ""): ChatMessage => ({ + id: "reply", role: "assistant", status: "streaming", text, tools, +}); + +test("groups adjacent calls while preserving public messages and Unicode boundaries", () => { + const parts = replyParts(message([ + tool("a", 0), tool("b", 0), tool("c", 2), tool("d", 2), + ], "🌊查结果")); + assert.deepEqual(parts, [ + { type: "tools", id: "a", tools: [tool("a", 0), tool("b", 0)] }, + { type: "text", id: "text-c", text: "🌊查" }, + { type: "tools", id: "c", tools: [tool("c", 2), tool("d", 2)] }, + { type: "text", id: "tail", text: "结果" }, + ]); +}); + +test("keeps one stable group identity as a large call sequence grows", () => { + const tools = Array.from({ length: 1000 }, (_, index) => tool(String(index), 0)); + const before = replyParts(message(tools.slice(0, 1))); + const after = replyParts(message(tools)); + assert.equal(before[0].id, after[0].id); + assert.equal(after.length, 1); + assert.equal(after[0].type, "tools"); + if (after[0].type === "tools") assert.equal(after[0].tools.length, 1000); + assert.equal(tools.length, 1000); +}); + +test("renders text-only and empty messages without synthetic activities", () => { + assert.deepEqual(replyParts(message([], "完成")), [{ type: "text", id: "tail", text: "完成" }]); + assert.deepEqual(replyParts(message([])), []); +}); + +test("clamps historical tool offsets without losing or repeating text", () => { + const parts = replyParts(message([tool("a", -1), tool("b", 20), tool("c", 1)], "结果")); + assert.deepEqual(parts.filter((part) => part.type === "text").map((part) => part.text), ["结果"]); + assert.deepEqual(parts.filter((part) => part.type === "tools").flatMap((part) => part.tools.map((call) => call.id)), ["a", "b", "c"]); +}); + +test("activity progress follows running calls, thinking and plan state", () => { + const reply = message([tool("a", 0, "running"), tool("b", 0, "running")]); + assert.equal(activityProgress(reply, true), "正在执行：读取网页（2 项进行中）"); + reply.tools = [tool("a", 0, "complete")]; + assert.equal(activityProgress(reply, true), "正在思考…"); + assert.equal(activityProgress(reply, false), "正在回复…"); + assert.equal(activityProgress({ ...reply, plan: [{ content: "检查结果", status: "in_progress" }] }, false), "正在执行计划：检查结果"); +}); diff --git a/tests/browser/assistant_activity.html b/tests/browser/assistant_activity.html new file mode 100644 index 000000000..127cc13e3 --- /dev/null +++ b/tests/browser/assistant_activity.html @@ -0,0 +1,8 @@ + + + Assistant activity test + +
+ + + diff --git a/tests/browser/assistant_activity.mjs b/tests/browser/assistant_activity.mjs new file mode 100644 index 000000000..7d31531a6 --- /dev/null +++ b/tests/browser/assistant_activity.mjs @@ -0,0 +1,79 @@ +/** Harness-independent activity rendering against Vite; no project or model calls. */ +import assert from "node:assert/strict"; +import { launchBrowser } from "./engine.mjs"; + +const base = process.env.CREATION_WEB_URL ?? "http://127.0.0.1:5179"; +const browser = await launchBrowser(); +const page = await browser.newPage({ viewport: { width: 900, height: 900 } }); +page.setDefaultTimeout(10000); +const errors = []; +page.on("pageerror", (error) => errors.push(error.message)); +const prefix = "先查询资料。🌊"; +const tools = (count) => Array.from({ length: count }, (_, index) => ({ + id: `call-${index}`, name: "web_fetch", textOffset: Array.from(prefix).length, + status: index === count - 1 ? "running" : "complete", + arguments: { url: `https://example.test/${index}` }, +})); +const message = (calls, text = prefix) => ({ + id: "reply", role: "assistant", status: "streaming", text, tools: calls, +}); +try { + await page.goto(base + "/tests/browser/assistant_activity.html"); + await page.evaluate(async () => { + const { mountReply } = await import("/tests/browser/assistant_activity_fixture.tsx"); + window.replyFixture = mountReply(document.getElementById("activity-test")); + }); + const render = (value, thinking = false) => page.evaluate(({ value, thinking }) => + window.replyFixture.render(value, thinking), { value, thinking }); + const group = page.locator(".cw-chat-tool-group > details"); + const calls = tools(1000); + await render(message(calls)); + await group.getByText("执行记录 · 1000 次调用", { exact: true }).waitFor(); + await page.locator(".cw-chat-tool-group-items").waitFor({ state: "detached" }); + assert.equal(await group.getAttribute("open"), null); + assert.equal(await page.locator('[role="status"]').innerText(), "正在执行：读取网页"); + assert.equal(await page.locator(".cw-chat-markdown").innerText(), prefix); + + await group.locator(":scope > summary").click(); + await page.waitForFunction(() => document.querySelectorAll(".cw-chat-tool-group-items > details").length === 50); + const first = group.locator(".cw-chat-tool-group-items > details").first(); + await first.locator(":scope > summary").click(); + calls.push({ ...calls.at(-1), id: "call-1000" }); + await render(message(calls)); + await page.waitForFunction(() => document.querySelectorAll(".cw-chat-tool-group-items > details").length === 51); + assert.notEqual(await first.getAttribute("open"), null); + assert.notEqual(await group.getAttribute("open"), null); + await group.getByRole("button", { name: "显示更早的 50 次调用", exact: true }).click(); + await page.waitForFunction(() => document.querySelectorAll(".cw-chat-tool-group-items > details").length === 101); + + await group.locator(":scope > summary").click(); + await page.waitForFunction(() => !document.querySelector(".cw-chat-tool-group-items")); + calls.forEach((call, index) => { + if (call.status === "running") calls[index] = { ...call, status: "complete" }; + }); + calls[calls.length - 1] = { ...calls.at(-1), status: "failed", error: "网页无法访问" }; + await render({ ...message(calls, prefix + "已完成。"), status: "complete" }); + await group.getByText("1000/1001 已完成 · 1 失败", { exact: true }).waitFor(); + assert.equal(await page.getByText("网页无法访问", { exact: true }).count(), 0); + assert.equal(await group.evaluate((element) => element.classList.contains('is-failed')), false); + await group.locator(":scope > summary").click(); + const failure = page.locator('.cw-chat-tool-group-items > details.is-failed'); + await failure.locator(':scope > summary').click(); + await page.getByText("网页无法访问", { exact: true }).waitFor(); + await group.locator(":scope > summary").click(); + await page.locator(".cw-chat-tool-group-items").waitFor({ state: "detached" }); + assert.equal(await page.locator(".cw-chat-markdown").last().innerText(), "已完成。"); + assert.equal(await page.locator('[role="status"]').count(), 0); + + await render({ ...message([]), plan: [{ content: "核对素材", status: "in_progress" }] }); + await page.getByText("正在执行计划：核对素材", { exact: true }).waitFor(); + await page.locator(".cw-chat-plan > summary").click(); + await page.getByText("核对素材", { exact: true }).waitFor(); + await render(message([]), true); + await page.getByText("正在思考…", { exact: true }).waitFor(); + assert.deepEqual(errors, []); + console.log(JSON.stringify({ passed: true, checks: ["1000 collapsed calls", "lazy pagination", "stable disclosure state", "muted failure with expandable details", "public messages", "plan and thinking"] })); +} finally { + await page.evaluate(() => window.replyFixture?.unmount()).catch(() => {}); + await browser.close(); +} diff --git a/tests/browser/assistant_activity_fixture.tsx b/tests/browser/assistant_activity_fixture.tsx new file mode 100644 index 000000000..635d7b36a --- /dev/null +++ b/tests/browser/assistant_activity_fixture.tsx @@ -0,0 +1,15 @@ +import { createElement } from "react"; +import { createRoot } from "react-dom/client"; +import type { ChatMessage } from "../../src/backend/chat"; +import { AssistantReply } from "../../src/canvas/AssistantReply"; +import "../../src/canvas/assistant-experience.css"; + +export function mountReply(element: HTMLElement) { + const root = createRoot(element); + return { + render(message: ChatMessage, thinking = false) { + root.render(createElement(AssistantReply, { message, thinking })); + }, + unmount() { root.unmount(); }, + }; +} diff --git a/tests/browser/canvas_generation_reference.mjs b/tests/browser/canvas_generation_reference.mjs new file mode 100644 index 000000000..065eea391 --- /dev/null +++ b/tests/browser/canvas_generation_reference.mjs @@ -0,0 +1,66 @@ +/** Real image-to-video canvas references using shared query/command contracts. No generation calls. */ +import assert from "node:assert/strict"; +import { launchBrowser } from "./engine.mjs"; +const base = process.env.CREATION_WEB_URL ?? "http://127.0.0.1:5179"; +const api = process.env.CREATION_API_URL ?? "http://127.0.0.1:8019"; +const headers = { "X-OpenDirector-Request": "1", "Content-Type": "application/json" }; +async function call(path, body) { + const response = await fetch(api + path, { headers, + ...(body ? { method: "POST", body: JSON.stringify(body) } : {}) }); + assert.ok(response.ok, await response.clone().text()); + return response.json(); +} +assert.equal((await call("/api/creation-test-health")).isolated, true); +const browser = await launchBrowser(); +const page = await browser.newPage({ viewport: { width: 1600, height: 1000 } }); +page.setDefaultTimeout(15000); +try { + for (const binding of ["follow_current", "pinned"]) { + const projectId = crypto.randomUUID(); + await call(`/api/creation/projects/${projectId}/commands`, { commandId: crypto.randomUUID(), projectId, + type: "project.create", payload: { name: `Connected generation ${binding} ${projectId.slice(0, 8)}` } }); + await page.goto(`${base}/ai-tool/ai-canvas/${projectId}?creation=unified`); + await page.getByText("已保存到本机", { exact: true }).waitFor(); + await page.getByLabel("选择本地素材文件", { exact: true }).setInputFiles({ + name: "reference.png", mimeType: "image/png", + buffer: Buffer.from("iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAGUlEQVR4nGNsYGhgIAUwkaR6VMOohiGlAQBCPQEgiSD+iQAAAABJRU5ErkJggg==", "base64"), + }); + let source; + for (let attempt = 0; attempt < 150; attempt++) { + const snapshot = await call(`/api/creation/projects/${projectId}`); + source = snapshot.canvasItems[0]; + if (source && snapshot.assets.length) break; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + assert.ok(source); + const created = await page.evaluate(async ({ projectId, binding, source }) => { + const { request } = await import("/src/backend/api.ts"); + const { creationClient } = await import("/src/backend/creationClient.ts"); + const client = creationClient(projectId); + await client.refresh(); + const status = await request(`/creation/projects/${projectId}/canvas/${source.id}/status`); + const fields = status.referenceSources[binding === "pinned" ? "pinned" : "followCurrent"]; + const specId = crypto.randomUUID(), itemId = crypto.randomUUID(), referenceId = crypto.randomUUID(); + const receipt = await client.command("canvas.createGeneration", { + spec: { id: specId, projectId, kind: "video", purpose: "generation", prompt: "Animate the reference", + parameters: { durationSeconds: 5, videoMode: "reference" } }, + item: { id: itemId, projectId, kind: "video", title: `Connected ${binding}`, + position: { x: source.position.x + 500, y: source.position.y }, + binding: { role: "generation", objectRef: { id: specId } } }, + references: [{ id: referenceId, projectId, targetSpecId: specId, ...fields }], + }); + if (receipt.status !== "applied") throw Error(JSON.stringify(receipt)); + return { sourceId: source.id, itemId, referenceId }; + }, { projectId, binding, source }); + await page.locator(`.react-flow__node[data-id="${created.itemId}"]`).waitFor(); + await page.locator(`.react-flow__edge[data-id="${created.referenceId}"]`).waitFor(); + const snapshot = await call(`/api/creation/projects/${projectId}`); + assert.equal(snapshot.jobs.length, 0); + assert.equal(snapshot.references[0].binding, binding); + assert.equal(snapshot.canvasItems.find((item) => item.id === created.itemId).visible, true); + if (binding === "pinned") assert.equal(snapshot.references[0].sourceVersion.id, created.sourceId); + } + console.log(JSON.stringify({ passed: true, checks: ["reusable source fields", "visible video before submission", "follow_current connection", "pinned connection"] })); +} finally { + await browser.close(); +} diff --git a/tests/browser/created_node_focus.mjs b/tests/browser/created_node_focus.mjs new file mode 100644 index 000000000..d75977fac --- /dev/null +++ b/tests/browser/created_node_focus.mjs @@ -0,0 +1,88 @@ +/** Remote creation changes only the viewport; no paid generation. */ +import assert from "node:assert/strict"; +import { launchBrowser } from "./engine.mjs"; +const base = process.env.CREATION_WEB_URL ?? "http://127.0.0.1:5179"; +const api = process.env.CREATION_API_URL ?? "http://127.0.0.1:8019"; +const headers = { "X-OpenDirector-Request": "1", "Content-Type": "application/json" }; +async function call(path, body) { + const response = await fetch(api + path, { headers, ...(body ? { method: "POST", body: JSON.stringify(body) } : {}) }); + assert.ok(response.ok, await response.clone().text()); + return response.json(); +} +assert.equal((await call("/api/creation-test-health")).isolated, true); +const browser = await launchBrowser(); +const page = await browser.newPage({ viewport: { width: 1600, height: 1000 } }); +page.setDefaultTimeout(15000); +const projectId = crypto.randomUUID(); +try { + await call(`/api/creation/projects/${projectId}/commands`, { commandId: crypto.randomUUID(), projectId, + type: "project.create", payload: { name: `Viewport focus ${projectId.slice(0, 8)}` } }); + await page.goto(`${base}/ai-tool/ai-canvas/${projectId}?creation=unified`); + await page.getByText("已保存到本机", { exact: true }).waitFor(); + await page.locator(".cw-chat-button").click(); + await page.locator(".cw-assistant-panel:not([hidden])").waitFor(); + headers["X-OpenDirector-Session"] = await page.evaluate(async () => (await import("/src/backend/api.ts")).sessionId); + const create = async (x, y) => { + const id = crypto.randomUUID(); + await call(`/api/creation/projects/${projectId}/commands`, { commandId: crypto.randomUUID(), projectId, + type: "canvas.show", payload: { item: { id, projectId, kind: "image", position: { x, y }, + width: 352, height: 528, title: "Remote node", binding: { role: "empty" } } } }); + await page.locator(`.react-flow__node[data-id="${id}"]`).waitFor({ state: "attached" }); + return id; + }; + const transform = () => page.locator(".react-flow__viewport").getAttribute("style"); + const checkCentered = async (id) => { + await page.waitForFunction((id) => { + const node = document.querySelector(`.react-flow__node[data-id="${id}"]`).getBoundingClientRect(); + const panel = document.querySelector(".cw-assistant-panel").getBoundingClientRect(); + return node.left > 75 && node.right < panel.left - 10 && node.top > 72 && node.bottom < innerHeight - 65 && + Math.abs((node.left + node.right) / 2 - (80 + panel.left - 16) / 2) < 50; + }, id); + }; + const editor = page.locator(".cw-assistant-panel [contenteditable=true]").first(); + await editor.fill("继续创作"); + const first = await create(0, 0); + await checkCentered(first); + assert.ok(await editor.evaluate((element) => element === document.activeElement)); + assert.equal(await page.locator(".react-flow__node.selected").count(), 0); + await page.screenshot({ path: "/tmp/created-node-focus.png" }); + await page.waitForTimeout(250); + await page.mouse.move(100, 800); + await page.mouse.down(); + const before = await transform(); + await create(5000, 5000); + await page.waitForTimeout(300); + assert.equal(await transform(), before, "creation must not move the viewport during a pointer gesture"); + await page.mouse.up(); + await page.waitForTimeout(250); + assert.equal(await transform(), before, "blocked creation must not steal the view after the gesture"); + const third = await create(7000, 3000); + await checkCentered(third); + await page.waitForTimeout(250); + await page.mouse.move(1000, 800); + await page.mouse.down({ button: 'middle' }); + await page.mouse.move(1100, 830, { steps: 5 }); + await page.mouse.up({ button: 'middle' }); + await page.waitForTimeout(250); + const navigated = await transform(); + await call(`/api/creation/projects/${projectId}/commands`, { commandId: crypto.randomUUID(), projectId, + type: 'canvas.update', payload: { itemId: third, title: 'Updated title' }, + expectedVersions: [{ objectType: 'canvas_item', id: third, revision: 1 }] }); + await page.getByText('Updated title', { exact: true }).waitFor({ state: 'attached' }); + await page.waitForTimeout(250); + assert.equal(await transform(), navigated, 'updates to existing nodes must not undo manual navigation'); + await page.locator('.cw-workspace').focus(); + await page.keyboard.press(process.platform === 'darwin' ? 'Meta+a' : 'Control+a'); + await page.waitForFunction(() => document.querySelectorAll('.react-flow__node.selected').length === 3); + const multiViewport = await transform(); + await create(9000, 9000); + await page.waitForTimeout(300); + assert.equal(await transform(), multiViewport, 'multi-selection must retain its view'); + assert.equal(await page.locator('.react-flow__node.selected').count(), 3); + const snapshot = await call(`/api/creation/projects/${projectId}`); + assert.deepEqual(snapshot.canvasItems.map((item) => item.position), [{ x: 0, y: 0 }, { x: 5000, y: 5000 }, { x: 7000, y: 3000 }, { x: 9000, y: 9000 }]); + assert.equal(snapshot.jobs.length, 0); + console.log(JSON.stringify({ passed: true, checks: ["assistant occlusion", "center origin and distant nodes", "preserve focus and selection", "respect pointer gesture and multi-selection", "unchanged authored positions"] })); +} finally { + await browser.close(); +} diff --git a/tests/canvasSelectionPresentation.test.ts b/tests/canvasSelectionPresentation.test.ts index 7a36de7df..d158c4457 100644 --- a/tests/canvasSelectionPresentation.test.ts +++ b/tests/canvasSelectionPresentation.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + centeredViewport, selectionPlacement, selectionViewport, type SelectionChrome, @@ -142,3 +143,17 @@ test("manual zoom and pan preserve the composer's canvas anchor and scaled gap", assert.deepEqual(view, { x: 300, y: 100, zoom }); } }); + + +test("created content centers in the canvas beside the assistant without enlarging", () => { + const chrome = { safe: { x: 80, y: 72, width: 800, height: 650 }, toolbar: { width: 0, height: 0 }, composer: { width: 0, height: 0 } }; + const node = { x: 0, y: 0, width: 352, height: 528 }; + const view = centeredViewport(node, { x: 0, y: 0, zoom: .8 }, chrome); + assert.equal(view.zoom, .8); + assert.equal(view.x + node.width * view.zoom / 2, 480); + assert.equal(view.y + node.height * view.zoom / 2, 397); + const large = centeredViewport({ ...node, x: 5000, width: 2000, height: 1200 }, { x: 0, y: 0, zoom: 1 }, chrome); + assert.ok(large.zoom < 1); + assert.ok(large.x + 5000 * large.zoom >= chrome.safe.x); + assert.ok(large.x + 7000 * large.zoom <= chrome.safe.x + chrome.safe.width); +}); diff --git a/tests/creationParameters.test.ts b/tests/creationParameters.test.ts index 248354681..9b8130e31 100644 --- a/tests/creationParameters.test.ts +++ b/tests/creationParameters.test.ts @@ -356,3 +356,19 @@ test("a music node follows a connected recording only through YuE2 and a transcr /扒谱模型仅用于/, ); }); + +test("video audio controls and validation consume declared audio choices", () => { + const config: ModelConfig = { + id: crypto.randomUUID(), kind: "video", url: "https://fixture.invalid/v1", + modelId: "fixture", protocol: "volcengine", connectionType: "api", authMode: "none", + inputRules: { profile: "compatible", allowedKinds: ["text", "image"], + maxImages: 8, maxVideos: 0, maxAudios: 0, videoModes: ["reference"], generateAudioOptions: [true] }, + }; + const settings = { ...generationDefaults("video"), modelConfigId: config.id, + videoMode: "reference" as const, prompt: "test", audioEnabled: false }; + assert.equal(backendBlocker(settings, [], [config]), "当前视频适配器不支持关闭生成音频。"); + assert.equal(serviceGenerationSettings(settings, config, [], true).audioEnabled, true); + const optional: ModelConfig = { ...config, inputRules: { ...config.inputRules, generateAudioOptions: [true, false] } }; + assert.equal(serviceGenerationSettings(settings, optional, [], true).audioEnabled, false); + assert.notEqual(backendBlocker(settings, [], [optional]), "当前视频适配器不支持关闭生成音频。"); +}); diff --git a/tests/test_agent_artifacts.py b/tests/test_agent_artifacts.py index 11c90399d..62bf1fac0 100644 --- a/tests/test_agent_artifacts.py +++ b/tests/test_agent_artifacts.py @@ -4,6 +4,7 @@ import subprocess from uuid import UUID, uuid4, uuid5 +from fastapi.testclient import TestClient from PIL import Image import pytest @@ -18,6 +19,7 @@ from opendirector.infrastructure.creation_projects import CreationProjects from opendirector.infrastructure.local_projects import LocalProjects from opendirector.infrastructure.model_configs import ModelConfigs +from opendirector_api.main import create_app @pytest.fixture @@ -210,7 +212,7 @@ def test_asset_id_preview_is_bounded_and_preserves_project(publication, kind): asset_id = UUID(published['asset']['id']) before = business.projects.snapshot(run.project_id) gateway = ToolGateway(business, run) - result = asyncio.run(gateway.call('preview', 'preview_asset', {'assetId': str(asset_id)})) + result = asyncio.run(gateway.call('preview', 'query', {'type': 'asset.preview', 'objectId': str(asset_id)})) assert not result.error, result preview = Path(result.value['preview']['path']) assert preview.is_relative_to(work / 'previews') and preview.stat().st_size > 0 @@ -222,5 +224,57 @@ def test_asset_id_preview_is_bounded_and_preserves_project(publication, kind): with wave.open(str(preview)) as audio: assert audio.getnframes() / audio.getframerate() <= 30 assert business.projects.snapshot(run.project_id) == before - missing = asyncio.run(gateway.call('missing', 'preview_asset', {'assetId': str(uuid4())})) + missing = asyncio.run(gateway.call('missing', 'query', {'type': 'asset.preview', 'objectId': str(uuid4())})) assert missing.error + + +@pytest.mark.anyio +async def test_public_queries_share_http_and_agent_results(publication): + business, run, work = publication + source = work / 'query-source.png' + Image.new('RGB', (24, 32), 'blue').save(source) + asset = publish(publication, source.name)['asset'] + item_id = uuid4() + business.projects.execute(parse_command({'type': 'canvas.show', 'projectId': run.project_id, + 'commandId': uuid4(), 'payload': {'item': {'id': item_id, 'projectId': run.project_id, + 'kind': 'image', 'binding': {'role': 'material', 'objectRef': asset_ref(business.assets.get(run.project_id, UUID(asset['id'])))}}}}), session=run.session) + before = business.projects.snapshot(run.project_id) + gateway = ToolGateway(business, run) + path = f'/api/creation/projects/{run.project_id}' + with TestClient(create_app(business.projects.local), headers={'X-OpenDirector-Request': '1'}) as client: + for kind, extra in [('project.get', {}), ('object.get', {'objectId': str(item_id)}), + ('canvas.status', {'objectId': str(item_id), 'waitSeconds': 0}), + ('asset.search', {'text': 'query-source', 'limit': 1}), + ('asset.usage', {'objectId': asset['id']})]: + args = {'type': kind, **extra} + result = await gateway.call(kind, 'query', args) + response = client.post(path + '/queries', json={**args, 'projectId': str(run.project_id), 'queryId': str(uuid4())}) + assert response.status_code == 200, response.text + assert not result.error, result + assert result.value == response.json() + args = {'type': 'asset.preview', 'objectId': asset['id']} + preview = await gateway.call('preview-shared', 'query', args) + assert not preview.error + assert Path(preview.value['preview']['path']).is_relative_to(work / 'previews') + response = client.post(path + '/queries', json={**args, 'projectId': str(run.project_id), 'queryId': str(uuid4())}) + assert response.status_code == 200, response.text + value = response.json() + assert 'path' not in value['preview'] + media = client.get(value['preview']['url']) + assert media.status_code == 200 and media.headers['content-type'] == 'image/webp' + assert media.content == Path(preview.value['preview']['path']).read_bytes() + assert value['metadata'] == preview.value['metadata'] + wrong = await gateway.call('wrong-fields', 'query', {'type': 'object.get', 'objectId': str(item_id), 'waitSeconds': 1}) + assert wrong.error and 'waitSeconds' in wrong.error + missing = await gateway.call('missing-id', 'query', {'type': 'canvas.status'}) + assert missing.error and 'objectId' in missing.error + destination = Path(preview.value['preview']['path']) + destination.unlink() + destination.symlink_to(source) + original = source.read_bytes() + protected = await gateway.call('preview-symlink', 'query', {'type': 'asset.preview', 'objectId': asset['id']}) + assert protected.error and source.read_bytes() == original + scoped = await gateway.call('wrong-scope', 'query', {'type': 'project.get', 'projectId': str(uuid4())}) + assert scoped.error + assert business.projects.snapshot(run.project_id) == before + await gateway.close() diff --git a/tests/test_agent_gateway.py b/tests/test_agent_gateway.py index 51c0b08d8..665205429 100644 --- a/tests/test_agent_gateway.py +++ b/tests/test_agent_gateway.py @@ -299,7 +299,7 @@ async def execute(*args): cancelled.set() monkeypatch.setattr(gateway.business, 'execute', execute) - waiter = asyncio.create_task(gateway.call('status', 'read_canvas_item', {'itemId': str(uuid4()), 'waitSeconds': 30})) + waiter = asyncio.create_task(gateway.call('status', 'query', {'type': 'canvas.status', 'objectId': str(uuid4()), 'waitSeconds': 30})) await entered.wait() await asyncio.wait_for(gateway.close(), timeout=1) assert cancelled.is_set() diff --git a/tests/test_agent_metadata.py b/tests/test_agent_metadata.py index 0b9ef0f53..d01a4e287 100644 --- a/tests/test_agent_metadata.py +++ b/tests/test_agent_metadata.py @@ -1,11 +1,15 @@ """CI gate for discoverable tool and command descriptions.""" -from pydantic import Field, create_model +import json +from uuid import uuid4 + +from pydantic import Field, JsonValue, create_model import pytest -from opendirector.agent.tools import COMMANDS, tool_schema, tools +from opendirector.agent.tools import AgentTools, COMMANDS, tool_schema, tools from opendirector.domain.creation.base import Contract from opendirector.domain.creation.commands import COMMAND_MODELS from opendirector.domain.creation.schema import CreationContracts +from opendirector.infrastructure.model_configs import ModelConfigs def test_all_registered_commands_have_explicit_metadata(): @@ -57,3 +61,44 @@ def test_exported_contract_fields_have_metadata(): for name, definition in {'CreationContracts': schema, **schema['$defs']}.items(): for field, value in definition.get('properties', {}).items(): assert value.get('description', '').strip(), f'{name}.{field}' + + +@pytest.mark.anyio +async def test_model_schema_exposes_supported_video_inputs_without_changing_contract(tmp_path): + configs = ModelConfigs(tmp_path) + config = configs.save('video', 'http://fixture.invalid/v1', 'FastVideo-FastH3-Omni-8-Step-V1', '', + connection_type='self_hosted', protocol='vllm_omni', auth_mode='none', video_adapter='fasth3') + business = AgentTools(None, None, configs) + async def schema(arguments): + return await business.execute(uuid4(), uuid4(), '', 'schema', 'command_schema', arguments, [], None) + full = await schema({'type': 'canvas.createGeneration'}) + compact = await schema({'type': 'canvas.createGeneration', 'modelConfigId': config['id']}) + assert len(json.dumps(compact)) < len(json.dumps(full)) * .65 + defs = compact['$defs'] + assert 'ImageCanvasItem' not in defs and 'AudioGenerationSpec' not in defs + fields = defs['VideoParameters']['properties'] + assert fields['videoMode']['enum'] == ['reference'] + assert 'default' not in fields['videoMode'] + assert 'videoMode' in defs['VideoParameters']['required'] + assert 'parameters' in defs['VideoGenerationSpec']['required'] + assert fields['generateAudio']['enum'] == [True] + assert 'PinnedReference' in defs and 'FollowingReference' in defs + assert full['$defs']['VideoParameters']['properties']['videoMode']['default'] == 'first_frame' + assert COMMANDS['canvas.createGeneration'].model_json_schema()['$defs']['VideoParameters'] == full['$defs']['VideoParameters'] + + +@pytest.mark.anyio +async def test_command_schemas_keep_every_reference_resolvable(): + business = AgentTools(None, None, None) + for kind in COMMANDS: + schema = await business.execute(uuid4(), uuid4(), '', kind, 'command_schema', {'type': kind}, [], None) + def check(value: JsonValue) -> None: + if isinstance(value, dict): + if '$ref' in value: + assert value['$ref'].rsplit('/', 1)[1] in schema['$defs'], kind + for child in value.values(): + check(child) + elif isinstance(value, list): + for child in value: + check(child) + check(schema) diff --git a/tests/test_creation_canvas_service.py b/tests/test_creation_canvas_service.py index 2a30d41d6..db29fcf0f 100644 --- a/tests/test_creation_canvas_service.py +++ b/tests/test_creation_canvas_service.py @@ -1,8 +1,11 @@ """Real committed commands: no alternate graph and no visible execution nodes.""" +from io import BytesIO from uuid import uuid4 +from PIL import Image import pytest +from opendirector.application.creation_assets import CreationAssets from opendirector.application.creation_canvas import CreationCanvas, asset_ref, resolved_inputs from opendirector.domain.creation.commands import parse_command from opendirector.infrastructure.creation_projects import CreationProjects @@ -388,3 +391,45 @@ def test_canvas_item_wait_returns_on_change_and_timeout(project, monkeypatch): assert result['status'] == 'succeeded' monkeypatch.setattr(canvas, 'item_status', lambda *args: {'status': 'running'}) assert asyncio.run(canvas.wait_item_status(pid, item.id, .01)) == {'status': 'running'} + + +@pytest.mark.parametrize('binding', ['follow_current', 'pinned']) +def test_canvas_reference_sources_create_a_connected_video_node(project, binding): + store, pid, session = project + assets = CreationAssets(store) + image = BytesIO() + Image.new('RGB', (16, 16), 'blue').save(image, 'PNG') + handle = uuid4() + writer = assets.uploads.begin(pid, handle, session, 'reference.png', expected_kind='image') + writer.write(image.getvalue()) + writer.finish() + assert send(store, pid, 'asset.import', session=session, uploadHandle=handle, + name='reference.png', expectedKind='image').status == 'applied' + asset = store.snapshot(pid).assets[-1] + source_id = uuid4() + assert send(store, pid, 'canvas.show', session=session, item=dict(id=source_id, + projectId=pid, kind='image', binding=dict(role='material', objectRef=asset_ref(asset)))).status == 'applied' + canvas = CreationCanvas(store) + before = store.snapshot(pid) + sources = canvas.item_status(pid, source_id)['referenceSources'] + assert store.snapshot(pid) == before + spec_id, target_id, ref_id = uuid4(), uuid4(), uuid4() + source = sources['followCurrent'] if binding == 'follow_current' else sources['pinned'] + receipt = send(store, pid, 'canvas.createGeneration', session=session, + spec=dict(id=spec_id, projectId=pid, kind='video', purpose='generation', prompt='Animate the reference', + parameters=dict(durationSeconds=5, videoMode='reference')), + item=dict(id=target_id, projectId=pid, kind='video', + binding=dict(role='generation', objectRef=dict(id=spec_id))), + references=[dict(id=ref_id, projectId=pid, targetSpecId=spec_id, **source)]) + assert receipt.status == 'applied', receipt.issues + projection = canvas.projection(pid) + assert projection['edges'] == [dict(referenceId=ref_id, sourceItemId=source_id, + targetSpecId=spec_id, state='pending')] + current = store.snapshot(pid) + assert next(item for item in current.canvasItems if item.id == target_id).visible + assert not current.jobs + pinned = current.references[0].source if binding == 'pinned' else None + assert resolved_inputs(current, spec_id)[0].asset.assetId == asset.id + assert canvas.item_status(pid, target_id)['referenceSources'] is None + if pinned is not None: + assert pinned == asset_ref(asset) diff --git a/tests/test_creation_contracts.py b/tests/test_creation_contracts.py index 8473cb537..ba931fe14 100644 --- a/tests/test_creation_contracts.py +++ b/tests/test_creation_contracts.py @@ -512,3 +512,33 @@ def test_audio_voice_names_reject_invalid_characters_and_lengths(voice): from opendirector.domain.creation.generation import AudioParameters with pytest.raises(ValidationError): AudioParameters(voice=voice) + + +def test_pinned_reference_schema_requires_canvas_provenance_type_and_preserves_legacy_json(): + from opendirector.domain.creation.generation import PinnedReference + schema = PinnedReference.model_json_schema() + assert schema['$defs']['CanvasItemVersion']['properties']['objectType']['const'] == 'canvas_item' + value = dict(id=uid(90), projectId=uid(1), targetSpecId=uid(30), binding='pinned', + source=asset_ref(asset(18, 'image'))) + without = PinnedReference.model_validate(value).model_dump(mode='json') + assert 'sourceVersion' not in without + provenance = dict(objectType='canvas_item', id=str(uid(40)), revision=1) + restored = PinnedReference.model_validate({**value, 'sourceVersion': provenance}) + assert restored.model_dump(mode='json')['sourceVersion'] == provenance + with pytest.raises(ValidationError, match='canvas_item'): + PinnedReference.model_validate({**value, 'sourceVersion': {**provenance, 'objectType': 'asset'}}) + + +@pytest.mark.parametrize(('protocol', 'allowed'), [('vllm_omni', (True,)), ('volcengine', (True, False))]) +def test_audio_generation_options_match_shared_validation(protocol, allowed): + config = model('video', protocol=protocol) + assert config.inputRules.generateAudioOptions == allowed + validate_generation(video_spec(generateAudio=True), config) + if False in allowed: + validate_generation(video_spec(generateAudio=False), config) + else: + with pytest.raises(ValueError, match='不支持关闭生成音频'): + validate_generation(video_spec(generateAudio=False), config) + legacy = config.inputRules.model_dump(mode='json', exclude={'generateAudioOptions'}) + from opendirector.domain.creation.models import InputRules + assert InputRules.model_validate(legacy).model_dump(mode='json') == legacy diff --git a/tests/test_deepseek_runtime.py b/tests/test_deepseek_runtime.py index 56fd2f720..5c8df48c8 100644 --- a/tests/test_deepseek_runtime.py +++ b/tests/test_deepseek_runtime.py @@ -58,6 +58,8 @@ def read(): return json.loads(sys.stdin.readline()) if mode=='wait': write(dict(method='session/update',params=dict(sessionId=sid,update=dict(sessionUpdate='agent_message_chunk',content=dict(type='text',text='Waiting'))))) continue + write(dict(method='session/update',params=dict(sessionId=sid,update=dict(sessionUpdate='agent_thought_chunk',content=dict(type='text',text='Test native thought'))))) + write(dict(method='session/update',params=dict(sessionId=sid,update=dict(sessionUpdate='plan',entries=[dict(content='Save the plan',priority='medium',status='in_progress')])))) if mode=='permission': write(dict(id='approval-1',method='session/request_permission',params=dict(sessionId=sid,options=[dict(optionId='allow',kind='allow_once'),dict(optionId='reject',kind='reject_once')]))) answer=read() @@ -70,7 +72,7 @@ def rpc(value): response=json.loads(bridge.stdout.readline());assert 'error' not in response,response return response['result'] rpc({'id':1,'method':'initialize'}) - assert len(rpc({'id':2,'method':'tools/list'})['tools'])==6 + assert len(rpc({'id':2,'method':'tools/list'})['tools'])==5 context=rpc({'id':20,'method':'tools/call','params':{'name':'read_project','arguments':{}}}) assert 'selected-image' in json.dumps(context),context outcome=rpc({'id':3,'method':'tools/call','params':{'name':'execute_command','arguments':{'type':'asset.writeText','payload':{'name':'deepseek.md','text':'CLI saved'}}}}) @@ -78,6 +80,7 @@ def rpc(value): bridge.stdin.close();bridge.wait() write(dict(method='session/update',params=dict(sessionId=sid,update=dict(sessionUpdate='tool_call',toolCallId='native-read',title='read',rawInput={'path':'file'},status='in_progress')))) write(dict(method='session/update',params=dict(sessionId=sid,update=dict(sessionUpdate='tool_call_update',toolCallId='native-read',status='completed',content=[])))) + write(dict(method='session/update',params=dict(sessionId=sid,update=dict(sessionUpdate='plan',entries=[dict(content='Save the plan',priority='medium',status='completed')])))) write(dict(method='session/update',params=dict(sessionId=sid,update=dict(sessionUpdate='agent_message_chunk',content=dict(type='text',text='Saved.'))))) result(req,dict(stopReason='max_tokens' if mode=='incomplete' else 'end_turn')) ''' @@ -113,6 +116,10 @@ def test_acp_shared_write_and_native_trace(cli, tmp_path, monkeypatch, mode): assert (events[-1]['error'] is None) == (mode != 'incomplete'), events message = events[-1]['conversation']['messages'][-1] assert message['text'] == 'Saved.' + assert {'type': 'thinking'} in events + assert 'Test native thought' not in response.text + assert message['plan'] == [{'content': 'Save the plan', 'status': 'completed'}] + assert client.get(path + '/assistant').json()['messages'][-1]['plan'] == message['plan'] assert any(tool['name']=='execute_command' and tool['status']=='complete' for tool in message['tools']) assert any(tool['name']=='read' and tool['status']=='complete' for tool in message['tools']) snapshot=client.get(path).json() diff --git a/tests/test_native_runtime.py b/tests/test_native_runtime.py index de1427bcf..e4348fc89 100644 --- a/tests/test_native_runtime.py +++ b/tests/test_native_runtime.py @@ -33,11 +33,11 @@ def rpc(value): assert 'error' not in response, response return response['result'] rpc({'id':1, 'method':'initialize'}) -assert len(rpc({'id':2, 'method':'tools/list'})['tools']) == 6 +assert len(rpc({'id':2, 'method':'tools/list'})['tools']) == 5 result = rpc({'id':3, 'method':'tools/call', 'params':{'name':'execute_command', 'arguments':{'type':'asset.writeText','payload':{'name':'native.md','text':'Native plan'}}}}) assert not result['isError'], result assert rpc({'id':3, 'method':'tools/call', 'params':{'name':'execute_command', 'arguments':{'type':'asset.writeText','payload':{'name':'native.md','text':'Native plan'}}}}) == result -preview = rpc({'id':4, 'method':'tools/call', 'params':{'name':'preview_asset', 'arguments':{'assetId':'00000000-0000-0000-0000-000000000001'}}}) +preview = rpc({'id':4, 'method':'tools/call', 'params':{'name':'query', 'arguments':{'type':'asset.preview','objectId':'00000000-0000-0000-0000-000000000001'}}}) assert not preview['isError'], preview assert preview['content'][1] == {'type':'image','data':'cHJldmlldw==','mimeType':'image/webp'}, preview bridge.stdin.close(); bridge.wait()