diff --git a/apps/api/src/opendirector_api/creation.py b/apps/api/src/opendirector_api/creation.py index 5d7377fd..d1dedeef 100644 --- a/apps/api/src/opendirector_api/creation.py +++ b/apps/api/src/opendirector_api/creation.py @@ -316,6 +316,13 @@ def thumbnail(project_id: UUID, asset_id: UUID): 'Cache-Control': 'private, no-cache', 'ETag': '"thumbnail-' + asset.fingerprint + '"', 'X-Content-Type-Options': 'nosniff'}) + @router.get('/projects/{project_id}/assets/{asset_id}/browser-image') + def browser_image(project_id: UUID, asset_id: UUID): + value, asset = assets.browser_image(project_id, asset_id) + return Response(value, media_type='image/png', headers={ + 'Cache-Control': 'private, no-cache', 'ETag': '"browser-image-' + asset.fingerprint + '"', + 'X-Content-Type-Options': 'nosniff'}) + @router.get('/projects/{project_id}/assets/{asset_id}/content') def content(project_id: UUID, asset_id: UUID, range_header: str | None = Header(default=None, alias='Range'), if_none_match: str | None = Header(default=None, alias='If-None-Match')): diff --git a/apps/api/src/opendirector_api/localization.py b/apps/api/src/opendirector_api/localization.py index c5a4cbf6..c5175a4f 100644 --- a/apps/api/src/opendirector_api/localization.py +++ b/apps/api/src/opendirector_api/localization.py @@ -47,7 +47,10 @@ "moss_speech_voice": "Enter a Voice ID available to your MOSS account.", "moss_speech_reference": "MOSS speech generation does not accept references. Enter speech text and a Voice ID.", "moss_prompt_limit": "MOSS requires a nonempty prompt of at most 4,000 characters, including text references.", - "minimax_image_reference": "MiniMax H3 references must be PNG, JPEG, or WebP, with edges of 128–8192 pixels and a long-to-short edge ratio of at most 4:1.", + "image_decode": "The image could not be decoded. Check its actual format and file integrity.", + "asset_preview_unsupported": "This asset type does not support this preview format.", + "image_pixel_limit": "Images must contain no more than 25 million pixels.", + "minimax_image_reference": "MiniMax H3 references must be PNG, JPEG, WebP, HEIC, or HEIF, with edges of 128–8192 pixels and a long-to-short edge ratio of at most 4:1.", "minimax_image_count": "MiniMax H3 image editing accepts up to 16 reference images.", "minimax_image_bytes": "MiniMax H3 image reference requests cannot exceed 64 MiB.", "moss_image_size": "MOSS image dimensions must be 256–2048, multiples of 16, and no more than 2,097,152 pixels in total.", diff --git a/docs/development/lessons.md b/docs/development/lessons.md index 13e6dfac..8de92709 100644 --- a/docs/development/lessons.md +++ b/docs/development/lessons.md @@ -18,6 +18,7 @@ When work exposes a consequential risk or a recurring problem, add a lesson or u - **Lesson:** Commit `e8c29c9b` fixed provenance and creation-order problems in a canvas archive round trip; `44857448` fixed asset context across turns; `00d8899c` fixed Agent runs stopping when switching projects. These are different defects, but each shows that the current display alone cannot establish state and context. - **Check:** As relevant to the change, inspect persisted object identities, references, and execution inputs after conversion or recovery. Reopen the project or reconnect to the run when needed, and verify the actual result. - **Async projection follow-up:** PR #193's structured-editor CI failure prompted a real-browser delayed-read regression. An older background projection could finish after an acknowledged editor save and restore an obsolete CAS basis, making the next save/publication fail. Recheck the editor transaction and captured snapshot at completion before installing either changed or apparently identical content. Keep publication versions fixed, test the actual save/retry/provenance path with a deterministic interleaving, and retain the original CI failure rather than treating a longer timeout or a rerun as its explanation. +- **Media representation follow-up:** [Issue #282](https://github.com/ThinkFlowLab/OpenDirector/issues/282) exposed copy/export paths that read displayed pixels and archive imports that displayed unsupported original bytes. A browser preview and an original asset need distinct read paths while retaining the same asset identity. Verify original bytes through download, fresh publication, cross-project paste fallback and archive round trips, including the first display after import. ## Baseline failures still need tracking diff --git a/docs/development/validation.md b/docs/development/validation.md index 71e2d2b5..c21c6f98 100644 --- a/docs/development/validation.md +++ b/docs/development/validation.md @@ -191,6 +191,24 @@ node tests/browser/first_last_frame_connections.mjs ### 提供商与模型设置 +HEIC/HEIF imports and MiniMax H3 image editing: + +```sh +PYTHONPATH=packages/core/src:apps/api/src .venv/bin/python -m pytest tests/test_heif.py tests/test_minimax_image.py tests/test_creation_assets.py -q +node tests/browser/heif_import.mjs +``` + +The browser script uses the isolated API and Vite above. It checks real HEVC-encoded +HEIC/HEIF fixtures through upload and mixed-format drag and drop, oriented pixels, +project reload, both display languages, corrupt/oversized upload rejection, original +downloads and canvas archive round trips, cross-project paste with an unavailable source, and +both MiniMax model submissions with fixture results. Python tests exercise Agent +artifact import and preview, and run the real MiniMax adapter against mocked HTTP through UI +and Agent command sources, checking original bytes, ordered frozen inputs, result +adoption, rejection before submission and request-size limits. These tests do not +call MiniMax, verify model quality, or exercise a live Agent conversation. Camera +HDR profiles and platform wheel installation require separate validation. + Kling 4.0 protocol and bilingual workflow checks: ```sh diff --git a/docs/guides/user-guide.md b/docs/guides/user-guide.md index d8a7346b..4ce01be6 100644 --- a/docs/guides/user-guide.md +++ b/docs/guides/user-guide.md @@ -66,6 +66,7 @@ The first launch creates an untitled canvas. Later launches can restore the most - Use the left toolbar or double-click empty canvas space to create text, image, video, audio, timeline, or subject nodes. - Right-click empty canvas space to create a node, paste, undo, or redo. - Click **Upload** on the left or drag files onto the canvas to add assets. Use **Asset library** to choose existing assets. +- HEIC and HEIF photos are supported. OpenDirector retains the original file and creates an oriented, full-resolution PNG for browser display. Downloads and canvas archives retain the original. Files must decode successfully and contain at most 25 million pixels. - Double-click a node's name to rename it. Double-click its content to open the appropriate editor or preview. The canvas saves automatically. Before closing it or switching projects, check that the top-left status reads **Saved locally**. If saving fails, keep the page open and follow the error message. @@ -81,6 +82,8 @@ After typing `@`, use the up and down arrow keys to select an asset, the left an An empty node must first receive an upload, saved text, or generated content before it can serve as a valid reference for other nodes. When uploading a node's own content, the file type must match the node. To add references, use connections, `@`, or the reference controls in the generation panel. +**MiniMax-H3-Image / MiniMax-H3-Image-Highspeed** can mix HEIC, HEIF, PNG, JPEG and WebP references in the selected order, through the canvas or Agent commands. Both use the original project assets. Each edge must be 128–8192 pixels, the long-to-short edge ratio at most 4:1, and there can be at most 16 images. The complete Base64-encoded request must fit within 64 MiB; oversized inputs are rejected without automatic compression. Other models retain their own format restrictions. + For video generation, choose a mode supported by the model: - **Multimodal references**: Add image, video, audio, or text references supported by the model. diff --git a/docs/guides/user-guide.zh-CN.md b/docs/guides/user-guide.zh-CN.md index 5328fcd7..dcd77819 100644 --- a/docs/guides/user-guide.zh-CN.md +++ b/docs/guides/user-guide.zh-CN.md @@ -66,6 +66,7 @@ HTTP 400。 - 通过左侧工具栏或双击空白画布创建文本、图片、视频、音频、时间线或主体节点。 - 右键空白画布可新建节点、粘贴、撤销或重做。 - 点击左侧“上传”或将文件拖入画布添加素材;“资产库”可选择已有素材。 +- 支持 HEIC 和 HEIF 照片。OpenDirector 保留原文件,并生成方向正确、完整分辨率的 PNG 供浏览器显示;下载和画布归档保留原件。文件须能完整解码,且像素总数不超过 2500 万。 - 双击节点名称重命名;双击节点内容进入相应编辑或预览界面。 画布会自动保存。关闭或切换前,请确认左上角显示“已保存到本机”;若显示保存失败,保留页面并按提示处理。 @@ -81,6 +82,8 @@ HTTP 400。 空白节点需要先上传、保存文本或生成内容,才能作为其他节点的有效参考。向已有节点上传本体素材时,文件类型应与节点一致;添加参考请使用连线、`@` 或生成面板的添加入口。 +**MiniMax-H3-Image / MiniMax-H3-Image-Highspeed** 支持通过画布或 Agent 命令,按所选顺序混用 HEIC、HEIF、PNG、JPEG 和 WebP 参考图;两种入口均使用工程中的原始资产。每条边须为 128–8192 像素,长短边之比不超过 4:1,最多 16 张。完整请求经 Base64 编码后不得超过 64 MiB;超限会明确拒绝,不会自动压缩。其他模型仍遵循各自的格式限制。 + 视频生成时按模型能力选择模式: - **全能参考**:添加模型支持的图片、视频、音频或文字参考。 diff --git a/packages/core/src/opendirector/application/creation_assets.py b/packages/core/src/opendirector/application/creation_assets.py index f4e56be8..d36b5388 100644 --- a/packages/core/src/opendirector/application/creation_assets.py +++ b/packages/core/src/opendirector/application/creation_assets.py @@ -285,6 +285,15 @@ def preview(self, project_id: UUID, asset_id: UUID, target: Path) -> dict: 'preview': {'path': str(output.resolve()), 'mimeType': mime, 'scope': 'first_frame' if asset.kind == 'video' else 'first_30_seconds' if asset.kind == 'audio' else 'thumbnail'}} + def browser_image(self, project_id, asset_id): + """Full-resolution display bytes; the asset and its fingerprint stay original.""" + from opendirector.media.processing import open_image, png + stream, asset = self.open_media(project_id, asset_id) + with stream: + if asset.metadata.mimeType not in {'image/heic', 'image/heif'}: + raise ProjectError('该素材类型不支持预览。', 422, code='asset_preview_unsupported') + return png(open_image(stream.read())), asset + def thumbnail(self, project_id, asset_id): """A bounded, disposable image preview; always verify its source first.""" from PIL import Image, ImageOps diff --git a/packages/core/src/opendirector/domain/minimax_image.py b/packages/core/src/opendirector/domain/minimax_image.py index 100c2921..a0dd2640 100644 --- a/packages/core/src/opendirector/domain/minimax_image.py +++ b/packages/core/src/opendirector/domain/minimax_image.py @@ -2,8 +2,8 @@ def validate_minimax_image_reference(mime, width, height): - if (mime not in {'image/png', 'image/jpeg', 'image/webp'} or not width or not height + if (mime not in {'image/png', 'image/jpeg', 'image/webp', 'image/heic', 'image/heif'} or not width or not height or not 128 <= min(width, height) <= max(width, height) <= 8192 or max(width, height) > 4 * min(width, height)): - raise ProjectError('MiniMax H3 图片参考须为 PNG、JPEG 或 WebP,边长 128–8192 像素,长短边之比不超过 4:1。', + raise ProjectError('MiniMax H3 图片参考须为 PNG、JPEG、WebP、HEIC 或 HEIF,边长 128–8192 像素,长短边之比不超过 4:1。', 422, code='minimax_image_reference') diff --git a/packages/core/src/opendirector/infrastructure/creation_uploads.py b/packages/core/src/opendirector/infrastructure/creation_uploads.py index eafe8698..215aa063 100644 --- a/packages/core/src/opendirector/infrastructure/creation_uploads.py +++ b/packages/core/src/opendirector/infrastructure/creation_uploads.py @@ -19,6 +19,7 @@ from opendirector.infrastructure.local_projects import child, read_json from opendirector.infrastructure.rename import rename_path from opendirector.media.probe import probe_file +from opendirector.media import heif_type from opendirector.media.fonts import FONT_SIGNATURES, inspect_font @@ -70,17 +71,19 @@ def inspect_file(path: Path, name: str, *, expected_kind=None): with path.open('rb') as stream: prefix = stream.read(16) image_bytes = (prefix.startswith((b'\x89PNG\r\n\x1a\n', b'\xff\xd8\xff', b'GIF87a', b'GIF89a')) - or prefix.startswith(b'RIFF') and prefix[8:12] == b'WEBP') - if suffix in {'.png', '.jpg', '.jpeg', '.gif', '.webp'} or image_bytes or expected_kind == 'image': + or prefix.startswith(b'RIFF') and prefix[8:12] == b'WEBP' or heif_type(prefix)) + if suffix in {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.heic', '.heif'} or image_bytes or expected_kind == 'image': try: - with Image.open(path, formats=['PNG', 'JPEG', 'GIF', 'WEBP']) as image: + with Image.open(path, formats=['PNG', 'JPEG', 'GIF', 'WEBP', 'HEIF']) as image: + if image.format == 'HEIF' and not heif_type(prefix): + raise ValueError('Unsupported HEIF subtype') if image.width * image.height > 25_000_000: - raise ProjectError('图片像素总数应不超过 2500 万。', 413) - metadata = MediaMetadata(size=size, mimeType=Image.MIME[image.format], width=image.width, height=image.height) + raise ProjectError('图片像素总数应不超过 2500 万。', 413, code='image_pixel_limit') image.load() + metadata = MediaMetadata(size=size, mimeType=image.get_format_mimetype(), width=image.width, height=image.height) return 'image', metadata - except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError): - raise ProjectError('无法读取图片,请检查实际格式及完整性。') from None + except (UnidentifiedImageError, OSError, ValueError, EOFError, Image.DecompressionBombError): + raise ProjectError('无法读取图片,请检查实际格式及完整性。', code='image_decode') from None kind = ('video' if suffix in {'.mp4', '.mov', '.webm'} else 'audio' if suffix in {'.mp3', '.wav', '.m4a', '.ogg', '.flac'} else expected_kind if expected_kind in {'video', 'audio'} else None) diff --git a/packages/core/src/opendirector/integrations/model_adapters.py b/packages/core/src/opendirector/integrations/model_adapters.py index a78204ab..ee3e9101 100644 --- a/packages/core/src/opendirector/integrations/model_adapters.py +++ b/packages/core/src/opendirector/integrations/model_adapters.py @@ -22,6 +22,7 @@ VideoRequest) from opendirector.domain.input_rules import input_rules from opendirector.domain.minimax_image import validate_minimax_image_reference +from opendirector.media import heif_type from opendirector.domain.moss_image import validate_moss_image_prompt, validate_moss_image_size from opendirector.domain.moss_speech import validate_moss_speech_input from opendirector.domain.projects import ProjectError @@ -223,15 +224,17 @@ async def image(self, config, request: ImageRequest) -> MediaResult: raise ProjectError('MiniMax H3 图片编辑最多接受 16 张参考图。', 422, code='minimax_image_count') encoded = [] for data in images: - mime = image_type(data) + if len(data) > 64 * 1024 * 1024: + raise ProjectError('MiniMax H3 图片参考请求超过 64 MiB。', 413, code='minimax_image_bytes') + mime = image_type(data) or heif_type(data) try: with Image.open(io.BytesIO(data)) as image: dimensions = image.size - except (OSError, ValueError, Image.DecompressionBombError): + validate_minimax_image_reference(mime, *dimensions) + image.load() + except (OSError, ValueError, EOFError, Image.DecompressionBombError): dimensions = (None, None) validate_minimax_image_reference(mime, *dimensions) - if len(data) > 64 * 1024 * 1024: - raise ProjectError('MiniMax H3 图片参考请求超过 64 MiB。', 413, code='minimax_image_bytes') encoded.append(data_uri(data, mime)) width, height = request.width, request.height ratio = min(3, max(1 / 3, width / height)) diff --git a/packages/core/src/opendirector/integrations/reference_inputs.py b/packages/core/src/opendirector/integrations/reference_inputs.py index 9914a035..bc687d7f 100644 --- a/packages/core/src/opendirector/integrations/reference_inputs.py +++ b/packages/core/src/opendirector/integrations/reference_inputs.py @@ -10,6 +10,7 @@ from opendirector.domain.projects import ProjectError from opendirector.domain.minimax_image import validate_minimax_image_reference from opendirector.domain.references import project_content +from opendirector.media import heif_type def reference_text(data): @@ -148,6 +149,8 @@ def generation_references(store, task, config, *, complete=True, overrides=None) entry['role'] = 'text' elif source_kind == 'image': mime = image_type(data) + if rules['profile'] == 'minimax-h3-image': + mime = mime or heif_type(data) if mime is None: raise ProjectError(f'“{meta["name"]}”不是已支持的参考图片格式。') if len(data) > rules['imageBytes']: diff --git a/packages/core/src/opendirector/media/__init__.py b/packages/core/src/opendirector/media/__init__.py index ee484fc5..08a855c0 100644 --- a/packages/core/src/opendirector/media/__init__.py +++ b/packages/core/src/opendirector/media/__init__.py @@ -1 +1,10 @@ """Local media metadata and processing.""" +from pillow_heif import get_file_mimetype, register_heif_opener + +register_heif_opener() + + +def heif_type(data: bytes) -> str | None: + """Detect still HEIF content, without admitting AVIF or image sequences.""" + mime = get_file_mimetype(data) + return mime if mime in {'image/heic', 'image/heif'} else None diff --git a/packages/core/src/opendirector/media/processing.py b/packages/core/src/opendirector/media/processing.py index 087fb612..a8e442fc 100644 --- a/packages/core/src/opendirector/media/processing.py +++ b/packages/core/src/opendirector/media/processing.py @@ -21,7 +21,7 @@ def open_image(data: bytes) -> Image.Image: try: image = Image.open(io.BytesIO(data)) image.load() - except (OSError, ValueError, Image.DecompressionBombError): + except (OSError, ValueError, EOFError, Image.DecompressionBombError): raise ProjectError('无法读取图片素材。', 422) from None return ImageOps.exif_transpose(image).convert('RGBA') diff --git a/packages/core/src/opendirector/media/thumbnails.py b/packages/core/src/opendirector/media/thumbnails.py index a98dd689..b855dcec 100644 --- a/packages/core/src/opendirector/media/thumbnails.py +++ b/packages/core/src/opendirector/media/thumbnails.py @@ -7,8 +7,13 @@ import subprocess import tempfile from collections import OrderedDict +from io import BytesIO from pathlib import Path +from opendirector.domain.projects import ProjectError +from opendirector.media import heif_type +from opendirector.media.processing import open_image + WIDTH = 480 _CAPACITY = 256 _CACHE: "OrderedDict[str, bytes]" = OrderedDict() @@ -47,6 +52,15 @@ def remember(key: str, rendered: bytes) -> None: def _render(data: bytes, kind: str) -> bytes | None: + if kind == 'image' and heif_type(data): + try: + image = open_image(data) + image.thumbnail((WIDTH, WIDTH)) + output = BytesIO() + image.convert('RGB').save(output, 'JPEG', quality=80) + return output.getvalue() + except ProjectError: + return None with tempfile.TemporaryDirectory(prefix='opendirector-preview-') as folder: source = Path(folder) / 'source' source.write_bytes(data) diff --git a/pyproject.toml b/pyproject.toml index dad124ee..058bf223 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ readme = "README.md" requires-python = ">=3.11" dependencies = [ "PyYAML>=6,<7", "filelock>=3.16,<4", "httpx>=0.27,<1", "anyio>=4,<5", - "pydantic>=2.12,<3", "typer>=0.27.3,<0.28", "Pillow>=12,<14", + "pydantic>=2.12,<3", "typer>=0.27.3,<0.28", "Pillow>=12,<14", "pillow-heif>=1.8.1,<2", "fonttools==4.68.0", "fastapi>=0.115,<1", "uvicorn>=0.30,<1", "starlette>=1.8,<2", "python-multipart>=0.0.33,<1", ] diff --git a/requirements.txt b/requirements.txt index c5aa40e3..90cd62a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -59,6 +59,10 @@ opentelemetry-api==1.45.1 packaging==26.3 # via pytest pillow==13.0.0 + # via + # opendirector (pyproject.toml) + # pillow-heif +pillow-heif==1.8.1 # via opendirector (pyproject.toml) pluggy==1.7.0 # via pytest diff --git a/src/backend/creationClient.ts b/src/backend/creationClient.ts index 6bfa53af..62047d85 100644 --- a/src/backend/creationClient.ts +++ b/src/backend/creationClient.ts @@ -356,11 +356,20 @@ export class CreationClient { } })(); } - async blob(assetId: string, options: ReadOptions = {}) { + async blob( + assetId: string, + options: ReadOptions & { browserImage?: boolean } = {}, + ) { + const mime = this.snapshot?.assets.find((asset) => asset.id === assetId) + ?.metadata.mimeType; + const representation = + options.browserImage && (mime === "image/heic" || mime === "image/heif") + ? "browser-image" + : "content"; return resilientRead( async (signal) => { const response = await fetch( - `/api${creationPath(this.id)}/assets/${assetId}/content`, + `/api${creationPath(this.id)}/assets/${assetId}/${representation}`, { headers, signal }, ); if (!response.ok) throw await responseError(response); @@ -375,7 +384,11 @@ export class CreationClient { if (!pending) { // Cached reads have their own bounded lifetime. Cancelling one history // consumer must not abort the same media read used by the visible node. - pending = this.blob(assetId, { ...options, signal: undefined }) + pending = this.blob(assetId, { + ...options, + signal: undefined, + browserImage: true, + }) .then((blob) => { const url = URL.createObjectURL(blob); this.sourceAssets.set(url, assetId); @@ -492,6 +505,8 @@ export class CreationClient { } /** Previews an asset from bytes the page already holds instead of reading them back. */ adoptMedia(assetId: string, blob: Blob) { + if (blob.type === "image/heic" || blob.type === "image/heif") + return this.mediaUrl(assetId); let pending = this.media.get(assetId); if (!pending) { const url = URL.createObjectURL(blob); diff --git a/src/backend/creationProjection.ts b/src/backend/creationProjection.ts index 842ec742..af062487 100644 --- a/src/backend/creationProjection.ts +++ b/src/backend/creationProjection.ts @@ -44,6 +44,7 @@ import { type CreationOperation, } from "./creationClient.ts"; import { sameJsonValue } from "./sameJsonValue.ts"; +import { classifyImportFile, readEditorResource } from "../canvas/editorMedia.ts"; /** Every comparison here is over a document that carries its media inline, so * it reads the values instead of serializing both sides to compare strings. */ @@ -79,6 +80,16 @@ export class CreationProjection { this.client = client; } + async importFile(file: File, signal?: AbortSignal): Promise { + const { mimeType } = classifyImportFile(file); + if (mimeType !== "image/heic" && mimeType !== "image/heif") + return readEditorResource(file, signal); + signal?.throwIfAborted(); + const ref = await this.client.importBlob(file, file.name, "image"); + signal?.throwIfAborted(); + return this.resource(ref); + } + async resource(ref: AssetRef, title?: string): Promise { const asset = this.client.snapshot.assets.find( (a) => a.id === ref.assetId, @@ -134,10 +145,14 @@ export class CreationProjection { resource.previous, ); if (!resource.src) throw new Error(tNow("errors.backend.mediaEmpty")); - const response = await fetch(resource.src); - if (!response.ok) - throw new Error(tNow("errors.backend.mediaReadFailed")); - const blob = await response.blob(); + let blob: Blob; + if (existing) blob = await this.client.blob(existing.id); + else { + const response = await fetch(resource.src); + if (!response.ok) + throw new Error(tNow("errors.backend.mediaReadFailed")); + blob = await response.blob(); + } const suffix = extension[blob.type] ?? blob.type.split("/")[1] ?? "bin"; const name = /\.[\w]+$/.test(resource.title) ? resource.title diff --git a/src/backend/workspace.ts b/src/backend/workspace.ts index e6a10754..90f93657 100644 --- a/src/backend/workspace.ts +++ b/src/backend/workspace.ts @@ -133,7 +133,8 @@ async function populateWorkspaceProject( await openWorkspaceProject(project); const imported = { ...document, id: project.id, title: project.name }; await saveWorkspaceDocument(imported); - return imported; + // Render registered assets through their project previews after importing originals. + return readWorkspaceDocument(project); } /** JSON imports become independent filesystem projects. */ diff --git a/src/canvas/AssistantPanel.tsx b/src/canvas/AssistantPanel.tsx index 3efd0d22..1322e3eb 100644 --- a/src/canvas/AssistantPanel.tsx +++ b/src/canvas/AssistantPanel.tsx @@ -794,7 +794,7 @@ export function AssistantPanel({ className="sr-only" type="file" multiple - accept="image/*,video/*,audio/*,text/*,.md,.markdown,.txt" + accept="image/*,.heic,.heif,video/*,audio/*,text/*,.md,.markdown,.txt" aria-label={t("assistant.panel.selectFiles")} onChange={(event) => { if (event.target.files?.length) void upload([...event.target.files]); diff --git a/src/canvas/GenerationReferenceEditor.tsx b/src/canvas/GenerationReferenceEditor.tsx index 23cd635f..53da53b6 100644 --- a/src/canvas/GenerationReferenceEditor.tsx +++ b/src/canvas/GenerationReferenceEditor.tsx @@ -490,7 +490,7 @@ export function GenerationReferenceEditor({ disabled={running || uploading} hidden multiple - accept="image/*,video/*,audio/*,.txt,.md" + accept="image/*,.heic,.heif,video/*,audio/*,.txt,.md" aria-label={t("generation.reference.upload")} onChange={(event) => { void uploadFiles([...(event.target.files ?? [])]); diff --git a/src/canvas/downloadAssets.ts b/src/canvas/downloadAssets.ts index 2c0c6078..6c095e31 100644 --- a/src/canvas/downloadAssets.ts +++ b/src/canvas/downloadAssets.ts @@ -44,6 +44,7 @@ const mediaExtensions: Record = { "image/png": ["png"], "image/jpeg": ["jpg", "jpeg"], "image/svg+xml": ["svg"], "image/webp": ["webp"], "image/gif": ["gif"], "image/avif": ["avif"], + "image/heic": ["heic"], "image/heif": ["heif"], "image/bmp": ["bmp"], "image/tiff": ["tiff", "tif"], "video/webm": ["webm"], "video/mp4": ["mp4", "m4v"], "video/quicktime": ["mov"], "video/ogg": ["ogv", "ogg"], diff --git a/src/canvas/editorMedia.ts b/src/canvas/editorMedia.ts index 14b32065..b6a8c99d 100644 --- a/src/canvas/editorMedia.ts +++ b/src/canvas/editorMedia.ts @@ -13,6 +13,8 @@ const mediaExtensions: Readonly> = { jpeg: "image/jpeg", png: "image/png", webp: "image/webp", + heic: "image/heic", + heif: "image/heif", bmp: "image/bmp", gif: "image/gif", svg: "image/svg+xml", diff --git a/src/canvas/workspace/entryPoints.ts b/src/canvas/workspace/entryPoints.ts index 6d4736c1..acc60c1f 100644 --- a/src/canvas/workspace/entryPoints.ts +++ b/src/canvas/workspace/entryPoints.ts @@ -35,12 +35,14 @@ export function uploadPickerPlan(input: { ? ".txt,.md,.markdown" : input.nodeType === "subject" ? input.subjectSlot === "auxiliary" - ? "image/*" + ? "image/*,.heic,.heif" : input.subjectSlot === "voice" ? "audio/*" - : "image/*,video/*" + : "image/*,.heic,.heif,video/*" : input.nodeType && ["image", "video", "audio"].includes(input.nodeType) - ? `${input.nodeType}/*` + ? input.nodeType === "image" + ? "image/*,.heic,.heif" + : `${input.nodeType}/*` : ""; return { blocked: null, multiple: !input.hasTarget, accept }; } diff --git a/src/canvas/workspace/useWorkspaceEditor.ts b/src/canvas/workspace/useWorkspaceEditor.ts index 219dd685..53b9f128 100644 --- a/src/canvas/workspace/useWorkspaceEditor.ts +++ b/src/canvas/workspace/useWorkspaceEditor.ts @@ -326,7 +326,7 @@ export function useWorkspaceEditor(rt: WorkspaceRuntime) { task, }; if (resultInput.current) { - resultInput.current.accept = "image/*,video/*,audio/*"; + resultInput.current.accept = "image/*,.heic,.heif,video/*,audio/*"; resultInput.current.click(); } }; diff --git a/src/canvas/workspace/useWorkspaceGeneration.ts b/src/canvas/workspace/useWorkspaceGeneration.ts index 9406382a..626bf69b 100644 --- a/src/canvas/workspace/useWorkspaceGeneration.ts +++ b/src/canvas/workspace/useWorkspaceGeneration.ts @@ -3,10 +3,10 @@ import { errorMessage } from "../../backend/api.ts"; import { creationClient } from "../../backend/creationClient.ts"; import { type GenerationRequest } from "../../backend/generation.ts"; import { modelLabel } from "../../backend/rules.ts"; +import { workspaceProjection } from "../../backend/workspace.ts"; import { deliverTool, pendingToolDelivery } from "../../backend/toolDeliveries"; import { acknowledgeToolPlacement, cancelToolTask, submitToolTask, ToolSubmissionError } from "../../backend/tools.ts"; -import { readEditorResource } from "../editorMedia"; import { expectedDuration, recordDuration } from "../generationDurations"; import { type GenerationJob as Job } from "../generationHistory"; @@ -640,7 +640,7 @@ export function useWorkspaceGeneration(rt: WorkspaceRuntime) { try { const outputs: LinkedMediaResult[] = []; for (const file of files) { - const resource = await readEditorResource(file, controller.signal); + const resource = await workspaceProjection(origin.documentId).importFile(file, controller.signal); if (resource.kind === "file" || resource.kind === "text") throw new Error(t("canvas.workspace.error.resultNotMedia")); outputs.push({ diff --git a/src/canvas/workspace/useWorkspaceTransfer.ts b/src/canvas/workspace/useWorkspaceTransfer.ts index 2ba612fa..19456db7 100644 --- a/src/canvas/workspace/useWorkspaceTransfer.ts +++ b/src/canvas/workspace/useWorkspaceTransfer.ts @@ -19,7 +19,7 @@ import { downloadFile, downloadFilename } from "../assets"; import { referenceIssueText, referenceIssue } from "../connectionPolicy"; import { sameNodeData } from "../dataBoundary.ts"; -import { classifyImportFile, readEditorResource } from "../editorMedia"; +import { classifyImportFile } from "../editorMedia"; import { lockedNodeIds } from "../hierarchy"; import { readImportBatch } from "../importBatch"; import { @@ -440,6 +440,7 @@ export function useWorkspaceTransfer(rt: WorkspaceRuntime) { const projectId = historyRef.current.present.id; const client = creationClient(projectId); return captureClipboardMedia(projectId, fragment, { + read: (src) => readProjectMedia(src, projectId), resolve: (src, assetId) => { // A just-imported node may show its upload before it records the asset. const asset = assetId @@ -561,11 +562,22 @@ export function useWorkspaceTransfer(rt: WorkspaceRuntime) { setSelected([]); setMenu(null); }; - const downloadNode = (node: CanvasNode) => { + const readProjectMedia = async (src: string, projectId: string) => { + const client = creationClient(projectId); + const asset = client.assetForSource(src); + if (asset && ["image/heic", "image/heif"].includes(asset.metadata.mimeType)) + return client.blob(asset.id); + const response = await fetch(src); + if (!response.ok) throw new Error(t("errors.archive.readMedia")); + return response.blob(); + }; + const downloadNode = async (node: CanvasNode) => { try { if (node.data.src) downloadFile( - node.data.src, + ["image/heic", "image/heif"].includes(node.data.mimeType ?? "") + ? await readProjectMedia(node.data.src, historyRef.current.present.id) + : node.data.src, downloadFilename(node.data.title, node.data.mimeType), ); else if (node.type === "text") @@ -582,7 +594,10 @@ export function useWorkspaceTransfer(rt: WorkspaceRuntime) { }; const exportDocument = async () => { try { - const archive = await buildCanvasArchive(historyRef.current.present); + const document = historyRef.current.present; + const archive = await buildCanvasArchive(document, { + read: (src) => readProjectMedia(src, document.id), + }); downloadFile(archive.blob, archive.filename); } catch (error) { notify(errorMessage(error)); @@ -687,7 +702,7 @@ export function useWorkspaceTransfer(rt: WorkspaceRuntime) { expected?.node.type, ); validate(classifyImportFile(file).kind); - let resource = await readEditorResource(file, signal); + let resource = await workspaceProjection(importDocumentId).importFile(file, signal); validate(resource.kind); if (expected?.node.type === "subject") { // Validate the slot before registering, then retain the canonical @@ -697,14 +712,13 @@ export function useWorkspaceTransfer(rt: WorkspaceRuntime) { resource, expected.subjectSlot, ); - { + if (!resource.assetId) { const ref = await creationClient(importDocumentId).importBlob( file, file.name, resource.kind, ); - resource = - await workspaceProjection(importDocumentId).resource(ref); + resource = await workspaceProjection(importDocumentId).resource(ref); subjectWithResource( expected.node.data.subject, resource, diff --git a/tests/browser/heif_import.mjs b/tests/browser/heif_import.mjs new file mode 100644 index 00000000..1bcb19f9 --- /dev/null +++ b/tests/browser/heif_import.mjs @@ -0,0 +1,198 @@ +/** Real HEIF bytes through the UI and project storage; generation uses the fixture provider. */ +import assert from "node:assert/strict"; +import { mkdir, readFile } from "node:fs/promises"; +import { readZip } from "../../src/canvas/zipArchive.ts"; +import { launchBrowser } from "./engine.mjs"; + +const base = process.env.CREATION_WEB_URL ?? "http://127.0.0.1:5179"; +const api = process.env.CREATION_API_URL ?? "http://127.0.0.1:8019"; +const headers = { "X-OpenDirector-Request": "1", "Content-Type": "application/json" }; +async function call(path, body) { + const response = await fetch(api + path, { headers, + ...(body ? { method: "POST", body: JSON.stringify(body) } : {}) }); + assert.ok(response.ok, await response.clone().text()); + return response.json(); +} +async function until(read, check, label) { + const deadline = Date.now() + 30000; + for (;;) { + const value = await read(); + if (check(value)) return value; + assert.ok(Date.now() < deadline, `${label}: ${JSON.stringify(value)}`); + await new Promise(resolve => setTimeout(resolve, 100)); + } +} +const health = await call("/api/creation-test-health"); +assert.equal(health.isolated, true); +assert.equal(health.provider, "fixture"); +assert.equal((await (await fetch(base + "/api/creation-test-health", { headers })).json()).root, health.root); +const models = []; +for (const modelId of ["MiniMax-H3-Image", "MiniMax-H3-Image-Highspeed"]) + models.push((await call("/api/model-configs")).find(model => model.modelId === modelId) ?? + await call("/api/model-configs", { kind: "image", url: "https://api.minimax.cn/v1", modelId, + apiKey: "fixture-only", protocol: "openai", connectionType: "api", authMode: "bearer" })); +const heic = await readFile(new URL("../fixtures/heif/oriented.heic", import.meta.url)); +const heif = await readFile(new URL("../fixtures/heif/oriented.heif", import.meta.url)); +const projectId = crypto.randomUUID(); +const project = `/api/creation/projects/${projectId}`; +await call(project + "/commands", { commandId: crypto.randomUUID(), projectId, + type: "project.create", payload: { name: `HEIF import ${projectId.slice(0, 8)}` } }); +const targetId = crypto.randomUUID(), targetName = `HEIF paste ${targetId.slice(0, 8)}`; +await call(`/api/creation/projects/${targetId}/commands`, { commandId: crypto.randomUUID(), projectId: targetId, + type: "project.create", payload: { name: targetName } }); +const snapshot = () => call(project); +const browser = await launchBrowser(); +const page = await browser.newPage({ locale: "zh-CN", viewport: { width: 1600, height: 1000 } }); +page.setDefaultTimeout(20000); +await page.addInitScript(() => localStorage.setItem("opendirector.modelSetup.v1", "dismissed")); +const errors = []; +page.on("pageerror", error => errors.push(error.message)); +await mkdir("test-results/heif", { recursive: true }); +try { + await page.goto(`${base}/ai-tool/ai-canvas/${projectId}?creation=unified`); + await page.getByText("已保存到本机", { exact: true }).waitFor(); + await page.getByLabel("选择本地素材文件", { exact: true }).setInputFiles({ + name: "portrait.HEIC", mimeType: "application/octet-stream", buffer: heic }); + await until(snapshot, value => value.canvasItems.length === 1 && value.assets.length === 1, "HEIC upload"); + await page.evaluate(async encoded => { + const transfer = new DataTransfer(); + transfer.items.add(new File([Uint8Array.from(atob(encoded), c => c.charCodeAt(0))], "portrait.heif")); + const canvas = document.createElement("canvas"); + canvas.width = canvas.height = 128; + canvas.getContext("2d").fillRect(0, 0, 128, 128); + for (const [suffix, type] of [["png", "image/png"], ["jpg", "image/jpeg"], ["webp", "image/webp"]]) { + const blob = await new Promise(resolve => canvas.toBlob(resolve, type)); + transfer.items.add(new File([blob], `reference.${suffix}`, { type })); + } + document.querySelector(".react-flow__pane").dispatchEvent(new DragEvent("drop", { + bubbles: true, cancelable: true, dataTransfer: transfer, clientX: 400, clientY: 650 })); + }, heif.toString("base64")); + const imported = await until(snapshot, value => value.canvasItems.length === 5 && value.assets.length === 5, "mixed drop"); + assert.deepEqual(imported.canvasItems.map(item => item.title), + ["portrait.HEIC", "portrait.heif", "reference.png", "reference.jpg", "reference.webp"]); + const orderedAssets = imported.canvasItems.map(item => imported.assets.find(asset => item.binding.objectRef.assetId === asset.id)); + const originals = imported.assets.filter(asset => /image\/hei[cf]/.test(asset.metadata.mimeType)); + assert.equal(originals.length, 2); + for (const asset of originals) { + assert.deepEqual([asset.metadata.width, asset.metadata.height], [192, 320]); + const response = await fetch(api + project + `/assets/${asset.id}/content`, { headers }); + assert.deepEqual(Buffer.from(await response.arrayBuffer()), asset.metadata.mimeType === "image/heic" ? heic : heif); + } + for (const locale of ["zh-CN", "en-US"]) { + await page.locator(".cw-language-toggle").getByRole("button", { name: locale === "en-US" ? "EN" : "中文", exact: true }).click(); + await page.reload(); + await page.getByText(locale === "en-US" ? "Saved on this device" : "已保存到本机", { exact: true }).waitFor(); + for (const asset of originals) { + const node = imported.canvasItems.find(item => item.binding.objectRef?.assetId === asset.id); + assert.ok(node, JSON.stringify(imported.canvasItems)); + const pixels = await page.locator(`.react-flow__node[data-id="${node.id}"] img.cw-image-preview`).evaluate(async image => { + await image.decode(); + const canvas = document.createElement("canvas"); + canvas.width = image.naturalWidth; canvas.height = image.naturalHeight; + const ctx = canvas.getContext("2d"); ctx.drawImage(image, 0, 0); + return { size: [canvas.width, canvas.height], red: ctx.getImageData(96, 40, 1, 1).data[0], + blue: ctx.getImageData(96, 280, 1, 1).data[2] }; + }); + assert.deepEqual(pixels.size, [192, 320]); + assert.ok(pixels.red > 200 && pixels.blue > 200); + } + await page.getByLabel(locale === "en-US" ? "Choose local asset files" : "选择本地素材文件", { exact: true }).setInputFiles({ + name: "corrupt.heic", mimeType: "image/heic", buffer: heic.subarray(0, 100) }); + await page.getByText(locale === "en-US" ? /image could not be decoded/ : /无法读取图片/).waitFor(); + const tooLarge = await page.evaluate(() => { + const canvas = document.createElement("canvas"); + canvas.width = 5001; canvas.height = 5000; + return canvas.toDataURL("image/png").split(",")[1]; + }); + await page.getByLabel(locale === "en-US" ? "Choose local asset files" : "选择本地素材文件", { exact: true }).setInputFiles({ + name: "too-large.heif", mimeType: "image/heif", buffer: Buffer.from(tooLarge, "base64") }); + await page.getByText(locale === "en-US" ? /25 million pixels/ : /2500 万/).waitFor(); + assert.equal((await snapshot()).assets.length, 5); + await page.screenshot({ path: `test-results/heif/${locale}.png`, fullPage: true }); + } + const originalNode = imported.canvasItems[0]; + await page.locator(`.react-flow__node[data-id="${originalNode.id}"]`).click({ button: "right" }); + const originalDownload = page.waitForEvent("download"); + await page.getByRole("menuitem", { name: "Download asset", exact: true }).click(); + assert.deepEqual(await readFile(await (await originalDownload).path()), heic); + const download = page.waitForEvent("download"); + await page.getByRole("button", { name: "Export canvas", exact: true }).click(); + const archiveFile = await (await download).path(); + const archive = await readZip(await readFile(archiveFile)); + for (const [suffix, bytes] of [["HEIC", heic], ["heif", heif]]) { + const entry = [...archive].find(([path]) => path.endsWith(`.${suffix}`)); + assert.ok(entry, JSON.stringify([...archive.keys()])); + assert.deepEqual(Buffer.from(entry[1]), bytes); + } + for (const model of models) { + const created = await page.evaluate(async ({ projectId, model, sources }) => { + const { creationClient } = await import("/src/backend/creationClient.ts"); + const client = creationClient(projectId); + await client.refresh(); + const specId = crypto.randomUUID(), itemId = crypto.randomUUID(); + const receipt = await client.command("canvas.createGeneration", { + spec: { id: specId, projectId, kind: "image", purpose: "generation", modelConfigId: model.id, + prompt: "Combine references in order", parameters: { size: "auto" } }, + item: { id: itemId, projectId, kind: "image", title: model.modelId, + position: { x: 500, y: 1000 }, binding: { role: "generation", objectRef: { id: specId } } }, + references: sources.map((asset, order) => ({ id: crypto.randomUUID(), projectId, targetSpecId: specId, + binding: "pinned", order, source: { assetId: asset.id, revision: asset.revision, fingerprint: asset.fingerprint, kind: "image" } })), + }); + if (receipt.status !== "applied") throw Error(JSON.stringify(receipt)); + return { itemId, specId }; + }, { projectId, model, sources: orderedAssets }); + await page.reload(); + await page.locator(`.react-flow__node[data-id="${created.itemId}"]`).click(); + await page.getByRole("button", { name: "Generate", exact: true }).click(); + const completed = await until(snapshot, value => value.jobs.some(job => job.inputSnapshot.spec.id === created.specId && job.status === "succeeded"), "image edit"); + const job = completed.jobs.find(job => job.inputSnapshot.spec.id === created.specId); + assert.equal(job.adopted, true); + assert.deepEqual(job.inputSnapshot.references.map(reference => reference.asset.assetId), orderedAssets.map(asset => asset.id)); + } + const freshCopy = await page.evaluate(async ({ projectId, assetId }) => { + const { creationClient } = await import("/src/backend/creationClient.ts"); + const { CreationProjection } = await import("/src/backend/creationProjection.ts"); + const client = creationClient(projectId); + await client.refresh(); + const copied = await new CreationProjection(client).importResource({ + kind: "image", title: "Copied output.heic", src: await client.mediaUrl(assetId), fresh: true, + identity: "publication:heif-fixture" }); + return client.snapshot.assets.find(asset => asset.id === copied.assetId); + }, { projectId, assetId: originalNode.binding.objectRef.assetId }); + assert.notEqual(freshCopy.id, originalNode.binding.objectRef.assetId); + assert.equal(freshCopy.metadata.mimeType, "image/heic"); + const freshBytes = await fetch(api + project + `/assets/${freshCopy.id}/content`, { headers }); + assert.deepEqual(Buffer.from(await freshBytes.arrayBuffer()), heic); + await page.locator(`.react-flow__node[data-id="${originalNode.id}"]`).click(); + await page.locator(".cw-workspace").focus(); + await page.keyboard.press("Control+c"); + await page.getByRole("button", { name: "Switch project", exact: true }).click(); + await page.getByRole("menuitemradio", { name: targetName, exact: true }).click(); + await page.locator(".cw-project-name").filter({ hasText: targetName }).waitFor(); + await page.getByText("Saved on this device", { exact: true }).waitFor(); + // A missing source forces the fallback to use bytes captured before switching. + await page.route("**/uploads/*/copy", route => route.fulfill({ status: 404, + contentType: "application/json", body: JSON.stringify({ detail: "Source unavailable" }) })); + await page.locator(".cw-workspace").focus(); + await page.keyboard.press("Control+v"); + const pasted = await until(() => call(`/api/creation/projects/${targetId}`), + value => value.canvasItems.length === 1 && value.assets.length === 1, "HEIF cross-project paste"); + const pastedBytes = await fetch(`${api}/api/creation/projects/${targetId}/assets/${pasted.assets[0].id}/content`, { headers }); + assert.deepEqual(Buffer.from(await pastedBytes.arrayBuffer()), heic); + const copiedImage = page.locator(".react-flow__node img.cw-image-preview"); + assert.deepEqual(await copiedImage.evaluate(async image => { + await image.decode(); return [image.naturalWidth, image.naturalHeight]; + }), [192, 320]); + await page.locator('input[type="file"][accept=".canvas,application/zip"]').setInputFiles(archiveFile); + await page.waitForFunction(() => document.querySelectorAll(".react-flow__node").length === 5); + for (const node of imported.canvasItems.slice(0, 2)) { + const image = page.locator(`.react-flow__node[data-id="${node.id}"] img.cw-image-preview`); + assert.deepEqual(await image.evaluate(async image => { + await image.decode(); return [image.naturalWidth, image.naturalHeight]; + }), [192, 320]); + } + assert.deepEqual(errors, []); + console.log(JSON.stringify({ passed: true, projectId, checks: ["upload", "mixed drop", "orientation", "reload", "bilingual rejection", "original download and archive round trip", "both model submissions with fixture results", "cross-project paste fallback"] })); +} finally { + await browser.close(); +} diff --git a/tests/browser/manifest.json b/tests/browser/manifest.json index c1f5773d..4267cb86 100644 --- a/tests/browser/manifest.json +++ b/tests/browser/manifest.json @@ -1,5 +1,10 @@ { "scripts": [ + { + "file": "heif_import.mjs", + "lanes": ["extended"], + "note": "Real HEIC/HEIF import, orientation, reload, bilingual errors and original export; fixture generation. Imports /src through Vite." + }, { "file": "kling_video.mjs", "lanes": ["extendedFixture"], diff --git a/tests/canvasWorkspaceEntry.test.ts b/tests/canvasWorkspaceEntry.test.ts index 75ab3347..fdd325b0 100644 --- a/tests/canvasWorkspaceEntry.test.ts +++ b/tests/canvasWorkspaceEntry.test.ts @@ -76,7 +76,7 @@ test("upload picker accept follows node kind and subject slot", () => { generating: false, nodeType: "image", }); - assert.deepEqual(image, { blocked: null, multiple: false, accept: "image/*" }); + assert.deepEqual(image, { blocked: null, multiple: false, accept: "image/*,.heic,.heif" }); assert.equal( uploadPickerPlan({ hasTarget: true, @@ -119,7 +119,7 @@ test("upload picker accept follows node kind and subject slot", () => { subjectSlot: "auxiliary", }) as { accept: string } ).accept, - "image/*", + "image/*,.heic,.heif", ); assert.equal( ( @@ -131,7 +131,7 @@ test("upload picker accept follows node kind and subject slot", () => { subjectSlot: "main", }) as { accept: string } ).accept, - "image/*,video/*", + "image/*,.heic,.heif,video/*", ); }); diff --git a/tests/editorMedia.test.ts b/tests/editorMedia.test.ts index be79174d..086bcb85 100644 --- a/tests/editorMedia.test.ts +++ b/tests/editorMedia.test.ts @@ -19,6 +19,8 @@ test("imports fall back only for missing or generic MIME and preserve explicit t for (const type of ["", "application/octet-stream"]) { for (const [name, kind, mimeType] of [ ["portrait.JPG", "image", "image/jpeg"], + ["portrait.HEIC", "image", "image/heic"], + ["portrait.heif", "image", "image/heif"], ["clip.MOV", "video", "video/quicktime"], ["voice.wav", "audio", "audio/wav"], ]) diff --git a/tests/fixtures/heif/README.md b/tests/fixtures/heif/README.md new file mode 100644 index 00000000..aacd83cd --- /dev/null +++ b/tests/fixtures/heif/README.md @@ -0,0 +1,27 @@ +# Oriented HEIF fixtures + +These original test images contain HEVC pixels, a container rotation and EXIF +orientation 6. Decoding must produce a 192 × 320 portrait, red on top and blue +below, without applying the EXIF rotation a second time. They contain no photos +or personal metadata. + +Generated with Pillow 13 and pillow-heif 1.8.1: + +```python +from pathlib import Path +from PIL import Image +from pillow_heif import register_heif_opener + +register_heif_opener() +image = Image.new('RGB', (320, 192), (240, 20, 20)) +image.paste((20, 20, 240), (160, 0, 320, 192)) +exif = Image.Exif() +exif[274] = 6 +image.save('oriented.heic', 'HEIF', quality=100, chroma=444, exif=exif.tobytes()) +data = Path('oriented.heic').read_bytes() +# Generic HEIF major brand; retain the encoder's compatible HEVC brands. +Path('oriented.heif').write_bytes(data[:8] + b'mif1' + data[12:]) +``` + +The tests verify both decoded pixels and MIME detection from bytes, including +when the upload filename has no image extension. diff --git a/tests/fixtures/heif/oriented.heic b/tests/fixtures/heif/oriented.heic new file mode 100644 index 00000000..4a0751c8 Binary files /dev/null and b/tests/fixtures/heif/oriented.heic differ diff --git a/tests/fixtures/heif/oriented.heif b/tests/fixtures/heif/oriented.heif new file mode 100644 index 00000000..ca021f8a Binary files /dev/null and b/tests/fixtures/heif/oriented.heif differ diff --git a/tests/test_heif.py b/tests/test_heif.py new file mode 100644 index 00000000..159d09a8 --- /dev/null +++ b/tests/test_heif.py @@ -0,0 +1,120 @@ +import asyncio +from io import BytesIO +from pathlib import Path +from uuid import uuid4 + +from fastapi.testclient import TestClient +from PIL import Image +import pytest + +from opendirector.application.creation_assets import CreationAssets +from opendirector.agent.gateway import ToolGateway +from opendirector.infrastructure.creation_projects import CreationProjects +from opendirector.infrastructure.local_projects import LocalProjects +from opendirector.media import heif_type +from opendirector.media.thumbnails import preview +from opendirector_api.main import create_app +from test_creation_assets import storage, import_asset +from test_agent_artifacts import publication + +FIXTURES = Path(__file__).parent / 'fixtures' / 'heif' + + +def assert_portrait(data): + with Image.open(BytesIO(data)) as image: + assert image.size == (192, 320) + assert image.getpixel((96, 40))[0] > 200 + assert image.getpixel((96, 280))[2] > 200 + + +@pytest.mark.parametrize('suffix', ['heic', 'heif']) +@pytest.mark.parametrize('name', ['photo', 'misnamed.bin']) +def test_heif_original_preview_orientation_and_reopen(storage, suffix, name): + projects, pid, session = storage + data = (FIXTURES / f'oriented.{suffix}').read_bytes() + asset, _, _ = import_asset(storage, f'{name}.{suffix}' if name == 'photo' else name, data) + assert asset.kind == 'image' and asset.metadata.mimeType == f'image/{suffix}' + assert (asset.metadata.width, asset.metadata.height) == (192, 320) + service = CreationAssets(projects) + assert_portrait(service.thumbnail(pid, asset.id)[0]) + assert_portrait(service.browser_image(pid, asset.id)[0]) + assert_portrait(preview(asset.fingerprint, data, 'image')) + before = projects.snapshot(pid) + projects.local.release(session) + reopened = CreationProjects(LocalProjects(projects.local.repository)) + service = CreationAssets(reopened) + with service.open_media(pid, asset.id)[0] as original: + assert original.read() == data + assert_portrait(service.browser_image(pid, asset.id)[0]) + assert reopened.snapshot(pid) == before + with TestClient(create_app(projects.local)) as client: + path = f'/api/creation/projects/{pid}/assets/{asset.id}' + headers = {'X-OpenDirector-Request': '1'} + original = client.get(path + '/content', headers=headers) + assert original.content == data and original.headers['content-type'] == f'image/{suffix}' + display = client.get(path + '/browser-image', headers=headers) + assert display.status_code == 200 and display.headers['content-type'] == 'image/png' + assert_portrait(display.content) + + +@pytest.mark.parametrize('locale', ['zh-CN', 'en-US']) +@pytest.mark.parametrize('invalid', ['corrupt', 'pixels', 'sequence']) +def test_invalid_image_upload_is_localized_and_does_not_publish(storage, locale, invalid): + projects, pid, session = storage + if invalid == 'corrupt': + data = (FIXTURES / 'oriented.heic').read_bytes()[:100] + elif invalid == 'sequence': + data = (FIXTURES / 'oriented.heic').read_bytes() + data = data[:8] + b'hevc' + data[12:] + else: + output = BytesIO() + Image.new('RGB', (5001, 5000)).save(output, 'PNG') + data = output.getvalue() + with TestClient(create_app(projects.local)) as client: + response = client.put(f'/api/creation/projects/{pid}/uploads/{uuid4()}', content=data, + headers={'X-OpenDirector-Request': '1', 'X-OpenDirector-Session': session, + 'X-Asset-Name': 'photo.heic', 'X-Asset-Kind': 'image', 'Accept-Language': locale}) + assert response.status_code == (413 if invalid == 'pixels' else 400) + message = response.json()['detail'] + expected = ('25 million' if invalid == 'pixels' else 'decoded') if locale == 'en-US' else ( + '2500 万' if invalid == 'pixels' else '无法读取图片') + assert expected in message + assert projects.snapshot(pid).assets == () + + +def test_heif_detection_does_not_admit_avif_video_or_sequences(): + for data in (b'', b'not an image', b'\x00\x00\x00\x18ftypavif\x00\x00\x00\x00', + b'\x00\x00\x00\x18ftyphevc\x00\x00\x00\x00', + b'\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00'): + assert heif_type(data) is None + + +@pytest.mark.parametrize('suffix', ['heic', 'heif']) +def test_agent_import_and_preview_use_the_same_validated_original(publication, suffix): + business, run, work = publication + data = (FIXTURES / f'oriented.{suffix}').read_bytes() + name = f'photo.{suffix}' + (work / name).write_bytes(data) + (work / f'corrupt.{suffix}').write_bytes(data[:100]) + + async def execute(): + gateway = ToolGateway(business, run) + try: + result = await gateway.call('import', 'execute_command', {'type': 'artifact.publish', + 'payload': {'path': name, 'name': name, 'expectedKind': 'image'}}) + assert result.error is None, result + asset = business.assets.get(run.project_id, result.value['changed'][0]['id']) + assert asset.metadata.mimeType == f'image/{suffix}' + with business.assets.open_media(run.project_id, asset.id)[0] as original: + assert original.read() == data + preview = await gateway.call('preview', 'query', {'type': 'asset.preview', 'objectId': str(asset.id)}) + assert preview.error is None, preview + assert_portrait(Path(preview.value['preview']['path']).read_bytes()) + corrupt = await gateway.call('corrupt', 'execute_command', {'type': 'artifact.publish', + 'payload': {'path': f'corrupt.{suffix}', 'name': f'corrupt.{suffix}', 'expectedKind': 'image'}}) + assert corrupt.error is not None + assert len(business.projects.snapshot(run.project_id).assets) == 1 + finally: + await gateway.close() + + asyncio.run(execute()) diff --git a/tests/test_minimax_image.py b/tests/test_minimax_image.py index 1fa62cb0..aa7d720d 100644 --- a/tests/test_minimax_image.py +++ b/tests/test_minimax_image.py @@ -2,6 +2,8 @@ import base64 import io import json +from pathlib import Path +from uuid import UUID, uuid4 import httpx import pytest @@ -12,10 +14,16 @@ from opendirector.infrastructure.model_configs import ModelConfigs from opendirector.integrations.model_adapters import adapter_for from opendirector.integrations.providers import GenerationProviders +from opendirector.application.creation_generation import CreationGeneration +from opendirector.domain.projects import ProjectError +from opendirector.domain.creation.commands import parse_command +from test_creation_assets import import_asset, ref +from test_creation_canvas_service import project, media, send, ver URL = 'https://api.minimax.cn/v1' RESULT = 'https://93.184.215.14/output.png' MODELS = ('MiniMax-H3-Image', 'MiniMax-H3-Image-Highspeed') +HEIF = Path(__file__).parent / 'fixtures' / 'heif' def png(color): @@ -88,3 +96,101 @@ def timeout(request): asyncio.run(adapter.image(config, ImageRequest('A landscape', 1024, 1024))) assert failure.value.details.category == 'submission_unknown' assert calls == ['POST'] + + +@pytest.mark.parametrize('model', MODELS) +@pytest.mark.parametrize('source', ['ui', 'agent']) +def test_mixed_heif_edits_share_original_assets_and_frozen_order(project, model, source): + projects, pid, session = project + configs = ModelConfigs(projects.local.repository) + saved = configs.save('image', URL, model, 'fixture-key', protocol='openai') + inputs = [('oriented.heif', (HEIF / 'oriented.heif').read_bytes()), ('red.png', png('red')), + ('oriented.heic', (HEIF / 'oriented.heic').read_bytes())] + for fmt in ('JPEG', 'WEBP'): + output = io.BytesIO() + Image.new('RGB', (128, 128), 'green').save(output, fmt) + inputs.append((f'green.{fmt.lower()}', output.getvalue())) + assets = [import_asset(project, name, data)[0] for name, data in inputs] + target, spec = media(project, kind='image') + spec = spec.model_copy(update={'prompt': 'Combine in reference order', 'modelConfigId': UUID(saved['id'])}) + assert send(projects, pid, 'generationSpec.update', session=session, + versions=[ver('generation_spec', spec)], spec=spec).status == 'applied' + for index, asset in enumerate(assets): + spec = projects.snapshot(pid).generationSpecs[-1] + target = projects.snapshot(pid).canvasItems[-1] + receipt = send(projects, pid, 'reference.add', session=session, + versions=[ver('generation_spec', spec), ver('canvas_item', target)], reference=dict(id=uuid4(), projectId=pid, + targetSpecId=spec.id, binding='pinned', source=ref(asset), order=index)) + assert receipt.status == 'applied', receipt + calls = [] + result = png('purple') + + def respond(request): + if request.method == 'GET': + return httpx.Response(200, content=result) + assert request.url.path == '/v1/images/edits' + calls.append(json.loads(request.content)) + return httpx.Response(200, json={'data': [{'url': RESULT}]}) + + async def run(): + transport = httpx.MockTransport(respond) + runner = CreationGeneration(projects, configs) + runner.provider = GenerationProviders(runner.store, transport) + try: + snapshot = projects.snapshot(pid) + spec, target = snapshot.generationSpecs[-1], snapshot.canvasItems[-1] + command = parse_command(dict(commandId=uuid4(), projectId=pid, type='generation.submit', + expectedVersions=[ver('generation_spec', spec), ver('canvas_item', target)], + payload=dict(specId=spec.id, expectedSpecRevision=spec.revision, originItemId=target.id))) + receipt = await runner.execute(command, session=session, source=source) + assert receipt.status == 'accepted', receipt + await asyncio.gather(*(job.future for job in list(runner.jobs.values()))) + snapshot = projects.snapshot(pid) + job = snapshot.jobs[0] + assert job.status == 'succeeded' and job.adopted, job + assert [r.asset for r in job.inputSnapshot.references] == [ref(asset) for asset in assets] + assert len(snapshot.results) == 1 + assert (await runner.execute(command, session=session, source=source)) == receipt + with runner.assets.open_media(pid, snapshot.results[0].asset.assetId)[0] as stream: + assert stream.read() == result + config = configs.resolve(saved['id'], 'image') + adapter = adapter_for(config, transport) + assert (await adapter.image(config, ImageRequest('Combine', 1024, 1024, + inputs[0][1], references=tuple(data for _, data in inputs[1:])))).data == result + finally: + await runner.close() + + asyncio.run(run()) + expected = [{'image_url': f'data:{asset.metadata.mimeType};base64,' + base64.b64encode(data).decode()} + for asset, (_, data) in zip(assets, inputs)] + assert len(calls) == 2 + assert calls[0]['images'] == calls[1]['images'] == expected + + +@pytest.mark.parametrize('invalid', ['corrupt', 'small', 'edge', 'ratio', 'count', 'bytes']) +def test_heif_reference_limits_reject_before_provider_request(tmp_path, invalid): + configs = ModelConfigs(tmp_path) + saved = configs.save('image', URL, MODELS[0], 'fixture-key', protocol='openai') + config = configs.resolve(saved['id'], 'image') + data = (HEIF / 'oriented.heic').read_bytes() + references = () + if invalid == 'corrupt': + data = data[:100] + elif invalid in {'small', 'edge', 'ratio'}: + output = io.BytesIO() + Image.new('RGB', {'small': (127, 128), 'edge': (8193, 2048), 'ratio': (128, 513)}[invalid]).save(output, 'HEIF') + data = output.getvalue() + elif invalid == 'count': + references = (data,) * 16 + elif invalid == 'bytes': + # Each valid file fits, but their Base64 JSON exceeds the request budget. + data += b'\0' * (25 * 1024 * 1024) + references = (data,) + + def unexpected(request): + pytest.fail('Invalid input reached the provider') + + with pytest.raises(ProjectError) as failure: + asyncio.run(adapter_for(config, httpx.MockTransport(unexpected)).image(config, + ImageRequest('Combine', 1024, 1024, data, references=references))) + assert failure.value.status == (413 if invalid == 'bytes' else 422)