diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6773877f..b34f5550 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,6 +78,7 @@ jobs: node tests/browser/minimax_h3.mjs node tests/browser/provider_settings.mjs node tests/browser/created_node_focus.mjs + node tests/browser/cross_canvas_paste.mjs node tests/browser/reference_reconnect.mjs node tests/browser/context_menu_dismissal.mjs node tests/browser/mp4_export.mjs diff --git a/src/canvas/CanvasWorkspace.tsx b/src/canvas/CanvasWorkspace.tsx index 59bd3eef..806067eb 100644 --- a/src/canvas/CanvasWorkspace.tsx +++ b/src/canvas/CanvasWorkspace.tsx @@ -245,6 +245,11 @@ import { type CanvasFragment, } from "./operations"; import { buildCanvasArchive, readCanvasArchive } from "./canvasArchive.ts"; +import { + captureClipboardMedia, + transferClipboardMedia, + type ClipboardMedia, +} from "./clipboardMedia.ts"; import { CanvasPointerGesture } from "./pointerGesture"; import { resourceNodeData } from "./projectAssets"; import { persistCurrentDocument } from "./projectManagement"; @@ -676,6 +681,8 @@ function Workspace() { const [assistantOpen, setAssistantOpen] = useState(false); const [assistantSelectionRequest, setAssistantSelectionRequest] = useState(0); const clipboard = useRef(null); + const clipboardMedia = useRef(null); + const pasting = useRef(false); const pasteCount = useRef(0); const fileInput = useRef(null); const uploadTarget = useRef(null); @@ -2271,6 +2278,10 @@ function Workspace() { historyRef.current.present, selectedRef.current, ); + clipboardMedia.current = captureClipboardMedia( + historyRef.current.present.id, + clipboard.current, + ); pasteCount.current = 0; notify(`已复制 ${clipboard.current.nodes.length} 个节点`); }; @@ -2288,6 +2299,10 @@ function Workspace() { const fragment = result.fragment; change(() => result.document); clipboard.current = fragment; + clipboardMedia.current = captureClipboardMedia( + historyRef.current.present.id, + fragment, + ); pasteCount.current = 0; setSelected([]); setMenu(null); @@ -2299,7 +2314,48 @@ function Workspace() { ? copySelection(before, selectedRef.current) : clipboard.current; if (!fragment?.nodes.length) return; - const distance = 40 * (duplicate ? 1 : ++pasteCount.current); + const media = clipboardMedia.current; + if (!duplicate && media && media.projectId !== before.id) { + void pasteFromProject(fragment, media); + return; + } + insertFragment(fragment, 40 * (duplicate ? 1 : ++pasteCount.current)); + }; + const pasteFromProject = async ( + fragment: CanvasFragment, + media: ClipboardMedia, + ) => { + if (pasting.current) return; + pasting.current = true; + const projectId = historyRef.current.present.id; + try { + const transferred = await transferClipboardMedia( + fragment, + media, + projectId, + async (blob, title, kind) => { + const client = creationClient(projectId); + const ref = await client.importBlob( + blob, + withMediaSuffix(title, blob.type), + kind, + ); + return { + assetId: ref.assetId, + src: await client.mediaUrl(ref.assetId), + }; + }, + ); + if (historyRef.current.present.id !== projectId) return; + insertFragment(transferred, 40 * ++pasteCount.current); + } catch (error) { + notify(`跨画布粘贴失败,未粘贴任何节点:${errorMessage(error)}`); + } finally { + pasting.current = false; + } + }; + const insertFragment = (fragment: CanvasFragment, distance: number) => { + const before = historyRef.current.present; const result = pasteFragment(before, fragment, { x: distance, y: distance, diff --git a/src/canvas/canvasArchive.ts b/src/canvas/canvasArchive.ts index e7c09ddc..9fa8b1e9 100644 --- a/src/canvas/canvasArchive.ts +++ b/src/canvas/canvasArchive.ts @@ -13,7 +13,7 @@ export const CANVAS_ASSET_DIRECTORY = "assets"; const UNSAFE_FILE_NAME = /[\\/:*?"<>|\u0000-\u001f\u007f]/g; /** Field names that hold media: node sources, palettes and adopted outputs. */ -const MEDIA_FIELDS = new Set(["src", "sourceImageUri", "output"]); +export const MEDIA_FIELDS = new Set(["src", "sourceImageUri", "output"]); export interface CanvasArchive { blob: Blob; diff --git a/src/canvas/clipboardMedia.ts b/src/canvas/clipboardMedia.ts new file mode 100644 index 00000000..8d40105a --- /dev/null +++ b/src/canvas/clipboardMedia.ts @@ -0,0 +1,155 @@ +import { MEDIA_FIELDS } from "./canvasArchive.ts"; +import type { CanvasResource } from "./model.ts"; +import type { CanvasFragment } from "./operations.ts"; + +const KINDS = new Set([ + "image", + "video", + "audio", + "text", + "file", +]); + +/** Media copied with a fragment. Switching projects revokes the source's object + * URLs, so the bytes are read while the copy is made. */ +export interface ClipboardMedia { + projectId: string; + blobs: Map>; + /** Registered copies per target project, so repeated pastes and retries reuse them. */ + registered: Map>; +} + +export interface RegisteredMedia { + assetId: string; + src: string; +} + +export type RegisterMedia = ( + blob: Blob, + title: string, + kind: CanvasResource["kind"], +) => Promise; + +const isTemporary = (value: unknown): value is string => + typeof value === "string" && /^(blob|data):/i.test(value); + +function visitMedia( + value: unknown, + found: (record: Record, key: string, src: string) => void, +): void { + if (!value || typeof value !== "object") return; + const record = value as Record; + for (const [key, entry] of Object.entries(record)) { + if (MEDIA_FIELDS.has(key) && isTemporary(entry)) found(record, key, entry); + else if (typeof entry === "object") visitMedia(entry, found); + } +} + +async function readBlob(src: string): Promise { + const response = await fetch(src); + if (!response.ok) throw new Error("素材读取失败。"); + return response.blob(); +} + +/** Starts reading every inline media field before the source project can close. */ +export function captureClipboardMedia( + projectId: string, + fragment: CanvasFragment, + read: (src: string) => Promise = readBlob, +): ClipboardMedia { + const blobs = new Map>(); + visitMedia(fragment.nodes, (_record, _key, src) => { + if (blobs.has(src)) return; + const pending = read(src); + // A failed read is reported by the paste that needs it. + pending.catch(() => {}); + blobs.set(src, pending); + }); + return { projectId, blobs, registered: new Map() }; +} + +/** + * Re-homes a fragment copied from another project: each media field is + * registered in the target project before any node appears, and source-project + * identities are dropped. A generated image's task stays in its own project, so + * its copy becomes a plain material image instead of claiming that provenance. + */ +export async function transferClipboardMedia( + fragment: CanvasFragment, + media: ClipboardMedia, + targetProjectId: string, + register: RegisterMedia, +): Promise { + const copy = structuredClone(fragment); + const registrations: Promise[] = []; + for (const node of copy.nodes) { + delete node.data.backendAssetId; + delete node.data.copiedSourceId; + const nodeKind = KINDS.has(node.type as CanvasResource["kind"]) + ? (node.type as CanvasResource["kind"]) + : undefined; + visitMedia(node, (record, key, src) => { + const blob = media.blobs.get(src); + if (!blob) throw new Error("复制的素材已失效,请重新复制。"); + const title = + typeof record.title === "string" ? record.title : node.data.title; + const kind = + typeof record.kind === "string" && + KINDS.has(record.kind as CanvasResource["kind"]) + ? (record.kind as CanvasResource["kind"]) + : record === node.data + ? nodeKind + : undefined; + const cacheKey = `${targetProjectId}\n${src}`; + let pending = media.registered.get(cacheKey); + if (!pending) { + pending = blob + .then((bytes) => + register(bytes, title, kind ?? mediaKind(bytes.type)), + ) + .catch((error) => { + media.registered.delete(cacheKey); + throw error; + }); + media.registered.set(cacheKey, pending); + } + registrations.push( + pending.then((registered) => { + record[key] = registered.src; + if (record === node.data && key === "src") + node.data.backendAssetId = registered.assetId; + else if (key === "src" && "assetId" in record) + record.assetId = registered.assetId; + }), + ); + }); + } + // Settle every upload before failing, so a retry finds the successful ones. + const failed = (await Promise.allSettled(registrations)).find( + (result) => result.status === "rejected", + ); + if (failed) throw failed.reason; + const materialized = new Set(); + for (const node of copy.nodes) + if ( + node.type === "image" && + node.data.mediaRole === "generation" && + node.data.src + ) { + node.data.mediaRole = "asset"; + node.data.prompt = ""; + delete node.data.promptContent; + delete node.data.promptMentions; + delete node.data.settings; + delete node.data.taskDraft; + materialized.add(node.id); + } + // A material image has no generation spec to receive references. + copy.edges = copy.edges.filter((edge) => !materialized.has(edge.target)); + return copy; +} + +function mediaKind(mimeType: string): CanvasResource["kind"] { + const family = mimeType.split("/")[0] as CanvasResource["kind"]; + return ["image", "video", "audio"].includes(family) ? family : "file"; +} diff --git a/tests/browser/cross_canvas_paste.mjs b/tests/browser/cross_canvas_paste.mjs new file mode 100644 index 00000000..966109d7 --- /dev/null +++ b/tests/browser/cross_canvas_paste.mjs @@ -0,0 +1,108 @@ +/** Isolated model double + real project storage: copy media between canvases (#69). */ +import assert from "node:assert/strict"; +import { launchBrowser } from "./engine.mjs"; +import { chooseGenerationModel } from "./model_picker.mjs"; +const base = process.env.CREATION_WEB_URL ?? "http://127.0.0.1:5179"; +const api = process.env.CREATION_API_URL ?? "http://127.0.0.1:8019"; +const headers = { "X-OpenDirector-Request": "1", "Content-Type": "application/json" }; +async function call(path, body) { + const response = await fetch(api + path, { headers, ...(body ? { method: "POST", body: JSON.stringify(body) } : {}) }); + assert.ok(response.ok, await response.clone().text()); + return response.json(); +} +async function until(check) { + const deadline = Date.now() + 20000; + while (Date.now() < deadline) { + const value = await check(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw Error("Expected persisted state not observed"); +} +assert.equal((await call("/api/creation-test-health")).isolated, true); +await call("/api/creation-test-gate", { open: true }); +const project = async (label) => { + const id = crypto.randomUUID(); + const name = `${label} ${id.slice(0, 8)}`; + await call(`/api/creation/projects/${id}/commands`, { projectId: id, commandId: crypto.randomUUID(), + type: "project.create", payload: { name } }); + return { id, name, state: () => call(`/api/creation/projects/${id}`) }; +}; +const source = await project("Paste source"); +const target = await project("Paste target"); +const browser = await launchBrowser(); +const errors = []; +try { + const page = await browser.newPage({ viewport: { width: 1500, height: 1000 } }); + page.setDefaultTimeout(15000); + page.on("pageerror", (error) => errors.push(error.message)); + const saved = () => page.getByText("已保存到本机", { exact: true }).waitFor(); + const nodes = page.locator(".react-flow__node"); + const loadedImages = (count) => page.waitForFunction((count) => { + const images = [...document.querySelectorAll(".react-flow__node img.cw-image-preview")]; + return images.length === count && images.every((image) => image.complete && image.naturalWidth > 0); + }, count); + await page.goto(`${base}/ai-tool/ai-canvas/${source.id}`); + await saved(); + // One generated image and one upload. + await page.getByRole("toolbar", { name: "画布创作工具" }).getByRole("button", { name: "添加图片", exact: true }).click(); + await chooseGenerationModel(page, { label: "fixture-image" }); + await page.getByRole("textbox", { name: "生成提示词", exact: true }).fill("Copied generation"); + await until(async () => (await source.state()).generationSpecs[0]?.modelConfigId); + await saved(); + await page.getByRole("button", { name: "提交生成", exact: true }).click(); + await until(async () => (await source.state()).jobs[0]?.status === "succeeded"); + await page.locator(".cw-generating").waitFor({ state: "hidden" }); + const png = await page.evaluate(() => { + const canvas = document.createElement("canvas"); + canvas.width = canvas.height = 64; + canvas.getContext("2d").fillRect(0, 0, 64, 64); + return canvas.toDataURL().split(",")[1]; + }); + await page.getByLabel("选择本地素材文件", { exact: true }).setInputFiles({ + name: "upload.png", mimeType: "image/png", buffer: Buffer.from(png, "base64"), + }); + await until(async () => (await source.state()).canvasItems.length === 2); + await saved(); + await loadedImages(2); + + await page.locator(".cw-workspace").focus(); + await page.keyboard.press("Control+a"); + await page.waitForFunction(() => document.querySelectorAll(".react-flow__node.selected").length === 2); + await page.keyboard.press("Control+c"); + // Switching closes the source client and revokes its preview URLs. + await page.getByRole("button", { name: "切换项目", exact: true }).click(); + await page.getByRole("menuitemradio", { name: target.name, exact: true }).click(); + await page.locator(".cw-project-name").filter({ hasText: target.name }).waitFor(); + await saved(); + await page.locator(".cw-workspace").focus(); + await page.keyboard.press("Control+v"); + await nodes.nth(1).waitFor(); + await loadedImages(2); + const pasted = await until(async () => { + const snapshot = await target.state(); + return snapshot.canvasItems.filter((item) => item.visible).length === 2 && snapshot; + }); + await saved(); + assert.equal(await page.getByText(/保存失败/).count(), 0); + assert.deepEqual(pasted.canvasItems.map((item) => item.binding.role).sort(), ["material", "material"]); + assert.equal(pasted.jobs.length, 0, "a copy must not claim the source project's generation task"); + assert.deepEqual(pasted.assets.map((asset) => asset.provenance.source), ["upload", "upload"]); + assert.ok(pasted.assets.every((asset) => asset.projectId === target.id)); + + // Paste again: the copied media is registered once per target project. + await page.keyboard.press("Control+v"); + await until(async () => (await target.state()).canvasItems.filter((item) => item.visible).length === 4); + await saved(); + assert.equal((await target.state()).assets.length, 2); + + // Reopening the target, with every preview cache gone, still reads the media. + await page.reload(); + await saved(); + await loadedImages(4); + assert.equal((await source.state()).canvasItems.length, 2, "the source canvas is unchanged"); + assert.deepEqual(errors, []); + console.log("PASS cross-canvas paste: media registered in the target, saved and readable after reopening"); +} finally { + await browser.close(); +} diff --git a/tests/clipboardMedia.test.ts b/tests/clipboardMedia.test.ts new file mode 100644 index 00000000..5aba6371 --- /dev/null +++ b/tests/clipboardMedia.test.ts @@ -0,0 +1,184 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + captureClipboardMedia, + transferClipboardMedia, + type RegisteredMedia, +} from "../src/canvas/clipboardMedia.ts"; +import { createNode } from "../src/canvas/model.ts"; +import type { CanvasFragment } from "../src/canvas/operations.ts"; + +function fixture() { + const image = URL.createObjectURL(new Blob(["png"], { type: "image/png" })); + const video = URL.createObjectURL(new Blob(["mp4"], { type: "video/mp4" })); + const portrait = URL.createObjectURL( + new Blob(["jpg"], { type: "image/jpeg" }), + ); + const generated = createNode( + "image", + { x: 0, y: 0 }, + { + title: "生成图片", + src: image, + backendAssetId: "source-image", + copiedSourceId: "source-item", + mediaRole: "generation", + prompt: "雨夜街道", + taskDraft: undefined, + }, + ); + const clip = createNode( + "video", + { x: 400, y: 0 }, + { + title: "镜头", + src: video, + backendAssetId: "source-video", + mediaRole: "generation", + prompt: "推镜", + }, + ); + const subject = createNode( + "subject", + { x: 800, y: 0 }, + { + title: "主角", + subject: { + description: "", + category: "人物", + main: { + id: "main", + assetId: "source-portrait", + kind: "image", + title: "正脸", + src: portrait, + }, + auxiliaries: [], + }, + }, + ); + const fragment: CanvasFragment = { + nodes: [generated, clip, subject], + edges: [ + { id: "into-image", source: subject.id, target: generated.id }, + { id: "into-video", source: generated.id, target: clip.id }, + ], + }; + return { fragment, urls: [image, video, portrait], generated, clip, subject }; +} + +function registrar() { + const calls: { title: string; kind: string; type: string }[] = []; + const register = async ( + blob: Blob, + title: string, + kind: string, + ): Promise => { + calls.push({ title, kind, type: blob.type }); + return { + assetId: `target-${calls.length}`, + src: `blob:target/${calls.length}`, + }; + }; + return { calls, register }; +} + +test("copied media survives the source project revoking its object URLs", async () => { + const { fragment, urls, generated, clip, subject } = fixture(); + const media = captureClipboardMedia("project-a", fragment); + // Switching canvases closes the source client, which revokes every preview URL. + for (const url of urls) URL.revokeObjectURL(url); + const { calls, register } = registrar(); + const pasted = await transferClipboardMedia( + fragment, + media, + "project-b", + register, + ); + assert.deepEqual(calls, [ + { title: "生成图片", kind: "image", type: "image/png" }, + { title: "镜头", kind: "video", type: "video/mp4" }, + { title: "正脸", kind: "image", type: "image/jpeg" }, + ]); + const [image, video, person] = pasted.nodes; + assert.equal(image!.data.src, "blob:target/1"); + assert.equal(image!.data.backendAssetId, "target-1"); + assert.equal(image!.data.copiedSourceId, undefined); + assert.equal(video!.data.src, "blob:target/2"); + assert.equal(video!.data.backendAssetId, "target-2"); + assert.deepEqual( + [person!.data.subject!.main!.src, person!.data.subject!.main!.assetId], + ["blob:target/3", "target-3"], + ); + // The clipboard itself stays untouched for later pastes. + assert.equal(generated.data.backendAssetId, "source-image"); + assert.equal(subject.data.subject!.main!.assetId, "source-portrait"); + assert.equal(clip.data.src, urls[1]); +}); + +test("a generated image pastes into another project as material without its task", async () => { + const { fragment } = fixture(); + const media = captureClipboardMedia("project-a", fragment); + const pasted = await transferClipboardMedia( + fragment, + media, + "project-b", + registrar().register, + ); + const [image, video] = pasted.nodes; + assert.equal(image!.data.mediaRole, "asset"); + assert.equal(image!.data.prompt, ""); + assert.equal(image!.data.settings, undefined); + // Generated videos may keep their configuration over imported media. + assert.equal(video!.data.mediaRole, "generation"); + assert.equal(video!.data.prompt, "推镜"); + // A material image cannot receive references; its outgoing reference stays. + assert.deepEqual( + pasted.edges.map((edge) => edge.id), + ["into-video"], + ); +}); + +test("repeated pastes reuse registrations and a failed one can be retried", async () => { + const { fragment } = fixture(); + const media = captureClipboardMedia("project-a", fragment); + let fail = true; + const { calls, register } = registrar(); + const flaky = (blob: Blob, title: string, kind: string) => { + if (kind === "video" && fail) return Promise.reject(new Error("上传失败")); + return register(blob, title, kind); + }; + await assert.rejects( + transferClipboardMedia(fragment, media, "project-b", flaky), + /上传失败/, + ); + assert.equal(calls.length, 2); + fail = false; + const pasted = await transferClipboardMedia( + fragment, + media, + "project-b", + flaky, + ); + assert.deepEqual( + calls.map((call) => call.kind), + ["image", "image", "video"], + ); + assert.equal(pasted.nodes[1]!.data.backendAssetId, "target-3"); + await transferClipboardMedia(fragment, media, "project-b", flaky); + assert.equal(calls.length, 3); + await transferClipboardMedia(fragment, media, "project-c", flaky); + assert.equal(calls.length, 6); +}); + +test("an unreadable copy fails the paste instead of inserting broken nodes", async () => { + const { fragment } = fixture(); + const media = captureClipboardMedia("project-a", fragment, async (src) => { + if (src === fragment.nodes[1]!.data.src) throw new Error("素材读取失败。"); + return new Blob(["x"], { type: "image/png" }); + }); + await assert.rejects( + transferClipboardMedia(fragment, media, "project-b", registrar().register), + /素材读取失败/, + ); +});