diff --git a/scripts/i18n-baseline.json b/scripts/i18n-baseline.json index b02000e8..1134439f 100644 --- a/scripts/i18n-baseline.json +++ b/scripts/i18n-baseline.json @@ -5,12 +5,13 @@ "src/canvas/CanvasWorkspace.tsx": 1, "src/canvas/dataBoundary.ts": 3, "src/canvas/generationCatalog.ts": 16, + "src/canvas/generationSettings.ts": 4, "src/canvas/library/AssetsPanel.tsx": 1, "src/canvas/library/libraryClient.ts": 1, "src/canvas/library/SubjectsPanel.tsx": 1, "src/canvas/library/TrashPanel.tsx": 1, "src/canvas/model.ts": 1, - "src/canvas/NodeComposer.tsx": 5, + "src/canvas/NodeComposer.tsx": 1, "src/canvas/referenceMentions.ts": 1, "src/canvas/toolCatalog.ts": 123 } diff --git a/src/canvas/CanvasWorkspace.tsx b/src/canvas/CanvasWorkspace.tsx index e8deb3f6..10f138f4 100644 --- a/src/canvas/CanvasWorkspace.tsx +++ b/src/canvas/CanvasWorkspace.tsx @@ -52,14 +52,7 @@ import { previewImageSelection } from "../backend/imageSelections"; import { MediaPublication } from "../backend/mediaPublication"; import { useBackendModels } from "../backend/models.ts"; import { openBackendProject } from "../backend/projects.ts"; -import { - backendBlocker, - backendModel, - backendReferences, - backendVideoMode, - modelLabel, - serviceVoice, -} from "../backend/rules.ts"; +import { backendModel, modelLabel } from "../backend/rules.ts"; import { backUpLibrarySubject, copyLibrarySubjects, @@ -94,7 +87,6 @@ import { studyImage, } from "./assets"; import { AssistantPanel } from "./AssistantPanel"; -import { assistantReferences } from "./assistantReferences"; import { CanvasActionsContext } from "./CanvasActionsContext"; import { CanvasConnectionLine, @@ -157,10 +149,7 @@ import { type GenerationJob as Job, } from "./generationHistory"; import { GenerationHistoryPanel } from "./GenerationHistoryPanel"; -import { - generationContent, - incomingReferenceNodes, -} from "./generationReferenceContent.ts"; +import { incomingReferenceNodes } from "./generationReferenceContent.ts"; import { referenceContentCommand } from "./generationReferenceEdit.ts"; import type { GenerationReferenceControls } from "./GenerationReferenceEditor"; import { absolutePosition, arrangeNodes, boundsOfNodes } from "./geometry"; @@ -254,7 +243,7 @@ import { CanvasPointerGesture } from "./pointerGesture"; import { resourceNodeData } from "./projectAssets"; import { persistCurrentDocument } from "./projectManagement"; import { ProjectSelector } from "./ProjectSelector"; -import { referencePresentation, referenceSnapshot } from "./referenceEvidence"; +import { referencePresentation } from "./referenceEvidence"; import "./settings.css"; import { modelSetupMissing, dismissModelSetup, shouldShowModelSetup } from "./ModelSetup"; import { WelcomeScreen } from "./WelcomeScreen"; @@ -271,7 +260,17 @@ import { subjectTargetIdentity, } from "./subjectLibraryImport.ts"; import { subjectWithResource, type SubjectMediaSlot } from "./subjectMedia.ts"; -import { agentGenerationBlocker, agentNodeSettings } from "./textGeneration"; +import { + authorFlowChanges, + canvasKeyAction, + canvasSelectAllIds, + connectionDragReleased, + droppedConnectionTarget, + mergeFlowSelection, + placeCreatedNode, + serviceGenerationPlan, + textGenerationPlan, +} from "./workspaceInteraction.ts"; import { TextNodeComposer } from "./TextNodeComposer"; import { useTheme } from "./theme"; import { useI18n } from "../i18n/I18nProvider"; @@ -1842,42 +1841,20 @@ function Workspace() { ...extra, }); if (kind === "timeline") node.width = 720; - node.position = { - x: where.x - node.width / 2, - y: where.y - node.height / 2, - }; - if (connected) { - // The dropped endpoint becomes the new node's input/output port. - node.position = { - x: where.x - (connected.direction === "target" ? node.width : 0), - y: where.y - node.height / 2, - }; - if (connected.side === "left") - node.position = { - x: where.x - node.width, - y: where.y - node.height / 2, - }; - if (connected.side === "top") - node.position = { - x: where.x - node.width / 2, - y: where.y - node.height, - }; - if (connected.side === "bottom") - node.position = { x: where.x - node.width / 2, y: where.y }; - } - if (menu?.parentId) { - const parent = current.nodes.find( - (candidate) => candidate.id === menu.parentId, - ); - if (parent) { - const origin = absolutePosition(parent, current.nodes); - node.parentId = parent.id; - node.position = { - x: node.position.x - origin.x, - y: node.position.y - origin.y, - }; - } - } + const parent = menu?.parentId + ? current.nodes.find((candidate) => candidate.id === menu.parentId) + : undefined; + // The dropped endpoint becomes the new node's input/output port. + const placed = placeCreatedNode( + node, + where, + connected, + parent + ? { id: parent.id, origin: absolutePosition(parent, current.nodes) } + : undefined, + ); + node.position = placed.position; + if (placed.parentId) node.parentId = placed.parentId; const commands: CanvasCommand[] = [ { type: "add-nodes", @@ -2018,17 +1995,24 @@ function Workspace() { const onConnectEnd: OnConnectEnd = (event, state) => { const start = connectionStarted.current; connectionStarted.current = null; - if (spacePressed || pointerGesture.current.cancelled) return; - const point = "changedTouches" in event ? event.changedTouches[0] : event; + const raw = "changedTouches" in event ? event.changedTouches[0] : event; + const point = raw ? { x: raw.clientX, y: raw.clientY } : null; if ( + !connectionDragReleased({ + spacePressed, + cancelled: pointerGesture.current.cancelled, + moved: pointerGesture.current.moved, + start, + point, + isValid: Boolean(state.isValid), + hasEndpoints: Boolean(state.fromNode && state.fromHandle), + }) || !point || - !start || - (!pointerGesture.current.moved && - Math.hypot(point.clientX - start.x, point.clientY - start.y) <= 4) + !state.fromNode || + !state.fromHandle ) return; - if (state.isValid || !state.fromNode || !state.fromHandle) return; - const target = document.elementFromPoint(point.clientX, point.clientY); + const target = document.elementFromPoint(point.x, point.y); const targetId = target ?.closest(".react-flow__node") ?.getAttribute("data-id"); @@ -2038,14 +2022,20 @@ function Workspace() { node.type === "group" && !lockedIds.has(node.id), ); - if (state.toNode && state.toNode.id !== group?.id) return; - if (!group && !target?.classList.contains("react-flow__pane")) return; + const drop = droppedConnectionTarget({ + toNodeId: state.toNode?.id, + groupId: group?.id, + onPane: Boolean(target?.classList.contains("react-flow__pane")), + fromNodeId: state.fromNode.id, + direction: state.fromHandle.type, + }); + if (!drop) return; setMenu({ - x: point.clientX, - y: point.clientY, - world: flow.screenToFlowPosition({ x: point.clientX, y: point.clientY }), - connected: { id: state.fromNode.id, direction: state.fromHandle.type }, - parentId: group?.id, + x: point.x, + y: point.y, + world: flow.screenToFlowPosition(point), + connected: { id: drop.id, direction: drop.direction }, + parentId: drop.parentId, }); }; const onNodesChange = useCallback( @@ -2070,50 +2060,23 @@ function Workspace() { } return changed ? next : current; }); - const selections = changes.filter((c) => c.type === "select"); + const selections = changes.flatMap((change) => + change.type === "select" + ? [{ id: change.id, selected: change.selected }] + : [], + ); if (selections.length) - setSelected((ids) => { - const next = new Set(ids); - for (const c of selections) - if (c.type === "select") { - if (c.selected) next.add(c.id); - else next.delete(c.id); - } - return [...next]; - }); - const patches: CanvasCommand[] = []; + setSelected((ids) => mergeFlowSelection(ids, selections)); // XYFlow may still deliver a final move/resize after cancellation. - if (pointerGesture.current.cancelled) return; // Capture the grab offset immediately, but do not author a move inside // click tolerance (or let that jitter snap a node to a nearby guide). - if (dragging.current && !pointerGesture.current.moved) return; - const resize = changes.find( - (c) => c.type === "dimensions" && c.dimensions && c.setAttributes, - ); - if (resize?.type === "dimensions" && resize.dimensions) { - const move = changes.find( - (c) => c.type === "position" && c.id === resize.id, - ); - // The document owns child-coordinate compensation, including locked children. - // Do not apply React Flow's child position patches a second time. - command({ - type: "resize-node", - id: resize.id, - ...resize.dimensions, - ...(move?.type === "position" && move.position - ? { position: move.position } - : {}), - }); - return; - } - const positions = changes.flatMap((c) => - c.type === "position" && c.position - ? [{ id: c.id, position: c.position }] - : [], - ); - for (const { id, position } of positions) - patches.push({ type: "update-node", id, patch: { position } }); - if (patches.length) command({ type: "batch", commands: patches }); + // A resize owns child-coordinate compensation, including locked children. + const authored = authorFlowChanges(changes, { + cancelled: pointerGesture.current.cancelled, + dragging: dragging.current, + moved: pointerGesture.current.moved, + }); + if (authored) command(authored); }, [command, flow], ); @@ -3572,97 +3535,29 @@ function Workspace() { }); } }, [jobs, ready, backendCatalog.connected, doc.id, change, persist, notify]); - const pinnedSubmissionInputs = (node: CanvasNode) => - (node.data.promptContent ?? []).flatMap((part) => { - if (part.type !== "reference" || part.binding !== "asset") return []; - const saved = creationClient(doc.id).snapshot?.assets.find( - (asset) => asset.id === part.assetId, - ); - const asset = - saved ?? - referenceAssets.assets.find((asset) => asset.id === part.assetId); - if ( - !asset || - !["text", "image", "video", "audio"].includes(asset.kind ?? "") - ) - throw new Error(t("canvas.workspace.error.pinnedMetadataUnloaded")); - return [ - { - binding: "asset" as const, - assetId: asset.id, - fingerprint: part.fingerprint, - referenceId: part.referenceId, - kind: asset.kind as "image" | "video" | "audio" | "text", - title: part.name, - }, - ]; - }); + const submissionAssets = () => [ + ...(creationClient(doc.id).snapshot?.assets ?? []), + ...referenceAssets.assets, + ]; const runServiceGeneration = (node: CanvasNode, incoming: CanvasNode[]) => { - const settings = getNodeGenerationSettings(node, backendCatalog.models); - const config = backendModel(settings, backendCatalog.models); - const blocked = backendBlocker(settings, incoming, backendCatalog.models); - if (blocked || !config) { - notify(blocked ?? t("canvas.workspace.error.modelUnavailable")); + const document = historyRef.current.present; + const plan = serviceGenerationPlan({ + document, + node, + incoming, + models: backendCatalog.models, + assets: submissionAssets(), + }); + if (plan.status === "blocked") { + notify(plan.message ?? t("canvas.workspace.error.modelUnavailable")); return; } - const references = backendReferences(settings, incoming, config); - const document = historyRef.current.present; - const request: GenerationRequest = { - documentId: document.id, - documentTitle: document.title, - nodeId: node.id, - promptContent: generationContent(node, incoming), - config, - kind: settings.mode, - prompt: settings.prompt, - title: node.data.title, - videoMode: backendVideoMode(settings, references, config), - durationSeconds: config.inputRules.videoCapabilities - ? settings.durationSeconds - : undefined, - aspectRatio: settings.aspectRatio, - resolution: settings.resolution, - voice: serviceVoice(settings), - references: [ - ...assistantReferences(incoming).map((input) => ({ - ...input, - contextOnly: !references.some( - ({ node: item }) => - item.id === input.nodeId && - item.data.kind === input.kind && - (input.kind === "text" || item.data.src === input.src), - ), - })), - ...pinnedSubmissionInputs(node).map((input) => ({ - ...input, - contextOnly: - settings.mode === "audio" || - (settings.mode === "video" && - settings.videoMode === "first-last" && - input.kind === "image" && - references.some((ref) => ref.role === "firstFrame")), - })), - ], - }; - const origin: ResultSource = { - documentId: document.id, - nodeId: node.id, - kind: node.type, - src: node.data.src, - referenceInputs: referenceSnapshot( - references.map( - (item) => - document.nodes.find((n) => n.id === item.node.id) ?? item.node, - ), - ), - }; - const count = settings.mode === "audio" ? 1 : (settings.count ?? 1); - const batchId = count > 1 ? crypto.randomUUID() : undefined; - for (let index = 0; index < count; index++) + const batchId = plan.count > 1 ? crypto.randomUUID() : undefined; + for (let index = 0; index < plan.count; index++) void runServiceTask( - { ...request, batchId, executionSlot: index }, - origin, - settings.prompt || node.data.title, + { ...plan.request, batchId, executionSlot: index }, + plan.origin, + plan.label, index === 0 ? "replace" : "additional", ); }; @@ -3713,45 +3608,18 @@ function Workspace() { return; const incoming = incomingNodes(current, nodeId); if (node.type === "text") { - const settings = agentNodeSettings(node); - const agents = backendCatalog.models.filter( - (model) => model.kind === "agent", - ); - const config = agents.find( - (model) => model.id === (settings.modelConfigId ?? agents[0]?.id), - ); - const blocker = agentGenerationBlocker(node, incoming, config); - if (blocker || !config) { - notify(blocker ?? t("canvas.workspace.error.agentUnavailable")); + const plan = textGenerationPlan({ + document: current, + node, + incoming, + models: backendCatalog.models, + assets: submissionAssets(), + }); + if (plan.status === "blocked") { + notify(plan.message ?? t("canvas.workspace.error.agentUnavailable")); return; } - void runServiceTask( - { - documentId: current.id, - documentTitle: current.title, - nodeId: node.id, - promptContent: generationContent(node, incoming), - config, - kind: "text", - prompt: node.data.prompt, - title: node.data.title, - videoMode: "first_frame", - thinking: settings.thinking, - maxTokens: settings.maxTokens, - references: [ - ...assistantReferences(incoming), - ...pinnedSubmissionInputs(node), - ], - }, - { - documentId: current.id, - nodeId: node.id, - kind: "text", - text: node.data.text, - referenceInputs: referenceSnapshot(incoming), - }, - node.data.title, - ); + void runServiceTask(plan.request, plan.origin, plan.label); return; } @@ -3920,106 +3788,60 @@ function Workspace() { !anchorElement?.closest(".react-flow") ) return; + const action = canvasKeyAction({ + key: event.key, + code: event.code, + mod, + shift: event.shiftKey, + hasClipboard: Boolean(clipboard.current), + editText: Boolean( + single?.type === "text" && + !lockedIds.has(single.id) && + (el === root.current || + el.closest(".react-flow__node")?.getAttribute("data-id") === + single.id), + ), + hasSingle: Boolean(single), + }); + if (!action) return; if ( - !mod && - event.key === "Enter" && - single?.type === "text" && - !lockedIds.has(single.id) && - (el === root.current || - el.closest(".react-flow__node")?.getAttribute("data-id") === single.id) - ) { + action.type !== "add-text" && + action.type !== "preview" && + action.type !== "help" + ) event.preventDefault(); + if (action.type === "edit-text" || action.type === "nudge") event.stopPropagation(); - setTextEditingId(single.id); - } else if (mod && key === "z") { - event.preventDefault(); - replayHistory(event.shiftKey); - } else if (mod && key === "y") { - event.preventDefault(); - replayHistory(true); - } else if (mod && key === "a") { - event.preventDefault(); - setSelected( - doc.nodes - .filter((n) => !projected.ownerByNodeId.has(n.id)) - .map((n) => n.id), - ); - } else if (mod && key === "c") { - event.preventDefault(); - copy(); - } else if (mod && key === "x") { - event.preventDefault(); - cut(); - } else if ( - mod && - (event.code === "BracketLeft" || event.code === "BracketRight") - ) { - event.preventDefault(); + if (action.type === "edit-text") { + if (single?.type === "text") setTextEditingId(single.id); + } else if (action.type === "history") replayHistory(action.forward); + else if (action.type === "select-all") + setSelected(canvasSelectAllIds(doc.nodes)); + else if (action.type === "copy") copy(); + else if (action.type === "cut") cut(); + else if (action.type === "reorder") command({ type: "reorder-nodes", ids: selectedRef.current, - direction: - event.code === "BracketLeft" - ? event.shiftKey - ? "back" - : "backward" - : event.shiftKey - ? "front" - : "forward", + direction: action.direction, }); - } else if (mod && key === "v" && clipboard.current) { - event.preventDefault(); - paste(); - } else if (mod && key === "d") { - event.preventDefault(); - paste(true); - } else if (mod && key === "g") { - event.preventDefault(); - if (event.shiftKey) ungroup(); + else if (action.type === "paste") paste(action.duplicate); + else if (action.type === "group") { + if (action.ungroup) ungroup(); else group(); - } else if (mod && key === "s") { - event.preventDefault(); - void persist(doc); - } else if (mod && key === "o") { - event.preventDefault(); - openFilePicker(); - } else if (event.key === "Delete" || event.key === "Backspace") { - event.preventDefault(); - removeSelected(); - } else if (event.shiftKey && event.code === "Digit1") { - event.preventDefault(); - fit(); - } else if (event.shiftKey && event.code === "Digit2") { - event.preventDefault(); - fit(true); - } else if (!mod && key === "t") addNode("text"); - else if (!mod && key === "f" && single) setPreview(single); - else if (!mod && key === "?") setHelp(true); - else if ( - ["ArrowLeft", "ArrowRight", "ArrowUp", "ArrowDown"].includes(event.key) - ) { - event.preventDefault(); - event.stopPropagation(); - const step = event.shiftKey ? 10 : 1; + } else if (action.type === "save") void persist(doc); + else if (action.type === "import") openFilePicker(); + else if (action.type === "delete") removeSelected(); + else if (action.type === "fit") fit(action.selection); + else if (action.type === "add-text") addNode("text"); + else if (action.type === "preview" && single) setPreview(single); + else if (action.type === "help") setHelp(true); + else if (action.type === "nudge") command({ type: "move-nodes", ids: selectedRef.current, - delta: { - x: - event.key === "ArrowLeft" - ? -step - : event.key === "ArrowRight" - ? step - : 0, - y: - event.key === "ArrowUp" - ? -step - : event.key === "ArrowDown" - ? step - : 0, - }, + delta: action.delta, }); - } }; useEffect(() => { const onPaste = (event: ClipboardEvent) => { diff --git a/src/canvas/NodeComposer.tsx b/src/canvas/NodeComposer.tsx index d7f451e4..e28f6c8b 100644 --- a/src/canvas/NodeComposer.tsx +++ b/src/canvas/NodeComposer.tsx @@ -12,7 +12,6 @@ import { ReferenceThumbnail } from "./ReferenceThumbnail"; import { changeGenerationSettings, generationBlocker, - generationDefaults, generationModel, generationReferenceIssue, generationReferenceRole, @@ -33,17 +32,13 @@ import { isMusicModel, hasMusicReference, MUSIC_SECONDS, - serviceAudioSettings, type ModelConfig, } from "../backend/rules.ts"; -import { - applyModelResolution, - resolveGenerationModel, -} from "./modelResolution.ts"; -import { copyGenerationSettings } from "./dataBoundary"; +import { resolveGenerationModel } from "./modelResolution.ts"; import { GenerationComposerShell } from "./GenerationComposerShell"; import { useI18n } from "../i18n/I18nProvider"; -import { generationPrompt } from "./generationReferenceContent.ts"; +import { getNodeGenerationSettings } from "./generationSettings.ts"; +export { getNodeGenerationSettings }; import { GenerationReferenceEditor, type GenerationReferenceHandle, @@ -77,69 +72,9 @@ function clockTime(seconds: number) { return `${Math.floor(whole / 60)}:${String(whole % 60).padStart(2, "0")}`; } -const LEGACY_MODELS = [ - "本地交互演示", - "本地音频工作流", - "配乐创意", - "环境音创意", -]; /** Built-in demo voice id; the visible label is localized separately. */ const DEMO_VOICE = "动漫海绵"; -/** Shared defaults let the workspace run the same parameters that the controls display. */ -export function getNodeGenerationSettings( - node: CanvasNode, - models?: readonly ModelConfig[], -): ComposerSettings { - const mode = - node.data.kind === "video" - ? "video" - : node.data.kind === "audio" - ? "audio" - : "image"; - const previous = node.data.settings - ? copyGenerationSettings(node.data.settings) - : undefined; - // Older saved projects may retain settings from a previous media kind. - // Model, resolution and duration belong to their original media mode. - const sameMode = previous?.mode === mode ? previous : undefined; - const defaults = generationDefaults(mode); - const legacyModel = LEGACY_MODELS.includes(sameMode?.model ?? ""); - const next: ComposerSettings = { - ...defaults, - ...sameMode, - mode, - model: sameMode?.model && !legacyModel ? sameMode.model : defaults.model, - aspectRatio: - previous?.aspectRatio === "auto" || - ratios.some((ratio) => ratio === previous?.aspectRatio) - ? previous!.aspectRatio - : defaults.aspectRatio, - resolution: - mode === "audio" - ? "auto" - : sameMode?.resolution && sameMode.resolution !== "1024" - ? sameMode.resolution - : defaults.resolution, - prompt: generationPrompt(node), - count: previous?.count ?? 1, - audioEnabled: sameMode?.audioEnabled ?? true, - voiceName: sameMode?.modelConfigId - ? sameMode.voiceName - : (sameMode?.voiceName ?? defaults.voiceName), - }; - // The creator only offers configured service models, so the displayed model - // has to be the resolved one: reading and running then agree, and reopening a - // node never drifts to whatever the catalog happens to define. - if (!models) return next; - const resolved = applyModelResolution( - next, - resolveGenerationModel(next, mode, models), - ); - const config = backendModel(resolved, models); - return config ? serviceAudioSettings(resolved, config) : resolved; -} - export function NodeComposer({ selectedNode, incomingNodes, diff --git a/src/canvas/generationSettings.ts b/src/canvas/generationSettings.ts new file mode 100644 index 00000000..5974ee44 --- /dev/null +++ b/src/canvas/generationSettings.ts @@ -0,0 +1,77 @@ +import { + backendModel, + serviceAudioSettings, + type ModelConfig, +} from "../backend/rules.ts"; +import { copyGenerationSettings } from "./dataBoundary.ts"; +import { + GENERATION_RATIOS, + generationDefaults, +} from "./generationCatalog.ts"; +import { generationPrompt } from "./generationReferenceContent.ts"; +import type { CanvasNode, GenerationSettings } from "./model.ts"; +import { + applyModelResolution, + resolveGenerationModel, +} from "./modelResolution.ts"; + +const LEGACY_MODELS = [ + "本地交互演示", + "本地音频工作流", + "配乐创意", + "环境音创意", +]; + +/** Shared defaults let the workspace run the same parameters that the controls display. */ +export function getNodeGenerationSettings( + node: CanvasNode, + models?: readonly ModelConfig[], +): GenerationSettings { + const mode = + node.data.kind === "video" + ? "video" + : node.data.kind === "audio" + ? "audio" + : "image"; + const previous = node.data.settings + ? copyGenerationSettings(node.data.settings) + : undefined; + // Older saved projects may retain settings from a previous media kind. + // Model, resolution and duration belong to their original media mode. + const sameMode = previous?.mode === mode ? previous : undefined; + const defaults = generationDefaults(mode); + const legacyModel = LEGACY_MODELS.includes(sameMode?.model ?? ""); + const next: GenerationSettings = { + ...defaults, + ...sameMode, + mode, + model: sameMode?.model && !legacyModel ? sameMode.model : defaults.model, + aspectRatio: + previous?.aspectRatio === "auto" || + GENERATION_RATIOS.some((ratio) => ratio === previous?.aspectRatio) + ? previous!.aspectRatio + : defaults.aspectRatio, + resolution: + mode === "audio" + ? "auto" + : sameMode?.resolution && sameMode.resolution !== "1024" + ? sameMode.resolution + : defaults.resolution, + prompt: generationPrompt(node), + count: previous?.count ?? 1, + audioEnabled: sameMode?.audioEnabled ?? true, + voiceName: sameMode?.modelConfigId + ? sameMode.voiceName + : (sameMode?.voiceName ?? defaults.voiceName), + }; + // The creator only offers configured service models, so the displayed model + // has to be the resolved one: reading and running then agree, and reopening a + // node never drifts to whatever the catalog happens to define. + if (!models) return next; + const resolved = applyModelResolution( + next, + resolveGenerationModel(next, mode, models), + ); + const config = backendModel(resolved, models); + return config ? serviceAudioSettings(resolved, config) : resolved; +} diff --git a/src/canvas/workspaceInteraction.ts b/src/canvas/workspaceInteraction.ts new file mode 100644 index 00000000..f70f6087 --- /dev/null +++ b/src/canvas/workspaceInteraction.ts @@ -0,0 +1,412 @@ +import type { GenerationRequest, ReferenceInput } from "../backend/generation.ts"; +import { + backendBlocker, + backendModel, + backendReferences, + backendVideoMode, + serviceVoice, + type ModelConfig, +} from "../backend/rules.ts"; +import { tNow } from "../i18n/messages.ts"; +import { assistantReferences } from "./assistantReferences.ts"; +import type { PortSide } from "./connectionPorts.ts"; +import { getNodeGenerationSettings } from "./generationSettings.ts"; +import { generationContent } from "./generationReferenceContent.ts"; +import type { ResultSource } from "./linkedResults.ts"; +import type { CanvasCommand, CanvasDocument, CanvasNode, Point } from "./model.ts"; +import { referenceSnapshot } from "./referenceEvidence.ts"; +import { agentGenerationBlocker, agentNodeSettings } from "./textGeneration.ts"; + +type FlowChange = { + type: string; + id?: string; + selected?: boolean; + position?: Point; + dimensions?: { width: number; height: number } | null; + setAttributes?: boolean | "width" | "height"; +}; + +/** React Flow select changes merge into the current ids. Existing order stays; new ids append. */ +export function mergeFlowSelection( + ids: readonly string[], + changes: readonly { id: string; selected: boolean }[], +): string[] { + const next = new Set(ids); + for (const change of changes) { + if (change.selected) next.add(change.id); + else next.delete(change.id); + } + return [...next]; +} + +/** Select-all addresses visible cards. Folded stack members stay behind their stack. */ +export function canvasSelectAllIds(nodes: readonly CanvasNode[]): string[] { + const hidden = new Set(); + for (const node of nodes) + if (node.type === "stack") + for (const id of node.data.stack?.nodeIds ?? []) hidden.add(id); + return nodes.filter((node) => !hidden.has(node.id)).map((node) => node.id); +} + +/** + * Position and resize changes become one canvas command. + * A cancelled gesture, or a drag still inside click slop, authors nothing. + * A resize owns the gesture, including the child-compensation position that arrives with it. + */ +export function authorFlowChanges( + changes: readonly FlowChange[], + gesture: { cancelled: boolean; dragging: boolean; moved: boolean }, +): CanvasCommand | null { + if (gesture.cancelled || (gesture.dragging && !gesture.moved)) return null; + const resize = changes.find( + (change) => + change.type === "dimensions" && + change.id && + change.dimensions && + change.setAttributes, + ); + if (resize?.id && resize.dimensions) { + const move = changes.find( + (change) => change.type === "position" && change.id === resize.id, + ); + return { + type: "resize-node", + id: resize.id, + width: resize.dimensions.width, + height: resize.dimensions.height, + ...(move?.position ? { position: move.position } : {}), + }; + } + const commands = changes.flatMap((change) => + change.type === "position" && change.id && change.position + ? [ + { + type: "update-node" as const, + id: change.id, + patch: { position: change.position }, + }, + ] + : [], + ); + return commands.length ? { type: "batch", commands } : null; +} + +/** A dropped connection places the new node so the drop point is its facing port. */ +export function placeCreatedNode( + size: { width: number; height: number }, + where: Point, + connected?: { direction: "source" | "target"; side?: PortSide }, + parent?: { id: string; origin: Point }, +): { position: Point; parentId?: string } { + let position = { + x: where.x - size.width / 2, + y: where.y - size.height / 2, + }; + if (connected) { + position = { + x: where.x - (connected.direction === "target" ? size.width : 0), + y: where.y - size.height / 2, + }; + if (connected.side === "left") + position = { x: where.x - size.width, y: where.y - size.height / 2 }; + if (connected.side === "top") + position = { x: where.x - size.width / 2, y: where.y - size.height }; + if (connected.side === "bottom") + position = { x: where.x - size.width / 2, y: where.y }; + } + if (parent) + position = { + x: position.x - parent.origin.x, + y: position.y - parent.origin.y, + }; + return { position, ...(parent ? { parentId: parent.id } : {}) }; +} + +/** A connection drag opens the creation menu only after it leaves the handle as an unfinished link. */ +export function connectionDragReleased(input: { + spacePressed: boolean; + cancelled: boolean; + moved: boolean; + start: Point | null; + point: Point | null; + isValid: boolean; + hasEndpoints: boolean; +}): boolean { + if (input.spacePressed || input.cancelled) return false; + if ( + !input.point || + !input.start || + (!input.moved && + Math.hypot(input.point.x - input.start.x, input.point.y - input.start.y) <= + 4) + ) + return false; + return !input.isValid && input.hasEndpoints; +} + +/** The menu anchors to the pane or an unlocked group. A drop on any other node is ignored. */ +export function droppedConnectionTarget(input: { + toNodeId?: string | null; + groupId?: string | null; + onPane: boolean; + fromNodeId: string; + direction: "source" | "target"; +}): { id: string; direction: "source" | "target"; parentId?: string } | null { + if (input.toNodeId && input.toNodeId !== input.groupId) return null; + if (!input.groupId && !input.onPane) return null; + return { + id: input.fromNodeId, + direction: input.direction, + ...(input.groupId ? { parentId: input.groupId } : {}), + }; +} + +export type CanvasKeyAction = + | { type: "edit-text" } + | { type: "history"; forward: boolean } + | { type: "select-all" } + | { type: "copy" } + | { type: "cut" } + | { type: "reorder"; direction: "front" | "back" | "forward" | "backward" } + | { type: "paste"; duplicate: boolean } + | { type: "group"; ungroup: boolean } + | { type: "save" } + | { type: "import" } + | { type: "delete" } + | { type: "fit"; selection: boolean } + | { type: "add-text" } + | { type: "preview" } + | { type: "help" } + | { type: "nudge"; delta: Point }; + +/** Keyboard actions once the workspace has decided the event belongs to the canvas. */ +export function canvasKeyAction(input: { + key: string; + code: string; + mod: boolean; + shift: boolean; + hasClipboard: boolean; + editText: boolean; + hasSingle: boolean; +}): CanvasKeyAction | null { + const key = input.key.toLowerCase(); + if (!input.mod && input.key === "Enter" && input.editText) + return { type: "edit-text" }; + if (input.mod && key === "z") return { type: "history", forward: input.shift }; + if (input.mod && key === "y") return { type: "history", forward: true }; + if (input.mod && key === "a") return { type: "select-all" }; + if (input.mod && key === "c") return { type: "copy" }; + if (input.mod && key === "x") return { type: "cut" }; + if ( + input.mod && + (input.code === "BracketLeft" || input.code === "BracketRight") + ) + return { + type: "reorder", + direction: + input.code === "BracketLeft" + ? input.shift + ? "back" + : "backward" + : input.shift + ? "front" + : "forward", + }; + if (input.mod && key === "v" && input.hasClipboard) + return { type: "paste", duplicate: false }; + if (input.mod && key === "d") return { type: "paste", duplicate: true }; + if (input.mod && key === "g") return { type: "group", ungroup: input.shift }; + if (input.mod && key === "s") return { type: "save" }; + if (input.mod && key === "o") return { type: "import" }; + if (input.key === "Delete" || input.key === "Backspace") + return { type: "delete" }; + if (input.shift && input.code === "Digit1") + return { type: "fit", selection: false }; + if (input.shift && input.code === "Digit2") + return { type: "fit", selection: true }; + if (!input.mod && key === "t") return { type: "add-text" }; + if (!input.mod && key === "f" && input.hasSingle) return { type: "preview" }; + if (!input.mod && input.key === "?") return { type: "help" }; + if ( + ["ArrowLeft", "ArrowRight", "ArrowUp", "ArrowDown"].includes(input.key) + ) { + const step = input.shift ? 10 : 1; + return { + type: "nudge", + delta: { + x: + input.key === "ArrowLeft" + ? -step + : input.key === "ArrowRight" + ? step + : 0, + y: + input.key === "ArrowUp" + ? -step + : input.key === "ArrowDown" + ? step + : 0, + }, + }; + } + return null; +} + +type SubmissionAsset = { id: string; kind?: string | null }; + +/** Pinned prompt tokens submit the saved asset, never a canvas node invented from the label. */ +export function pinnedSubmissionInputs( + node: CanvasNode, + assets: readonly SubmissionAsset[], +): ReferenceInput[] { + return (node.data.promptContent ?? []).flatMap((part) => { + if (part.type !== "reference" || part.binding !== "asset") return []; + const asset = assets.find((item) => item.id === part.assetId); + if ( + !asset?.kind || + !["text", "image", "video", "audio"].includes(asset.kind) + ) + throw new Error(tNow("canvas.workspace.error.pinnedMetadataUnloaded")); + return [ + { + binding: "asset" as const, + assetId: asset.id, + fingerprint: part.fingerprint, + referenceId: part.referenceId, + kind: asset.kind as "image" | "video" | "audio" | "text", + title: part.name, + }, + ]; + }); +} + +export type ServiceGenerationPlan = + | { status: "blocked"; message: string | null } + | { + status: "ready"; + request: GenerationRequest; + origin: ResultSource; + count: number; + label: string; + }; + +/** The request the workspace submits for an image, video, or audio node. */ +export function serviceGenerationPlan(input: { + document: Pick; + node: CanvasNode; + incoming: readonly CanvasNode[]; + models: readonly ModelConfig[]; + assets: readonly SubmissionAsset[]; +}): ServiceGenerationPlan { + const settings = getNodeGenerationSettings(input.node, input.models); + const config = backendModel(settings, input.models); + const blocked = backendBlocker(settings, input.incoming, input.models); + if (blocked || !config) + return { status: "blocked", message: blocked }; + const references = backendReferences(settings, input.incoming, config); + const request: GenerationRequest = { + documentId: input.document.id, + documentTitle: input.document.title, + nodeId: input.node.id, + promptContent: generationContent(input.node, input.incoming), + config, + kind: settings.mode, + prompt: settings.prompt, + title: input.node.data.title, + videoMode: backendVideoMode(settings, references, config), + durationSeconds: config.inputRules.videoCapabilities + ? settings.durationSeconds + : undefined, + aspectRatio: settings.aspectRatio, + resolution: settings.resolution, + voice: serviceVoice(settings), + references: [ + ...assistantReferences(input.incoming).map((item) => ({ + ...item, + contextOnly: !references.some( + ({ node: source }) => + source.id === item.nodeId && + source.data.kind === item.kind && + (item.kind === "text" || source.data.src === item.src), + ), + })), + ...pinnedSubmissionInputs(input.node, input.assets).map((item) => ({ + ...item, + contextOnly: + settings.mode === "audio" || + (settings.mode === "video" && + settings.videoMode === "first-last" && + item.kind === "image" && + references.some((ref) => ref.role === "firstFrame")), + })), + ], + }; + return { + status: "ready", + request, + origin: { + documentId: input.document.id, + nodeId: input.node.id, + kind: input.node.type, + src: input.node.data.src, + referenceInputs: referenceSnapshot( + references.map( + (item) => + input.document.nodes.find((node) => node.id === item.node.id) ?? + item.node, + ), + ), + }, + count: settings.mode === "audio" ? 1 : (settings.count ?? 1), + label: settings.prompt || input.node.data.title, + }; +} + +export type TextGenerationPlan = + | { status: "blocked"; message: string | null } + | { status: "ready"; request: GenerationRequest; origin: ResultSource; label: string }; + +/** The request the workspace submits for a text node. */ +export function textGenerationPlan(input: { + document: Pick; + node: CanvasNode; + incoming: readonly CanvasNode[]; + models: readonly ModelConfig[]; + assets: readonly SubmissionAsset[]; +}): TextGenerationPlan { + const settings = agentNodeSettings(input.node); + const agents = input.models.filter((model) => model.kind === "agent"); + const config = agents.find( + (model) => model.id === (settings.modelConfigId ?? agents[0]?.id), + ); + const blocker = agentGenerationBlocker(input.node, input.incoming, config); + if (blocker || !config) return { status: "blocked", message: blocker }; + return { + status: "ready", + label: input.node.data.title, + request: { + documentId: input.document.id, + documentTitle: input.document.title, + nodeId: input.node.id, + promptContent: generationContent(input.node, input.incoming), + config, + kind: "text", + prompt: input.node.data.prompt, + title: input.node.data.title, + videoMode: "first_frame", + thinking: settings.thinking, + maxTokens: settings.maxTokens, + references: [ + ...assistantReferences(input.incoming), + ...pinnedSubmissionInputs(input.node, input.assets), + ], + }, + origin: { + documentId: input.document.id, + nodeId: input.node.id, + kind: "text", + text: input.node.data.text, + referenceInputs: referenceSnapshot([...input.incoming]), + }, + }; +} diff --git a/tests/canvasInteraction.test.ts b/tests/canvasInteraction.test.ts new file mode 100644 index 00000000..4e8ed77f --- /dev/null +++ b/tests/canvasInteraction.test.ts @@ -0,0 +1,503 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { setActiveLocale } from "../src/i18n/messages.ts"; +import { + buildCanvasArchive, + readCanvasArchive, +} from "../src/canvas/canvasArchive.ts"; +import { generationDefaults } from "../src/canvas/generationCatalog.ts"; +import { + beginTransaction, + commitTransaction, + createHistory, + executeCommand, + undo, + updateTransaction, +} from "../src/canvas/history.ts"; +import { + applyCommand, + createDocument, + createNode, +} from "../src/canvas/model.ts"; +import type { ModelConfig } from "../src/backend/rules.ts"; +import { + authorFlowChanges, + canvasKeyAction, + canvasSelectAllIds, + connectionDragReleased, + droppedConnectionTarget, + mergeFlowSelection, + placeCreatedNode, + serviceGenerationPlan, + textGenerationPlan, +} from "../src/canvas/workspaceInteraction.ts"; + +setActiveLocale("zh-CN"); + +const key = ( + patch: Partial[0]> = {}, +) => + canvasKeyAction({ + key: "", + code: "", + mod: false, + shift: false, + hasClipboard: false, + editText: false, + hasSingle: false, + ...patch, + }); + +const model = ( + kind: ModelConfig["kind"], + rules: Partial = {}, +): ModelConfig => ({ + id: `${kind}-model`, + kind, + url: "http://127.0.0.1:8010/v1", + modelId: "fixture/model", + connectionType: "self_hosted", + protocol: kind === "agent" ? "openai" : "vllm_omni", + authMode: "none", + inputRules: { + profile: "fixture", + allowedKinds: ["text", "image", "video", "audio"], + maxImages: 4, + maxVideos: 1, + maxAudios: 1, + videoModes: ["first_frame", "reference", "continuation"], + ...rules, + }, +}); + +const ready = ( + kind: "image" | "video" | "audio" | "text", + data: Record = {}, +) => + createNode(kind, { x: 0, y: 0 }, { + ...(kind === "text" + ? { text: "旁白" } + : { src: `data:${kind}/x;base64,AA==`, assetState: "ready" as const }), + ...data, + }); + +test("selection merges flow changes in place and select-all skips folded stack members", () => { + assert.deepEqual( + mergeFlowSelection( + ["a", "b"], + [ + { id: "c", selected: true }, + { id: "a", selected: false }, + { id: "d", selected: true }, + { id: "d", selected: false }, + ], + ), + ["b", "c"], + ); + const member = createNode("image"); + const loose = createNode("text"); + const stack = createNode("stack", undefined, { + stack: { nodeIds: [member.id], activeIndex: 0 }, + }); + assert.deepEqual(canvasSelectAllIds([member, loose, stack]), [ + loose.id, + stack.id, + ]); +}); + +test("a drag inside click slop or after cancel authors nothing; a real drag is one undo step", () => { + const node = createNode("image", { x: 0, y: 0 }); + const move = [ + { type: "position", id: node.id, position: { x: 40, y: 12 } }, + ]; + assert.equal( + authorFlowChanges(move, { cancelled: false, dragging: true, moved: false }), + null, + ); + assert.equal( + authorFlowChanges(move, { cancelled: true, dragging: true, moved: true }), + null, + ); + const authored = authorFlowChanges(move, { + cancelled: false, + dragging: true, + moved: true, + }); + assert.equal(authored?.type, "batch"); + let history = beginTransaction( + createHistory({ ...createDocument(), nodes: [node] }), + ); + history = updateTransaction( + history, + applyCommand(history.present, authored!), + ); + history = commitTransaction(history); + assert.deepEqual(history.present.nodes[0]!.position, { x: 40, y: 12 }); + assert.equal(history.past.length, 1); + assert.deepEqual(undo(history).present.nodes[0]!.position, { x: 0, y: 0 }); +}); + +test("a resize authors one command and keeps the compensating position", () => { + const node = createNode("group", { x: 10, y: 20 }); + const authored = authorFlowChanges( + [ + { + type: "dimensions", + id: node.id, + dimensions: { width: 800, height: 500 }, + setAttributes: true, + }, + { type: "position", id: node.id, position: { x: 8, y: 16 } }, + { type: "position", id: "child", position: { x: 1, y: 1 } }, + ], + { cancelled: false, dragging: false, moved: false }, + ); + assert.deepEqual(authored, { + type: "resize-node", + id: node.id, + width: 800, + height: 500, + position: { x: 8, y: 16 }, + }); +}); + +test("connection drops ignore clicks, cancelled drags, and landings on other nodes", () => { + const start = { x: 10, y: 10 }; + const base = { + spacePressed: false, + cancelled: false, + moved: false, + start, + point: { x: 14, y: 10 }, + isValid: false, + hasEndpoints: true, + }; + assert.equal(connectionDragReleased(base), false); + assert.equal( + connectionDragReleased({ ...base, point: { x: 15, y: 10 } }), + true, + ); + assert.equal( + connectionDragReleased({ ...base, moved: true, point: start }), + true, + ); + assert.equal(connectionDragReleased({ ...base, moved: true, spacePressed: true }), false); + assert.equal(connectionDragReleased({ ...base, moved: true, cancelled: true }), false); + assert.equal(connectionDragReleased({ ...base, moved: true, isValid: true }), false); + assert.equal( + connectionDragReleased({ ...base, moved: true, hasEndpoints: false }), + false, + ); + assert.equal( + droppedConnectionTarget({ + toNodeId: "image", + groupId: null, + onPane: false, + fromNodeId: "source", + direction: "source", + }), + null, + ); + assert.deepEqual( + droppedConnectionTarget({ + toNodeId: null, + groupId: null, + onPane: true, + fromNodeId: "source", + direction: "source", + }), + { id: "source", direction: "source" }, + ); + assert.deepEqual( + droppedConnectionTarget({ + toNodeId: "group", + groupId: "group", + onPane: false, + fromNodeId: "source", + direction: "target", + }), + { id: "source", direction: "target", parentId: "group" }, + ); +}); + +test("a connected node is placed on the dropped port and then inside its group", () => { + const size = { width: 300, height: 200 }; + const where = { x: 100, y: 80 }; + assert.deepEqual(placeCreatedNode(size, where).position, { x: -50, y: -20 }); + assert.deepEqual( + placeCreatedNode(size, where, { direction: "source", side: "right" }).position, + { x: 100, y: -20 }, + ); + assert.deepEqual( + placeCreatedNode(size, where, { direction: "target", side: "left" }).position, + { x: -200, y: -20 }, + ); + assert.deepEqual( + placeCreatedNode(size, where, { direction: "source", side: "top" }).position, + { x: -50, y: -120 }, + ); + const nested = placeCreatedNode(size, where, { direction: "source", side: "bottom" }, { + id: "group", + origin: { x: 30, y: 10 }, + }); + assert.deepEqual(nested, { + position: { x: -80, y: 70 }, + parentId: "group", + }); +}); + +test("canvas shortcuts map to history, selection, import, and nudges", () => { + assert.deepEqual(key({ key: "z", mod: true }), { type: "history", forward: false }); + assert.deepEqual(key({ key: "z", mod: true, shift: true }), { + type: "history", + forward: true, + }); + assert.deepEqual(key({ key: "y", mod: true }), { type: "history", forward: true }); + assert.equal(key({ key: "a", mod: true })?.type, "select-all"); + assert.equal(key({ key: "o", mod: true })?.type, "import"); + assert.equal(key({ key: "s", mod: true })?.type, "save"); + assert.equal(key({ key: "v", mod: true })?.type, undefined); + assert.deepEqual(key({ key: "v", mod: true, hasClipboard: true }), { + type: "paste", + duplicate: false, + }); + assert.deepEqual(key({ key: "d", mod: true }), { type: "paste", duplicate: true }); + assert.deepEqual(key({ key: "g", code: "KeyG", mod: true, shift: true }), { + type: "group", + ungroup: true, + }); + assert.deepEqual(key({ code: "BracketLeft", mod: true }), { + type: "reorder", + direction: "backward", + }); + assert.deepEqual(key({ code: "BracketRight", mod: true, shift: true }), { + type: "reorder", + direction: "front", + }); + assert.deepEqual(key({ key: "ArrowLeft", shift: true }), { + type: "nudge", + delta: { x: -10, y: 0 }, + }); + assert.deepEqual(key({ key: "ArrowDown" }), { + type: "nudge", + delta: { x: 0, y: 1 }, + }); + assert.equal(key({ key: "Enter", editText: true })?.type, "edit-text"); + assert.equal(key({ key: "Enter" }), null); + assert.equal(key({ key: "f", hasSingle: true })?.type, "preview"); + assert.equal(key({ key: "t" })?.type, "add-text"); + assert.equal(key({ key: "Delete" })?.type, "delete"); + + const node = createNode("image", { x: 5, y: 6 }); + const nudge = key({ key: "ArrowRight" }); + assert.equal(nudge?.type, "nudge"); + const history = executeCommand( + createHistory({ ...createDocument(), nodes: [node] }), + { type: "move-nodes", ids: [node.id], delta: nudge!.type === "nudge" ? nudge.delta : { x: 0, y: 0 } }, + ); + assert.deepEqual(history.present.nodes[0]!.position, { x: 6, y: 6 }); + assert.deepEqual(undo(history).present.nodes[0]!.position, { x: 5, y: 6 }); +}); + +test("generation submission marks unused inputs context-only and keeps one audio run", () => { + const video = model("video"); + const first = ready("image"); + const extra = ready("image"); + const caption = ready("text"); + const node = createNode("video", { x: 20, y: 30 }, { + title: "镜头", + prompt: "推进", + promptContent: [ + { type: "text", text: "推进" }, + { + type: "reference", + binding: "asset", + assetId: "pin", + name: "定妆", + fingerprint: "abc", + }, + ], + settings: { + ...generationDefaults("video"), + modelConfigId: video.id, + prompt: "推进", + videoMode: "first-last", + firstFrameNodeId: first.id, + resolution: "auto", + aspectRatio: "auto", + count: 2, + voiceName: "voice_1", + }, + }); + const document = { + ...createDocument("短片"), + nodes: [first, extra, caption, node], + }; + const plan = serviceGenerationPlan({ + document, + node, + incoming: [first, extra, caption], + models: [video], + assets: [{ id: "pin", kind: "image" }], + }); + assert.equal(plan.status, "ready"); + if (plan.status !== "ready") return; + assert.equal(plan.count, 2); + assert.equal(plan.request.videoMode, "first_frame"); + assert.equal(plan.request.voice, "voice_1"); + assert.equal(plan.request.durationSeconds, undefined); + assert.equal(plan.label, "推进"); + assert.deepEqual( + plan.request.references.map((item) => [item.title, item.contextOnly]), + [ + [first.data.title, false], + [extra.data.title, true], + [caption.data.title, false], + ["定妆", true], + ], + ); + assert.deepEqual(Object.keys(plan.origin.referenceInputs ?? {}).sort(), [ + caption.id, + first.id, + ].sort()); + + const speech = model("audio", { allowedKinds: ["text"], videoModes: [] }); + const voice = createNode("audio", undefined, { + title: "旁白", + prompt: "你好", + settings: { + ...generationDefaults("audio"), + modelConfigId: speech.id, + prompt: "你好", + count: 4, + audioMode: "tts", + voiceName: "voice_1", + }, + }); + const audioPlan = serviceGenerationPlan({ + document, + node: voice, + incoming: [], + models: [speech], + assets: [], + }); + assert.equal(audioPlan.status, "ready"); + if (audioPlan.status !== "ready") return; + assert.equal(audioPlan.count, 1); + assert.equal(audioPlan.request.voice, "voice_1"); + assert.equal(audioPlan.request.kind, "audio"); + + const blocked = serviceGenerationPlan({ + document, + node: createNode("image", undefined, { + settings: { + ...generationDefaults("image"), + modelConfigId: model("image").id, + resolution: "auto", + }, + }), + incoming: [], + models: [model("image")], + assets: [], + }); + assert.equal(blocked.status, "blocked"); + if (blocked.status === "blocked") assert.match(blocked.message ?? "", /生成要求/); + + assert.throws( + () => + serviceGenerationPlan({ + document, + node, + incoming: [first], + models: [video], + assets: [], + }), + /固定参考元数据/, + ); +}); + +test("a text node submits its agent settings and refuses to run without one", () => { + const agent = model("agent", { allowedKinds: ["text"], maxImages: 0, videoModes: [] }); + const node = createNode("text", undefined, { + title: "大纲", + prompt: "写一幕", + text: "旧稿", + agentSettings: { modelConfigId: agent.id, thinking: "low", maxTokens: 1000 }, + }); + const document = { ...createDocument("短片"), nodes: [node] }; + const plan = textGenerationPlan({ + document, + node, + incoming: [], + models: [agent], + assets: [], + }); + assert.equal(plan.status, "ready"); + if (plan.status !== "ready") return; + assert.equal(plan.request.kind, "text"); + assert.equal(plan.request.videoMode, "first_frame"); + assert.equal(plan.request.thinking, "low"); + assert.equal(plan.request.maxTokens, 1000); + assert.equal(plan.request.prompt, "写一幕"); + assert.equal(plan.origin.text, "旧稿"); + assert.equal(plan.label, "大纲"); + + const missing = textGenerationPlan({ + document, + node, + incoming: [], + models: [], + assets: [], + }); + assert.equal(missing.status, "blocked"); + if (missing.status === "blocked") assert.match(missing.message ?? "", /Agent/); +}); + +test("placed nodes, connections, and history survive a canvas archive round trip", async () => { + const source = createNode("text", { x: 0, y: 0 }, { text: "第一场", title: "场次" }); + const target = createNode("image", { x: 0, y: 0 }, { + title: "画面", + prompt: "雨", + settings: { ...generationDefaults("image"), prompt: "雨" }, + }); + const placed = placeCreatedNode(target, { x: 480, y: 200 }, { + direction: "source", + side: "right", + }); + target.position = placed.position; + assert.deepEqual(target.position, { x: 480, y: 50 }); + let document = { + ...createDocument("导出"), + nodes: [source, target], + }; + document = applyCommand(document, { + type: "add-edge", + edge: { + id: "edge-1", + source: source.id, + target: target.id, + }, + }); + const nudge = key({ key: "ArrowUp", shift: true }); + assert.equal(nudge?.type, "nudge"); + document = applyCommand(document, { + type: "move-nodes", + ids: [target.id], + delta: nudge!.type === "nudge" ? nudge.delta : { x: 0, y: 0 }, + }); + const moved = document.nodes.find((node) => node.id === target.id); + assert.deepEqual(moved?.position, { x: 480, y: 40 }); + const archive = await buildCanvasArchive(document, { + now: new Date("2026-11-06T00:00:00.000Z"), + }); + const imported = await readCanvasArchive(archive.blob); + const image = imported.document.nodes.find((node) => node.id === target.id); + assert.equal(imported.document.title, "导出"); + assert.deepEqual(image?.position, { x: 480, y: 40 }); + assert.equal(image?.data.prompt, "雨"); + assert.equal(image?.data.settings?.prompt, "雨"); + assert.deepEqual( + imported.document.edges.map((edge) => [edge.source, edge.target]), + [[source.id, target.id]], + ); +}); diff --git a/tests/rulesDerivation.test.ts b/tests/rulesDerivation.test.ts new file mode 100644 index 00000000..15d8a65e --- /dev/null +++ b/tests/rulesDerivation.test.ts @@ -0,0 +1,321 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { setActiveLocale } from "../src/i18n/messages.ts"; +import { generationDefaults } from "../src/canvas/generationCatalog.ts"; +import { createNode } from "../src/canvas/model.ts"; +import type { CanvasNode, GenerationSettings } from "../src/canvas/model.ts"; +import { + MUSIC_SECONDS, + backendAspectRatios, + backendBlocker, + backendModel, + backendReferenceIssue, + backendVideoMode, + isMusicModel, + isScoreModel, + modelLabel, + referenceBudgetIssue, + serviceAudioSettings, + serviceGenerationSettings, + serviceVoice, + type ModelConfig, +} from "../src/backend/rules.ts"; + +setActiveLocale("zh-CN"); + +const model = ( + kind: ModelConfig["kind"], + patch: Partial = {}, + rules: Partial = {}, +): ModelConfig => { + const { inputRules: rulePatch, id, ...rest } = patch; + return { + id: id ?? `${kind}-config`, + kind, + url: "http://127.0.0.1:8010/v1", + modelId: "org/fixture-model", + connectionType: "self_hosted", + protocol: "openai", + authMode: "none", + ...rest, + inputRules: { + profile: "fixture", + allowedKinds: ["text", "image", "video", "audio"], + maxImages: 2, + maxVideos: 1, + maxAudios: 1, + videoModes: ["first_frame", "reference"], + ...rules, + ...rulePatch, + }, + }; +}; + +const media = (kind: "image" | "video" | "audio" | "text"): CanvasNode => + createNode( + kind, + { x: 0, y: 0 }, + kind === "text" + ? { text: "旁白" } + : { src: `data:${kind}/x;base64,AA==`, assetState: "ready" }, + ); + +const settings = ( + mode: GenerationSettings["mode"], + patch: Partial = {}, +): GenerationSettings => ({ + ...generationDefaults(mode), + modelConfigId: `${mode}-config`, + prompt: "灯塔", + resolution: "auto", + aspectRatio: "auto", + ...patch, +}); + +test("model identity, voice ids, and audio role follow the published configuration", () => { + const music = model("audio", { + id: "music", + modelId: "m-a-p/YuE2-3B", + protocol: "vllm_omni_music", + }); + const speech = model("audio", { id: "speech", modelId: "tts" }); + assert.equal(modelLabel(music), "YuE2-3B"); + assert.equal(modelLabel({ modelId: "bare" }), "bare"); + assert.equal(modelLabel({ modelId: "org/" }), "org/"); + assert.equal(backendModel(settings("audio", { modelConfigId: "" }), [music]), undefined); + assert.equal(backendModel(settings("audio", { modelConfigId: "missing" }), [music]), undefined); + assert.equal(backendModel(settings("audio", { modelConfigId: "music" }), [speech, music]), music); + assert.equal(serviceVoice({ voiceName: "Voice.1" }), "Voice.1"); + assert.equal(serviceVoice({ voiceName: "a".repeat(64) }), "a".repeat(64)); + assert.equal(serviceVoice({ voiceName: "a".repeat(65) }), undefined); + assert.equal(serviceVoice({ voiceName: "动漫海绵" }), undefined); + assert.equal(serviceVoice({ voiceName: "has space" }), undefined); + assert.equal(serviceVoice({}), undefined); + + const image = settings("image"); + assert.equal(serviceAudioSettings(image, music), image); + const short = settings("audio", { modelConfigId: "music", durationSeconds: 5 }); + assert.equal(serviceAudioSettings(short, music).audioCreationType, "music"); + assert.equal(serviceAudioSettings(short, music).durationSeconds, MUSIC_SECONDS.initial); + const kept = settings("audio", { modelConfigId: "music", durationSeconds: 40 }); + assert.equal(serviceAudioSettings(kept, music).durationSeconds, 40); + assert.equal(serviceAudioSettings(short, speech).audioCreationType, "audio"); + assert.equal(serviceAudioSettings(short, speech).durationSeconds, 5); + assert.equal(isMusicModel(music), true); + assert.equal(isMusicModel(speech), false); + assert.equal(isScoreModel({ kind: "audio", protocol: "vllm_omni_score" }), true); + assert.equal(isScoreModel(music), false); +}); + +test("changing models clears choices the new adapter does not publish", () => { + const image = model("image", { id: "image" }, { + imageOutputs: [{ aspectRatio: "1:1", size: "1024x1024" }], + videoModes: [], + allowedKinds: ["text"], + maxImages: 0, + }); + const changed = serviceGenerationSettings( + settings("image", { + modelConfigId: image.id, + imagePreset: "product-views", + voiceName: "音色", + aspectRatio: "16:9", + resolution: "2K", + }), + image, + [], + true, + ); + assert.equal(changed.imagePreset, "none"); + assert.equal(changed.voiceName, undefined); + assert.equal(changed.aspectRatio, "auto"); + assert.equal(changed.resolution, "auto"); + const kept = serviceGenerationSettings( + settings("image", { modelConfigId: image.id, voiceName: "Voice.1" }), + image, + [], + true, + ); + assert.equal(kept.voiceName, "Voice.1"); +}); + +test("reference checks reject empty, missing, and frame-incompatible sources", () => { + const video = model("video", { id: "video" }); + const videoSettings = settings("video", { modelConfigId: video.id }); + const empty = createNode("text"); + assert.match(backendReferenceIssue(empty, videoSettings, video)!, /文本节点为空/); + const foreign = model("video", { id: "video" }, { allowedKinds: ["image"] }); + assert.match( + backendReferenceIssue(media("text"), videoSettings, foreign)!, + /不接收文本/, + ); + const lost = createNode("image", undefined, { src: "data:image/png;base64,AA==", assetState: "missing" }); + assert.match(backendReferenceIssue(lost, videoSettings, video)!, /丢失/); + assert.match(backendReferenceIssue(createNode("group"), videoSettings, video)!, /不接收此类/); + const frames = settings("video", { modelConfigId: video.id, videoMode: "first-last" }); + assert.match(backendReferenceIssue(media("video"), frames, video)!, /视频或音频/); + assert.match(backendReferenceIssue(media("audio"), frames, video)!, /视频或音频/); +}); + +test("derived video mode and ratios follow the adapter, not the requested label", () => { + const referenceOnly = model("video", { id: "video" }, { videoModes: ["reference"] }); + const requested = settings("video", { modelConfigId: referenceOnly.id, videoMode: "reference" }); + assert.equal(backendVideoMode(requested, [], referenceOnly), "reference"); + assert.equal( + backendVideoMode( + settings("video", { videoMode: "continuation" }), + [], + referenceOnly, + ), + "continuation", + ); + assert.equal( + backendVideoMode(settings("video", { videoMode: "first-last" }), [], referenceOnly), + "first_frame", + ); + const volcengine = model("video", { + id: "video", + protocol: "volcengine", + inputRules: { + profile: "volcengine", + allowedKinds: ["text", "image"], + maxImages: 1, + maxVideos: 0, + maxAudios: 0, + videoModes: ["reference"], + }, + }); + assert.deepEqual( + backendAspectRatios(settings("video", { videoMode: "continuation" }), volcengine), + ["auto"], + ); + assert.ok( + backendAspectRatios(settings("video", { videoMode: "reference" }), volcengine).includes( + "16:9", + ), + ); +}); + +test("submission blockers name the rule that fails before a request is sent", () => { + const image = model("image", { id: "image" }, { videoModes: [] }); + const video = model("video", { id: "video" }); + assert.match( + backendBlocker(settings("image", { modelConfigId: video.id }), [], [video])!, + /类型与节点不符/, + ); + assert.match( + backendBlocker( + settings("image", { modelConfigId: image.id, imagePreset: "product-views" }), + [], + [image], + )!, + /三视图/, + ); + const music = model("audio", { + id: "music", + protocol: "vllm_omni_music", + inputRules: { + profile: "music", + allowedKinds: ["text"], + maxImages: 0, + maxVideos: 0, + maxAudios: 0, + videoModes: [], + }, + }); + const composing = serviceGenerationSettings( + settings("audio", { modelConfigId: music.id, prompt: "钢琴" }), + music, + [], + true, + ); + assert.match( + backendBlocker({ ...composing, lyrics: "词".repeat(5001) }, [], [music])!, + /5000/, + ); + const speech = model("audio", { id: "speech" }); + assert.match( + backendBlocker( + settings("audio", { modelConfigId: speech.id, audioCreationType: "music" }), + [], + [speech], + )!, + /文本朗读/, + ); + assert.match( + backendBlocker( + settings("audio", { + modelConfigId: speech.id, + audioMode: "tts", + voiceName: "bad voice", + }), + [], + [speech], + )!, + /音色标识/, + ); + const first = media("image"); + assert.match( + backendBlocker( + settings("video", { + modelConfigId: video.id, + videoMode: "first-last", + firstFrameNodeId: first.id, + }), + [], + [video], + )!, + /首帧素材/, + ); +}); + +test("reference budgets count each source once and skip continuation clips", () => { + const image = model("image", { id: "image" }, { + imageBytes: 20, + requestBytes: 12, + maxImages: 4, + videoModes: [], + }); + const photo = createNode("image", undefined, { src: "picture", byteSize: 8 }); + assert.equal(referenceBudgetIssue([photo, photo], image), null); + const other = createNode("image", undefined, { src: "other", byteSize: 8 }); + assert.match(referenceBudgetIssue([photo, other], image)!, /预算/); + photo.data.byteSize = 1; + other.data.byteSize = 1; + const limited = model("image", { id: "limited" }, { + imageBytes: 20, + requestBytes: 12, + maxImages: 1, + videoModes: [], + }); + assert.match(referenceBudgetIssue([photo, other], limited)!, /数量/); + + const agent = model("agent", { id: "agent" }, { maxImages: 0, maxVideos: 0 }); + const clip = createNode("video", undefined, { src: "clip", byteSize: 4 }); + assert.equal(referenceBudgetIssue([clip], agent), null); + assert.match(referenceBudgetIssue([photo], agent)!, /数量/); + + const video = model("video", { id: "video" }, { + videoBytes: 1, + videoModes: ["continuation"], + maxVideos: 1, + }); + const previous = createNode("video", undefined, { + src: "data:video/mp4;base64,AA==", + assetState: "ready", + byteSize: 20, + }); + assert.match(referenceBudgetIssue([previous], video)!, /大小限制/); + assert.equal( + referenceBudgetIssue( + [previous], + video, + settings("video", { modelConfigId: video.id, videoMode: "continuation" }), + ), + null, + ); + + const novel = createNode("text", undefined, { text: "字".repeat(70_000) }); + const textModel = model("image", { id: "image" }, { maxImages: 0 }); + assert.match(referenceBudgetIssue([novel], textModel)!, /大小限制/); +});