From c0596baad7ea884201bc2dbdbfc7008e0176d04a Mon Sep 17 00:00:00 2001 From: YAN YIXIN Date: Thu, 5 Nov 2026 21:04:18 +0800 Subject: [PATCH] [bugfix] canvas: fill the last frame for models that declare it Connecting two images to a first-last video node skipped the last-frame slot for every configured service model, so a MiniMax V2 model that declares frameRoles=[first_frame, last_frame] only ever received the first image; the second stayed a context-only reference and was never sent. The command layer now takes optional frame capabilities. The workspace derives them from the configured model's inputRules, so the last slot is filled in edge order only when the model accepts it. Without the capabilities (legacy initialisation, other callers) a configured model is still treated as first-frame only. Fixes #83 Co-Authored-By: Claude Opus 5.5 --- src/canvas/CanvasWorkspace.tsx | 9 +++- src/canvas/frameReferences.ts | 42 +++++++++++++++- src/canvas/history.ts | 4 +- src/canvas/model.ts | 18 +++++-- tests/frameReferences.test.ts | 92 +++++++++++++++++++++++++++++++++- 5 files changed, 156 insertions(+), 9 deletions(-) diff --git a/src/canvas/CanvasWorkspace.tsx b/src/canvas/CanvasWorkspace.tsx index b0f2aa962..df0002473 100644 --- a/src/canvas/CanvasWorkspace.tsx +++ b/src/canvas/CanvasWorkspace.tsx @@ -164,6 +164,7 @@ import { referenceContentCommand } from "./generationReferenceEdit.ts"; import type { GenerationReferenceControls } from "./GenerationReferenceEditor"; import { absolutePosition, arrangeNodes, boundsOfNodes } from "./geometry"; import { lockedNodeIds } from "./hierarchy"; +import { modelFrameCapabilities } from "./frameReferences"; import { beginTransaction, commitTransaction, @@ -905,8 +906,14 @@ function Workspace() { if (actionSession && !actionNode) setActionSession(null); }, [actionSession, actionNode]); useEffect(() => () => quickAction.current?.abort(), [doc.id]); + const frameCapabilities = useRef(modelFrameCapabilities([])); + frameCapabilities.current = useMemo( + () => modelFrameCapabilities(backendCatalog.models), + [backendCatalog.models], + ); const command = useCallback( - (value: CanvasCommand) => setHistory((h) => executeCommand(h, value)), + (value: CanvasCommand) => + setHistory((h) => executeCommand(h, value, frameCapabilities.current)), [], ); const change = useCallback( diff --git a/src/canvas/frameReferences.ts b/src/canvas/frameReferences.ts index d98475c0c..ca372f25f 100644 --- a/src/canvas/frameReferences.ts +++ b/src/canvas/frameReferences.ts @@ -4,6 +4,7 @@ import type { GenerationSettings, } from "./model.ts"; import { tNow } from "../i18n/messages.ts"; +import type { ModelConfig } from "../backend/rules.ts"; export const FRAME_IMAGE_LIMIT = 2; export const frameImageLimitMessage = () => @@ -22,11 +23,45 @@ export function frameImageLimitIssue( : null; } +/** + * What the canvas command layer needs to know about the selected model. The + * document model has no access to the configured service catalog, so the + * workspace supplies it. Without it, a configured service model is treated as + * first-frame only: filling a slot the model rejects would block generation. + */ +export interface FrameCapabilities { + supportsLastFrame(settings: GenerationSettings): boolean; +} + +/** Read last-frame support from the configured model's input rules. */ +export function modelFrameCapabilities( + models: readonly Pick[], +): FrameCapabilities { + return { + supportsLastFrame: (settings) => { + if (!settings.modelConfigId) return true; + const config = models.find( + (model) => model.id === settings.modelConfigId, + ); + return Boolean(config?.inputRules.frameRoles?.includes("last_frame")); + }, + }; +} + +function supportsLastFrame( + settings: GenerationSettings, + capabilities?: FrameCapabilities, +) { + if (capabilities) return capabilities.supportsLastFrame(settings); + return !settings.modelConfigId; +} + /** Connection/mode-change default only; rendering and manual clearing never * refill a slot. Keep explicit selections and the order of incoming edges. */ export function fillConnectedFrames( settings: GenerationSettings, incoming: readonly CanvasNode[], + capabilities?: FrameCapabilities, ): GenerationSettings { if ( settings.mode !== "video" || @@ -44,8 +79,11 @@ export function fillConnectedFrames( ? settings : { ...settings, frameSlotsInitialized: true }; for (const field of ["firstFrameNodeId", "lastFrameNodeId"] as const) { - // Current local adapters accept a first frame only. - if (settings.modelConfigId && field === "lastFrameNodeId") continue; + if ( + field === "lastFrameNodeId" && + !supportsLastFrame(settings, capabilities) + ) + continue; if (next[field]) continue; const image = images.find( (node) => diff --git a/src/canvas/history.ts b/src/canvas/history.ts index 324394fc5..75a0629f3 100644 --- a/src/canvas/history.ts +++ b/src/canvas/history.ts @@ -1,6 +1,7 @@ import { applyCommand } from "./model.ts"; import { sameGenerationSettings, sameNodeData } from "./dataBoundary.ts"; import type { CanvasCommand, CanvasDocument, CanvasNode } from "./model.ts"; +import type { FrameCapabilities } from "./frameReferences.ts"; import { tNow } from "../i18n/messages.ts"; export interface CanvasHistory { @@ -38,8 +39,9 @@ export function recordChange( export function executeCommand( history: CanvasHistory, command: CanvasCommand, + frames?: FrameCapabilities, ): CanvasHistory { - return recordChange(history, applyCommand(history.present, command)); + return recordChange(history, applyCommand(history.present, command, frames)); } /** Start once on drag start, update during drag, then commit once on drag stop. */ diff --git a/src/canvas/model.ts b/src/canvas/model.ts index 2ec495ec2..bfab18c00 100644 --- a/src/canvas/model.ts +++ b/src/canvas/model.ts @@ -20,6 +20,7 @@ import { import { fillConnectedFrames, frameImageLimitIssue, + type FrameCapabilities, } from "./frameReferences.ts"; /** Authored local canvas model, compatible with React Flow's node/edge shape. */ export type CanvasNodeKind = @@ -459,8 +460,9 @@ export function canAddReference( export function applyCommand( document: CanvasDocument, command: CanvasCommand, + frames?: FrameCapabilities, ): CanvasDocument { - let next = applyCanvasCommand(document, command); + let next = applyCanvasCommand(document, command, frames); if ( next !== document && (command.type === "remove-edges" || @@ -564,9 +566,13 @@ export function applyCommand( function applyCanvasCommand( document: CanvasDocument, command: CanvasCommand, + frames?: FrameCapabilities, ): CanvasDocument { if (command.type === "batch") - return command.commands.reduce(applyCommand, document); + return command.commands.reduce( + (current, item) => applyCommand(current, item, frames), + document, + ); if (command.type === "reference-sources") { const ids = new Set( [...document.nodes, ...(document.hiddenNodes ?? [])].map( @@ -770,7 +776,7 @@ function applyCanvasCommand( settings.videoMode === "first-last" && node.data.settings?.videoMode !== "first-last" ) - settings = fillConnectedFrames(settings, incoming); + settings = fillConnectedFrames(settings, incoming, frames); if (settings !== data.settings) data = { ...data, settings }; } changed = true; @@ -1017,7 +1023,11 @@ function applyCanvasCommand( const incoming = incomingIds.flatMap( (id) => logicalNodes.find((node) => node.id === id) ?? [], ); - const settings = fillConnectedFrames(target.data.settings, incoming); + const settings = fillConnectedFrames( + target.data.settings, + incoming, + frames, + ); return { ...document, edges, diff --git a/tests/frameReferences.test.ts b/tests/frameReferences.test.ts index e2da8acb5..21e2d67f5 100644 --- a/tests/frameReferences.test.ts +++ b/tests/frameReferences.test.ts @@ -14,7 +14,10 @@ import { import { createHistory, recordChange, undo } from "../src/canvas/history.ts"; import { parseDocument, serializeDocument } from "../src/canvas/storage.ts"; import { referenceIssue } from "../src/canvas/connectionPolicy.ts"; -import { initializeConnectedFrames } from "../src/canvas/frameReferences.ts"; +import { + initializeConnectedFrames, + modelFrameCapabilities, +} from "../src/canvas/frameReferences.ts"; const image = () => createNode("image", undefined, { @@ -202,6 +205,93 @@ test("first-frame-only adapters fill the first slot without creating an unsuppor assert.equal(doc.nodes.at(-1)!.data.settings!.lastFrameNodeId, undefined); }); +test("configured models that declare a last frame fill both slots in edge order", () => { + const a = image(), + b = image(), + target = video(true); + const configId = target.data.settings!.modelConfigId!; + const frames = modelFrameCapabilities([ + { + id: configId, + inputRules: { frameRoles: ["first_frame", "last_frame"] }, + } as never, + ]); + let doc: CanvasDocument = { ...createDocument(), nodes: [a, b, target] }; + for (const source of [a, b]) + doc = applyCommand( + doc, + { + type: "add-edge", + edge: { id: crypto.randomUUID(), source: source.id, target: target.id }, + }, + frames, + ); + const settings = doc.nodes.at(-1)!.data.settings!; + assert.equal(settings.firstFrameNodeId, a.id); + assert.equal(settings.lastFrameNodeId, b.id); +}); + +test("frame capabilities reach slots filled by a batch or a switch to first-last mode", () => { + const a = image(), + b = image(), + target = video(true); + target.data.settings!.videoMode = "reference"; + const configId = target.data.settings!.modelConfigId!; + const frames = modelFrameCapabilities([ + { + id: configId, + inputRules: { frameRoles: ["first_frame", "last_frame"] }, + } as never, + ]); + const edges = [a, b].map((source) => ({ + type: "add-edge" as const, + edge: { id: crypto.randomUUID(), source: source.id, target: target.id }, + })); + let doc = applyCommand( + { ...createDocument(), nodes: [a, b, target] }, + { type: "batch", commands: edges }, + frames, + ); + doc = applyCommand( + doc, + { + type: "update-node", + id: target.id, + patch: { + data: { + settings: { ...target.data.settings!, videoMode: "first-last" }, + }, + }, + }, + frames, + ); + const settings = doc.nodes.at(-1)!.data.settings!; + assert.equal(settings.firstFrameNodeId, a.id); + assert.equal(settings.lastFrameNodeId, b.id); +}); + +test("unknown or first-frame-only configured models never receive a last frame", () => { + const target = video(true); + const settings = target.data.settings!; + assert.equal(modelFrameCapabilities([]).supportsLastFrame(settings), false); + assert.equal( + modelFrameCapabilities([ + { + id: settings.modelConfigId!, + inputRules: { frameRoles: ["first_frame"] }, + } as never, + ]).supportsLastFrame(settings), + false, + ); + assert.equal( + modelFrameCapabilities([]).supportsLastFrame({ + ...settings, + modelConfigId: undefined, + }), + true, + ); +}); + test("over-limit reference mode cannot switch to frames or submit a legacy over-limit draft", () => { const images = [image(), image(), image()], target = video();