diff --git a/docs/generation-sessions.md b/docs/generation-sessions.md new file mode 100644 index 0000000..79059df --- /dev/null +++ b/docs/generation-sessions.md @@ -0,0 +1,33 @@ +# Persistent generation sessions (proposal) + +Create a session with an API key that has images or videos scope: + +```http +POST /v1/generation-sessions +Authorization: Bearer +Content-Type: application/json + +{"title":"Sequential shots"} +``` + +Save the returned `id` (for example, `gs_EXAMPLE`) and pass it to related image/video requests: + +```json +{ + "prompt": "Create the next shot using the established character design.", + "generation_session_id": "gs_EXAMPLE", + "async": true +} +``` + +Use `GET /v1/generation-sessions/{id}` to inspect state, busy status, active job and completed turn count. The upstream thread and account identifiers are not exposed. Session ownership is scoped to the creating API key. + +Turns in one session execute exclusively and remain pinned to the same account. A busy session requeues waiting jobs without consuming an attempt. Omit the optional field to retain the existing independent-job behavior. + +Session turns do not automatically retry generation or switch accounts. A failure after binding the upstream account, or a detected orphaned turn, conservatively marks the session uncertain. Inspect the existing job and upstream outcome before explicitly starting another session; a timeout does not establish that generation stopped. The inspection endpoint remains available for uncertain sessions. + +## Review limitations + +This is an opt-in API proposal. Retention expiry, explicit close/delete, and a user-facing reconciliation operation for uncertain sessions are not implemented. Upstream thread lifetime and exact visual continuity are not guaranteed. Crash recovery uses the existing stale-heartbeat sweep; it is not an immediate unlock guarantee. Interrupted/aborted turn behavior and multi-node races deserve further integration review. + +Offline architecture tests cover ownership, exclusive locking, account affinity, busy requeueing, uncertainty, retained outputs, and exclusion of previous-turn media. Syntax and Studio checks also pass. No new upstream generation was submitted while preparing this contribution. Real MongoDB crash recovery and distributed load were not rerun for this PR. diff --git a/src/core/client.js b/src/core/client.js index 205c8d1..2cb96ec 100644 --- a/src/core/client.js +++ b/src/core/client.js @@ -1107,6 +1107,24 @@ class MuseClient extends EventEmitter { options.signal?.throwIfAborted(); const onToken = typeof options === 'function' ? options : options.onToken; const onEvent = options.onEvent; + const mediaPaths = p => [p?.image_path, ...(p?.data?.images || []).map(x => x.path), ...(p?.data?.videos || []).map(x => x.path)].filter(Boolean); + const previousPaths = new Set(); + let previousUserIndex = -1; + if (options.sessionId && options.waitForKind) { + const history = await this.sendRequest('GET', `/chat/history?session_id=${options.sessionId}`, null, [], false, null, { signal: options.signal }); + if (history.status !== 200 && history.status !== 404) throw new Error('Could not establish the retained session turn boundary.'); + if (history.status === 200 && history.body) { + const events = JSON.parse(history.body).result?.chat_events || []; + previousUserIndex = events.map(e => e.role === 'user' || e.event_name === 'message.user').lastIndexOf(true); + for (const event of events) { + const p = event.payload || event; + for (const presentation of [p, ...(p.presentations || [])]) { + for (const path of mediaPaths(presentation)) previousPaths.add(path); + } + } + } + } + const isCurrentMedia = p => !mediaPaths(p).some(path => previousPaths.has(path)); return new Promise((resolve, reject) => { let fullResponse = ''; @@ -1117,10 +1135,10 @@ class MuseClient extends EventEmitter { const hasExpectedMedia = () => { if (options.waitForKind === 'video') { - return presentations.some(p => p.kind === 'video' || p.data?.videos?.length > 0); + return presentations.some(p => (p.kind === 'video' || p.data?.videos?.length > 0) && mediaPaths(p).length > 0); } if (options.waitForKind === 'image') { - return presentations.some(p => p.kind === 'image' || p.data?.images?.length > 0); + return presentations.some(p => (p.kind === 'image' || p.data?.images?.length > 0) && mediaPaths(p).length > 0); } return false; }; @@ -1236,7 +1254,7 @@ class MuseClient extends EventEmitter { const handlePresentation = (payload, seq) => { if (isDone || !isForSession(payload)) return; // Do not treat browser_task as a media presentation - if (payload?.kind === 'browser_task' || payload?.data?.browser_task_id) return; + if (payload?.kind === 'browser_task' || payload?.data?.browser_task_id || !isCurrentMedia(payload)) return; presentations.push(payload); emit('presentation', payload); @@ -1300,7 +1318,7 @@ class MuseClient extends EventEmitter { // Extract any presentations from payload if (Array.isArray(payload?.presentations)) { for (const pres of payload.presentations) { - if (!presentations.some(p => p.id === pres.id || p.image_path === pres.image_path)) { + if (isCurrentMedia(pres) && !presentations.some(p => (p.id && p.id === pres.id) || (p.image_path && p.image_path === pres.image_path))) { presentations.push(pres); emit('presentation', pres); } @@ -1339,20 +1357,21 @@ class MuseClient extends EventEmitter { // Find the LAST user message in this session const lastUserIdx = evs.map(e => e.role === 'user' || e.event_name === 'message.user').lastIndexOf(true); + if (options.sessionId && options.waitForKind && lastUserIdx <= previousUserIndex) return; const turnEvents = lastUserIdx !== -1 ? evs.slice(lastUserIdx + 1) : evs; // Extract any presentations from ALL turn events (including delta.presentation) for (const evItem of turnEvents) { const p = evItem.payload || evItem; if (evItem.event_name === 'delta.presentation' || p.kind === 'image' || p.kind === 'video' || p.data?.images?.length > 0 || p.data?.videos?.length > 0) { - if (!presentations.some(pr => (pr.id && pr.id === p.id) || (pr.image_path && pr.image_path === p.image_path) || (pr.data?.images?.[0]?.path && pr.data?.images?.[0]?.path === p.data?.images?.[0]?.path))) { + if (isCurrentMedia(p) && !presentations.some(pr => (pr.id && pr.id === p.id) || (pr.image_path && pr.image_path === p.image_path) || (pr.data?.images?.[0]?.path && pr.data?.images?.[0]?.path === p.data?.images?.[0]?.path))) { presentations.push(p); emit('presentation', p); } } if (Array.isArray(p.presentations)) { for (const pres of p.presentations) { - if (!presentations.some(pr => (pr.id && pr.id === pres.id) || (pr.image_path && pr.image_path === pres.image_path))) { + if (isCurrentMedia(pres) && !presentations.some(pr => (pr.id && pr.id === pres.id) || (pr.image_path && pr.image_path === pres.image_path))) { presentations.push(pres); emit('presentation', pres); } @@ -1645,8 +1664,8 @@ class MuseClient extends EventEmitter { } const chatResult = await this.chat(formattedPrompt, { - sessionId: null, - new_session: true, + sessionId: options.sessionId || null, + new_session: !options.sessionId, ref_image: refImage, ref_video: refVideo, timeout: options.timeout || 180000, @@ -1662,7 +1681,7 @@ class MuseClient extends EventEmitter { // Find image presentation widget const imagePres = chatResult.presentations.find(p => p.kind === 'image' || p.data?.images?.length > 0); if (!imagePres) { - if (chatResult.sessionId) this.deleteSession(chatResult.sessionId).catch(() => {}); + if (chatResult.sessionId && !options.retainSession) this.deleteSession(chatResult.sessionId).catch(() => {}); return { success: false, text: chatResult.text, @@ -1683,11 +1702,12 @@ class MuseClient extends EventEmitter { base64 = buffer.toString('base64'); } } finally { - if (chatResult.sessionId) await this.deleteSession(chatResult.sessionId).catch(() => {}); + if (chatResult.sessionId && !options.retainSession) await this.deleteSession(chatResult.sessionId).catch(() => {}); } return { success: true, + sessionId: chatResult.sessionId, text: chatResult.text, prompt, path: imgPath, @@ -1717,8 +1737,8 @@ class MuseClient extends EventEmitter { } const chatResult = await this.chat(formattedPrompt, { - sessionId: null, - new_session: true, + sessionId: options.sessionId || null, + new_session: !options.sessionId, ref_image: refImage, ref_video: refVideo, timeout: options.timeout || 180000, @@ -1739,7 +1759,7 @@ class MuseClient extends EventEmitter { // Only this session's presentation/history may identify an output file. if (!vidPath) { - if (chatResult.sessionId) await this.deleteSession(chatResult.sessionId).catch(() => {}); + if (chatResult.sessionId && !options.retainSession) await this.deleteSession(chatResult.sessionId).catch(() => {}); return { success: false, text: chatResult.text, @@ -1757,11 +1777,12 @@ class MuseClient extends EventEmitter { base64 = buffer.toString('base64'); } } finally { - if (chatResult.sessionId) await this.deleteSession(chatResult.sessionId).catch(() => {}); + if (chatResult.sessionId && !options.retainSession) await this.deleteSession(chatResult.sessionId).catch(() => {}); } return { success: true, + sessionId: chatResult.sessionId, text: chatResult.text, prompt, path: vidPath, diff --git a/src/docs/openapi.js b/src/docs/openapi.js index b23fb11..3e091a5 100644 --- a/src/docs/openapi.js +++ b/src/docs/openapi.js @@ -127,6 +127,16 @@ function buildOpenApiSpec() { { name: 'Health', description: 'Liveness and readiness probes.' } ], paths: { + '/v1/generation-sessions': { + post: { tags: ['Generation sessions'], summary: 'Create a retained image/video project session (images or videos scope; no upstream generation)', + requestBody: { required: true, content: json({ type: 'object', additionalProperties: false, properties: { title: { type: 'string', maxLength: 200 } } }) }, + responses: { 201: { description: 'Session created', content: json(ref('GenerationSession')) }, 401: resp('Error') } } + }, + '/v1/generation-sessions/{id}': { + get: { tags: ['Generation sessions'], summary: 'Inspect an owned generation session', + parameters: [{ name: 'id', in: 'path', required: true, schema: { type: 'string' } }], + responses: { 200: { description: 'Session state (upstream account/session identifiers are private)', content: json(ref('GenerationSession')) }, 404: resp('Error') } } + }, // ------------------------------------------------------------------ Chat '/v1/chat/completions': { post: { @@ -448,11 +458,17 @@ function buildOpenApiSpec() { } }, + GenerationSession: { + type: 'object', properties: { id: { type: 'string' }, object: { const: 'generation_session' }, title: { type: 'string' }, + state: { type: 'string', enum: ['ready', 'uncertain'] }, busy: { type: 'boolean' }, active_job_id: { type: ['string', 'null'] }, + turn_count: { type: 'integer' }, created_at: { type: 'string', format: 'date-time' }, updated_at: { type: 'string', format: 'date-time' } } + }, ImageGenerationRequest: { type: 'object', required: ['prompt'], additionalProperties: false, properties: { + generation_session_id: { type: 'string', pattern: '^gs_[A-Za-z0-9_-]+$', description: 'Retain and continue one upstream session on its owning account. Turns execute exclusively. Interrupted turns mark the session uncertain and are not automatically retried.' }, prompt: { type: 'string', minLength: 1, maxLength: 4000 }, model: { type: 'string' }, n: { type: 'integer', const: 1 }, @@ -480,6 +496,7 @@ function buildOpenApiSpec() { required: ['prompt'], additionalProperties: false, properties: { + generation_session_id: { type: 'string', pattern: '^gs_[A-Za-z0-9_-]+$', description: 'Retain and continue one upstream session on its owning account. Turns execute exclusively. Interrupted turns mark the session uncertain and are not automatically retried.' }, prompt: { type: 'string', minLength: 1, maxLength: 4000 }, model: { type: 'string' }, aspect_ratio: { type: 'string', enum: ['16:9', '9:16', '1:1'], default: '16:9' }, @@ -496,6 +513,7 @@ function buildOpenApiSpec() { properties: { id: { type: 'string' }, object: { const: 'job' }, + generation_session_id: { type: ['string', 'null'] }, type: { type: 'string', enum: ['image', 'video'] }, status: { type: 'string', enum: ['queued', 'running', 'succeeded', 'failed', 'canceled'] }, progress: { type: 'integer', minimum: 0, maximum: 100, description: 'Estimated completion percentage.' }, diff --git a/src/models/GenerationSession.js b/src/models/GenerationSession.js new file mode 100644 index 0000000..e1ac630 --- /dev/null +++ b/src/models/GenerationSession.js @@ -0,0 +1,17 @@ +'use strict'; +const mongoose = require('mongoose'); +const { randomUUID } = require('crypto'); +const { generateId } = require('../utils/crypto'); +const schema = new mongoose.Schema({ + _id: { type: String, default: () => generateId('gs') }, + api_key_id: { type: String, required: true, index: true }, + title: { type: String, default: '', maxlength: 200 }, + session_id: { type: String, default: randomUUID, required: true }, + account_id: { type: String, default: null }, + state: { type: String, enum: ['ready', 'uncertain'], default: 'ready' }, + active_execution: { type: String, default: null }, + active_job_id: { type: String, default: null }, + upstream_started: { type: Boolean, default: false }, + turn_count: { type: Number, default: 0 } +}, { timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }, versionKey: false }); +module.exports = { GenerationSession: mongoose.models.GenerationSession || mongoose.model('GenerationSession', schema) }; diff --git a/src/models/Job.js b/src/models/Job.js index f77c835..9ccdd79 100644 --- a/src/models/Job.js +++ b/src/models/Job.js @@ -19,6 +19,8 @@ const JobSchema = new mongoose.Schema( api_key_id: { type: String, required: true }, status: { type: String, enum: JOB_STATUSES, default: 'queued' }, prompt: { type: String, required: true, maxlength: 4000 }, + generation_session_id: { type: String, default: null }, + session_wait_until: { type: Date, default: null }, params: { aspect_ratio: { type: String, default: null }, duration: { type: Number, default: null }, diff --git a/src/routes/generation.js b/src/routes/generation.js index 53a741d..140bef7 100644 --- a/src/routes/generation.js +++ b/src/routes/generation.js @@ -63,6 +63,7 @@ async function submitGeneration(req, res, { type, sync, params, references, onSu await dispatcher.assertCanEnqueue(type, apiKey); const webhookUrl = body.webhook_url ? webhook.validateWebhookUrl(body.webhook_url) : null; + if (body.generation_session_id) await require('../services/generationSessions').get(body.generation_session_id, apiKey._id); const reference = { image_file_id: null, video_file_id: null }; if (references.image) { const doc = await media.resolveReference(references.image, { expectedKind: 'image', apiKeyId: apiKey._id, param: 'reference_image' }); @@ -77,6 +78,7 @@ async function submitGeneration(req, res, { type, sync, params, references, onSu apiKey, type, prompt: body.prompt, + generationSessionId: body.generation_session_id, params, reference, webhookUrl, diff --git a/src/routes/generationSessions.js b/src/routes/generationSessions.js new file mode 100644 index 0000000..2dc07fa --- /dev/null +++ b/src/routes/generationSessions.js @@ -0,0 +1,20 @@ +'use strict'; +const express = require('express'); +const { GenerationSession } = require('../models/GenerationSession'); +const sessions = require('../services/generationSessions'); +const { requireScope } = require('../middleware/auth'); +const { validate } = require('../middleware/validate'); +const { z } = require('./helpers'); +const { ApiError, ErrorCodes } = require('../errors'); +const router = express.Router(); +router.use(requireScope('images', 'videos')); +router.post('/', validate(z.object({ title: z.string().trim().max(200).default('') }).strict()), async (req, res) => { + const doc = await GenerationSession.create({ api_key_id: req.auth.apiKey._id, title: req.valid.body.title }); + res.status(201).json(sessions.serialize(doc)); +}); +router.get('/:id', async (req, res) => { + const doc = await GenerationSession.findOne({ _id: req.params.id, api_key_id: req.auth.apiKey._id }).lean(); + if (!doc) throw new ApiError(ErrorCodes.RESOURCE_NOT_FOUND); + res.json(sessions.serialize(doc)); +}); +module.exports = router; diff --git a/src/routes/images.js b/src/routes/images.js index 97b577b..ff541b3 100644 --- a/src/routes/images.js +++ b/src/routes/images.js @@ -23,6 +23,7 @@ const router = express.Router(); const IMAGE_ASPECT_RATIOS = ['1:1', '16:9', '9:16', '4:3', '3:4', '3:2', '2:3']; const imageRequestSchema = z.object({ + generation_session_id: z.string().regex(/^gs_[A-Za-z0-9_-]+$/).optional(), prompt: z.string().trim().min(1).max(4000), model: z.string().max(64).optional(), n: z.literal(1).optional().describe('Only one image per request is supported.'), diff --git a/src/routes/index.js b/src/routes/index.js index d8cb4ee..5c58dc5 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -22,6 +22,7 @@ v1.use('/files', filesRouter); // Every other endpoint requires an API key and is rate limited per key. v1.use(authenticate, keyRateLimit); +v1.use('/generation-sessions', require('./generationSessions')); v1.use('/images', imagesRouter); v1.use('/videos', videosRouter); v1.use('/jobs', jobsRouter); diff --git a/src/routes/videos.js b/src/routes/videos.js index 79c9e48..472d29d 100644 --- a/src/routes/videos.js +++ b/src/routes/videos.js @@ -23,6 +23,7 @@ const VIDEO_ASPECT_RATIOS = ['16:9', '9:16', '1:1']; const videoRequestSchema = z .object({ + generation_session_id: z.string().regex(/^gs_[A-Za-z0-9_-]+$/).optional(), prompt: z.string().trim().min(1).max(4000), model: z.string().max(64).optional(), aspect_ratio: z.enum(VIDEO_ASPECT_RATIOS).default('16:9'), diff --git a/src/serializers.js b/src/serializers.js index bd0d524..fde0712 100644 --- a/src/serializers.js +++ b/src/serializers.js @@ -46,6 +46,7 @@ function serializeJob(doc) { id: doc._id, object: 'job', type: doc.type, + generation_session_id: doc.generation_session_id || null, status: doc.status, progress: doc.progress ?? 0, progress_text: doc.progress_text || null, @@ -56,6 +57,7 @@ function serializeJob(doc) { }, reference: { image_file_id: doc.reference?.image_file_id || null, + ...(doc.reference?.images?.length ? { images: doc.reference.images.map(item => ({ file_id: item.file_id, label: item.label, role: item.role })) } : {}), ...(doc.type === 'video' ? { video_file_id: doc.reference?.video_file_id || null } : {}) }, output, diff --git a/src/services/dispatcher.js b/src/services/dispatcher.js index a7f8e2f..05f9f7c 100644 --- a/src/services/dispatcher.js +++ b/src/services/dispatcher.js @@ -31,6 +31,7 @@ const { Job, TERMINAL_STATUSES } = require('../models'); const { pool } = require('./pool'); const storage = require('./storage'); const media = require('./media'); +const generationSessions = require('./generationSessions'); const usage = require('./usage'); const webhook = require('./webhook'); const { serializeJob } = require('../serializers'); @@ -103,7 +104,9 @@ class Dispatcher extends EventEmitter { if (!ctx) continue; ctx.controller.abort(new Error('Job interrupted by server shutdown.')); await Job.updateOne(this._executionFilter(ctx.job), - { $set: { status: 'queued', node_id: null, execution_id: null, account_id: null, progress: 0, progress_text: 'Re-queued after server restart', heartbeat_at: null }, $inc: { attempts: -1 } } + ctx.job.generation_session_id + ? { $set: { status: 'failed', completed_at: new Date(), error: { code: ErrorCodes.JOB_INTERRUPTED.code, message: 'Session turn interrupted; inspect before creating another job.' } } } + : { $set: { status: 'queued', node_id: null, execution_id: null, account_id: null, progress: 0, progress_text: 'Re-queued after server restart', heartbeat_at: null }, $inc: { attempts: -1 } } ).catch((err) => log.error({ err: err.message }, 'Failed to re-queue job')); } } @@ -167,7 +170,7 @@ class Dispatcher extends EventEmitter { * Persists a new queued job. * @returns {Promise<{ job: object, replayed: boolean }>} */ - async createJob({ apiKey, type, prompt, params, reference, webhookUrl, idempotencyKey, fingerprint, metadata }) { + async createJob({ apiKey, type, prompt, params, reference, webhookUrl, idempotencyKey, fingerprint, metadata, generationSessionId }) { const queueTimeout = config.jobs[type].queueTimeoutMs; try { const doc = await Job.create({ @@ -176,11 +179,12 @@ class Dispatcher extends EventEmitter { prompt, params, reference, + generation_session_id: generationSessionId || null, webhook: { url: webhookUrl || null }, idempotency_key: idempotencyKey || null, request_fingerprint: fingerprint || null, metadata: metadata || {}, - max_attempts: config.jobs.maxAttempts, + max_attempts: generationSessionId ? 1 : config.jobs.maxAttempts, queue_deadline: queueTimeout > 0 ? new Date(Date.now() + queueTimeout) : null }); this.pendingCount += 1; @@ -291,7 +295,7 @@ class Dispatcher extends EventEmitter { _claim(type) { const now = new Date(); - const filter = { status: 'queued', type }; + const filter = { status: 'queued', type, $and: [{ $or: [{ session_wait_until: null }, { session_wait_until: { $lte: now } }] }] }; if (config.jobs[type].queueTimeoutMs > 0) { filter.$or = [{ queue_deadline: null }, { queue_deadline: { $gt: now } }]; } @@ -334,10 +338,11 @@ class Dispatcher extends EventEmitter { let outputFile = null; try { + if (job.generation_session_id) ctx.generationSession = await generationSessions.lock(job); const references = await this._loadReferences(job); ctx.controller.signal.throwIfAborted(); - ctx.lease = await pool.acquire(job.type, { exclude: job.excluded_accounts || [], waitMs: ACQUIRE_WAIT_MS }); + ctx.lease = await pool.acquire(job.type, { preferredAccountId: ctx.generationSession?.account_id || null, exclude: job.excluded_accounts || [], waitMs: ACQUIRE_WAIT_MS, signal: ctx.controller.signal }); ctx.controller.signal.throwIfAborted(); releaseAcquiring(); ctx.progress = 3; @@ -349,11 +354,14 @@ class Dispatcher extends EventEmitter { ctx.controller.signal.throwIfAborted(); } + if (ctx.generationSession) await generationSessions.bind(job, ctx.lease.accountId); + const sessionOptions = ctx.generationSession ? { sessionId: ctx.generationSession.session_id, retainSession: true } : {}; const isVideo = job.type === 'video'; const timeout = config.jobs[job.type].timeoutMs; const onProgress = (text) => this._onProgress(ctx, text); const call = isVideo ? ctx.lease.client.generateVideo(job.prompt, { + ...sessionOptions, ref_image: references.image, ref_video: references.video, aspect_ratio: job.params?.aspect_ratio || undefined, @@ -364,6 +372,7 @@ class Dispatcher extends EventEmitter { onProgress }) : ctx.lease.client.generateImage(job.prompt, { + ...sessionOptions, ref_image: references.image, aspect_ratio: job.params?.aspect_ratio || undefined, timeout, @@ -375,6 +384,9 @@ class Dispatcher extends EventEmitter { // Hard guard on top of the client's own timeout (covers the download phase). const result = await withTimeout(call, timeout + 60000, `${isVideo ? 'Video' : 'Image'} generation timed out.`, ctx.controller); ctx.controller.signal.throwIfAborted(); + if (ctx.generationSession && result?.sessionId !== ctx.generationSession.session_id) { + throw new ApiError(ErrorCodes.CONVERSATION_UNAVAILABLE, { message: 'Upstream returned a different session; continuity could not be verified.' }); + } if (!result?.success) { // The account worked; the model simply declined / produced nothing. @@ -419,6 +431,10 @@ class Dispatcher extends EventEmitter { } log.info({ job: job._id, type: job.type, account: ctx.lease.accountId, ms: Date.now() - ctx.startedAt, bytes: result.buffer.length }, 'Job succeeded'); outputFile = null; // The succeeded job now owns the stored file. + if (ctx.generationSession) { + await generationSessions.finish(job, true); + ctx.sessionFinished = true; + } this._finalize(updated); } catch (err) { releaseAcquiring(); @@ -428,6 +444,9 @@ class Dispatcher extends EventEmitter { if (ctx.lease && !ctx.lease.released) ctx.lease.release(ctx.canceled || this.stopping ? { success: true } : { success: false, error: apiError }); await this._handleFailure(ctx, apiError); } finally { + if (ctx.generationSession && !ctx.sessionFinished) { + await generationSessions.finish(job, false).catch(err => log.error({ job: job._id, err: sanitize(err.message) }, 'Failed to release generation session')); + } clearInterval(progressTimer); releaseAcquiring(); if (this.running.get(job._id) === ctx) this.running.delete(job._id); @@ -479,16 +498,16 @@ class Dispatcher extends EventEmitter { // Capacity race (slot taken between scheduling and acquisition) or node // shutting down: put the job back without consuming an attempt. - if (apiError.code === 'capacity_exceeded' || apiError.code === 'service_shutting_down') { + if (apiError.code === 'capacity_exceeded' || apiError.code === 'service_shutting_down' || apiError.code === 'conversation_busy') { await Job.updateOne(guard, { - $set: { status: 'queued', node_id: null, execution_id: null, progress: 0, progress_text: 'Waiting in queue', heartbeat_at: null }, + $set: { status: 'queued', node_id: null, execution_id: null, progress: 0, progress_text: 'Waiting in queue', heartbeat_at: null, session_wait_until: apiError.code === 'conversation_busy' ? new Date(Date.now() + 1000) : null }, $inc: { attempts: -1 } }).catch(() => {}); return; } const excluded = [...(job.excluded_accounts || []), ...(lease ? [lease.accountId] : [])]; - const retryable = isAccountLevelFailure(apiError) + const retryable = !job.generation_session_id && isAccountLevelFailure(apiError) && job.attempts < job.max_attempts && !this.stopping && pool.hasAlternative(job.type, excluded); @@ -557,7 +576,8 @@ class Dispatcher extends EventEmitter { for (const orphan of orphans) { if (orphan.node_id === this.nodeId && this.running.has(orphan._id)) continue; const guard = { _id: orphan._id, status: 'running', heartbeat_at: orphan.heartbeat_at, execution_id: orphan.execution_id ?? null }; - if (orphan.attempts < orphan.max_attempts) { + if (orphan.generation_session_id) await generationSessions.finish(orphan, false); + if (!orphan.generation_session_id && orphan.attempts < orphan.max_attempts) { await Job.updateOne(guard, { $set: { status: 'queued', node_id: null, execution_id: null, account_id: null, progress: 0, progress_text: 'Re-queued after interruption', heartbeat_at: null, last_attempt_error: { code: ErrorCodes.JOB_INTERRUPTED.code, message: 'The node processing this job stopped responding.' } } }); diff --git a/src/services/generationSessions.js b/src/services/generationSessions.js new file mode 100644 index 0000000..7217550 --- /dev/null +++ b/src/services/generationSessions.js @@ -0,0 +1,36 @@ +'use strict'; +const { GenerationSession } = require('../models/GenerationSession'); +const { ApiError, ErrorCodes } = require('../errors'); + +function serialize(doc) { + return { id: doc._id, object: 'generation_session', title: doc.title, state: doc.state, + busy: Boolean(doc.active_execution), active_job_id: doc.active_job_id || null, + turn_count: doc.turn_count, created_at: doc.created_at, updated_at: doc.updated_at }; +} +async function get(id, apiKeyId) { + const doc = await GenerationSession.findOne({ _id: id, api_key_id: apiKeyId }).lean(); + if (!doc) throw new ApiError(ErrorCodes.RESOURCE_NOT_FOUND, { param: 'generation_session_id' }); + if (doc.state !== 'ready') throw new ApiError(ErrorCodes.CONVERSATION_UNAVAILABLE, { message: 'This generation session has an uncertain upstream turn. Inspect the existing job before starting a new session.', param: 'generation_session_id' }); + return doc; +} +async function lock(job) { + const doc = await GenerationSession.findOneAndUpdate({ _id: job.generation_session_id, api_key_id: job.api_key_id, state: 'ready', active_execution: null }, + { $set: { active_execution: job.execution_id, active_job_id: job._id, upstream_started: false } }, { returnDocument: 'after' }).lean(); + if (doc) return doc; + await get(job.generation_session_id, job.api_key_id); + throw new ApiError(ErrorCodes.CONVERSATION_BUSY); +} +async function bind(job, accountId) { + const result = await GenerationSession.updateOne({ _id: job.generation_session_id, active_execution: job.execution_id, + $or: [{ account_id: null }, { account_id: accountId }] }, { $set: { account_id: accountId, upstream_started: true } }); + if (!result.matchedCount) throw new ApiError(ErrorCodes.CONVERSATION_UNAVAILABLE); +} +async function finish(job, succeeded) { + const filter = { _id: job.generation_session_id, active_execution: job.execution_id }; + if (!succeeded) { + await GenerationSession.updateOne({ ...filter, upstream_started: true }, { $set: { state: 'uncertain' } }); + } + await GenerationSession.updateOne(filter, { $set: { active_execution: null, active_job_id: null, upstream_started: false }, + ...(succeeded ? { $inc: { turn_count: 1 } } : {}) }); +} +module.exports = { get, lock, bind, finish, serialize }; diff --git a/tests/architecture.test.js b/tests/architecture.test.js index 37bc634..7a9fe32 100644 --- a/tests/architecture.test.js +++ b/tests/architecture.test.js @@ -221,7 +221,7 @@ test('authentication failure renews the credential and reconnects without recurs function dispatcher(mocks = {}, globals = {}) { return isolated('src/services/dispatcher.js', { - '../models': {}, './pool': { pool: {} }, './storage': {}, './media': {}, './usage': { track() {} }, + '../models': {}, './pool': { pool: {} }, './storage': {}, './media': {}, './generationSessions': {}, './usage': { track() {} }, './webhook': { startWebhooks() {}, stopWebhooks() {}, dispatchJobEvent: async () => {} }, '../serializers': { serializeJob: x => x }, ...mocks }, '', globals).Dispatcher; @@ -602,3 +602,117 @@ test('NoiseFrameDecoder evicts stale or excessive partial frames and serializeAc ); assert.equal(serialized.live.is_renewing, true); }); + +test('retained image/video turns use the supplied upstream session and never delete it on success', async () => { + for (const kind of ['image', 'video']) { + const c = client(); + c.chat = async (_, options) => { + assert.equal(options.sessionId, A); + assert.equal(options.new_session, false); + return { sessionId: A, text: 'done', presentations: [{ kind, data: { [kind === 'image' ? 'images' : 'videos']: [{ path: `${kind}.bin` }] } }] }; + }; + c.readFile = async () => Buffer.from('media'); + c.deleteSession = async () => assert.fail('Retained session was deleted'); + const result = await c[kind === 'image' ? 'generateImage' : 'generateVideo']('next turn', { sessionId: A, retainSession: true }); + assert.equal(result.sessionId, A); + assert.equal(result.success, true); + } +}); + +test('generation sessions enforce ownership, atomic exclusivity, account affinity and uncertainty', async () => { + const doc = { _id: 'gs_test', api_key_id: 'owner', session_id: A, account_id: null, state: 'ready', active_execution: null, turn_count: 0 }; + function matches(filter) { + return Object.entries(filter).every(([key, value]) => key === '$or' ? value.some(matches) : doc[key] === value); + } + function update(changes) { Object.assign(doc, changes.$set || {}); for (const [key, n] of Object.entries(changes.$inc || {})) doc[key] += n; } + const model = { + findOne: filter => ({ lean: async () => matches(filter) ? { ...doc } : null }), + findOneAndUpdate: (filter, changes) => ({ lean: async () => { if (!matches(filter)) return null; update(changes); return { ...doc }; } }), + updateOne: async (filter, changes) => { if (!matches(filter)) return { matchedCount: 0 }; update(changes); return { matchedCount: 1 }; } + }; + const sessions = isolated('src/services/generationSessions.js', { '../models/GenerationSession': { GenerationSession: model } }); + const job = { _id: 'job_one', api_key_id: 'owner', generation_session_id: 'gs_test', execution_id: 'run_one' }; + await assert.rejects(sessions.get('gs_test', 'foreign'), /resource_not_found/); + const [first, second] = await Promise.allSettled([sessions.lock(job), sessions.lock({ ...job, execution_id: 'run_two' })]); + assert.equal(first.status, 'fulfilled'); assert.equal(second.status, 'rejected'); + await sessions.bind(job, 'account_one'); + await assert.rejects(sessions.bind(job, 'account_two'), /conversation_unavailable/); + await sessions.finish({ ...job, execution_id: 'foreign_execution' }, true); + assert.equal(doc.active_execution, 'run_one'); + await sessions.finish(job, true); + assert.equal(doc.turn_count, 1);assert.equal(doc.active_execution, null);assert.equal(doc.state, 'ready'); + await sessions.lock(job);await sessions.bind(job, 'account_one');await sessions.finish(job, false); + assert.equal(doc.state, 'uncertain'); + await assert.rejects(sessions.get('gs_test', 'owner'), /uncertain upstream turn/); + const publicDoc = sessions.serialize(doc); + assert.equal(publicDoc.session_id, undefined);assert.equal(publicDoc.account_id, undefined);assert.equal(publicDoc.api_key_id, undefined); +}); + +test('dispatcher pins a retained session and commits one turn after media succeeds', async () => { + const events = []; + const session = { session_id: A, account_id: 'pinned' }; + const D = dispatcher({ + './generationSessions': { lock: async () => session, bind: async (_, id) => events.push(['bind', id]), finish: async (_, success) => events.push(['finish', success]) }, + '../models': { Job: { updateOne: async () => ({ matchedCount: 1 }), findOneAndUpdate: () => ({ lean: async () => ({ _id: 'job', status: 'succeeded', type: 'image' }) }) } }, + './pool': { pool: { acquire: async (_, options) => { + assert.equal(options.preferredAccountId, 'pinned'); + return { accountId: 'pinned', released: false, release() { this.released = true; }, client: { generateImage: async (_, options) => { + assert.equal(options.sessionId, A); assert.equal(options.retainSession, true); + return { success: true, sessionId: A, buffer: Buffer.from('image') }; + } } }; + } } }, + './storage': { saveFile: async () => ({ _id: 'file' }) } + }); + const d = new D();d._loadReferences = async () => ({});d._finalize = () => {}; + await d._run({ _id: 'job', generation_session_id: 'gs_test', api_key_id: 'owner', execution_id: 'run', type: 'image', attempts: 1, max_attempts: 1 }); + assert.deepEqual(events, [['bind', 'pinned'], ['finish', true]]); +}); + +test('a failed retained turn is marked uncertain without account failover or paid retry', async () => { + const finished = []; + const D = dispatcher({ + './generationSessions': { lock: async () => ({ session_id: A, account_id: 'pinned' }), bind: async () => {}, finish: async (_, ok) => finished.push(ok) }, + '../models': { Job: { updateOne: async () => ({ matchedCount: 1 }), findOneAndUpdate: () => ({ lean: async () => ({ _id: 'job', status: 'failed' }) }) } }, + './pool': { pool: { hasAlternative: () => assert.fail('Retained turn must not fail over'), acquire: async () => ({ accountId: 'pinned', released: false, release() { this.released = true; }, client: { generateImage: async () => { throw new ApiError(ErrorCodes.UPSTREAM_ERROR); } } }) } } + }); + const d = new D();d._loadReferences = async () => ({});d._finalize = () => {}; + await d._run({ _id: 'job', generation_session_id: 'gs_test', api_key_id: 'owner', execution_id: 'run', type: 'image', attempts: 1, max_attempts: 2 }); + assert.deepEqual(finished, [false]); +}); + +test('another turn in a busy generation session waits without acquiring an upstream slot', async () => { + let update; + const D = dispatcher({ + './generationSessions': { lock: async () => { throw new ApiError(ErrorCodes.CONVERSATION_BUSY); } }, + '../models': { Job: { updateOne: async (_, value) => { update = value; return { matchedCount: 1 }; } } }, + './pool': { pool: { acquire: () => assert.fail('Busy session must wait before upstream work') } } + }); + const d = new D(); + await d._run({ _id: 'job', generation_session_id: 'gs_test', execution_id: 'run', type: 'video', attempts: 1, max_attempts: 1 }); + assert.equal(update.$set.status, 'queued'); + assert.equal(update.$inc.attempts, -1); + assert.ok(update.$set.session_wait_until > new Date()); +}); + +test('retained media turns ignore replayed video presentations and history from the preceding user turn', async () => { + const c = client({ setTimeout: (fn, ms) => setTimeout(fn, [300, 1500, 2500, 3500].includes(ms) ? 2 : ms), setInterval: (fn, ms) => setInterval(fn, ms === 400 ? 5 : ms) }); + const old = { kind: 'video', id: 'old-video', session_id: A, data: { videos: [{ path: 'old.mp4' }] } }; + const fresh = { kind: 'video', id: 'new-video', session_id: A, data: { videos: [{ path: 'new.mp4' }] } }; + let newTurnVisible = false; + c.sendRequest = async method => ({ status: 200, body: method === 'GET' ? JSON.stringify({ result: { chat_events: [ + { role: 'user', display_text: 'previous request' }, { event_name: 'delta.presentation', payload: old }, + ...(newTurnVisible ? [{ role: 'user', display_text: 'current request' }, { event_name: 'delta.presentation', payload: fresh }] : []) + ] } }) : '{}' }); + let resolved = false; + const pending = c.chat('current request', { sessionId: A, waitForKind: 'video', timeout: 1000 }).then(result => { resolved = true; return result; }); + await delay(5); + c.emit('presentation', old, 1); + c.emit('message_done', { session_id: A, presentations: [old] }, 2); + c.emit('agent_status', { session_id: A, activity_code: 'idle' }, 3); + await delay(25); + assert.equal(resolved, false); + newTurnVisible = true; + const result = await pending; + assert.equal(result.presentations.some(p => p.data?.videos?.[0]?.path === 'old.mp4'), false); + assert.equal(result.presentations.some(p => p.data?.videos?.[0]?.path === 'new.mp4'), true); +});