diff --git a/apps/sim/lib/mothership/async-runs/execution-lease.ts b/apps/sim/lib/mothership/async-runs/execution-lease.ts index 602c643c3d8..8e3a3f4bdf3 100644 --- a/apps/sim/lib/mothership/async-runs/execution-lease.ts +++ b/apps/sim/lib/mothership/async-runs/execution-lease.ts @@ -20,11 +20,11 @@ export const INTERRUPTED_SIM_TOOL_MESSAGE = 'Tool execution lost its owner before a result was recorded. Its outcome is unknown; inspect current state before retrying a mutation.' /** - * A takeover re-runs an interrupted call only with at least this much of its deadline left; - * otherwise the call fails with {@link DEADLINE_SIM_TOOL_MESSAGE} instead of starting work - * that could not finish in time. + * A call starts, the first time or again after a takeover, only with at least this much of its + * deadline left; otherwise it fails as not run instead of starting work that would likely be cut + * off with its outcome unknown. About 99% of non-workflow tool calls finish within a minute. */ -export const SIM_TOOL_RERUN_MIN_REMAINING_MS = 60_000 +export const SIM_TOOL_MIN_START_BUDGET_MS = 60_000 export const DEADLINE_SIM_TOOL_MESSAGE = 'Tool execution was interrupted because the Sim server running it stopped, and too little of its time remained to run it again. Its outcome is unknown; inspect current state before retrying a mutation.' diff --git a/apps/sim/lib/mothership/async-runs/repository.ts b/apps/sim/lib/mothership/async-runs/repository.ts index 3822af296d7..7f826b28f96 100644 --- a/apps/sim/lib/mothership/async-runs/repository.ts +++ b/apps/sim/lib/mothership/async-runs/repository.ts @@ -42,7 +42,7 @@ import { INTERRUPTED_SIM_TOOL_MESSAGE, SIM_TOOL_EXECUTION_LEASE_SECONDS, SIM_TOOL_EXECUTION_MAX_ATTEMPTS, - SIM_TOOL_RERUN_MIN_REMAINING_MS, + SIM_TOOL_MIN_START_BUDGET_MS, SimToolExecutionLeaseLostError, type SimToolExecutionOwner, } from '@/lib/mothership/async-runs/execution-lease' @@ -891,8 +891,11 @@ const unfinishedSimExecution = allOf( isNull(copilotAsyncToolCalls.clientWorkflowExecutionId) ) -/** A call whose deadline leaves less than a re-run needs; a call without one never matches. */ -const tooLateToRerun = sql`(${copilotAsyncToolCalls.executionDeadlineAt} IS NOT NULL AND ${copilotAsyncToolCalls.executionDeadlineAt} < clock_timestamp() + ${SIM_TOOL_RERUN_MIN_REMAINING_MS} * interval '1 millisecond')` +/** + * A call whose deadline leaves less than the minimum start budget, so nothing may start it: no + * re-run and no claim. A call without a deadline never matches. + */ +const tooLateToStart = sql`(${copilotAsyncToolCalls.executionDeadlineAt} IS NOT NULL AND ${copilotAsyncToolCalls.executionDeadlineAt} < clock_timestamp() + ${SIM_TOOL_MIN_START_BUDGET_MS} * interval '1 millisecond')` /** * A Sim execution whose owner died before recording a result: its lease lapsed, which a live @@ -903,7 +906,7 @@ const interruptedSimExecution = allOf( unfinishedSimExecution, sql`${copilotAsyncToolCalls.executionLeaseExpiresAt} <= clock_timestamp()`, lt(copilotAsyncToolCalls.executionAttempts, SIM_TOOL_EXECUTION_MAX_ATTEMPTS), - not(tooLateToRerun) + not(tooLateToStart) ) /** @@ -1030,6 +1033,7 @@ export async function claimDesktopToolCall( and( thisCall, eq(copilotAsyncToolCalls.status, ASYNC_TOOL_STATUS.pending), + not(tooLateToStart), // The executor takes only a call Sim offered it, inside its pickup window; once the // window closed the call belongs to its not-started settlement. executor @@ -1135,7 +1139,7 @@ export async function getChatViewDesktopLeaseRemainingMs( * Why a lapsed execution ended without a result: once, after every re-run it was allowed, or * with too little of its deadline left to run again. */ -const interruptionMessage = sql`CASE WHEN ${copilotAsyncToolCalls.executionAttempts} >= ${SIM_TOOL_EXECUTION_MAX_ATTEMPTS} THEN ${EXHAUSTED_SIM_TOOL_MESSAGE}::text WHEN ${tooLateToRerun} THEN ${DEADLINE_SIM_TOOL_MESSAGE}::text ELSE ${INTERRUPTED_SIM_TOOL_MESSAGE}::text END` +const interruptionMessage = sql`CASE WHEN ${copilotAsyncToolCalls.executionAttempts} >= ${SIM_TOOL_EXECUTION_MAX_ATTEMPTS} THEN ${EXHAUSTED_SIM_TOOL_MESSAGE}::text WHEN ${tooLateToStart} THEN ${DEADLINE_SIM_TOOL_MESSAGE}::text ELSE ${INTERRUPTED_SIM_TOOL_MESSAGE}::text END` /** * Whether one of the run's Sim handlers is still running with its execution lease held. Its @@ -1192,7 +1196,7 @@ export async function revokeExhaustedSimToolExecutions(runId: string) { eq(copilotAsyncToolCalls.runId, runId), or( gte(copilotAsyncToolCalls.executionAttempts, SIM_TOOL_EXECUTION_MAX_ATTEMPTS), - tooLateToRerun + tooLateToStart ) ) ) @@ -1776,6 +1780,7 @@ export async function claimWorkflowToolExecution( .where( and( eq(copilotAsyncToolCalls.toolCallId, toolCallId), + not(tooLateToStart), or( and( isNull(copilotAsyncToolCalls.claimedBy), diff --git a/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts b/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts index d2e0f593e55..2a954363179 100644 --- a/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts +++ b/apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts @@ -549,6 +549,32 @@ describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', () } }) + it.each([ + [30, false], + [600, true], + ])( + 'a browser claims a workflow call with %is of its deadline left: %s', + async (seconds, claimable) => { + const { runId, toolCallId } = await runAbandonedMidTool({ toolName: 'run_workflow' }) + /** Offered to a browser and not yet picked up, with `seconds` of its deadline left. */ + await db + .update(copilotAsyncToolCalls) + .set({ + claimedBy: null, + executionStartedAt: null, + executionOwnerToken: null, + executionDeadlineAt: sql`clock_timestamp() + ${seconds} * interval '1 second'`, + }) + .where(eq(copilotAsyncToolCalls.toolCallId, toolCallId)) + try { + const claim = await claimWorkflowToolExecution(toolCallId, generateId(), 'client') + expect(claim !== null).toBe(claimable) + } finally { + await db.update(copilotRuns).set({ status: 'cancelled' }).where(eq(copilotRuns.id, runId)) + } + } + ) + it('fails an exhausted workflow run before a takeover offers it to anyone', async () => { const { runId, toolCallId } = await runAbandonedMidTool({ toolName: 'run_workflow', diff --git a/apps/sim/lib/mothership/request/handlers/handlers.test.ts b/apps/sim/lib/mothership/request/handlers/handlers.test.ts index 014bbf6d88a..358d6e4cc9c 100644 --- a/apps/sim/lib/mothership/request/handlers/handlers.test.ts +++ b/apps/sim/lib/mothership/request/handlers/handlers.test.ts @@ -775,6 +775,37 @@ describe('sse-handlers tool lifecycle', () => { expectFailedWith(onEvent, TOOL_DEADLINE_BEFORE_START_MESSAGE) }) + it('does not offer an unclaimed client call with less than a minute of its deadline left', async () => { + mothershipAsyncRunsMockFns.mockCompletePendingAsyncToolCall.mockImplementationOnce( + async (input) => ({ ...input }) + ) + const onEvent = await runLocalRead(TOOL_DEADLINE_SETTLE_MARGIN_MS + 30_000) + await Promise.allSettled(context.pendingToolPromises.values()) + + expect(waitForClientToolCompletion).not.toHaveBeenCalled() + expectFailedWith(onEvent, TOOL_DEADLINE_BEFORE_START_MESSAGE) + }) + + it('settles a client call nothing claimed by its deadline as not run', async () => { + vi.useFakeTimers() + try { + waitForClientToolCompletion.mockImplementationOnce( + () => new Promise((resolve) => setTimeout(() => resolve(null), 65_000)) + ) + mothershipAsyncRunsMockFns.mockCompletePendingAsyncToolCall.mockImplementationOnce( + async (input) => ({ ...input }) + ) + const onEvent = await runLocalRead(TOOL_DEADLINE_SETTLE_MARGIN_MS + 65_000) + await vi.advanceTimersByTimeAsync(65_000) + await Promise.allSettled(context.pendingToolPromises.values()) + + expect(waitForClientToolCompletion).toHaveBeenCalledOnce() + expectFailedWith(onEvent, TOOL_DEADLINE_BEFORE_START_MESSAGE) + } finally { + vi.useRealTimers() + } + }) + it('reads a claimed call past its deadline and reports its outcome as unknown', async () => { const onEvent = await runLocalRead(TOOL_DEADLINE_SETTLE_MARGIN_MS) await Promise.allSettled(context.pendingToolPromises.values()) diff --git a/apps/sim/lib/mothership/request/handlers/tool.ts b/apps/sim/lib/mothership/request/handlers/tool.ts index b0ed204f692..533cee21dc5 100644 --- a/apps/sim/lib/mothership/request/handlers/tool.ts +++ b/apps/sim/lib/mothership/request/handlers/tool.ts @@ -41,6 +41,7 @@ import { TOOL_DEADLINE_EXCEEDED_MESSAGE, toDeadlineDate, toolDeadlinePassed, + toolStartWindowClosed, } from '@/lib/mothership/request/tools/deadline' import { waitForDesktopToolCall } from '@/lib/mothership/request/tools/desktop-wait' import { @@ -1000,7 +1001,7 @@ async function dispatchToolExecution( // provably never ran; a claimed one (a takeover re-hands calls) is read, then settled // below as outcome unknown. if ( - toolDeadlinePassed(toolCall.deadlineAt, Date.now()) && + toolStartWindowClosed(toolCall.deadlineAt, Date.now()) && (await failUnstartedToolCall( toolCallId, context, @@ -1079,13 +1080,22 @@ async function dispatchToolExecution( } span.setAttribute(TraceAttr.ToolExecutor, MothershipStreamV1ToolExecutor.client) if (completion === null && toolDeadlinePassed(toolCall.deadlineAt, Date.now())) { - // Settled durably, so a desktop that claimed the call stops renewing it. - await failPendingToolCall( - toolCallId, - context, - execContext, - TOOL_DEADLINE_EXCEEDED_MESSAGE + // A call nothing claimed never ran. A claimed one is settled durably as outcome + // unknown, so a desktop that claimed it stops renewing it. + if ( + !(await failUnstartedToolCall( + toolCallId, + context, + execContext, + TOOL_DEADLINE_BEFORE_START_MESSAGE + )) ) + await failPendingToolCall( + toolCallId, + context, + execContext, + TOOL_DEADLINE_EXCEEDED_MESSAGE + ) completion = settledClientCompletion(toolCall) } // Both waiters resolve `T | null`, never undefined — comparing against diff --git a/apps/sim/lib/mothership/request/tools/deadline.ts b/apps/sim/lib/mothership/request/tools/deadline.ts index ad95cf31130..4a013afdcc8 100644 --- a/apps/sim/lib/mothership/request/tools/deadline.ts +++ b/apps/sim/lib/mothership/request/tools/deadline.ts @@ -1,3 +1,4 @@ +import { SIM_TOOL_MIN_START_BUDGET_MS } from '@/lib/mothership/async-runs/execution-lease' import { TOOL_DEADLINE_SETTLE_MARGIN_MS } from '@/lib/mothership/generated/protocol' /** The longest worker wait a frame may name; a larger or malformed value is ignored. */ @@ -27,14 +28,28 @@ export const TOOL_DEADLINE_BEFORE_START_MESSAGE = export const TOOL_DEADLINE_EXCEEDED_MESSAGE = 'This tool call did not finish within its time limit and was abandoned. Its outcome is unknown: it may already have taken effect, so inspect the current state before repeating it.' -/** - * Whether the call's deadline has passed, so nothing more may start for it: no handler, approval - * wait, server fallback or allowed decision. A call without a deadline never passes one. - */ +/** Whether the call's deadline has passed. A call without a deadline never passes one. */ export function toolDeadlinePassed(deadlineAt: number | undefined, now: number): boolean { return deadlineAt !== undefined && now >= deadlineAt } +/** + * The last moment anything may start for a call: its deadline less the minimum start budget, + * since work started later would likely be abandoned mid-flight, its effects unknown. + */ +export function lastToolStartAt(deadlineAt: number | undefined): number | undefined { + return deadlineAt === undefined ? undefined : deadlineAt - SIM_TOOL_MIN_START_BUDGET_MS +} + +/** + * Whether it is too late to start anything for the call: no handler, approval prompt, allowed + * decision, server fallback or client pickup. Exactly the minimum budget left still starts. + */ +export function toolStartWindowClosed(deadlineAt: number | undefined, now: number): boolean { + const lastStartAt = lastToolStartAt(deadlineAt) + return lastStartAt !== undefined && now > lastStartAt +} + /** The column value for a call's deadline. */ export function toDeadlineDate(deadlineAt: number | undefined): Date | undefined { return deadlineAt === undefined ? undefined : new Date(deadlineAt) diff --git a/apps/sim/lib/mothership/request/tools/executor.test.ts b/apps/sim/lib/mothership/request/tools/executor.test.ts index fa7c939e93c..3d7406e0c1e 100644 --- a/apps/sim/lib/mothership/request/tools/executor.test.ts +++ b/apps/sim/lib/mothership/request/tools/executor.test.ts @@ -706,7 +706,7 @@ describe('executeToolAndReport provenance isolation', () => { return new Promise(() => {}) }) upsertAsyncToolCall.mockResolvedValueOnce({ - executionDeadlineAt: new Date(Date.now() + 30_000), + executionDeadlineAt: new Date(Date.now() + 90_000), }) const tool: ToolCallState = { ...buildPendingToolCall(), @@ -718,12 +718,47 @@ describe('executeToolAndReport provenance isolation', () => { workflowId: 'workflow-1', }) - await vi.advanceTimersByTimeAsync(30_000) + await vi.advanceTimersByTimeAsync(90_000) expect(signal?.aborted).toBe(true) expect((await pending).status).toBe('error') }) + it('starts a handler with exactly the minimum budget of its deadline left', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-10-15T12:00:00Z')) + executeTool.mockResolvedValue({ success: true, output: {} }) + const tool: ToolCallState = { + ...buildPendingToolCall(), + name: 'generate_audio', + deadlineAt: Date.now() + 60_000, + } + await executeToolAndReport(tool.id, buildStreamingContext(tool), { + userId: 'user-1', + workflowId: 'workflow-1', + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), + }) + + expect(executeTool).toHaveBeenCalledOnce() + }) + + it('does not start a handler with less than a minute of its deadline left', async () => { + const tool: ToolCallState = { + ...buildPendingToolCall(), + name: 'generate_audio', + deadlineAt: Date.now() + 30_000, + } + const completion = await executeToolAndReport(tool.id, buildStreamingContext(tool), { + userId: 'user-1', + workflowId: 'workflow-1', + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), + }) + + expect(executeTool).not.toHaveBeenCalled() + expect(completion.status).toBe('error') + expect(tool.error).toBe(TOOL_DEADLINE_BEFORE_START_MESSAGE) + }) + it('aborts a timed-out handler without cancelling a parallel tool', async () => { vi.useFakeTimers() const parent = new AbortController() diff --git a/apps/sim/lib/mothership/request/tools/executor.ts b/apps/sim/lib/mothership/request/tools/executor.ts index 67c9046f254..60f10f7662e 100644 --- a/apps/sim/lib/mothership/request/tools/executor.ts +++ b/apps/sim/lib/mothership/request/tools/executor.ts @@ -66,7 +66,7 @@ import { capToToolDeadline, TOOL_DEADLINE_BEFORE_START_MESSAGE, toDeadlineDate, - toolDeadlinePassed, + toolStartWindowClosed, } from '@/lib/mothership/request/tools/deadline' import { desktopToolNotStarted } from '@/lib/mothership/request/tools/desktop-wait' import { @@ -376,8 +376,8 @@ async function executeToolWithWatchdog( // execution always dispatches on the model's real tool name. const executableName = toolCall.execName ?? toolCall.name const now = Date.now() - // Admission or approval can spend the whole deadline; the handler then never starts. - if (toolDeadlinePassed(toolCall.deadlineAt, now)) + // Admission or approval can leave too little time to finish; the handler then never starts. + if (toolStartWindowClosed(toolCall.deadlineAt, now)) throw new Error(TOOL_DEADLINE_BEFORE_START_MESSAGE) const timeoutMs = capToToolDeadline( toolWatchdogTimeoutMs(executableName), diff --git a/apps/sim/lib/mothership/request/tools/permission.test.ts b/apps/sim/lib/mothership/request/tools/permission.test.ts index 903e5b10b72..e5d74997645 100644 --- a/apps/sim/lib/mothership/request/tools/permission.test.ts +++ b/apps/sim/lib/mothership/request/tools/permission.test.ts @@ -22,6 +22,7 @@ vi.mock('@/lib/mothership/persistence/tool-permission', async (importOriginal) = import { MothershipStreamV1ToolExecutor } from '@/lib/mothership/generated/mothership-stream-v1' import { createStreamingContext } from '@/lib/mothership/request/context/request-context' +import { TOOL_DEADLINE_BEFORE_START_MESSAGE } from '@/lib/mothership/request/tools/deadline' import { runGatedToolExecution, toolCallNeedsApproval, @@ -236,6 +237,28 @@ describe('runGatedToolExecution', () => { } }) + it('does not honor an approval that leaves less than a minute to run the call', async () => { + vi.useFakeTimers() + try { + waitForToolPermissionDecision.mockImplementation( + () => + new Promise((resolve) => + setTimeout(() => resolve({ toolCallId: 'call-1', decision: 'allow' }), 40_000) + ) + ) + const toolCall = { ...makeToolCall(), deadlineAt: Date.now() + 90_000 } + const execute = vi.fn() + const pending = gate(makeContext(), toolCall, execute, []) + + await vi.advanceTimersByTimeAsync(40_000) + + expect(await pending).toMatchObject({ status: 'error' }) + expect(execute).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + it('stops waiting for an answer at the deadline the worker named for the call', async () => { vi.useFakeTimers() try { @@ -254,7 +277,7 @@ describe('runGatedToolExecution', () => { expect(settled).toBe(true) expect(await pending).toMatchObject({ status: 'error' }) - expect(toolCall.error).toBe('Timed out waiting for the user to approve this tool.') + expect(toolCall.error).toBe(TOOL_DEADLINE_BEFORE_START_MESSAGE) expect(execute).not.toHaveBeenCalled() } finally { vi.useRealTimers() diff --git a/apps/sim/lib/mothership/request/tools/permission.ts b/apps/sim/lib/mothership/request/tools/permission.ts index 98db4021487..a62c965ff86 100644 --- a/apps/sim/lib/mothership/request/tools/permission.ts +++ b/apps/sim/lib/mothership/request/tools/permission.ts @@ -21,7 +21,13 @@ import { import { withCopilotSpan } from '@/lib/mothership/request/otel' import { markToolResultSeen } from '@/lib/mothership/request/sse-utils' import { setTerminalToolCallState } from '@/lib/mothership/request/tool-call-state' -import { capToToolDeadline, toolDeadlinePassed } from '@/lib/mothership/request/tools/deadline' +import { + capToToolDeadline, + lastToolStartAt, + TOOL_DEADLINE_BEFORE_START_MESSAGE, + toolDeadlinePassed, + toolStartWindowClosed, +} from '@/lib/mothership/request/tools/deadline' import type { OrchestratorOptions, StreamingContext, @@ -247,15 +253,19 @@ export function runGatedToolExecution( return { status: MothershipStreamV1ToolOutcome.success, message: output.message } } - // No prompt opens past the call's deadline, and an answer that lands after it is not honored. - const answer = toolDeadlinePassed(toolCall.deadlineAt, Date.now()) + // No prompt opens once it is too late to start the call, and an answer after that is not honored. + const answer = toolStartWindowClosed(toolCall.deadlineAt, Date.now()) ? null : await waitForToolPermissionDecision( toolCallId, - capToToolDeadline(PERMISSION_WAIT_TIMEOUT_MS, toolCall.deadlineAt, Date.now()), + capToToolDeadline( + PERMISSION_WAIT_TIMEOUT_MS, + lastToolStartAt(toolCall.deadlineAt), + Date.now() + ), options.abortSignal ) - const decision = toolDeadlinePassed(toolCall.deadlineAt, Date.now()) ? null : answer + const decision = toolStartWindowClosed(toolCall.deadlineAt, Date.now()) ? null : answer if (!decision) { // Timed out or the turn was stopped. Fail the call rather than running @@ -264,7 +274,9 @@ export function runGatedToolExecution( span.setAttribute(TraceAttr.ToolOutcome, MothershipStreamV1ToolOutcome.cancelled) const error = options.abortSignal?.aborted ? 'Stopped before the user approved this tool.' - : 'Timed out waiting for the user to approve this tool.' + : toolDeadlinePassed(lastToolStartAt(toolCall.deadlineAt), Date.now()) + ? TOOL_DEADLINE_BEFORE_START_MESSAGE + : 'Timed out waiting for the user to approve this tool.' setTerminalToolCallState(toolCall, { status: MothershipStreamV1ToolOutcome.cancelled, output: { error }, diff --git a/apps/sim/lib/mothership/request/tools/workflow-client-fallback.test.ts b/apps/sim/lib/mothership/request/tools/workflow-client-fallback.test.ts index 3a85cd92a53..ec29de68d56 100644 --- a/apps/sim/lib/mothership/request/tools/workflow-client-fallback.test.ts +++ b/apps/sim/lib/mothership/request/tools/workflow-client-fallback.test.ts @@ -114,6 +114,17 @@ describe('raceWorkflowToolClientPickup', () => { expect(params.runOnServer).not.toHaveBeenCalled() }) + it('does not take a call over with less than a minute of its deadline left', async () => { + waitForWorkflowToolCompletion.mockResolvedValue(null) + claimWorkflowToolExecution.mockResolvedValue({ toolCallId: 'tool-1' }) + const params = baseParams({ timeoutMs: 0, deadlineAt: Date.now() + 30_000 }) + + await raceWorkflowToolClientPickup(params as never) + + expect(claimWorkflowToolExecution).not.toHaveBeenCalled() + expect(params.runOnServer).not.toHaveBeenCalled() + }) + it('claims immediately when the caller declared no client pickup (graceMs 0)', async () => { // ChatRequest.clientCapabilities without 'workflow-tool-pickup' resolves to graceMs 0 // at the dispatch site: same claim arbitration, no dead-air grace for a client that diff --git a/apps/sim/lib/mothership/request/tools/workflow-client-fallback.ts b/apps/sim/lib/mothership/request/tools/workflow-client-fallback.ts index 38528260bd4..0c346b84418 100644 --- a/apps/sim/lib/mothership/request/tools/workflow-client-fallback.ts +++ b/apps/sim/lib/mothership/request/tools/workflow-client-fallback.ts @@ -10,7 +10,7 @@ import { claimWorkflowToolExecution } from '@/lib/mothership/async-runs/reposito import { CopilotDegradedReason } from '@/lib/mothership/generated/trace-attribute-values-v1' import { recordDegraded } from '@/lib/mothership/request/metrics' import { waitForWorkflowToolCompletion } from '@/lib/mothership/request/tools/client' -import { toolDeadlinePassed } from '@/lib/mothership/request/tools/deadline' +import { toolStartWindowClosed } from '@/lib/mothership/request/tools/deadline' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const logger = createLogger('CopilotWorkflowClientFallback') @@ -98,8 +98,8 @@ export async function raceWorkflowToolClientPickup( return { winner: 'client', completion: first.completion } } - // Never claim work on a turn the user already stopped, or past the call's deadline. - if (abortSignal?.aborted || toolDeadlinePassed(params.deadlineAt, Date.now())) { + // Never claim work on a turn the user already stopped, or too late to start the call. + if (abortSignal?.aborted || toolStartWindowClosed(params.deadlineAt, Date.now())) { return { winner: 'client', completion: await clientWait } } diff --git a/apps/sim/lib/mothership/tools/client/desktop-tool-authorization.integration.ts b/apps/sim/lib/mothership/tools/client/desktop-tool-authorization.integration.ts index 3e1afd3576f..5f9acf767cb 100644 --- a/apps/sim/lib/mothership/tools/client/desktop-tool-authorization.integration.ts +++ b/apps/sim/lib/mothership/tools/client/desktop-tool-authorization.integration.ts @@ -332,6 +332,20 @@ describe.runIf(Boolean(redisUrl))('desktop tool calls the server no longer admit expect(await storedCall(toolCallId)).toMatchObject({ status: 'pending', claimedBy: null }) }) + it('refuses the claim with less than a minute of the call deadline left', async () => { + const { runId } = await startRun() + const toolCallId = await agentCalls(runId, 'terminal', { operation: 'read' }) + await db + .update(copilotAsyncToolCalls) + .set({ executionDeadlineAt: new Date(Date.now() + 30_000) }) + .where(eq(copilotAsyncToolCalls.toolCallId, toolCallId)) + + expect( + await claimDesktopToolCall({ toolCallId, runId, userId, claimedBy: 'desktop-terminal' }) + ).toEqual({ outcome: 'existing' }) + expect(await storedCall(toolCallId)).toMatchObject({ status: 'pending', claimedBy: null }) + }) + it('wakes the waiting turn with what Stop did to a call the desktop already claimed', async () => { const { runId, streamId } = await startRun() const registry = new ResolvedSecretTraceRegistry([])