fix(mothership): start no tool call with less than a minute of its deadline left

apps/sim/lib/mothership/async-runs/execution-lease.ts

@@ -20,11 +20,11 @@ export const INTERRUPTED_SIM_TOOL_MESSAGE =
   'Tool execution lost its owner before a result was recorded. Its outcome is unknown; inspect current state before retrying a mutation.'
 
 /**
- * A takeover re-runs an interrupted call only with at least this much of its deadline left;
- * otherwise the call fails with {@link DEADLINE_SIM_TOOL_MESSAGE} instead of starting work
- * that could not finish in time.
+ * A call starts, the first time or again after a takeover, only with at least this much of its
+ * deadline left; otherwise it fails as not run instead of starting work that would likely be cut
+ * off with its outcome unknown. About 99% of non-workflow tool calls finish within a minute.
  */
-export const SIM_TOOL_RERUN_MIN_REMAINING_MS = 60_000
+export const SIM_TOOL_MIN_START_BUDGET_MS = 60_000
 
 export const DEADLINE_SIM_TOOL_MESSAGE =
   'Tool execution was interrupted because the Sim server running it stopped, and too little of its time remained to run it again. Its outcome is unknown; inspect current state before retrying a mutation.'

apps/sim/lib/mothership/async-runs/repository.ts

@@ -42,7 +42,7 @@ import {
   INTERRUPTED_SIM_TOOL_MESSAGE,
   SIM_TOOL_EXECUTION_LEASE_SECONDS,
   SIM_TOOL_EXECUTION_MAX_ATTEMPTS,
-  SIM_TOOL_RERUN_MIN_REMAINING_MS,
+  SIM_TOOL_MIN_START_BUDGET_MS,
   SimToolExecutionLeaseLostError,
   type SimToolExecutionOwner,
 } from '@/lib/mothership/async-runs/execution-lease'
@@ -891,8 +891,11 @@ const unfinishedSimExecution = allOf(
   isNull(copilotAsyncToolCalls.clientWorkflowExecutionId)
 )
 
-/** A call whose deadline leaves less than a re-run needs; a call without one never matches. */
-const tooLateToRerun = sql`(${copilotAsyncToolCalls.executionDeadlineAt} IS NOT NULL AND ${copilotAsyncToolCalls.executionDeadlineAt} < clock_timestamp() + ${SIM_TOOL_RERUN_MIN_REMAINING_MS} * interval '1 millisecond')`
+/**
+ * A call whose deadline leaves less than the minimum start budget, so nothing may start it: no
+ * re-run and no claim. A call without a deadline never matches.
+ */
+const tooLateToStart = sql`(${copilotAsyncToolCalls.executionDeadlineAt} IS NOT NULL AND ${copilotAsyncToolCalls.executionDeadlineAt} < clock_timestamp() + ${SIM_TOOL_MIN_START_BUDGET_MS} * interval '1 millisecond')`
 
 /**
  * A Sim execution whose owner died before recording a result: its lease lapsed, which a live
@@ -903,7 +906,7 @@ const interruptedSimExecution = allOf(
   unfinishedSimExecution,
   sql`${copilotAsyncToolCalls.executionLeaseExpiresAt} <= clock_timestamp()`,
   lt(copilotAsyncToolCalls.executionAttempts, SIM_TOOL_EXECUTION_MAX_ATTEMPTS),
-  not(tooLateToRerun)
+  not(tooLateToStart)
 )
 
 /**
@@ -1030,6 +1033,7 @@ export async function claimDesktopToolCall(
             and(
               thisCall,
               eq(copilotAsyncToolCalls.status, ASYNC_TOOL_STATUS.pending),
+              not(tooLateToStart),
               // The executor takes only a call Sim offered it, inside its pickup window; once the
               // window closed the call belongs to its not-started settlement.
               executor
@@ -1135,7 +1139,7 @@ export async function getChatViewDesktopLeaseRemainingMs(
  * Why a lapsed execution ended without a result: once, after every re-run it was allowed, or
  * with too little of its deadline left to run again.
  */
-const interruptionMessage = sql<string>`CASE WHEN ${copilotAsyncToolCalls.executionAttempts} >= ${SIM_TOOL_EXECUTION_MAX_ATTEMPTS} THEN ${EXHAUSTED_SIM_TOOL_MESSAGE}::text WHEN ${tooLateToRerun} THEN ${DEADLINE_SIM_TOOL_MESSAGE}::text ELSE ${INTERRUPTED_SIM_TOOL_MESSAGE}::text END`
+const interruptionMessage = sql<string>`CASE WHEN ${copilotAsyncToolCalls.executionAttempts} >= ${SIM_TOOL_EXECUTION_MAX_ATTEMPTS} THEN ${EXHAUSTED_SIM_TOOL_MESSAGE}::text WHEN ${tooLateToStart} THEN ${DEADLINE_SIM_TOOL_MESSAGE}::text ELSE ${INTERRUPTED_SIM_TOOL_MESSAGE}::text END`
 
 /**
  * Whether one of the run's Sim handlers is still running with its execution lease held. Its
@@ -1192,7 +1196,7 @@ export async function revokeExhaustedSimToolExecutions(runId: string) {
         eq(copilotAsyncToolCalls.runId, runId),
         or(
           gte(copilotAsyncToolCalls.executionAttempts, SIM_TOOL_EXECUTION_MAX_ATTEMPTS),
-          tooLateToRerun
+          tooLateToStart
         )
       )
     )
@@ -1776,6 +1780,7 @@ export async function claimWorkflowToolExecution(
           .where(
             and(
               eq(copilotAsyncToolCalls.toolCallId, toolCallId),
+              not(tooLateToStart),
               or(
                 and(
                   isNull(copilotAsyncToolCalls.claimedBy),

apps/sim/lib/mothership/request/application/abandoned-runs.integration.ts

@@ -549,6 +549,32 @@ describe.runIf(Boolean(redisUrl))('taking over a run whose controller died', ()
     }
   })
 
+  it.each([
+    [30, false],
+    [600, true],
+  ])(
+    'a browser claims a workflow call with %is of its deadline left: %s',
+    async (seconds, claimable) => {
+      const { runId, toolCallId } = await runAbandonedMidTool({ toolName: 'run_workflow' })
+      /** Offered to a browser and not yet picked up, with `seconds` of its deadline left. */
+      await db
+        .update(copilotAsyncToolCalls)
+        .set({
+          claimedBy: null,
+          executionStartedAt: null,
+          executionOwnerToken: null,
+          executionDeadlineAt: sql`clock_timestamp() + ${seconds} * interval '1 second'`,
+        })
+        .where(eq(copilotAsyncToolCalls.toolCallId, toolCallId))
+      try {
+        const claim = await claimWorkflowToolExecution(toolCallId, generateId(), 'client')
+        expect(claim !== null).toBe(claimable)
+      } finally {
+        await db.update(copilotRuns).set({ status: 'cancelled' }).where(eq(copilotRuns.id, runId))
+      }
+    }
+  )
+
   it('fails an exhausted workflow run before a takeover offers it to anyone', async () => {
     const { runId, toolCallId } = await runAbandonedMidTool({
       toolName: 'run_workflow',

apps/sim/lib/mothership/request/handlers/handlers.test.ts

@@ -775,6 +775,37 @@ describe('sse-handlers tool lifecycle', () => {
       expectFailedWith(onEvent, TOOL_DEADLINE_BEFORE_START_MESSAGE)
     })
 
+    it('does not offer an unclaimed client call with less than a minute of its deadline left', async () => {
+      mothershipAsyncRunsMockFns.mockCompletePendingAsyncToolCall.mockImplementationOnce(
+        async (input) => ({ ...input })
+      )
+      const onEvent = await runLocalRead(TOOL_DEADLINE_SETTLE_MARGIN_MS + 30_000)
+      await Promise.allSettled(context.pendingToolPromises.values())
+
+      expect(waitForClientToolCompletion).not.toHaveBeenCalled()
+      expectFailedWith(onEvent, TOOL_DEADLINE_BEFORE_START_MESSAGE)
+    })
+
+    it('settles a client call nothing claimed by its deadline as not run', async () => {
+      vi.useFakeTimers()
+      try {
+        waitForClientToolCompletion.mockImplementationOnce(
+          () => new Promise((resolve) => setTimeout(() => resolve(null), 65_000))
+        )
+        mothershipAsyncRunsMockFns.mockCompletePendingAsyncToolCall.mockImplementationOnce(
+          async (input) => ({ ...input })
+        )
+        const onEvent = await runLocalRead(TOOL_DEADLINE_SETTLE_MARGIN_MS + 65_000)
+        await vi.advanceTimersByTimeAsync(65_000)
+        await Promise.allSettled(context.pendingToolPromises.values())
+
+        expect(waitForClientToolCompletion).toHaveBeenCalledOnce()
+        expectFailedWith(onEvent, TOOL_DEADLINE_BEFORE_START_MESSAGE)
+      } finally {
+        vi.useRealTimers()
+      }
+    })
+
     it('reads a claimed call past its deadline and reports its outcome as unknown', async () => {
       const onEvent = await runLocalRead(TOOL_DEADLINE_SETTLE_MARGIN_MS)
       await Promise.allSettled(context.pendingToolPromises.values())

apps/sim/lib/mothership/request/handlers/tool.ts

@@ -41,6 +41,7 @@ import {
   TOOL_DEADLINE_EXCEEDED_MESSAGE,
   toDeadlineDate,
   toolDeadlinePassed,
+  toolStartWindowClosed,
 } from '@/lib/mothership/request/tools/deadline'
 import { waitForDesktopToolCall } from '@/lib/mothership/request/tools/desktop-wait'
 import {
@@ -1000,7 +1001,7 @@ async function dispatchToolExecution(
         // provably never ran; a claimed one (a takeover re-hands calls) is read, then settled
         // below as outcome unknown.
         if (
-          toolDeadlinePassed(toolCall.deadlineAt, Date.now()) &&
+          toolStartWindowClosed(toolCall.deadlineAt, Date.now()) &&
           (await failUnstartedToolCall(
             toolCallId,
             context,
@@ -1079,13 +1080,22 @@ async function dispatchToolExecution(
         }
         span.setAttribute(TraceAttr.ToolExecutor, MothershipStreamV1ToolExecutor.client)
         if (completion === null && toolDeadlinePassed(toolCall.deadlineAt, Date.now())) {
-          // Settled durably, so a desktop that claimed the call stops renewing it.
-          await failPendingToolCall(
-            toolCallId,
-            context,
-            execContext,
-            TOOL_DEADLINE_EXCEEDED_MESSAGE
+          // A call nothing claimed never ran. A claimed one is settled durably as outcome
+          // unknown, so a desktop that claimed it stops renewing it.
+          if (
+            !(await failUnstartedToolCall(
+              toolCallId,
+              context,
+              execContext,
+              TOOL_DEADLINE_BEFORE_START_MESSAGE
+            ))
           )
+            await failPendingToolCall(
+              toolCallId,
+              context,
+              execContext,
+              TOOL_DEADLINE_EXCEEDED_MESSAGE
+            )
           completion = settledClientCompletion(toolCall)
         }
         // Both waiters resolve `T | null`, never undefined — comparing against

apps/sim/lib/mothership/request/tools/deadline.ts

@@ -1,3 +1,4 @@
+import { SIM_TOOL_MIN_START_BUDGET_MS } from '@/lib/mothership/async-runs/execution-lease'
 import { TOOL_DEADLINE_SETTLE_MARGIN_MS } from '@/lib/mothership/generated/protocol'
 
 /** The longest worker wait a frame may name; a larger or malformed value is ignored. */
@@ -27,14 +28,28 @@ export const TOOL_DEADLINE_BEFORE_START_MESSAGE =
 export const TOOL_DEADLINE_EXCEEDED_MESSAGE =
   'This tool call did not finish within its time limit and was abandoned. Its outcome is unknown: it may already have taken effect, so inspect the current state before repeating it.'
 
-/**
- * Whether the call's deadline has passed, so nothing more may start for it: no handler, approval
- * wait, server fallback or allowed decision. A call without a deadline never passes one.
- */
+/** Whether the call's deadline has passed. A call without a deadline never passes one. */
 export function toolDeadlinePassed(deadlineAt: number | undefined, now: number): boolean {
   return deadlineAt !== undefined && now >= deadlineAt
 }
 
+/**
+ * The last moment anything may start for a call: its deadline less the minimum start budget,
+ * since work started later would likely be abandoned mid-flight, its effects unknown.
+ */
+export function lastToolStartAt(deadlineAt: number | undefined): number | undefined {
+  return deadlineAt === undefined ? undefined : deadlineAt - SIM_TOOL_MIN_START_BUDGET_MS
+}
+
+/**
+ * Whether it is too late to start anything for the call: no handler, approval prompt, allowed
+ * decision, server fallback or client pickup. Exactly the minimum budget left still starts.
+ */
+export function toolStartWindowClosed(deadlineAt: number | undefined, now: number): boolean {
+  const lastStartAt = lastToolStartAt(deadlineAt)
+  return lastStartAt !== undefined && now > lastStartAt
+}
+
 /** The column value for a call's deadline. */
 export function toDeadlineDate(deadlineAt: number | undefined): Date | undefined {
   return deadlineAt === undefined ? undefined : new Date(deadlineAt)

apps/sim/lib/mothership/request/tools/executor.test.ts

@@ -706,7 +706,7 @@ describe('executeToolAndReport provenance isolation', () => {
       return new Promise(() => {})
     })
     upsertAsyncToolCall.mockResolvedValueOnce({
-      executionDeadlineAt: new Date(Date.now() + 30_000),
+      executionDeadlineAt: new Date(Date.now() + 90_000),
     })
     const tool: ToolCallState = {
       ...buildPendingToolCall(),
@@ -718,12 +718,47 @@ describe('executeToolAndReport provenance isolation', () => {
       workflowId: 'workflow-1',
     })
 
-    await vi.advanceTimersByTimeAsync(30_000)
+    await vi.advanceTimersByTimeAsync(90_000)
 
     expect(signal?.aborted).toBe(true)
     expect((await pending).status).toBe('error')
   })
 
+  it('starts a handler with exactly the minimum budget of its deadline left', async () => {
+    vi.useFakeTimers()
+    vi.setSystemTime(new Date('2026-10-15T12:00:00Z'))
+    executeTool.mockResolvedValue({ success: true, output: {} })
+    const tool: ToolCallState = {
+      ...buildPendingToolCall(),
+      name: 'generate_audio',
+      deadlineAt: Date.now() + 60_000,
+    }
+    await executeToolAndReport(tool.id, buildStreamingContext(tool), {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+      resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(),
+    })
+
+    expect(executeTool).toHaveBeenCalledOnce()
+  })
+
+  it('does not start a handler with less than a minute of its deadline left', async () => {
+    const tool: ToolCallState = {
+      ...buildPendingToolCall(),
+      name: 'generate_audio',
+      deadlineAt: Date.now() + 30_000,
+    }
+    const completion = await executeToolAndReport(tool.id, buildStreamingContext(tool), {
+      userId: 'user-1',
+      workflowId: 'workflow-1',
+      resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(),
+    })
+
+    expect(executeTool).not.toHaveBeenCalled()
+    expect(completion.status).toBe('error')
+    expect(tool.error).toBe(TOOL_DEADLINE_BEFORE_START_MESSAGE)
+  })
+
   it('aborts a timed-out handler without cancelling a parallel tool', async () => {
     vi.useFakeTimers()
     const parent = new AbortController()

apps/sim/lib/mothership/request/tools/executor.ts

@@ -66,7 +66,7 @@ import {
   capToToolDeadline,
   TOOL_DEADLINE_BEFORE_START_MESSAGE,
   toDeadlineDate,
-  toolDeadlinePassed,
+  toolStartWindowClosed,
 } from '@/lib/mothership/request/tools/deadline'
 import { desktopToolNotStarted } from '@/lib/mothership/request/tools/desktop-wait'
 import {
@@ -376,8 +376,8 @@ async function executeToolWithWatchdog(
   // execution always dispatches on the model's real tool name.
   const executableName = toolCall.execName ?? toolCall.name
   const now = Date.now()
-  // Admission or approval can spend the whole deadline; the handler then never starts.
-  if (toolDeadlinePassed(toolCall.deadlineAt, now))
+  // Admission or approval can leave too little time to finish; the handler then never starts.
+  if (toolStartWindowClosed(toolCall.deadlineAt, now))
     throw new Error(TOOL_DEADLINE_BEFORE_START_MESSAGE)
   const timeoutMs = capToToolDeadline(
     toolWatchdogTimeoutMs(executableName),

apps/sim/lib/mothership/request/tools/permission.test.ts

@@ -22,6 +22,7 @@ vi.mock('@/lib/mothership/persistence/tool-permission', async (importOriginal) =
 
 import { MothershipStreamV1ToolExecutor } from '@/lib/mothership/generated/mothership-stream-v1'
 import { createStreamingContext } from '@/lib/mothership/request/context/request-context'
+import { TOOL_DEADLINE_BEFORE_START_MESSAGE } from '@/lib/mothership/request/tools/deadline'
 import {
   runGatedToolExecution,
   toolCallNeedsApproval,
@@ -236,6 +237,28 @@ describe('runGatedToolExecution', () => {
     }
   })
 
+  it('does not honor an approval that leaves less than a minute to run the call', async () => {
+    vi.useFakeTimers()
+    try {
+      waitForToolPermissionDecision.mockImplementation(
+        () =>
+          new Promise((resolve) =>
+            setTimeout(() => resolve({ toolCallId: 'call-1', decision: 'allow' }), 40_000)
+          )
+      )
+      const toolCall = { ...makeToolCall(), deadlineAt: Date.now() + 90_000 }
+      const execute = vi.fn()
+      const pending = gate(makeContext(), toolCall, execute, [])
+
+      await vi.advanceTimersByTimeAsync(40_000)
+
+      expect(await pending).toMatchObject({ status: 'error' })
+      expect(execute).not.toHaveBeenCalled()
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('stops waiting for an answer at the deadline the worker named for the call', async () => {
     vi.useFakeTimers()
     try {
@@ -254,7 +277,7 @@ describe('runGatedToolExecution', () => {
 
       expect(settled).toBe(true)
       expect(await pending).toMatchObject({ status: 'error' })
-      expect(toolCall.error).toBe('Timed out waiting for the user to approve this tool.')
+      expect(toolCall.error).toBe(TOOL_DEADLINE_BEFORE_START_MESSAGE)
       expect(execute).not.toHaveBeenCalled()
     } finally {
       vi.useRealTimers()