From a4e28b0f48901b6ec638d0025611cfd22796924e Mon Sep 17 00:00:00 2001 From: Ben Reitz Date: Mon, 2 Nov 2026 10:18:14 +0000 Subject: [PATCH] computer: Let the exec tool start runs with metadata A run's metadata tells host modules who the run is for, but the exec tool started every run without any, and the AI SDK and TanStack tools passed only the abort signal on, so an app had no way to supply it. `createExecTool` now takes `metadata` and every tool set takes `execMetadata`: a record of strings, or a function that picks one from the tool call's id, which each agent library passes on. A throw from the function fails the call before anything runs. A pi caller runs each call itself, so it can pass `metadata` on the context it hands `execute` instead, and metadata passed with a call wins, since the caller that passed it knows that call best. --- .changeset/exec-tool-metadata.md | 5 ++ docs/09_tool_interface.md | 18 +++- .../computer/src/tools/ai-sdk/index.test.ts | 47 ++++++++++ packages/computer/src/tools/ai-sdk/tools.ts | 3 +- .../computer/src/tools/common/exec.test.ts | 85 +++++++++++++++++++ packages/computer/src/tools/common/exec.ts | 28 +++++- packages/computer/src/tools/common/options.ts | 9 +- packages/computer/src/tools/core.ts | 1 + packages/computer/src/tools/index.ts | 1 + .../computer/src/tools/pi-ai/index.test.ts | 26 ++++++ packages/computer/src/tools/pi-ai/index.ts | 14 ++- .../src/tools/tanstack-ai/index.test.ts | 25 ++++++ .../computer/src/tools/tanstack-ai/index.ts | 5 +- 13 files changed, 259 insertions(+), 8 deletions(-) create mode 100644 .changeset/exec-tool-metadata.md create mode 100644 packages/computer/src/tools/common/exec.test.ts diff --git a/.changeset/exec-tool-metadata.md b/.changeset/exec-tool-metadata.md new file mode 100644 index 000000000..e78787dbf --- /dev/null +++ b/.changeset/exec-tool-metadata.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": minor +--- + +The `exec` tool can start each run with `metadata` for host modules. `createExecTool` takes `metadata` and the tool sets take `execMetadata`: a record of strings, or a function that picks one from the tool call's id. A pi caller can pass `metadata` on the context it hands `execute` instead, which wins. diff --git a/docs/09_tool_interface.md b/docs/09_tool_interface.md index bd2bfa970..d161248a7 100644 --- a/docs/09_tool_interface.md +++ b/docs/09_tool_interface.md @@ -176,6 +176,7 @@ createAITools({ edit?, exec?, execOutput?, + execMetadata?, root?, }); ``` @@ -191,6 +192,7 @@ createAITools({ | `edit` | default caps | Options passed to `createEditTool`. | | `exec` | every backend | An `ExecBackends` map from backend id to `ExecBackendOptions` (`{ description? }`). `{}` omits `exec`. Both types are exported from `@cloudflare/computer/tools`. | | `execOutput` | 2000 lines, 64 KiB | `{ maxLines?, maxBytes? }`: how much of each `exec` stream the model sees. See [`exec`](#exec). | +| `execMetadata` | none | The `metadata` each `exec` run starts with, or `({ toolCallId }) => metadata` to pick it per call. See [Run metadata](#run-metadata). | | `shell` | omitted | Deprecated, and ignored when `exec` is given. `{ backends }` becomes `exec: backends`; `defaultBackend` is ignored. | `createPiTools` and `createTanStackTools` take the same options, plus their own listed above. @@ -357,7 +359,7 @@ The tool uses forced removal, so deleting a missing path succeeds. Set `recursiv ## `exec` -`exec` calls `workspace.runtime.exec` on the chosen backend and streams bounded output. `createExecTool({ workspace, backends?, maxBytes?, maxLines? })` takes the same `backends` as the `exec` option, plus output limits. +`exec` calls `workspace.runtime.exec` on the chosen backend and streams bounded output. `createExecTool({ workspace, backends?, maxBytes?, maxLines?, metadata? })` takes the same `backends` as the `exec` option, plus output limits and [metadata](#run-metadata). Each backend's entry in the tool description joins two parts: your text, if any, and what the backend says about itself (`backend.description`, read through `workspace.runtime.backends()`). `WorkerJavaScriptBackend` describes its source language and every module code can import, so the list stays in step with `modules`. `WorkerShellBackend` and `ContainerBackend` describe their command sets, network access, and startup cost. A backend that says nothing gets a one-line default, so add text for a custom backend. @@ -384,6 +386,20 @@ line 3000 The model can open that file with `read` or search it with `grep`. `streamMaxBytes` is ignored; memory per stream stays within a few times `maxBytes`. +### Run metadata + +A run can carry `metadata`, a record of strings that host modules read and the model never sees (see [Host modules](./17_isolate_javascript.md#host-modules)). `metadata` on `createExecTool`, or `execMetadata` on a tool set, is either the record itself or a function that picks it from the tool call's id, which each agent library passes. A throw fails the call without running anything. + +```ts +// Inside an agent with one instance per conversation. +const tools = createAITools({ + workspace, + execMetadata: ({ toolCallId }) => ({ conversation: this.name, toolCall: toolCallId }), +}); +``` + +A pi caller runs each call itself, so it can instead pass the metadata with the call: `execute(call, { metadata })`. Metadata passed with a call wins over `execMetadata`. A caller of `defineExec` passes `metadata` or `toolCallId` on each `execute` the same way. + Wire this tool carefully: it executes arbitrary shell commands inside the configured backend. Treat its output as untrusted text when including it in later model input. Pass `exec: {}` or `readonly: true` when command execution is not part of the agent's job, and list backends explicitly when the Workspace has one the model should not use directly. ## `publish` diff --git a/packages/computer/src/tools/ai-sdk/index.test.ts b/packages/computer/src/tools/ai-sdk/index.test.ts index 1195dd90b..7cb29eaee 100644 --- a/packages/computer/src/tools/ai-sdk/index.test.ts +++ b/packages/computer/src/tools/ai-sdk/index.test.ts @@ -8,6 +8,7 @@ import { Workspace } from "../../workspace"; import { createDeleteTool, createEditTool, + createExecTool, createFindTool, createGrepTool, createReadTool, @@ -2280,6 +2281,52 @@ describe("createAITools exec streaming", () => { }); }); +describe("createAITools exec metadata", () => { + function recordingWorkspace(): { + workspace: Parameters[0]["workspace"]; + metadata: unknown[]; + } { + const metadata: unknown[] = []; + return { + metadata, + workspace: { + runtime: { + async exec(_command, options) { + metadata.push(options.metadata); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }, + backends: () => [{ id: "sh", protocol: "command", callable: false }], + }, + }, + }; + } + + it("picks each run's metadata from the AI SDK tool call's id", async () => { + const { workspace, metadata } = recordingWorkspace(); + const tool = createExecTool({ + workspace, + metadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await executeTool(tool, { command: "true" }); + + expect(metadata).toEqual([{ toolCall: "test-call" }]); + }); + + it("takes the metadata on the tool set", async () => { + const { workspace, metadata } = recordingWorkspace(); + const tools = createAITools({ + // SAFETY: The test only runs exec, which reads nothing but the runtime. + workspace: workspace as Parameters[0]["workspace"], + execMetadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await executeTool(tools.exec, { command: "true" }); + + expect(metadata).toEqual([{ toolCall: "test-call" }]); + }); +}); + describe("createAITools publish tool", () => { it("adds publish by default when assets are configured", async () => { const calls: Array<{ path: string; expiresAfter: number; prefix?: string }> = []; diff --git a/packages/computer/src/tools/ai-sdk/tools.ts b/packages/computer/src/tools/ai-sdk/tools.ts index 1278950d5..14a0d4b09 100644 --- a/packages/computer/src/tools/ai-sdk/tools.ts +++ b/packages/computer/src/tools/ai-sdk/tools.ts @@ -125,7 +125,8 @@ export function createExecTool(options: ExecToolOptions): Tool exec.execute(input, { abortSignal }), + execute: (input, { abortSignal, toolCallId }) => + exec.execute(input, { abortSignal, toolCallId }), }); } diff --git a/packages/computer/src/tools/common/exec.test.ts b/packages/computer/src/tools/common/exec.test.ts new file mode 100644 index 000000000..6ffeefac3 --- /dev/null +++ b/packages/computer/src/tools/common/exec.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { defineExec, type ExecToolOutput, type ExecWorkspaceLike } from "./exec"; + +// A runtime that records the metadata each run was started with. +function recordingWorkspace(): { workspace: ExecWorkspaceLike; metadata: unknown[] } { + const metadata: unknown[] = []; + const workspace: ExecWorkspaceLike = { + runtime: { + async exec(_command, options) { + metadata.push(options.metadata); + return { + result: async () => ({ exitCode: 0, stdout: "", stderr: "" }), + }; + }, + backends: () => [{ id: "sh", protocol: "command", callable: false }], + }, + }; + return { workspace, metadata }; +} + +async function last(output: AsyncIterable): Promise { + let value: ExecToolOutput | undefined; + for await (const chunk of output) value = chunk; + return value; +} + +describe("defineExec metadata", () => { + it("starts each run with the metadata it was given", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ workspace, metadata: { conversation: "c1" } }); + + await last(exec.execute({ command: "true" })); + + expect(metadata).toEqual([{ conversation: "c1" }]); + }); + + it("picks each run's metadata from its tool call's id", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ + workspace, + metadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await last(exec.execute({ command: "true" }, { toolCallId: "call-1" })); + await last(exec.execute({ command: "true" }, { toolCallId: "call-2" })); + + expect(metadata).toEqual([{ toolCall: "call-1" }, { toolCall: "call-2" }]); + }); + + it("uses the metadata a call passes instead", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ workspace, metadata: { conversation: "configured" } }); + + await last(exec.execute({ command: "true" }, { metadata: { conversation: "passed" } })); + + expect(metadata).toEqual([{ conversation: "passed" }]); + }); + + it("starts runs without metadata when none is given", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ workspace }); + + await last(exec.execute({ command: "true" })); + + expect(metadata).toEqual([undefined]); + }); + + it("reports a failure to pick metadata as the call's error, without running", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ + workspace, + metadata: () => { + throw new Error("no conversation"); + }, + }); + + await expect(last(exec.execute({ command: "true" }))).resolves.toEqual({ + command: "true", + cwd: null, + backend: "sh", + error: "no conversation", + }); + expect(metadata).toEqual([]); + }); +}); diff --git a/packages/computer/src/tools/common/exec.ts b/packages/computer/src/tools/common/exec.ts index 437611efa..45673ab2c 100644 --- a/packages/computer/src/tools/common/exec.ts +++ b/packages/computer/src/tools/common/exec.ts @@ -65,6 +65,7 @@ export interface ExecWorkspaceLike { backend?: string; env?: Record; input?: WorkspaceRuntimeValue; + metadata?: Record; output?: { maxBytes: number; maxLines: number }; }, ): Promise; @@ -91,6 +92,15 @@ export interface ExecBackendOptions { */ export type ExecBackends = Readonly>; +/** + * The `metadata` each exec run starts with, for host modules; the model + * never sees it. A function picks it per tool call. A throw fails the + * call without running it. + */ +export type ExecMetadata = + | Record + | ((call: { toolCallId: string }) => Record); + export interface ExecToolOptions { workspace: ExecWorkspaceLike; // Omit to offer every backend the Workspace has. With one backend @@ -109,6 +119,11 @@ export interface ExecToolOptions { * times `maxBytes`. */ streamMaxBytes?: number; + /** + * Each run's `metadata`, such as the conversation the tool call + * belongs to. Metadata a call passes itself, as a pi caller can, wins. + */ + metadata?: ExecMetadata; // Clock backing the running-snapshot coalescing floor. Defaults to // Date.now; injectable so tests can drive the interval deterministically. now?: () => number; @@ -145,6 +160,10 @@ export interface ExecInput { export interface ExecCallContext { abortSignal?: AbortSignal; + /** The tool call's id, handed to an `ExecToolOptions.metadata` function. */ + toolCallId?: string; + /** The run's metadata. When set, `ExecToolOptions.metadata` isn't used. */ + metadata?: Record; } /** The exec tool with no agent library attached. Each library wraps it in its own tool shape. */ @@ -230,7 +249,8 @@ export function defineExec(options: ExecToolOptions): ExecDefinition { description, inputSchema, prepareArguments: callableBackendIds.size > 0 ? parseInputObject : (args) => args, - execute: async function* ({ command, cwd, backend, env, input }, { abortSignal } = {}) { + execute: async function* ({ command, cwd, backend, env, input }, context = {}) { + const { abortSignal } = context; // With one backend there is nothing to choose. With several the // schema requires `backend`; a caller that skips the schema gets // the same answer as an error. @@ -246,12 +266,18 @@ export function defineExec(options: ExecToolOptions): ExecDefinition { } let handle: ExecRuntimeHandle; try { + const metadata = + context.metadata ?? + (typeof options.metadata === "function" + ? options.metadata({ toolCallId: context.toolCallId ?? "" }) + : options.metadata); handle = await options.workspace.runtime.exec(command, { cwd, encoding: "utf8", backend: selectedBackend, env, input, + ...(metadata === undefined ? {} : { metadata }), // The runtime cuts and saves by the same limits the tool // shows, so whatever the model does not see is in the file. output: limits, diff --git a/packages/computer/src/tools/common/options.ts b/packages/computer/src/tools/common/options.ts index 169f3f71e..90449d476 100644 --- a/packages/computer/src/tools/common/options.ts +++ b/packages/computer/src/tools/common/options.ts @@ -1,4 +1,4 @@ -import type { ExecBackends, ExecToolOptions, ExecWorkspaceLike } from "./exec"; +import type { ExecBackends, ExecMetadata, ExecToolOptions, ExecWorkspaceLike } from "./exec"; import { confineWorkspace } from "./fs/confine"; import type { EditToolOptions } from "./fs/edit"; import type { ReadToolOptions } from "./fs/read"; @@ -24,6 +24,12 @@ export interface CreateToolsOptions { */ exec?: ExecBackends; execOutput?: Pick; + /** + * The `metadata` each `exec` run starts with, or a function that picks + * it from the tool call's id. Host modules read it; the model never + * sees it. + */ + execMetadata?: ExecMetadata; /** * @deprecated Use `exec`. `{ backends }` becomes `exec: backends`; * `defaultBackend` is ignored, because the model names a backend @@ -81,6 +87,7 @@ function execOptions(options: CreateToolsOptions): ExecToolOptions | undefined { ...exec, ...(output?.maxBytes === undefined ? {} : { maxBytes: output.maxBytes }), ...(output?.maxLines === undefined ? {} : { maxLines: output.maxLines }), + ...(options.execMetadata === undefined ? {} : { metadata: options.execMetadata }), }; } diff --git a/packages/computer/src/tools/core.ts b/packages/computer/src/tools/core.ts index 3727711e1..a959be5b4 100644 --- a/packages/computer/src/tools/core.ts +++ b/packages/computer/src/tools/core.ts @@ -5,6 +5,7 @@ export { type ExecCallContext, type ExecDefinition, type ExecInput, + type ExecMetadata, type ExecRuntimeHandle, type ExecStreamEvent, type ExecToolOptions, diff --git a/packages/computer/src/tools/index.ts b/packages/computer/src/tools/index.ts index 8c0bba8cf..521b460b3 100644 --- a/packages/computer/src/tools/index.ts +++ b/packages/computer/src/tools/index.ts @@ -18,6 +18,7 @@ export { export type { ExecBackendOptions, ExecBackends, + ExecMetadata, ExecRuntimeHandle, ExecStreamEvent, ExecToolOptions, diff --git a/packages/computer/src/tools/pi-ai/index.test.ts b/packages/computer/src/tools/pi-ai/index.test.ts index 2cad3c135..8ad52809a 100644 --- a/packages/computer/src/tools/pi-ai/index.test.ts +++ b/packages/computer/src/tools/pi-ai/index.test.ts @@ -211,6 +211,32 @@ describe("createPiTools execution", () => { expect(parsed.entries[0].name).toBe("a.txt"); }); + it("starts exec with the metadata the caller passes for the call", async () => { + const workspace = makeWorkspace(); + const metadata: unknown[] = []; + (workspace.runtime as unknown as Record).exec = async ( + _command: string, + options: { metadata?: unknown }, + ) => { + metadata.push(options.metadata); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }; + fakeBackends(workspace, [{ id: "sh", protocol: "command", callable: false }]); + const tools = createPiTools({ + workspace, + exec: { sh: {} }, + execMetadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await tools.execute({ id: "1", name: "exec", arguments: { command: "true" } }); + await tools.execute( + { id: "2", name: "exec", arguments: { command: "true" } }, + { metadata: { conversation: "c1" } }, + ); + + expect(metadata).toEqual([{ toolCall: "1" }, { conversation: "c1" }]); + }); + it("returns the tool's own output as details", async () => { const workspace = makeWorkspace(); (workspace.runtime as unknown as Record).exec = async () => ({ diff --git a/packages/computer/src/tools/pi-ai/index.ts b/packages/computer/src/tools/pi-ai/index.ts index 4909ef0a9..b1510b522 100644 --- a/packages/computer/src/tools/pi-ai/index.ts +++ b/packages/computer/src/tools/pi-ai/index.ts @@ -40,15 +40,23 @@ export { export interface ToolCallContext { abortSignal?: AbortSignal; + /** + * The `metadata` an `exec` call's run starts with, for host modules; + * the model never sees it. When set, `execMetadata` isn't used. + */ + metadata?: Record; } +// What each tool's executor gets: the caller's context and the call's id. +type PiToolCallContext = ToolCallContext & { toolCallId: string }; + interface PiToolEntry { name: string; description: string; inputSchema: z.ZodType; strictArguments?: boolean; prepareArguments?: (args: unknown) => unknown; - execute: (input: never, context: ToolCallContext) => Promise | AsyncIterable; + execute: (input: never, context: PiToolCallContext) => Promise | AsyncIterable; toModelOutput?: (args: { input: never; output: never }) => ModelOutput; } @@ -248,13 +256,13 @@ function dispatcher( const run = entry.execute as ( i: unknown, - c: ToolCallContext, + c: PiToolCallContext, ) => Promise | AsyncIterable; const toOutput = entry.toModelOutput as | ((args: { input: unknown; output: unknown }) => ModelOutput) | undefined; try { - const output = await settle(run(parsed.data, context)); + const output = await settle(run(parsed.data, { ...context, toolCallId: call.id })); const modelOutput = toOutput ? toOutput({ input: parsed.data, output }) : defaultModelOutput(output); diff --git a/packages/computer/src/tools/tanstack-ai/index.test.ts b/packages/computer/src/tools/tanstack-ai/index.test.ts index 77f4091f6..a71439772 100644 --- a/packages/computer/src/tools/tanstack-ai/index.test.ts +++ b/packages/computer/src/tools/tanstack-ai/index.test.ts @@ -398,6 +398,31 @@ describe("createTanStackTools", () => { await workspace.close(); }); + it("picks each exec run's metadata from the TanStack tool call", async () => { + const workspace = makeWorkspace(); + const metadata: unknown[] = []; + (workspace.runtime as unknown as Record).exec = async ( + _command: string, + options: { metadata?: unknown }, + ) => { + metadata.push(options.metadata); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }; + (workspace.runtime as unknown as Record).backends = () => [ + { id: "sh", protocol: "command", callable: false }, + ]; + const tools = createTanStackTools({ + workspace, + format: "object", + execMetadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await tools.exec.execute({ command: "true" } as never, { toolCallId: "call-1" }); + + expect(metadata).toEqual([{ toolCall: "call-1" }]); + await workspace.close(); + }); + it("kills exec when the chat run's abort signal fires", async () => { const quiet = quietCommandBackend(); const workspace = new Workspace({ diff --git a/packages/computer/src/tools/tanstack-ai/index.ts b/packages/computer/src/tools/tanstack-ai/index.ts index 4ab92161d..44c50812e 100644 --- a/packages/computer/src/tools/tanstack-ai/index.ts +++ b/packages/computer/src/tools/tanstack-ai/index.ts @@ -185,7 +185,10 @@ export function createTanStackTools mutates: true, streams: true, run: async (input: never, context?: TanStackToolExecutionContext) => { - const returned = exec.execute(input, { abortSignal: context?.abortSignal }); + const returned = exec.execute(input, { + abortSignal: context?.abortSignal, + toolCallId: context?.toolCallId, + }); const output = options.streamEventName ? await settleWithEvents(returned, options.streamEventName, context) : await settle(returned);