diff --git a/.changeset/exec-tool-metadata.md b/.changeset/exec-tool-metadata.md new file mode 100644 index 000000000..e78787dbf --- /dev/null +++ b/.changeset/exec-tool-metadata.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": minor +--- + +The `exec` tool can start each run with `metadata` for host modules. `createExecTool` takes `metadata` and the tool sets take `execMetadata`: a record of strings, or a function that picks one from the tool call's id. A pi caller can pass `metadata` on the context it hands `execute` instead, which wins. diff --git a/docs/09_tool_interface.md b/docs/09_tool_interface.md index bd2bfa970..d161248a7 100644 --- a/docs/09_tool_interface.md +++ b/docs/09_tool_interface.md @@ -176,6 +176,7 @@ createAITools({ edit?, exec?, execOutput?, + execMetadata?, root?, }); ``` @@ -191,6 +192,7 @@ createAITools({ | `edit` | default caps | Options passed to `createEditTool`. | | `exec` | every backend | An `ExecBackends` map from backend id to `ExecBackendOptions` (`{ description? }`). `{}` omits `exec`. Both types are exported from `@cloudflare/computer/tools`. | | `execOutput` | 2000 lines, 64 KiB | `{ maxLines?, maxBytes? }`: how much of each `exec` stream the model sees. See [`exec`](#exec). | +| `execMetadata` | none | The `metadata` each `exec` run starts with, or `({ toolCallId }) => metadata` to pick it per call. See [Run metadata](#run-metadata). | | `shell` | omitted | Deprecated, and ignored when `exec` is given. `{ backends }` becomes `exec: backends`; `defaultBackend` is ignored. | `createPiTools` and `createTanStackTools` take the same options, plus their own listed above. @@ -357,7 +359,7 @@ The tool uses forced removal, so deleting a missing path succeeds. Set `recursiv ## `exec` -`exec` calls `workspace.runtime.exec` on the chosen backend and streams bounded output. `createExecTool({ workspace, backends?, maxBytes?, maxLines? })` takes the same `backends` as the `exec` option, plus output limits. +`exec` calls `workspace.runtime.exec` on the chosen backend and streams bounded output. `createExecTool({ workspace, backends?, maxBytes?, maxLines?, metadata? })` takes the same `backends` as the `exec` option, plus output limits and [metadata](#run-metadata). Each backend's entry in the tool description joins two parts: your text, if any, and what the backend says about itself (`backend.description`, read through `workspace.runtime.backends()`). `WorkerJavaScriptBackend` describes its source language and every module code can import, so the list stays in step with `modules`. `WorkerShellBackend` and `ContainerBackend` describe their command sets, network access, and startup cost. A backend that says nothing gets a one-line default, so add text for a custom backend. @@ -384,6 +386,20 @@ line 3000 The model can open that file with `read` or search it with `grep`. `streamMaxBytes` is ignored; memory per stream stays within a few times `maxBytes`. +### Run metadata + +A run can carry `metadata`, a record of strings that host modules read and the model never sees (see [Host modules](./17_isolate_javascript.md#host-modules)). `metadata` on `createExecTool`, or `execMetadata` on a tool set, is either the record itself or a function that picks it from the tool call's id, which each agent library passes. A throw fails the call without running anything. + +```ts +// Inside an agent with one instance per conversation. +const tools = createAITools({ + workspace, + execMetadata: ({ toolCallId }) => ({ conversation: this.name, toolCall: toolCallId }), +}); +``` + +A pi caller runs each call itself, so it can instead pass the metadata with the call: `execute(call, { metadata })`. Metadata passed with a call wins over `execMetadata`. A caller of `defineExec` passes `metadata` or `toolCallId` on each `execute` the same way. + Wire this tool carefully: it executes arbitrary shell commands inside the configured backend. Treat its output as untrusted text when including it in later model input. Pass `exec: {}` or `readonly: true` when command execution is not part of the agent's job, and list backends explicitly when the Workspace has one the model should not use directly. ## `publish` diff --git a/packages/computer/src/tools/ai-sdk/index.test.ts b/packages/computer/src/tools/ai-sdk/index.test.ts index 1195dd90b..7cb29eaee 100644 --- a/packages/computer/src/tools/ai-sdk/index.test.ts +++ b/packages/computer/src/tools/ai-sdk/index.test.ts @@ -8,6 +8,7 @@ import { Workspace } from "../../workspace"; import { createDeleteTool, createEditTool, + createExecTool, createFindTool, createGrepTool, createReadTool, @@ -2280,6 +2281,52 @@ describe("createAITools exec streaming", () => { }); }); +describe("createAITools exec metadata", () => { + function recordingWorkspace(): { + workspace: Parameters[0]["workspace"]; + metadata: unknown[]; + } { + const metadata: unknown[] = []; + return { + metadata, + workspace: { + runtime: { + async exec(_command, options) { + metadata.push(options.metadata); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }, + backends: () => [{ id: "sh", protocol: "command", callable: false }], + }, + }, + }; + } + + it("picks each run's metadata from the AI SDK tool call's id", async () => { + const { workspace, metadata } = recordingWorkspace(); + const tool = createExecTool({ + workspace, + metadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await executeTool(tool, { command: "true" }); + + expect(metadata).toEqual([{ toolCall: "test-call" }]); + }); + + it("takes the metadata on the tool set", async () => { + const { workspace, metadata } = recordingWorkspace(); + const tools = createAITools({ + // SAFETY: The test only runs exec, which reads nothing but the runtime. + workspace: workspace as Parameters[0]["workspace"], + execMetadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await executeTool(tools.exec, { command: "true" }); + + expect(metadata).toEqual([{ toolCall: "test-call" }]); + }); +}); + describe("createAITools publish tool", () => { it("adds publish by default when assets are configured", async () => { const calls: Array<{ path: string; expiresAfter: number; prefix?: string }> = []; diff --git a/packages/computer/src/tools/ai-sdk/tools.ts b/packages/computer/src/tools/ai-sdk/tools.ts index 1278950d5..14a0d4b09 100644 --- a/packages/computer/src/tools/ai-sdk/tools.ts +++ b/packages/computer/src/tools/ai-sdk/tools.ts @@ -125,7 +125,8 @@ export function createExecTool(options: ExecToolOptions): Tool exec.execute(input, { abortSignal }), + execute: (input, { abortSignal, toolCallId }) => + exec.execute(input, { abortSignal, toolCallId }), }); } diff --git a/packages/computer/src/tools/common/exec.test.ts b/packages/computer/src/tools/common/exec.test.ts new file mode 100644 index 000000000..6ffeefac3 --- /dev/null +++ b/packages/computer/src/tools/common/exec.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { defineExec, type ExecToolOutput, type ExecWorkspaceLike } from "./exec"; + +// A runtime that records the metadata each run was started with. +function recordingWorkspace(): { workspace: ExecWorkspaceLike; metadata: unknown[] } { + const metadata: unknown[] = []; + const workspace: ExecWorkspaceLike = { + runtime: { + async exec(_command, options) { + metadata.push(options.metadata); + return { + result: async () => ({ exitCode: 0, stdout: "", stderr: "" }), + }; + }, + backends: () => [{ id: "sh", protocol: "command", callable: false }], + }, + }; + return { workspace, metadata }; +} + +async function last(output: AsyncIterable): Promise { + let value: ExecToolOutput | undefined; + for await (const chunk of output) value = chunk; + return value; +} + +describe("defineExec metadata", () => { + it("starts each run with the metadata it was given", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ workspace, metadata: { conversation: "c1" } }); + + await last(exec.execute({ command: "true" })); + + expect(metadata).toEqual([{ conversation: "c1" }]); + }); + + it("picks each run's metadata from its tool call's id", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ + workspace, + metadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await last(exec.execute({ command: "true" }, { toolCallId: "call-1" })); + await last(exec.execute({ command: "true" }, { toolCallId: "call-2" })); + + expect(metadata).toEqual([{ toolCall: "call-1" }, { toolCall: "call-2" }]); + }); + + it("uses the metadata a call passes instead", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ workspace, metadata: { conversation: "configured" } }); + + await last(exec.execute({ command: "true" }, { metadata: { conversation: "passed" } })); + + expect(metadata).toEqual([{ conversation: "passed" }]); + }); + + it("starts runs without metadata when none is given", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ workspace }); + + await last(exec.execute({ command: "true" })); + + expect(metadata).toEqual([undefined]); + }); + + it("reports a failure to pick metadata as the call's error, without running", async () => { + const { workspace, metadata } = recordingWorkspace(); + const exec = defineExec({ + workspace, + metadata: () => { + throw new Error("no conversation"); + }, + }); + + await expect(last(exec.execute({ command: "true" }))).resolves.toEqual({ + command: "true", + cwd: null, + backend: "sh", + error: "no conversation", + }); + expect(metadata).toEqual([]); + }); +}); diff --git a/packages/computer/src/tools/common/exec.ts b/packages/computer/src/tools/common/exec.ts index 437611efa..45673ab2c 100644 --- a/packages/computer/src/tools/common/exec.ts +++ b/packages/computer/src/tools/common/exec.ts @@ -65,6 +65,7 @@ export interface ExecWorkspaceLike { backend?: string; env?: Record; input?: WorkspaceRuntimeValue; + metadata?: Record; output?: { maxBytes: number; maxLines: number }; }, ): Promise; @@ -91,6 +92,15 @@ export interface ExecBackendOptions { */ export type ExecBackends = Readonly>; +/** + * The `metadata` each exec run starts with, for host modules; the model + * never sees it. A function picks it per tool call. A throw fails the + * call without running it. + */ +export type ExecMetadata = + | Record + | ((call: { toolCallId: string }) => Record); + export interface ExecToolOptions { workspace: ExecWorkspaceLike; // Omit to offer every backend the Workspace has. With one backend @@ -109,6 +119,11 @@ export interface ExecToolOptions { * times `maxBytes`. */ streamMaxBytes?: number; + /** + * Each run's `metadata`, such as the conversation the tool call + * belongs to. Metadata a call passes itself, as a pi caller can, wins. + */ + metadata?: ExecMetadata; // Clock backing the running-snapshot coalescing floor. Defaults to // Date.now; injectable so tests can drive the interval deterministically. now?: () => number; @@ -145,6 +160,10 @@ export interface ExecInput { export interface ExecCallContext { abortSignal?: AbortSignal; + /** The tool call's id, handed to an `ExecToolOptions.metadata` function. */ + toolCallId?: string; + /** The run's metadata. When set, `ExecToolOptions.metadata` isn't used. */ + metadata?: Record; } /** The exec tool with no agent library attached. Each library wraps it in its own tool shape. */ @@ -230,7 +249,8 @@ export function defineExec(options: ExecToolOptions): ExecDefinition { description, inputSchema, prepareArguments: callableBackendIds.size > 0 ? parseInputObject : (args) => args, - execute: async function* ({ command, cwd, backend, env, input }, { abortSignal } = {}) { + execute: async function* ({ command, cwd, backend, env, input }, context = {}) { + const { abortSignal } = context; // With one backend there is nothing to choose. With several the // schema requires `backend`; a caller that skips the schema gets // the same answer as an error. @@ -246,12 +266,18 @@ export function defineExec(options: ExecToolOptions): ExecDefinition { } let handle: ExecRuntimeHandle; try { + const metadata = + context.metadata ?? + (typeof options.metadata === "function" + ? options.metadata({ toolCallId: context.toolCallId ?? "" }) + : options.metadata); handle = await options.workspace.runtime.exec(command, { cwd, encoding: "utf8", backend: selectedBackend, env, input, + ...(metadata === undefined ? {} : { metadata }), // The runtime cuts and saves by the same limits the tool // shows, so whatever the model does not see is in the file. output: limits, diff --git a/packages/computer/src/tools/common/options.ts b/packages/computer/src/tools/common/options.ts index 169f3f71e..90449d476 100644 --- a/packages/computer/src/tools/common/options.ts +++ b/packages/computer/src/tools/common/options.ts @@ -1,4 +1,4 @@ -import type { ExecBackends, ExecToolOptions, ExecWorkspaceLike } from "./exec"; +import type { ExecBackends, ExecMetadata, ExecToolOptions, ExecWorkspaceLike } from "./exec"; import { confineWorkspace } from "./fs/confine"; import type { EditToolOptions } from "./fs/edit"; import type { ReadToolOptions } from "./fs/read"; @@ -24,6 +24,12 @@ export interface CreateToolsOptions { */ exec?: ExecBackends; execOutput?: Pick; + /** + * The `metadata` each `exec` run starts with, or a function that picks + * it from the tool call's id. Host modules read it; the model never + * sees it. + */ + execMetadata?: ExecMetadata; /** * @deprecated Use `exec`. `{ backends }` becomes `exec: backends`; * `defaultBackend` is ignored, because the model names a backend @@ -81,6 +87,7 @@ function execOptions(options: CreateToolsOptions): ExecToolOptions | undefined { ...exec, ...(output?.maxBytes === undefined ? {} : { maxBytes: output.maxBytes }), ...(output?.maxLines === undefined ? {} : { maxLines: output.maxLines }), + ...(options.execMetadata === undefined ? {} : { metadata: options.execMetadata }), }; } diff --git a/packages/computer/src/tools/core.ts b/packages/computer/src/tools/core.ts index 3727711e1..a959be5b4 100644 --- a/packages/computer/src/tools/core.ts +++ b/packages/computer/src/tools/core.ts @@ -5,6 +5,7 @@ export { type ExecCallContext, type ExecDefinition, type ExecInput, + type ExecMetadata, type ExecRuntimeHandle, type ExecStreamEvent, type ExecToolOptions, diff --git a/packages/computer/src/tools/index.ts b/packages/computer/src/tools/index.ts index 8c0bba8cf..521b460b3 100644 --- a/packages/computer/src/tools/index.ts +++ b/packages/computer/src/tools/index.ts @@ -18,6 +18,7 @@ export { export type { ExecBackendOptions, ExecBackends, + ExecMetadata, ExecRuntimeHandle, ExecStreamEvent, ExecToolOptions, diff --git a/packages/computer/src/tools/pi-ai/index.test.ts b/packages/computer/src/tools/pi-ai/index.test.ts index 2cad3c135..8ad52809a 100644 --- a/packages/computer/src/tools/pi-ai/index.test.ts +++ b/packages/computer/src/tools/pi-ai/index.test.ts @@ -211,6 +211,32 @@ describe("createPiTools execution", () => { expect(parsed.entries[0].name).toBe("a.txt"); }); + it("starts exec with the metadata the caller passes for the call", async () => { + const workspace = makeWorkspace(); + const metadata: unknown[] = []; + (workspace.runtime as unknown as Record).exec = async ( + _command: string, + options: { metadata?: unknown }, + ) => { + metadata.push(options.metadata); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }; + fakeBackends(workspace, [{ id: "sh", protocol: "command", callable: false }]); + const tools = createPiTools({ + workspace, + exec: { sh: {} }, + execMetadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await tools.execute({ id: "1", name: "exec", arguments: { command: "true" } }); + await tools.execute( + { id: "2", name: "exec", arguments: { command: "true" } }, + { metadata: { conversation: "c1" } }, + ); + + expect(metadata).toEqual([{ toolCall: "1" }, { conversation: "c1" }]); + }); + it("returns the tool's own output as details", async () => { const workspace = makeWorkspace(); (workspace.runtime as unknown as Record).exec = async () => ({ diff --git a/packages/computer/src/tools/pi-ai/index.ts b/packages/computer/src/tools/pi-ai/index.ts index 4909ef0a9..b1510b522 100644 --- a/packages/computer/src/tools/pi-ai/index.ts +++ b/packages/computer/src/tools/pi-ai/index.ts @@ -40,15 +40,23 @@ export { export interface ToolCallContext { abortSignal?: AbortSignal; + /** + * The `metadata` an `exec` call's run starts with, for host modules; + * the model never sees it. When set, `execMetadata` isn't used. + */ + metadata?: Record; } +// What each tool's executor gets: the caller's context and the call's id. +type PiToolCallContext = ToolCallContext & { toolCallId: string }; + interface PiToolEntry { name: string; description: string; inputSchema: z.ZodType; strictArguments?: boolean; prepareArguments?: (args: unknown) => unknown; - execute: (input: never, context: ToolCallContext) => Promise | AsyncIterable; + execute: (input: never, context: PiToolCallContext) => Promise | AsyncIterable; toModelOutput?: (args: { input: never; output: never }) => ModelOutput; } @@ -248,13 +256,13 @@ function dispatcher( const run = entry.execute as ( i: unknown, - c: ToolCallContext, + c: PiToolCallContext, ) => Promise | AsyncIterable; const toOutput = entry.toModelOutput as | ((args: { input: unknown; output: unknown }) => ModelOutput) | undefined; try { - const output = await settle(run(parsed.data, context)); + const output = await settle(run(parsed.data, { ...context, toolCallId: call.id })); const modelOutput = toOutput ? toOutput({ input: parsed.data, output }) : defaultModelOutput(output); diff --git a/packages/computer/src/tools/tanstack-ai/index.test.ts b/packages/computer/src/tools/tanstack-ai/index.test.ts index 77f4091f6..a71439772 100644 --- a/packages/computer/src/tools/tanstack-ai/index.test.ts +++ b/packages/computer/src/tools/tanstack-ai/index.test.ts @@ -398,6 +398,31 @@ describe("createTanStackTools", () => { await workspace.close(); }); + it("picks each exec run's metadata from the TanStack tool call", async () => { + const workspace = makeWorkspace(); + const metadata: unknown[] = []; + (workspace.runtime as unknown as Record).exec = async ( + _command: string, + options: { metadata?: unknown }, + ) => { + metadata.push(options.metadata); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }; + (workspace.runtime as unknown as Record).backends = () => [ + { id: "sh", protocol: "command", callable: false }, + ]; + const tools = createTanStackTools({ + workspace, + format: "object", + execMetadata: ({ toolCallId }) => ({ toolCall: toolCallId }), + }); + + await tools.exec.execute({ command: "true" } as never, { toolCallId: "call-1" }); + + expect(metadata).toEqual([{ toolCall: "call-1" }]); + await workspace.close(); + }); + it("kills exec when the chat run's abort signal fires", async () => { const quiet = quietCommandBackend(); const workspace = new Workspace({ diff --git a/packages/computer/src/tools/tanstack-ai/index.ts b/packages/computer/src/tools/tanstack-ai/index.ts index 4ab92161d..44c50812e 100644 --- a/packages/computer/src/tools/tanstack-ai/index.ts +++ b/packages/computer/src/tools/tanstack-ai/index.ts @@ -185,7 +185,10 @@ export function createTanStackTools mutates: true, streams: true, run: async (input: never, context?: TanStackToolExecutionContext) => { - const returned = exec.execute(input, { abortSignal: context?.abortSignal }); + const returned = exec.execute(input, { + abortSignal: context?.abortSignal, + toolCallId: context?.toolCallId, + }); const output = options.streamEventName ? await settleWithEvents(returned, options.streamEventName, context) : await settle(returned);