diff --git a/dato-agent/.gitignore b/dato-agent/.gitignore index ebb1242d..b4621e9b 100644 --- a/dato-agent/.gitignore +++ b/dato-agent/.gitignore @@ -2,3 +2,8 @@ dist node_modules .DS_Store coverage + +# Dato Agent live tests: secrets, spend ledger and eval output +e2e/.env.local +e2e/.spend.jsonl +e2e/results/ diff --git a/dato-agent/e2e/README.md b/dato-agent/e2e/README.md new file mode 100644 index 00000000..b3afc85b --- /dev/null +++ b/dato-agent/e2e/README.md @@ -0,0 +1,41 @@ +# Live tests and evals + +These tests run the real agent runtime against real provider APIs (Anthropic, OpenAI) and the real DatoCMS Remote MCP. Model output varies from run to run, so: + +- **Live tests** (`*.live.test.ts`) check that something still works. Each task is small and likely to succeed, and each test asserts on the project's end state, never on the script text. +- **Evals** (`*.eval.ts`) compare request variants. They run each case several times and write a table of rates and costs to `e2e/results/`. They never fail on a model's choices, only on harness errors. + +Neither one runs in `npm test`, and neither one gates commits, pushes or releases. + +## Setup, once + +1. Put the provider test keys in the monorepo's root `.env`: + `ANTHROPIC_PLUGIN_TESTING_KEY=…` and `OPENAI_PLUGIN_TESTING_KEY=…`. Give each key a spend cap in the provider's console. +2. Put the seed project in `e2e/.env.local` (git-ignored): `E2E_PROJECT_ID`, and `E2E_CMA_TOKEN` (a full-access API token of that project). +3. Run `npm run e2e:seed` to create the schema the tests expect. It only adds what is missing. +4. Run `npm run e2e:login`. In the browser window: + 1. Sign in. + 2. Pick **Only selected projects**, then the seed project. + 3. Pick **Read and edit content**. + 4. Click **Authorize**. + + The script saves `E2E_MCP_ACCESS_TOKEN` to `e2e/.env.local`. When a test reports a 401 from the MCP, run it again. + +## Run + +```bash +npm run test:live # every live test +npm run eval -- e2e/output-limit.eval.ts # one eval, every case, 3 repeats +E2E_EVAL_CASES=64k-batch-nobudget E2E_EVAL_REPEATS=1 npm run eval -- e2e/output-limit.eval.ts +``` + +## How a run works + +1. `forkSeedEnvironment` forks the seed project's primary environment. Every run starts from the same schema and content, and the fork carries the seed's upgrade flags. +2. `runTask` builds the runtime with `createAgentRuntime`, as the inspector does. It sends one message and approves every script the model proposes, until the turn ends. +3. `installProviderTap` wraps `fetch` for provider calls. It records each response's usage and stop reason, and evals use it to rewrite requests (max_tokens, system prompt, beta features) without changing production code. +4. The test reads the fork with the CMA client and asserts on what it finds. Then it deletes the fork. + +## Spend + +Every provider call goes into `e2e/.spend.jsonl` (git-ignored) with its estimated cost. A run refuses to start once a provider's total reaches `E2E_SPEND_CAP_USD` (default $10). Prices live in `e2e/lib/spend.ts`; add a model there before a test uses it. diff --git a/dato-agent/e2e/lib/environments.ts b/dato-agent/e2e/lib/environments.ts new file mode 100644 index 00000000..6a6fb67d --- /dev/null +++ b/dato-agent/e2e/lib/environments.ts @@ -0,0 +1,34 @@ +import { buildClient, type Client } from '@datocms/cma-client-browser'; + +export type ForkedEnvironment = { + client: Client; + environment: string; + destroy(): Promise; +}; + +/** + * Forks the seed project's primary environment so a live run starts from the + * same schema and content every time. Forks inherit the source's opt-in + * upgrade flags and plugins. + */ +export async function forkSeedEnvironment( + cmaToken: string, + label: string, +): Promise { + const admin = buildClient({ apiToken: cmaToken }); + const primary = (await admin.environments.list()).find( + (environment) => environment.meta.primary, + ); + if (!primary) throw new Error('The seed project has no primary environment'); + const environment = `e2e-${label}-${Date.now().toString(36)}` + .toLowerCase() + .replaceAll(/[^a-z0-9-]/g, '-'); + await admin.environments.fork(primary.id, { id: environment }); + return { + client: buildClient({ apiToken: cmaToken, environment }), + environment, + destroy: async () => { + await admin.environments.destroy(environment); + }, + }; +} diff --git a/dato-agent/e2e/lib/liveEnv.ts b/dato-agent/e2e/lib/liveEnv.ts new file mode 100644 index 00000000..c4641467 --- /dev/null +++ b/dato-agent/e2e/lib/liveEnv.ts @@ -0,0 +1,33 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { parseEnv } from 'node:util'; + +/** Secrets and IDs the live tests need. Every field is optional so suites can skip cleanly. */ +export type LiveEnv = { + projectId?: string; + cmaToken?: string; + mcpAccessToken?: string; + anthropicApiKey?: string; + openAiApiKey?: string; +}; + +const readEnvFile = (url: URL): Record => + existsSync(url) ? parseEnv(readFileSync(url, 'utf8')) : {}; + +/** + * Reads `e2e/.env.local` (seed project and MCP token) and the monorepo's root + * `.env` (provider test keys). Real environment variables win over both files. + */ +export function loadLiveEnv(): LiveEnv { + const values = { + ...readEnvFile(new URL('../../../.env', import.meta.url)), + ...readEnvFile(new URL('../.env.local', import.meta.url)), + ...process.env, + }; + return { + projectId: values.E2E_PROJECT_ID, + cmaToken: values.E2E_CMA_TOKEN, + mcpAccessToken: values.E2E_MCP_ACCESS_TOKEN, + anthropicApiKey: values.ANTHROPIC_PLUGIN_TESTING_KEY, + openAiApiKey: values.OPENAI_PLUGIN_TESTING_KEY, + }; +} diff --git a/dato-agent/e2e/lib/providerTap.ts b/dato-agent/e2e/lib/providerTap.ts new file mode 100644 index 00000000..c5b4cdf2 --- /dev/null +++ b/dato-agent/e2e/lib/providerTap.ts @@ -0,0 +1,164 @@ +/** + * Wraps `globalThis.fetch` for provider calls made by the real SDKs inside the + * runtime. It records usage and stop reasons from each streamed response and can + * rewrite request bodies, so evals compare request variants without changing + * production code. + */ + +export type ProviderCall = { + provider: 'anthropic' | 'openai'; + model: string; + maxTokens?: number; + stopReason?: string; + inputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + outputTokens: number; +}; + +export type RequestRewrite = ( + body: Record, + headers: Headers, +) => void; + +const providerOf = (url: string): ProviderCall['provider'] | undefined => { + if (url.startsWith('https://api.anthropic.com/v1/messages')) + return 'anthropic'; + if (url.startsWith('https://api.openai.com/v1/responses')) return 'openai'; + return undefined; +}; + +const numberAt = (value: unknown, key: string): number => { + const found = (value as Record | undefined)?.[key]; + return typeof found === 'number' ? found : 0; +}; + +/** Folds one parsed SSE `data:` payload into the call's totals. */ +function absorbEvent(call: ProviderCall, event: Record): void { + if (call.provider === 'anthropic') { + const message = event.message as Record | undefined; + const usage = (event.usage ?? message?.usage) as + | Record + | undefined; + if (usage) { + call.inputTokens = Math.max( + call.inputTokens, + numberAt(usage, 'input_tokens'), + ); + call.cacheReadTokens = Math.max( + call.cacheReadTokens, + numberAt(usage, 'cache_read_input_tokens'), + ); + call.cacheWriteTokens = Math.max( + call.cacheWriteTokens, + numberAt(usage, 'cache_creation_input_tokens'), + ); + call.outputTokens = Math.max( + call.outputTokens, + numberAt(usage, 'output_tokens'), + ); + } + const delta = event.delta as Record | undefined; + if (typeof delta?.stop_reason === 'string') + call.stopReason = delta.stop_reason; + return; + } + const response = event.response as Record | undefined; + const usage = response?.usage as Record | undefined; + if (!usage) return; + const cached = numberAt(usage.input_tokens_details, 'cached_tokens'); + call.inputTokens = numberAt(usage, 'input_tokens') - cached; + call.cacheReadTokens = cached; + call.outputTokens = numberAt(usage, 'output_tokens'); + const incomplete = response?.incomplete_details as + | Record + | undefined; + call.stopReason = + typeof incomplete?.reason === 'string' + ? incomplete.reason + : String(response?.status ?? ''); +} + +/** Reads a copy of the SSE stream in the background; the SDK reads the other copy. */ +async function drain(stream: ReadableStream, call: ProviderCall) { + const decoder = new TextDecoder(); + let buffer = ''; + for await (const chunk of stream) { + buffer += decoder.decode(chunk, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() ?? ''; + for (const line of lines) { + if (!line.startsWith('data:')) continue; + try { + absorbEvent(call, JSON.parse(line.slice(5))); + } catch { + // Keep-alive or non-JSON frames. + } + } + } +} + +/** + * Installs the tap and returns the calls it records plus a function that + * restores the original fetch. + */ +export function installProviderTap(rewrite?: RequestRewrite) { + const originalFetch = globalThis.fetch; + const calls: ProviderCall[] = []; + const pending: Promise[] = []; + + globalThis.fetch = async (input, init) => { + const request = new Request(input, init); + const provider = providerOf(request.url); + if (!provider || request.method !== 'POST') + return originalFetch(input, init); + + const body = (await request.json()) as Record; + const headers = new Headers(request.headers); + rewrite?.(body, headers); + const call: ProviderCall = { + provider, + model: String(body.model), + maxTokens: + typeof body.max_tokens === 'number' + ? body.max_tokens + : typeof body.max_output_tokens === 'number' + ? body.max_output_tokens + : undefined, + inputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 0, + }; + calls.push(call); + + const response = await originalFetch(request.url, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: request.signal, + }).catch((error: unknown) => { + // The runtime maps network failures to a generic message; keep the real cause. + call.stopReason = `fetch_error: ${error instanceof Error ? `${error.message} ${String((error as { cause?: unknown }).cause ?? '')}` : String(error)}`; + throw error; + }); + if (!response.ok) call.stopReason = `http_${response.status}`; + if (!response.body) return response; + const [forSdk, forTap] = response.body.tee(); + pending.push(drain(forTap, call).catch(() => undefined)); + return new Response(forSdk, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); + }; + + return { + calls, + /** Waits for every tapped stream to finish, then puts the original fetch back. */ + async restore() { + await Promise.all(pending); + globalThis.fetch = originalFetch; + }, + }; +} diff --git a/dato-agent/e2e/lib/runTask.ts b/dato-agent/e2e/lib/runTask.ts new file mode 100644 index 00000000..3bb58cf0 --- /dev/null +++ b/dato-agent/e2e/lib/runTask.ts @@ -0,0 +1,96 @@ +import { + type AgentApprovalRequest, + type AgentRuntimeConfig, + type AgentTurnResult, + createAgentRuntime, +} from '../../src/lib/agentRuntime'; +import { + installProviderTap, + type ProviderCall, + type RequestRewrite, +} from './providerTap'; +import { assertUnderCap, recordCalls } from './spend'; + +export type TaskRun = { + result: AgentTurnResult; + /** Every unsafe call the model asked to run, in order. All were approved. */ + approvals: AgentApprovalRequest[]; + calls: ProviderCall[]; + usd: number; + text: string; +}; + +export type RunTaskOptions = { + label: string; + config: Omit & { + provider: NonNullable; + }; + projectId: string; + environment: string; + message: string; + rewrite?: RequestRewrite; + /** Approval rounds before the run counts as stuck. */ + maxApprovalRounds?: number; +}; + +const acknowledge = () => ({ presented: true }); + +/** + * Runs one chat request the way the inspector does, approving every unsafe + * script the model proposes, until the turn ends. Usage goes to the spend ledger. + */ +export async function runTask(options: RunTaskOptions): Promise { + assertUnderCap( + options.config.provider === 'anthropic' ? 'anthropic' : 'openai', + ); + const tap = installProviderTap(options.rewrite); + const approvals: AgentApprovalRequest[] = []; + let text = ''; + try { + // The SDKs bind fetch when the runtime builds them, so the tap goes in first. + const runtime = createAgentRuntime({ + ...options.config, + mcpAccessLevel: 'unrestricted', + context: { + siteId: options.projectId, + environment: options.environment, + isEnvironmentPrimary: false, + siteName: 'claude-e2e-dato-agent', + surface: 'project', + }, + navigation: { + openRecord: acknowledge, + showRecords: acknowledge, + presentRecords: acknowledge, + presentAssets: acknowledge, + }, + }); + let result = await runtime.runTurn({ message: options.message }); + text += result.text; + for ( + let round = 0; + result.status === 'approval_required' && result.responseId; + round++ + ) { + if (round >= (options.maxApprovalRounds ?? 20)) break; + approvals.push(...result.approvals); + result = await runtime.submitApprovals({ + responseId: result.responseId, + decisions: result.approvals.map((approval) => ({ + approvalRequestId: approval.approvalRequestId, + approve: true, + })), + }); + text += result.text; + } + await runtime.dispose?.(); + return { result, approvals, calls: tap.calls, usd: 0, text }; + } finally { + await tap.restore(); + } +} + +/** Records the run's provider calls in the ledger and fills in its cost. */ +export function settle(run: TaskRun, label: string): TaskRun { + return { ...run, usd: recordCalls(label, run.calls) }; +} diff --git a/dato-agent/e2e/lib/spend.ts b/dato-agent/e2e/lib/spend.ts new file mode 100644 index 00000000..e0830d92 --- /dev/null +++ b/dato-agent/e2e/lib/spend.ts @@ -0,0 +1,80 @@ +import { appendFileSync, existsSync, readFileSync } from 'node:fs'; +import type { ProviderCall } from './providerTap'; + +/** USD per million tokens. Add a model here before a live test may use it. */ +const PRICES: Record< + string, + { input: number; output: number; longInput?: number; longOutput?: number } +> = { + // $0.50 / $2.50 above 100K prompt tokens. + 'claude-haiku-5-5': { + input: 0.1, + output: 0.5, + longInput: 0.5, + longOutput: 2.5, + }, +}; + +/** Stops every live run once a provider key's recorded spend reaches this. */ +export const SPEND_CAP_USD = Number(process.env.E2E_SPEND_CAP_USD ?? 10); + +const LEDGER = new URL('../.spend.jsonl', import.meta.url); + +/** Estimated cost of one call: cache writes at 1.25× input, cache reads at 0.1×. */ +export function callCost(call: ProviderCall): number { + const price = PRICES[call.model]; + if (!price) throw new Error(`No price for ${call.model} in e2e/lib/spend.ts`); + const prompt = + call.inputTokens + call.cacheReadTokens + call.cacheWriteTokens; + const isLong = prompt > 100_000 && price.longInput !== undefined; + const input = isLong ? (price.longInput ?? price.input) : price.input; + const output = isLong ? (price.longOutput ?? price.output) : price.output; + return ( + (call.inputTokens * input + + call.cacheWriteTokens * input * 1.25 + + call.cacheReadTokens * input * 0.1 + + call.outputTokens * output) / + 1_000_000 + ); +} + +/** Total recorded spend per provider, across all runs on this machine. */ +export function recordedSpend(): Record { + const totals = { anthropic: 0, openai: 0 }; + if (!existsSync(LEDGER)) return totals; + for (const line of readFileSync(LEDGER, 'utf8').split('\n')) { + if (!line) continue; + const entry = JSON.parse(line) as { + provider: ProviderCall['provider']; + usd: number; + }; + totals[entry.provider] += entry.usd; + } + return totals; +} + +/** Throws before a run when the provider's ledger already reached the cap. */ +export function assertUnderCap(provider: ProviderCall['provider']): void { + const spent = recordedSpend()[provider]; + if (spent >= SPEND_CAP_USD) + throw new Error( + `Live-test spend for ${provider} is $${spent.toFixed(2)}, at or above the $${SPEND_CAP_USD} cap (E2E_SPEND_CAP_USD).`, + ); +} + +/** Appends one line per call to the git-ignored ledger and returns the run's cost. */ +export function recordCalls( + label: string, + calls: readonly ProviderCall[], +): number { + let total = 0; + for (const call of calls) { + const usd = callCost(call); + total += usd; + appendFileSync( + LEDGER, + `${JSON.stringify({ at: new Date().toISOString(), label, ...call, usd })}\n`, + ); + } + return total; +} diff --git a/dato-agent/e2e/output-limit.eval.ts b/dato-agent/e2e/output-limit.eval.ts new file mode 100644 index 00000000..f32a7b37 --- /dev/null +++ b/dato-agent/e2e/output-limit.eval.ts @@ -0,0 +1,273 @@ +/** + * Eval for large writes against Anthropic's output limit (PR #160). Each case + * asks for many long records in one request and varies what the request sends: + * the max_tokens ceiling, the system prompt's script-size guidance, and an + * advisory task budget. It reports rates and costs rather than pass/fail. + * + * Run: npm run eval -- e2e/output-limit.eval.ts + * Narrow it: E2E_EVAL_CASES=64k-batch-nobudget E2E_EVAL_REPEATS=1 npm run eval -- e2e/output-limit.eval.ts + */ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { forkSeedEnvironment } from './lib/environments'; +import { loadLiveEnv } from './lib/liveEnv'; +import type { RequestRewrite } from './lib/providerTap'; +import { runTask, settle } from './lib/runTask'; + +const RECORD_COUNT = 30; +const MIN_BODY_WORDS = 400; +const MESSAGE = `Create ${RECORD_COUNT} new Article records, one per European city (${RECORD_COUNT} different cities). Each body must be a travel guide of about 600 words covering history, neighbourhoods, food, transport and tips. Save them as drafts (do not publish) and do not ask any questions first.`; + +/** The sentence in src/lib/systemPrompt.ts that steers the model toward one large script. */ +export const CURRENT_SCRIPT_GUIDANCE = + 'Prepare one focused script containing the exact mutation set and result verification so a correct request normally needs one approval.'; +export const BATCHED_SCRIPT_GUIDANCE = + 'Prepare focused scripts containing the exact mutation set and result verification. When a write creates or updates more than about 10 records, or includes long text, split it into batches of about 10 records per script and run them one after another. Never shorten the requested content to fit fewer scripts.'; + +const TASK_BUDGET_BETA = 'task-budgets-2026-03-13'; +const TASK_BUDGET_TOKENS = 200_000; + +type EvalCase = { + name: string; + maxTokens: number; + isBatchedPrompt: boolean; + hasTaskBudget: boolean; + /** Strips the runtime's cache_control markers, to measure what caching saves. */ + isUncached?: boolean; +}; + +const variant = ( + maxTokens: number, + isBatchedPrompt: boolean, + hasTaskBudget: boolean, + isUncached = false, +): EvalCase => ({ + name: `${maxTokens / 1000}k-${isBatchedPrompt ? 'batch' : 'current'}-${hasTaskBudget ? 'budget' : 'nobudget'}${isUncached ? '-nocache' : ''}`, + maxTokens, + isBatchedPrompt, + hasTaskBudget, + isUncached, +}); + +const ALL_CASES: EvalCase[] = [ + variant(64_000, false, false), // #160 as submitted, plus caching + variant(64_000, true, false), + variant(64_000, false, true), + variant(64_000, true, true), + variant(16_000, false, false), // master's low-effort limit, with #160's retry + variant(64_000, false, false, true), +]; + +const selected = process.env.E2E_EVAL_CASES?.split(','); +const CASES = selected + ? ALL_CASES.filter((evalCase) => selected.includes(evalCase.name)) + : ALL_CASES; +const REPEATS = Number(process.env.E2E_EVAL_REPEATS ?? 3); +const CONCURRENCY = Number(process.env.E2E_EVAL_CONCURRENCY ?? 4); + +/** Rewrites each Anthropic request to match the case. */ +const rewriteFor = + (evalCase: EvalCase): RequestRewrite => + (body, headers) => { + body.max_tokens = evalCase.maxTokens; + if (evalCase.isUncached) { + delete body.cache_control; + body.system = (body.system as { cache_control?: unknown }[]).map( + ({ cache_control: _, ...block }) => block, + ); + } + if (evalCase.isBatchedPrompt) { + const system = JSON.stringify(body.system); + // A retried request can arrive already rewritten; only fail when neither text is there. + if ( + !system.includes(CURRENT_SCRIPT_GUIDANCE) && + !system.includes(BATCHED_SCRIPT_GUIDANCE) + ) + throw new Error('System prompt no longer contains the script guidance'); + body.system = JSON.parse( + system.replace(CURRENT_SCRIPT_GUIDANCE, BATCHED_SCRIPT_GUIDANCE), + ); + } + if (evalCase.hasTaskBudget) { + body.output_config = { + ...(body.output_config as object | undefined), + task_budget: { type: 'tokens', total: TASK_BUDGET_TOKENS }, + }; + const betas = headers.get('anthropic-beta'); + headers.set( + 'anthropic-beta', + betas ? `${betas},${TASK_BUDGET_BETA}` : TASK_BUDGET_BETA, + ); + } + }; + +type Outcome = { + name: string; + repeat: number; + status: string; + created: number; + longBodies: number; + meanBodyWords: number; + approvals: number; + truncations: number; + outputTokens: number; + /** Share of prompt tokens read from the cache. */ + cachedShare: number; + usd: number; + seconds: number; + error?: string; +}; + +const env = loadLiveEnv(); +const isConfigured = Boolean( + env.projectId && env.cmaToken && env.mcpAccessToken && env.anthropicApiKey, +); + +async function runCase(evalCase: EvalCase, repeat: number): Promise { + const label = `${evalCase.name}-r${repeat}`; + const startedAt = Date.now(); + const fork = await forkSeedEnvironment(env.cmaToken ?? '', label); + try { + const run = settle( + await runTask({ + label: `output-limit:${label}`, + projectId: env.projectId ?? '', + environment: fork.environment, + message: MESSAGE, + rewrite: rewriteFor(evalCase), + config: { + provider: 'anthropic', + apiKey: env.anthropicApiKey, + mcpAccessToken: env.mcpAccessToken ?? '', + model: 'claude-haiku-5-5', + modelMaxOutputTokens: 128_000, + reasoningEffort: 'low', + }, + }), + `output-limit:${label}`, + ); + const articles = await fork.client.items.list({ + filter: { type: 'article' }, + version: 'current', + page: { limit: 100 }, + }); + const created = articles.filter((article) => article.title !== 'Welcome'); + const bodyWords = created.map( + (article) => + String(article.body ?? '') + .split(/\s+/) + .filter(Boolean).length, + ); + return { + name: evalCase.name, + repeat, + status: run.result.status, + created: created.length, + longBodies: bodyWords.filter((words) => words >= MIN_BODY_WORDS).length, + meanBodyWords: Math.round( + bodyWords.reduce((sum, words) => sum + words, 0) / + Math.max(1, bodyWords.length), + ), + approvals: run.approvals.length, + truncations: run.calls.filter((call) => call.stopReason === 'max_tokens') + .length, + outputTokens: run.calls.reduce((sum, call) => sum + call.outputTokens, 0), + cachedShare: + run.calls.reduce((sum, call) => sum + call.cacheReadTokens, 0) / + Math.max( + 1, + run.calls.reduce( + (sum, call) => + sum + + call.inputTokens + + call.cacheReadTokens + + call.cacheWriteTokens, + 0, + ), + ), + usd: run.usd, + seconds: Math.round((Date.now() - startedAt) / 1000), + ...(run.result.error ? { error: run.result.error.message } : {}), + }; + } catch (error) { + return { + name: evalCase.name, + repeat, + status: 'harness_error', + created: 0, + longBodies: 0, + meanBodyWords: 0, + approvals: 0, + truncations: 0, + outputTokens: 0, + cachedShare: 0, + usd: 0, + seconds: Math.round((Date.now() - startedAt) / 1000), + error: error instanceof Error ? error.message : String(error), + }; + } finally { + await fork.destroy(); + } +} + +/** Runs jobs with at most `limit` in flight. */ +async function pool( + jobs: (() => Promise)[], + limit: number, +): Promise { + const results: T[] = []; + let next = 0; + const worker = async () => { + while (next < jobs.length) { + const index = next++; + results[index] = await (jobs[index] as () => Promise)(); + } + }; + await Promise.all(Array.from({ length: limit }, worker)); + return results; +} + +/** Done, every record created, and at least 90% of bodies as long as asked. */ +const isSuccess = (outcome: Outcome) => + outcome.status === 'completed' && + outcome.created === RECORD_COUNT && + outcome.longBodies >= RECORD_COUNT * 0.9; + +function summarize(outcomes: Outcome[]): string { + const rows = CASES.map((evalCase) => { + const runs = outcomes.filter((outcome) => outcome.name === evalCase.name); + const mean = (pick: (outcome: Outcome) => number) => + runs.reduce((sum, outcome) => sum + pick(outcome), 0) / runs.length; + return `| ${evalCase.name} | ${runs.filter(isSuccess).length}/${runs.length} | ${mean((o) => o.created).toFixed(1)} | ${mean((o) => o.longBodies).toFixed(1)} | ${Math.round(mean((o) => o.meanBodyWords))} | ${mean((o) => o.truncations).toFixed(1)} | ${mean((o) => o.approvals).toFixed(1)} | ${Math.round(mean((o) => o.outputTokens)).toLocaleString('en')} | ${Math.round(mean((o) => o.cachedShare) * 100)}% | $${mean((o) => o.usd).toFixed(3)} | ${Math.round(mean((o) => o.seconds))} |`; + }); + return [ + '| Case | Success | Records | Long bodies | Words/body | Truncations | Approvals | Output tokens | Cached input | Cost | Seconds |', + '|---|---|---|---|---|---|---|---|---|---|---|', + ...rows, + ].join('\n'); +} + +describe.skipIf(!isConfigured)('eval: large writes vs the output limit', () => { + it('measures every case', async () => { + const outcomes = await pool( + CASES.flatMap((evalCase) => + Array.from( + { length: REPEATS }, + (_, repeat) => () => runCase(evalCase, repeat + 1), + ), + ), + CONCURRENCY, + ); + const table = summarize(outcomes); + const folder = new URL('./results/', import.meta.url); + mkdirSync(folder, { recursive: true }); + const stamp = new Date().toISOString().replaceAll(/[:.]/g, '-'); + writeFileSync( + new URL(`output-limit-${stamp}.json`, folder), + JSON.stringify({ message: MESSAGE, cases: CASES, outcomes }, null, 2), + ); + writeFileSync(new URL(`output-limit-${stamp}.md`, folder), `${table}\n`); + console.log(`\n${table}\n`); + expect(outcomes.filter((o) => o.status === 'harness_error')).toEqual([]); + }); +}); diff --git a/dato-agent/e2e/output-limit.live.test.ts b/dato-agent/e2e/output-limit.live.test.ts new file mode 100644 index 00000000..a0d0c3f7 --- /dev/null +++ b/dato-agent/e2e/output-limit.live.test.ts @@ -0,0 +1,89 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + type ForkedEnvironment, + forkSeedEnvironment, +} from './lib/environments'; +import { loadLiveEnv } from './lib/liveEnv'; +import { runTask, settle } from './lib/runTask'; + +const env = loadLiveEnv(); +const isConfigured = Boolean( + env.projectId && env.cmaToken && env.mcpAccessToken && env.anthropicApiKey, +); + +const anthropicConfig = () => ({ + provider: 'anthropic' as const, + apiKey: env.anthropicApiKey, + mcpAccessToken: env.mcpAccessToken ?? '', + model: 'claude-haiku-5-5', + modelMaxOutputTokens: 128_000, + reasoningEffort: 'low' as const, +}); + +const articleTitles = async (fork: ForkedEnvironment) => + ( + await fork.client.items.list({ + filter: { type: 'article' }, + version: 'current', + page: { limit: 100 }, + }) + ) + .map((article) => String(article.title)) + .filter((title) => title !== 'Welcome'); + +describe.skipIf(!isConfigured)('live: Anthropic output limit', () => { + let fork: ForkedEnvironment; + beforeEach(async (context) => { + fork = await forkSeedEnvironment(env.cmaToken ?? '', context.task.name); + }); + afterEach(async () => { + await fork?.destroy(); + }); + + it('completes a large write of 30 long records with the default output limit', async () => { + // Master's 16k low-effort limit truncated a single-script version of this request (PR #160). + const run = settle( + await runTask({ + label: 'output-limit:large-write', + projectId: env.projectId ?? '', + environment: fork.environment, + message: + 'Create 30 new Article records, one per European city (30 different cities). Each body must be a travel guide of about 600 words covering history, neighbourhoods, food, transport and tips. Save them as drafts (do not publish) and do not ask any questions first.', + config: anthropicConfig(), + }), + 'output-limit:large-write', + ); + + const titles = await articleTitles(fork); + expect(run.result.status).toBe('completed'); + expect(titles).toHaveLength(30); + expect(new Set(titles).size).toBe(30); + }); + + it('discards a truncated reply, retries in smaller scripts, and writes each record once', async () => { + // A deliberately low ceiling makes the first write script hit max_tokens. + const run = settle( + await runTask({ + label: 'output-limit:recovery', + projectId: env.projectId ?? '', + environment: fork.environment, + message: + 'Create 6 new Article records, one per Italian city (6 different cities). Each body must be a travel guide of about 500 words. Save them as drafts and do not ask any questions first.', + rewrite: (body) => { + body.max_tokens = 5_000; + }, + config: anthropicConfig(), + }), + 'output-limit:recovery', + ); + + const titles = await articleTitles(fork); + expect( + run.calls.some((call) => call.stopReason === 'max_tokens'), + 'the low ceiling must truncate at least one reply', + ).toBe(true); + expect(run.result.status).toBe('completed'); + expect(titles).toHaveLength(6); + expect(new Set(titles).size).toBe(6); + }); +}); diff --git a/dato-agent/e2e/setup/mcp-login.ts b/dato-agent/e2e/setup/mcp-login.ts new file mode 100644 index 00000000..5a870fa2 --- /dev/null +++ b/dato-agent/e2e/setup/mcp-login.ts @@ -0,0 +1,172 @@ +/** + * Mints a DatoCMS Remote MCP access token for the live tests and saves it as + * `E2E_MCP_ACCESS_TOKEN` in `e2e/.env.local`. It runs the same OAuth flow as the + * plugin (dynamic client registration, PKCE, consent at oauth.datocms.com), with a + * loopback redirect that the browser never actually reaches. + * + * The token reaches only the seed project, with "Read and edit content" access. + * Pass `--anything` for schema-changing tests and `--project ` for + * another project. + * + * Usage, from the plugin folder: + * node e2e/setup/mcp-login.ts # headed: sign in and authorize by hand + * E2E_LOGIN_EMAIL=… E2E_LOGIN_PASSWORD=… node e2e/setup/mcp-login.ts # headless + */ +import { createHash, randomBytes } from 'node:crypto'; +import { chmodSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { chromium } from 'playwright-core'; + +// Same values as src/lib/oauth.ts (this script runs under plain Node, so it can't import it). +const MCP_RESOURCE = 'https://mcp.datocms.com/'; +const OAUTH_BASE_URL = 'https://oauth.datocms.com/oauth'; +const SCOPES = 'read_account read_sites read_organizations'; +const REDIRECT_URI = 'http://127.0.0.1:53682/callback'; +const ENV_FILE = new URL('../.env.local', import.meta.url); + +/** Value after a `--flag` on the command line. */ +const argument = (flag: string) => { + const index = process.argv.indexOf(flag); + return index === -1 ? undefined : process.argv[index + 1]; +}; + +const base64Url = (bytes: Buffer) => bytes.toString('base64url'); + +/** Replaces or appends one `KEY=value` line, keeping the file owner-only. */ +function saveEnvValue(key: string, value: string): void { + const lines = existsSync(ENV_FILE) + ? readFileSync(ENV_FILE, 'utf8') + .split('\n') + .filter((line) => line && !line.startsWith(`${key}=`)) + : []; + writeFileSync(ENV_FILE, `${[...lines, `${key}=${value}`].join('\n')}\n`); + chmodSync(ENV_FILE, 0o600); +} + +const registration = await fetch(`${OAUTH_BASE_URL}/register`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + client_name: 'DatoCMS Agent Plugin (live tests)', + redirect_uris: [REDIRECT_URI], + grant_types: ['authorization_code'], + response_types: ['code'], + token_endpoint_auth_method: 'none', + scope: SCOPES, + }), +}); +if (!registration.ok) + throw new Error(`Client registration failed: ${registration.status}`); +const { client_id: clientId } = (await registration.json()) as { + client_id: string; +}; + +const codeVerifier = base64Url(randomBytes(32)); +const state = base64Url(randomBytes(32)); +const authorizationUrl = `${OAUTH_BASE_URL}/authorize?${new URLSearchParams({ + client_id: clientId, + response_type: 'code', + resource: MCP_RESOURCE, + redirect_uri: REDIRECT_URI, + code_challenge: base64Url(createHash('sha256').update(codeVerifier).digest()), + code_challenge_method: 'S256', + state, + scope: SCOPES, +})}`; + +const projectName = argument('--project') ?? 'claude-e2e-dato-agent'; +const isAnything = process.argv.includes('--anything'); +const email = process.env.E2E_LOGIN_EMAIL; +const password = process.env.E2E_LOGIN_PASSWORD; +const isHeadless = Boolean(email && password); +const browser = await chromium.launch({ + channel: 'chromium', + headless: isHeadless, +}); +const page = await browser.newPage(); + +// The loopback redirect carries the code; answer it here instead of running a server. +const callback = new Promise((resolve) => { + void page.route(`${new URL(REDIRECT_URI).origin}/**`, async (route) => { + resolve(new URL(route.request().url())); + await route.fulfill({ + contentType: 'text/html', + body: '

Signed in. You can close this window.

', + }); + }); +}); + +await page.goto(authorizationUrl); +if (isHeadless && email && password) { + const authorize = page.getByRole('button', { name: 'Authorize' }); + await page + .getByLabel('Email') + .or(authorize) + .first() + .waitFor({ timeout: 30_000 }); + if (await page.getByLabel('Email').count()) { + await page.getByLabel('Email').fill(email); + await page.getByLabel('Password').fill(password); + await page.getByLabel('Password').press('Enter'); + } + await authorize.waitFor({ timeout: 30_000 }); + // Least privilege: only the seed project, content edits only unless --anything. + await page.getByRole('radio', { name: /^Only selected projects/ }).check(); + await page.getByRole('button', { name: 'Select projects…' }).click(); + await page.getByRole('textbox', { name: 'Filter projects…' }).fill(projectName); + await page.getByRole('checkbox', { name: projectName, exact: true }).check(); + await page.keyboard.press('Escape'); + await page + .getByRole('radio', { + name: isAnything ? /^Anything you can/ : /^Read and edit content/, + }) + .check(); + await authorize.click({ timeout: 30_000 }); +} else { + console.log( + `In the browser window: sign in, pick "Only selected projects" → ${projectName}, pick "${isAnything ? 'Anything you can' : 'Read and edit content'}", then click Authorize.`, + ); +} + +const redirected = await Promise.race([ + callback, + new Promise((_, reject) => + setTimeout(async () => { + // Shows where the flow stopped, e.g. an unexpected consent or project picker. + const screenshot = `${tmpdir()}/dato-agent-mcp-login-failure.png`; + await page.screenshot({ path: screenshot }).catch(() => undefined); + reject(new Error(`No OAuth redirect after 2 minutes at ${new URL(page.url()).pathname}. See ${screenshot}`)); + }, 120_000), + ), +]); +await browser.close(); +if (redirected.searchParams.get('state') !== state) + throw new Error('OAuth state mismatch'); +const code = redirected.searchParams.get('code'); +if (!code) + throw new Error( + `Authorization failed: ${redirected.searchParams.get('error') ?? 'no code'}`, + ); + +const tokenResponse = await fetch(`${OAUTH_BASE_URL}/token`, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + grant_type: 'authorization_code', + resource: MCP_RESOURCE, + code, + client_id: clientId, + code_verifier: codeVerifier, + redirect_uri: REDIRECT_URI, + }), +}); +if (!tokenResponse.ok) + throw new Error(`Token exchange failed: ${tokenResponse.status}`); +const token = (await tokenResponse.json()) as { + access_token: string; + expires_in?: number; +}; +saveEnvValue('E2E_MCP_ACCESS_TOKEN', token.access_token); +console.log( + `Saved E2E_MCP_ACCESS_TOKEN to e2e/.env.local${token.expires_in ? ` (expires in ${Math.round(token.expires_in / 3600)} h)` : ''}.`, +); diff --git a/dato-agent/e2e/setup/seed.ts b/dato-agent/e2e/setup/seed.ts new file mode 100644 index 00000000..8a8c0871 --- /dev/null +++ b/dato-agent/e2e/setup/seed.ts @@ -0,0 +1,38 @@ +/** + * Creates the schema the live tests expect in the seed project's primary + * environment. Safe to rerun: it only adds what is missing. + * Usage, from the plugin folder: node e2e/setup/seed.ts + */ +import { buildClient } from '@datocms/cma-client-browser'; +import { loadLiveEnv } from '../lib/liveEnv.ts'; + +const { cmaToken } = loadLiveEnv(); +if (!cmaToken) throw new Error('E2E_CMA_TOKEN is missing from e2e/.env.local'); +const client = buildClient({ apiToken: cmaToken }); + +const itemTypes = await client.itemTypes.list(); +let article = itemTypes.find((itemType) => itemType.api_key === 'article'); +if (!article) { + article = await client.itemTypes.create({ + name: 'Article', + api_key: 'article', + draft_mode_active: true, + }); + await client.fields.create(article.id, { + label: 'Title', + api_key: 'title', + field_type: 'string', + validators: { required: {} }, + }); + await client.fields.create(article.id, { + label: 'Body', + api_key: 'body', + field_type: 'text', + }); + await client.items.create({ + item_type: { type: 'item_type', id: article.id }, + title: 'Welcome', + body: 'The one record every forked environment starts with.', + }); +} +console.log(`Seed ready: article model ${article.id}`); diff --git a/dato-agent/e2e/smoke.live.test.ts b/dato-agent/e2e/smoke.live.test.ts new file mode 100644 index 00000000..a0852095 --- /dev/null +++ b/dato-agent/e2e/smoke.live.test.ts @@ -0,0 +1,53 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { + type ForkedEnvironment, + forkSeedEnvironment, +} from './lib/environments'; +import { loadLiveEnv } from './lib/liveEnv'; +import { runTask, settle } from './lib/runTask'; + +const env = loadLiveEnv(); +const isConfigured = Boolean( + env.projectId && env.cmaToken && env.mcpAccessToken && env.anthropicApiKey, +); + +describe.skipIf(!isConfigured)('live smoke: one write, Anthropic', () => { + let fork: ForkedEnvironment; + beforeAll(async () => { + fork = await forkSeedEnvironment(env.cmaToken ?? '', 'smoke'); + }); + afterAll(async () => { + await fork?.destroy(); + }); + + it('creates a draft article after one approval', async () => { + const run = settle( + await runTask({ + label: 'smoke', + projectId: env.projectId ?? '', + environment: fork.environment, + message: + 'Create one draft Article titled "Live smoke test" with the body "Created by the live smoke test." Do not ask questions first.', + config: { + provider: 'anthropic', + apiKey: env.anthropicApiKey, + mcpAccessToken: env.mcpAccessToken ?? '', + model: 'claude-haiku-5-5', + modelMaxOutputTokens: 128_000, + reasoningEffort: 'low', + }, + }), + 'smoke', + ); + console.log( + `smoke: ${run.result.status}, ${run.approvals.length} approvals, ${run.calls.length} calls, $${run.usd.toFixed(4)}`, + ); + + const articles = await fork.client.items.list({ + filter: { type: 'article', fields: { title: { eq: 'Live smoke test' } } }, + version: 'current', + }); + expect(run.result.status).toBe('completed'); + expect(articles).toHaveLength(1); + }); +}); diff --git a/dato-agent/e2e/tsconfig.json b/dato-agent/e2e/tsconfig.json new file mode 100644 index 00000000..58def064 --- /dev/null +++ b/dato-agent/e2e/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../tsconfig.app.json", + "compilerOptions": { + "tsBuildInfoFile": "../node_modules/.tmp/tsconfig.e2e.tsbuildinfo", + "target": "ES2023", + "lib": ["ES2023", "DOM", "DOM.Iterable"], + "allowImportingTsExtensions": true, + "types": ["node", "vite/client"] + }, + "include": [".", "../vitest.live.config.ts"] +} diff --git a/dato-agent/package-lock.json b/dato-agent/package-lock.json index c709ea50..283ff4da 100644 --- a/dato-agent/package-lock.json +++ b/dato-agent/package-lock.json @@ -42,6 +42,7 @@ "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^6.0.3", "jsdom": "^29.1.1", + "playwright-core": "1.63.0", "typescript": "^7.0.2", "vite": "^8.1.5", "vitest": "^4.1.10" @@ -529,6 +530,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=20.19.0" }, @@ -577,6 +579,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=20.19.0" } @@ -611,31 +614,6 @@ "async-scheduler": "^1.4.4" } }, - "node_modules/@emnapi/core": { - "version": "2.0.0-alpha.3", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-2.0.0-alpha.3.tgz", - "integrity": "sha512-AZypUeJ/yByuxyS7BlSNRDOMLMlROYtjYdIAuBmJssVz1UJDSeYxLrdizhXCFYhedC5bqd/ASy8EuNXbVVXp9g==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@emnapi/wasi-threads": "2.0.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "2.0.0-alpha.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-2.0.0-alpha.3.tgz", - "integrity": "sha512-hFPAhMUjJD9BSyCANEISPOogeXC9Zo9ZQl7L6vKnaVsMkCtzznaW/naYypeyl0Gv5rYfWYsZbpixTMpjDJzQeA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/wasi-threads": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-2.0.1.tgz", @@ -643,7 +621,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "tslib": "^2.4.0" } @@ -812,6 +789,7 @@ "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", "license": "MIT", + "peer": true, "dependencies": { "@floating-ui/core": "^1.8.0", "@floating-ui/utils": "^0.2.12" @@ -1364,6 +1342,7 @@ "resolved": "https://registry.npmjs.org/@tiptap/core/-/core-3.30.6.tgz", "integrity": "sha512-M32+WOx1IrPRbPunhG1Xl7ETgUZJGABpGi/y969Uio6rR4tPBnOgFG5xXE3Xw6OQtYUqUE2n9nnnckoObZwBzA==", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/ueberdosis" @@ -1587,6 +1566,7 @@ "resolved": "https://registry.npmjs.org/@tiptap/extension-list/-/extension-list-3.30.6.tgz", "integrity": "sha512-vNzXMap27plACDvyeEUlUyB01Rw1IOWn+wABIEuiIyJzVpZ6n5T8ck2nwHXp6lTy5xxGi7MnVq8epOjqCG1g8A==", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/ueberdosis" @@ -1720,6 +1700,7 @@ "resolved": "https://registry.npmjs.org/@tiptap/extensions/-/extensions-3.30.6.tgz", "integrity": "sha512-8SYpu3E+ZuWhTGWtjcWPtcJmcotrADYBuXIu+bgdZjcuaxRHs+6IpNvdLabl3q46W7+0A11GginPDJNqrMKk2A==", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/ueberdosis" @@ -1734,6 +1715,7 @@ "resolved": "https://registry.npmjs.org/@tiptap/pm/-/pm-3.30.6.tgz", "integrity": "sha512-FVzgc+7TMKUreBqFHmyhDSkFUMZPgIzWdeNt4sWLFEzwzdklgKy/+MZClAq9m+bJJU2FKC2+SVmWTatZD6didg==", "license": "MIT", + "peer": true, "dependencies": { "prosemirror-changeset": "^2.4.1", "prosemirror-commands": "^1.7.1", @@ -1822,6 +1804,7 @@ "resolved": "https://registry.npmjs.org/@tiptap/suggestion/-/suggestion-3.30.6.tgz", "integrity": "sha512-Zv/nJoRSRRlB8bL7fl3oc7I1aOyahANaEyy1Yv1/D3sUoXePorBItTUb9IhnfzK9Dhm4ijOoT2V3QPI3NEdpFA==", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/ueberdosis" @@ -1848,8 +1831,7 @@ "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/chai": { "version": "5.2.3", @@ -1930,6 +1912,7 @@ "integrity": "sha512-Vu4a5UFA9rIIFJ7rB/Vaafh9lrCQszopTCx6KjFboXTGQbPNasehVR5TEiithSDGyd1DEiUByggTZsg8jukeIg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~8.3.0" } @@ -1957,6 +1940,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.17.tgz", "integrity": "sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -1966,6 +1950,7 @@ "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -2534,7 +2519,6 @@ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -2545,7 +2529,6 @@ "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -3123,8 +3106,7 @@ "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/dom-helpers": { "version": "5.2.1", @@ -3316,6 +3298,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -3642,6 +3625,7 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.13.tgz", "integrity": "sha512-CQ46U0ZkAGmbT/4UxdzzGJpacP2IeKgY4a5/tOI9AABbpOMfK739wfDXmv1usCk+3RkKj1hQy4/fjhiwa2xlrA==", "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -3884,6 +3868,7 @@ "integrity": "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@asamuzakjp/css-color": "^5.1.11", "@asamuzakjp/dom-selector": "^7.1.1", @@ -4273,7 +4258,6 @@ "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true, "license": "MIT", - "peer": true, "bin": { "lz-string": "bin/bin.js" } @@ -5486,6 +5470,7 @@ "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -5502,6 +5487,19 @@ "node": ">=16.20.0" } }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/postcss": { "version": "8.5.24", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.24.tgz", @@ -5537,7 +5535,6 @@ "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -5802,6 +5799,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -5811,6 +5809,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -5838,8 +5837,7 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/react-markdown": { "version": "10.1.0", @@ -6821,6 +6819,7 @@ "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", @@ -7097,6 +7096,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/dato-agent/package.json b/dato-agent/package.json index 439c1573..e446691b 100644 --- a/dato-agent/package.json +++ b/dato-agent/package.json @@ -31,6 +31,10 @@ "typecheck": "tsc -b", "test": "vitest run", "test:watch": "vitest", + "test:live": "vitest run --config vitest.live.config.ts e2e/*.live.test.ts", + "eval": "vitest run --config vitest.live.config.ts", + "e2e:login": "node e2e/setup/mcp-login.ts", + "e2e:seed": "node e2e/setup/seed.ts", "lint": "biome check ./src", "prepublishOnly": "npm run build" }, @@ -68,6 +72,7 @@ "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^6.0.3", "jsdom": "^29.1.1", + "playwright-core": "1.63.0", "typescript": "^7.0.2", "vite": "^8.1.5", "vitest": "^4.1.10" diff --git a/dato-agent/src/entrypoints/ConfigScreen.tsx b/dato-agent/src/entrypoints/ConfigScreen.tsx index 1ebea212..7ccbd1bd 100644 --- a/dato-agent/src/entrypoints/ConfigScreen.tsx +++ b/dato-agent/src/entrypoints/ConfigScreen.tsx @@ -41,6 +41,7 @@ import { listProviderModels, type ProviderModel, preferredProviderModel, + providerModelOptionLabel, providerModelSupportsFastMode, } from '../lib/providerModels'; import styles from './ConfigScreen.module.css'; @@ -142,7 +143,7 @@ function modelOptions( } return models.map((model) => ({ - label: model.label, + label: providerModelOptionLabel(model), value: model.id, })); } diff --git a/dato-agent/src/lib/agentRuntime.test.ts b/dato-agent/src/lib/agentRuntime.test.ts index 3d221ff5..a5d86186 100644 --- a/dato-agent/src/lib/agentRuntime.test.ts +++ b/dato-agent/src/lib/agentRuntime.test.ts @@ -424,6 +424,41 @@ function attachment({ } describe('AgentRuntime', () => { + it.each([ + { + reason: 'max_output_tokens', + guidance: 'Ask for fewer records or shorter content at a time.', + }, + { reason: 'content_filter', guidance: 'Rephrase the request' }, + ])( + 'turns an incomplete $reason response into guidance the editor can act on', + async ({ reason, guidance }) => { + const incomplete = { + ...response('resp-incomplete', [], 'incomplete'), + incomplete_details: { reason }, + } as unknown as Response; + const runtime = runtimeWith( + new QueueResponsesClient([ + [ + { + type: 'response.incomplete', + response: incomplete, + sequence_number: 0, + } as ResponseStreamEvent, + ], + ]), + ); + + const { result } = await drain( + runtime.streamTurn({ message: 'Create the records.' }), + ); + + expect(result.status).toBe('incomplete'); + expect(result.error?.message).toContain(guidance); + expect(result.error?.message).not.toContain(reason); + }, + ); + it('rejects an oversized message before contacting OpenAI', async () => { const client = new QueueResponsesClient([]); const runtime = runtimeWith(client); diff --git a/dato-agent/src/lib/agentRuntime.ts b/dato-agent/src/lib/agentRuntime.ts index 923d6365..00a6f8ca 100644 --- a/dato-agent/src/lib/agentRuntime.ts +++ b/dato-agent/src/lib/agentRuntime.ts @@ -80,8 +80,12 @@ export const DEFAULT_MAX_TOOL_CALLS = 20; export const MAX_AGENT_TOOL_CALLS_PER_TURN = 200; export const MAX_AGENT_REQUEST_CHARACTERS = 600_000; export const MAX_TOOL_RESULT_CHARACTERS = 60_000; -export const DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS = 16_000; -export const DEEP_ANTHROPIC_MAX_OUTPUT_TOKENS = 64_000; +/** + * Output ceiling when the configured model's own maximum is unknown (it is + * normally discovered from the Models API when an admin picks the model). + * Every active Claude model allows at least this much (Haiku 4.5 is the smallest). + */ +export const DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS = 64_000; export const MAX_AGENT_HISTORY_CHARACTERS = 120_000; export const MAX_TOOL_RESULT_CHARACTERS_PER_TURN = 240_000; export const MAX_HOST_CONTEXT_CHARACTERS = 20_000; @@ -3013,6 +3017,20 @@ function terminalVisibleText(response: Response): string { .join(''); } +/** Editor-facing guidance for each reason OpenAI gives for an incomplete response. */ +function openAiIncompleteMessage(reason: string | undefined): string { + switch (reason) { + case 'max_output_tokens': + return 'The reply got too long and stopped early. Changes you already approved still apply. Ask for fewer records or shorter content at a time.'; + case 'content_filter': + return 'OpenAI’s content filter stopped the reply. Rephrase the request and try again.'; + case undefined: + return 'The model could not complete the response. Try again, or ask for a smaller step.'; + default: + return `The response stopped early (${reason}). Try again, or ask for a smaller step.`; + } +} + function responseError( response: Response, fallbackCode: AgentRuntimeError['code'], @@ -3020,9 +3038,7 @@ function responseError( if (fallbackCode === 'incomplete') { return { code: 'incomplete', - message: response.incomplete_details?.reason - ? `The response stopped early: ${response.incomplete_details.reason}.` - : 'The model could not complete the response.', + message: openAiIncompleteMessage(response.incomplete_details?.reason), retryable: true, }; } @@ -4866,6 +4882,37 @@ interface AnthropicLoopState { viewedProtectedRepairScript: boolean; failedRepairCallIdentities: Set; repairPolicy?: AgentRepairPolicy; + outputLimitRecoveryUsed?: boolean; +} + +const OUTPUT_LIMIT_RECOVERY_INSTRUCTION = + 'Your previous response reached the output limit and was discarded before any of its tool calls ran, so nothing from it changed the project. Continue the same task in smaller steps: keep reasoning brief, and split large writes into several smaller scripts (for example two or three records per script), each reviewed separately.'; + +/** Shown while the runtime retries a reply that hit the output limit. */ +export const OUTPUT_LIMIT_RETRY_LABEL = + 'That reply got too long to finish, so nothing in it ran. Redoing it in smaller steps.'; + +/** Shown when the retry also hits the limit. Editors can act on it; they can't change the model. */ +export const OUTPUT_LIMIT_ERROR_MESSAGE = + 'The reply got too long to finish, even in smaller steps, so nothing in it ran. Changes you already approved still apply. Ask for fewer records or shorter content at a time.'; + +export const CONTEXT_WINDOW_ERROR_MESSAGE = + 'This chat got too long for the model to read. Changes you already approved still apply. Start a new chat and say what to do next.'; + +function appendAnthropicUserText(messages: MessageParam[], text: string): void { + const last = messages.at(-1); + if (last?.role !== 'user') { + messages.push({ role: 'user', content: text }); + return; + } + const content = + typeof last.content === 'string' + ? [{ type: 'text' as const, text: last.content }] + : last.content; + messages[messages.length - 1] = { + ...last, + content: [...content, { type: 'text', text }], + }; } function anthropicToolResultCharacters( @@ -5062,22 +5109,20 @@ function resolveAnthropicEffort( return 'high'; } +/** + * Requests the model's full output maximum. max_tokens is a hard cutoff the + * model cannot see, so it is no tool for keeping scripts small (the system + * prompt does that). Anthropic bills only generated tokens, and max_tokens + * does not count toward output rate limits, so the ceiling costs nothing. + */ function resolveAnthropicMaxOutputTokens( - effort: AnthropicReasoningEffort, discoveredMaximum: number | undefined, ): number { - const target = - effort === 'xhigh' || effort === 'max' - ? DEEP_ANTHROPIC_MAX_OUTPUT_TOKENS - : DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS; - const boundedMaximum = - typeof discoveredMaximum === 'number' && + return typeof discoveredMaximum === 'number' && Number.isSafeInteger(discoveredMaximum) && discoveredMaximum > 0 - ? discoveredMaximum - : DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS; - - return Math.min(target, boundedMaximum); + ? discoveredMaximum + : DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS; } function anthropicInputSchema( @@ -5419,7 +5464,6 @@ export class AnthropicAgentRuntime implements AgentRuntimeHandle { config.mcpAccessLevel ?? 'unrestricted', ); this.maxOutputTokens = resolveAnthropicMaxOutputTokens( - this.reasoningEffort, config.modelMaxOutputTokens, ); this.additionalInstructions = @@ -6011,7 +6055,18 @@ export class AnthropicAgentRuntime implements AgentRuntimeHandle { // to runtime state (also important for proxy/client implementations that // retain the params object until serialization). messages: [...state.messages], - system: state.system, + // Prompt caching. Every tool-loop step resends tools, system and history. + // The explicit breakpoint caches tools + system for the whole turn; the + // top-level one moves with the growing conversation, so each step reads + // the previous step's prefix instead of paying full input price again. + system: [ + { + type: 'text', + text: state.system, + cache_control: { type: 'ephemeral' }, + }, + ], + cache_control: { type: 'ephemeral' }, tools, tool_choice: { type: 'auto', @@ -6116,6 +6171,39 @@ export class AnthropicAgentRuntime implements AgentRuntimeHandle { // has been stopped. throwIfAborted(signal); state.lastResponseId = summary.message.id; + + if ( + summary.message.stop_reason === 'max_tokens' && + !state.outputLimitRecoveryUsed + ) { + state.outputLimitRecoveryUsed = true; + if (summary.text) { + state.accumulatedText += `${summary.text}\n\n`; + yield { + type: 'text_delta', + responseId: summary.message.id, + delta: '\n\n', + }; + } + appendAnthropicUserText( + state.messages, + OUTPUT_LIMIT_RECOVERY_INSTRUCTION, + ); + // Tell the editor why more, smaller approvals are about to follow. + yield { + type: 'activity', + responseId: summary.message.id, + activity: { + id: `output-limit-retry:${summary.message.id}`, + kind: 'continuation', + status: 'completed', + label: OUTPUT_LIMIT_RETRY_LABEL, + }, + }; + state.nextContinuation += 1; + continue; + } + state.accumulatedText += summary.text; state.messages.push(anthropicAssistantMessage(summary.message)); @@ -6127,8 +6215,8 @@ export class AnthropicAgentRuntime implements AgentRuntimeHandle { code: 'incomplete', message: summary.message.stop_reason === 'max_tokens' - ? 'The response stopped after reaching its output limit.' - : 'The conversation exceeded this model’s context window.', + ? OUTPUT_LIMIT_ERROR_MESSAGE + : CONTEXT_WINDOW_ERROR_MESSAGE, retryable: true, }; const result: AgentTurnResult = { diff --git a/dato-agent/src/lib/anthropicAgentRuntime.test.ts b/dato-agent/src/lib/anthropicAgentRuntime.test.ts index 057cab4b..41d67799 100644 --- a/dato-agent/src/lib/anthropicAgentRuntime.test.ts +++ b/dato-agent/src/lib/anthropicAgentRuntime.test.ts @@ -3,7 +3,6 @@ import type { Message, MessageCreateParamsStreaming, MessageStreamEvent, - StopReason, ToolResultBlockParam, ToolUseBlock, } from '@anthropic-ai/sdk/resources/messages/messages'; @@ -22,9 +21,9 @@ import type { import { ANTHROPIC_FAST_MODE_BETA, ANTHROPIC_FILES_API_BETA, + CONTEXT_WINDOW_ERROR_MESSAGE, createAgentRepairPolicy, createAgentRuntime, - DEEP_ANTHROPIC_MAX_OUTPUT_TOKENS, DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS, MAX_AGENT_HISTORY_CHARACTERS, MAX_AGENT_REQUEST_CHARACTERS, @@ -38,6 +37,8 @@ import { MAX_TOOL_RESULT_CHARACTERS, MAX_TOOL_RESULT_CHARACTERS_PER_TURN, normalizeAgentHistory, + OUTPUT_LIMIT_ERROR_MESSAGE, + OUTPUT_LIMIT_RETRY_LABEL, } from './agentRuntime'; import { MAX_CONVERSATION_MESSAGE_CHARACTERS } from './conversations'; import type { @@ -992,6 +993,14 @@ describe('AnthropicAgentRuntime', () => { }); expect(request.speed).toBeUndefined(); expect(request.betas).toBeUndefined(); + // Tools + system are cached for the turn; the top-level marker caches the growing history. + expect(request.cache_control).toEqual({ type: 'ephemeral' }); + expect(request.system).toEqual([ + expect.objectContaining({ + type: 'text', + cache_control: { type: 'ephemeral' }, + }), + ]); expect(JSON.stringify(request.system)).toContain( 'models:\\n- article | Article | title:string', ); @@ -1195,8 +1204,10 @@ describe('AnthropicAgentRuntime', () => { expect(toolNames).toContain('upsert_and_execute_safe_script'); expect(toolNames).not.toContain('upsert_and_execute_unsafe_script'); expect(toolNames).not.toContain('create_dato_asset'); - expect(firstRequest?.system).toContain('READ ONLY MODE'); - expect(firstRequest?.system).not.toContain('create_dato_asset'); + expect(JSON.stringify(firstRequest?.system)).toContain('READ ONLY MODE'); + expect(JSON.stringify(firstRequest?.system)).not.toContain( + 'create_dato_asset', + ); expect(createDatoAsset).not.toHaveBeenCalled(); expect(mcp.callTool).toHaveBeenCalledOnce(); expect(mcp.callTool).toHaveBeenCalledWith( @@ -1294,8 +1305,8 @@ describe('AnthropicAgentRuntime', () => { expect(JSON.stringify(request?.messages)).toContain( 'Writing tools are unavailable because Read Only is enabled.', ); - expect(request?.system).toContain('WRITABLE MODE'); - expect(request?.system).toContain( + expect(JSON.stringify(request?.system)).toContain('WRITABLE MODE'); + expect(JSON.stringify(request?.system)).toContain( "Plugin restrictions, OAuth access, and the account's project role all apply", ); expect((request?.tools ?? []).map((tool) => tool.name)).toContain( @@ -1485,27 +1496,27 @@ describe('AnthropicAgentRuntime', () => { { effort: 'low', discoveredMaximum: 128_000, - expectedMaximum: DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS, + expectedMaximum: 128_000, }, { effort: 'medium', discoveredMaximum: 128_000, - expectedMaximum: DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS, + expectedMaximum: 128_000, }, { effort: 'high', discoveredMaximum: 128_000, - expectedMaximum: DEFAULT_ANTHROPIC_MAX_OUTPUT_TOKENS, + expectedMaximum: 128_000, }, { effort: 'xhigh', discoveredMaximum: 128_000, - expectedMaximum: DEEP_ANTHROPIC_MAX_OUTPUT_TOKENS, + expectedMaximum: 128_000, }, { effort: 'max', discoveredMaximum: 128_000, - expectedMaximum: DEEP_ANTHROPIC_MAX_OUTPUT_TOKENS, + expectedMaximum: 128_000, }, { effort: 'xhigh', @@ -1523,7 +1534,7 @@ describe('AnthropicAgentRuntime', () => { expectedMaximum: 8_000, }, ] as const)( - 'maps $effort effort to a model-safe $expectedMaximum token budget', + 'requests the model maximum ($expectedMaximum) at $effort effort', async ({ discoveredMaximum, effort, expectedMaximum }) => { const anthropic = new QueueAnthropicClient([ message(`msg-effort-${effort}-${expectedMaximum}`, [ @@ -3710,35 +3721,108 @@ describe('AnthropicAgentRuntime', () => { expect(mcp.close).toHaveBeenCalledOnce(); }); - it.each([ - ['max_tokens', 'output limit'], - ['model_context_window_exceeded', 'context window'], - ] satisfies Array<[StopReason, string]>)( - 'maps %s to a retryable incomplete turn', - async (stopReason, messagePart) => { - const anthropic = new QueueAnthropicClient([ - message('msg-incomplete', [textBlock('Partial answer.')], stopReason), - ]); - const mcp = mcpClientWith(); - const runtime = runtimeWith(anthropic, mcp.client); + it('maps model_context_window_exceeded to a retryable incomplete turn', async () => { + const anthropic = new QueueAnthropicClient([ + message( + 'msg-incomplete', + [textBlock('Partial answer.')], + 'model_context_window_exceeded', + ), + ]); + const mcp = mcpClientWith(); + const runtime = runtimeWith(anthropic, mcp.client); - const { result } = await drain( - runtime.streamTurn({ message: 'Describe this project.' }), - ); + const { result } = await drain( + runtime.streamTurn({ message: 'Describe this project.' }), + ); - expect(result).toMatchObject({ - status: 'incomplete', - responseId: 'msg-incomplete', - text: 'Partial answer.', - error: { - code: 'incomplete', - retryable: true, - message: expect.stringContaining(messagePart), - }, - }); - expect(mcp.close).toHaveBeenCalledOnce(); - }, - ); + expect(result).toMatchObject({ + status: 'incomplete', + responseId: 'msg-incomplete', + text: 'Partial answer.', + error: { + code: 'incomplete', + retryable: true, + message: CONTEXT_WINDOW_ERROR_MESSAGE, + }, + }); + expect(anthropic.requests).toHaveLength(1); + expect(mcp.close).toHaveBeenCalledOnce(); + }); + + it('discards a response cut off at the output limit and retries once in smaller steps', async () => { + const anthropic = new QueueAnthropicClient([ + message( + 'msg-truncated', + [ + textBlock('Creating all sections now.'), + toolUse('toolu-truncated', 'upsert_and_execute_script', {}), + ], + 'max_tokens', + ), + message('msg-recovered', [textBlock('Done in smaller steps.')]), + ]); + const mcp = mcpClientWith(); + const runtime = runtimeWith(anthropic, mcp.client); + + const { events, result } = await drain( + runtime.streamTurn({ message: 'Build the landing page.' }), + ); + + expect(result).toMatchObject({ + status: 'completed', + responseId: 'msg-recovered', + text: 'Creating all sections now.\n\nDone in smaller steps.', + }); + expect(mcp.callTool).not.toHaveBeenCalled(); + expect(events.some((event) => event.type === 'approval_required')).toBe( + false, + ); + expect(anthropic.requests).toHaveLength(2); + const retryMessages = anthropic.requests[1]?.messages ?? []; + expect(retryMessages.some((entry) => entry.role === 'assistant')).toBe( + false, + ); + expect(JSON.stringify(retryMessages.at(-1))).toContain( + 'reached the output limit', + ); + // The editor learns why smaller approvals follow. + expect(events).toContainEqual( + expect.objectContaining({ + type: 'activity', + activity: expect.objectContaining({ + kind: 'continuation', + label: OUTPUT_LIMIT_RETRY_LABEL, + }), + }), + ); + }); + + it('stops with a retryable incomplete turn when the retry also reaches the output limit', async () => { + const anthropic = new QueueAnthropicClient([ + message('msg-truncated-1', [textBlock('First.')], 'max_tokens'), + message('msg-truncated-2', [textBlock('Second.')], 'max_tokens'), + ]); + const mcp = mcpClientWith(); + const runtime = runtimeWith(anthropic, mcp.client); + + const { result } = await drain( + runtime.streamTurn({ message: 'Build the landing page.' }), + ); + + expect(result).toMatchObject({ + status: 'incomplete', + responseId: 'msg-truncated-2', + text: 'First.\n\nSecond.', + error: { + code: 'incomplete', + retryable: true, + message: OUTPUT_LIMIT_ERROR_MESSAGE, + }, + }); + expect(anthropic.requests).toHaveLength(2); + expect(mcp.close).toHaveBeenCalledOnce(); + }); it('surfaces an Anthropic refusal explanation as a non-retryable provider error', async () => { const refusedMessage: Message = { diff --git a/dato-agent/src/lib/providerModels.test.ts b/dato-agent/src/lib/providerModels.test.ts index beca546d..30b96e60 100644 --- a/dato-agent/src/lib/providerModels.test.ts +++ b/dato-agent/src/lib/providerModels.test.ts @@ -1,10 +1,12 @@ import { describe, expect, it, vi } from 'vitest'; import { + formatTokenCount, isCompatibleOpenAiAgentModel, listAnthropicProviderModels, listOpenAiProviderModels, listProviderModels, preferredProviderModel, + providerModelOptionLabel, providerModelSupportsFastMode, } from './providerModels'; @@ -397,3 +399,57 @@ describe('provider model discovery', () => { ); }); }); + +describe('model selector limits', () => { + it('reads the context window and shows both limits in the label', async () => { + const fetchModels = vi.fn().mockResolvedValueOnce( + jsonResponse({ + data: [ + { + ...anthropicModel({ + id: 'claude-haiku-5-5', + displayName: 'Claude Haiku 5.5', + maxTokens: 128_000, + }), + max_input_tokens: 1_000_000, + }, + ], + has_more: false, + }), + ); + + const [model] = await listAnthropicProviderModels( + 'sk-ant-project', + undefined, + fetchModels, + ); + + expect(model).toMatchObject({ + contextWindowTokens: 1_000_000, + maxOutputTokens: 128_000, + }); + expect(model && providerModelOptionLabel(model)).toBe( + 'Claude Haiku 5.5 · 1M context · 128K output', + ); + }); + + it('keeps the plain label when the provider reports no limits', () => { + expect( + providerModelOptionLabel({ + id: 'gpt-5.6-terra', + label: 'gpt-5.6-terra', + reasoningEfforts: ['low'], + }), + ).toBe('gpt-5.6-terra'); + }); + + it.each([ + [1_000_000, '1M'], + [200_000, '200K'], + [64_000, '64K'], + [1_500_000, '1.5M'], + [12_345, '12,345'], + ])('formats %i tokens as %s', (tokens, expected) => { + expect(formatTokenCount(tokens)).toBe(expected); + }); +}); diff --git a/dato-agent/src/lib/providerModels.ts b/dato-agent/src/lib/providerModels.ts index 20d4f0f4..1957a9b1 100644 --- a/dato-agent/src/lib/providerModels.ts +++ b/dato-agent/src/lib/providerModels.ts @@ -19,6 +19,8 @@ type FetchModels = ( export type ProviderModel = { id: string; label: string; + /** Input context window, when the provider's model list reports it. */ + contextWindowTokens?: number; maxOutputTokens?: number; reasoningEfforts: ReasoningEffort[]; }; @@ -279,10 +281,12 @@ function extractAnthropicModel(entry: unknown): ProviderModel | undefined { const displayName = typeof entry.display_name === 'string' ? entry.display_name.trim() : ''; const maxOutputTokens = positiveSafeInteger(entry.max_tokens); + const contextWindowTokens = positiveSafeInteger(entry.max_input_tokens); return { id, label: displayName || id, + ...(contextWindowTokens ? { contextWindowTokens } : {}), ...(maxOutputTokens ? { maxOutputTokens } : {}), reasoningEfforts: [...reasoningEfforts], }; @@ -429,3 +433,26 @@ export function preferredProviderModel( return models.find((model) => model.id === DEFAULT_MODEL) ?? models[0]; } + +/** Compact token count for labels: 1000000 → "1M", 128000 → "128K". */ +export function formatTokenCount(tokens: number): string { + if (tokens >= 1_000_000 && tokens % 100_000 === 0) + return `${tokens / 1_000_000}M`; + if (tokens >= 1_000 && tokens % 1_000 === 0) return `${tokens / 1_000}K`; + return tokens.toLocaleString('en'); +} + +/** Model selector label with the limits developers rarely know offhand. */ +export function providerModelOptionLabel(model: ProviderModel): string { + const limits = [ + model.contextWindowTokens + ? `${formatTokenCount(model.contextWindowTokens)} context` + : undefined, + model.maxOutputTokens + ? `${formatTokenCount(model.maxOutputTokens)} output` + : undefined, + ].filter(Boolean); + return limits.length > 0 + ? `${model.label} · ${limits.join(' · ')}` + : model.label; +} diff --git a/dato-agent/vitest.config.ts b/dato-agent/vitest.config.ts index d4f7d909..e8ef1c5b 100644 --- a/dato-agent/vitest.config.ts +++ b/dato-agent/vitest.config.ts @@ -1,5 +1,5 @@ import react from '@vitejs/plugin-react'; -import { defineConfig } from 'vitest/config'; +import { configDefaults, defineConfig } from 'vitest/config'; import { recordCommentsAliases } from './configAliases'; export default defineConfig({ @@ -11,5 +11,7 @@ export default defineConfig({ environment: 'jsdom', setupFiles: ['./src/test/setup.ts'], restoreMocks: true, + // Live tests hit real APIs; they run only through vitest.live.config.ts. + exclude: [...configDefaults.exclude, 'e2e/**'], }, }); diff --git a/dato-agent/vitest.live.config.ts b/dato-agent/vitest.live.config.ts new file mode 100644 index 00000000..79a95639 --- /dev/null +++ b/dato-agent/vitest.live.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from 'vitest/config'; +import { recordCommentsAliases } from './configAliases'; + +/** + * Live tests and evals: real provider APIs, the real DatoCMS MCP and a forked + * environment of the seed project. Never part of `npm test`; run on demand + * with `npm run test:live` or `npm run eval`. + */ +export default defineConfig({ + resolve: { alias: recordCommentsAliases }, + test: { + environment: 'node', + include: ['e2e/**/*.live.test.ts', 'e2e/**/*.eval.ts'], + fileParallelism: false, + testTimeout: 30 * 60_000, + hookTimeout: 5 * 60_000, + }, +});