|
| 1 | +// Generic, objective-driven browser task — the production path for browser agents |
| 2 | +// (issue #69/#71). It REUSES apply's proven engine (`runApplyLoop` + `ApplyDeps` + |
| 3 | +// the runner `/browser/*` primitives + captcha/stuck/needs_input handoffs); only the |
| 4 | +// prompt, the tool set, and the decision step are generalized from job-specific to |
| 5 | +// "achieve this objective on this page". The agent supplies the objective as DATA, so |
| 6 | +// the platform stays domain-agnostic (no "facebook"/"friends" in core). |
| 7 | +import { runUserWorkersAi } from "./user-ai.js"; |
| 8 | +import { type ApplyDecision, type BrowserAction, type BrowserJobBase, type PageSnapshot } from "./apply-loop.js"; |
| 9 | +import type { UsageContext } from "./usage.js"; |
| 10 | +import type { Env } from "../types.js"; |
| 11 | + |
| 12 | +/** The minimal job a generic browser task needs. Everything task-specific is `objective`. */ |
| 13 | +export interface BrowserTaskJob extends BrowserJobBase { |
| 14 | + /** Where the run starts (the agent's declared start URL, or a per-run override). */ |
| 15 | + url: string; |
| 16 | + /** What to accomplish — the agent's declared goal + the subscriber's settings, |
| 17 | + * rendered by the trigger route. This is the ONLY task-specific input. */ |
| 18 | + objective: string; |
| 19 | + /** The subscriber's free-text rules (Special Instructions); injected at the top. */ |
| 20 | + specialInstructions?: string; |
| 21 | + /** Live free-text message the user sent while the agent was paused/stuck. */ |
| 22 | + userHint?: string; |
| 23 | + /** Today's date (YYYY-MM-DD), stamped at trigger time — for any date fields. */ |
| 24 | + today?: string; |
| 25 | + /** Test mode: the workflow's act layer blocks the commit action (click Confirm/ |
| 26 | + * Submit/Send/…) so a run can be walked safely without committing anything. */ |
| 27 | + dryRun?: boolean; |
| 28 | + /** Values the user supplied mid-run via a needs_input handoff (field → value), |
| 29 | + * fed back so the agent never re-asks. */ |
| 30 | + providedAnswers?: Record<string, string>; |
| 31 | +} |
| 32 | + |
| 33 | +/** Verbs that COMMIT an irreversible action — blocked in dry-run, and the model is told |
| 34 | + * to treat them as the final step. Kept here (not the workflow) so both agree. */ |
| 35 | +export const COMMIT_VERB_RE = |
| 36 | + /\b(confirm|accept|add friend|add|submit|send|post|publish|delete|remove|pay|purchase|buy|apply|approve|agree|save)\b/i; |
| 37 | + |
| 38 | +function tool(name: string, description: string, props: Record<string, { type: string; description: string }>, required: string[]) { |
| 39 | + return { type: "function" as const, function: { name, description, parameters: { type: "object", properties: props, required } } }; |
| 40 | +} |
| 41 | + |
| 42 | +const REF = { type: "string", description: 'the element\'s ref from the snapshot, e.g. "e42" — ALWAYS include it; it targets the exact element even when two share a label' }; |
| 43 | + |
| 44 | +/** The browser actions exposed to Claude for a generic task. Same primitives as apply, |
| 45 | + * minus résumé upload; `finish` uses generic outcomes (done|blocked). */ |
| 46 | +export const BROWSER_TASK_TOOLS = [ |
| 47 | + tool("click", "Click a button, link, or control. Target it by its snapshot ref.", { |
| 48 | + ref: REF, |
| 49 | + role: { type: "string", description: '"button" or "link"' }, |
| 50 | + name: { type: "string", description: "the control's visible text/label" }, |
| 51 | + }, ["ref"]), |
| 52 | + tool("type", "Type text into a field. Target it by its snapshot ref.", { |
| 53 | + ref: REF, |
| 54 | + name: { type: "string", description: "the field's label/accessible name" }, |
| 55 | + text: { type: "string", description: "text to type" }, |
| 56 | + }, ["ref", "text"]), |
| 57 | + tool("select", "Choose an option in a dropdown/combobox. Target the control by its snapshot ref.", { |
| 58 | + ref: REF, |
| 59 | + name: { type: "string", description: "the select's label" }, |
| 60 | + value: { type: "string", description: "the option label to choose" }, |
| 61 | + }, ["ref", "value"]), |
| 62 | + tool("check", "Check a checkbox or radio. Target it by its snapshot ref.", { |
| 63 | + ref: REF, |
| 64 | + name: { type: "string", description: "the option's label" }, |
| 65 | + }, ["ref"]), |
| 66 | + tool("navigate", "Go to a URL.", { url: { type: "string", description: "absolute http(s) URL" } }, ["url"]), |
| 67 | + tool("scroll", "Scroll the page vertically to reveal more.", { dy: { type: "number", description: "pixels (positive = down)" } }, []), |
| 68 | + tool("press_key", "Press a keyboard key like Enter or Tab.", { key: { type: "string", description: "e.g. Enter, Tab, Escape" } }, ["key"]), |
| 69 | + tool("request_user_info", "Ask the USER for a REQUIRED value you don't have. Use this INSTEAD of inventing one. The run pauses until they answer, then resumes.", { |
| 70 | + field: { type: "string", description: "the value you need" }, |
| 71 | + why: { type: "string", description: "brief reason / where it's needed" }, |
| 72 | + }, ["field"]), |
| 73 | + tool("finish", "End the run: status done (objective achieved / nothing left to do) or blocked (cannot proceed truthfully).", { |
| 74 | + status: { type: "string", description: "done | blocked" }, |
| 75 | + detail: { type: "string", description: "short summary of what you did / why you stopped" }, |
| 76 | + }, ["status", "detail"]), |
| 77 | +]; |
| 78 | + |
| 79 | +/** Build the objective-driven system prompt (replaces the ~job-specific apply prompt). */ |
| 80 | +export function browserTaskSystemPrompt(job: BrowserTaskJob): string { |
| 81 | + const lines = [ |
| 82 | + "You are a web-automation agent operating a real web browser through tools.", |
| 83 | + "You see each page as an accessibility snapshot: every element shows its role, accessible name, current value, state (e.g. [disabled], [checked], [expanded]), and a stable reference like [ref=e42]. You act ONLY through the provided tools — no CSS selectors.", |
| 84 | + "- TARGET BY REF: always pass the element's exact `ref` from the snapshot (e.g. \"e42\"). Include the accessible name too for clarity.", |
| 85 | + "- Read element STATE: a control marked [disabled]/[readonly] can't be changed — move on. [checked]/[expanded] tell you current state.", |
| 86 | + job.userHint |
| 87 | + ? `\n‼️ LIVE MESSAGE FROM THE USER — they are watching right now and just sent this; it describes the CURRENT screen. TRUST IT: re-read the snapshot and act on it, do NOT repeat the action you were stuck on. Message: "${job.userHint}"\n` |
| 88 | + : "", |
| 89 | + job.specialInstructions |
| 90 | + ? `\n★ USER'S SPECIAL INSTRUCTIONS — follow strictly; they OVERRIDE the defaults below:\n${job.specialInstructions}\n` |
| 91 | + : "", |
| 92 | + "", |
| 93 | + "## YOUR OBJECTIVE", |
| 94 | + job.objective, |
| 95 | + "", |
| 96 | + ...(job.providedAnswers && Object.keys(job.providedAnswers).length |
| 97 | + ? ["Values you asked the user for (use these, don't ask again):", ...Object.entries(job.providedAnswers).map(([k, v]) => ` • ${k}: ${v}`), ""] |
| 98 | + : []), |
| 99 | + "## RULES", |
| 100 | + "- Do exactly ONE tool call per turn — the next page snapshot follows.", |
| 101 | + "- After you click something that changes the page (Next/Confirm/a link), act on the NEW snapshot, not the element you just clicked.", |
| 102 | + "- If the action log shows your last action FAILED, do NOT repeat it — re-read the CURRENT snapshot and act on what's there now.", |
| 103 | + "- DROPDOWNS/COMBOBOXES: first try `select`. If it fails (\"not a <select>\"), it's a custom widget — `click` it to open, then `click` the matching option (or type a few letters to filter, then click).", |
| 104 | + "- Stay STRICTLY within the objective. Do NOT take any action beyond it — no posting, messaging, changing settings, or interacting with anything the objective doesn't ask for.", |
| 105 | + "- NEVER invent data. For a REQUIRED value you weren't given, call request_user_info(field, why) and wait — do not guess.", |
| 106 | + "- LOGIN / CHECKPOINT / CAPTCHA: if you see a sign-in screen, security check, 2FA, or a \"verify you are human\" challenge, do NOTHING — the system hands off to a human automatically. NEVER attempt to sign in or solve a captcha yourself.", |
| 107 | + "- Respect any count limit in the objective (e.g. \"up to 20\"): track how many you've done and STOP at the limit.", |
| 108 | + "- When the objective is complete, or there is nothing left to act on, call finish(status:\"done\", detail:\"<what you did>\").", |
| 109 | + "- If you genuinely cannot proceed, call finish(status:\"blocked\", detail:\"<why>\").", |
| 110 | + "- Be decisive and brief. Do not narrate.", |
| 111 | + job.dryRun |
| 112 | + ? "- ⛔ TEST MODE (dry run): walk the flow but do NOT perform the committing action. When you would click a final Confirm/Add/Submit/Send/Post/Delete/Pay button, STOP and call finish(status:\"done\", detail:\"test mode — reached <button>\") instead. (Such clicks are blocked anyway.)" |
| 113 | + : "", |
| 114 | + ]; |
| 115 | + return lines.filter((l) => l !== "").join("\n"); |
| 116 | +} |
| 117 | + |
| 118 | +/** Map a Claude tool call to a loop decision. `finish` uses generic outcomes mapped |
| 119 | + * onto the shared ApplyDecision shape (done→"submitted" success, blocked→"blocked"). */ |
| 120 | +export function browserToolCallToDecision(call: { name: string; arguments: Record<string, unknown> }, thought?: string): ApplyDecision { |
| 121 | + const a = call.arguments || {}; |
| 122 | + const str = (v: unknown) => (typeof v === "string" ? v : v == null ? undefined : String(v)); |
| 123 | + const num = (v: unknown) => (typeof v === "number" ? v : undefined); |
| 124 | + switch (call.name) { |
| 125 | + case "click": return { thought, action: { action: "click", ref: str(a.ref), role: str(a.role) || "button", name: str(a.name) } }; |
| 126 | + case "type": return { thought, action: { action: "type", ref: str(a.ref), role: "textbox", name: str(a.name), text: str(a.text) ?? "" } }; |
| 127 | + case "select": return { thought, action: { action: "select", ref: str(a.ref), role: "combobox", name: str(a.name), text: str(a.value) } }; |
| 128 | + case "check": return { thought, action: { action: "check", ref: str(a.ref), role: "checkbox", name: str(a.name) } }; |
| 129 | + case "navigate": return { thought, action: { action: "navigate", url: str(a.url) } }; |
| 130 | + case "scroll": return { thought, action: { action: "scroll", dy: num(a.dy) ?? 600 } }; |
| 131 | + case "press_key": return { thought, action: { action: "key", key: str(a.key) || "Enter" } }; |
| 132 | + case "request_user_info": return { thought, needsInput: { field: str(a.field) || "this value", why: str(a.why) } }; |
| 133 | + case "finish": { |
| 134 | + const done = str(a.status) === "done"; |
| 135 | + return { thought, finish: { status: done ? "submitted" : "blocked", detail: str(a.detail) || "" } }; |
| 136 | + } |
| 137 | + default: |
| 138 | + // Unknown tool — re-read the page and try again (step cap prevents runaway). |
| 139 | + return { thought, action: { action: "wait", ms: 300 } }; |
| 140 | + } |
| 141 | +} |
| 142 | + |
| 143 | +/** The decision step: ask Claude (BYOK) for the next action given the page + objective. */ |
| 144 | +export async function decideBrowserTask( |
| 145 | + env: Env, |
| 146 | + userId: string, |
| 147 | + params: { job: BrowserTaskJob; actionLog: string[]; snapshot: PageSnapshot }, |
| 148 | + usageCtx?: UsageContext, |
| 149 | +): Promise<ApplyDecision> { |
| 150 | + const MAX_LOG = 30; |
| 151 | + const log = |
| 152 | + params.actionLog.length > MAX_LOG |
| 153 | + ? [`… (${params.actionLog.length - MAX_LOG} earlier steps omitted) …`, ...params.actionLog.slice(-MAX_LOG)] |
| 154 | + : params.actionLog; |
| 155 | + const userMsg = [ |
| 156 | + `Actions so far:\n${log.length ? log.map((a, i) => `${i + 1}. ${a}`).join("\n") : "(none yet)"}`, |
| 157 | + `\nCURRENT PAGE — ${params.snapshot.title || ""} <${params.snapshot.url}>`, |
| 158 | + params.snapshot.snapshot, |
| 159 | + "\nDo the single next action toward the objective. Call exactly one tool.", |
| 160 | + ].join("\n"); |
| 161 | + |
| 162 | + const res = (await runUserWorkersAi(env, userId, "claude-sonnet-4-6", { |
| 163 | + messages: [ |
| 164 | + { role: "system", content: browserTaskSystemPrompt(params.job) }, |
| 165 | + { role: "user", content: userMsg }, |
| 166 | + ], |
| 167 | + tools: BROWSER_TASK_TOOLS, |
| 168 | + timeoutMs: 60_000, |
| 169 | + }, usageCtx)) as { response?: string; tool_calls?: Array<{ name: string; arguments: Record<string, unknown> }>; usage?: { input: number; output: number } }; |
| 170 | + |
| 171 | + const call = res.tool_calls?.[0]; |
| 172 | + if (!call) { |
| 173 | + return { thought: res.response, finish: { status: "blocked", detail: res.response || "no action chosen" }, usage: res.usage }; |
| 174 | + } |
| 175 | + const decision = browserToolCallToDecision(call, res.response); |
| 176 | + decision.usage = res.usage; |
| 177 | + return decision; |
| 178 | +} |
| 179 | + |
| 180 | +/** Is this action the committing step (blocked in dry-run)? Used by the workflow's act layer. */ |
| 181 | +export function isCommitClick(action: BrowserAction): boolean { |
| 182 | + return action.action === "click" && COMMIT_VERB_RE.test(action.name ?? ""); |
| 183 | +} |
0 commit comments