diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 4ba1a0f54ba..a2ba4e7629e 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -106,6 +106,18 @@ func requireNonSecure(_ element: AXUIElement) throws { func rectJSON(_ rect: CGRect) -> [String: Double] { ["x": rect.minX, "y": rect.minY, "width": rect.width, "height": rect.height] } +/// Whether keyboard focus is on `element` or an editor inside it, such as a combo box's field. +func holdsFocus(_ focused: CFTypeRef, _ element: AXUIElement) -> Bool { + guard CFGetTypeID(focused) == AXUIElementGetTypeID() else { return false } + var current: AXUIElement? = unsafeBitCast(focused, to: AXUIElement.self) + for _ in 0..<8 { + guard let candidate = current else { return false } + if CFEqual(candidate, element) { return true } + guard let parent = attribute(candidate, kAXParentAttribute), CFGetTypeID(parent) == AXUIElementGetTypeID() else { return false } + current = unsafeBitCast(parent, to: AXUIElement.self) + } + return false +} func axActions(_ element: AXUIElement) -> [String] { var names: CFArray? return AXUIElementCopyActionNames(element, &names) == .success ? names as? [String] ?? [] : [] @@ -224,6 +236,8 @@ final class Snapshot { /// The Accessibility window behind each observed windowId, so input checks compare identity, not /// just bounds that two windows can share. var windowElements: [String: AXUIElement] = [:] + /// The window this observation selected; coordinate input without a windowId targets it. + var selectedWindowId: String? var nodes: [[String: Any]] = [] var truncated = false init(app: NSRunningApplication) { pid = app.processIdentifier; launchDate = app.launchDate } @@ -423,14 +437,34 @@ func requireForegroundTarget(pid: pid_t, windowID: String, window: AXUIElement, throw ComputerError("window_occluded", "Another of this app's windows covers the target point; observe that window and target it instead.") } } else if ![kAXMenuRole, kAXMenuItemRole, kAXMenuBarRole, kAXMenuBarItemRole].contains(attribute(hit, kAXRoleAttribute) as? String ?? "") { - throw ComputerError("window_occluded", "Cannot confirm which window receives this point; observe again before retrying.") + // Out-of-process panels (open and save) expose their contents without a window; they are + // reachable only inside the app's focused sheet, which must belong to the observed window. + let root = AXUIElementCreateApplication(pid) + guard let rawSheet = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(rawSheet) == AXUIElementGetTypeID(), + case let sheet = unsafeBitCast(rawSheet, to: AXUIElement.self), attribute(sheet, kAXRoleAttribute) as? String == kAXSheetRole, + elementRect(sheet)?.contains(point) == true, acceptsInput(sheet, observed: [(window, frame)]) else { + throw ComputerError("window_occluded", "Cannot confirm which window receives this point; observe again before retrying.") + } } } +/// The window a sheet is attached to: its parent, or for an out-of-process panel that reports no +/// parent, the window it reports as its own. +func sheetOwner(_ sheet: AXUIElement) -> AXUIElement? { + for name in [kAXParentAttribute, kAXWindowAttribute] { + if let raw = attribute(sheet, name), CFGetTypeID(raw) == AXUIElementGetTypeID() { + let owner = unsafeBitCast(raw, to: AXUIElement.self) + if attribute(owner, kAXRoleAttribute) as? String == kAXWindowRole { return owner } + } + } + return nil +} /// Whether input aimed at an observed window may go to `window`: the observed window itself, or one of /// the app's dialogs or floating panels above it. Another document window, or one whose kind cannot be /// read, is refused. func acceptsInput(_ window: AXUIElement, observed: [(element: AXUIElement, frame: CGRect)]) -> Bool { if observed.contains(where: { CFEqual($0.element, window) && elementRect(window) == $0.frame }) { return true } + // A sheet (Go to Folder, an open or save panel) belongs to the window it is attached to. + if attribute(window, kAXRoleAttribute) as? String == kAXSheetRole { return sheetOwner(window).map { acceptsInput($0, observed: observed) } ?? false } let auxiliary: Set = [kAXDialogSubrole, kAXSystemDialogSubrole, kAXFloatingWindowSubrole, kAXSystemFloatingWindowSubrole] return (attribute(window, kAXSubroleAttribute) as? String).map(auxiliary.contains) ?? false } @@ -502,7 +536,7 @@ final class Driver { guard let frame = elementRect(try snapshot.element(elementId)) else { throw ComputerError("element_unavailable", "Element has no usable frame.") } return CGPoint(x: frame.midX, y: frame.midY) } - let windowId = try required(p.windowId, "windowId") + let windowId = try required(p.windowId ?? snapshot.selectedWindowId, "windowId") let frame = try windowRect(windowId, pid: app.processIdentifier) guard snapshot.windowFrames[windowId] == frame else { throw ComputerError("stale_window", "Window moved or changed; observe it again.") } let x = try required(end ? p.toX : p.x, end ? "toX" : "x"); let y = try required(end ? p.toY : p.y, end ? "toY" : "y") @@ -579,6 +613,7 @@ final class Driver { } guard let selectedWindowId = p.windowId ?? focusedID ?? windows.first?["windowId"] as? String else { throw ComputerError("app_window_unavailable", "No accessible app window is available. Open a window, then observe the app again.") } guard snapshot.windowFrames[selectedWindowId] != nil else { throw ComputerError("window_unavailable", "Selected window does not belong to the app.") } + snapshot.selectedWindowId = selectedWindowId var result: [String: Any] = ["kind": "state", "windowId": selectedWindowId, "snapshotId": snapshot.id, "bundleId": app.bundleIdentifier!, "nodes": snapshot.nodes, "windows": windows, "truncated": snapshot.truncated] if p.includeScreenshot == true { do { @@ -633,7 +668,8 @@ final class Driver { } else { try await coordinateAction { dispatch in let position = try point(p, app: app, snapshot: snapshot); let mouse: CGMouseButton = button == "right" ? .right : .left - let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId") + // Coordinates are window-local to the requested (or selected) window; an element target uses its own. + let windowID = try required(p.windowId ?? (p.elementId == nil ? snapshot.selectedWindowId : snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key), "windowId") let frame = try required(snapshot.windowFrames[windowID], "window frame") for click in 1...count { try checkCancellation() @@ -660,18 +696,6 @@ final class Driver { try axCheck(AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)) try await Task.sleep(for: .milliseconds(50)) } - /// Focus may land on the element itself or on an editor inside it, such as a combo box's field. - func holdsFocus(_ focused: CFTypeRef) -> Bool { - guard CFGetTypeID(focused) == AXUIElementGetTypeID() else { return false } - var current: AXUIElement? = unsafeBitCast(focused, to: AXUIElement.self) - for _ in 0..<8 { - guard let candidate = current else { return false } - if CFEqual(candidate, element) { return true } - guard let parent = attribute(candidate, kAXParentAttribute), CFGetTypeID(parent) == AXUIElementGetTypeID() else { return false } - current = unsafeBitCast(parent, to: AXUIElement.self) - } - return false - } let observedWindows = snapshot.observedWindows() func validateFocus() throws { try checkCancellation(); try requireNonSecure(element) @@ -680,7 +704,7 @@ final class Driver { acceptsInput(unsafeBitCast(rawWindow, to: AXUIElement.self), observed: observedWindows) else { throw ComputerError("stale_window", "The editor's window moved or is not an observed window; observe again.") } - guard let current = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(current) else { throw ComputerError("focus_changed", "The exact editor is no longer focused; remaining input was not dispatched. Observe again.") } + guard let current = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(current, element) else { throw ComputerError("focus_changed", "The exact editor is no longer focused; remaining input was not dispatched. Observe again.") } if let focused = attribute(root, kAXFocusedUIElementAttribute), CFGetTypeID(focused) == AXUIElementGetTypeID() { try requireNonSecure(unsafeBitCast(focused, to: AXUIElement.self)) } } // AX focus requests and app activation may settle asynchronously. Wait without @@ -730,6 +754,21 @@ final class Driver { return frame } let root = AXUIElementCreateApplication(pid) + // A targeted key first moves keyboard focus to that element, as clicking into it would. + if let elementId = p.elementId { + let element = try snapshot.element(elementId) + if attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) != true { + try axCheck(AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)) + } + // Focus can settle asynchronously; the key is sent only once this element holds it. + var focused = false + for _ in 0..<10 { + try checkCancellation() + if attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) == true { focused = true; break } + try await Task.sleep(for: .milliseconds(50)) + } + guard focused else { throw ComputerError("focus_changed", "The element did not take keyboard focus; no key was sent. Observe again.") } + } // Keys go to the app's key window: the observed window, or a dialog, panel or popover above it, // but never another document window the snapshot did not observe. guard let rawWindow = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(rawWindow) == AXUIElementGetTypeID() else { throw ComputerError("focus_failed", "The app has no focused window to receive keys; click a window or call activate_app, then observe again.") } @@ -755,7 +794,8 @@ final class Driver { let position = try point(p, app: app, snapshot: snapshot); let dx = p.deltaX ?? 0; let dy = p.deltaY ?? 0 guard dx.isFinite, dy.isFinite, abs(dx) <= 10000, abs(dy) <= 10000 else { throw ComputerError("invalid_arguments", "Scroll deltas exceed bounds.") } guard let event = CGEvent(scrollWheelEvent2Source: CGEventSource(stateID: .privateState), units: .pixel, wheelCount: 2, wheel1: Int32(-dy), wheel2: Int32(-dx), wheel3: 0) else { throw ComputerError("input_failed", "Could not create scroll event.") } - let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId") + // Coordinates are window-local to the requested (or selected) window; an element target uses its own. + let windowID = try required(p.windowId ?? (p.elementId == nil ? snapshot.selectedWindowId : snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key), "windowId") let frame = try required(snapshot.windowFrames[windowID], "window frame") try requireForegroundTarget(pid: pid, windowID: windowID, window: try required(snapshot.windowElements[windowID], "observed window"), point: position, frame: frame) event.location = position; dispatch.willDispatch(); event.post(tap: .cghidEventTap) @@ -763,7 +803,7 @@ final class Driver { case "drag": try await coordinateAction { dispatch in let start = try point(p, app: app, snapshot: snapshot); let end = try point(p, app: app, snapshot: snapshot, end: true) - let windowID = try required(p.windowId, "windowId"); let frame = try required(snapshot.windowFrames[windowID], "window frame") + let windowID = try required(p.windowId ?? snapshot.selectedWindowId, "windowId"); let frame = try required(snapshot.windowFrames[windowID], "window frame") try postMouse(pid: pid, type: .leftMouseDown, point: start, button: .left, windowID: windowID, window: snapshot.windowElements[windowID], frame: frame, dispatch: dispatch) var lastPoint = start defer { try? postMouse(pid: pid, type: .leftMouseUp, point: lastPoint, button: .left, windowID: windowID, window: snapshot.windowElements[windowID], frame: frame, release: true, dispatch: dispatch) } diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index 62ca40c9f22..1a9ea084d22 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -10,9 +10,9 @@ Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` return The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized; menu traversal is deferred and capped at 80 nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise. -Coordinates in action inputs are window-local points; node frames are global screen points. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps and System Settings are protected targets. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`. +Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps and System Settings are protected targets. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`. -AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A press that times out while the app opens a sheet or modal (`cannotComplete`) counts as dispatched. A menu-item press that leaves its menu open with the item highlighted (Calendar on macOS 26) is completed with Return. Open menus and their submenus are observed first, and menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. +AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A press that times out while the app opens a sheet or modal (`cannotComplete`) counts as dispatched. A menu-item press that leaves its menu open with the item highlighted (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Open menus and their submenus are observed first, and menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. `type_text` and `input_sequence` target any observed element that can take keyboard focus (text fields, terminals, combo boxes, web editors). `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index f348397a78a..e3149dc6aed 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -228,12 +228,18 @@ export class ComputerUseService { } this.check(work) const owner = 'snapshotId' in input ? this.snapshots.get(input.snapshotId) : undefined + const elementId = 'elementId' in input ? input.elementId : undefined + /** + * Element and coordinate calls observe the window they acted on. An untargeted key keeps the + * focused-window default, so a key that closes the window still returns fresh state. + */ const windowId = - 'windowId' in input - ? input.windowId - : 'elementId' in input - ? (owner?.elementWindows.get(input.elementId) ?? owner?.windowId) - : undefined + ('windowId' in input ? input.windowId : undefined) ?? + (elementId + ? (owner?.elementWindows.get(elementId) ?? owner?.windowId) + : 'x' in input + ? owner?.windowId + : undefined) if ('snapshotId' in input) this.snapshots.delete(input.snapshotId) /** Mutations accept includeScreenshot as shorthand for observeAfter.includeScreenshot. */ const isMutation = input.action !== 'get_app_state' diff --git a/apps/sim/lib/mothership/generated/computer-use.ts b/apps/sim/lib/mothership/generated/computer-use.ts index 378cd1b288c..006d387b6c6 100644 --- a/apps/sim/lib/mothership/generated/computer-use.ts +++ b/apps/sim/lib/mothership/generated/computer-use.ts @@ -33,7 +33,15 @@ const InputSequenceStep = z.discriminatedUnion("action", [ ]); const ElementTarget = { bundleId: BundleId, snapshotId: SnapshotId, elementId: ElementId }; const WindowTarget = { bundleId: BundleId, snapshotId: SnapshotId, windowId: WindowId }; -const PointTarget = { ...WindowTarget, x: Coordinate, y: Coordinate }; +const PointTarget = { + bundleId: BundleId, + snapshotId: SnapshotId, + windowId: WindowId.optional().describe( + "Window reference from that snapshot. Defaults to the window the snapshot selected.", + ), + x: Coordinate, + y: Coordinate, +}; const ClickOptions = { button: z.enum(["left", "right"]).optional(), clickCount: z.number().int().min(1).max(3).optional(), @@ -43,14 +51,19 @@ const ScrollOffsets = { .number() .min(-10_000) .max(10_000) - .describe("Horizontal scroll distance in points; positive scrolls right, negative left."), + .optional() + .describe("Horizontal scroll distance in points; positive scrolls right, negative left. Defaults to 0."), deltaY: z .number() .min(-10_000) .max(10_000) - .describe("Vertical scroll distance in points; positive scrolls down, negative up."), + .optional() + .describe("Vertical scroll distance in points; positive scrolls down, negative up. Defaults to 0."), }; +const hasScrollDistance = (input: { deltaX?: number; deltaY?: number }) => + input.deltaX !== undefined || input.deltaY !== undefined; + /** Strict branches require an observed, unambiguous target before native input. */ export const ComputerUseSchema = z.union([ z.strictObject({ action: z.literal("status") }), @@ -81,10 +94,11 @@ export const ComputerUseSchema = z.union([ windowId: WindowId.optional().describe( "Window reference from that snapshot. Omit to send the key to the app's focused window.", ), + elementId: ElementId.optional().describe("Focus this element from the snapshot before pressing the key."), key: z.string().min(1).max(128).describe("One key or chord, such as Enter, Tab, Escape, or Cmd+A."), }), - z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }), - z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ObserveAfter, ...ScrollOffsets }), + z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }).refine(hasScrollDistance, "Scroll needs deltaX or deltaY."), + z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ObserveAfter, ...ScrollOffsets }).refine(hasScrollDistance, "Scroll needs deltaX or deltaY."), z.strictObject({ action: z.literal("drag"), ...PointTarget, diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index 378cd1b288c..006d387b6c6 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -33,7 +33,15 @@ const InputSequenceStep = z.discriminatedUnion("action", [ ]); const ElementTarget = { bundleId: BundleId, snapshotId: SnapshotId, elementId: ElementId }; const WindowTarget = { bundleId: BundleId, snapshotId: SnapshotId, windowId: WindowId }; -const PointTarget = { ...WindowTarget, x: Coordinate, y: Coordinate }; +const PointTarget = { + bundleId: BundleId, + snapshotId: SnapshotId, + windowId: WindowId.optional().describe( + "Window reference from that snapshot. Defaults to the window the snapshot selected.", + ), + x: Coordinate, + y: Coordinate, +}; const ClickOptions = { button: z.enum(["left", "right"]).optional(), clickCount: z.number().int().min(1).max(3).optional(), @@ -43,14 +51,19 @@ const ScrollOffsets = { .number() .min(-10_000) .max(10_000) - .describe("Horizontal scroll distance in points; positive scrolls right, negative left."), + .optional() + .describe("Horizontal scroll distance in points; positive scrolls right, negative left. Defaults to 0."), deltaY: z .number() .min(-10_000) .max(10_000) - .describe("Vertical scroll distance in points; positive scrolls down, negative up."), + .optional() + .describe("Vertical scroll distance in points; positive scrolls down, negative up. Defaults to 0."), }; +const hasScrollDistance = (input: { deltaX?: number; deltaY?: number }) => + input.deltaX !== undefined || input.deltaY !== undefined; + /** Strict branches require an observed, unambiguous target before native input. */ export const ComputerUseSchema = z.union([ z.strictObject({ action: z.literal("status") }), @@ -81,10 +94,11 @@ export const ComputerUseSchema = z.union([ windowId: WindowId.optional().describe( "Window reference from that snapshot. Omit to send the key to the app's focused window.", ), + elementId: ElementId.optional().describe("Focus this element from the snapshot before pressing the key."), key: z.string().min(1).max(128).describe("One key or chord, such as Enter, Tab, Escape, or Cmd+A."), }), - z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }), - z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ObserveAfter, ...ScrollOffsets }), + z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }).refine(hasScrollDistance, "Scroll needs deltaX or deltaY."), + z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ObserveAfter, ...ScrollOffsets }).refine(hasScrollDistance, "Scroll needs deltaX or deltaY."), z.strictObject({ action: z.literal("drag"), ...PointTarget,