diff --git a/apps/desktop/README.md b/apps/desktop/README.md index ebb50f9a412..63760a2a52c 100644 --- a/apps/desktop/README.md +++ b/apps/desktop/README.md @@ -218,3 +218,56 @@ Raw local file bytes are never exposed through the preload bridge and cannot be ## Electron upgrades Cadence: Electron ships a major every ~8 weeks and supports the latest 3 — budget a bump every ~4–6 months and adopt security patches within ~2 weeks. Follow `docs/electron-upgrade-checklist.md`; the `desktop-e2e.yml` canary leg (electron@latest) is the early-warning signal. + + +## Native Mac computer use + +Mothership's `computer` tool is available on macOS 14+ when the global runtime flag +`mothership-computer-use` is enabled and the user opts in under **Settings → Desktop → Computer Use**. +The flag defaults off; local/self-hosted deployments use `MSHIP_COMPUTER_USE=true`, while hosted +rollout uses the existing AppConfig feature-flags document. Older desktop builds omit this optional +bridge and continue to work. + +The user grants Accessibility and Screen Recording through macOS, then approves each app for the +current task or persistently. Persistent app approvals are listed in Settings and can be revoked. +Sign-out and deployment changes clear computer-use approval and opt-in. The conversation shows the +active app/action and a Stop button; **⌘⇧Esc** also stops active work when the shortcut is available. +Stop cancels queued actions, in-flight authorization, approval prompts and the native helper. +Already completed input cannot be undone. + +The tool lists installed/running apps, reads window accessibility trees, captures selected-window +screenshots, activates apps explicitly, and performs clicks, text input, keyboard shortcuts, +scrolling, dragging, value changes and accessibility actions. Semantic actions can run in the +background. Coordinate mouse actions require explicit foreground activation and a fresh observation; +they validate the target app/window before dispatch. Each mutation consumes its snapshot and the +model observes again to verify results. Secure accessibility values are suppressed. See +[`native/computer-use/README.md`](native/computer-use/README.md) for native bounds and compatibility +limits; screenshots can still contain sensitive content visibly rendered by the approved app. + +Every execution is authorized and claimed once by `/api/desktop/computer/authorize` using the +stored tool arguments, current chat access, run/Stop state and current flag. Renderer-supplied +arguments are not execution authority. The main process serializes native work and binds snapshots +to the authorized chat. The Swift helper communicates over private stdio and is packaged outside +ASAR as `Contents/Resources/Sim Computer Use.app`. + +Distribution signing must cover the nested helper with the same stable Developer ID identity. +Development ad-hoc builds can lose macOS grants when rebuilt. macOS can attribute permission +requests to the responsible parent app (Sim, Electron, or the terminal/IDE that launched a test), +so confirm the name in the system permission dialog. Test a Finder/LaunchServices-launched signed +build for release acceptance; terminal-launched helper tests do not establish that app's TCC grants. + +Validation includes desktop lifecycle/transport unit tests, the Electron authorization/Stop test +in `e2e/computer-use.spec.ts`, the native protocol and live fixture tests, Sim authorization/stream +regressions, and Mothership's opt-in `apps/server/scripts/computer-use-live.ts` real-model trial. +The live trial uses only the disposable fixture and verifies mutations by fresh state; visual mode +also asks the model to identify a drawn shape unavailable in accessibility text. + + +September 24 acceptance: 1,909 desktop regression tests, the Electron canonical-authorization/ +replay/Stop test, native pure/protocol tests, and the live fixture GUI suite passed. A real native +Calculator background test verified `2 + 3 = 5` without changing the foreground app. The signed +arm64 test bundle and nested universal helper passed strict code-signature verification. Its +LaunchServices-launched acceptance remained blocked on that app identity's Accessibility and +Screen Recording grants; the separately granted development helper's tests do not satisfy this +last release-onboarding check. The disposable packaged test follows the existing packaged-smoke +suite's mock-keychain convention; native TCC permissions remain real. diff --git a/apps/desktop/e2e/computer-use.spec.ts b/apps/desktop/e2e/computer-use.spec.ts new file mode 100644 index 00000000000..d171febdf3d --- /dev/null +++ b/apps/desktop/e2e/computer-use.spec.ts @@ -0,0 +1,150 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer, type Server } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { type ElectronApplication, _electron as electron, expect, test } from '@playwright/test' +import type { SimDesktopApi } from '@sim/desktop-bridge' + +const DESKTOP_DIR = fileURLToPath(new URL('..', import.meta.url)) + +test('computer use reaches the actual helper only with canonical one-shot authorization', async () => { + test.skip(process.platform !== 'darwin', 'The native helper is macOS-only.') + const root = mkdtempSync(join(tmpdir(), 'sim-computer-use-e2e-')) + let server: Server | undefined + let app: ElectronApplication | undefined + let claimed = false + let authorizationCount = 0 + let releaseAuthorization: () => void = () => {} + let markAuthorizationStarted: () => void = () => {} + const authorizationStarted = new Promise((resolve) => { + markAuthorizationStarted = resolve + }) + const authorizationReleased = new Promise((resolve) => { + releaseAuthorization = resolve + }) + try { + server = createServer(async (request, response) => { + const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname + if (path === '/api/auth/get-session') { + response.writeHead(200, { 'Content-Type': 'application/json' }).end( + JSON.stringify({ + user: { id: 'computer-test' }, + session: { id: 'computer-test-session' }, + }) + ) + return + } + if (path === '/api/desktop/computer/authorize') { + authorizationCount += 1 + let body = '' + for await (const chunk of request) body += chunk.toString() + const input = JSON.parse(body) + expect(Object.keys(input)).toEqual(['toolCallId']) + if (input.toolCallId === 'delayed') { + markAuthorizationStarted() + await authorizationReleased + } else if (input.toolCallId !== 'status-once' || claimed) { + response.writeHead(403, { 'Content-Type': 'application/json' }).end('{}') + return + } + claimed = true + response.writeHead(200, { 'Content-Type': 'application/json' }).end( + JSON.stringify({ + chatId: 'computer-chat', + toolName: 'computer', + args: { action: 'status' }, + }) + ) + return + } + response + .writeHead(200, { + 'Content-Type': 'text/html', + 'Set-Cookie': 'better-auth.session_token=fixture; HttpOnly; SameSite=Lax; Path=/', + }) + .end( + `Computer Use fixture

Computer Use

` + ) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Missing local fixture port') + const profile = join(root, 'profile') + mkdirSync(profile) + writeFileSync( + join(profile, 'settings.json'), + JSON.stringify({ origin: `http://127.0.0.1:${address.port}`, computerUseEnabled: true }) + ) + app = await electron.launch({ + args: ['.'], + cwd: DESKTOP_DIR, + env: { + ...process.env, + SIM_DESKTOP_ORIGIN: `http://127.0.0.1:${address.port}`, + SIM_DESKTOP_USER_DATA: profile, + }, + }) + const window = await app.firstWindow() + await expect(window.getByRole('heading')).toHaveText('Computer Use') + const status = await window.evaluate(async () => { + const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop + return api.computerUse?.getStatus() + }) + expect(status).toMatchObject({ supported: true, enabled: true, activeAction: null }) + const result = await window.evaluate(async () => { + const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop + return api.computerUse?.executeTool('status-once', { + action: 'get_app_state', + bundleId: 'com.apple.systempreferences', + }) + }) + expect(result).toMatchObject({ kind: 'status', platform: 'darwin' }) + const replay = await window.evaluate(async () => { + const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop + try { + await api.computerUse?.executeTool('status-once', { action: 'status' }) + return 'unexpected success' + } catch { + return 'rejected' + } + }) + expect(replay).toBe('rejected') + expect(authorizationCount).toBe(2) + const delayed = window.evaluate(async () => { + const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop + try { + await api.computerUse?.executeTool('delayed', { action: 'status' }) + return 'unexpected success' + } catch { + return 'stopped' + } + }) + await authorizationStarted + await window.evaluate(async () => { + const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop + await api.computerUse?.cancel('delayed') + }) + releaseAuthorization() + expect(await delayed).toBe('stopped') + expect(authorizationCount).toBe(3) + await window.getByRole('button', { name: 'Disable Computer Use' }).click() + await expect(window.locator('#status')).toHaveText('false') + const disabled = await window.evaluate(async () => { + const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop + try { + await api.computerUse?.executeTool('disabled', { action: 'status' }) + return 'unexpected success' + } catch { + return 'rejected' + } + }) + expect(disabled).toBe('rejected') + expect(authorizationCount).toBe(3) + } finally { + releaseAuthorization() + await app?.close() + await new Promise((resolve) => (server ? server.close(() => resolve()) : resolve())) + rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/electron-builder.yml b/apps/desktop/electron-builder.yml index 25856fb9dc8..c9219a003db 100644 --- a/apps/desktop/electron-builder.yml +++ b/apps/desktop/electron-builder.yml @@ -8,6 +8,7 @@ directories: files: - dist/** + - "!dist/native/Sim Computer Use.app/**" - static/** - package.json - from: ../sim/app/_styles/fonts/season @@ -15,6 +16,10 @@ files: filter: - SeasonSansUprightsVF.woff2 +extraResources: + - from: dist/native/Sim Computer Use.app + to: Sim Computer Use.app + asar: true # Native modules cannot be dlopen'd from inside an asar. The Help-search addon @@ -64,6 +69,7 @@ mac: # macOS refuses to show the microphone prompt at all — it kills the process — # unless the bundle declares why it wants the device. extendInfo: + NSScreenCaptureUsageDescription: Mothership captures approved app windows to understand and verify computer-use actions. NSMicrophoneUsageDescription: Sim uses your microphone for voice input in Chat and for meetings you join in the built-in browser. NSCameraUsageDescription: Sim uses your camera for meetings you join in the built-in browser, such as Google Meet. NSBluetoothAlwaysUsageDescription: Sim uses Bluetooth to complete passkey sign-ins with a nearby phone in the built-in browser. diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift new file mode 100644 index 00000000000..ef207fed561 --- /dev/null +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -0,0 +1,447 @@ +import AppKit +import ApplicationServices +import ScreenCaptureKit + +nonisolated(unsafe) var cancellationRequested: sig_atomic_t = 0 +func checkCancellation() throws { + if cancellationRequested != 0 { throw ComputerError("cancelled", "Computer-use operation was cancelled.") } +} + +struct ComputerError: Error { + let code: String + let message: String + init(_ code: String, _ message: String) { self.code = code; self.message = message } +} +struct Parameters: Decodable { + var permission: String? + var bundleId: String? + var snapshotId: String? + var elementId: String? + var windowId: String? + var includeScreenshot: Bool? + var x: Double? + var y: Double? + var toX: Double? + var toY: Double? + var deltaX: Double? + var deltaY: Double? + var text: String? + var value: String? + var key: String? + var button: String? + var clickCount: Int? + var accessibilityAction: String? +} +struct Request: Decodable { let id: String; let method: String; let params: Parameters } +func required(_ value: T?, _ name: String) throws -> T { + guard let value else { throw ComputerError("invalid_arguments", "Missing \(name).") }; return value +} +func axCheck(_ result: AXError) throws { + guard result == .success else { throw ComputerError("accessibility_error", "Accessibility operation failed (\(result.rawValue)); observe the app again before retrying.") } +} +func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? { + var value: CFTypeRef? + return AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success ? value : nil +} +func secure(_ element: AXUIElement) -> Bool { + attribute(element, kAXSubroleAttribute) as? String == kAXSecureTextFieldSubrole +} +func requireNonSecure(_ element: AXUIElement) throws { + var ancestor: AXUIElement? = element + for _ in 0..<32 { + try checkCancellation() + guard let current = ancestor else { break } + AXUIElementSetMessagingTimeout(current, 0.25) + guard !secure(current) else { throw ComputerError("secure_element", "Secure input controls are unavailable to computer use.") } + guard let parent = attribute(current, kAXParentAttribute), CFGetTypeID(parent) == AXUIElementGetTypeID() else { break } + ancestor = unsafeBitCast(parent, to: AXUIElement.self) + } +} +func rectJSON(_ rect: CGRect) -> [String: Double] { + ["x": rect.minX, "y": rect.minY, "width": rect.width, "height": rect.height] +} +func elementRect(_ element: AXUIElement) -> CGRect? { + guard let rawPosition = attribute(element, kAXPositionAttribute), CFGetTypeID(rawPosition) == AXValueGetTypeID(), + let rawSize = attribute(element, kAXSizeAttribute), CFGetTypeID(rawSize) == AXValueGetTypeID() else { return nil } + var position = CGPoint.zero; var size = CGSize.zero + guard AXValueGetValue(unsafeBitCast(rawPosition, to: AXValue.self), .cgPoint, &position), + AXValueGetValue(unsafeBitCast(rawSize, to: AXValue.self), .cgSize, &size) else { return nil } + return CGRect(origin: position, size: size) +} +func appFor(_ bundleId: String?, launch: Bool = false) async throws -> NSRunningApplication { + let bundle = try required(bundleId, "bundleId") + guard bundle != Bundle.main.bundleIdentifier, !bundle.hasPrefix("com.simstudio."), !bundle.hasPrefix("ai.sim.desktop"), bundle != "com.apple.systempreferences" else { + throw ComputerError("protected_app", "Computer use cannot control its own app or macOS security settings.") + } + var apps = NSRunningApplication.runningApplications(withBundleIdentifier: bundle).filter { !$0.isTerminated } + if apps.isEmpty && launch { + guard let url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: bundle) else { throw ComputerError("app_unavailable", "No installed app matches this bundle ID.") } + let configuration = NSWorkspace.OpenConfiguration(); configuration.activates = false + let app = try await NSWorkspace.shared.openApplication(at: url, configuration: configuration) + guard app.bundleIdentifier == bundle, !app.isTerminated else { throw ComputerError("app_identity_mismatch", "Launched app identity does not match the requested bundle ID.") } + let root = AXUIElementCreateApplication(app.processIdentifier) + AXUIElementSetMessagingTimeout(root, 0.1) + var windowReady = false + for _ in 0..<20 { + try checkCancellation() + if let windows = attribute(root, kAXWindowsAttribute) as? [AXUIElement], !windows.isEmpty { windowReady = true; break } + guard !app.isTerminated else { throw ComputerError("app_unavailable", "App exited while opening its window.") } + try await Task.sleep(for: .milliseconds(100)) + } + guard windowReady else { throw ComputerError("app_window_unavailable", "The app has not opened an accessible window yet. Observe it again after the window appears.") } + apps = [app] + } + guard apps.count == 1, let app = apps.first else { + throw ComputerError("app_unavailable", "Expected exactly one running app matching the bundle ID.") + } + guard app.processIdentifier != getppid() else { throw ComputerError("protected_app", "Computer use cannot control the desktop process that hosts its permission controls.") } + return app +} +func requireAccessibility() throws { + guard AXIsProcessTrusted() else { throw ComputerError("accessibility_permission_required", "Enable Accessibility for Mothership in System Settings, then retry.") } +} +func windowsFor(_ pid: pid_t) -> [[String: Any]] { + guard let entries = CGWindowListCopyWindowInfo([.optionAll, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { return [] } + return entries.filter { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid && ($0[kCGWindowLayer as String] as? Int) == 0 } +} +func windowRect(_ id: String?, pid: pid_t) throws -> CGRect { + let id = try required(id, "windowId") + guard let number = UInt32(id), let entry = windowsFor(pid).first(where: { ($0[kCGWindowNumber as String] as? NSNumber)?.uint32Value == number }), + let bounds = entry[kCGWindowBounds as String] as? [String: Any], let rect = CGRect(dictionaryRepresentation: bounds as CFDictionary), rect.width > 0, rect.height > 0 else { + throw ComputerError("window_unavailable", "Window no longer belongs to the selected app; observe again.") + } + return rect +} +final class Snapshot { + let id = UUID().uuidString + let pid: pid_t + let launchDate: Date? + let created = Date() + var elements: [String: AXUIElement] = [:] + var windowFrames: [String: CGRect] = [:] + var nodes: [[String: Any]] = [] + var truncated = false + init(app: NSRunningApplication) { pid = app.processIdentifier; launchDate = app.launchDate } + func read() throws { + try requireAccessibility() + let root = AXUIElementCreateApplication(pid) + AXUIElementSetMessagingTimeout(root, 0.25) + var queue: [(AXUIElement, String?, Int)] = [(root, nil, 0)] + var cursor = 0 + while cursor < queue.count && nodes.count < 500 && Date().timeIntervalSince(created) < 8 { + try checkCancellation() + let (element, parent, depth) = queue[cursor]; cursor += 1 + AXUIElementSetMessagingTimeout(element, 0.25) + if elements.values.contains(where: { CFEqual($0, element) }) { continue } + let id = "e\(nodes.count)"; elements[id] = element + var node: [String: Any] = ["elementId": id, "role": attribute(element, kAXRoleAttribute) as? String ?? "AXUnknown", "actions": [String]()] + if let parent { node["parentId"] = parent } + let labels = [kAXTitleAttribute, kAXDescriptionAttribute].compactMap { attribute(element, $0) as? String }.filter { !$0.isEmpty } + if !secure(element), !labels.isEmpty { node["label"] = String(labels.joined(separator: " ").prefix(1024)) } + if !secure(element), let raw = attribute(element, kAXValueAttribute) { + if let value = raw as? String { node["value"] = String(value.prefix(2048)) } + else if let value = raw as? NSNumber { node["value"] = value.stringValue } + } + if let enabled = attribute(element, kAXEnabledAttribute) as? Bool { node["enabled"] = enabled } + if let rect = elementRect(element) { node.merge(rectJSON(rect)) { _, new in new } } + var actions: CFArray? + if AXUIElementCopyActionNames(element, &actions) == .success { node["actions"] = Array((actions as? [String] ?? []).prefix(128)).map { String($0.prefix(128)) } } + nodes.append(node) + if !secure(element), depth < 15, let children = attribute(element, kAXChildrenAttribute) as? [AXUIElement] { + let available = max(0, 1000 - queue.count) + if children.count > available { truncated = true } + for child in children.prefix(available) { queue.append((child, id, depth + 1)) } + } else if depth >= 15 { truncated = true } + } + if cursor < queue.count { truncated = true } + } + func element(_ id: String?) throws -> AXUIElement { + let id = try required(id, "elementId") + guard let element = elements[id], attribute(element, kAXRoleAttribute) != nil else { throw ComputerError("stale_element", "Element is stale; observe the app again.") } + try requireNonSecure(element) + return element + } +} +@available(macOS 14.0, *) +func screenshot(pid: pid_t, windowID: String) async throws -> [String: Any] { + guard CGPreflightScreenCaptureAccess() else { throw ComputerError("screen_capture_permission_required", "Enable Screen Recording for Mothership in System Settings, then retry.") } + let content = try await SCShareableContent.excludingDesktopWindows(true, onScreenWindowsOnly: false) + guard let number = UInt32(windowID), let window = content.windows.first(where: { $0.windowID == number && $0.owningApplication?.processID == pid }) else { throw ComputerError("window_unavailable", "Selected window is unavailable for capture.") } + let filter = SCContentFilter(desktopIndependentWindow: window) + let config = SCStreamConfiguration() + let scale = min(CGFloat(filter.pointPixelScale), 1600 / max(filter.contentRect.width, filter.contentRect.height, 1)) + config.width = max(1, Int(filter.contentRect.width * scale)); config.height = max(1, Int(filter.contentRect.height * scale)) + config.showsCursor = false + let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) + guard let data = NSBitmapImageRep(cgImage: image).representation(using: .png, properties: [:]), data.count <= 8 * 1024 * 1024 else { throw ComputerError("capture_failed", "Screenshot encoding exceeded the allowed size.") } + return ["base64": data.base64EncodedString(), "mimeType": "image/png", "width": image.width, "height": image.height] +} +func parseKey(_ key: String) throws -> (CGKeyCode, CGEventFlags) { + let parts = key.lowercased().split(separator: "+").map(String.init) + guard let name = parts.last else { throw ComputerError("invalid_arguments", "Key is empty.") } + var flags: CGEventFlags = [] + for modifier in parts.dropLast() { switch modifier { case "cmd", "command": flags.insert(.maskCommand); case "shift": flags.insert(.maskShift); case "alt", "option": flags.insert(.maskAlternate); case "ctrl", "control": flags.insert(.maskControl); default: throw ComputerError("invalid_arguments", "Unknown key modifier.") } } + let keys: [String: CGKeyCode] = ["a":0,"s":1,"d":2,"f":3,"h":4,"g":5,"z":6,"x":7,"c":8,"v":9,"b":11,"q":12,"w":13,"e":14,"r":15,"y":16,"t":17,"1":18,"2":19,"3":20,"4":21,"6":22,"5":23,"9":25,"7":26,"8":28,"0":29,"=":24,"-":27,"]":30,"[":33,"o":31,"u":32,"i":34,"p":35,"return":36,"enter":36,"l":37,"j":38,"'":39,";":41,"\\":42,",":43,"/":44,".":47,"`":50,"k":40,"n":45,"m":46,"tab":48,"space":49,"backspace":51,"escape":53,"delete":117,"left":123,"right":124,"down":125,"up":126,"home":115,"end":119,"pageup":116,"pagedown":121,"f1":122,"f2":120,"f3":99,"f4":118,"f5":96,"f6":97,"f7":98,"f8":100,"f9":101,"f10":109,"f11":103,"f12":111] + guard let code = keys[name] else { throw ComputerError("unsupported_key", "Unsupported key name.") }; return (code, flags) +} +func postKey(pid: pid_t, code: CGKeyCode, flags: CGEventFlags = [], text: String? = nil) throws { + guard let source = CGEventSource(stateID: .privateState), let down = CGEvent(keyboardEventSource: source, virtualKey: code, keyDown: true), let up = CGEvent(keyboardEventSource: source, virtualKey: code, keyDown: false) else { throw ComputerError("input_failed", "Could not create keyboard event.") } + down.flags = flags; up.flags = flags + if let text { let chars = Array(text.utf16); chars.withUnsafeBufferPointer { down.keyboardSetUnicodeString(stringLength: chars.count, unicodeString: $0.baseAddress) } } + down.postToPid(pid); up.postToPid(pid) +} +func mapPoint(frame: CGRect, x: Double, y: Double) throws -> CGPoint { + guard x.isFinite, y.isFinite, x >= 0, y >= 0, x < frame.width, y < frame.height else { throw ComputerError("invalid_coordinates", "Coordinates must lie within the selected window in points.") } + return CGPoint(x: frame.minX + x, y: frame.minY + y) +} +func requireForegroundTarget(pid: pid_t, windowID: String, point: CGPoint, frame: CGRect) throws { + guard NSWorkspace.shared.frontmostApplication?.processIdentifier == pid else { throw ComputerError("foreground_required", "Coordinate mouse input requires the target app in front. Call activate_app, then observe again.") } + guard try windowRect(windowID, pid: pid) == frame else { throw ComputerError("stale_window", "Window moved during input; observe it again.") } + guard let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { throw ComputerError("window_unavailable", "Cannot verify the target window.") } + let top = entries.first { entry in + guard (entry[kCGWindowAlpha as String] as? Double ?? 0) > 0, let bounds = entry[kCGWindowBounds as String] as? [String: Any], let rect = CGRect(dictionaryRepresentation: bounds as CFDictionary) else { return false } + return rect.contains(point) + } + guard let top, (top[kCGWindowNumber as String] as? NSNumber)?.stringValue == windowID else { throw ComputerError("window_occluded", "The target point is covered by another window; bring the intended window forward and observe again.") } +} +func postMouse(pid: pid_t, type: CGEventType, point: CGPoint, button: CGMouseButton, count: Int = 1, windowID: String? = nil, frame: CGRect? = nil, release: Bool = false) throws { + guard let windowID, let frame else { throw ComputerError("invalid_window", "Mouse input requires an observed app window.") } + if !release { try requireForegroundTarget(pid: pid, windowID: windowID, point: point, frame: frame) } + guard let source = CGEventSource(stateID: .privateState), let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else { throw ComputerError("input_failed", "Could not create mouse event.") } + event.setIntegerValueField(.mouseEventClickState, value: Int64(count)) + event.post(tap: .cghidEventTap) +} + +func discoverApps() throws -> [[String: Any]] { + var entries: [String: [String: Any]] = [:] + let roots = [URL(fileURLWithPath: "/Applications"), URL(fileURLWithPath: "/System/Applications"), FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Applications")] + var inspected = 0 + for root in roots { + guard let enumerator = FileManager.default.enumerator(at: root, includingPropertiesForKeys: [.isDirectoryKey], options: [.skipsHiddenFiles]) else { continue } + for case let url as URL in enumerator { + try checkCancellation() + inspected += 1 + if inspected > 5000 || entries.count >= 1000 { break } + if url.pathExtension == "app" { + enumerator.skipDescendants() + guard let bundle = Bundle(url: url), let id = bundle.bundleIdentifier, id.utf8.count <= 255 else { continue } + let name = bundle.object(forInfoDictionaryKey: "CFBundleDisplayName") as? String ?? bundle.object(forInfoDictionaryKey: "CFBundleName") as? String ?? url.deletingPathExtension().lastPathComponent + entries[id] = ["bundleId": id, "name": String(name.prefix(1024)), "isActive": false] + } else if enumerator.level >= 3 { enumerator.skipDescendants() } + } + } + for app in NSWorkspace.shared.runningApplications where app.activationPolicy == .regular { + guard let id = app.bundleIdentifier, id.utf8.count <= 255 else { continue } + entries[id] = ["bundleId": id, "name": String((app.localizedName ?? id).prefix(1024)), "pid": app.processIdentifier, "isActive": app.isActive] + } + return Array(entries.values.sorted { ($0["bundleId"] as? String ?? "") < ($1["bundleId"] as? String ?? "") }.prefix(1000)) +} +@MainActor +final class Driver { + var snapshots: [String: Snapshot] = [:] + func point(_ p: Parameters, app: NSRunningApplication, snapshot: Snapshot, end: Bool = false) throws -> CGPoint { + if let elementId = p.elementId, !end { + guard let frame = elementRect(try snapshot.element(elementId)) else { throw ComputerError("element_unavailable", "Element has no usable frame.") } + return CGPoint(x: frame.midX, y: frame.midY) + } + let windowId = try required(p.windowId, "windowId") + let frame = try windowRect(windowId, pid: app.processIdentifier) + guard snapshot.windowFrames[windowId] == frame else { throw ComputerError("stale_window", "Window moved or changed; observe it again.") } + let x = try required(end ? p.toX : p.x, end ? "toX" : "x"); let y = try required(end ? p.toY : p.y, end ? "toY" : "y") + let position = try mapPoint(frame: frame, x: x, y: y) + let root = AXUIElementCreateApplication(app.processIdentifier) + AXUIElementSetMessagingTimeout(root, 0.25) + var target: AXUIElement? + if AXUIElementCopyElementAtPosition(root, Float(position.x), Float(position.y), &target) == .success, let target { try requireNonSecure(target) } + return position + } + func execute(_ request: Request) async throws -> [String: Any] { + try checkCancellation() + let p = request.params + switch request.method { + case "diagnose_screen_capture": + guard #available(macOS 14.0, *) else { throw ComputerError("unsupported_os", "ScreenCaptureKit requires macOS 14.") } + let content = try await SCShareableContent.excludingDesktopWindows(true, onScreenWindowsOnly: false) + _ = content + return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": true] + case "request_permission": + switch p.permission { + case "accessibility": + let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary + _ = AXIsProcessTrustedWithOptions(options) + case "screenCapture": _ = CGRequestScreenCaptureAccess() + default: throw ComputerError("invalid_arguments", "Unknown permission.") + } + return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess()] + case "status": return ["kind": "status", "platform": "darwin", "accessibility": AXIsProcessTrusted(), "screenRecording": CGPreflightScreenCaptureAccess()] + case "list_apps": return ["kind": "apps", "apps": try discoverApps()] + case "activate_app": + try requireAccessibility() + let app = try await appFor(p.bundleId, launch: true) + guard app.activate(options: []) else { throw ComputerError("activation_failed", "macOS did not accept app activation.") } + for _ in 0..<20 { + try checkCancellation() + if app.isActive { snapshots.removeValue(forKey: app.bundleIdentifier!); return ["kind": "action", "action": "activate_app", "bundleId": app.bundleIdentifier!, "dispatched": true, "verified": true] } + try await Task.sleep(for: .milliseconds(50)) + } + throw ComputerError("activation_failed", "App did not become active; observe before retrying.") + case "get_app_state": + try requireAccessibility() + let app = try await appFor(p.bundleId, launch: true); let snapshot = Snapshot(app: app); try snapshot.read() + let axWindowFrames = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole }.compactMap(elementRect) + let windows = windowsFor(app.processIdentifier).prefix(100).compactMap { entry -> [String: Any]? in + guard let id = entry[kCGWindowNumber as String] as? NSNumber, let bounds = entry[kCGWindowBounds as String] as? [String: Any], let rect = CGRect(dictionaryRepresentation: bounds as CFDictionary), rect.width > 0, rect.height > 0, axWindowFrames.contains(rect) else { return nil } + snapshot.windowFrames[id.stringValue] = rect + var result: [String: Any] = ["windowId": id.stringValue, "title": entry[kCGWindowName as String] as? String ?? ""] + if let title = result["title"] as? String { result["title"] = String(title.prefix(4096)) } + result.merge(rectJSON(rect)) { _, new in new }; return result + } + if snapshots.count >= 16 { snapshots.removeAll() } + snapshots[app.bundleIdentifier!] = snapshot + let root = AXUIElementCreateApplication(app.processIdentifier) + var focusedFrame: CGRect? + if let focused = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(focused) == AXUIElementGetTypeID() { focusedFrame = elementRect(unsafeBitCast(focused, to: AXUIElement.self)) } + let focusedID = snapshot.windowFrames.first { $0.value == focusedFrame }?.key + for index in snapshot.nodes.indices { + if let element = snapshot.elements[snapshot.nodes[index]["elementId"] as? String ?? ""], attribute(element, kAXRoleAttribute) as? String == kAXWindowRole, let frame = elementRect(element), let windowID = snapshot.windowFrames.first(where: { $0.value == frame })?.key { snapshot.nodes[index]["windowId"] = windowID } + else if let parentID = snapshot.nodes[index]["parentId"] as? String, let parent = snapshot.nodes.first(where: { $0["elementId"] as? String == parentID }), let windowID = parent["windowId"] as? String { snapshot.nodes[index]["windowId"] = windowID } + } + guard let selectedWindowId = p.windowId ?? focusedID ?? windows.first?["windowId"] as? String else { throw ComputerError("app_window_unavailable", "No accessible app window is available. Open a window, then observe the app again.") } + guard snapshot.windowFrames[selectedWindowId] != nil else { throw ComputerError("window_unavailable", "Selected window does not belong to the app.") } + var result: [String: Any] = ["kind": "state", "windowId": selectedWindowId, "snapshotId": snapshot.id, "bundleId": app.bundleIdentifier!, "nodes": snapshot.nodes, "windows": windows, "truncated": snapshot.truncated] + if p.includeScreenshot == true { + do { + guard #available(macOS 14.0, *) else { throw ComputerError("unsupported_os", "Screenshots require macOS 14 or later.") } + result["screenshot"] = try await screenshot(pid: app.processIdentifier, windowID: selectedWindowId) + } catch let error as ComputerError { result["screenshotError"] = String((error.code + ": " + error.message).prefix(2000)) } + catch { result["screenshotError"] = String(("capture_failed: " + String(describing: error)).prefix(2000)) } + } + return result + case "click", "type_text", "press_key", "scroll", "drag", "set_value", "perform_action": break + default: throw ComputerError("unknown_method", "Unknown computer-use method.") + } + try requireAccessibility() + let app = try await appFor(p.bundleId); let bundle = try required(app.bundleIdentifier, "bundleId") + guard let snapshot = snapshots[bundle], snapshot.id == p.snapshotId, snapshot.pid == app.processIdentifier, snapshot.launchDate == app.launchDate, Date().timeIntervalSince(snapshot.created) < 60 else { throw ComputerError("stale_snapshot", "Observe the app again before acting.") } + snapshots.removeValue(forKey: bundle) + let pid = app.processIdentifier + switch request.method { + case "perform_action": + let element = try snapshot.element(p.elementId); let action = try required(p.accessibilityAction, "accessibilityAction") + var names: CFArray?; try axCheck(AXUIElementCopyActionNames(element, &names)) + guard (names as? [String] ?? []).contains(action) else { throw ComputerError("unsupported_action", "Action was not advertised by the selected element.") } + try axCheck(AXUIElementPerformAction(element, action as CFString)) + case "set_value": + let element = try snapshot.element(p.elementId); let value = try required(p.value, "value") + guard value.utf16.count <= 32000 else { throw ComputerError("invalid_arguments", "Value exceeds 32000 UTF-16 units.") } + var writable: DarwinBoolean = false; try axCheck(AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &writable)) + guard writable.boolValue else { throw ComputerError("unsupported_action", "This element does not support setting its value.") } + try axCheck(AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFString)) + case "click": + let button = p.button ?? "left"; let count = p.clickCount ?? 1 + guard ["left", "right"].contains(button), (1...3).contains(count) else { throw ComputerError("invalid_arguments", "Invalid button or clickCount.") } + if p.elementId != nil && button == "left" && count == 1 { try axCheck(AXUIElementPerformAction(try snapshot.element(p.elementId), kAXPressAction as CFString)) } + else { + let position = try point(p, app: app, snapshot: snapshot); let mouse: CGMouseButton = button == "right" ? .right : .left + let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId") + let frame = try required(snapshot.windowFrames[windowID], "window frame") + for click in 1...count { + try checkCancellation() + try postMouse(pid: pid, type: button == "right" ? .rightMouseDown : .leftMouseDown, point: position, button: mouse, count: click, windowID: windowID, frame: frame) + try postMouse(pid: pid, type: button == "right" ? .rightMouseUp : .leftMouseUp, point: position, button: mouse, count: click, windowID: windowID, frame: frame, release: true) + } + } + case "type_text": + let element = try snapshot.element(p.elementId); let text = try required(p.text, "text") + guard text.utf16.count <= 32000 else { throw ComputerError("invalid_arguments", "Text exceeds 32000 UTF-16 units.") } + try axCheck(AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)) + let root = AXUIElementCreateApplication(pid) + guard let focused = attribute(root, kAXFocusedUIElementAttribute), CFEqual(focused, element) else { throw ComputerError("focus_failed", "App did not focus the requested element.") } + for character in text { + try checkCancellation() + guard let current = attribute(root, kAXFocusedUIElementAttribute), CFEqual(current, element) else { throw ComputerError("focus_changed", "Focus changed during typing; observe again.") } + try postKey(pid: pid, code: 0, text: String(character)) + } + case "press_key": + let windowId = try required(p.windowId, "windowId"); let frame = try windowRect(windowId, pid: pid) + guard snapshot.windowFrames[windowId] == frame else { throw ComputerError("stale_window", "Observe the target window again.") } + let root = AXUIElementCreateApplication(pid) + guard let rawWindow = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(rawWindow) == AXUIElementGetTypeID(), elementRect(unsafeBitCast(rawWindow, to: AXUIElement.self)) == frame else { throw ComputerError("focus_failed", "Requested window is not the app's focused window.") } + if let focused = attribute(root, kAXFocusedUIElementAttribute), CFGetTypeID(focused) == AXUIElementGetTypeID() { try requireNonSecure(unsafeBitCast(focused, to: AXUIElement.self)) } + let (code, flags) = try parseKey(required(p.key, "key")) + var selectedAll = false + if code == 0, flags == .maskCommand, let rawFocused = attribute(root, kAXFocusedUIElementAttribute), CFGetTypeID(rawFocused) == AXUIElementGetTypeID() { + let element = unsafeBitCast(rawFocused, to: AXUIElement.self) + var settable: DarwinBoolean = false + if AXUIElementIsAttributeSettable(element, kAXSelectedTextRangeAttribute as CFString, &settable) == .success, settable.boolValue, let value = attribute(element, kAXValueAttribute) as? String { + var range = CFRange(location: 0, length: value.utf16.count) + if let selectedRange = AXValueCreate(.cfRange, &range) { try axCheck(AXUIElementSetAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, selectedRange)); selectedAll = true } + } + } + if !selectedAll { try postKey(pid: pid, code: code, flags: flags) } + case "scroll": + let position = try point(p, app: app, snapshot: snapshot); let dx = p.deltaX ?? 0; let dy = p.deltaY ?? 0 + guard dx.isFinite, dy.isFinite, abs(dx) <= 10000, abs(dy) <= 10000 else { throw ComputerError("invalid_arguments", "Scroll deltas exceed bounds.") } + guard let event = CGEvent(scrollWheelEvent2Source: CGEventSource(stateID: .privateState), units: .pixel, wheelCount: 2, wheel1: Int32(-dy), wheel2: Int32(-dx), wheel3: 0) else { throw ComputerError("input_failed", "Could not create scroll event.") } + let windowID = try required(p.windowId ?? snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key, "windowId") + let frame = try required(snapshot.windowFrames[windowID], "window frame") + try requireForegroundTarget(pid: pid, windowID: windowID, point: position, frame: frame) + event.location = position; event.post(tap: .cghidEventTap) + case "drag": + let start = try point(p, app: app, snapshot: snapshot); let end = try point(p, app: app, snapshot: snapshot, end: true) + let windowID = try required(p.windowId, "windowId"); let frame = try required(snapshot.windowFrames[windowID], "window frame") + try postMouse(pid: pid, type: .leftMouseDown, point: start, button: .left, windowID: windowID, frame: frame) + var lastPoint = start + defer { try? postMouse(pid: pid, type: .leftMouseUp, point: lastPoint, button: .left, windowID: windowID, frame: frame, release: true) } + try await Task.sleep(for: .milliseconds(10)) + for step in 1...10 { + try checkCancellation() + let amount = CGFloat(step) / 10 + let position = CGPoint(x: start.x + (end.x - start.x) * amount, y: start.y + (end.y - start.y) * amount) + try postMouse(pid: pid, type: .leftMouseDragged, point: position, button: .left, windowID: windowID, frame: frame) + lastPoint = position + try await Task.sleep(for: .milliseconds(10)) + } + default: throw ComputerError("unknown_method", "Unknown method.") + } + return ["kind": "action", "action": request.method, "bundleId": bundle, "dispatched": true, "verified": false] + } +} +func nextLine() throws -> String? { + var bytes = [UInt8](); bytes.reserveCapacity(4096) + var oversized = false + while true { + let value = getchar() + if value == EOF { if bytes.isEmpty && !oversized { return nil }; break } + if value == 10 { break } + if bytes.count < 128 * 1024 { bytes.append(UInt8(value)) } else { oversized = true } + } + if oversized { throw ComputerError("invalid_request", "Request exceeds 128 KiB.") } + guard let line = String(bytes: bytes, encoding: .utf8) else { throw ComputerError("invalid_request", "Request is not UTF-8.") } + return line +} +#if !COMPUTER_USE_TEST +@main +struct Main { + @MainActor static func main() async { + signal(SIGTERM) { _ in cancellationRequested = 1 } + signal(SIGINT) { _ in cancellationRequested = 1 } + let driver = Driver() + while cancellationRequested == 0 { + var id = "invalid-request" + var response: [String: Any] + do { + guard let line = try nextLine() else { break } + let request = try JSONDecoder().decode(Request.self, from: Data(line.utf8)); id = request.id + guard !id.isEmpty, id.utf8.count <= 128 else { throw ComputerError("invalid_request", "Request ID must contain 1 to 128 bytes.") } + response = ["id": id, "result": try await driver.execute(request)] + } catch let error as ComputerError { response = ["id": id, "error": ["code": error.code, "message": error.message]] } + catch { response = ["id": id, "error": ["code": "request_failed", "message": String(String(describing: error).prefix(2000))]] } + if let data = try? JSONSerialization.data(withJSONObject: response, options: [.sortedKeys]) { + FileHandle.standardOutput.write(data); FileHandle.standardOutput.write(Data([10])) + } + } + } +} + +#endif diff --git a/apps/desktop/native/computer-use/Info.plist b/apps/desktop/native/computer-use/Info.plist new file mode 100644 index 00000000000..2ba6eb3d71d --- /dev/null +++ b/apps/desktop/native/computer-use/Info.plist @@ -0,0 +1,15 @@ + + + +CFBundleIdentifiercom.simstudio.computer-use +CFBundleNameSim Computer Use +CFBundleDisplayNameSim Computer Use +CFBundleExecutableSimComputerUse +CFBundlePackageTypeAPPL +CFBundleVersion1 +CFBundleShortVersionString1.0 +LSMinimumSystemVersion14.0 +LSUIElement +NSHighResolutionCapable +NSScreenCaptureUsageDescriptionMothership captures the app window you select to understand and verify computer-use actions. + diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md new file mode 100644 index 00000000000..6aeb839dff6 --- /dev/null +++ b/apps/desktop/native/computer-use/README.md @@ -0,0 +1,29 @@ +# macOS computer-use helper + +Independent native implementation using public Accessibility, Core Graphics and ScreenCaptureKit APIs. Requires macOS 14+. Package `ComputerUse.swift` as `Sim Computer Use.app/Contents/MacOS/SimComputerUse` with this directory's `Info.plist`; sign the nested app with the desktop distribution identity so TCC grants survive upgrades. + +Compile each architecture with `swiftc -parse-as-library -O -target arm64-apple-macosx14.0 ComputerUse.swift -o SimComputerUse` (repeat for `x86_64`, combine with `lipo`). No Apple Events, shell execution, private APIs, global event listeners or clipboard access is used. Coordinate mouse input posts public foreground events only after checking the active app, observed window geometry and topmost target point. + +## Protocol + +Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` returns `{id,result}` or `{id,error:{code,message}}`. The producer contract is `worker/packages/contracts/src/computer-use.ts` in mothership, copied into the desktop bridge by contract sync. `request_permission` is a local onboarding method, never a model tool; params are `{permission:"accessibility"|"screenCapture"}`. Status/preflight never triggers prompts. Errors go through the protocol, and stdout contains no logs. + +The helper observes up to 500 accessibility nodes (depth 15, queue 1000, traversal deadline 8 seconds, each AX call timeout 250ms), 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every attempted action. Restarts invalidate all references. Screen captures are selected-window only and capped to 1600 pixels on the longest side and 8 MiB encoded. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. + +Coordinates in action inputs are window-local points; node frames are global screen points. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps and System Settings are protected targets. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`. + +AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and topmost window at the point. Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. + +## Cancellation + +SIGTERM/SIGINT sets a cancellation flag. Traversal, typing and drag loops check it; mouse up is sent through a defer before returning from a cancelled drag. Keyboard down/up are paired in one function. The owner should close stdin and send SIGTERM, allowing a grace period before SIGKILL. Foreground drags also stop if the active app or window geometry changes; this does not detect every physical user input within the same app. Forced termination cannot guarantee a final release event; input routines deliberately avoid long waits with held buttons. Cancellation cannot undo an action already dispatched. + +## Tests and limitations + +Compile pure native tests with `swiftc -parse-as-library -D COMPUTER_USE_TEST ComputerUse.swift tests/NativeTests.swift -o native-tests`. They cover multi-display negative origins, coordinate bounds/nonfinite input, shortcut modifiers, invalid key handling and cooperative cancellation. Real status and app discovery outputs were validated against the generated worker Zod schema (156 apps, including 147 installed-only apps). + +`python3 tests/protocol.py ` checks persistent framing, bounded-request recovery, Unicode decoding, no-prompt status, invalid methods/permissions, permission gates and clean EOF. `tests/Fixture.swift` compiles into a disposable AppKit app with a text field, secure field, increment button/counter, slider, scroll area and a visual-only marker. `python3 tests/live.py ` requires both grants and verifies actual outcomes. `--allow-missing-screen` explicitly reports screenshots blocked rather than passing them; `--existing-fixture` leaves an existing fixture running for coordinated model trials. Give the fixture bundle ID `com.mothership.computer-use-fixture` when packaging it for integration tests. + +Both arm64 and x86_64 compilation passed on September 24, 2026. Protocol tests passed. The strict live fixture suite passed with real Accessibility reads, secure-value suppression, selected-window PNG capture, AX button presses/value edits, Unicode typing, Cmd+A/Backspace, verified explicit activation, coordinate drag moving a slider, and scrolling changing visible content. Every action was checked using fresh state. A separate macOS Calculator smoke test pressed All Clear, 2, Add, 3 and Equals through Accessibility while Calculator remained in the background, verified result 5, and confirmed the foreground app was unchanged. These checks establish tested behavior, not Codex-level app compatibility. Foreground coordinate tests ran on a Mac with multiple displays. Lock-screen operation, arbitrary hidden/private windows, non-US shortcut layouts and broad third-party app compatibility remain unverified. + +TCC attribution matters: a helper directly spawned by Codex was attributed to Codex in macOS logs; launching it through LaunchServices made it independently responsible. Ad-hoc signing generates a cdhash-only requirement, so rebuilding invalidates independent grants. Distribution and meaningful onboarding tests must use a stable Developer ID and the real parent app responsibility. Never weaken the designated requirement or edit TCC databases to bypass a grant. diff --git a/apps/desktop/native/computer-use/tests/Fixture.swift b/apps/desktop/native/computer-use/tests/Fixture.swift new file mode 100644 index 00000000000..9e934a8ee93 --- /dev/null +++ b/apps/desktop/native/computer-use/tests/Fixture.swift @@ -0,0 +1,62 @@ +import AppKit + +final class MarkerView: NSView { + override func draw(_ dirtyRect: NSRect) { + NSColor.white.setFill(); bounds.fill() + NSColor.systemBlue.setFill() + let triangle = NSBezierPath(); triangle.move(to: NSPoint(x: bounds.midX, y: bounds.maxY - 5)); triangle.line(to: NSPoint(x: 5, y: 5)); triangle.line(to: NSPoint(x: bounds.maxX - 5, y: 5)); triangle.close(); triangle.fill() + } +} + +final class FixtureDelegate: NSObject, NSApplicationDelegate { + var window: NSWindow! + var count = 0 + let counter = NSTextField(labelWithString: "Count: 0") + let scrollCounter = NSTextField(labelWithString: "Scroll: 0") + func applicationDidFinishLaunching(_ notification: Notification) { + NSEvent.addLocalMonitorForEvents(matching: [.leftMouseDown, .leftMouseDragged, .leftMouseUp, .scrollWheel]) { event in + print("fixture-event type=\(event.type.rawValue) window=\(event.windowNumber) x=\(event.locationInWindow.x) y=\(event.locationInWindow.y)"); fflush(stdout) + return event + } + let menu = NSMenu() + let editItem = NSMenuItem(title: "Edit", action: nil, keyEquivalent: "") + let editMenu = NSMenu(title: "Edit") + editMenu.addItem(withTitle: "Select All", action: #selector(NSText.selectAll(_:)), keyEquivalent: "a") + editItem.submenu = editMenu; menu.addItem(editItem); NSApplication.shared.mainMenu = menu + window = NSWindow(contentRect: NSRect(x: 100, y: 100, width: 600, height: 720), styleMask: [.titled, .closable, .resizable], backing: .buffered, defer: false) + window.title = "Mothership Computer Use Test Fixture" + let stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16 + stack.translatesAutoresizingMaskIntoConstraints = false + let field = NSTextField(string: "Fixture text"); field.setAccessibilityIdentifier("fixture-text"); field.setAccessibilityLabel("Fixture editable text") + let secure = NSSecureTextField(string: "fixture-only-secret"); secure.setAccessibilityIdentifier("fixture-password"); secure.setAccessibilityLabel("Fixture password") + let button = NSButton(title: "Increment", target: self, action: #selector(increment)); button.setAccessibilityIdentifier("fixture-increment") + let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider") + let marker = MarkerView(); marker.setAccessibilityElement(false) + marker.translatesAutoresizingMaskIntoConstraints = false + let scroll = NSScrollView(); scroll.hasVerticalScroller = true + scroll.translatesAutoresizingMaskIntoConstraints = false + let document = NSTextView(frame: NSRect(x: 0, y: 0, width: 520, height: 3000)); document.isEditable = false + document.string = (1...150).map { "Fixture scrolling line \($0)" }.joined(separator: "\n") + scroll.documentView = document + scroll.contentView.postsBoundsChangedNotifications = true + NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView) + for view in [field, secure, button, counter, slider, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) } + window.contentView!.addSubview(stack) + NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)]) + window.makeKeyAndOrderFront(nil) + window.makeFirstResponder(field) + NSApplication.shared.activate() + print("fixture-ready"); fflush(stdout) + } + @objc func increment() { count += 1; counter.stringValue = "Count: \(count)" } + @objc func scrolled(_ notification: Notification) { + guard let clip = notification.object as? NSClipView else { return } + scrollCounter.stringValue = "Scroll: \(Int(clip.bounds.origin.y))" + } +} +@main struct Fixture { + @MainActor static func main() { + let app = NSApplication.shared; let delegate = FixtureDelegate(); app.delegate = delegate + app.setActivationPolicy(.regular); app.run() + } +} diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift new file mode 100644 index 00000000000..98320b57e16 --- /dev/null +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -0,0 +1,31 @@ +import AppKit + +@main struct NativeTests { + static func expectError(_ code: String, _ operation: () throws -> Void) { + do { try operation(); fatalError("Expected \(code)") } + catch let error as ComputerError { precondition(error.code == code) } + catch { fatalError("Unexpected \(error)") } + } + static func main() throws { + let frame = CGRect(x: -1920, y: 120, width: 800, height: 600) + let interior = try mapPoint(frame: frame, x: 20, y: 30) + precondition(interior == CGPoint(x: -1900, y: 150)) + let origin = try mapPoint(frame: frame, x: 0, y: 0) + precondition(origin == frame.origin) + for point in [(Double.nan, 0.0), (Double.infinity, 0.0), (-1.0, 1.0), (800.0, 0.0), (0.0, 600.0)] { + expectError("invalid_coordinates") { _ = try mapPoint(frame: frame, x: point.0, y: point.1) } + } + let (code, flags) = try parseKey("Cmd+Shift+A") + precondition(code == 0 && flags == [.maskCommand, .maskShift]) + let enter = try parseKey("Enter"); precondition(enter.0 == 36) + let optionLeft = try parseKey("Option+Left"); precondition(optionLeft.1 == .maskAlternate) + expectError("unsupported_key") { _ = try parseKey("F99") } + expectError("invalid_arguments") { _ = try parseKey("Hyper+A") } + expectError("invalid_arguments") { _ = try required(Optional.none, "example") } + cancellationRequested = 1 + expectError("cancelled") { try checkCancellation() } + cancellationRequested = 0 + try checkCancellation() + print("PASS: negative-display geometry, boundaries, nonfinite coordinates, shortcut modifiers, invalid keys, cancellation") + } +} diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py new file mode 100644 index 00000000000..0bac9b9f3d0 --- /dev/null +++ b/apps/desktop/native/computer-use/tests/live.py @@ -0,0 +1,126 @@ +"""Strict live test: requires existing TCC grants; controls only our disposable fixture.""" +import argparse +import base64 +import json +import os +import pathlib +import plistlib +import selectors +import subprocess +import tempfile +import time + +parser = argparse.ArgumentParser() +parser.add_argument('helper', help='Packaged/signed SimComputerUse executable with existing user grants') +parser.add_argument('--allow-missing-screen', action='store_true', help='Run AX/input checks and explicitly report screenshot as blocked') +parser.add_argument('--existing-fixture', action='store_true', help='Use an already running disposable fixture and leave it open') +args = parser.parse_args() +source = pathlib.Path(__file__).resolve().parent +helper = subprocess.Popen([args.helper], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) +fixture = None +selector = selectors.DefaultSelector() +selector.register(helper.stdout, selectors.EVENT_READ) +sequence = 0 +bundle = 'com.mothership.computer-use-fixture' + + +def call(method, **params): + global sequence + sequence += 1 + helper.stdin.write(json.dumps({'id': str(sequence), 'method': method, 'params': params}) + '\n') + helper.stdin.flush() + if not selector.select(timeout=35): + raise AssertionError('Native response timed out: ' + method) + response = json.loads(helper.stdout.readline()) + assert response['id'] == str(sequence) + if 'error' in response: + raise AssertionError(method + ': ' + json.dumps(response['error'])) + return response['result'] + + +def state(screenshot=False): + return call('get_app_state', bundleId=bundle, includeScreenshot=screenshot) + + +def find(snapshot, role=None, label=None): + return next(node for node in snapshot['nodes'] if (role is None or node['role'] == role) and (label is None or label in node.get('label', ''))) + + +def mutate(method, snapshot, **params): + result = call(method, bundleId=bundle, snapshotId=snapshot['snapshotId'], **params) + assert result['dispatched'] and not result['verified'] + time.sleep(0.15) + return state() + + +def coordinate(snapshot, node, fx=0.5, fy=0.5): + window = next(w for w in snapshot['windows'] if w['windowId'] == snapshot['windowId']) + return {'windowId': window['windowId'], 'x': node['x'] - window['x'] + node['width'] * fx, 'y': node['y'] - window['y'] + node['height'] * fy} + + +try: + permissions = call('status') + assert permissions['accessibility'], 'BLOCKED: Accessibility permission is missing.' + assert permissions['screenRecording'] or args.allow_missing_screen, 'BLOCKED: Screen Recording permission is missing.' + with tempfile.TemporaryDirectory(prefix='mship-computer-use-live-') as temporary: + app = pathlib.Path(temporary) / 'Fixture.app' + executable = app / 'Contents/MacOS/Fixture' + executable.parent.mkdir(parents=True) + (app / 'Contents/Info.plist').write_bytes(plistlib.dumps({'CFBundleIdentifier': bundle, 'CFBundleName': 'Mothership Computer Use Fixture', 'CFBundleExecutable': 'Fixture', 'CFBundlePackageType': 'APPL', 'LSMinimumSystemVersion': '14.0'})) + if not args.existing_fixture: + subprocess.run(['swiftc', '-parse-as-library', str(source / 'Fixture.swift'), '-o', str(executable)], check=True) + fixture = subprocess.Popen([str(executable)], stdout=open('/private/tmp/mship-fixture-events.log', 'w'), stderr=subprocess.DEVNULL) + time.sleep(1) + snapshot = state(screenshot=True) + assert 'fixture-only-secret' not in json.dumps(snapshot), 'Secure value leaked' + image = snapshot.get('screenshot') + if permissions['screenRecording']: + assert image and base64.b64decode(image['base64']).startswith(b'\x89PNG\r\n\x1a\n'), snapshot.get('screenshotError') + pathlib.Path('/private/tmp/mship-computer-use-fixture.png').write_bytes(base64.b64decode(image['base64'])) + else: + assert 'screen_capture_permission_required' in snapshot.get('screenshotError', '') + print('BLOCKED: screenshot verification skipped because Screen Recording grant is missing', flush=True) + baseline = int(next(node.get('value', node.get('label', '')) for node in snapshot['nodes'] if node.get('value', node.get('label', '')).startswith('Count: ')).split(': ')[1]) + button = find(snapshot, role='AXButton', label='Increment') + snapshot = mutate('click', snapshot, elementId=button['elementId']) + assert any(f'Count: {baseline + 1}' in node.get('value', '') or f'Count: {baseline + 1}' in node.get('label', '') for node in snapshot['nodes']), 'AX click did not increment counter' + field = find(snapshot, role='AXTextField', label='Fixture editable text') + snapshot = mutate('set_value', snapshot, elementId=field['elementId'], value='Mship α😀') + assert find(snapshot, role='AXTextField', label='Fixture editable text')['value'] == 'Mship α😀' + snapshot = mutate('type_text', snapshot, elementId=find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'], text=' typed') + assert ' typed' in find(snapshot, role='AXTextField', label='Fixture editable text')['value'], 'Unicode text event did not arrive' + snapshot = mutate('press_key', snapshot, windowId=snapshot['windowId'], key='Cmd+A') + snapshot = mutate('press_key', snapshot, windowId=snapshot['windowId'], key='Backspace') + assert find(snapshot, role='AXTextField', label='Fixture editable text').get('value', '') == '', 'Keyboard chord did not clear text' + snapshot = mutate('type_text', snapshot, elementId=find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'], text='Verified 😀') + assert find(snapshot, role='AXTextField', label='Fixture editable text')['value'] == 'Verified 😀' + activation = call('activate_app', bundleId=bundle) + assert activation['verified'] + snapshot = state() + slider = find(snapshot, role='AXSlider') + initial = float(slider['value']) + start = coordinate(snapshot, slider, 0.25) + end = coordinate(snapshot, slider, 0.8) + snapshot = mutate('drag', snapshot, **start, toX=end['x'], toY=end['y']) + assert float(find(snapshot, role='AXSlider')['value']) > initial + 10, 'Drag did not move slider' + scroll = find(snapshot, role='AXScrollArea') + snapshot = mutate('scroll', snapshot, **coordinate(snapshot, scroll), deltaX=0, deltaY=180) + assert any((node.get('value', '').startswith('Scroll: ') and node['value'] != 'Scroll: 0') or (node.get('label', '').startswith('Scroll: ') and node['label'] != 'Scroll: 0') for node in snapshot['nodes']), 'Scroll did not move fixture content' + button = find(snapshot, role='AXButton', label='Increment') + snapshot = mutate('perform_action', snapshot, elementId=button['elementId'], accessibilityAction='AXPress') + assert any(f'Count: {baseline + 2}' in node.get('value', '') or f'Count: {baseline + 2}' in node.get('label', '') for node in snapshot['nodes']), 'Explicit AX action did not increment' + print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) +finally: + if fixture: + fixture.terminate() + try: + fixture.wait(timeout=3) + except subprocess.TimeoutExpired: + fixture.kill() + helper.stdin.close() + helper.terminate() + try: + helper.wait(timeout=3) + except subprocess.TimeoutExpired: + helper.kill() + selector.close() diff --git a/apps/desktop/native/computer-use/tests/protocol.py b/apps/desktop/native/computer-use/tests/protocol.py new file mode 100644 index 00000000000..6ee0c6ddd6a --- /dev/null +++ b/apps/desktop/native/computer-use/tests/protocol.py @@ -0,0 +1,32 @@ +"""Run against a compiled helper; no permission requests or personal UI reads.""" +import json +import subprocess +import sys + +helper = sys.argv[1] +process = subprocess.Popen([helper], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + +def call(method, params=None): + process.stdin.write(json.dumps({'id': method, 'method': method, 'params': params or {}}) + '\n') + process.stdin.flush() + return json.loads(process.stdout.readline()) + +status = call('status')['result'] +assert status['kind'] == 'status' and status['platform'] == 'darwin' +assert isinstance(status['accessibility'], bool) and isinstance(status['screenRecording'], bool) +assert call('unknown')['error']['code'] == 'unknown_method' +assert call('request_permission', {'permission': 'invalid'})['error']['code'] == 'invalid_arguments' +if not status['accessibility']: + for action in ['get_app_state', 'click', 'type_text', 'press_key', 'scroll', 'drag', 'set_value', 'perform_action']: + assert call(action, {'bundleId': 'com.mothership.computer-use-fixture'})['error']['code'] == 'accessibility_permission_required' +else: + assert call('get_app_state', {'bundleId': 'com.apple.systempreferences'})['error']['code'] == 'protected_app' + assert call('get_app_state', {'bundleId': 'com.simstudio.computer-use'})['error']['code'] == 'protected_app' +process.stdin.write('x' * (129 * 1024) + '\n') +process.stdin.flush() +assert json.loads(process.stdout.readline())['error']['code'] == 'invalid_request' +assert call('unknown', {'text': 'A😀é漢字'})['error']['code'] == 'unknown_method' +assert call('status')['result'] == status +process.stdin.close() +assert process.wait(timeout=5) == 0 +print('PASS: persistent JSONL, status, malformed method, invalid permission, permission gates, clean EOF') diff --git a/apps/desktop/scripts/build.ts b/apps/desktop/scripts/build.ts index b15dc2fb354..6d9e059565b 100644 --- a/apps/desktop/scripts/build.ts +++ b/apps/desktop/scripts/build.ts @@ -80,6 +80,42 @@ function compileNativeHelpSearch(): void { console.log('• Compiled native macOS documentation Help search') } +/** Bundle the native helper; packaged builds sign it with the desktop distribution identity. */ +function compileComputerUse(): void { + if (process.platform !== 'darwin') return + const bundle = 'dist/native/Sim Computer Use.app' + const binaryDirectory = join(bundle, 'Contents', 'MacOS') + mkdirSync(binaryDirectory, { recursive: true }) + cpSync('native/computer-use/Info.plist', join(bundle, 'Contents', 'Info.plist')) + const parts: string[] = [] + for (const arch of ['arm64', 'x86_64']) { + const output = join('dist/native', `computer-use-${arch}`) + execFileSync( + 'xcrun', + [ + 'swiftc', + '-parse-as-library', + '-O', + '-target', + `${arch}-apple-macosx14.0`, + 'native/computer-use/ComputerUse.swift', + '-o', + output, + ], + { stdio: 'inherit' } + ) + parts.push(output) + } + execFileSync( + 'xcrun', + ['lipo', '-create', ...parts, '-output', join(binaryDirectory, 'SimComputerUse')], + { stdio: 'inherit' } + ) + for (const part of parts) rmSync(part) + execFileSync('codesign', ['--force', '--sign', '-', bundle], { stdio: 'inherit' }) + console.log('• Compiled native macOS Computer Use helper') +} + const common = { bundle: true, platform: 'node' as const, @@ -139,6 +175,7 @@ const renderer: BuildOptions = { async function run(): Promise { compileNativeHelpSearch() + compileComputerUse() if (watch) { const { context } = await import('esbuild') const rendererCtx = await context(renderer) diff --git a/apps/desktop/src/main/computer-use/native-client.test.ts b/apps/desktop/src/main/computer-use/native-client.test.ts new file mode 100644 index 00000000000..d53c90a3be5 --- /dev/null +++ b/apps/desktop/src/main/computer-use/native-client.test.ts @@ -0,0 +1,98 @@ +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { NativeComputerUseClient } from '@/main/computer-use/native-client' + +const roots: string[] = [] +const clients: NativeComputerUseClient[] = [] +afterEach(() => { + for (const client of clients.splice(0)) client.stop() + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +/** A real child process exercises framing and shutdown without OS permissions. */ +function helper(body: string) { + const root = mkdtempSync(join(tmpdir(), 'sim-native-transport-')) + roots.push(root) + const executable = join(root, 'helper') + writeFileSync(executable, `#!${process.execPath}\n${body}\n`) + chmodSync(executable, 0o700) + const reset = vi.fn() + const client = new NativeComputerUseClient(executable, reset) + clients.push(client) + return { client, reset } +} + +const status = JSON.stringify({ + kind: 'status', + platform: 'darwin', + accessibility: true, + screenRecording: false, +}) +const reader = `require('node:readline').createInterface({input:process.stdin}).on('line',line=>{const request=JSON.parse(line);` + +describe('native computer use transport', () => { + it('decodes split frames and correlates replies that arrive out of order', async () => { + const { client } = helper(`let first; ${reader} + if(!first){first=request;return} + const replies=[request,first].map(value=>JSON.stringify({id:value.id,result:${status}})+'\\n').join(''); + process.stdout.write(replies.slice(0,13)); + setTimeout(()=>process.stdout.write(replies.slice(13)),1); + });`) + const results = await Promise.all([client.request('status', {}), client.request('status', {})]) + expect(results).toEqual([JSON.parse(status), JSON.parse(status)]) + }) + + it('rejects malformed output and can start a fresh helper afterward', async () => { + const { client, reset } = helper(`${reader} + process.stdout.write(request.method==='bad'?'not json\\n':JSON.stringify({id:request.id,result:${status}})+'\\n'); + });`) + await expect(client.request('bad', {})).rejects.toThrow('invalid response') + expect(reset).toHaveBeenCalledOnce() + await expect(client.request('status', {})).resolves.toMatchObject({ kind: 'status' }) + }) + + it('rejects oversized output instead of retaining an unbounded buffer', async () => { + const { client } = helper(`${reader}process.stdout.write('x'.repeat(17*1024*1024));});`) + await expect(client.request('status', {})).rejects.toThrow('oversized response') + }) + + it('can recover after the helper executable is initially unavailable', async () => { + const root = mkdtempSync(join(tmpdir(), 'sim-native-missing-')) + roots.push(root) + const executable = join(root, 'helper') + const client = new NativeComputerUseClient(executable, () => {}) + clients.push(client) + await expect(client.request('status', {})).rejects.toThrow(/could not start|disconnected/) + writeFileSync( + executable, + `#!${process.execPath}\n${reader}process.stdout.write(JSON.stringify({id:request.id,result:${status}})+'\\n');});` + ) + chmodSync(executable, 0o700) + await expect(client.request('status', {})).resolves.toMatchObject({ kind: 'status' }) + }) + + it('rejects pending requests if the child exits', async () => { + const { client } = helper(`${reader}process.exit(0);});`) + await expect(client.request('status', {})).rejects.toThrow('helper exited') + }) + + it('Stop rejects pending input and prevents waiting requests crossing another Stop', async () => { + const { client } = + helper(`process.on('SIGTERM',()=>setTimeout(()=>process.exit(0),20));${reader} + if(request.method==='status')process.stdout.write(JSON.stringify({id:request.id,result:${status}})+'\\n'); + });`) + await client.request('status', {}) + const pending = client.request('hang', {}) + const stopped = expect(pending).rejects.toThrow('stopped') + await Promise.resolve() + client.stop() + await stopped + const waiting = client.request('status', {}) + const waitingStopped = expect(waiting).rejects.toThrow('stopped') + client.stop() + await waitingStopped + await expect(client.request('status', {})).resolves.toMatchObject({ kind: 'status' }) + }) +}) diff --git a/apps/desktop/src/main/computer-use/native-client.ts b/apps/desktop/src/main/computer-use/native-client.ts new file mode 100644 index 00000000000..9468f8621c5 --- /dev/null +++ b/apps/desktop/src/main/computer-use/native-client.ts @@ -0,0 +1,140 @@ +import { type ChildProcessWithoutNullStreams, spawn } from 'node:child_process' +import { + ComputerUseNativeReplySchema, + type ComputerUseResult, +} from '@sim/desktop-bridge/computer-use' +import { generateId } from '@sim/utils/id' + +const MAX_FRAME_BYTES = 16 * 1024 * 1024 +const REQUEST_TIMEOUT_MS = 30_000 + +interface PendingRequest { + resolve: (result: ComputerUseResult) => void + reject: (error: Error) => void + timer: ReturnType +} + +export interface ComputerUseNativeClient { + request(method: string, params: Record): Promise + stop(): void +} + +/** A private stdio channel avoids an unauthenticated local control port. */ +export class NativeComputerUseClient implements ComputerUseNativeClient { + private child: ChildProcessWithoutNullStreams | null = null + private pending = new Map() + private buffer = Buffer.alloc(0) + private stopping: Promise = Promise.resolve() + private generation = 0 + + constructor( + private readonly executable: string, + private readonly onReset: () => void + ) {} + + async request(method: string, params: Record): Promise { + const generation = this.generation + await this.stopping + if (generation !== this.generation) throw new Error('Computer Use stopped.') + if (this.pending.size >= 16) return Promise.reject(new Error('Computer Use is busy.')) + const child = this.start() + const id = generateId() + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.stopWithError(new Error('Computer Use timed out; inspect the app before retrying.')) + }, REQUEST_TIMEOUT_MS) + this.pending.set(id, { resolve, reject, timer }) + child.stdin.write(`${JSON.stringify({ id, method, params })}\n`, (error) => { + if (error && this.child === child) + this.stopWithError(new Error('Computer Use helper disconnected.')) + }) + }) + } + + stop(): void { + this.stopWithError(new Error('Computer Use stopped. Observe the app again before continuing.')) + } + + private start(): ChildProcessWithoutNullStreams { + if (this.child) return this.child + const child = spawn(this.executable, [], { + stdio: 'pipe', + env: { PATH: '/usr/bin:/bin', HOME: process.env.HOME, TMPDIR: process.env.TMPDIR }, + }) + this.child = child + child.stdout.on('data', (chunk: Buffer) => { + if (this.child !== child) return + this.buffer = Buffer.concat([this.buffer, chunk]) + if (this.buffer.length > MAX_FRAME_BYTES) { + this.stopWithError(new Error('Computer Use returned an oversized response.')) + return + } + let newline = this.buffer.indexOf(10) + while (newline >= 0) { + const line = this.buffer.subarray(0, newline).toString('utf8') + this.buffer = this.buffer.subarray(newline + 1) + try { + const reply = ComputerUseNativeReplySchema.parse(JSON.parse(line)) + const pending = this.pending.get(reply.id) + if (pending) { + clearTimeout(pending.timer) + this.pending.delete(reply.id) + if ('error' in reply) + pending.reject(new Error(`${reply.error.code}: ${reply.error.message}`)) + else pending.resolve(reply.result) + } + } catch { + this.stopWithError(new Error('Computer Use returned an invalid response.')) + return + } + newline = this.buffer.indexOf(10) + } + }) + /** Native diagnostics must never copy app contents into application logs. */ + child.stderr.resume() + child.stdin.on('error', () => { + if (this.child === child) this.stopWithError(new Error('Computer Use helper disconnected.')) + }) + child.on('error', () => { + if (this.child === child) + this.stopWithError(new Error('Computer Use helper could not start.')) + }) + child.on('exit', () => { + if (this.child === child) this.stopWithError(new Error('Computer Use helper exited.')) + }) + return child + } + + private stopWithError(error: Error): void { + this.generation += 1 + const child = this.child + this.child = null + this.buffer = Buffer.alloc(0) + for (const pending of this.pending.values()) { + clearTimeout(pending.timer) + pending.reject(error) + } + this.pending.clear() + this.onReset() + if (child) { + this.stopping = new Promise((resolve) => { + if (!child.pid || child.exitCode !== null || child.signalCode !== null) { + resolve() + return + } + const force = setTimeout(() => child.kill('SIGKILL'), 1_000) + force.unref() + child.once('exit', () => { + clearTimeout(force) + resolve() + }) + child.once('error', () => { + clearTimeout(force) + resolve() + }) + child.stdin.end() + child.kill('SIGTERM') + }) + } + } +} diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts new file mode 100644 index 00000000000..86ec661f0d5 --- /dev/null +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -0,0 +1,319 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { ComputerUseResult } from '@sim/desktop-bridge/computer-use' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ComputerUseService } from '@/main/computer-use/service' +import { createConfigStore } from '@/main/config' + +const roots: string[] = [] +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +function setup(enabled = true) { + const root = mkdtempSync(join(tmpdir(), 'sim-computer-use-test-')) + roots.push(root) + const config = createConfigStore(join(root, 'settings.json')) + config.set('computerUseEnabled', enabled) + let sequence = 0 + const request = vi.fn(async (method: string): Promise => { + if (method === 'status' || method === 'request_permission') + return { kind: 'status', platform: 'darwin', accessibility: true, screenRecording: false } + if (method === 'list_apps') + return { + kind: 'apps', + apps: [{ bundleId: 'com.example.Fixture', name: 'Fixture', pid: 42, isActive: false }], + } + if (method === 'get_app_state') + return { + kind: 'state', + bundleId: 'com.example.Fixture', + snapshotId: `s${++sequence}`, + windowId: '1', + windows: [], + nodes: [], + truncated: false, + } + return { + kind: 'action', + action: 'click', + bundleId: 'com.example.Fixture', + dispatched: true, + verified: false, + } + }) + const native = { request, stop: vi.fn() } + const approveApp = vi.fn(async (): Promise<'always' | 'deny' | 'once'> => 'always') + const onActivity = vi.fn() + const openPermissionSettings = vi.fn(async () => {}) + const setStopShortcutActive = vi.fn((active: boolean) => active) + const service = new ComputerUseService({ + config, + native, + supported: true, + approveApp, + onActivity, + setStopShortcutActive, + openPermissionSettings, + }) + const state = (chat = 'chat') => + service.execute(`state-${sequence}`, chat, { + action: 'get_app_state', + bundleId: 'com.example.Fixture', + }) + const click = (snapshotId: string, chat = 'chat', call = 'click') => + service.execute(call, chat, { + action: 'click', + bundleId: 'com.example.Fixture', + snapshotId, + elementId: 'e1', + }) + return { + service, + config, + native, + approveApp, + onActivity, + setStopShortcutActive, + openPermissionSettings, + state, + click, + } +} + +describe('native computer use authority and lifecycle', () => { + it('rejects execution while switched off before touching the helper', async () => { + const { service, native } = setup(false) + await expect(service.execute('call', 'chat', { action: 'list_apps' })).rejects.toThrow( + 'switched off' + ) + expect(native.request).not.toHaveBeenCalled() + }) + + it('reports native permissions separately from the opt-in switch', async () => { + const { service } = setup(false) + expect(await service.getStatus()).toEqual({ + supported: true, + enabled: false, + permissions: { accessibility: true, screenCapture: false }, + activeAction: null, + }) + }) + + it('does not read app state when the user denies app access', async () => { + const { state, approveApp, native } = setup() + approveApp.mockResolvedValue('deny') + await expect(state()).rejects.toThrow('denied') + expect(native.request.mock.calls.map(([method]) => method)).toEqual(['list_apps']) + }) + + it('binds observed elements to their originating chat', async () => { + const { state, click, native } = setup() + await state('owner') + await expect(click('s1', 'another-chat')).rejects.toThrow('another chat') + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('click') + }) + + it('consumes an observation even when the dispatched action is unverified', async () => { + const { state, click, native } = setup() + await state() + expect(await click('s1')).toMatchObject({ dispatched: true, verified: false }) + await expect(click('s1', 'chat', 'replay')).rejects.toThrow('stale') + expect(native.request.mock.calls.filter(([method]) => method === 'click')).toHaveLength(1) + }) + + it('invalidates previous references after a fresh observation of the app', async () => { + const { state, click } = setup() + await state() + await state() + await expect(click('s1')).rejects.toThrow('stale') + await expect(click('s2')).resolves.toMatchObject({ kind: 'action' }) + }) + + it('requires fresh observations after a helper restart', async () => { + const { state, click, service } = setup() + await state() + service.invalidateSnapshots() + await expect(click('s1')).rejects.toThrow('stale') + }) + + it('clears grants and opt-in on account reset', async () => { + const { state, service, native } = setup() + await state() + expect(service.listAppPermissions()).toHaveLength(1) + service.reset() + expect(service.isEnabled()).toBe(false) + expect(service.listAppPermissions()).toEqual([]) + expect(native.stop).toHaveBeenCalledOnce() + }) + + it('revoking app access invalidates its cached observation', async () => { + const { state, service, click } = setup() + await state() + service.revokeApp('com.example.Fixture') + expect(service.listAppPermissions()).toEqual([]) + await expect(click('s1')).rejects.toThrow('stale') + }) + + it('blocks self-automation before showing an approval', async () => { + const { service, approveApp, native } = setup() + await expect( + service.execute('self', 'chat', { action: 'get_app_state', bundleId: 'ai.sim.desktop' }) + ).rejects.toThrow('own permission') + expect(approveApp).not.toHaveBeenCalled() + expect(native.request).not.toHaveBeenCalled() + }) + + it('stops work waiting for app approval and does not persist that approval', async () => { + const { service, approveApp, state, native } = setup() + let resolveApproval: (answer: 'always') => void = () => {} + approveApp.mockImplementation( + () => + new Promise((resolve) => { + resolveApproval = resolve + }) + ) + const pending = state() + const rejection = expect(pending).rejects.toThrow('stopped') + await vi.waitFor(() => expect(approveApp).toHaveBeenCalledOnce()) + service.cancel() + resolveApproval('always') + await rejection + expect(service.listAppPermissions()).toEqual([]) + expect(native.request.mock.calls.map(([method]) => method)).not.toContain('get_app_state') + }) + + it('turning off cancels pending and queued work', async () => { + const { service, approveApp, state } = setup() + let resolveApproval: (answer: 'once') => void = () => {} + approveApp.mockImplementation( + () => + new Promise((resolve) => { + resolveApproval = resolve + }) + ) + const pending = state() + const rejection = expect(pending).rejects.toThrow('stopped') + const queued = service.execute('queued', 'chat', { action: 'list_apps' }) + const queuedRejection = expect(queued).rejects.toThrow('stopped') + await vi.waitFor(() => expect(approveApp).toHaveBeenCalledOnce()) + await service.setEnabled(false) + resolveApproval('once') + await rejection + await queuedRejection + }) + + it('keeps temporary app approval in one task and clears it on Stop', async () => { + const { state, approveApp, service } = setup() + approveApp.mockResolvedValue('once') + await state('first-task') + await state('first-task') + expect(approveApp).toHaveBeenCalledTimes(1) + expect(service.listAppPermissions()).toEqual([]) + await state('second-task') + expect(approveApp).toHaveBeenCalledTimes(2) + service.cancel() + await state('first-task') + expect(approveApp).toHaveBeenCalledTimes(3) + }) + + it('Stop cancels a tool still waiting on server authorization', async () => { + const { service, native } = setup() + let finish: (value: { scopeId: string; input: unknown }) => void = () => {} + const authorize = vi.fn( + () => + new Promise<{ scopeId: string; input: unknown }>((resolve) => { + finish = resolve + }) + ) + const pending = service.executeAuthorized('pending-authorization', authorize) + const rejected = expect(pending).rejects.toThrow('stopped') + service.cancel('pending-authorization') + finish({ scopeId: 'chat', input: { action: 'list_apps' } }) + await rejected + expect(native.request).not.toHaveBeenCalled() + }) + + it('rejects concurrent duplicate claims before contacting the server again', async () => { + const { service } = setup() + let finish: (value: { scopeId: string; input: unknown }) => void = () => {} + const pending = service.executeAuthorized( + 'same-call', + () => + new Promise((resolve) => { + finish = resolve + }) + ) + const secondAuthorization = vi.fn() + await expect(service.executeAuthorized('same-call', secondAuthorization)).rejects.toThrow( + 'already running' + ) + expect(secondAuthorization).not.toHaveBeenCalled() + finish({ scopeId: 'chat', input: { action: 'list_apps' } }) + await expect(pending).resolves.toMatchObject({ kind: 'apps' }) + }) + + it('offers the global Stop shortcut only while a native action is active', async () => { + const { state, onActivity, setStopShortcutActive } = setup() + await state() + expect(setStopShortcutActive.mock.calls).toEqual([[true], [false]]) + expect(onActivity.mock.calls[0][0]).toMatchObject({ stopShortcutAvailable: true }) + expect(onActivity.mock.calls.at(-1)).toEqual([null]) + }) + + it('clears the global Stop shortcut and activity after helper failure', async () => { + const { state, native, onActivity, setStopShortcutActive } = setup() + native.request.mockRejectedValueOnce(new Error('Helper disconnected')) + await expect(state()).rejects.toThrow('Helper disconnected') + expect(setStopShortcutActive.mock.calls).toEqual([[true], [false]]) + expect(onActivity.mock.calls.at(-1)).toEqual([null]) + }) + + it('does not return an outdated enabled state after a pending permission read', async () => { + const { service, native } = setup() + let finish: (value: ComputerUseResult) => void = () => {} + native.request.mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve + }) + ) + const pending = service.getStatus() + await service.setEnabled(false) + finish({ kind: 'status', platform: 'darwin', accessibility: true, screenRecording: false }) + expect(await pending).toMatchObject({ enabled: false, activeAction: null }) + }) + + it('fails closed if device opt-in cannot be persisted', async () => { + const { service, config, native } = setup(false) + vi.spyOn(config, 'flush').mockReturnValue(false) + await expect(service.setEnabled(true)).rejects.toThrow('Could not save') + expect(service.isEnabled()).toBe(false) + expect(native.request).not.toHaveBeenCalled() + }) + + it('reports failed grant erasure so account teardown retains its recovery marker', async () => { + const { service, config, state } = setup() + await state() + vi.spyOn(config, 'flush').mockReturnValue(false) + expect(() => service.reset()).toThrow('Could not clear') + expect(service.isEnabled()).toBe(false) + expect(service.listAppPermissions()).toEqual([]) + }) + + it('does not operate an app if persistent approval cannot be saved', async () => { + const { config, state, native, service } = setup() + vi.spyOn(config, 'flush').mockReturnValue(false) + await expect(state()).rejects.toThrow('Could not save the app permission') + expect(native.request.mock.calls.map(([method]) => method)).toEqual(['list_apps']) + expect(service.listAppPermissions()).toEqual([]) + }) + + it('opens the requested permission settings even when access is already denied', async () => { + const { service, openPermissionSettings } = setup() + const result = await service.requestPermission('screenCapture') + expect(openPermissionSettings).toHaveBeenCalledWith('screenCapture') + expect(result.permissions.screenCapture).toBe(false) + }) +}) diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts new file mode 100644 index 00000000000..984f2fb1aa9 --- /dev/null +++ b/apps/desktop/src/main/computer-use/service.ts @@ -0,0 +1,269 @@ +import type { + ComputerUseActivity, + ComputerUseAppPermission, + ComputerUseStatus, +} from '@sim/desktop-bridge' +import { + type ComputerUseInput, + type ComputerUseResult, + ComputerUseSchema, +} from '@sim/desktop-bridge/computer-use' +import { omit } from '@sim/utils/object' +import type { ComputerUseNativeClient } from '@/main/computer-use/native-client' +import type { ConfigStore } from '@/main/config' + +interface ComputerUseServiceDeps { + config: Pick + supported: boolean + native: ComputerUseNativeClient + approveApp: ( + app: ComputerUseAppPermission, + signal: AbortSignal + ) => Promise<'once' | 'always' | 'deny'> + onActivity: (activity: ComputerUseActivity | null) => void + setStopShortcutActive?: (active: boolean) => boolean + openPermissionSettings?: (permission: 'accessibility' | 'screenCapture') => Promise +} + +interface ToolWork { + toolCallId: string + scopeId: string + input: ComputerUseInput + controller: AbortController +} + +/** Native snapshots cannot transfer authority between chats or restarted helpers. */ +export class ComputerUseService { + private admissions = new Map() + private active: ToolWork | null = null + private tail: Promise = Promise.resolve() + private queued = new Map() + private snapshots = new Map() + private activity: ComputerUseActivity | null = null + private taskGrants = new Map>() + + constructor(private readonly deps: ComputerUseServiceDeps) {} + + isEnabled(): boolean { + return this.deps.supported && this.deps.config.get('computerUseEnabled') === true + } + + async getStatus(): Promise { + const status: ComputerUseStatus = { + supported: this.deps.supported, + enabled: this.isEnabled(), + permissions: { accessibility: false, screenCapture: false }, + activeAction: this.activity, + } + if (!status.supported) return status + const native = await this.deps.native.request('status', {}) + if (native.kind !== 'status') throw new Error('Computer Use permission status is unavailable.') + status.permissions = { + accessibility: native.accessibility, + screenCapture: native.screenRecording, + } + status.enabled = this.isEnabled() + status.activeAction = this.activity + return status + } + + async setEnabled(enabled: boolean): Promise { + if (enabled && !this.deps.supported) throw new Error('Computer Use requires macOS 14 or later.') + this.deps.config.set('computerUseEnabled', enabled) + if (!enabled) this.cancel() + if (!this.deps.config.flush()) { + this.deps.config.set('computerUseEnabled', false) + this.cancel() + throw new Error('Could not save Computer Use settings. Computer Use has been switched off.') + } + this.deps.onActivity(this.activity) + return this.getStatus() + } + + async requestPermission( + permission: 'accessibility' | 'screenCapture' + ): Promise { + if (!this.deps.supported) return this.getStatus() + await this.deps.native.request('request_permission', { permission }) + await this.deps.openPermissionSettings?.(permission) + return this.getStatus() + } + + listAppPermissions(): ComputerUseAppPermission[] { + return (this.deps.config.get('computerUseAllowedApps') ?? []).map((entry) => ({ ...entry })) + } + + revokeApp(bundleId: string): void { + this.deps.config.set( + 'computerUseAllowedApps', + this.listAppPermissions().filter((app) => app.bundleId !== bundleId) + ) + const persisted = this.deps.config.flush() + for (const work of this.queued.values()) { + if ('bundleId' in work.input && work.input.bundleId === bundleId) this.cancel(work.toolCallId) + } + for (const apps of this.taskGrants.values()) apps.delete(bundleId) + this.invalidateSnapshots() + if (!persisted) throw new Error('Could not save the revoked app permission.') + } + + invalidateSnapshots(): void { + this.snapshots.clear() + } + + reset(): void { + this.cancel() + this.deps.config.set('computerUseEnabled', false) + this.deps.config.set('computerUseAllowedApps', []) + if (!this.deps.config.flush()) + throw new Error('Could not clear saved Computer Use permissions.') + } + + cancel(toolCallId?: string): void { + for (const [id, controller] of this.admissions) { + if (!toolCallId || id === toolCallId) controller.abort() + } + if (!toolCallId) this.taskGrants.clear() + else { + const work = this.queued.get(toolCallId) + if (work) this.taskGrants.delete(work.scopeId) + } + for (const work of this.queued.values()) { + if (!toolCallId || work.toolCallId === toolCallId) work.controller.abort() + } + if (!toolCallId || this.active?.toolCallId === toolCallId) this.deps.native.stop() + this.invalidateSnapshots() + } + + /** Register Stop before awaiting the server's one-shot tool authorization. */ + async executeAuthorized( + toolCallId: string, + authorize: () => Promise<{ scopeId: string; input: unknown }> + ): Promise { + if (!this.isEnabled()) throw new Error('Computer Use is switched off on this Mac.') + if (this.admissions.size + this.queued.size >= 16) + throw new Error('Computer Use has too many queued actions.') + if (this.admissions.has(toolCallId) || this.queued.has(toolCallId)) + throw new Error('This Computer Use action is already running.') + const controller = new AbortController() + this.admissions.set(toolCallId, controller) + try { + const authorized = await authorize() + if (controller.signal.aborted || !this.isEnabled()) throw new Error('Computer Use stopped.') + this.admissions.delete(toolCallId) + return await this.execute(toolCallId, authorized.scopeId, authorized.input) + } finally { + this.admissions.delete(toolCallId) + } + } + + execute(toolCallId: string, scopeId: string, raw: unknown): Promise { + if (!this.isEnabled()) + return Promise.reject(new Error('Computer Use is switched off on this Mac.')) + if (this.admissions.size + this.queued.size >= 16) + return Promise.reject(new Error('Computer Use has too many queued actions.')) + if (this.queued.has(toolCallId)) + return Promise.reject(new Error('This Computer Use action is already running.')) + const input = ComputerUseSchema.parse(raw) + const work: ToolWork = { toolCallId, scopeId, input, controller: new AbortController() } + this.queued.set(toolCallId, work) + const result = this.tail.then(() => this.run(work)) + this.tail = result.then( + () => undefined, + () => undefined + ) + return result.finally(() => this.queued.delete(toolCallId)) + } + + private check(work: ToolWork): void { + if (work.controller.signal.aborted || !this.isEnabled()) + throw new Error('Computer Use stopped.') + } + + private async run(work: ToolWork): Promise { + this.check(work) + this.active = work + const { input } = work + try { + this.activity = { + toolCallId: work.toolCallId, + scopeId: work.scopeId, + action: input.action, + ...('bundleId' in input ? { bundleId: input.bundleId } : {}), + startedAt: Date.now(), + stopShortcutAvailable: this.deps.setStopShortcutActive?.(true) ?? false, + } + this.deps.onActivity(this.activity) + if ('bundleId' in input) { + if (/^ai\.sim\.desktop(?:\.|$)/.test(input.bundleId)) { + throw new Error('Computer Use cannot operate Sim or its own permission controls.') + } + if ('snapshotId' in input) { + const owner = this.snapshots.get(input.snapshotId) + if (owner?.scopeId !== work.scopeId || owner.bundleId !== input.bundleId) { + throw new Error( + 'This snapshot is stale or belongs to another chat. Observe the app again.' + ) + } + } + const appName = await this.authorizeApp(input.bundleId, work) + this.activity = { ...this.activity, appName } + this.deps.onActivity(this.activity) + } + this.check(work) + if ('snapshotId' in input) this.snapshots.delete(input.snapshotId) + const result = await this.deps.native.request(input.action, omit(input, ['action'])) + this.check(work) + if (result.kind === 'state') { + if (!('bundleId' in input) || result.bundleId !== input.bundleId) { + throw new Error('Computer Use returned state for a different app.') + } + for (const [id, owner] of this.snapshots) { + if (owner.bundleId === result.bundleId) this.snapshots.delete(id) + } + if (this.snapshots.size >= 16) this.snapshots.clear() + this.snapshots.set(result.snapshotId, { scopeId: work.scopeId, bundleId: result.bundleId }) + } + return result + } finally { + this.deps.setStopShortcutActive?.(false) + this.active = null + this.activity = null + this.deps.onActivity(null) + } + } + + private async authorizeApp(bundleId: string, work: ToolWork): Promise { + const approved = this.listAppPermissions().find((app) => app.bundleId === bundleId) + if (approved) return approved.displayName + const taskName = this.taskGrants.get(work.scopeId)?.get(bundleId) + if (taskName) return taskName + const apps = await this.deps.native.request('list_apps', {}) + this.check(work) + const app = + apps.kind === 'apps' ? apps.apps.find((app) => app.bundleId === bundleId) : undefined + const target = { bundleId, displayName: app?.name ?? bundleId } + const answer = await this.deps.approveApp(target, work.controller.signal) + this.check(work) + if (answer === 'deny') + throw new Error(`Computer Use access to ${target.displayName} was denied.`) + if (answer === 'once') { + if (this.taskGrants.size >= 64 && !this.taskGrants.has(work.scopeId)) { + const oldest = this.taskGrants.keys().next().value + if (oldest) this.taskGrants.delete(oldest) + } + const apps = this.taskGrants.get(work.scopeId) ?? new Map() + apps.set(bundleId, target.displayName) + this.taskGrants.set(work.scopeId, apps) + } + if (answer === 'always') { + const previous = this.listAppPermissions() + this.deps.config.set('computerUseAllowedApps', [...previous, target]) + if (!this.deps.config.flush()) { + this.deps.config.set('computerUseAllowedApps', previous) + throw new Error('Could not save the app permission. Computer Use did not run this action.') + } + } + return target.displayName + } +} diff --git a/apps/desktop/src/main/config.ts b/apps/desktop/src/main/config.ts index 30c00528b10..c3c34b6faab 100644 --- a/apps/desktop/src/main/config.ts +++ b/apps/desktop/src/main/config.ts @@ -104,6 +104,9 @@ export interface DesktopSettings { /** Whether omnibox typing may request live Google search completions. */ browserSearchSuggestionsEnabled?: boolean terminalEnabled?: boolean + /** Native app access is opt-in and cleared when the signed-in account changes. */ + computerUseEnabled?: boolean + computerUseAllowedApps?: Array<{ bundleId: string; displayName: string }> /** Device-wide browser page appearance; `app` follows Sim. */ browserTheme?: 'app' | 'light' | 'dark' /** Device-wide default zoom for built-in browser pages. */ diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 56888943323..1bc85bba4fb 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -1,8 +1,18 @@ +import { release } from 'node:os' import { join } from 'node:path' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { OpenDialogOptions, Session, WebContents } from 'electron' -import { app, BrowserWindow, crashReporter, dialog, net, session, shell } from 'electron' +import { + app, + BrowserWindow, + crashReporter, + dialog, + globalShortcut, + net, + session, + shell, +} from 'electron' import { beginAccountDataTeardown, completeDeploymentScopedTeardown, @@ -36,6 +46,8 @@ import { setPanelFocused as setBrowserAgentPanelFocused, } from '@/main/browser-agent/session' import { attachClientInfo } from '@/main/client-info' +import { NativeComputerUseClient } from '@/main/computer-use/native-client' +import { ComputerUseService } from '@/main/computer-use/service' import { APP_NAME_FOR_CHANNEL, channelForOrigin, @@ -65,7 +77,7 @@ import { registerLocalPageScheme, } from '@/main/local-pages' import { installApplicationMenu } from '@/main/menu' -import { openExternalSafe } from '@/main/navigation' +import { isAppOrigin, openExternalSafe } from '@/main/navigation' import { createEventLog, installMainProcessFailureObservers } from '@/main/observability' import { ScopedEventRouter } from '@/main/scoped-event-router' import { installGlobalGuards } from '@/main/security-guards' @@ -146,6 +158,61 @@ function main(): void { join(userDataPath, 'local-filesystem-grants.json') ), }) + const computerNative = new NativeComputerUseClient( + join( + app.isPackaged ? process.resourcesPath : join(__dirname, 'native'), + 'Sim Computer Use.app', + 'Contents', + 'MacOS', + 'SimComputerUse' + ), + () => computerUse.invalidateSnapshots() + ) + let computerStopShortcutRegistered = false + const computerStopShortcut = 'CommandOrControl+Shift+Escape' + const computerUse = new ComputerUseService({ + config, + supported: process.platform === 'darwin' && Number.parseInt(release(), 10) >= 23, + native: computerNative, + openPermissionSettings: (permission) => + shell.openExternal( + permission === 'accessibility' + ? 'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility' + : 'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture' + ), + setStopShortcutActive: (active) => { + if (active && !computerStopShortcutRegistered) { + computerStopShortcutRegistered = globalShortcut.register(computerStopShortcut, () => + computerUse.cancel() + ) + } else if (!active && computerStopShortcutRegistered) { + globalShortcut.unregister(computerStopShortcut) + computerStopShortcutRegistered = false + } + return computerStopShortcutRegistered + }, + approveApp: async (target, signal) => { + const result = await dialog.showMessageBox({ + type: 'question', + title: 'Computer Use', + message: `Allow Mothership to use ${target.displayName}?`, + detail: `Mothership can read and operate this app, including taking screenshots. App: ${target.bundleId}`, + buttons: ['Allow for This Task', 'Always Allow', 'Cancel'], + defaultId: 2, + cancelId: 2, + noLink: true, + signal, + }) + return result.response === 0 ? 'once' : result.response === 1 ? 'always' : 'deny' + }, + onActivity: (activity) => { + for (const win of getWindows()) { + if (isAppOrigin(win.webContents.getURL(), appOrigin())) { + win.webContents.send('computer-use:activity', activity) + } + } + }, + }) const scopeEvents = new ScopedEventRouter() const terminal = new TerminalRegistry({ load: (scopeId) => desktopChatSessions.getTerminal(processOrigin, scopeId) ?? undefined, @@ -285,6 +352,7 @@ function main(): void { origin: appOrigin, events, getWindows, + stopLocalActions: () => computerUse.cancel(), clearHandoffState: async () => { const stores = [ { label: 'sign-in handoff state', clear: () => handoff.clear() }, @@ -309,6 +377,7 @@ function main(): void { { label: 'terminal sessions', clear: () => terminal.dispose() }, { label: 'task resource state', clear: clearDesktopChatSessions }, { label: 'local filesystem grants', clear: () => localFilesystem.forgetAll() }, + { label: 'computer use', clear: () => computerUse.reset() }, ] const outcomes = await Promise.allSettled( stores.map(({ clear }) => Promise.resolve().then(clear)) @@ -537,7 +606,10 @@ function main(): void { preloadPath, isPackaged: app.isPackaged, getParentWindow: getMainWindow, - prepareDeploymentScopedStateChange: () => beginAccountDataTeardown('deployment', appOrigin()), + prepareDeploymentScopedStateChange: () => { + computerUse.cancel() + return beginAccountDataTeardown('deployment', appOrigin()) + }, clearDeploymentScopedState: async () => { await waitForAccountDataMutations() // allSettled, not sequential awaits: these are independent stores, and a @@ -546,6 +618,7 @@ function main(): void { // access. Each failure is named so the picker can say what survived. const stores = [ { label: 'local file access', clear: () => localFilesystem.forgetAll() }, + { label: 'computer use', clear: () => computerUse.reset() }, { label: 'built-in browser sessions', clear: () => clearAgentBrowserProfile({ settingsPersistence: 'server-repair' }), @@ -627,6 +700,8 @@ function main(): void { // now be released without leaving a cancelled quit in a degraded state. tray?.destroy() tray = null + computerUse.cancel() + if (computerStopShortcutRegistered) globalShortcut.unregister(computerStopShortcut) localFilesystem.close() quiesceBrowserSessions() terminal.dispose() @@ -664,6 +739,7 @@ function main(): void { const stores = [ { label: 'built-in browser sessions', clear: () => clearAgentBrowserProfile() }, { label: 'local filesystem grants', clear: () => localFilesystem.forgetAll() }, + { label: 'computer use', clear: () => computerUse.reset() }, { label: 'browser site history', clear: () => { @@ -760,6 +836,7 @@ function main(): void { scopeEvents.sendTerminal(scopeId, 'terminal:command', { ...event, scopeId }), }) registerIpcHandlers({ + computerUse, appOrigin, allowHttpLocalhost, accountDataAvailable, diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 840334ce29c..96ef6fb0a8e 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -77,6 +77,7 @@ import { } from '@/main/browser-import' import { getSearchSuggestions } from '@/main/browser-search/suggestions' import { listSites } from '@/main/browser-sites' +import type { ComputerUseService } from '@/main/computer-use/service' import { isSafeInternalPath } from '@/main/config' import type { DesktopSettingsService } from '@/main/desktop-settings' import { isDesktopPreferenceKey } from '@/main/desktop-settings' @@ -324,6 +325,7 @@ export function parseDesktopNotificationPayload(raw: unknown): DesktopNotificati } export interface IpcDeps { + computerUse?: ComputerUseService appOrigin: () => string allowHttpLocalhost: () => boolean /** False while local account-data persistence is unavailable or teardown must be retried. */ @@ -510,21 +512,19 @@ async function fetchDesktopToolAuthorization( deps: IpcDeps, toolCallId: unknown, claim = false, - onFailureStatus?: (status: number) => void + onFailureStatus?: (status: number) => void, + authorizationPath = '/api/desktop/tool/authorize' ): Promise { if (!isDesktopToolCallId(toolCallId)) return null const startedAt = Date.now() try { - const response = await event.sender.session.fetch( - `${deps.appOrigin()}/api/desktop/tool/authorize`, - { - method: 'POST', - credentials: 'include', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ toolCallId, ...(claim ? { claim: true } : {}) }), - signal: AbortSignal.timeout(BROWSER_TOOL_AUTHORIZATION_TIMEOUT_MS), - } - ) + const response = await event.sender.session.fetch(`${deps.appOrigin()}${authorizationPath}`, { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ toolCallId, ...(claim ? { claim: true } : {}) }), + signal: AbortSignal.timeout(BROWSER_TOOL_AUTHORIZATION_TIMEOUT_MS), + }) if (!response.ok) { onFailureStatus?.(response.status) logger.warn('Desktop tool authorization was rejected', { @@ -680,6 +680,77 @@ export function registerIpcHandlers(deps: IpcDeps): void { } const channels: Record = { + 'computer-use:status': { + kind: 'invoke', + gate: 'app-origin', + requiresAccountData: true, + denied: null, + handler: () => deps.computerUse?.getStatus(), + }, + 'computer-use:set-enabled': { + kind: 'invoke', + gate: 'app-origin', + requiresAccountData: true, + needsUserActivation: true, + denied: null, + handler: (enabled) => { + if (typeof enabled !== 'boolean') + throw new Error('A boolean Computer Use preference is required.') + return deps.computerUse?.setEnabled(enabled) + }, + }, + 'computer-use:request-permission': { + kind: 'invoke', + gate: 'app-origin', + requiresAccountData: true, + needsUserActivation: true, + denied: null, + handler: (permission) => { + if (permission !== 'accessibility' && permission !== 'screenCapture') + throw new Error('Unknown Computer Use permission.') + return deps.computerUse?.requestPermission(permission) + }, + }, + 'computer-use:list-permissions': { + kind: 'invoke', + gate: 'app-origin', + requiresAccountData: true, + denied: [], + handler: () => deps.computerUse?.listAppPermissions() ?? [], + }, + 'computer-use:revoke-app': { + kind: 'invoke', + gate: 'app-origin', + requiresAccountData: true, + needsUserActivation: true, + denied: undefined, + handler: (bundleId) => { + if (typeof bundleId !== 'string' || bundleId.length > 255) + throw new Error('Invalid app identifier.') + deps.computerUse?.revokeApp(bundleId) + }, + }, + 'computer-use:cancel': { + kind: 'invoke', + gate: 'app-origin', + denied: undefined, + handler: (toolCallId) => { + if (toolCallId !== undefined && !isDesktopToolCallId(toolCallId)) + throw new Error('Invalid tool call identifier.') + deps.computerUse?.cancel(toolCallId) + }, + }, + 'computer-use:execute-tool': { + kind: 'invoke', + gate: 'app-origin', + requiresAccountData: true, + denied: undefined, + handler: (scopeId, toolCallId, params) => { + if (!deps.computerUse || typeof scopeId !== 'string' || typeof toolCallId !== 'string') + throw new Error('Computer Use is unavailable.') + return deps.computerUse.execute(toolCallId, scopeId, params) + }, + }, 'desktop:open-external': { kind: 'invoke', gate: 'any', @@ -1989,6 +2060,28 @@ export function registerIpcHandlers(deps: IpcDeps): void { return spec.denied } let handlerArgs = args + if (channel === 'computer-use:execute-tool') { + if (!deps.computerUse?.isEnabled()) + throw new Error('Computer Use is switched off on this Mac.') + const toolCallId = args[0] + if (!isDesktopToolCallId(toolCallId)) throw new Error('Invalid tool call identifier.') + return deps.computerUse.executeAuthorized(toolCallId, async () => { + const authorization = await fetchDesktopToolAuthorization( + event, + deps, + toolCallId, + false, + undefined, + '/api/desktop/computer/authorize' + ) + if (!authorization || authorization.toolName !== 'computer') { + throw new Error('This is not an authorized pending Computer Use action.') + } + if (!senderAllowed(event, spec.gate) || !deps.accountDataAvailable()) + throw new Error('Computer Use session ended.') + return { scopeId: authorization.chatId, input: authorization.args } + }) + } if (channel === 'browser-agent:execute-tool') { const toolCallId = args[0] const requestedTool = args[1] diff --git a/apps/desktop/src/main/session-lifecycle.test.ts b/apps/desktop/src/main/session-lifecycle.test.ts index bf441fe3f9f..f04ec8fc41c 100644 --- a/apps/desktop/src/main/session-lifecycle.test.ts +++ b/apps/desktop/src/main/session-lifecycle.test.ts @@ -240,11 +240,12 @@ describe('tearDownSession', () => { expect(clearCache).toHaveBeenCalledOnce() }) - it('does not erase local data when the recovery marker cannot be written', async () => { + it('stops runtime work without erasing local data when the recovery marker cannot be written', async () => { const directory = mkdtempSync(join(tmpdir(), 'sim-account-recovery-')) const blockedParent = join(directory, 'blocked') initializeAccountDataRecovery(join(blockedParent, 'teardown-required.json')) writeFileSync(blockedParent, 'not a directory') + const stopLocalActions = vi.fn() const clearHandoffState = vi.fn(async () => {}) const clearBrowserProfile = vi.fn(async () => {}) const clearStorageData = vi.fn(async () => {}) @@ -258,10 +259,12 @@ describe('tearDownSession', () => { clearHandoffState, { filePath: '/tmp/events.log', record: vi.fn() }, clearBrowserProfile, - async () => {} + async () => {}, + stopLocalActions ) ).rejects.toThrow('recovery marker') + expect(stopLocalActions).toHaveBeenCalledOnce() expect(clearHandoffState).not.toHaveBeenCalled() expect(clearBrowserProfile).not.toHaveBeenCalled() expect(clearStorageData).not.toHaveBeenCalled() @@ -389,6 +392,7 @@ describe('createSessionLifecycleCoordinator', () => { appSession: session, origin: () => APP, events: { filePath: '/tmp/events.log', record: vi.fn() }, + stopLocalActions: vi.fn(), clearHandoffState, clearBrowserProfile: vi.fn(async () => {}), getWindows: () => [first, second], @@ -418,6 +422,54 @@ describe('createSessionLifecycleCoordinator', () => { expect(clearHandoffState).toHaveBeenCalledOnce() }) + it.each(['menu', 'navigation'] as const)( + 'stops local actions synchronously before a delayed %s sign-out request', + async (trigger) => { + let finishRevoke: () => void = () => {} + const revoke = new Promise((resolve) => { + finishRevoke = resolve + }) + const win = new BrowserWindow() + vi.mocked(win.webContents.getURL).mockReturnValue(`${APP}/home`) + vi.mocked(win.webContents.executeJavaScript).mockImplementationOnce(() => revoke) + const stopLocalActions = vi.fn() + const clearHandoffState = vi.fn(async () => {}) + const coordinator = createSessionLifecycleCoordinator({ + appSession: { + cookies: { on: vi.fn() }, + clearStorageData: vi.fn(async () => {}), + clearCache: vi.fn(async () => {}), + } as unknown as Session, + origin: () => APP, + events: { filePath: '/tmp/events.log', record: vi.fn() }, + stopLocalActions, + clearHandoffState, + clearBrowserProfile: vi.fn(async () => {}), + getWindows: () => [win], + }) + if (trigger === 'menu') { + void coordinator.signOut() + } else { + coordinator.attachWindow(win) + const windowEventCalls = vi.mocked(win.webContents.on).mock.calls as unknown as Array< + [string, (...args: unknown[]) => unknown] + > + const navigation = windowEventCalls.find(([event]) => event === 'did-navigate-in-page')?.[1] + if (!navigation) throw new Error('Missing navigation listener') + navigation({}, `${APP}/login?fromLogout=true`) + } + expect(stopLocalActions).toHaveBeenCalledOnce() + expect(win.webContents.executeJavaScript).toHaveBeenCalledOnce() + expect(stopLocalActions.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(win.webContents.executeJavaScript).mock.invocationCallOrder[0] + ) + expect(clearHandoffState).not.toHaveBeenCalled() + finishRevoke() + await expect(coordinator.awaitTeardown()).resolves.toBe(true) + expect(clearHandoffState).toHaveBeenCalledOnce() + } + ) + it('shares one awaitable teardown and does not open login when clearing fails', async () => { let releaseBrowserClear: (() => void) | undefined const browserClear = new Promise((resolve) => { @@ -434,6 +486,7 @@ describe('createSessionLifecycleCoordinator', () => { } as unknown as Session, origin: () => APP, events: { filePath: '/tmp/events.log', record: vi.fn() }, + stopLocalActions: vi.fn(), clearHandoffState: vi.fn(async () => {}), clearBrowserProfile: vi.fn(() => browserClear), getWindows: () => [win], diff --git a/apps/desktop/src/main/session-lifecycle.ts b/apps/desktop/src/main/session-lifecycle.ts index cf16a6fa675..d6e3a5cf57c 100644 --- a/apps/desktop/src/main/session-lifecycle.ts +++ b/apps/desktop/src/main/session-lifecycle.ts @@ -264,8 +264,10 @@ export async function tearDownSession( clearHandoffState: () => void | Promise, events: EventRecorder, clearBrowserProfile: () => Promise, - revokeSession: () => Promise + revokeSession: () => Promise, + stopLocalActions?: () => void ): Promise { + stopLocalActions?.() if (!beginAccountDataTeardown('account', origin)) { throw new Error('Could not persist account-data recovery marker.') } @@ -303,6 +305,8 @@ export interface SessionLifecycleDeps { appSession: Session origin: () => string events: EventRecorder + /** Stops live device actions before account persistence or server revocation can block. */ + stopLocalActions: () => void clearHandoffState: () => void | Promise /** Clears the embedded browser's own partition. See {@link tearDownSession}. */ clearBrowserProfile: () => Promise @@ -366,7 +370,8 @@ export function createSessionLifecycleCoordinator( if (win) { await revokeAppSession(win, origin) } - } + }, + deps.stopLocalActions ) .then(() => { for (const win of deps.getWindows()) { diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index 50fe5d3ff25..0ebea00450a 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -26,6 +26,7 @@ import type { BrowserPasswordImportResult, BrowserSiteInfo, BrowserToolbarCommand, + ComputerUseActivity, DesktopAppearanceTheme, DesktopCommand, DesktopLocalFileRequest, @@ -46,6 +47,7 @@ import type { TerminalShortcutCommand, TerminalThemeProfile, } from '@sim/desktop-bridge' +import type { ComputerUseInput } from '@sim/desktop-bridge/computer-use' import { type ScopedTerminalCommandEvent, type ScopedTerminalTabsState, @@ -118,6 +120,27 @@ function shellVersion(): string { */ const api: SimDesktopApi = { version: shellVersion(), + ...(process.platform === 'darwin' + ? { + computerUse: { + getStatus: () => ipcRenderer.invoke('computer-use:status'), + setEnabled: (enabled: boolean) => ipcRenderer.invoke('computer-use:set-enabled', enabled), + requestPermission: (permission: 'accessibility' | 'screenCapture') => + ipcRenderer.invoke('computer-use:request-permission', permission), + listAppPermissions: () => ipcRenderer.invoke('computer-use:list-permissions'), + revokeApp: (bundleId: string) => ipcRenderer.invoke('computer-use:revoke-app', bundleId), + executeTool: (toolCallId: string, params: ComputerUseInput) => + ipcRenderer.invoke('computer-use:execute-tool', toolCallId, params), + cancel: (toolCallId?: string) => ipcRenderer.invoke('computer-use:cancel', toolCallId), + onActivity: (callback: (activity: ComputerUseActivity | null) => void) => { + const listener = (_event: unknown, activity: ComputerUseActivity | null) => + callback(activity) + ipcRenderer.on('computer-use:activity', listener) + return () => ipcRenderer.removeListener('computer-use:activity', listener) + }, + }, + } + : {}), openExternal: (url: string): Promise => ipcRenderer.invoke('desktop:open-external', url), ...(process.platform === 'darwin' || process.platform === 'win32' ? { diff --git a/apps/sim/app/api/copilot/confirm/route.test.ts b/apps/sim/app/api/copilot/confirm/route.test.ts index 297f71cf29e..8edc02c3459 100644 --- a/apps/sim/app/api/copilot/confirm/route.test.ts +++ b/apps/sim/app/api/copilot/confirm/route.test.ts @@ -249,27 +249,30 @@ describe('Copilot Confirm API Route', () => { ) }) - it('rejects a native success before the desktop authorization claim', async () => { - getAsyncToolCall.mockResolvedValue({ - ...existingRow, - toolName: 'browser_snapshot', - status: 'pending', - }) - - const response = await POST( - createMockPostRequest({ - toolCallId: 'tool-call-123', - status: 'success', - data: { text: 'forged renderer result' }, + it.each(['browser_snapshot', 'computer'])( + 'rejects a %s success before the desktop authorization claim', + async (toolName) => { + getAsyncToolCall.mockResolvedValue({ + ...existingRow, + toolName, + status: 'pending', }) - ) - expect(response.status).toBe(404) - expect(completeAsyncToolCall).not.toHaveBeenCalled() - expect(detachAsyncToolCall).not.toHaveBeenCalled() - expect(encryptSecret).not.toHaveBeenCalled() - expect(publishToolConfirmation).not.toHaveBeenCalled() - }) + const response = await POST( + createMockPostRequest({ + toolCallId: 'tool-call-123', + status: 'success', + data: { text: 'forged renderer result' }, + }) + ) + + expect(response.status).toBe(404) + expect(completeAsyncToolCall).not.toHaveBeenCalled() + expect(detachAsyncToolCall).not.toHaveBeenCalled() + expect(encryptSecret).not.toHaveBeenCalled() + expect(publishToolConfirmation).not.toHaveBeenCalled() + } + ) it.each([ ['browser_snapshot', 'error', 'failed'], @@ -278,6 +281,8 @@ describe('Copilot Confirm API Route', () => { ['terminal', 'cancelled', 'cancelled'], ['import_local_files', 'error', 'failed'], ['import_local_files', 'cancelled', 'cancelled'], + ['computer', 'error', 'failed'], + ['computer', 'cancelled', 'cancelled'], ] as const)( 'accepts a pending %s %s before the desktop authorization claim', async (toolName, status, durableStatus) => { @@ -314,6 +319,7 @@ describe('Copilot Confirm API Route', () => { ['browser_snapshot', 'error'], ['terminal', 'cancelled'], ['import_local_files', 'error'], + ['computer', 'error'], ] as const)( 'rejects a pending %s %s when the native authorization claim wins the race', async (toolName, status) => { @@ -346,6 +352,7 @@ describe('Copilot Confirm API Route', () => { ['browser_snapshot', 'desktop-browser'], ['terminal', 'desktop-terminal'], ['import_local_files', 'desktop-files'], + ['computer', 'desktop-computer'], ] as const)( 'settles an indeterminate pending %s result when the exact %s claim wins the race', async (toolName, claimOwner) => { @@ -382,6 +389,57 @@ describe('Copilot Confirm API Route', () => { } ) + it.each(['desktop-browser', null])( + 'rejects a running computer completion owned by %s', + async (claimedBy) => { + getAsyncToolCall.mockResolvedValue({ ...existingRow, toolName: 'computer', claimedBy }) + const response = await POST( + createMockPostRequest({ toolCallId: 'tool-call-123', status: 'success' }) + ) + expect(response.status).toBe(404) + expect(completeAsyncToolCall).not.toHaveBeenCalled() + expect(completeClaimedAsyncToolCall).not.toHaveBeenCalled() + expect(publishToolConfirmation).not.toHaveBeenCalled() + } + ) + + it('completes a computer result only through its exact native claim', async () => { + getAsyncToolCall.mockResolvedValue({ + ...existingRow, + toolName: 'computer', + claimedBy: 'desktop-computer', + }) + const response = await POST( + createMockPostRequest({ toolCallId: 'tool-call-123', status: 'success', data: { ok: true } }) + ) + expect(response.status).toBe(200) + expect(completeClaimedAsyncToolCall).toHaveBeenCalledWith( + { + toolCallId: 'tool-call-123', + status: 'completed', + result: { __sealedClientToolCompletionV1: 'sealed-client-result' }, + error: null, + }, + 'desktop-computer' + ) + expect(completeAsyncToolCall).not.toHaveBeenCalled() + expect(publishToolConfirmation).toHaveBeenCalledOnce() + }) + + it('rejects background computer results without detaching the native claim', async () => { + getAsyncToolCall.mockResolvedValue({ + ...existingRow, + toolName: 'computer', + claimedBy: 'desktop-computer', + }) + const response = await POST( + createMockPostRequest({ toolCallId: 'tool-call-123', status: 'background' }) + ) + expect(response.status).toBe(404) + expect(detachAsyncToolCall).not.toHaveBeenCalled() + expect(publishToolConfirmation).not.toHaveBeenCalled() + }) + it('does not publish when another terminal transition wins indeterminate claim reconciliation', async () => { getAsyncToolCall.mockResolvedValue({ ...existingRow, diff --git a/apps/sim/app/api/copilot/confirm/route.ts b/apps/sim/app/api/copilot/confirm/route.ts index b34ba42a32b..d40e10c10ea 100644 --- a/apps/sim/app/api/copilot/confirm/route.ts +++ b/apps/sim/app/api/copilot/confirm/route.ts @@ -290,11 +290,13 @@ export const POST = withRouteHandler((req: NextRequest) => { const isNativeClientTool = isBrowserToolName(existing.toolName) || isTerminalToolName(existing.toolName) || - existing.toolName === 'import_local_files' + existing.toolName === 'import_local_files' || + existing.toolName === 'computer' const isPreclaimNativeTerminalOutcome = (isCurrentBrowserToolName(existing.toolName) || isTerminalToolName(existing.toolName) || - existing.toolName === 'import_local_files') && + existing.toolName === 'import_local_files' || + existing.toolName === 'computer') && existing.status === ASYNC_TOOL_STATUS.pending && isErrorOrCancelledOutcome const nativeClaimOwner = isCurrentBrowserToolName(existing.toolName) @@ -303,7 +305,9 @@ export const POST = withRouteHandler((req: NextRequest) => { ? DESKTOP_TOOL_CLAIM_OWNER.terminal : existing.toolName === 'import_local_files' ? DESKTOP_TOOL_CLAIM_OWNER.files - : undefined + : existing.toolName === 'computer' + ? DESKTOP_TOOL_CLAIM_OWNER.computer + : undefined const isIndeterminateNativeExit = isPreclaimNativeTerminalOutcome && status === ASYNC_TOOL_CONFIRMATION_STATUS.error && @@ -318,6 +322,16 @@ export const POST = withRouteHandler((req: NextRequest) => { return createNotFoundResponse('Running client tool call not found') } + if ( + existing.toolName === 'computer' && + (status === ASYNC_TOOL_CONFIRMATION_STATUS.background || + (existing.status === ASYNC_TOOL_STATUS.running && + existing.claimedBy !== DESKTOP_TOOL_CLAIM_OWNER.computer)) + ) { + span.setAttribute(TraceAttr.CopilotConfirmOutcome, CopilotConfirmOutcome.ToolCallNotFound) + return createNotFoundResponse('Claimed computer tool call not found') + } + let effectiveStatus = status let executionId = submittedExecutionId let launchError: WorkflowToolLaunchError | undefined @@ -433,7 +447,14 @@ export const POST = withRouteHandler((req: NextRequest) => { ...(isWorkflowTool && executionId ? { executionId } : {}), ...(isPreclaimNativeTerminalOutcome ? { completionGuard: { status: ASYNC_TOOL_STATUS.pending } as const } - : {}), + : existing.toolName === 'computer' + ? { + completionGuard: { + status: ASYNC_TOOL_STATUS.running, + claimedBy: DESKTOP_TOOL_CLAIM_OWNER.computer, + } as const, + } + : {}), } ) diff --git a/apps/sim/app/api/desktop/computer/authorize/route.test.ts b/apps/sim/app/api/desktop/computer/authorize/route.test.ts new file mode 100644 index 00000000000..32736c7fad3 --- /dev/null +++ b/apps/sim/app/api/desktop/computer/authorize/route.test.ts @@ -0,0 +1,80 @@ +/** @vitest-environment node */ +import { authMockFns } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OrchestrationError } from '@/lib/core/orchestration/types' + +const mocks = vi.hoisted(() => ({ execute: vi.fn(), rate: vi.fn() })) +vi.mock('@/lib/computer-use/application/authorize', () => ({ + authorizeComputerUse: { + operation: { + id: 'desktop.computer.execute', + capability: 'copilot.use', + principalKinds: ['session'], + }, + execute: mocks.execute, + }, +})) +vi.mock('@/lib/core/rate-limiter', async (importOriginal) => ({ + ...(await importOriginal()), + enforceUserRateLimit: mocks.rate, +})) + +import { POST } from '@/app/api/desktop/computer/authorize/route' + +const request = (body: unknown) => + new NextRequest('http://localhost/api/desktop/computer/authorize', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + }) +describe('computer authorization route', () => { + beforeEach(() => { + vi.clearAllMocks() + authMockFns.mockGetSession.mockResolvedValue({ + user: { id: 'user-1' }, + session: { id: 'session-1' }, + }) + mocks.rate.mockResolvedValue(null) + mocks.execute.mockResolvedValue({ + toolName: 'computer', + chatId: 'chat-1', + args: { action: 'list_apps' }, + }) + }) + it('authenticates before parsing malformed requests', async () => { + authMockFns.mockGetSession.mockResolvedValueOnce(null) + expect((await POST(request({ args: 'forged' }))).status).toBe(401) + expect(mocks.execute).not.toHaveBeenCalled() + }) + it('accepts only a tool ID, never renderer-provided target or chat authority', async () => { + expect( + (await POST(request({ toolCallId: 'call-1', args: { action: 'click' }, chatId: 'chat-2' }))) + .status + ).toBe(400) + expect(mocks.execute).not.toHaveBeenCalled() + }) + it('passes the authenticated session principal and returns validated canonical arguments', async () => { + const response = await POST(request({ toolCallId: 'call-1' })) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + toolName: 'computer', + chatId: 'chat-1', + args: { action: 'list_apps' }, + }) + expect(mocks.execute).toHaveBeenCalledWith( + expect.objectContaining({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { toolCallId: 'call-1' }, + }) + ) + expect(mocks.rate).toHaveBeenCalledWith('desktop-computer-use', 'user-1', undefined) + }) + it.each([ + ['not_found', 404], + ['forbidden', 403], + ] as const)('projects %s without leaking protected state', async (code, status) => { + mocks.execute.mockRejectedValueOnce(new OrchestrationError(code, 'Computer action unavailable')) + expect((await POST(request({ toolCallId: 'call-1' }))).status).toBe(status) + }) +}) diff --git a/apps/sim/app/api/desktop/computer/authorize/route.ts b/apps/sim/app/api/desktop/computer/authorize/route.ts new file mode 100644 index 00000000000..1ad068f30fd --- /dev/null +++ b/apps/sim/app/api/desktop/computer/authorize/route.ts @@ -0,0 +1,19 @@ +import { authorizeComputerUseContract } from '@/lib/api/contracts/computer-use' +import { + defineInternalJsonRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { authorizeComputerUse } from '@/lib/computer-use/application/authorize' + +export const POST = defineInternalJsonRoute({ + contract: authorizeComputerUseContract, + auth: internalSessionAuth, + operation: authorizeComputerUse.operation, + rateLimit: internalRateLimits.user({ bucketName: 'desktop-computer-use' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ body }) => body, + useCase: authorizeComputerUse, + present: (result) => result, +}) diff --git a/apps/sim/app/api/desktop/computer/availability/route.ts b/apps/sim/app/api/desktop/computer/availability/route.ts new file mode 100644 index 00000000000..a6a2310afff --- /dev/null +++ b/apps/sim/app/api/desktop/computer/availability/route.ts @@ -0,0 +1,21 @@ +import { computerUseAvailabilityContract } from '@/lib/api/contracts/computer-use' +import { + defineInternalJsonRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { readComputerUseAvailability } from '@/lib/computer-use/application/availability' + +export const GET = defineInternalJsonRoute({ + contract: computerUseAvailabilityContract, + auth: internalSessionAuth, + operation: readComputerUseAvailability.operation, + rateLimit: internalRateLimits.none({ + reason: 'Read-only rollout switch used on each desktop chat admission.', + }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: () => undefined, + useCase: readComputerUseAvailability, + present: (result) => result, +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index aa6cf08303b..4b2970c9e28 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -14,6 +14,7 @@ import { Chip, cn, Tooltip, toast } from '@sim/emcn' import { Paperclip, Plus, Slash } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { useParams } from 'next/navigation' +import { ComputerUseActivity } from '@/components/computer-use/activity' import { getMothershipAttachmentPreviewUrl } from '@/lib/mothership/chat/attachment-preview' import { MOTHERSHIP_ADD_CONTEXT_EVENT } from '@/lib/mothership/events' import { SIM_RESOURCE_DRAG_TYPE, SIM_RESOURCES_DRAG_TYPE } from '@/lib/mothership/resource-types' @@ -566,6 +567,7 @@ const UserInputImpl = forwardRef(function UserI onDragOver={handleContainerDragOver} onDrop={handleContainerDrop} > + { + it('routes a complete native computer call and never executes a partial or settled call', () => { + const deps = makeStreamLoopDeps() + const ctx = createStreamLoopContext(deps) + const args = { action: 'list_apps' } + dispatchStreamEvent( + ctx, + toolEnv({ + phase: 'call', + executor: 'client', + mode: 'async', + toolCallId: 'computer', + toolName: 'computer', + arguments: args, + partial: true, + }) + ) + expect(deps.startClientComputerTool).not.toHaveBeenCalled() + dispatchStreamEvent( + ctx, + toolEnv({ + phase: 'call', + executor: 'client', + mode: 'async', + toolCallId: 'computer', + toolName: 'computer', + arguments: args, + }) + ) + expect(deps.startClientComputerTool).toHaveBeenCalledWith('computer', args, '') + dispatchStreamEvent(ctx, toolResult('computer', true, 'computer')) + vi.mocked(deps.startClientComputerTool).mockClear() + dispatchStreamEvent( + ctx, + toolEnv({ + phase: 'call', + executor: 'client', + mode: 'async', + toolCallId: 'computer', + toolName: 'computer', + arguments: args, + }) + ) + expect(deps.startClientComputerTool).not.toHaveBeenCalled() + }) + + it('redelivers an unsettled computer call through the replay-safe native executor after reconnect', () => { + const deps = makeStreamLoopDeps() + const ctx = createStreamLoopContext(deps) + dispatchStreamEvent( + ctx, + toStreamBatchEvent( + toolEnv({ + phase: 'call', + executor: 'client', + mode: 'async', + toolCallId: 'computer-recovery', + toolName: 'computer', + arguments: { action: 'list_apps' }, + }) + ).event + ) + expect(deps.startClientComputerTool).toHaveBeenCalledOnce() + }) + it.each([false, true])( 'refreshes credential lists and selectors after Slack connection (replay=%s)', (replay) => { diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.ts index 0fb9877fa43..6375f2327a1 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-tool-event.ts @@ -210,6 +210,16 @@ export function handleToolEvent(ctx: StreamLoopContext, parsed: ToolEvent): void deps.startClientLocalFilesystemTool(rawId, name, localFilesystemArgs ?? {}) } } + if ( + name === 'computer' && + !isPartial && + !deps.options.suppressedWorkflowToolStartIds?.has(rawId) && + node?.kind === 'tool' && + node.status === 'running' && + !node.result + ) { + deps.startClientComputerTool(rawId, payload.arguments ?? {}, parsed.ts) + } if (isCurrentBrowserToolName(name) && !isPartial) { const shouldStartBrowserTool = !deps.options.suppressedWorkflowToolStartIds?.has(rawId) && diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts index 2fbc786cbe5..7f621aabe79 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-context.ts @@ -113,6 +113,7 @@ export interface StreamLoopDeps { ) => void startClientWorkflowTool: (id: string, name: string, args: Record) => void startClientLocalFilesystemTool: (id: string, name: string, args: Record) => void + startClientComputerTool: (id: string, args: Record, ts?: string) => void startClientBrowserTool: ( id: string, name: string, diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-test-helpers.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-test-helpers.ts index 0a8e7351333..c79edad20d4 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-test-helpers.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/stream-test-helpers.ts @@ -42,6 +42,7 @@ export function makeStreamLoopDeps(overrides: Partial = {}): Str removeResource: vi.fn(), startClientWorkflowTool: vi.fn(), startClientLocalFilesystemTool: vi.fn(), + startClientComputerTool: vi.fn(), startClientBrowserTool: vi.fn(), startClientTerminalTool: vi.fn(), startBrowserAgentRun: vi.fn(), diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts index 7bad75a9d2e..ff4b362e03d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts @@ -75,6 +75,7 @@ import { sanitizeChatResources, } from '@/lib/mothership/resources/types' import { executeBrowserToolOnClient } from '@/lib/mothership/tools/client/browser-tool-execution' +import { executeComputerToolOnClient } from '@/lib/mothership/tools/client/computer-tool-execution' import { bindRunToolToExecution, executeRunToolOnClient, @@ -2173,6 +2174,9 @@ export function useChat( removeResource, startClientWorkflowTool, startClientLocalFilesystemTool, + startClientComputerTool: (toolCallId, args, eventTs) => { + void executeComputerToolOnClient(toolCallId, args, eventTs, streamAbortSignal) + }, startClientBrowserTool: startClientBrowserToolForStream, startClientTerminalTool: startClientTerminalToolForStream, startBrowserAgentRun: startBrowserAgentRunForStream, diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx new file mode 100644 index 00000000000..b4c7a4e26dc --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/computer-use.tsx @@ -0,0 +1,114 @@ +'use client' + +import { useEffect, useState } from 'react' +import type { ComputerUseAppPermission } from '@sim/desktop-bridge' +import { Chip, ChipSwitch, Label, toast } from '@sim/emcn' +import { ComputerUseActivity } from '@/components/computer-use/activity' +import { getDesktopBridge } from '@/lib/desktop' +import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' +import { useComputerUseAvailability } from '@/hooks/queries/computer-use' +import { useComputerUseStatus } from '@/hooks/use-computer-use-status' + +export function ComputerUseSettings() { + const bridge = getDesktopBridge()?.computerUse + const availability = useComputerUseAvailability(Boolean(bridge)) + const { status, setStatus, refresh, error } = useComputerUseStatus() + const [apps, setApps] = useState([]) + const [pending, setPending] = useState(false) + useEffect(() => { + if (!bridge || !availability.data?.enabled) return + void bridge + .listAppPermissions() + .then(setApps) + .catch(() => toast.error('Could not load approved apps')) + }, [bridge, availability.data?.enabled, status?.activeAction]) + if (!bridge || !availability.data?.enabled || status?.supported === false) return null + const update = async (action: () => Promise) => { + setPending(true) + try { + await action() + } catch { + toast.error('Could not update Computer Use settings') + } finally { + setPending(false) + } + } + return ( + +
+ +
+ + + void update(async () => setStatus(await bridge.setEnabled(value === 'on'))) + } + /> +
+

+ Off by default. Each app requires your approval. You can stop an action from the + conversation or revoke an app below. +

+ {error && ( +
+ Could not connect to the computer helper. + void refresh()}>Retry +
+ )} + {(['accessibility', 'screenCapture'] as const).map((permission) => ( +
+ + {status?.permissions[permission] ? ( + Allowed + ) : ( + + void update(async () => setStatus(await bridge.requestPermission(permission))) + } + > + Open System Settings + + )} +
+ ))} +

+ Accessibility allows interaction with approved apps. Screen Recording allows screenshots. + Return here after granting access to refresh the status. +

+ {apps.length > 0 && ( +
+ + {apps.map((app) => ( +
+ + {app.displayName} + + + void update(async () => { + await bridge.revokeApp(app.bundleId) + setApps(await bridge.listAppPermissions()) + }) + } + > + Revoke + +
+ ))} +
+ )} +
+ + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx index 20d72828869..bc195b1be07 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/desktop/desktop.tsx @@ -5,6 +5,7 @@ import type { DesktopPreferenceKey, DesktopPreferences } from '@sim/desktop-brid import { Label, Switch, toast } from '@sim/emcn' import { useParams, useRouter } from 'next/navigation' import { getDesktopBridge, getDesktopShellVersion } from '@/lib/desktop' +import { ComputerUseSettings } from '@/app/workspace/[workspaceId]/settings/components/desktop/computer-use' import { SettingsPanel } from '@/app/workspace/[workspaceId]/settings/components/settings-panel' import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' import { useDesktopUpdateState } from '@/hooks/use-desktop-update-state' @@ -69,6 +70,7 @@ export function Desktop() { return ( +
{shellVersion && ( diff --git a/apps/sim/components/computer-use/activity.tsx b/apps/sim/components/computer-use/activity.tsx new file mode 100644 index 00000000000..3fd3924f273 --- /dev/null +++ b/apps/sim/components/computer-use/activity.tsx @@ -0,0 +1,57 @@ +'use client' + +import { Chip, toast } from '@sim/emcn' +import { getDesktopBridge } from '@/lib/desktop' +import { useComputerUseStatus } from '@/hooks/use-computer-use-status' + +const ACTION_LABELS: Record = { + status: 'Checking permissions', + list_apps: 'Finding apps', + get_app_state: 'Reading', + activate_app: 'Bringing app forward', + click: 'Clicking', + type_text: 'Typing', + press_key: 'Pressing keys', + scroll: 'Scrolling', + drag: 'Dragging', + set_value: 'Editing', + perform_action: 'Interacting', +} + +/** Remains visible while a native action is active, even if rollout or device access is revoked. */ +export function ComputerUseActivity() { + const { status } = useComputerUseStatus() + const activity = status?.activeAction + if (!activity) return null + return ( +
+ + Computer Use · {activity.appName ?? 'Mac app'} ·{' '} + {ACTION_LABELS[activity.action] ?? 'Working'} + + +
+ ) +} diff --git a/apps/sim/hooks/queries/computer-use.ts b/apps/sim/hooks/queries/computer-use.ts new file mode 100644 index 00000000000..66ef4b5ecce --- /dev/null +++ b/apps/sim/hooks/queries/computer-use.ts @@ -0,0 +1,18 @@ +import { useQuery } from '@tanstack/react-query' +import { requestJson } from '@/lib/api/client' +import { computerUseAvailabilityContract } from '@/lib/api/contracts/computer-use' + +export const computerUseKeys = { + all: ['computer-use'] as const, + availability: () => [...computerUseKeys.all, 'availability'] as const, +} + +/** Resolve the rollout on the server; devices never infer feature access from their own preference. */ +export function useComputerUseAvailability(enabled: boolean) { + return useQuery({ + queryKey: computerUseKeys.availability(), + queryFn: ({ signal }) => requestJson(computerUseAvailabilityContract, { signal }), + enabled, + staleTime: 0, + }) +} diff --git a/apps/sim/hooks/use-computer-use-status.ts b/apps/sim/hooks/use-computer-use-status.ts new file mode 100644 index 00000000000..223ca4be316 --- /dev/null +++ b/apps/sim/hooks/use-computer-use-status.ts @@ -0,0 +1,37 @@ +import { useCallback, useEffect, useState } from 'react' +import type { ComputerUseStatus } from '@sim/desktop-bridge' +import { getDesktopBridge } from '@/lib/desktop' + +/** Refresh native permissions when the user returns from System Settings and on activity changes. */ +export function useComputerUseStatus() { + const [status, setStatus] = useState(null) + const [error, setError] = useState(false) + const refresh = useCallback(async () => { + const bridge = getDesktopBridge()?.computerUse + if (!bridge) return + try { + setStatus(await bridge.getStatus()) + setError(false) + } catch { + setError(true) + } + }, []) + useEffect(() => { + const bridge = getDesktopBridge()?.computerUse + if (!bridge) return + void refresh() + const unsubscribe = bridge.onActivity((activeAction) => { + setStatus((current) => (current ? { ...current, activeAction } : current)) + void refresh() + }) + const onFocus = () => { + void refresh() + } + window.addEventListener('focus', onFocus) + return () => { + unsubscribe() + window.removeEventListener('focus', onFocus) + } + }, [refresh]) + return { status, setStatus, refresh, error } +} diff --git a/apps/sim/lib/api/contracts/computer-use.ts b/apps/sim/lib/api/contracts/computer-use.ts new file mode 100644 index 00000000000..e88cb2ed890 --- /dev/null +++ b/apps/sim/lib/api/contracts/computer-use.ts @@ -0,0 +1,31 @@ +import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use' +import { z } from 'zod' +import { desktopToolCallIdSchema } from '@/lib/api/contracts/desktop-tool-authorization' +import { defineRouteContract } from '@/lib/api/contracts/types' + +export const computerUseAvailabilityResponseSchema = z.object({ enabled: z.boolean() }).strict() +export type ComputerUseAvailabilityResponse = z.output +export const computerUseAvailabilityContract = defineRouteContract({ + method: 'GET', + path: '/api/desktop/computer/availability', + response: { mode: 'json', schema: computerUseAvailabilityResponseSchema }, +}) + +export const authorizeComputerUseBodySchema = z + .object({ toolCallId: desktopToolCallIdSchema }) + .strict() +export type AuthorizeComputerUseBody = z.input +export const authorizeComputerUseResponseSchema = z + .object({ + toolName: z.literal('computer'), + args: ComputerUseSchema, + chatId: z.string().min(1), + }) + .strict() +export type AuthorizeComputerUseResponse = z.output +export const authorizeComputerUseContract = defineRouteContract({ + method: 'POST', + path: '/api/desktop/computer/authorize', + body: authorizeComputerUseBodySchema, + response: { mode: 'json', schema: authorizeComputerUseResponseSchema }, +}) diff --git a/apps/sim/lib/computer-use/application/authorize.test.ts b/apps/sim/lib/computer-use/application/authorize.test.ts new file mode 100644 index 00000000000..728ba3261f9 --- /dev/null +++ b/apps/sim/lib/computer-use/application/authorize.test.ts @@ -0,0 +1,145 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + getTool: vi.fn(), + getRun: vi.fn(), + ownedChat: vi.fn(), + permission: vi.fn(), + available: vi.fn(), + claim: vi.fn(), + organization: vi.fn(), +})) +vi.mock('@/lib/mothership/async-runs/repository', () => ({ + getAsyncToolCall: mocks.getTool, + getRunSegment: mocks.getRun, +})) +vi.mock('@/lib/mothership/chat/application/context', () => ({ + resolveOwnedChatContext: mocks.ownedChat, +})) +vi.mock('@/lib/computer-use/availability.server', () => ({ + isComputerUseAvailable: mocks.available, +})) +vi.mock('@/lib/computer-use/repository', () => ({ claimComputerUseTool: mocks.claim })) +vi.mock('@sim/platform-authz/workspace', async (importOriginal) => ({ + ...(await importOriginal()), + resolveEffectiveWorkspacePermission: mocks.permission, +})) +vi.mock('@/lib/core/application/organization-authorization', () => ({ + authorizeOrganizationOperation: mocks.organization, +})) + +import { authorizeComputerUse } from '@/lib/computer-use/application/authorize' + +const principal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } as const +const input = { toolCallId: 'call-1' } +const run = { + id: 'run-1', + userId: 'user-1', + chatId: 'chat-1', + workspaceId: 'ws-1', + organizationId: null, + status: 'active', + toolAdmissionClosedAt: null, +} +const context = { + chatId: 'chat-1', + userId: 'user-1', + workspaceId: 'ws-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'user-1', + mode: 'agent', +} + +describe('native computer authorization', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.getTool.mockResolvedValue({ toolName: 'computer', status: 'pending', runId: 'run-1' }) + mocks.getRun.mockResolvedValue(run) + mocks.ownedChat.mockResolvedValue(context) + mocks.permission.mockResolvedValue('read') + mocks.available.mockResolvedValue(true) + mocks.claim.mockResolvedValue({ args: { action: 'list_apps' } }) + }) + it('returns canonical arguments only after current chat access and an atomic claim', async () => { + await expect(authorizeComputerUse.execute({ principal, input })).resolves.toEqual({ + toolName: 'computer', + args: { action: 'list_apps' }, + chatId: 'chat-1', + }) + expect(mocks.claim).toHaveBeenCalledWith({ + ...input, + userId: 'user-1', + runId: 'run-1', + chatId: 'chat-1', + }) + }) + it('rejects non-session principals before protected loading', async () => { + await expect( + authorizeComputerUse.execute({ + principal: { kind: 'personal_api_key', userId: 'user-1', keyId: 'key-1' }, + input, + }) + ).rejects.toThrow() + expect(mocks.getTool).not.toHaveBeenCalled() + }) + it('rejects disabled rollout without claiming an action', async () => { + mocks.available.mockResolvedValue(false) + await expect(authorizeComputerUse.execute({ principal, input })).rejects.toThrow('unavailable') + expect(mocks.claim).not.toHaveBeenCalled() + }) + it('rechecks current workspace membership', async () => { + mocks.permission.mockResolvedValue(null) + await expect(authorizeComputerUse.execute({ principal, input })).rejects.toThrow() + expect(mocks.claim).not.toHaveBeenCalled() + }) + it.each([ + { userId: 'another-user' }, + { status: 'cancelled' }, + { toolAdmissionClosedAt: new Date() }, + ])('rejects a foreign or stopped run %j', async (change) => { + mocks.getRun.mockResolvedValue({ ...run, ...change }) + await expect(authorizeComputerUse.execute({ principal, input })).rejects.toThrow('not found') + expect(mocks.claim).not.toHaveBeenCalled() + }) + it('rejects canonical chat scope mismatch', async () => { + mocks.ownedChat.mockResolvedValue({ ...context, workspaceId: 'another-workspace' }) + await expect(authorizeComputerUse.execute({ principal, input })).rejects.toThrow( + 'does not belong' + ) + expect(mocks.claim).not.toHaveBeenCalled() + }) + it('rejects a replay or Stop that wins the final claim race', async () => { + mocks.claim.mockResolvedValue(null) + await expect(authorizeComputerUse.execute({ principal, input })).rejects.toThrow( + 'may already have started' + ) + }) + it('uses the organization policy for an organization-owned private chat', async () => { + mocks.getRun.mockResolvedValue({ ...run, workspaceId: null, organizationId: 'org-1' }) + mocks.ownedChat.mockResolvedValue({ + chatId: 'chat-1', + userId: 'user-1', + organizationId: 'org-1', + mode: 'agent', + }) + await authorizeComputerUse.execute({ principal, input }) + expect(mocks.organization).toHaveBeenCalledWith( + principal, + expect.objectContaining({ + id: 'desktop.computer.execute', + minimumRole: 'member', + principalKinds: ['session'], + }), + { organizationId: 'org-1' } + ) + }) + it('propagates infrastructure failures without disguising them as access refusals', async () => { + mocks.getTool.mockRejectedValue(new Error('database unavailable')) + await expect(authorizeComputerUse.execute({ principal, input })).rejects.toThrow( + 'database unavailable' + ) + expect(mocks.claim).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/computer-use/application/authorize.ts b/apps/sim/lib/computer-use/application/authorize.ts new file mode 100644 index 00000000000..74706fd6987 --- /dev/null +++ b/apps/sim/lib/computer-use/application/authorize.ts @@ -0,0 +1,82 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use' +import { isComputerUseAvailable } from '@/lib/computer-use/availability.server' +import { claimComputerUseTool } from '@/lib/computer-use/repository' +import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' +import { defineWorkspaceOperation } from '@/lib/core/application/workspace-operation' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { getAsyncToolCall, getRunSegment } from '@/lib/mothership/async-runs/repository' +import { defineAuthorizedChatUseCase } from '@/lib/mothership/chat/application/authorized-chat-use-case' +import { resolveOwnedChatContext } from '@/lib/mothership/chat/application/context' + +interface AuthorizeComputerUseInput { + toolCallId: string +} + +/** Authorizes and consumes one server-authored native action under the current private-chat policy. */ +export const authorizeComputerUse = defineAuthorizedChatUseCase({ + operation: defineWorkspaceOperation({ + id: 'desktop.computer.execute', + minimumRole: 'read', + workspaceApiKey: 'deny', + capability: 'copilot.use', + principalKinds: ['session'], + }), + organizationOperation: defineOrganizationOperation({ + id: 'desktop.computer.execute', + minimumRole: 'member', + capability: 'copilot.use', + principalKinds: ['session'], + }), + async resolveContext({ + principal, + input, + }: { + principal: SessionPrincipal + input: AuthorizeComputerUseInput + }) { + const tool = await getAsyncToolCall(input.toolCallId) + if (!tool || tool.toolName !== 'computer' || tool.status !== 'pending') + throw new OrchestrationError('not_found', 'Pending computer action not found') + const run = await getRunSegment(tool.runId) + if ( + !run || + run.userId !== principal.userId || + run.toolAdmissionClosedAt || + ['complete', 'error', 'cancelled'].includes(run.status) + ) + throw new OrchestrationError('not_found', 'Pending computer action not found') + const context = await resolveOwnedChatContext(principal, run.chatId) + if ( + context.mode === 'assistant' || + context.workspaceId !== (run.workspaceId ?? undefined) || + context.organizationId !== (run.organizationId ?? undefined) + ) + throw new OrchestrationError( + 'not_found', + 'Computer action does not belong to this conversation' + ) + return { ...context, runId: run.id } + }, + authorizationOptions: {}, + async execute({ principal, context, input }) { + if (!(await isComputerUseAvailable())) + throw new OrchestrationError('forbidden', 'Computer use is unavailable on this deployment') + const claimed = await claimComputerUseTool({ + toolCallId: input.toolCallId, + runId: context.runId, + chatId: context.chatId, + userId: principal.userId, + }) + if (!claimed) + throw new OrchestrationError( + 'not_found', + 'Pending computer action not found; it may already have started' + ) + return { + toolName: 'computer' as const, + args: ComputerUseSchema.parse(claimed.args), + chatId: context.chatId, + } + }, +}) diff --git a/apps/sim/lib/computer-use/application/availability.ts b/apps/sim/lib/computer-use/application/availability.ts new file mode 100644 index 00000000000..3ccca27a8c2 --- /dev/null +++ b/apps/sim/lib/computer-use/application/availability.ts @@ -0,0 +1,26 @@ +import { isComputerUseAvailable } from '@/lib/computer-use/availability.server' +import { + assertOperationPrincipal, + defineOperation, + type OperationUseCase, +} from '@/lib/core/application/operation' + +const availabilityOperation = defineOperation({ + id: 'desktop.computer.availability', + /** permission-group-exempt: reports only the deployment-wide rollout switch; actions require chat authorization. */ + capability: 'none', + principalKinds: ['session'], +}) + +/** The authenticated settings and capability read; no device or app data leaves the desktop. */ +export const readComputerUseAvailability: OperationUseCase< + typeof availabilityOperation, + undefined, + { enabled: boolean } +> = { + operation: availabilityOperation, + async execute({ principal }) { + assertOperationPrincipal(principal, availabilityOperation) + return { enabled: await isComputerUseAvailable() } + }, +} diff --git a/apps/sim/lib/computer-use/availability.server.ts b/apps/sim/lib/computer-use/availability.server.ts new file mode 100644 index 00000000000..4a2d5d10dd3 --- /dev/null +++ b/apps/sim/lib/computer-use/availability.server.ts @@ -0,0 +1,6 @@ +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' + +/** Re-evaluates the global rollout at every server admission boundary. */ +export async function isComputerUseAvailable(): Promise { + return isFeatureEnabled('mothership-computer-use') +} diff --git a/apps/sim/lib/computer-use/repository.postgres.test.ts b/apps/sim/lib/computer-use/repository.postgres.test.ts new file mode 100644 index 00000000000..db04c1fb57f --- /dev/null +++ b/apps/sim/lib/computer-use/repository.postgres.test.ts @@ -0,0 +1,105 @@ +/** @vitest-environment node */ +import { generateShortId } from '@sim/utils/id' +import { drizzle, type PostgresJsDatabase } from 'drizzle-orm/postgres-js' +import postgres from 'postgres' +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { database } = vi.hoisted(() => ({ + database: { current: undefined as PostgresJsDatabase | undefined }, +})) +vi.unmock('@sim/db/schema') +vi.unmock('drizzle-orm') +vi.mock('@sim/db', () => ({ + db: { + transaction: (...args: unknown[]) => { + if (!database.current) throw new Error('Computer use test database is not initialized') + return Reflect.apply(database.current.transaction, database.current, args) + }, + }, +})) + +import { claimComputerUseTool } from '@/lib/computer-use/repository' + +const databaseUrl = process.env.COMPUTER_USE_TEST_DATABASE_URL +if (databaseUrl && !['localhost', '127.0.0.1', '[::1]'].includes(new URL(databaseUrl).hostname)) + throw new Error('Computer use tests require an isolated local PostgreSQL schema') +const schema = `computer_use_${generateShortId() + .replace(/[^a-zA-Z0-9]/g, '') + .toLowerCase()}` +const connection = databaseUrl + ? postgres(databaseUrl, { max: 6, connection: { search_path: schema } }) + : undefined +const runId = '11111111-1111-4111-8111-111111111111' +const chatId = '22222222-2222-4222-8222-222222222222' +const input = { toolCallId: 'computer-1', runId, chatId, userId: 'user-1' } + +afterAll(async () => { + if (!connection) return + await connection.unsafe(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`) + await connection.end() +}) + +describe.skipIf(!connection)('computer use one-shot admission in PostgreSQL', () => { + beforeAll(async () => { + if (!connection) throw new Error('Test database missing') + await connection.unsafe(`CREATE SCHEMA "${schema}"`) + await connection.unsafe( + `CREATE TABLE copilot_chats (id uuid PRIMARY KEY, user_id text NOT NULL, deleted_at timestamp)` + ) + await connection.unsafe( + `CREATE TABLE copilot_runs (id uuid PRIMARY KEY, chat_id uuid NOT NULL, user_id text NOT NULL, status text NOT NULL, tool_admission_closed_at timestamp)` + ) + await connection.unsafe( + `CREATE TABLE copilot_async_tool_calls (tool_call_id text PRIMARY KEY, run_id uuid NOT NULL, tool_name text NOT NULL, args jsonb NOT NULL, status text NOT NULL, claimed_by text, claimed_at timestamp, created_at timestamp DEFAULT now(), updated_at timestamp DEFAULT now())` + ) + database.current = drizzle(connection) + }) + beforeEach(async () => { + if (!connection) throw new Error('Test database missing') + await connection`TRUNCATE copilot_chats, copilot_runs, copilot_async_tool_calls` + await connection`INSERT INTO copilot_chats (id,user_id) VALUES (${chatId},'user-1')` + await connection`INSERT INTO copilot_runs (id,chat_id,user_id,status) VALUES (${runId},${chatId},'user-1','active')` + await connection`INSERT INTO copilot_async_tool_calls (tool_call_id,run_id,tool_name,args,status) VALUES ('computer-1',${runId},'computer','{"action":"list_apps","activity":{"title":"Inspecting apps"}}','pending')` + }) + it('admits one of twelve concurrent executions and returns only canonical business arguments', async () => { + const results = await Promise.all(Array.from({ length: 12 }, () => claimComputerUseTool(input))) + expect(results.filter(Boolean)).toEqual([{ args: { action: 'list_apps' } }]) + expect(await claimComputerUseTool(input)).toBeNull() + const [row] = await connection!`SELECT status, claimed_by FROM copilot_async_tool_calls` + expect(row).toEqual({ status: 'running', claimed_by: 'desktop-computer' }) + }) + it.each(['complete', 'error', 'cancelled'])('refuses a %s run', async (status) => { + await connection!`UPDATE copilot_runs SET status = ${status}` + expect(await claimComputerUseTool(input)).toBeNull() + }) + it('refuses a closed admission even while the run is still active', async () => { + await connection!`UPDATE copilot_runs SET tool_admission_closed_at = now()` + expect(await claimComputerUseTool(input)).toBeNull() + }) + it('refuses old calls and leaves them unclaimed', async () => { + await connection!`UPDATE copilot_async_tool_calls SET created_at = now() - interval '3 minutes'` + expect(await claimComputerUseTool(input)).toBeNull() + const [row] = await connection!`SELECT status FROM copilot_async_tool_calls` + expect(row.status).toBe('pending') + }) + it('binds run, chat and human owner independently', async () => { + expect(await claimComputerUseTool({ ...input, userId: 'user-2' })).toBeNull() + expect(await claimComputerUseTool({ ...input, chatId: runId })).toBeNull() + expect(await claimComputerUseTool({ ...input, runId: chatId })).toBeNull() + await connection!`UPDATE copilot_chats SET user_id = 'user-2'` + expect(await claimComputerUseTool(input)).toBeNull() + }) + it('rejects malformed canonical targets before consuming a claim', async () => { + await connection!`UPDATE copilot_async_tool_calls SET args = '{"action":"click","bundleId":"com.apple.Notes"}'` + expect(await claimComputerUseTool(input)).toBeNull() + const [row] = await connection!`SELECT status, claimed_by FROM copilot_async_tool_calls` + expect(row).toEqual({ status: 'pending', claimed_by: null }) + }) + it('rejects archived chats and non-computer tool names', async () => { + await connection!`UPDATE copilot_chats SET deleted_at = now()` + expect(await claimComputerUseTool(input)).toBeNull() + await connection!`UPDATE copilot_chats SET deleted_at = NULL` + await connection!`UPDATE copilot_async_tool_calls SET tool_name = 'terminal'` + expect(await claimComputerUseTool(input)).toBeNull() + }) +}) diff --git a/apps/sim/lib/computer-use/repository.ts b/apps/sim/lib/computer-use/repository.ts new file mode 100644 index 00000000000..94b8c1e5966 --- /dev/null +++ b/apps/sim/lib/computer-use/repository.ts @@ -0,0 +1,66 @@ +import { db } from '@sim/db' +import { copilotAsyncToolCalls, copilotChats, copilotRuns } from '@sim/db/schema' +import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use' +import { omit, toRecord } from '@sim/utils/object' +import { and, eq, isNull, notInArray, sql } from 'drizzle-orm' +import { DESKTOP_TOOL_CLAIM_OWNER } from '@/lib/mothership/async-runs/lifecycle' + +interface ComputerUseClaim { + toolCallId: string + runId: string + chatId: string + userId: string +} + +/** Locks the run with its action so Stop and native admission have a single ordering point. */ +export async function claimComputerUseTool(input: ComputerUseClaim) { + return db.transaction(async (tx) => { + const [pending] = await tx + .select({ args: copilotAsyncToolCalls.args }) + .from(copilotRuns) + .innerJoin(copilotChats, eq(copilotChats.id, copilotRuns.chatId)) + .innerJoin(copilotAsyncToolCalls, eq(copilotAsyncToolCalls.runId, copilotRuns.id)) + .where( + and( + eq(copilotRuns.id, input.runId), + eq(copilotRuns.chatId, input.chatId), + eq(copilotRuns.userId, input.userId), + notInArray(copilotRuns.status, ['complete', 'error', 'cancelled']), + isNull(copilotRuns.toolAdmissionClosedAt), + eq(copilotChats.userId, input.userId), + isNull(copilotChats.deletedAt), + eq(copilotAsyncToolCalls.toolCallId, input.toolCallId), + eq(copilotAsyncToolCalls.toolName, 'computer'), + eq(copilotAsyncToolCalls.status, 'pending'), + isNull(copilotAsyncToolCalls.claimedBy), + sql`${copilotAsyncToolCalls.createdAt} > now() - interval '2 minutes'` + ) + ) + .for('update', { of: copilotRuns }) + .limit(1) + if ( + !pending || + !ComputerUseSchema.safeParse(omit(toRecord(pending.args), ['activity'])).success + ) + return null + const now = new Date() + const [claimed] = await tx + .update(copilotAsyncToolCalls) + .set({ + status: 'running', + claimedBy: DESKTOP_TOOL_CLAIM_OWNER.computer, + claimedAt: now, + updatedAt: now, + }) + .where( + and( + eq(copilotAsyncToolCalls.toolCallId, input.toolCallId), + eq(copilotAsyncToolCalls.status, 'pending') + ) + ) + .returning({ args: copilotAsyncToolCalls.args }) + return claimed + ? { args: ComputerUseSchema.parse(omit(toRecord(claimed.args), ['activity'])) } + : null + }) +} diff --git a/apps/sim/lib/computer-use/transport.test.ts b/apps/sim/lib/computer-use/transport.test.ts new file mode 100644 index 00000000000..5e8730d807c --- /dev/null +++ b/apps/sim/lib/computer-use/transport.test.ts @@ -0,0 +1,53 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ status: vi.fn(), execute: vi.fn() })) +vi.mock('@/lib/desktop', () => ({ + getDesktopBridge: () => ({ + computerUse: { getStatus: mocks.status, executeTool: mocks.execute }, + }), +})) + +import { executeComputerUseTool } from '@/lib/computer-use/transport' + +describe('computer transport', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.status.mockResolvedValue({ supported: true, enabled: true }) + mocks.execute.mockResolvedValue({ kind: 'apps', apps: [] }) + }) + it('does not dispatch when Stop happens during the asynchronous device status lookup', async () => { + const controller = new AbortController() + let resolveStatus: ((value: { supported: boolean; enabled: boolean }) => void) | undefined + mocks.status.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveStatus = resolve + }) + ) + const result = executeComputerUseTool('call', { action: 'list_apps' }, controller.signal) + controller.abort() + resolveStatus?.({ supported: true, enabled: true }) + await expect(result).rejects.toThrow() + expect(mocks.execute).not.toHaveBeenCalled() + }) + it.each([ + { supported: false, enabled: true }, + { supported: true, enabled: false }, + ])('refuses unavailable device state %j', async (status) => { + mocks.status.mockResolvedValue(status) + await expect(executeComputerUseTool('call', { action: 'list_apps' })).rejects.toThrow( + 'Enable Computer Use' + ) + expect(mocks.execute).not.toHaveBeenCalled() + }) + it('validates native results at the renderer boundary', async () => { + mocks.execute.mockResolvedValue({ + kind: 'action', + action: 'click', + bundleId: 'com.apple.Notes', + dispatched: true, + }) + await expect(executeComputerUseTool('call', { action: 'list_apps' })).rejects.toThrow() + }) +}) diff --git a/apps/sim/lib/computer-use/transport.ts b/apps/sim/lib/computer-use/transport.ts new file mode 100644 index 00000000000..9dda4f47344 --- /dev/null +++ b/apps/sim/lib/computer-use/transport.ts @@ -0,0 +1,23 @@ +import type { ComputerUseInput, ComputerUseResult } from '@sim/desktop-bridge/computer-use' +import { ComputerUseResultSchema } from '@sim/desktop-bridge/computer-use' +import { getDesktopBridge } from '@/lib/desktop' + +/** Desktop main independently authorizes the tool ID and executes only the server's canonical args. */ +export async function executeComputerUseTool( + toolCallId: string, + input: ComputerUseInput, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const bridge = getDesktopBridge()?.computerUse + if (!bridge) throw new Error('Computer use requires the macOS desktop app') + const status = await bridge.getStatus() + signal?.throwIfAborted() + if (!status.supported || !status.enabled) + throw new Error('Enable Computer Use in Desktop settings first') + return ComputerUseResultSchema.parse(await bridge.executeTool(toolCallId, input)) +} + +export async function cancelComputerUseTool(toolCallId?: string): Promise { + await getDesktopBridge()?.computerUse?.cancel(toolCallId) +} diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 933f829cccd..15c426aaf60 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -592,6 +592,7 @@ export const env = createEnv({ AGENTMAIL_API_KEY: z.string().min(1).optional(), // AgentMail API key for mothership email inbox AGENTMAIL_DOMAIN: z.string().optional(), // Custom domain for AgentMail inboxes (default: agentmail.to) MSHIP_PLAN_MODE: z.boolean().optional(), + MSHIP_COMPUTER_USE: z.boolean().optional(), MSHIP_MODEL_SELECTOR: z.boolean().optional(), SIM_SEARCH_LIVE: z.boolean().optional(), // Query connected providers directly; false preserves indexed search INBOX_ENABLED: z.boolean().optional(), // Enable inbox (Sim Mailer) on self-hosted (bypasses hosted requirements) diff --git a/apps/sim/lib/core/config/feature-flags.test.ts b/apps/sim/lib/core/config/feature-flags.test.ts index b10655e4cee..e2028e8d9d5 100644 --- a/apps/sim/lib/core/config/feature-flags.test.ts +++ b/apps/sim/lib/core/config/feature-flags.test.ts @@ -17,6 +17,7 @@ const { mockFetch, mockIsPlatformAdmin, envRef } = vi.hoisted(() => ({ TABLE_ROW_TTL: undefined as boolean | undefined, MSHIP_MODEL_SELECTOR: undefined as boolean | undefined, MSHIP_PLAN_MODE: undefined as boolean | undefined, + MSHIP_COMPUTER_USE: undefined as boolean | undefined, AGENT_MEMORY_HISTORY: undefined as boolean | undefined, CREDENTIAL_GROUPS: undefined as boolean | undefined, KNOWLEDGE_MEMBER_ACCESS: undefined as boolean | undefined, @@ -76,6 +77,18 @@ const enabled = (flag: string, ctx?: FeatureFlagContext) => afterAll(resetEnvFlagsMock) describe('getFeatureFlags', () => { + it('gates computer use globally and defaults off without AppConfig', async () => { + withAppConfig({ 'mothership-computer-use': { enabled: true } }) + expect(await isFeatureEnabled('mothership-computer-use')).toBe(true) + withAppConfig({ 'mothership-computer-use': { enabled: false, userIds: ['user-1'] } }) + expect(await isFeatureEnabled('mothership-computer-use')).toBe(false) + setEnvFlags({ isAppConfigEnabled: false }) + expect(await isFeatureEnabled('mothership-computer-use')).toBe(false) + envRef.MSHIP_COMPUTER_USE = true + expect(await isFeatureEnabled('mothership-computer-use')).toBe(true) + envRef.MSHIP_COMPUTER_USE = undefined + }) + beforeEach(() => { vi.clearAllMocks() setEnvFlags({ isAppConfigEnabled: false }) diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index db2a3c6b33a..68c1ff3b711 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -46,6 +46,11 @@ interface FeatureFlagDefinition { /** The single registry of known flags. To add a flag, add one entry here. */ const FEATURE_FLAGS = { + 'mothership-computer-use': { + description: + 'Enable native macOS computer use in Mothership. Global on/off only; each device must also opt in.', + fallback: 'MSHIP_COMPUTER_USE', + }, 'mothership-model-selector': { description: 'Show the Mothership model selector, model-specific effort levels, and Fast for supported ' + diff --git a/apps/sim/lib/desktop/index.test.ts b/apps/sim/lib/desktop/index.test.ts index 9381567d2f7..e31835b2e8a 100644 --- a/apps/sim/lib/desktop/index.test.ts +++ b/apps/sim/lib/desktop/index.test.ts @@ -2,6 +2,10 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const requestAvailability = vi.hoisted(() => vi.fn()) +vi.mock('@/lib/api/client', () => ({ requestJson: requestAvailability })) + import { getDesktopChatCapabilities, hasBrowserAgent, @@ -26,6 +30,49 @@ function installBridge(value: unknown): void { } describe('desktop surface availability', () => { + it.each([ + [true, true, true, true], + [true, true, false, false], + [true, false, true, false], + [false, true, true, false], + ])( + 'advertises computer use only for a supported enabled device and server rollout (%s,%s,%s)', + async (supported, enabled, rollout, expected) => { + installBridge({ + computerUse: { + getStatus: vi.fn(async () => ({ + supported, + enabled, + permissions: { accessibility: false, screenCapture: false }, + activeAction: null, + })), + }, + }) + setDesktopPreferencesSnapshot({ + ...ENABLED_PREFERENCES, + browserEnabled: false, + terminalEnabled: false, + }) + requestAvailability.mockResolvedValueOnce({ enabled: rollout }) + const result = await getDesktopChatCapabilities('chat-1') + expect(result.desktopCapabilities?.computerUse ?? false).toBe(expected) + } + ) + it('fails closed when the computer rollout cannot be resolved', async () => { + installBridge({ + computerUse: { getStatus: vi.fn(async () => ({ supported: true, enabled: true })) }, + }) + setDesktopPreferencesSnapshot({ + ...ENABLED_PREFERENCES, + browserEnabled: false, + terminalEnabled: false, + }) + requestAvailability.mockRejectedValueOnce(new Error('offline')) + expect( + (await getDesktopChatCapabilities('chat-1')).desktopCapabilities?.computerUse + ).toBeUndefined() + }) + beforeEach(() => { setDesktopPreferencesSnapshot(ENABLED_PREFERENCES) }) diff --git a/apps/sim/lib/desktop/index.ts b/apps/sim/lib/desktop/index.ts index 7ab60648924..0f50ad791e2 100644 --- a/apps/sim/lib/desktop/index.ts +++ b/apps/sim/lib/desktop/index.ts @@ -24,6 +24,8 @@ import type { BrowserKnownSession } from '@sim/browser-protocol' import type { DesktopPreferences, SimDesktopApi } from '@sim/desktop-bridge' import { truncate } from '@sim/utils/string' +import { requestJson } from '@/lib/api/client' +import { computerUseAvailabilityContract } from '@/lib/api/contracts/computer-use' import { DESKTOP_TERMINAL_HINT_ID_MAX_LENGTH, DESKTOP_TERMINAL_HINT_TEXT_MAX_LENGTH, @@ -158,6 +160,7 @@ export interface DesktopChatCapabilities { localFilesystem?: true browser?: true terminal?: true + computerUse?: true browserSessions?: BrowserKnownSession[] terminals?: DesktopTerminalHint[] } @@ -179,6 +182,14 @@ export async function getDesktopChatCapabilities( const localFilesystem = hasLocalFilesystem() const browser = isBrowserAgentEnabled() const terminal = isTerminalEnabled() + const computerUse = bridge?.computerUse + ? await Promise.all([ + bridge.computerUse.getStatus(), + requestJson(computerUseAvailabilityContract, { signal: AbortSignal.timeout(5000) }), + ]) + .then(([status, rollout]) => status.supported && status.enabled && rollout.enabled) + .catch(() => false) + : false // Sent every request so the agent knows what is already running without // spending a tool call to ask — and, more importantly, so it notices a // terminal that is occupied instead of launching a second copy into it. @@ -213,13 +224,14 @@ export async function getDesktopChatCapabilities( .catch(() => []) : [] return { - ...(localFiles || localFilesystem || browser || terminal + ...(localFiles || localFilesystem || browser || terminal || computerUse ? { desktopCapabilities: { ...(localFiles ? { localFiles: true as const } : {}), ...(localFilesystem ? { localFilesystem: true as const } : {}), ...(browser ? { browser: true as const } : {}), ...(terminal ? { terminal: true as const } : {}), + ...(computerUse ? { computerUse: true as const } : {}), ...(terminals.length > 0 ? { terminals } : {}), ...(browserSessions.length > 0 ? { browserSessions } : {}), }, diff --git a/apps/sim/lib/mothership/async-runs/lifecycle.ts b/apps/sim/lib/mothership/async-runs/lifecycle.ts index 060c5dba3c7..8f593487150 100644 --- a/apps/sim/lib/mothership/async-runs/lifecycle.ts +++ b/apps/sim/lib/mothership/async-runs/lifecycle.ts @@ -13,6 +13,7 @@ export const EXECUTABLE_TOOL_PERMISSION_DECISIONS = [ ] as const satisfies readonly CopilotToolPermissionDecision[] export const DESKTOP_TOOL_CLAIM_OWNER = { + computer: 'desktop-computer', browser: 'desktop-browser', terminal: 'desktop-terminal', files: 'desktop-files', diff --git a/apps/sim/lib/mothership/async-runs/repository.ts b/apps/sim/lib/mothership/async-runs/repository.ts index fae022827de..9062077e4e5 100644 --- a/apps/sim/lib/mothership/async-runs/repository.ts +++ b/apps/sim/lib/mothership/async-runs/repository.ts @@ -379,6 +379,7 @@ export async function getRunSegment(runId: string) { id: copilotRuns.id, userId: copilotRuns.userId, status: copilotRuns.status, + toolAdmissionClosedAt: copilotRuns.toolAdmissionClosedAt, workflowId: copilotRuns.workflowId, // Needed to scope an "allow for this chat" decision to its chat. chatId: copilotRuns.chatId, diff --git a/apps/sim/lib/mothership/chat/payload.test.ts b/apps/sim/lib/mothership/chat/payload.test.ts index afe2a8f4f42..9f6d9e31ebf 100644 --- a/apps/sim/lib/mothership/chat/payload.test.ts +++ b/apps/sim/lib/mothership/chat/payload.test.ts @@ -16,7 +16,9 @@ const { mockTrackChatUpload, mockSearchApprovals, mockSecretNames, + mockComputerUseAvailable, } = vi.hoisted(() => ({ + mockComputerUseAvailable: vi.fn(async () => false), mockCreateUserToolSchema: vi.fn(() => ({ type: 'object', properties: {} })), mockGetHighestPrioritySubscription: vi.fn(), mockGetUserPermissionConfig: vi.fn(), @@ -27,6 +29,10 @@ const { mockSecretNames: vi.fn(async () => ({ names: [] as string[] })), })) +vi.mock('@/lib/computer-use/availability.server', () => ({ + isComputerUseAvailable: mockComputerUseAvailable, +})) + // The inventory reads nine application worlds; these suites exercise the request shape, not the reads. vi.mock('@/lib/mothership/application/execute-organization-secret-use-case', () => ({ executeOrganizationSecretUseCase: mockSecretNames, @@ -823,6 +829,27 @@ describe('Assistant payload', () => { }) describe('desktop request capabilities', () => { + it.each([false, true])( + 'rechecks the server rollout for a forged computer capability (enabled=%s)', + async (enabled) => { + mockComputerUseAvailable.mockResolvedValueOnce(enabled) + const payload = await buildCopilotRequestPayload( + { + message: 'Use Notes', + workspaceId: 'workspace', + userId: 'user', + userMessageId: 'message', + mode: 'agent', + model: 'gpt-6-astra', + computerUse: true, + }, + { selectedModel: 'gpt-6-astra' } + ) + expect(payload.desktop?.computerUse ?? false).toBe(enabled) + expect(mockComputerUseAvailable).toHaveBeenCalledWith() + } + ) + it('preserves desktop capabilities and current session hints on the worker wire', async () => { const payload = await buildCopilotRequestPayload( { @@ -842,6 +869,7 @@ describe('desktop request capabilities', () => { { selectedModel: 'gpt-6-astra' } ) expect(payload.desktop).toEqual({ + computerUse: false, browser: true, terminal: true, terminals: [{ id: 'terminal-1', cwd: '/work/app', active: true }], @@ -865,6 +893,7 @@ describe('desktop request capabilities', () => { { selectedModel: 'gpt-6-astra' } ) expect(payload.desktop).toEqual({ + computerUse: false, localFiles: true, browser: false, terminal: false, diff --git a/apps/sim/lib/mothership/chat/payload.ts b/apps/sim/lib/mothership/chat/payload.ts index 69f367f0d55..feabc6f2ca6 100644 --- a/apps/sim/lib/mothership/chat/payload.ts +++ b/apps/sim/lib/mothership/chat/payload.ts @@ -6,6 +6,7 @@ import { toError } from '@sim/utils/errors' import { LRUCache } from 'lru-cache' import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription' import { isPaid } from '@/lib/billing/plan-helpers' +import { isComputerUseAvailable } from '@/lib/computer-use/availability.server' import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' import { isHosted, isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' import { isOAuthServiceDeploymentAvailable } from '@/lib/integrations/availability.server' @@ -88,6 +89,7 @@ interface BuildPayloadParams { desktopLocalFilesystem?: boolean browser?: boolean terminalCapable?: boolean + computerUse?: boolean terminals?: Array<{ id: string cwd?: string @@ -322,6 +324,8 @@ export async function buildCopilotRequestPayload( params const effectiveMode = mode === 'agent' ? 'build' : mode const isAssistant = effectiveMode === 'assistant' + const computerUse = + !isAssistant && params.computerUse === true && (await isComputerUseAvailable()) // Track uploaded files in the DB and build context tags instead of base64 inlining. // Tracking writes `workspace_files` rows, so it needs the same write grant the @@ -471,12 +475,14 @@ export async function buildCopilotRequestPayload( ...(params.effort ? { effort: params.effort } : {}), ...(params.modelSelection ? { modelSelection: params.modelSelection } : {}), ...(inventory ? { inventory } : {}), - ...(!isAssistant && (params.desktopLocalFiles || params.browser || params.terminalCapable) + ...(!isAssistant && + (params.desktopLocalFiles || params.browser || params.terminalCapable || computerUse) ? { desktop: { ...(params.desktopLocalFiles ? { localFiles: true } : {}), browser: params.browser === true, terminal: params.terminalCapable === true, + computerUse, terminals: params.terminalCapable ? (params.terminals ?? []).slice(0, 20) : [], browserSessions: params.browser ? (params.browserSessions ?? []).slice(0, 20) : [], }, diff --git a/apps/sim/lib/mothership/chat/post.ts b/apps/sim/lib/mothership/chat/post.ts index 7216025e601..b4bb937a39f 100644 --- a/apps/sim/lib/mothership/chat/post.ts +++ b/apps/sim/lib/mothership/chat/post.ts @@ -294,6 +294,7 @@ const ChatMessageSchema = z localFiles: z.boolean().optional(), browser: z.boolean().optional(), terminal: z.boolean().optional(), + computerUse: z.boolean().optional(), terminals: z .array( z.object({ @@ -399,6 +400,7 @@ type UnifiedChatBranch = desktopLocalFilesystem?: boolean browser?: boolean terminalCapable?: boolean + computerUse?: boolean terminals?: Terminals browserSessions?: BrowserSessions }) => Promise @@ -441,6 +443,7 @@ type UnifiedChatBranch = desktopLocalFilesystem?: boolean browser?: boolean terminalCapable?: boolean + computerUse?: boolean terminals?: Terminals browserSessions?: BrowserSessions }) => Promise @@ -831,6 +834,7 @@ async function resolveBranch(params: { desktopLocalFiles: payloadParams.desktopLocalFiles, browser: payloadParams.browser, terminalCapable: payloadParams.terminalCapable, + computerUse: payloadParams.computerUse, terminals: payloadParams.terminals, browserSessions: payloadParams.browserSessions, }, @@ -897,6 +901,7 @@ async function resolveBranch(params: { desktopLocalFiles: payloadParams.desktopLocalFiles, browser: payloadParams.browser, terminalCapable: payloadParams.terminalCapable, + computerUse: payloadParams.computerUse, terminals: payloadParams.terminals, browserSessions: payloadParams.browserSessions, }, @@ -1387,6 +1392,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { desktopLocalFiles: body.desktopCapabilities?.localFiles === true, browser: body.desktopCapabilities?.browser === true, terminalCapable: body.desktopCapabilities?.terminal === true, + computerUse: body.desktopCapabilities?.computerUse === true, terminals: body.desktopCapabilities?.terminals, browserSessions: body.desktopCapabilities?.browserSessions, }) @@ -1414,6 +1420,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { desktopLocalFiles: body.desktopCapabilities?.localFiles === true, browser: body.desktopCapabilities?.browser === true, terminalCapable: body.desktopCapabilities?.terminal === true, + computerUse: body.desktopCapabilities?.computerUse === true, terminals: body.desktopCapabilities?.terminals, browserSessions: body.desktopCapabilities?.browserSessions, }) diff --git a/apps/sim/lib/mothership/generated/computer-use.ts b/apps/sim/lib/mothership/generated/computer-use.ts new file mode 100644 index 00000000000..5ae90ef85cc --- /dev/null +++ b/apps/sim/lib/mothership/generated/computer-use.ts @@ -0,0 +1,173 @@ +// GENERATED — do not edit. Source of truth: mothership worker packages/contracts/src/computer-use.ts +// Regenerate with `bun run contracts:sync` in the worker. + +import { z } from "zod"; + +const BundleId = z + .string() + .min(1) + .max(255) + .describe( + "Exact bundleId returned by list_apps or a known installed macOS app. get_app_state may launch it.", + ); +const SnapshotId = z + .string() + .min(1) + .max(128) + .describe("Latest unconsumed snapshotId from get_app_state for this app. Actions consume it."); +const ElementId = z.string().min(1).max(128).describe("Element reference from that snapshot."); +const WindowId = z.string().min(1).max(128).describe("Window reference from that snapshot."); +const Coordinate = z.number().nonnegative().max(100_000); +const ElementTarget = { bundleId: BundleId, snapshotId: SnapshotId, elementId: ElementId }; +const WindowTarget = { bundleId: BundleId, snapshotId: SnapshotId, windowId: WindowId }; +const PointTarget = { ...WindowTarget, x: Coordinate, y: Coordinate }; +const ClickOptions = { + button: z.enum(["left", "right"]).optional(), + clickCount: z.number().int().min(1).max(3).optional(), +}; +const ScrollOffsets = { + deltaX: z + .number() + .min(-10_000) + .max(10_000) + .describe("Horizontal scroll distance in points; positive scrolls right, negative left."), + deltaY: z + .number() + .min(-10_000) + .max(10_000) + .describe("Vertical scroll distance in points; positive scrolls down, negative up."), +}; + +/** Strict branches require an observed, unambiguous target before native input. */ +export const ComputerUseSchema = z.union([ + z.strictObject({ action: z.literal("status") }), + z.strictObject({ action: z.literal("list_apps") }), + z.strictObject({ action: z.literal("activate_app"), bundleId: BundleId }), + z.strictObject({ + action: z.literal("get_app_state"), + bundleId: BundleId, + windowId: WindowId.optional(), + includeScreenshot: z + .boolean() + .optional() + .describe("Set true to include an image of the selected window."), + }), + z.strictObject({ action: z.literal("click"), ...ElementTarget, ...ClickOptions }), + z.strictObject({ action: z.literal("click"), ...PointTarget, ...ClickOptions }), + z.strictObject({ + action: z.literal("type_text"), + ...ElementTarget, + text: z.string().max(32_000).describe("Literal text to type into the observed editable element."), + }), + z.strictObject({ + action: z.literal("press_key"), + ...WindowTarget, + key: z.string().min(1).max(128).describe("One key or chord, such as Enter, Tab, Escape, or Cmd+A."), + }), + z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ScrollOffsets }), + z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ScrollOffsets }), + z.strictObject({ action: z.literal("drag"), ...PointTarget, toX: Coordinate, toY: Coordinate }), + z.strictObject({ + action: z.literal("set_value"), + ...ElementTarget, + value: z.string().max(32_000).describe("Replacement value for an accessibility-writable element."), + }), + z.strictObject({ + action: z.literal("perform_action"), + ...ElementTarget, + accessibilityAction: z.string().min(1).max(128).describe("Exact action advertised by this element."), + }), +]); +export type ComputerUseInput = z.infer; + +export const ComputerUseStatusSchema = z.strictObject({ + kind: z.literal("status"), + platform: z.literal("darwin"), + accessibility: z.boolean(), + screenRecording: z.boolean(), +}); +export type ComputerUseStatus = z.infer; + +export const ComputerUseAppSchema = z.strictObject({ + bundleId: BundleId, + name: z.string().max(1024), + pid: z.number().int().positive().optional(), + isActive: z.boolean(), +}); +export type ComputerUseApp = z.infer; + +export const ComputerUseWindowSchema = z.strictObject({ + windowId: WindowId, + title: z.string().max(4096), + x: z.number(), + y: z.number(), + width: z.number().nonnegative(), + height: z.number().nonnegative(), +}); +export type ComputerUseWindow = z.infer; + +const ComputerUseNodeSchema = z.strictObject({ + elementId: ElementId, + role: z.string().max(128), + parentId: ElementId.optional(), + label: z.string().max(8192).optional(), + value: z.string().max(32_000).optional(), + enabled: z.boolean().optional(), + actions: z.array(z.string().max(128)).max(128), + windowId: WindowId.optional(), + x: z.number().optional().describe("Global screen coordinate in points, not window-local."), + y: z.number().optional().describe("Global screen coordinate in points, not window-local."), + width: z.number().nonnegative().optional(), + height: z.number().nonnegative().optional(), +}); + +export const ComputerUseScreenshotSchema = z.strictObject({ + base64: z.string().min(1).max(11_200_000), + mimeType: z.literal("image/png"), + width: z.number().int().positive(), + height: z.number().int().positive(), +}); + +export const ComputerUseSnapshotSchema = z.strictObject({ + kind: z.literal("state"), + bundleId: BundleId, + snapshotId: SnapshotId, + windowId: WindowId, + windows: z.array(ComputerUseWindowSchema).max(100), + nodes: z.array(ComputerUseNodeSchema).max(2000), + truncated: z.boolean(), + screenshot: ComputerUseScreenshotSchema.optional(), + screenshotError: z.string().max(2000).optional(), +}); +export type ComputerUseSnapshot = z.infer; + +export const ComputerUseResultSchema = z.discriminatedUnion("kind", [ + ComputerUseStatusSchema, + z.strictObject({ kind: z.literal("apps"), apps: z.array(ComputerUseAppSchema).max(1000) }), + ComputerUseSnapshotSchema, + z.strictObject({ + kind: z.literal("action"), + action: z.enum([ + "activate_app", + "click", + "type_text", + "press_key", + "scroll", + "drag", + "set_value", + "perform_action", + ]), + bundleId: BundleId, + dispatched: z.literal(true), + verified: z.boolean(), + }), +]); +export type ComputerUseResult = z.infer; + +export const ComputerUseNativeReplySchema = z.union([ + z.strictObject({ id: z.string().min(1).max(128), result: ComputerUseResultSchema }), + z.strictObject({ + id: z.string().min(1).max(128), + error: z.strictObject({ code: z.string().min(1).max(128), message: z.string().min(1).max(2000) }), + }), +]); diff --git a/apps/sim/lib/mothership/generated/protocol.ts b/apps/sim/lib/mothership/generated/protocol.ts index c140abe0c68..fbb375a3b8b 100644 --- a/apps/sim/lib/mothership/generated/protocol.ts +++ b/apps/sim/lib/mothership/generated/protocol.ts @@ -103,6 +103,7 @@ export type ModelSelection = z.infer; /** Desktop capabilities and bounded session hints, supplied by Sim for this turn. */ export const DesktopContextSchema = z.object({ localFiles: z.boolean().optional(), + computerUse: z.boolean().default(false), browser: z.boolean().default(false), terminal: z.boolean().default(false), terminals: z diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-execution.test.ts b/apps/sim/lib/mothership/tools/client/computer-tool-execution.test.ts new file mode 100644 index 00000000000..75522b291ef --- /dev/null +++ b/apps/sim/lib/mothership/tools/client/computer-tool-execution.test.ts @@ -0,0 +1,139 @@ +/** @vitest-environment jsdom */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + execute: vi.fn(), + cancel: vi.fn(), + complete: vi.fn(), + pageExit: vi.fn(), +})) +vi.mock('@/lib/computer-use/transport', () => ({ + executeComputerUseTool: mocks.execute, + cancelComputerUseTool: mocks.cancel, +})) +vi.mock('@/lib/mothership/tools/client/completion', () => ({ + reportClientToolCompletion: mocks.complete, + reportClientToolCompletionOnPageExit: mocks.pageExit, +})) + +import { executeComputerToolOnClient } from '@/lib/mothership/tools/client/computer-tool-execution' + +let sequence = 0 +const nextId = () => `computer-test-${++sequence}` +const now = () => new Date().toISOString() +describe('computer action delivery', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.execute.mockResolvedValue({ kind: 'apps', apps: [] }) + mocks.cancel.mockResolvedValue(undefined) + mocks.complete.mockResolvedValue(undefined) + mocks.pageExit.mockResolvedValue(undefined) + }) + it('strips UI activity and runs each action only once across redelivery', async () => { + const id = nextId() + await executeComputerToolOnClient( + id, + { action: 'list_apps', activity: { title: 'Inspecting apps' } }, + now() + ) + await executeComputerToolOnClient(id, { action: 'list_apps' }, now()) + expect(mocks.execute).toHaveBeenCalledExactlyOnceWith( + id, + { action: 'list_apps' }, + expect.any(AbortSignal) + ) + expect(mocks.complete).toHaveBeenLastCalledWith( + id, + 'error', + expect.stringContaining('may already have run'), + expect.objectContaining({ doNotRetry: true }) + ) + }) + it.each([undefined, 'invalid', new Date(Date.now() - 121000).toISOString()])( + 'rejects stale or missing timestamps %s', + async (ts) => { + await executeComputerToolOnClient(nextId(), { action: 'list_apps' }, ts) + expect(mocks.execute).not.toHaveBeenCalled() + } + ) + it('rejects ambiguous target arguments before native dispatch', async () => { + await executeComputerToolOnClient( + nextId(), + { + action: 'click', + bundleId: 'com.apple.Notes', + snapshotId: 's1', + elementId: 'e1', + windowId: 'w1', + x: 1, + y: 1, + }, + now() + ) + expect(mocks.execute).not.toHaveBeenCalled() + }) + it('retries result delivery without repeating a native effect', async () => { + const id = nextId() + mocks.complete.mockRejectedValueOnce(new Error('offline')) + await executeComputerToolOnClient(id, { action: 'list_apps' }, now()) + await executeComputerToolOnClient(id, { action: 'list_apps' }, now()) + expect(mocks.execute).toHaveBeenCalledTimes(1) + expect(mocks.complete).toHaveBeenCalledTimes(2) + }) + it('cancels native work when Stop aborts the stream', async () => { + const controller = new AbortController() + let resolveAction: ((value: { kind: 'apps'; apps: [] }) => void) | undefined + mocks.execute.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveAction = resolve + }) + ) + const id = nextId() + const execution = executeComputerToolOnClient( + id, + { action: 'list_apps' }, + now(), + controller.signal + ) + controller.abort() + resolveAction?.({ kind: 'apps', apps: [] }) + await execution + expect(mocks.cancel).toHaveBeenCalledWith(id) + expect(mocks.complete).toHaveBeenCalledWith( + id, + 'cancelled', + expect.any(String), + expect.objectContaining({ doNotRetry: true }) + ) + }) + it('does not dispatch a tool when Stop already won', async () => { + const controller = new AbortController() + controller.abort() + await executeComputerToolOnClient(nextId(), { action: 'list_apps' }, now(), controller.signal) + expect(mocks.execute).not.toHaveBeenCalled() + }) + it('forwards screenshot bytes as a visual observation with a point coordinate mapping', async () => { + mocks.execute.mockResolvedValueOnce({ + kind: 'state', + bundleId: 'com.apple.Notes', + snapshotId: 's1', + windowId: 'w1', + windows: [{ windowId: 'w1', title: 'Notes', x: 20, y: 40, width: 400, height: 300 }], + nodes: [], + truncated: false, + screenshot: { base64: 'YWJj', mimeType: 'image/png', width: 800, height: 600 }, + }) + await executeComputerToolOnClient( + nextId(), + { action: 'get_app_state', bundleId: 'com.apple.Notes' }, + now() + ) + const output = mocks.complete.mock.calls[0][3] + expect(output).not.toHaveProperty('screenshot') + expect(output.observations).toEqual([ + { name: 'Computer screenshot', mediaType: 'image/png', data: 'YWJj' }, + ]) + expect(output.content).toContain('x = imageX * 400 / 800') + }) +}) diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts new file mode 100644 index 00000000000..83998390cef --- /dev/null +++ b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts @@ -0,0 +1,186 @@ +import { ComputerUseSchema } from '@sim/desktop-bridge/computer-use' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { omit } from '@sim/utils/object' +import { cancelComputerUseTool, executeComputerUseTool } from '@/lib/computer-use/transport' +import { + ASYNC_TOOL_CONFIRMATION_STATUS, + type AsyncConfirmationStatus, +} from '@/lib/mothership/async-runs/lifecycle' +import { BrowserToolReplayLedger } from '@/lib/mothership/tools/client/browser-tool-replay-ledger' +import { + reportClientToolCompletion, + reportClientToolCompletionOnPageExit, +} from '@/lib/mothership/tools/client/completion' +import { computerToolResultForModel } from '@/lib/mothership/tools/client/computer-tool-result' + +const logger = createLogger('ComputerToolExecution') +const MAX_EVENT_AGE_MS = 120_000 +const MAX_UNDELIVERED_RESULTS = 8 +const MAX_ACTION_MS = 90_000 +const replayLedger = new BrowserToolReplayLedger({ + storageKey: 'sim:computer-tool-ledger:v1', + legacyStoragePrefix: 'sim:computer-tool-executed:', + maxEntries: 2048, + ttlMs: 5 * 60_000, + protectedWindowMs: MAX_EVENT_AGE_MS, +}) +interface Completion { + status: AsyncConfirmationStatus + message: string + data?: unknown +} +interface Execution { + completion?: Completion + reporting?: Promise +} +const executions = new Map() + +async function deliver(toolCallId: string, execution: Execution): Promise { + if (execution.reporting) return execution.reporting + const completion = execution.completion + if (!completion) return + execution.reporting = reportClientToolCompletion( + toolCallId, + completion.status, + completion.message, + completion.data + ) + .then(() => { + executions.delete(toolCallId) + }) + .catch((error) => { + logger.warn('Computer action result delivery failed; retained for redelivery', { + toolCallId, + error: getErrorMessage(error), + }) + }) + .finally(() => { + execution.reporting = undefined + }) + return execution.reporting +} + +/** A live stream may dispatch each server-persisted action once; reconnects only redeliver its result. */ +export async function executeComputerToolOnClient( + toolCallId: string, + params: Record, + eventTs?: string, + signal?: AbortSignal +): Promise { + const existing = executions.get(toolCallId) + if (existing) return deliver(toolCallId, existing) + const execution: Execution = {} + const reject = async (message: string, data?: Record) => { + await reportClientToolCompletion(toolCallId, ASYNC_TOOL_CONFIRMATION_STATUS.error, message, { + error: message, + ...data, + }).catch((error) => + logger.warn('Could not report computer action rejection', { + toolCallId, + error: getErrorMessage(error), + }) + ) + } + if (executions.size >= MAX_UNDELIVERED_RESULTS) + return reject( + 'Computer use is waiting for earlier action results to reach the server. Try again after the connection recovers.' + ) + const emittedAt = eventTs ? Date.parse(eventTs) : Number.NaN + if ( + !Number.isFinite(emittedAt) || + Date.now() - emittedAt > MAX_EVENT_AGE_MS || + emittedAt - Date.now() > 5000 + ) + return reject( + 'This computer action is stale. Inspect the app again before deciding what to do.', + { doNotRetry: true, outcomeUnknown: true } + ) + const parsed = ComputerUseSchema.safeParse(omit(params, ['activity'])) + if (!parsed.success) + return reject('Computer action arguments are invalid. Inspect the tool schema and try again.') + const claim = replayLedger.claim(toolCallId) + if (claim !== 'claimed') + return reject( + claim === 'duplicate' + ? 'This computer action may already have run. Inspect the app before repeating it.' + : 'Computer use could not establish reload-safe replay protection. Enable browser storage and try again.', + claim === 'duplicate' ? { doNotRetry: true, outcomeUnknown: true } : undefined + ) + executions.set(toolCallId, execution) + const actionController = new AbortController() + let dispatched = false + let cancelled = signal?.aborted === true + const cancel = () => { + cancelled = true + actionController.abort() + void cancelComputerUseTool(toolCallId).catch((error) => + logger.warn('Computer action cancellation failed', { + toolCallId, + error: getErrorMessage(error), + }) + ) + } + const onPageHide = () => { + cancel() + void reportClientToolCompletionOnPageExit( + toolCallId, + ASYNC_TOOL_CONFIRMATION_STATUS.error, + 'The desktop view closed during a computer action. Inspect the app before repeating it.', + { outcomeUnknown: dispatched, doNotRetry: dispatched } + ).catch((error) => + logger.warn('Computer action page-exit result failed', { + toolCallId, + error: getErrorMessage(error), + }) + ) + } + signal?.addEventListener('abort', cancel, { once: true }) + window.addEventListener('pagehide', onPageHide) + let timer: ReturnType | undefined + try { + if (!cancelled) { + dispatched = true + const result = await Promise.race([ + executeComputerUseTool(toolCallId, parsed.data, actionController.signal), + new Promise((_resolve, rejectTimeout) => { + timer = setTimeout(() => { + cancel() + rejectTimeout(new Error('Computer action timed out; its effect may be incomplete')) + }, MAX_ACTION_MS) + }), + ]) + execution.completion = cancelled + ? { + status: ASYNC_TOOL_CONFIRMATION_STATUS.cancelled, + message: 'Computer action stopped. Inspect the app before repeating it.', + data: { outcomeUnknown: true, doNotRetry: true }, + } + : { + status: ASYNC_TOOL_CONFIRMATION_STATUS.success, + message: + result.kind === 'action' && !result.verified + ? 'Input was dispatched. Inspect the app to verify its effect.' + : 'Computer observation completed', + data: computerToolResultForModel(result), + } + } else + execution.completion = { + status: ASYNC_TOOL_CONFIRMATION_STATUS.cancelled, + message: 'Computer action stopped before execution', + } + } catch (error) { + execution.completion = { + status: cancelled + ? ASYNC_TOOL_CONFIRMATION_STATUS.cancelled + : ASYNC_TOOL_CONFIRMATION_STATUS.error, + message: getErrorMessage(error, 'Computer action failed'), + data: { doNotRetry: dispatched, outcomeUnknown: dispatched }, + } + } finally { + if (timer) clearTimeout(timer) + signal?.removeEventListener('abort', cancel) + window.removeEventListener('pagehide', onPageHide) + } + await deliver(toolCallId, execution) +} diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts new file mode 100644 index 00000000000..5f8b81eccda --- /dev/null +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts @@ -0,0 +1,18 @@ +import type { ComputerUseResult } from '@sim/desktop-bridge/computer-use' + +/** Screenshots become model image observations; base64 is never duplicated inside textual output. */ +export function computerToolResultForModel(result: ComputerUseResult) { + if (result.kind !== 'state' || !result.screenshot) return result + const { screenshot, ...state } = result + const window = state.windows.find((candidate) => candidate.windowId === state.windowId) + return { + ...state, + screenshotSize: { width: screenshot.width, height: screenshot.height }, + content: window + ? `Screenshot of window ${state.windowId}. Coordinate actions use window-local macOS points. Convert encoded image coordinates: x = imageX * ${window.width} / ${screenshot.width}; y = imageY * ${window.height} / ${screenshot.height}. Accessibility node x/y are global screen points; subtract window origin (${window.x}, ${window.y}) before a coordinate action. Use element IDs when available.` + : 'Screenshot coordinate mapping is unavailable. Use accessibility element IDs or take a fresh state before acting.', + observations: [ + { name: 'Computer screenshot', mediaType: screenshot.mimeType, data: screenshot.base64 }, + ], + } +} diff --git a/biome.json b/biome.json index f87153397c8..817a825af6c 100644 --- a/biome.json +++ b/biome.json @@ -28,6 +28,7 @@ "!**/public/fallback-*.js", "!**/apps/sim/tools/generated", "!**/apps/sim/lib/mothership/generated/**", + "!**/packages/desktop-bridge/src/computer-use.generated.ts", "!**/apps/docs/.source", "!**/apps/desktop/release", "!**/venv", diff --git a/bun.lock b/bun.lock index 7949c9faee5..05c42bc148d 100644 --- a/bun.lock +++ b/bun.lock @@ -511,6 +511,7 @@ "@sim/browser-protocol": "workspace:*", "@sim/terminal-protocol": "workspace:*", "@sim/utils": "workspace:*", + "zod": "4.3.6", }, "devDependencies": { "@sim/tsconfig": "workspace:*", diff --git a/packages/desktop-bridge/package.json b/packages/desktop-bridge/package.json index da9dcbd76f2..d1f8695f145 100644 --- a/packages/desktop-bridge/package.json +++ b/packages/desktop-bridge/package.json @@ -17,6 +17,10 @@ "./local-filesystem-limits": { "types": "./src/local-filesystem-limits.ts", "default": "./src/local-filesystem-limits.ts" + }, + "./computer-use": { + "types": "./src/computer-use.generated.ts", + "default": "./src/computer-use.generated.ts" } }, "scripts": { @@ -29,7 +33,8 @@ "dependencies": { "@sim/browser-protocol": "workspace:*", "@sim/terminal-protocol": "workspace:*", - "@sim/utils": "workspace:*" + "@sim/utils": "workspace:*", + "zod": "4.3.6" }, "devDependencies": { "@sim/tsconfig": "workspace:*", diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts new file mode 100644 index 00000000000..5ae90ef85cc --- /dev/null +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -0,0 +1,173 @@ +// GENERATED — do not edit. Source of truth: mothership worker packages/contracts/src/computer-use.ts +// Regenerate with `bun run contracts:sync` in the worker. + +import { z } from "zod"; + +const BundleId = z + .string() + .min(1) + .max(255) + .describe( + "Exact bundleId returned by list_apps or a known installed macOS app. get_app_state may launch it.", + ); +const SnapshotId = z + .string() + .min(1) + .max(128) + .describe("Latest unconsumed snapshotId from get_app_state for this app. Actions consume it."); +const ElementId = z.string().min(1).max(128).describe("Element reference from that snapshot."); +const WindowId = z.string().min(1).max(128).describe("Window reference from that snapshot."); +const Coordinate = z.number().nonnegative().max(100_000); +const ElementTarget = { bundleId: BundleId, snapshotId: SnapshotId, elementId: ElementId }; +const WindowTarget = { bundleId: BundleId, snapshotId: SnapshotId, windowId: WindowId }; +const PointTarget = { ...WindowTarget, x: Coordinate, y: Coordinate }; +const ClickOptions = { + button: z.enum(["left", "right"]).optional(), + clickCount: z.number().int().min(1).max(3).optional(), +}; +const ScrollOffsets = { + deltaX: z + .number() + .min(-10_000) + .max(10_000) + .describe("Horizontal scroll distance in points; positive scrolls right, negative left."), + deltaY: z + .number() + .min(-10_000) + .max(10_000) + .describe("Vertical scroll distance in points; positive scrolls down, negative up."), +}; + +/** Strict branches require an observed, unambiguous target before native input. */ +export const ComputerUseSchema = z.union([ + z.strictObject({ action: z.literal("status") }), + z.strictObject({ action: z.literal("list_apps") }), + z.strictObject({ action: z.literal("activate_app"), bundleId: BundleId }), + z.strictObject({ + action: z.literal("get_app_state"), + bundleId: BundleId, + windowId: WindowId.optional(), + includeScreenshot: z + .boolean() + .optional() + .describe("Set true to include an image of the selected window."), + }), + z.strictObject({ action: z.literal("click"), ...ElementTarget, ...ClickOptions }), + z.strictObject({ action: z.literal("click"), ...PointTarget, ...ClickOptions }), + z.strictObject({ + action: z.literal("type_text"), + ...ElementTarget, + text: z.string().max(32_000).describe("Literal text to type into the observed editable element."), + }), + z.strictObject({ + action: z.literal("press_key"), + ...WindowTarget, + key: z.string().min(1).max(128).describe("One key or chord, such as Enter, Tab, Escape, or Cmd+A."), + }), + z.strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ScrollOffsets }), + z.strictObject({ action: z.literal("scroll"), ...PointTarget, ...ScrollOffsets }), + z.strictObject({ action: z.literal("drag"), ...PointTarget, toX: Coordinate, toY: Coordinate }), + z.strictObject({ + action: z.literal("set_value"), + ...ElementTarget, + value: z.string().max(32_000).describe("Replacement value for an accessibility-writable element."), + }), + z.strictObject({ + action: z.literal("perform_action"), + ...ElementTarget, + accessibilityAction: z.string().min(1).max(128).describe("Exact action advertised by this element."), + }), +]); +export type ComputerUseInput = z.infer; + +export const ComputerUseStatusSchema = z.strictObject({ + kind: z.literal("status"), + platform: z.literal("darwin"), + accessibility: z.boolean(), + screenRecording: z.boolean(), +}); +export type ComputerUseStatus = z.infer; + +export const ComputerUseAppSchema = z.strictObject({ + bundleId: BundleId, + name: z.string().max(1024), + pid: z.number().int().positive().optional(), + isActive: z.boolean(), +}); +export type ComputerUseApp = z.infer; + +export const ComputerUseWindowSchema = z.strictObject({ + windowId: WindowId, + title: z.string().max(4096), + x: z.number(), + y: z.number(), + width: z.number().nonnegative(), + height: z.number().nonnegative(), +}); +export type ComputerUseWindow = z.infer; + +const ComputerUseNodeSchema = z.strictObject({ + elementId: ElementId, + role: z.string().max(128), + parentId: ElementId.optional(), + label: z.string().max(8192).optional(), + value: z.string().max(32_000).optional(), + enabled: z.boolean().optional(), + actions: z.array(z.string().max(128)).max(128), + windowId: WindowId.optional(), + x: z.number().optional().describe("Global screen coordinate in points, not window-local."), + y: z.number().optional().describe("Global screen coordinate in points, not window-local."), + width: z.number().nonnegative().optional(), + height: z.number().nonnegative().optional(), +}); + +export const ComputerUseScreenshotSchema = z.strictObject({ + base64: z.string().min(1).max(11_200_000), + mimeType: z.literal("image/png"), + width: z.number().int().positive(), + height: z.number().int().positive(), +}); + +export const ComputerUseSnapshotSchema = z.strictObject({ + kind: z.literal("state"), + bundleId: BundleId, + snapshotId: SnapshotId, + windowId: WindowId, + windows: z.array(ComputerUseWindowSchema).max(100), + nodes: z.array(ComputerUseNodeSchema).max(2000), + truncated: z.boolean(), + screenshot: ComputerUseScreenshotSchema.optional(), + screenshotError: z.string().max(2000).optional(), +}); +export type ComputerUseSnapshot = z.infer; + +export const ComputerUseResultSchema = z.discriminatedUnion("kind", [ + ComputerUseStatusSchema, + z.strictObject({ kind: z.literal("apps"), apps: z.array(ComputerUseAppSchema).max(1000) }), + ComputerUseSnapshotSchema, + z.strictObject({ + kind: z.literal("action"), + action: z.enum([ + "activate_app", + "click", + "type_text", + "press_key", + "scroll", + "drag", + "set_value", + "perform_action", + ]), + bundleId: BundleId, + dispatched: z.literal(true), + verified: z.boolean(), + }), +]); +export type ComputerUseResult = z.infer; + +export const ComputerUseNativeReplySchema = z.union([ + z.strictObject({ id: z.string().min(1).max(128), result: ComputerUseResultSchema }), + z.strictObject({ + id: z.string().min(1).max(128), + error: z.strictObject({ code: z.string().min(1).max(128), message: z.string().min(1).max(2000) }), + }), +]); diff --git a/packages/desktop-bridge/src/index.ts b/packages/desktop-bridge/src/index.ts index ca0eef82909..810b336d506 100644 --- a/packages/desktop-bridge/src/index.ts +++ b/packages/desktop-bridge/src/index.ts @@ -1,3 +1,4 @@ +import type { ComputerUseInput, ComputerUseResult } from './computer-use.generated' import type { DesktopLocalFileRequest, DesktopLocalFileResponse } from './local-files' export type { @@ -35,6 +36,42 @@ import type { export const PENDING_DESKTOP_SCOPE_PREFIX = 'pending:' as const +/** Native work is bound to the server-authorized chat and tool call. */ +export interface ComputerUseActivity { + toolCallId: string + scopeId: string + bundleId?: string + appName?: string + action: string + startedAt: number + stopShortcutAvailable?: boolean +} + +/** Device permissions are independent of the server rollout flag. */ +export interface ComputerUseStatus { + supported: boolean + enabled: boolean + permissions: { accessibility: boolean; screenCapture: boolean } + activeAction: ComputerUseActivity | null +} + +export interface ComputerUseAppPermission { + bundleId: string + displayName: string +} + +/** Optional so a new web deployment remains compatible with older desktop shells. */ +export interface SimDesktopComputerUseApi { + getStatus(): Promise + setEnabled(enabled: boolean): Promise + requestPermission(permission: 'accessibility' | 'screenCapture'): Promise + listAppPermissions(): Promise + revokeApp(bundleId: string): Promise + executeTool(toolCallId: string, params: ComputerUseInput): Promise + cancel(toolCallId?: string): Promise + onActivity(callback: (activity: ComputerUseActivity | null) => void): () => void +} + /** Boolean results preserve compatibility with older installed desktop shells. */ export type TerminalPasteResult = boolean | 'too-large' @@ -1075,6 +1112,7 @@ export interface SimDesktopServerApi { } export interface SimDesktopApi { + computerUse?: SimDesktopComputerUseApi /** Installed shell version (plain semver, e.g. `0.3.1`). */ version: string openExternal(url: string): Promise