Add gated native macOS computer use to Mothership desktop

apps/desktop/README.md

@@ -218,3 +218,56 @@ Raw local file bytes are never exposed through the preload bridge and cannot be
 ## Electron upgrades
 
 Cadence: Electron ships a major every ~8 weeks and supports the latest 3 — budget a bump every ~4–6 months and adopt security patches within ~2 weeks. Follow `docs/electron-upgrade-checklist.md`; the `desktop-e2e.yml` canary leg (electron@latest) is the early-warning signal.
+
+
+## Native Mac computer use
+
+Mothership's `computer` tool is available on macOS 14+ when the global runtime flag
+`mothership-computer-use` is enabled and the user opts in under **Settings → Desktop → Computer Use**.
+The flag defaults off; local/self-hosted deployments use `MSHIP_COMPUTER_USE=true`, while hosted
+rollout uses the existing AppConfig feature-flags document. Older desktop builds omit this optional
+bridge and continue to work.
+
+The user grants Accessibility and Screen Recording through macOS, then approves each app for the
+current task or persistently. Persistent app approvals are listed in Settings and can be revoked.
+Sign-out and deployment changes clear computer-use approval and opt-in. The conversation shows the
+active app/action and a Stop button; **⌘⇧Esc** also stops active work when the shortcut is available.
+Stop cancels queued actions, in-flight authorization, approval prompts and the native helper.
+Already completed input cannot be undone.
+
+The tool lists installed/running apps, reads window accessibility trees, captures selected-window
+screenshots, activates apps explicitly, and performs clicks, text input, keyboard shortcuts,
+scrolling, dragging, value changes and accessibility actions. Semantic actions can run in the
+background. Coordinate mouse actions require explicit foreground activation and a fresh observation;
+they validate the target app/window before dispatch. Each mutation consumes its snapshot and the
+model observes again to verify results. Secure accessibility values are suppressed. See
+[`native/computer-use/README.md`](native/computer-use/README.md) for native bounds and compatibility
+limits; screenshots can still contain sensitive content visibly rendered by the approved app.
+
+Every execution is authorized and claimed once by `/api/desktop/computer/authorize` using the
+stored tool arguments, current chat access, run/Stop state and current flag. Renderer-supplied
+arguments are not execution authority. The main process serializes native work and binds snapshots
+to the authorized chat. The Swift helper communicates over private stdio and is packaged outside
+ASAR as `Contents/Resources/Sim Computer Use.app`.
+
+Distribution signing must cover the nested helper with the same stable Developer ID identity.
+Development ad-hoc builds can lose macOS grants when rebuilt. macOS can attribute permission
+requests to the responsible parent app (Sim, Electron, or the terminal/IDE that launched a test),
+so confirm the name in the system permission dialog. Test a Finder/LaunchServices-launched signed
+build for release acceptance; terminal-launched helper tests do not establish that app's TCC grants.
+
+Validation includes desktop lifecycle/transport unit tests, the Electron authorization/Stop test
+in `e2e/computer-use.spec.ts`, the native protocol and live fixture tests, Sim authorization/stream
+regressions, and Mothership's opt-in `apps/server/scripts/computer-use-live.ts` real-model trial.
+The live trial uses only the disposable fixture and verifies mutations by fresh state; visual mode
+also asks the model to identify a drawn shape unavailable in accessibility text.
+
+
+September 24 acceptance: 1,909 desktop regression tests, the Electron canonical-authorization/
+replay/Stop test, native pure/protocol tests, and the live fixture GUI suite passed. A real native
+Calculator background test verified `2 + 3 = 5` without changing the foreground app. The signed
+arm64 test bundle and nested universal helper passed strict code-signature verification. Its
+LaunchServices-launched acceptance remained blocked on that app identity's Accessibility and
+Screen Recording grants; the separately granted development helper's tests do not satisfy this
+last release-onboarding check. The disposable packaged test follows the existing packaged-smoke
+suite's mock-keychain convention; native TCC permissions remain real.

apps/desktop/e2e/computer-use.spec.ts

@@ -0,0 +1,150 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { createServer, type Server } from 'node:http'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { type ElectronApplication, _electron as electron, expect, test } from '@playwright/test'
+import type { SimDesktopApi } from '@sim/desktop-bridge'
+
+const DESKTOP_DIR = fileURLToPath(new URL('..', import.meta.url))
+
+test('computer use reaches the actual helper only with canonical one-shot authorization', async () => {
+  test.skip(process.platform !== 'darwin', 'The native helper is macOS-only.')
+  const root = mkdtempSync(join(tmpdir(), 'sim-computer-use-e2e-'))
+  let server: Server | undefined
+  let app: ElectronApplication | undefined
+  let claimed = false
+  let authorizationCount = 0
+  let releaseAuthorization: () => void = () => {}
+  let markAuthorizationStarted: () => void = () => {}
+  const authorizationStarted = new Promise<void>((resolve) => {
+    markAuthorizationStarted = resolve
+  })
+  const authorizationReleased = new Promise<void>((resolve) => {
+    releaseAuthorization = resolve
+  })
+  try {
+    server = createServer(async (request, response) => {
+      const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname
+      if (path === '/api/auth/get-session') {
+        response.writeHead(200, { 'Content-Type': 'application/json' }).end(
+          JSON.stringify({
+            user: { id: 'computer-test' },
+            session: { id: 'computer-test-session' },
+          })
+        )
+        return
+      }
+      if (path === '/api/desktop/computer/authorize') {
+        authorizationCount += 1
+        let body = ''
+        for await (const chunk of request) body += chunk.toString()
+        const input = JSON.parse(body)
+        expect(Object.keys(input)).toEqual(['toolCallId'])
+        if (input.toolCallId === 'delayed') {
+          markAuthorizationStarted()
+          await authorizationReleased
+        } else if (input.toolCallId !== 'status-once' || claimed) {
+          response.writeHead(403, { 'Content-Type': 'application/json' }).end('{}')
+          return
+        }
+        claimed = true
+        response.writeHead(200, { 'Content-Type': 'application/json' }).end(
+          JSON.stringify({
+            chatId: 'computer-chat',
+            toolName: 'computer',
+            args: { action: 'status' },
+          })
+        )
+        return
+      }
+      response
+        .writeHead(200, {
+          'Content-Type': 'text/html',
+          'Set-Cookie': 'better-auth.session_token=fixture; HttpOnly; SameSite=Lax; Path=/',
+        })
+        .end(
+          `<!doctype html><title>Computer Use fixture</title><h1>Computer Use</h1><button id="disable">Disable Computer Use</button><output id="status"></output><script>document.getElementById('disable').onclick=async()=>{const state=await window.simDesktop.computerUse.setEnabled(false);document.getElementById('status').textContent=String(state.enabled)}</script>`
+        )
+    })
+    await new Promise<void>((resolve) => server?.listen(0, '127.0.0.1', resolve))
+    const address = server.address()
+    if (!address || typeof address === 'string') throw new Error('Missing local fixture port')
+    const profile = join(root, 'profile')
+    mkdirSync(profile)
+    writeFileSync(
+      join(profile, 'settings.json'),
+      JSON.stringify({ origin: `http://127.0.0.1:${address.port}`, computerUseEnabled: true })
+    )
+    app = await electron.launch({
+      args: ['.'],
+      cwd: DESKTOP_DIR,
+      env: {
+        ...process.env,
+        SIM_DESKTOP_ORIGIN: `http://127.0.0.1:${address.port}`,
+        SIM_DESKTOP_USER_DATA: profile,
+      },
+    })
+    const window = await app.firstWindow()
+    await expect(window.getByRole('heading')).toHaveText('Computer Use')
+    const status = await window.evaluate(async () => {
+      const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop
+      return api.computerUse?.getStatus()
+    })
+    expect(status).toMatchObject({ supported: true, enabled: true, activeAction: null })
+    const result = await window.evaluate(async () => {
+      const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop
+      return api.computerUse?.executeTool('status-once', {
+        action: 'get_app_state',
+        bundleId: 'com.apple.systempreferences',
+      })
+    })
+    expect(result).toMatchObject({ kind: 'status', platform: 'darwin' })
+    const replay = await window.evaluate(async () => {
+      const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop
+      try {
+        await api.computerUse?.executeTool('status-once', { action: 'status' })
+        return 'unexpected success'
+      } catch {
+        return 'rejected'
+      }
+    })
+    expect(replay).toBe('rejected')
+    expect(authorizationCount).toBe(2)
+    const delayed = window.evaluate(async () => {
+      const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop
+      try {
+        await api.computerUse?.executeTool('delayed', { action: 'status' })
+        return 'unexpected success'
+      } catch {
+        return 'stopped'
+      }
+    })
+    await authorizationStarted
+    await window.evaluate(async () => {
+      const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop
+      await api.computerUse?.cancel('delayed')
+    })
+    releaseAuthorization()
+    expect(await delayed).toBe('stopped')
+    expect(authorizationCount).toBe(3)
+    await window.getByRole('button', { name: 'Disable Computer Use' }).click()
+    await expect(window.locator('#status')).toHaveText('false')
+    const disabled = await window.evaluate(async () => {
+      const api = (globalThis as typeof globalThis & { simDesktop: SimDesktopApi }).simDesktop
+      try {
+        await api.computerUse?.executeTool('disabled', { action: 'status' })
+        return 'unexpected success'
+      } catch {
+        return 'rejected'
+      }
+    })
+    expect(disabled).toBe('rejected')
+    expect(authorizationCount).toBe(3)
+  } finally {
+    releaseAuthorization()
+    await app?.close()
+    await new Promise<void>((resolve) => (server ? server.close(() => resolve()) : resolve()))
+    rmSync(root, { recursive: true, force: true })
+  }
+})

apps/desktop/electron-builder.yml

@@ -8,13 +8,18 @@ directories:
 
 files:
   - dist/**
+  - "!dist/native/Sim Computer Use.app/**"
   - static/**
   - package.json
   - from: ../sim/app/_styles/fonts/season
     to: static
     filter:
       - SeasonSansUprightsVF.woff2
 
+extraResources:
+  - from: dist/native/Sim Computer Use.app
+    to: Sim Computer Use.app
+
 asar: true
 
 # Native modules cannot be dlopen'd from inside an asar. The Help-search addon
@@ -64,6 +69,7 @@ mac:
   # macOS refuses to show the microphone prompt at all — it kills the process —
   # unless the bundle declares why it wants the device.
   extendInfo:
+    NSScreenCaptureUsageDescription: Mothership captures approved app windows to understand and verify computer-use actions.
     NSMicrophoneUsageDescription: Sim uses your microphone for voice input in Chat and for meetings you join in the built-in browser.
     NSCameraUsageDescription: Sim uses your camera for meetings you join in the built-in browser, such as Google Meet.
     NSBluetoothAlwaysUsageDescription: Sim uses Bluetooth to complete passkey sign-ins with a nearby phone in the built-in browser.