Fix macOS keyboard actions targeting the wrong app (#3960)

* fix: guard macos synthetic input focus * Tighten macOS keyboard focus safety Co-authored-by: Orca <help@stably.ai> --------- Co-authored-by: Jinwoo-H <jinwoo0825@gmail.com> Co-authored-by: Orca <help@stably.ai>
2026-06-03 16:00:34 -07:00 · 2026-06-03 16:00:34 -07:00 · 09872bfe7b
parent 885f3c3b25
commit 09872bfe7b
5 changed files with 166 additions and 6 deletions
--- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift
+++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift
@ -273,11 +273,9 @@ final class Provider {
    }

    private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot {
-        let snapshot = try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement })
-        if params["restoreWindow"]?.bool != true && !isTargetWindowFocused(snapshot) {
-            throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; retry with --restore-window or use set-value for editable elements")
-        }
-        return snapshot
+        // Why: AX text replacement/select-all do not post global input, so only
+        // synthetic fallback paths require the target window to be focused.
+        try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement })
    }

    private func cachedSnapshot(params: [String: JSONValue]) throws -> Snapshot? {
@ -671,12 +669,14 @@ final class Provider {

    private func typeText(params: [String: JSONValue]) throws -> [String: Any] {
        let snapshot = try currentKeyboardSnapshot(params: params)
+        try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
        try Input.typeText(try requiredString(params, "text"), pid: snapshot.app.pid)
        return actionMetadata(path: "synthetic")
    }

    private func pressKey(params: [String: JSONValue]) throws -> [String: Any] {
        let snapshot = try currentKeyboardSnapshot(params: params)
+        try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
        try Input.pressKey(try requiredString(params, "key"), pid: snapshot.app.pid)
        return actionMetadata(path: "synthetic")
    }
@ -691,6 +691,7 @@ final class Provider {
                verification: TextInput.selectionVerification(focused.element)
            )
        }
+        try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
        try Input.pressKey(key, pid: snapshot.app.pid)
        return actionMetadata(
            path: "synthetic",
@ -705,6 +706,7 @@ final class Provider {
        if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) {
            return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
        }
+        try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
        try Input.pasteText(text, pid: snapshot.app.pid)
        return actionMetadata(
            path: "clipboard",
@ -936,6 +938,21 @@ private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool {
    return !intersection.isNull && intersection.area >= min(frame.area, snapshot.windowBounds.area) * 0.75
 }

+private func requireTargetWindowFocused(_ snapshot: Snapshot, restoreWindowRequested: Bool) throws {
+    guard let failure = KeyboardInputSafety.syntheticInputFocusFailure(
+        targetWindowFocused: isTargetWindowFocused(snapshot),
+        restoreWindowRequested: restoreWindowRequested
+    ) else {
+        return
+    }
+    switch failure {
+    case .targetNotFocused:
+        throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; retry with --restore-window or use set-value for editable elements")
+    case .targetNotFocusedAfterRestore:
+        throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; --restore-window was requested but the target is still not focused; bring it forward manually or check Accessibility permissions")
+    }
+}
+
 private func matchingWindow(appElement: AXUIElement, capture: WindowCapture, focused: AXUIElement, explicitTarget: Bool) -> AXUIElement? {
    guard let windows = copyArray(appElement, kAXWindowsAttribute as String) else {
        return nil
@ -3168,7 +3185,11 @@ private func isTrustedOrcaApplication(_ pid: pid_t) -> Bool {
    else {
        return false
    }
-    return bundleId == "com.stablyai.orca" || bundleId == "com.github.Electron"
+    // Why: dev validation runs from per-worktree wrapper apps with stable
+    // Orca-owned bundle ids; the sidecar peer check must still authorize them.
+    return bundleId == "com.stablyai.orca" ||
+        bundleId.hasPrefix("com.stablyai.orca.dev.") ||
+        bundleId == "com.github.Electron"
 }

 private func parentProcessId(_ pid: pid_t) -> pid_t? {
--- a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/KeyboardInputSafety.swift
+++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/KeyboardInputSafety.swift
@ -0,0 +1,13 @@
+public enum KeyboardInputSafety {
+    public enum FocusFailure: Equatable {
+        case targetNotFocused
+        case targetNotFocusedAfterRestore
+    }
+
+    public static func syntheticInputFocusFailure(targetWindowFocused: Bool, restoreWindowRequested: Bool) -> FocusFailure? {
+        guard !targetWindowFocused else {
+            return nil
+        }
+        return restoreWindowRequested ? .targetNotFocusedAfterRestore : .targetNotFocused
+    }
+}
--- a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/KeyboardInputSafetyTests.swift
+++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/KeyboardInputSafetyTests.swift
@ -0,0 +1,23 @@
+import XCTest
+@testable import OrcaComputerUseMacOSCore
+
+final class KeyboardInputSafetyTests: XCTestCase {
+    func testSyntheticInputRequiresFocusedTargetWindow() {
+        let cases: [(focused: Bool, restoreWindow: Bool, expectedFailure: KeyboardInputSafety.FocusFailure?)] = [
+            (focused: true, restoreWindow: false, expectedFailure: nil),
+            (focused: true, restoreWindow: true, expectedFailure: nil),
+            (focused: false, restoreWindow: false, expectedFailure: .targetNotFocused),
+            (focused: false, restoreWindow: true, expectedFailure: .targetNotFocusedAfterRestore),
+        ]
+
+        for testCase in cases {
+            XCTAssertEqual(
+                KeyboardInputSafety.syntheticInputFocusFailure(
+                    targetWindowFocused: testCase.focused,
+                    restoreWindowRequested: testCase.restoreWindow
+                ),
+                testCase.expectedFailure
+            )
+        }
+    }
+}
--- a/tests/e2e/computer-mac.e2e.ts
+++ b/tests/e2e/computer-mac.e2e.ts
@ -5,6 +5,7 @@ import type {
  ComputerSnapshotResult
 } from '../../src/shared/runtime-types'
 import {
+  activateFinder,
  ensureTextEditLaunched,
  findRoleIndex,
  killTextEdit,
@ -141,6 +142,103 @@ describe.skipIf(!isMac || !e2eOptIn)('computer-use macOS e2e (TextEdit)', () =>
    expect(after.result.snapshot.treeText).not.toContain('orca paste first')
  })

+  test('accessibility text actions work when TextEdit is not frontmost', async () => {
+    const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'get-app-state',
+          '--app',
+          'TextEdit',
+          '--restore-window',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    const textTarget = findRoleIndex(
+      before.result.snapshot.treeText,
+      /^\s*(\d+)\s+(text entry area|text field|HTML content)(?:\s|$)/m
+    )
+    expect(textTarget).toBeGreaterThanOrEqual(0)
+
+    await runOrcaCli([
+      'computer',
+      'click',
+      '--app',
+      'TextEdit',
+      '--element-index',
+      String(textTarget),
+      '--restore-window',
+      '--no-screenshot'
+    ])
+
+    await runOrcaCli([
+      'computer',
+      'paste-text',
+      '--app',
+      'TextEdit',
+      '--text',
+      'orca unfocused first',
+      '--no-screenshot'
+    ])
+    await activateFinder()
+
+    const selectAll = parseJsonOutput<{ result: ComputerActionResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'hotkey',
+          '--app',
+          'TextEdit',
+          '--key',
+          'CmdOrCtrl+A',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(selectAll.result.action?.actionName).toBe('AXSelectAll')
+    expect(selectAll.result.action?.verification?.state).toBe('verified')
+
+    const marker = `orca unfocused final ${Date.now()}`
+    const replacement = parseJsonOutput<{ result: ComputerActionResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'paste-text',
+          '--app',
+          'TextEdit',
+          '--text',
+          marker,
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(replacement.result.action?.actionName).toBe('AXReplaceSelection')
+    expect(replacement.result.action?.verification).toMatchObject({
+      state: 'verified',
+      property: 'focusedText',
+      expected: marker
+    })
+
+    const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'get-app-state',
+          '--app',
+          'TextEdit',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(after.result.snapshot.treeText).toContain(marker)
+    expect(after.result.snapshot.treeText).not.toContain('orca unfocused first')
+  })
+
  test('screenshot capture returns image metadata', async () => {
    const result = await runOrcaCli(['computer', 'get-app-state', '--app', 'TextEdit', '--json'])
    const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout)
--- a/tests/e2e/helpers/computer-driver.ts
+++ b/tests/e2e/helpers/computer-driver.ts
@ -62,6 +62,11 @@ export async function killTextEdit(): Promise<void> {
  }
 }

+export async function activateFinder(): Promise<void> {
+  await execFileAsync('open', ['-a', 'Finder'])
+  await delay(1000)
+}
+
 export async function ensureGeditLaunched(): Promise<void> {
  await killGedit()
  linuxTempDir = await mkdtemp(join(tmpdir(), 'orca-computer-linux-e2e-'))