From 09872bfe7bffe7fb2df86b31bc60b045070016be Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Wed, 3 Jun 2026 16:00:34 -0700 Subject: [PATCH] Fix macOS keyboard actions targeting the wrong app (#3960) * fix: guard macos synthetic input focus * Tighten macOS keyboard focus safety Co-authored-by: Orca --------- Co-authored-by: Jinwoo-H Co-authored-by: Orca --- .../Sources/OrcaComputerUseMacOS/main.swift | 33 +++++-- .../KeyboardInputSafety.swift | 13 +++ .../KeyboardInputSafetyTests.swift | 23 +++++ tests/e2e/computer-mac.e2e.ts | 98 +++++++++++++++++++ tests/e2e/helpers/computer-driver.ts | 5 + 5 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/KeyboardInputSafety.swift create mode 100644 native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/KeyboardInputSafetyTests.swift diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 1094fc9e5..ac15bf064 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -273,11 +273,9 @@ final class Provider { } private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot { - let snapshot = try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }) - if params["restoreWindow"]?.bool != true && !isTargetWindowFocused(snapshot) { - throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; retry with --restore-window or use set-value for editable elements") - } - return snapshot + // Why: AX text replacement/select-all do not post global input, so only + // synthetic fallback paths require the target window to be focused. + try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }) } private func cachedSnapshot(params: [String: JSONValue]) throws -> Snapshot? { @@ -671,12 +669,14 @@ final class Provider { private func typeText(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) + try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true) try Input.typeText(try requiredString(params, "text"), pid: snapshot.app.pid) return actionMetadata(path: "synthetic") } private func pressKey(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) + try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true) try Input.pressKey(try requiredString(params, "key"), pid: snapshot.app.pid) return actionMetadata(path: "synthetic") } @@ -691,6 +691,7 @@ final class Provider { verification: TextInput.selectionVerification(focused.element) ) } + try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true) try Input.pressKey(key, pid: snapshot.app.pid) return actionMetadata( path: "synthetic", @@ -705,6 +706,7 @@ final class Provider { if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } + try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true) try Input.pasteText(text, pid: snapshot.app.pid) return actionMetadata( path: "clipboard", @@ -936,6 +938,21 @@ private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool { return !intersection.isNull && intersection.area >= min(frame.area, snapshot.windowBounds.area) * 0.75 } +private func requireTargetWindowFocused(_ snapshot: Snapshot, restoreWindowRequested: Bool) throws { + guard let failure = KeyboardInputSafety.syntheticInputFocusFailure( + targetWindowFocused: isTargetWindowFocused(snapshot), + restoreWindowRequested: restoreWindowRequested + ) else { + return + } + switch failure { + case .targetNotFocused: + throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; retry with --restore-window or use set-value for editable elements") + case .targetNotFocusedAfterRestore: + throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; --restore-window was requested but the target is still not focused; bring it forward manually or check Accessibility permissions") + } +} + private func matchingWindow(appElement: AXUIElement, capture: WindowCapture, focused: AXUIElement, explicitTarget: Bool) -> AXUIElement? { guard let windows = copyArray(appElement, kAXWindowsAttribute as String) else { return nil @@ -3168,7 +3185,11 @@ private func isTrustedOrcaApplication(_ pid: pid_t) -> Bool { else { return false } - return bundleId == "com.stablyai.orca" || bundleId == "com.github.Electron" + // Why: dev validation runs from per-worktree wrapper apps with stable + // Orca-owned bundle ids; the sidecar peer check must still authorize them. + return bundleId == "com.stablyai.orca" || + bundleId.hasPrefix("com.stablyai.orca.dev.") || + bundleId == "com.github.Electron" } private func parentProcessId(_ pid: pid_t) -> pid_t? { diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/KeyboardInputSafety.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/KeyboardInputSafety.swift new file mode 100644 index 000000000..d739ce436 --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/KeyboardInputSafety.swift @@ -0,0 +1,13 @@ +public enum KeyboardInputSafety { + public enum FocusFailure: Equatable { + case targetNotFocused + case targetNotFocusedAfterRestore + } + + public static func syntheticInputFocusFailure(targetWindowFocused: Bool, restoreWindowRequested: Bool) -> FocusFailure? { + guard !targetWindowFocused else { + return nil + } + return restoreWindowRequested ? .targetNotFocusedAfterRestore : .targetNotFocused + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/KeyboardInputSafetyTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/KeyboardInputSafetyTests.swift new file mode 100644 index 000000000..425c8453b --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/KeyboardInputSafetyTests.swift @@ -0,0 +1,23 @@ +import XCTest +@testable import OrcaComputerUseMacOSCore + +final class KeyboardInputSafetyTests: XCTestCase { + func testSyntheticInputRequiresFocusedTargetWindow() { + let cases: [(focused: Bool, restoreWindow: Bool, expectedFailure: KeyboardInputSafety.FocusFailure?)] = [ + (focused: true, restoreWindow: false, expectedFailure: nil), + (focused: true, restoreWindow: true, expectedFailure: nil), + (focused: false, restoreWindow: false, expectedFailure: .targetNotFocused), + (focused: false, restoreWindow: true, expectedFailure: .targetNotFocusedAfterRestore), + ] + + for testCase in cases { + XCTAssertEqual( + KeyboardInputSafety.syntheticInputFocusFailure( + targetWindowFocused: testCase.focused, + restoreWindowRequested: testCase.restoreWindow + ), + testCase.expectedFailure + ) + } + } +} diff --git a/tests/e2e/computer-mac.e2e.ts b/tests/e2e/computer-mac.e2e.ts index 46f54f785..2b100905f 100644 --- a/tests/e2e/computer-mac.e2e.ts +++ b/tests/e2e/computer-mac.e2e.ts @@ -5,6 +5,7 @@ import type { ComputerSnapshotResult } from '../../src/shared/runtime-types' import { + activateFinder, ensureTextEditLaunched, findRoleIndex, killTextEdit, @@ -141,6 +142,103 @@ describe.skipIf(!isMac || !e2eOptIn)('computer-use macOS e2e (TextEdit)', () => expect(after.result.snapshot.treeText).not.toContain('orca paste first') }) + test('accessibility text actions work when TextEdit is not frontmost', async () => { + const before = parseJsonOutput<{ result: ComputerSnapshotResult }>( + ( + await runOrcaCli([ + 'computer', + 'get-app-state', + '--app', + 'TextEdit', + '--restore-window', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + const textTarget = findRoleIndex( + before.result.snapshot.treeText, + /^\s*(\d+)\s+(text entry area|text field|HTML content)(?:\s|$)/m + ) + expect(textTarget).toBeGreaterThanOrEqual(0) + + await runOrcaCli([ + 'computer', + 'click', + '--app', + 'TextEdit', + '--element-index', + String(textTarget), + '--restore-window', + '--no-screenshot' + ]) + + await runOrcaCli([ + 'computer', + 'paste-text', + '--app', + 'TextEdit', + '--text', + 'orca unfocused first', + '--no-screenshot' + ]) + await activateFinder() + + const selectAll = parseJsonOutput<{ result: ComputerActionResult }>( + ( + await runOrcaCli([ + 'computer', + 'hotkey', + '--app', + 'TextEdit', + '--key', + 'CmdOrCtrl+A', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + expect(selectAll.result.action?.actionName).toBe('AXSelectAll') + expect(selectAll.result.action?.verification?.state).toBe('verified') + + const marker = `orca unfocused final ${Date.now()}` + const replacement = parseJsonOutput<{ result: ComputerActionResult }>( + ( + await runOrcaCli([ + 'computer', + 'paste-text', + '--app', + 'TextEdit', + '--text', + marker, + '--no-screenshot', + '--json' + ]) + ).stdout + ) + expect(replacement.result.action?.actionName).toBe('AXReplaceSelection') + expect(replacement.result.action?.verification).toMatchObject({ + state: 'verified', + property: 'focusedText', + expected: marker + }) + + const after = parseJsonOutput<{ result: ComputerSnapshotResult }>( + ( + await runOrcaCli([ + 'computer', + 'get-app-state', + '--app', + 'TextEdit', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + expect(after.result.snapshot.treeText).toContain(marker) + expect(after.result.snapshot.treeText).not.toContain('orca unfocused first') + }) + test('screenshot capture returns image metadata', async () => { const result = await runOrcaCli(['computer', 'get-app-state', '--app', 'TextEdit', '--json']) const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout) diff --git a/tests/e2e/helpers/computer-driver.ts b/tests/e2e/helpers/computer-driver.ts index 15c6cd3a9..83c4b3b36 100644 --- a/tests/e2e/helpers/computer-driver.ts +++ b/tests/e2e/helpers/computer-driver.ts @@ -62,6 +62,11 @@ export async function killTextEdit(): Promise { } } +export async function activateFinder(): Promise { + await execFileAsync('open', ['-a', 'Finder']) + await delay(1000) +} + export async function ensureGeditLaunched(): Promise { await killGedit() linuxTempDir = await mkdtemp(join(tmpdir(), 'orca-computer-linux-e2e-'))