From abcd2d58362bd9846ad87f55290747b2b53d2ade Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Wed, 13 May 2026 15:37:47 -0400 Subject: [PATCH] fix: harden Windows computer use runtime (#1757) --- native/computer-use-windows/runtime.ps1 | 196 +++++++++++++-------- src/cli/handlers/computer.test.ts | 6 +- tests/e2e/computer-windows-store.e2e.ts | 96 ++++++++++ tests/e2e/computer-windows.e2e.ts | 222 +++++++++++++++++++++++- tests/e2e/helpers/computer-driver.ts | 73 +++++++- 5 files changed, 502 insertions(+), 91 deletions(-) create mode 100644 tests/e2e/computer-windows-store.e2e.ts diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1 index f87daa87e..4e3020bdf 100644 --- a/native/computer-use-windows/runtime.ps1 +++ b/native/computer-use-windows/runtime.ps1 @@ -4,6 +4,10 @@ param( ) $ErrorActionPreference = "Stop" +$utf8NoBom = New-Object System.Text.UTF8Encoding $false +[Console]::InputEncoding = $utf8NoBom +[Console]::OutputEncoding = $utf8NoBom +$OutputEncoding = $utf8NoBom Add-Type -AssemblyName UIAutomationClient Add-Type -AssemblyName UIAutomationTypes @@ -46,6 +50,12 @@ public static class OrcaDesktopWin32 { [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + + [DllImport("user32.dll")] + public static extern bool SetCursorPos(int x, int y); + + [DllImport("user32.dll")] + public static extern void mouse_event(uint dwFlags, uint dx, uint dy, int dwData, UIntPtr dwExtraInfo); } "@ @@ -75,6 +85,16 @@ $WindowsMessages = @{ Wheel = 0x020A } +$MouseEvents = @{ + LeftDown = 0x0002 + LeftUp = 0x0004 + RightDown = 0x0008 + RightUp = 0x0010 + MiddleDown = 0x0020 + MiddleUp = 0x0040 + Wheel = 0x0800 +} + function Write-OrcaJson($Payload) { $Payload | ConvertTo-Json -Depth 100 -Compress } @@ -85,7 +105,7 @@ function New-OrcaFrame([double]$X, [double]$Y, [double]$Width, [double]$Height) } function Read-OrcaOperation([string]$Path) { - Get-Content -Raw -Path $Path | ConvertFrom-Json + Get-Content -Raw -Encoding UTF8 -Path $Path | ConvertFrom-Json } function ConvertTo-OrcaLParam([int]$X, [int]$Y) { @@ -108,10 +128,10 @@ function Find-OrcaProcess([string]$Query) { $needle = $needle.Substring(4) } - $pid = 0 + $parsedProcessId = 0 $processes = Get-OrcaWindowProcesses - if ([int]::TryParse($needle, [ref]$pid)) { - $match = $processes | Where-Object { $_.Id -eq $pid } | Select-Object -First 1 + if ([int]::TryParse($needle, [ref]$parsedProcessId)) { + $match = $processes | Where-Object { $_.Id -eq $parsedProcessId } | Select-Object -First 1 if ($null -ne $match) { Assert-OrcaProcessAllowed $match return $match @@ -174,6 +194,22 @@ function Get-OrcaWindowId($Process) { [int64]$Process.MainWindowHandle } +function Get-OrcaAppName($Process) { + if ($Process.ProcessName -eq "ApplicationFrameHost" -and -not [string]::IsNullOrWhiteSpace($Process.MainWindowTitle)) { + return [string]$Process.MainWindowTitle + } + [string]$Process.ProcessName +} + +function New-OrcaAppRecord($Process) { + [pscustomobject]@{ + name = Get-OrcaAppName $Process + bundleIdentifier = $Process.ProcessName + bundleId = $Process.ProcessName + pid = [int]$Process.Id + } +} + function Assert-OrcaWindowTarget($Process, $WindowId, $WindowIndex) { if ($null -ne $WindowIndex -and [int]$WindowIndex -ne 0) { throw "windowNotFound(`"$WindowIndex`")" @@ -446,12 +482,7 @@ function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = [pscustomobject]@{ snapshotId = [guid]::NewGuid().ToString() - app = [pscustomobject]@{ - name = $process.ProcessName - bundleIdentifier = $process.ProcessName - bundleId = $process.ProcessName - pid = [int]$process.Id - } + app = New-OrcaAppRecord $process windowTitle = $process.MainWindowTitle windowId = Get-OrcaWindowId $process windowBounds = $windowFrame @@ -468,12 +499,7 @@ function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId = function Get-OrcaAppList { @(Get-OrcaWindowProcesses | ForEach-Object { - [pscustomobject]@{ - name = $_.ProcessName - bundleIdentifier = $_.ProcessName - bundleId = $_.ProcessName - pid = [int]$_.Id - } + New-OrcaAppRecord $_ }) } @@ -491,12 +517,7 @@ function Get-OrcaWindowList([string]$Query) { $width = [int][Math]::Max(0, [Math]::Round($windowFrame.width)) $height = [int][Math]::Max(0, [Math]::Round($windowFrame.height)) } - $app = [pscustomobject]@{ - name = $process.ProcessName - bundleIdentifier = $process.ProcessName - bundleId = $process.ProcessName - pid = [int]$process.Id - } + $app = New-OrcaAppRecord $process [pscustomobject]@{ app = $app windows = @([pscustomobject]@{ @@ -643,60 +664,45 @@ function Get-OrcaElementScreenPoint($Element) { } function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count) { - $point = New-Object OrcaDesktopWin32+POINT - $point.X = $ScreenX - $point.Y = $ScreenY - [void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$point) - - $down = $WindowsMessages.LeftDown - $up = $WindowsMessages.LeftUp - $flag = 1 + [void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle) + [void][OrcaDesktopWin32]::SetCursorPos($ScreenX, $ScreenY) + $down = $MouseEvents.LeftDown + $up = $MouseEvents.LeftUp if ($Button -eq "right") { - $down = $WindowsMessages.RightDown - $up = $WindowsMessages.RightUp - $flag = 2 + $down = $MouseEvents.RightDown + $up = $MouseEvents.RightUp } elseif ($Button -eq "middle") { - $down = $WindowsMessages.MiddleDown - $up = $WindowsMessages.MiddleUp - $flag = 16 + $down = $MouseEvents.MiddleDown + $up = $MouseEvents.MiddleUp } - $position = ConvertTo-OrcaLParam $point.X $point.Y for ($i = 0; $i -lt [Math]::Max(1, $Count); $i++) { - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]::Zero, $position) - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $down, [IntPtr]$flag, $position) + [OrcaDesktopWin32]::mouse_event($down, 0, 0, 0, [UIntPtr]::Zero) Start-Sleep -Milliseconds 35 - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $up, [IntPtr]::Zero, $position) + [OrcaDesktopWin32]::mouse_event($up, 0, 0, 0, [UIntPtr]::Zero) } } function Send-OrcaDrag([IntPtr]$WindowHandle, $From, $To) { - $start = New-Object OrcaDesktopWin32+POINT - $start.X = [int]$From.x - $start.Y = [int]$From.y - [void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$start) - - $end = New-Object OrcaDesktopWin32+POINT - $end.X = [int]$To.x - $end.Y = [int]$To.y - [void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$end) - - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]::Zero, (ConvertTo-OrcaLParam $start.X $start.Y)) - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.LeftDown, [IntPtr]1, (ConvertTo-OrcaLParam $start.X $start.Y)) + [void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle) + $startX = [int]$From.x + $startY = [int]$From.y + $endX = [int]$To.x + $endY = [int]$To.y + [void][OrcaDesktopWin32]::SetCursorPos($startX, $startY) + [OrcaDesktopWin32]::mouse_event($MouseEvents.LeftDown, 0, 0, 0, [UIntPtr]::Zero) for ($step = 1; $step -le 12; $step++) { - $x = [int][Math]::Round($start.X + (($end.X - $start.X) * $step / 12)) - $y = [int][Math]::Round($start.Y + (($end.Y - $start.Y) * $step / 12)) - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]1, (ConvertTo-OrcaLParam $x $y)) + $x = [int][Math]::Round($startX + (($endX - $startX) * $step / 12)) + $y = [int][Math]::Round($startY + (($endY - $startY) * $step / 12)) + [void][OrcaDesktopWin32]::SetCursorPos($x, $y) Start-Sleep -Milliseconds 20 } - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.LeftUp, [IntPtr]::Zero, (ConvertTo-OrcaLParam $end.X $end.Y)) + [OrcaDesktopWin32]::mouse_event($MouseEvents.LeftUp, 0, 0, 0, [UIntPtr]::Zero) } function Send-OrcaText([IntPtr]$WindowHandle, [string]$Text) { - foreach ($character in $Text.ToCharArray()) { - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.Char, [IntPtr][int][char]$character, [IntPtr]::Zero) - Start-Sleep -Milliseconds 8 - } + [void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle) + [System.Windows.Forms.SendKeys]::SendWait((ConvertTo-OrcaSendKeysText $Text)) } function Get-OrcaVirtualKey([string]$Key) { @@ -712,10 +718,8 @@ function Get-OrcaVirtualKey([string]$Key) { } function Send-OrcaKey([IntPtr]$WindowHandle, [string]$Key) { - $virtualKey = Get-OrcaVirtualKey $Key - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyDown, [IntPtr]$virtualKey, [IntPtr]::Zero) - Start-Sleep -Milliseconds 25 - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyUp, [IntPtr]$virtualKey, [IntPtr]::Zero) + [void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle) + [System.Windows.Forms.SendKeys]::SendWait((ConvertTo-OrcaSendKeysKey $Key)) } function Get-OrcaModifierVirtualKey([string]$Modifier) { @@ -732,16 +736,58 @@ function Send-OrcaHotkey([IntPtr]$WindowHandle, [string]$KeySpec) { $parts = @($KeySpec.Split("+") | ForEach-Object { $_.Trim() } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) if ($parts.Count -eq 0) { throw "Unsupported key: $KeySpec" } $key = $parts[$parts.Count - 1] - $modifiers = @() + $prefix = "" if ($parts.Count -gt 1) { - $modifiers = @($parts[0..($parts.Count - 2)] | ForEach-Object { Get-OrcaModifierVirtualKey $_ }) + foreach ($modifier in $parts[0..($parts.Count - 2)]) { + $prefix += ConvertTo-OrcaSendKeysModifier $modifier + } } - foreach ($modifier in $modifiers) { - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyDown, [IntPtr]$modifier, [IntPtr]::Zero) + [void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle) + [System.Windows.Forms.SendKeys]::SendWait($prefix + (ConvertTo-OrcaSendKeysKey $key)) +} + +function ConvertTo-OrcaSendKeysText([string]$Text) { + $builder = New-Object System.Text.StringBuilder + foreach ($character in $Text.ToCharArray()) { + $value = [string]$character + if ($value -eq "`r") { continue } + if ($value -eq "`n") { [void]$builder.Append("{ENTER}"); continue } + if ("+^%~(){}[]".Contains($value)) { + [void]$builder.Append("{").Append($value).Append("}") + } else { + [void]$builder.Append($value) + } } - Send-OrcaKey $WindowHandle $key - for ($i = $modifiers.Count - 1; $i -ge 0; $i--) { - [void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyUp, [IntPtr]$modifiers[$i], [IntPtr]::Zero) + $builder.ToString() +} + +function ConvertTo-OrcaSendKeysKey([string]$Key) { + switch ($Key.ToLowerInvariant()) { + { $_ -in @("return", "enter") } { return "{ENTER}" } + "tab" { return "{TAB}" } + { $_ -in @("escape", "esc") } { return "{ESC}" } + "backspace" { return "{BACKSPACE}" } + "delete" { return "{DELETE}" } + "space" { return " " } + "left" { return "{LEFT}" } + "up" { return "{UP}" } + "right" { return "{RIGHT}" } + "down" { return "{DOWN}" } + "home" { return "{HOME}" } + "end" { return "{END}" } + default { + if ($Key.Length -eq 1) { return (ConvertTo-OrcaSendKeysText $Key) } + throw "Unsupported key: $Key" + } + } +} + +function ConvertTo-OrcaSendKeysModifier([string]$Modifier) { + switch ($Modifier.ToLowerInvariant()) { + { $_ -in @("ctrl", "control", "cmdorctrl", "commandorcontrol") } { return "^" } + "shift" { return "+" } + { $_ -in @("alt", "option") } { return "%" } + default { throw "Unsupported modifier: $Modifier" } } } @@ -755,9 +801,9 @@ function Send-OrcaPasteText([IntPtr]$WindowHandle, [string]$Text) { Send-OrcaHotkey $WindowHandle "Ctrl+v" } finally { if ($hadPrevious) { - [System.Windows.Forms.Clipboard]::SetDataObject($previous, $true) + try { [System.Windows.Forms.Clipboard]::SetDataObject($previous, $true) } catch {} } else { - [System.Windows.Forms.Clipboard]::Clear() + try { [System.Windows.Forms.Clipboard]::Clear() } catch {} } } } @@ -819,7 +865,9 @@ function Invoke-OrcaOperation($Operation) { if ($Operation.direction -eq "down" -or $Operation.direction -eq "right") { $delta = -1 * $delta } $point = Get-OrcaElementScreenPoint $element if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame } - [void][OrcaDesktopWin32]::PostMessage($handle, $WindowsMessages.Wheel, (ConvertTo-OrcaWheelParam $delta), (ConvertTo-OrcaLParam $point.x $point.y)) + [void][OrcaDesktopWin32]::SetForegroundWindow($handle) + [void][OrcaDesktopWin32]::SetCursorPos([int]$point.x, [int]$point.y) + [OrcaDesktopWin32]::mouse_event($MouseEvents.Wheel, 0, 0, $delta, [UIntPtr]::Zero) $action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $null } } "drag" { diff --git a/src/cli/handlers/computer.test.ts b/src/cli/handlers/computer.test.ts index d7e8d07ed..dca8f050b 100644 --- a/src/cli/handlers/computer.test.ts +++ b/src/cli/handlers/computer.test.ts @@ -364,9 +364,11 @@ describe('orca computer CLI handlers', () => { expect(parsed.result.screenshot).toMatchObject({ dataOmitted: true, format: 'png', - expiresAt: expect.any(String), - path: expect.stringContaining('orca-computer-use/req_state-screenshot.png') + expiresAt: expect.any(String) }) + expect(String(parsed.result.screenshot.path).replaceAll('\\', '/')).toContain( + 'orca-computer-use/req_state-screenshot.png' + ) }) it('shows coordinate space and truncation in pretty state output', async () => { diff --git a/tests/e2e/computer-windows-store.e2e.ts b/tests/e2e/computer-windows-store.e2e.ts new file mode 100644 index 000000000..7c57863ac --- /dev/null +++ b/tests/e2e/computer-windows-store.e2e.ts @@ -0,0 +1,96 @@ +import { describe, expect, test } from 'vitest' +import type { ComputerListAppsResult, ComputerSnapshotResult } from '../../src/shared/runtime-types' +import { findRoleIndex, parseJsonOutput, runOrcaCli } from './helpers/computer-driver' + +const isWindows = process.platform === 'win32' +const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1' + +describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Store apps)', () => { + test('Store app windows are discoverable by title and clickable', async () => { + await launchCalculator() + try { + const apps = parseJsonOutput<{ result: ComputerListAppsResult }>( + (await runOrcaCli(['computer', 'list-apps', '--json'])).stdout + ) + expect(apps.result.apps).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'Calculator', bundleId: 'ApplicationFrameHost' }) + ]) + ) + + let state = parseJsonOutput<{ result: ComputerSnapshotResult }>( + ( + await runOrcaCli([ + 'computer', + 'get-app-state', + '--app', + 'Calculator', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + const one = findRoleIndex(state.result.snapshot.treeText, 'button One') + const plus = findRoleIndex(state.result.snapshot.treeText, 'button Plus') + const two = findRoleIndex(state.result.snapshot.treeText, 'button Two') + const equals = findRoleIndex(state.result.snapshot.treeText, 'button Equals') + expect([one, plus, two, equals].every((index) => index >= 0)).toBe(true) + + for (const index of [one, plus, two, equals]) { + state = parseJsonOutput<{ result: ComputerSnapshotResult }>( + ( + await runOrcaCli([ + 'computer', + 'click', + '--app', + 'Calculator', + '--element-index', + String(index), + '--no-screenshot', + '--json' + ]) + ).stdout + ) + } + expect(state.result.snapshot.treeText).toMatch(/Display is 3\b/) + } finally { + await killCalculator() + } + }) +}) + +async function launchCalculator(): Promise { + await runPowerShell('Start-Process calc.exe') + await runPowerShell( + [ + '$deadline = (Get-Date).AddSeconds(15)', + '$target = $null', + 'while ((Get-Date) -lt $deadline -and $null -eq $target) {', + ' Start-Sleep -Milliseconds 250', + ' $target = Get-Process |', + ' Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -eq "Calculator" } |', + ' Select-Object -First 1', + '}', + 'if ($null -eq $target) { throw "No visible Calculator window found" }' + ].join('\n') + ) +} + +async function killCalculator(): Promise { + await runPowerShell( + 'Get-Process CalculatorApp -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue' + ) +} + +async function runPowerShell(script: string): Promise { + const { execFile } = await import('child_process') + await new Promise((resolve, reject) => { + execFile('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], (error) => { + if (error) { + reject(error) + return + } + resolve() + }) + }) +} diff --git a/tests/e2e/computer-windows.e2e.ts b/tests/e2e/computer-windows.e2e.ts index 09086e327..3f344f8ed 100644 --- a/tests/e2e/computer-windows.e2e.ts +++ b/tests/e2e/computer-windows.e2e.ts @@ -1,7 +1,14 @@ import { afterAll, beforeAll, describe, expect, test } from 'vitest' -import type { ComputerActionResult, ComputerSnapshotResult } from '../../src/shared/runtime-types' +import type { + ComputerActionResult, + ComputerListAppsResult, + ComputerListWindowsResult, + ComputerSnapshotResult +} from '../../src/shared/runtime-types' import { ensureNotepadLaunched, + findRoleIndex, + getNotepadAppSelector, killNotepad, parseJsonOutput, runOrcaCli @@ -19,18 +26,88 @@ describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', ( await killNotepad() }) + test('list-apps includes the test-owned Notepad process', async () => { + const result = await runOrcaCli(['computer', 'list-apps', '--json']) + const envelope = parseJsonOutput<{ result: ComputerListAppsResult }>(result.stdout) + const pid = Number.parseInt(getNotepadAppSelector().slice(4), 10) + + expect(envelope.result.apps).toEqual( + expect.arrayContaining([expect.objectContaining({ name: 'Notepad', pid })]) + ) + }) + + test('list-windows returns a targetable Notepad window', async () => { + const app = getNotepadAppSelector() + const result = await runOrcaCli(['computer', 'list-windows', '--app', app, '--json']) + const envelope = parseJsonOutput<{ result: ComputerListWindowsResult }>(result.stdout) + + expect(envelope.result.windows).toEqual([ + expect.objectContaining({ + index: 0, + app: expect.objectContaining({ name: 'Notepad' }), + id: expect.any(Number), + title: expect.stringContaining('Notepad'), + width: expect.any(Number), + height: expect.any(Number) + }) + ]) + }) + test('Notepad exposes a basic accessibility tree', async () => { - const result = await runOrcaCli(['computer', 'get-app-state', '--app', 'Notepad', '--json']) + const result = await runOrcaCli([ + 'computer', + 'get-app-state', + '--app', + getNotepadAppSelector(), + '--json' + ]) const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout) + expect(envelope.result.snapshot.app.name).toBe('Notepad') + expect(envelope.result.snapshot.window.title).toContain('Notepad') expect(envelope.result.snapshot.elementCount).toBeGreaterThan(0) expect(envelope.result.snapshot.coordinateSpace).toBe('window') expect(envelope.result.snapshot.truncation?.truncated).toBe(false) + expect(envelope.result.screenshotStatus.state).toBe('captured') + expect(envelope.result.screenshot?.format).toBe('png') expect(envelope.result.screenshot?.data).toBeUndefined() - expect(envelope.result.screenshot?.path).toContain('orca-computer-use-') + expect(envelope.result.screenshot?.dataOmitted).toBe(true) + expect(envelope.result.screenshot?.path).toContain('orca-computer-use') + }) + + test('set-value mutates the document through UI Automation', async () => { + const app = getNotepadAppSelector() + const before = parseJsonOutput<{ result: ComputerSnapshotResult }>( + (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json'])) + .stdout + ) + const documentIndex = findRoleIndex(before.result.snapshot.treeText, 'document') + expect(documentIndex).toBeGreaterThanOrEqual(0) + + const marker = `orca-windows-set-${Date.now()}` + const action = parseJsonOutput<{ result: ComputerActionResult }>( + ( + await runOrcaCli([ + 'computer', + 'set-value', + '--app', + app, + '--element-index', + String(documentIndex), + '--value', + marker, + '--no-screenshot', + '--json' + ]) + ).stdout + ) + expect(action.result.action?.path).toBe('accessibility') + + expect(action.result.snapshot.treeText).toContain(marker) }) test('paste-text mutates the test-owned document', async () => { + const app = getNotepadAppSelector() const marker = `orca-windows-paste-${Date.now()}` const action = parseJsonOutput<{ result: ComputerActionResult }>( ( @@ -38,7 +115,7 @@ describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', ( 'computer', 'paste-text', '--app', - 'Notepad', + app, '--text', marker, '--restore-window', @@ -50,17 +127,148 @@ describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', ( expect(action.result.action?.path).toBe('clipboard') const after = parseJsonOutput<{ result: ComputerSnapshotResult }>( + (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json'])) + .stdout + ) + expect(after.result.snapshot.treeText).toContain(marker) + }) + + test('Unicode payloads survive set-value and paste-text', async () => { + const app = getNotepadAppSelector() + const before = parseJsonOutput<{ result: ComputerSnapshotResult }>( + (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json'])) + .stdout + ) + const documentIndex = findRoleIndex(before.result.snapshot.treeText, 'document') + expect(documentIndex).toBeGreaterThanOrEqual(0) + + const unicode = `orca unicode café Ω 漢字 ${Date.now()}` + const set = parseJsonOutput<{ result: ComputerActionResult }>( ( await runOrcaCli([ 'computer', - 'get-app-state', + 'set-value', '--app', - 'Notepad', + app, + '--element-index', + String(documentIndex), + '--value', + unicode, '--no-screenshot', '--json' ]) ).stdout ) - expect(after.result.snapshot.treeText).toContain(marker) + expect(set.result.snapshot.treeText).toContain(unicode) + + const pasted = parseJsonOutput<{ result: ComputerActionResult }>( + ( + await runOrcaCli([ + 'computer', + 'paste-text', + '--app', + app, + '--text', + unicode, + '--restore-window', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + expect(pasted.result.snapshot.treeText).toContain(unicode) + }) + + test('hotkey and paste-text can replace the document selection', async () => { + const app = getNotepadAppSelector() + const first = `orca-windows-first-${Date.now()}` + await runOrcaCli([ + 'computer', + 'paste-text', + '--app', + app, + '--text', + first, + '--restore-window', + '--no-screenshot', + '--json' + ]) + + const selectAll = parseJsonOutput<{ result: ComputerActionResult }>( + ( + await runOrcaCli([ + 'computer', + 'hotkey', + '--app', + app, + '--key', + 'CmdOrCtrl+A', + '--restore-window', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + expect(selectAll.result.action?.actionName).toBe('hotkey') + + const marker = `orca-windows-replaced-${Date.now()}` + const second = parseJsonOutput<{ result: ComputerActionResult }>( + ( + await runOrcaCli([ + 'computer', + 'paste-text', + '--app', + app, + '--text', + marker, + '--restore-window', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + expect(second.result.snapshot.treeText).toContain(marker) + expect(second.result.snapshot.treeText).not.toContain(first) + }) + + test('click and type-text send synthetic input to the document', async () => { + const app = getNotepadAppSelector() + const before = parseJsonOutput<{ result: ComputerSnapshotResult }>( + (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json'])) + .stdout + ) + const documentIndex = findRoleIndex(before.result.snapshot.treeText, 'document') + expect(documentIndex).toBeGreaterThanOrEqual(0) + + await runOrcaCli([ + 'computer', + 'click', + '--app', + app, + '--element-index', + String(documentIndex), + '--restore-window', + '--no-screenshot', + '--json' + ]) + + const marker = ` typed-${Date.now()}` + const typed = parseJsonOutput<{ result: ComputerActionResult }>( + ( + await runOrcaCli([ + 'computer', + 'type-text', + '--app', + app, + '--text', + marker, + '--restore-window', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + expect(typed.result.action?.path).toBe('synthetic') + expect(typed.result.snapshot.treeText).toContain(marker.trim()) }) }) diff --git a/tests/e2e/helpers/computer-driver.ts b/tests/e2e/helpers/computer-driver.ts index fcddfb499..15c6cd3a9 100644 --- a/tests/e2e/helpers/computer-driver.ts +++ b/tests/e2e/helpers/computer-driver.ts @@ -10,6 +10,7 @@ let linuxTempDir: string | null = null let windowsTempDir: string | null = null let geditProcess: ChildProcess | null = null let notepadProcess: ChildProcess | null = null +let notepadAppSelector: string | null = null export type CliResult = { stdout: string @@ -18,8 +19,14 @@ export type CliResult = { export async function runOrcaCli(args: string[]): Promise { const devCli = join(process.cwd(), 'config/scripts/orca-dev') - const command = process.env.ORCA_COMPUTER_CLI ?? devCli - const cliArgs = process.env.ORCA_COMPUTER_CLI ? args : args + const builtCli = join(process.cwd(), 'out/cli/index.js') + const command = + process.env.ORCA_COMPUTER_CLI ?? (process.platform === 'win32' ? process.execPath : devCli) + const cliArgs = process.env.ORCA_COMPUTER_CLI + ? args + : process.platform === 'win32' + ? [builtCli, ...args] + : args try { const result = await execFileAsync(command, cliArgs, { maxBuffer: 20 * 1024 * 1024 @@ -83,21 +90,29 @@ export async function killGedit(): Promise { export async function ensureNotepadLaunched(): Promise { await killNotepad() windowsTempDir = await mkdtemp(join(tmpdir(), 'orca-computer-windows-e2e-')) - const filePath = join(windowsTempDir, 'notepad-target.txt') + const filePath = join(windowsTempDir, `orca-notepad-${Date.now()}.txt`) await writeFile(filePath, 'seed', 'utf8') - notepadProcess = spawn('notepad.exe', [filePath], { detached: true, stdio: 'ignore' }) - notepadProcess.unref() - await delay(2500) + await execFileAsync('powershell.exe', [ + '-NoProfile', + '-NonInteractive', + '-Command', + `Start-Process notepad.exe -ArgumentList ${powerShellSingleQuoted(filePath)}` + ]) + notepadAppSelector = `pid:${await findNotepadWindowPid(filePath)}` } export async function killNotepad(): Promise { - if (notepadProcess?.pid) { + const notepadPid = notepadAppSelector?.startsWith('pid:') + ? Number.parseInt(notepadAppSelector.slice(4), 10) + : notepadProcess?.pid + if (notepadPid) { try { - await execFileAsync('taskkill.exe', ['/PID', String(notepadProcess.pid), '/T', '/F']) + await execFileAsync('taskkill.exe', ['/PID', String(notepadPid), '/T', '/F']) } catch { // The test-owned Notepad process may already be closed. } notepadProcess = null + notepadAppSelector = null } if (windowsTempDir) { await rm(windowsTempDir, { force: true, recursive: true }) @@ -105,6 +120,13 @@ export async function killNotepad(): Promise { } } +export function getNotepadAppSelector(): string { + if (!notepadAppSelector) { + throw new Error('Notepad has not been launched') + } + return notepadAppSelector +} + export function findRoleIndex(treeText: string, role: string | RegExp): number { const matcher = typeof role === 'string' @@ -122,6 +144,41 @@ function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } +async function findNotepadWindowPid(filePath: string): Promise { + const targetName = filePath.split(/[\\/]/).at(-1) ?? filePath + const script = [ + `$targetName = ${powerShellSingleQuoted(targetName)}`, + '$deadline = (Get-Date).AddSeconds(15)', + '$target = $null', + 'while ((Get-Date) -lt $deadline -and $null -eq $target) {', + ' Start-Sleep -Milliseconds 250', + ' $target = Get-Process Notepad -ErrorAction SilentlyContinue |', + ' Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like "*$targetName*" } |', + ' Sort-Object StartTime -Descending |', + ' Select-Object -First 1', + '}', + 'if ($null -eq $target) {', + ' $target = Get-Process Notepad -ErrorAction SilentlyContinue |', + ' Where-Object { $_.MainWindowHandle -ne 0 } |', + ' Sort-Object StartTime -Descending |', + ' Select-Object -First 1', + '}', + 'if ($null -eq $target) { throw "No visible Notepad window found for $targetName" }', + 'Write-Output $target.Id' + ].join('\n') + const result = await execFileAsync('powershell.exe', [ + '-NoProfile', + '-NonInteractive', + '-Command', + script + ]) + return Number.parseInt(result.stdout.trim(), 10) +} + +function powerShellSingleQuoted(value: string): string { + return `'${value.replaceAll("'", "''")}'` +} + function escapeRegExp(input: string): string { return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') }