fix: harden Windows computer use runtime (#1757)
This commit is contained in:
Jinwoo Hong
GitHub
parent
a9ea3c129d
commit
abcd2d5836
|
|
@ -4,6 +4,10 @@ param(
|
|||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
|
||||
[Console]::InputEncoding = $utf8NoBom
|
||||
[Console]::OutputEncoding = $utf8NoBom
|
||||
$OutputEncoding = $utf8NoBom
|
||||
|
||||
Add-Type -AssemblyName UIAutomationClient
|
||||
Add-Type -AssemblyName UIAutomationTypes
|
||||
|
|
@ -46,6 +50,12 @@ public static class OrcaDesktopWin32 {
|
|||
|
||||
[DllImport("user32.dll")]
|
||||
public static extern IntPtr GetForegroundWindow();
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
public static extern bool SetCursorPos(int x, int y);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
public static extern void mouse_event(uint dwFlags, uint dx, uint dy, int dwData, UIntPtr dwExtraInfo);
|
||||
}
|
||||
"@
|
||||
|
||||
|
|
@ -75,6 +85,16 @@ $WindowsMessages = @{
|
|||
Wheel = 0x020A
|
||||
}
|
||||
|
||||
$MouseEvents = @{
|
||||
LeftDown = 0x0002
|
||||
LeftUp = 0x0004
|
||||
RightDown = 0x0008
|
||||
RightUp = 0x0010
|
||||
MiddleDown = 0x0020
|
||||
MiddleUp = 0x0040
|
||||
Wheel = 0x0800
|
||||
}
|
||||
|
||||
function Write-OrcaJson($Payload) {
|
||||
$Payload | ConvertTo-Json -Depth 100 -Compress
|
||||
}
|
||||
|
|
@ -85,7 +105,7 @@ function New-OrcaFrame([double]$X, [double]$Y, [double]$Width, [double]$Height)
|
|||
}
|
||||
|
||||
function Read-OrcaOperation([string]$Path) {
|
||||
Get-Content -Raw -Path $Path | ConvertFrom-Json
|
||||
Get-Content -Raw -Encoding UTF8 -Path $Path | ConvertFrom-Json
|
||||
}
|
||||
|
||||
function ConvertTo-OrcaLParam([int]$X, [int]$Y) {
|
||||
|
|
@ -108,10 +128,10 @@ function Find-OrcaProcess([string]$Query) {
|
|||
$needle = $needle.Substring(4)
|
||||
}
|
||||
|
||||
$pid = 0
|
||||
$parsedProcessId = 0
|
||||
$processes = Get-OrcaWindowProcesses
|
||||
if ([int]::TryParse($needle, [ref]$pid)) {
|
||||
$match = $processes | Where-Object { $_.Id -eq $pid } | Select-Object -First 1
|
||||
if ([int]::TryParse($needle, [ref]$parsedProcessId)) {
|
||||
$match = $processes | Where-Object { $_.Id -eq $parsedProcessId } | Select-Object -First 1
|
||||
if ($null -ne $match) {
|
||||
Assert-OrcaProcessAllowed $match
|
||||
return $match
|
||||
|
|
@ -174,6 +194,22 @@ function Get-OrcaWindowId($Process) {
|
|||
[int64]$Process.MainWindowHandle
|
||||
}
|
||||
|
||||
function Get-OrcaAppName($Process) {
|
||||
if ($Process.ProcessName -eq "ApplicationFrameHost" -and -not [string]::IsNullOrWhiteSpace($Process.MainWindowTitle)) {
|
||||
return [string]$Process.MainWindowTitle
|
||||
}
|
||||
[string]$Process.ProcessName
|
||||
}
|
||||
|
||||
function New-OrcaAppRecord($Process) {
|
||||
[pscustomobject]@{
|
||||
name = Get-OrcaAppName $Process
|
||||
bundleIdentifier = $Process.ProcessName
|
||||
bundleId = $Process.ProcessName
|
||||
pid = [int]$Process.Id
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-OrcaWindowTarget($Process, $WindowId, $WindowIndex) {
|
||||
if ($null -ne $WindowIndex -and [int]$WindowIndex -ne 0) {
|
||||
throw "windowNotFound(`"$WindowIndex`")"
|
||||
|
|
@ -446,12 +482,7 @@ function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId =
|
|||
|
||||
[pscustomobject]@{
|
||||
snapshotId = [guid]::NewGuid().ToString()
|
||||
app = [pscustomobject]@{
|
||||
name = $process.ProcessName
|
||||
bundleIdentifier = $process.ProcessName
|
||||
bundleId = $process.ProcessName
|
||||
pid = [int]$process.Id
|
||||
}
|
||||
app = New-OrcaAppRecord $process
|
||||
windowTitle = $process.MainWindowTitle
|
||||
windowId = Get-OrcaWindowId $process
|
||||
windowBounds = $windowFrame
|
||||
|
|
@ -468,12 +499,7 @@ function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId =
|
|||
|
||||
function Get-OrcaAppList {
|
||||
@(Get-OrcaWindowProcesses | ForEach-Object {
|
||||
[pscustomobject]@{
|
||||
name = $_.ProcessName
|
||||
bundleIdentifier = $_.ProcessName
|
||||
bundleId = $_.ProcessName
|
||||
pid = [int]$_.Id
|
||||
}
|
||||
New-OrcaAppRecord $_
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -491,12 +517,7 @@ function Get-OrcaWindowList([string]$Query) {
|
|||
$width = [int][Math]::Max(0, [Math]::Round($windowFrame.width))
|
||||
$height = [int][Math]::Max(0, [Math]::Round($windowFrame.height))
|
||||
}
|
||||
$app = [pscustomobject]@{
|
||||
name = $process.ProcessName
|
||||
bundleIdentifier = $process.ProcessName
|
||||
bundleId = $process.ProcessName
|
||||
pid = [int]$process.Id
|
||||
}
|
||||
$app = New-OrcaAppRecord $process
|
||||
[pscustomobject]@{
|
||||
app = $app
|
||||
windows = @([pscustomobject]@{
|
||||
|
|
@ -643,60 +664,45 @@ function Get-OrcaElementScreenPoint($Element) {
|
|||
}
|
||||
|
||||
function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count) {
|
||||
$point = New-Object OrcaDesktopWin32+POINT
|
||||
$point.X = $ScreenX
|
||||
$point.Y = $ScreenY
|
||||
[void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$point)
|
||||
|
||||
$down = $WindowsMessages.LeftDown
|
||||
$up = $WindowsMessages.LeftUp
|
||||
$flag = 1
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
||||
[void][OrcaDesktopWin32]::SetCursorPos($ScreenX, $ScreenY)
|
||||
$down = $MouseEvents.LeftDown
|
||||
$up = $MouseEvents.LeftUp
|
||||
if ($Button -eq "right") {
|
||||
$down = $WindowsMessages.RightDown
|
||||
$up = $WindowsMessages.RightUp
|
||||
$flag = 2
|
||||
$down = $MouseEvents.RightDown
|
||||
$up = $MouseEvents.RightUp
|
||||
} elseif ($Button -eq "middle") {
|
||||
$down = $WindowsMessages.MiddleDown
|
||||
$up = $WindowsMessages.MiddleUp
|
||||
$flag = 16
|
||||
$down = $MouseEvents.MiddleDown
|
||||
$up = $MouseEvents.MiddleUp
|
||||
}
|
||||
|
||||
$position = ConvertTo-OrcaLParam $point.X $point.Y
|
||||
for ($i = 0; $i -lt [Math]::Max(1, $Count); $i++) {
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]::Zero, $position)
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $down, [IntPtr]$flag, $position)
|
||||
[OrcaDesktopWin32]::mouse_event($down, 0, 0, 0, [UIntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 35
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $up, [IntPtr]::Zero, $position)
|
||||
[OrcaDesktopWin32]::mouse_event($up, 0, 0, 0, [UIntPtr]::Zero)
|
||||
}
|
||||
}
|
||||
|
||||
function Send-OrcaDrag([IntPtr]$WindowHandle, $From, $To) {
|
||||
$start = New-Object OrcaDesktopWin32+POINT
|
||||
$start.X = [int]$From.x
|
||||
$start.Y = [int]$From.y
|
||||
[void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$start)
|
||||
|
||||
$end = New-Object OrcaDesktopWin32+POINT
|
||||
$end.X = [int]$To.x
|
||||
$end.Y = [int]$To.y
|
||||
[void][OrcaDesktopWin32]::ScreenToClient($WindowHandle, [ref]$end)
|
||||
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]::Zero, (ConvertTo-OrcaLParam $start.X $start.Y))
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.LeftDown, [IntPtr]1, (ConvertTo-OrcaLParam $start.X $start.Y))
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
||||
$startX = [int]$From.x
|
||||
$startY = [int]$From.y
|
||||
$endX = [int]$To.x
|
||||
$endY = [int]$To.y
|
||||
[void][OrcaDesktopWin32]::SetCursorPos($startX, $startY)
|
||||
[OrcaDesktopWin32]::mouse_event($MouseEvents.LeftDown, 0, 0, 0, [UIntPtr]::Zero)
|
||||
for ($step = 1; $step -le 12; $step++) {
|
||||
$x = [int][Math]::Round($start.X + (($end.X - $start.X) * $step / 12))
|
||||
$y = [int][Math]::Round($start.Y + (($end.Y - $start.Y) * $step / 12))
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.MouseMove, [IntPtr]1, (ConvertTo-OrcaLParam $x $y))
|
||||
$x = [int][Math]::Round($startX + (($endX - $startX) * $step / 12))
|
||||
$y = [int][Math]::Round($startY + (($endY - $startY) * $step / 12))
|
||||
[void][OrcaDesktopWin32]::SetCursorPos($x, $y)
|
||||
Start-Sleep -Milliseconds 20
|
||||
}
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.LeftUp, [IntPtr]::Zero, (ConvertTo-OrcaLParam $end.X $end.Y))
|
||||
[OrcaDesktopWin32]::mouse_event($MouseEvents.LeftUp, 0, 0, 0, [UIntPtr]::Zero)
|
||||
}
|
||||
|
||||
function Send-OrcaText([IntPtr]$WindowHandle, [string]$Text) {
|
||||
foreach ($character in $Text.ToCharArray()) {
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.Char, [IntPtr][int][char]$character, [IntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 8
|
||||
}
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
||||
[System.Windows.Forms.SendKeys]::SendWait((ConvertTo-OrcaSendKeysText $Text))
|
||||
}
|
||||
|
||||
function Get-OrcaVirtualKey([string]$Key) {
|
||||
|
|
@ -712,10 +718,8 @@ function Get-OrcaVirtualKey([string]$Key) {
|
|||
}
|
||||
|
||||
function Send-OrcaKey([IntPtr]$WindowHandle, [string]$Key) {
|
||||
$virtualKey = Get-OrcaVirtualKey $Key
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyDown, [IntPtr]$virtualKey, [IntPtr]::Zero)
|
||||
Start-Sleep -Milliseconds 25
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyUp, [IntPtr]$virtualKey, [IntPtr]::Zero)
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
||||
[System.Windows.Forms.SendKeys]::SendWait((ConvertTo-OrcaSendKeysKey $Key))
|
||||
}
|
||||
|
||||
function Get-OrcaModifierVirtualKey([string]$Modifier) {
|
||||
|
|
@ -732,16 +736,58 @@ function Send-OrcaHotkey([IntPtr]$WindowHandle, [string]$KeySpec) {
|
|||
$parts = @($KeySpec.Split("+") | ForEach-Object { $_.Trim() } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
|
||||
if ($parts.Count -eq 0) { throw "Unsupported key: $KeySpec" }
|
||||
$key = $parts[$parts.Count - 1]
|
||||
$modifiers = @()
|
||||
$prefix = ""
|
||||
if ($parts.Count -gt 1) {
|
||||
$modifiers = @($parts[0..($parts.Count - 2)] | ForEach-Object { Get-OrcaModifierVirtualKey $_ })
|
||||
foreach ($modifier in $parts[0..($parts.Count - 2)]) {
|
||||
$prefix += ConvertTo-OrcaSendKeysModifier $modifier
|
||||
}
|
||||
}
|
||||
foreach ($modifier in $modifiers) {
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyDown, [IntPtr]$modifier, [IntPtr]::Zero)
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
|
||||
[System.Windows.Forms.SendKeys]::SendWait($prefix + (ConvertTo-OrcaSendKeysKey $key))
|
||||
}
|
||||
|
||||
function ConvertTo-OrcaSendKeysText([string]$Text) {
|
||||
$builder = New-Object System.Text.StringBuilder
|
||||
foreach ($character in $Text.ToCharArray()) {
|
||||
$value = [string]$character
|
||||
if ($value -eq "`r") { continue }
|
||||
if ($value -eq "`n") { [void]$builder.Append("{ENTER}"); continue }
|
||||
if ("+^%~(){}[]".Contains($value)) {
|
||||
[void]$builder.Append("{").Append($value).Append("}")
|
||||
} else {
|
||||
[void]$builder.Append($value)
|
||||
}
|
||||
}
|
||||
Send-OrcaKey $WindowHandle $key
|
||||
for ($i = $modifiers.Count - 1; $i -ge 0; $i--) {
|
||||
[void][OrcaDesktopWin32]::PostMessage($WindowHandle, $WindowsMessages.KeyUp, [IntPtr]$modifiers[$i], [IntPtr]::Zero)
|
||||
$builder.ToString()
|
||||
}
|
||||
|
||||
function ConvertTo-OrcaSendKeysKey([string]$Key) {
|
||||
switch ($Key.ToLowerInvariant()) {
|
||||
{ $_ -in @("return", "enter") } { return "{ENTER}" }
|
||||
"tab" { return "{TAB}" }
|
||||
{ $_ -in @("escape", "esc") } { return "{ESC}" }
|
||||
"backspace" { return "{BACKSPACE}" }
|
||||
"delete" { return "{DELETE}" }
|
||||
"space" { return " " }
|
||||
"left" { return "{LEFT}" }
|
||||
"up" { return "{UP}" }
|
||||
"right" { return "{RIGHT}" }
|
||||
"down" { return "{DOWN}" }
|
||||
"home" { return "{HOME}" }
|
||||
"end" { return "{END}" }
|
||||
default {
|
||||
if ($Key.Length -eq 1) { return (ConvertTo-OrcaSendKeysText $Key) }
|
||||
throw "Unsupported key: $Key"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ConvertTo-OrcaSendKeysModifier([string]$Modifier) {
|
||||
switch ($Modifier.ToLowerInvariant()) {
|
||||
{ $_ -in @("ctrl", "control", "cmdorctrl", "commandorcontrol") } { return "^" }
|
||||
"shift" { return "+" }
|
||||
{ $_ -in @("alt", "option") } { return "%" }
|
||||
default { throw "Unsupported modifier: $Modifier" }
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -755,9 +801,9 @@ function Send-OrcaPasteText([IntPtr]$WindowHandle, [string]$Text) {
|
|||
Send-OrcaHotkey $WindowHandle "Ctrl+v"
|
||||
} finally {
|
||||
if ($hadPrevious) {
|
||||
[System.Windows.Forms.Clipboard]::SetDataObject($previous, $true)
|
||||
try { [System.Windows.Forms.Clipboard]::SetDataObject($previous, $true) } catch {}
|
||||
} else {
|
||||
[System.Windows.Forms.Clipboard]::Clear()
|
||||
try { [System.Windows.Forms.Clipboard]::Clear() } catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -819,7 +865,9 @@ function Invoke-OrcaOperation($Operation) {
|
|||
if ($Operation.direction -eq "down" -or $Operation.direction -eq "right") { $delta = -1 * $delta }
|
||||
$point = Get-OrcaElementScreenPoint $element
|
||||
if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
|
||||
[void][OrcaDesktopWin32]::PostMessage($handle, $WindowsMessages.Wheel, (ConvertTo-OrcaWheelParam $delta), (ConvertTo-OrcaLParam $point.x $point.y))
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($handle)
|
||||
[void][OrcaDesktopWin32]::SetCursorPos([int]$point.x, [int]$point.y)
|
||||
[OrcaDesktopWin32]::mouse_event($MouseEvents.Wheel, 0, 0, $delta, [UIntPtr]::Zero)
|
||||
$action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $null }
|
||||
}
|
||||
"drag" {
|
||||
|
|
|
|||
|
|
@ -364,9 +364,11 @@ describe('orca computer CLI handlers', () => {
|
|||
expect(parsed.result.screenshot).toMatchObject({
|
||||
dataOmitted: true,
|
||||
format: 'png',
|
||||
expiresAt: expect.any(String),
|
||||
path: expect.stringContaining('orca-computer-use/req_state-screenshot.png')
|
||||
expiresAt: expect.any(String)
|
||||
})
|
||||
expect(String(parsed.result.screenshot.path).replaceAll('\\', '/')).toContain(
|
||||
'orca-computer-use/req_state-screenshot.png'
|
||||
)
|
||||
})
|
||||
|
||||
it('shows coordinate space and truncation in pretty state output', async () => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
import { describe, expect, test } from 'vitest'
|
||||
import type { ComputerListAppsResult, ComputerSnapshotResult } from '../../src/shared/runtime-types'
|
||||
import { findRoleIndex, parseJsonOutput, runOrcaCli } from './helpers/computer-driver'
|
||||
|
||||
const isWindows = process.platform === 'win32'
|
||||
const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
|
||||
|
||||
describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Store apps)', () => {
|
||||
test('Store app windows are discoverable by title and clickable', async () => {
|
||||
await launchCalculator()
|
||||
try {
|
||||
const apps = parseJsonOutput<{ result: ComputerListAppsResult }>(
|
||||
(await runOrcaCli(['computer', 'list-apps', '--json'])).stdout
|
||||
)
|
||||
expect(apps.result.apps).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ name: 'Calculator', bundleId: 'ApplicationFrameHost' })
|
||||
])
|
||||
)
|
||||
|
||||
let state = parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'get-app-state',
|
||||
'--app',
|
||||
'Calculator',
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
const one = findRoleIndex(state.result.snapshot.treeText, 'button One')
|
||||
const plus = findRoleIndex(state.result.snapshot.treeText, 'button Plus')
|
||||
const two = findRoleIndex(state.result.snapshot.treeText, 'button Two')
|
||||
const equals = findRoleIndex(state.result.snapshot.treeText, 'button Equals')
|
||||
expect([one, plus, two, equals].every((index) => index >= 0)).toBe(true)
|
||||
|
||||
for (const index of [one, plus, two, equals]) {
|
||||
state = parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'click',
|
||||
'--app',
|
||||
'Calculator',
|
||||
'--element-index',
|
||||
String(index),
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
}
|
||||
expect(state.result.snapshot.treeText).toMatch(/Display is 3\b/)
|
||||
} finally {
|
||||
await killCalculator()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
async function launchCalculator(): Promise<void> {
|
||||
await runPowerShell('Start-Process calc.exe')
|
||||
await runPowerShell(
|
||||
[
|
||||
'$deadline = (Get-Date).AddSeconds(15)',
|
||||
'$target = $null',
|
||||
'while ((Get-Date) -lt $deadline -and $null -eq $target) {',
|
||||
' Start-Sleep -Milliseconds 250',
|
||||
' $target = Get-Process |',
|
||||
' Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -eq "Calculator" } |',
|
||||
' Select-Object -First 1',
|
||||
'}',
|
||||
'if ($null -eq $target) { throw "No visible Calculator window found" }'
|
||||
].join('\n')
|
||||
)
|
||||
}
|
||||
|
||||
async function killCalculator(): Promise<void> {
|
||||
await runPowerShell(
|
||||
'Get-Process CalculatorApp -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue'
|
||||
)
|
||||
}
|
||||
|
||||
async function runPowerShell(script: string): Promise<void> {
|
||||
const { execFile } = await import('child_process')
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
execFile('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script], (error) => {
|
||||
if (error) {
|
||||
reject(error)
|
||||
return
|
||||
}
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
@ -1,7 +1,14 @@
|
|||
import { afterAll, beforeAll, describe, expect, test } from 'vitest'
|
||||
import type { ComputerActionResult, ComputerSnapshotResult } from '../../src/shared/runtime-types'
|
||||
import type {
|
||||
ComputerActionResult,
|
||||
ComputerListAppsResult,
|
||||
ComputerListWindowsResult,
|
||||
ComputerSnapshotResult
|
||||
} from '../../src/shared/runtime-types'
|
||||
import {
|
||||
ensureNotepadLaunched,
|
||||
findRoleIndex,
|
||||
getNotepadAppSelector,
|
||||
killNotepad,
|
||||
parseJsonOutput,
|
||||
runOrcaCli
|
||||
|
|
@ -19,18 +26,88 @@ describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', (
|
|||
await killNotepad()
|
||||
})
|
||||
|
||||
test('list-apps includes the test-owned Notepad process', async () => {
|
||||
const result = await runOrcaCli(['computer', 'list-apps', '--json'])
|
||||
const envelope = parseJsonOutput<{ result: ComputerListAppsResult }>(result.stdout)
|
||||
const pid = Number.parseInt(getNotepadAppSelector().slice(4), 10)
|
||||
|
||||
expect(envelope.result.apps).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ name: 'Notepad', pid })])
|
||||
)
|
||||
})
|
||||
|
||||
test('list-windows returns a targetable Notepad window', async () => {
|
||||
const app = getNotepadAppSelector()
|
||||
const result = await runOrcaCli(['computer', 'list-windows', '--app', app, '--json'])
|
||||
const envelope = parseJsonOutput<{ result: ComputerListWindowsResult }>(result.stdout)
|
||||
|
||||
expect(envelope.result.windows).toEqual([
|
||||
expect.objectContaining({
|
||||
index: 0,
|
||||
app: expect.objectContaining({ name: 'Notepad' }),
|
||||
id: expect.any(Number),
|
||||
title: expect.stringContaining('Notepad'),
|
||||
width: expect.any(Number),
|
||||
height: expect.any(Number)
|
||||
})
|
||||
])
|
||||
})
|
||||
|
||||
test('Notepad exposes a basic accessibility tree', async () => {
|
||||
const result = await runOrcaCli(['computer', 'get-app-state', '--app', 'Notepad', '--json'])
|
||||
const result = await runOrcaCli([
|
||||
'computer',
|
||||
'get-app-state',
|
||||
'--app',
|
||||
getNotepadAppSelector(),
|
||||
'--json'
|
||||
])
|
||||
const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout)
|
||||
|
||||
expect(envelope.result.snapshot.app.name).toBe('Notepad')
|
||||
expect(envelope.result.snapshot.window.title).toContain('Notepad')
|
||||
expect(envelope.result.snapshot.elementCount).toBeGreaterThan(0)
|
||||
expect(envelope.result.snapshot.coordinateSpace).toBe('window')
|
||||
expect(envelope.result.snapshot.truncation?.truncated).toBe(false)
|
||||
expect(envelope.result.screenshotStatus.state).toBe('captured')
|
||||
expect(envelope.result.screenshot?.format).toBe('png')
|
||||
expect(envelope.result.screenshot?.data).toBeUndefined()
|
||||
expect(envelope.result.screenshot?.path).toContain('orca-computer-use-')
|
||||
expect(envelope.result.screenshot?.dataOmitted).toBe(true)
|
||||
expect(envelope.result.screenshot?.path).toContain('orca-computer-use')
|
||||
})
|
||||
|
||||
test('set-value mutates the document through UI Automation', async () => {
|
||||
const app = getNotepadAppSelector()
|
||||
const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
|
||||
.stdout
|
||||
)
|
||||
const documentIndex = findRoleIndex(before.result.snapshot.treeText, 'document')
|
||||
expect(documentIndex).toBeGreaterThanOrEqual(0)
|
||||
|
||||
const marker = `orca-windows-set-${Date.now()}`
|
||||
const action = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'set-value',
|
||||
'--app',
|
||||
app,
|
||||
'--element-index',
|
||||
String(documentIndex),
|
||||
'--value',
|
||||
marker,
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
expect(action.result.action?.path).toBe('accessibility')
|
||||
|
||||
expect(action.result.snapshot.treeText).toContain(marker)
|
||||
})
|
||||
|
||||
test('paste-text mutates the test-owned document', async () => {
|
||||
const app = getNotepadAppSelector()
|
||||
const marker = `orca-windows-paste-${Date.now()}`
|
||||
const action = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(
|
||||
|
|
@ -38,7 +115,7 @@ describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', (
|
|||
'computer',
|
||||
'paste-text',
|
||||
'--app',
|
||||
'Notepad',
|
||||
app,
|
||||
'--text',
|
||||
marker,
|
||||
'--restore-window',
|
||||
|
|
@ -50,17 +127,148 @@ describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', (
|
|||
expect(action.result.action?.path).toBe('clipboard')
|
||||
|
||||
const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
|
||||
.stdout
|
||||
)
|
||||
expect(after.result.snapshot.treeText).toContain(marker)
|
||||
})
|
||||
|
||||
test('Unicode payloads survive set-value and paste-text', async () => {
|
||||
const app = getNotepadAppSelector()
|
||||
const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
|
||||
.stdout
|
||||
)
|
||||
const documentIndex = findRoleIndex(before.result.snapshot.treeText, 'document')
|
||||
expect(documentIndex).toBeGreaterThanOrEqual(0)
|
||||
|
||||
const unicode = `orca unicode café Ω 漢字 ${Date.now()}`
|
||||
const set = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'get-app-state',
|
||||
'set-value',
|
||||
'--app',
|
||||
'Notepad',
|
||||
app,
|
||||
'--element-index',
|
||||
String(documentIndex),
|
||||
'--value',
|
||||
unicode,
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
expect(after.result.snapshot.treeText).toContain(marker)
|
||||
expect(set.result.snapshot.treeText).toContain(unicode)
|
||||
|
||||
const pasted = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'paste-text',
|
||||
'--app',
|
||||
app,
|
||||
'--text',
|
||||
unicode,
|
||||
'--restore-window',
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
expect(pasted.result.snapshot.treeText).toContain(unicode)
|
||||
})
|
||||
|
||||
test('hotkey and paste-text can replace the document selection', async () => {
|
||||
const app = getNotepadAppSelector()
|
||||
const first = `orca-windows-first-${Date.now()}`
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'paste-text',
|
||||
'--app',
|
||||
app,
|
||||
'--text',
|
||||
first,
|
||||
'--restore-window',
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
|
||||
const selectAll = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'hotkey',
|
||||
'--app',
|
||||
app,
|
||||
'--key',
|
||||
'CmdOrCtrl+A',
|
||||
'--restore-window',
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
expect(selectAll.result.action?.actionName).toBe('hotkey')
|
||||
|
||||
const marker = `orca-windows-replaced-${Date.now()}`
|
||||
const second = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'paste-text',
|
||||
'--app',
|
||||
app,
|
||||
'--text',
|
||||
marker,
|
||||
'--restore-window',
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
expect(second.result.snapshot.treeText).toContain(marker)
|
||||
expect(second.result.snapshot.treeText).not.toContain(first)
|
||||
})
|
||||
|
||||
test('click and type-text send synthetic input to the document', async () => {
|
||||
const app = getNotepadAppSelector()
|
||||
const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
|
||||
.stdout
|
||||
)
|
||||
const documentIndex = findRoleIndex(before.result.snapshot.treeText, 'document')
|
||||
expect(documentIndex).toBeGreaterThanOrEqual(0)
|
||||
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'click',
|
||||
'--app',
|
||||
app,
|
||||
'--element-index',
|
||||
String(documentIndex),
|
||||
'--restore-window',
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
|
||||
const marker = ` typed-${Date.now()}`
|
||||
const typed = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'type-text',
|
||||
'--app',
|
||||
app,
|
||||
'--text',
|
||||
marker,
|
||||
'--restore-window',
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
expect(typed.result.action?.path).toBe('synthetic')
|
||||
expect(typed.result.snapshot.treeText).toContain(marker.trim())
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ let linuxTempDir: string | null = null
|
|||
let windowsTempDir: string | null = null
|
||||
let geditProcess: ChildProcess | null = null
|
||||
let notepadProcess: ChildProcess | null = null
|
||||
let notepadAppSelector: string | null = null
|
||||
|
||||
export type CliResult = {
|
||||
stdout: string
|
||||
|
|
@ -18,8 +19,14 @@ export type CliResult = {
|
|||
|
||||
export async function runOrcaCli(args: string[]): Promise<CliResult> {
|
||||
const devCli = join(process.cwd(), 'config/scripts/orca-dev')
|
||||
const command = process.env.ORCA_COMPUTER_CLI ?? devCli
|
||||
const cliArgs = process.env.ORCA_COMPUTER_CLI ? args : args
|
||||
const builtCli = join(process.cwd(), 'out/cli/index.js')
|
||||
const command =
|
||||
process.env.ORCA_COMPUTER_CLI ?? (process.platform === 'win32' ? process.execPath : devCli)
|
||||
const cliArgs = process.env.ORCA_COMPUTER_CLI
|
||||
? args
|
||||
: process.platform === 'win32'
|
||||
? [builtCli, ...args]
|
||||
: args
|
||||
try {
|
||||
const result = await execFileAsync(command, cliArgs, {
|
||||
maxBuffer: 20 * 1024 * 1024
|
||||
|
|
@ -83,21 +90,29 @@ export async function killGedit(): Promise<void> {
|
|||
export async function ensureNotepadLaunched(): Promise<void> {
|
||||
await killNotepad()
|
||||
windowsTempDir = await mkdtemp(join(tmpdir(), 'orca-computer-windows-e2e-'))
|
||||
const filePath = join(windowsTempDir, 'notepad-target.txt')
|
||||
const filePath = join(windowsTempDir, `orca-notepad-${Date.now()}.txt`)
|
||||
await writeFile(filePath, 'seed', 'utf8')
|
||||
notepadProcess = spawn('notepad.exe', [filePath], { detached: true, stdio: 'ignore' })
|
||||
notepadProcess.unref()
|
||||
await delay(2500)
|
||||
await execFileAsync('powershell.exe', [
|
||||
'-NoProfile',
|
||||
'-NonInteractive',
|
||||
'-Command',
|
||||
`Start-Process notepad.exe -ArgumentList ${powerShellSingleQuoted(filePath)}`
|
||||
])
|
||||
notepadAppSelector = `pid:${await findNotepadWindowPid(filePath)}`
|
||||
}
|
||||
|
||||
export async function killNotepad(): Promise<void> {
|
||||
if (notepadProcess?.pid) {
|
||||
const notepadPid = notepadAppSelector?.startsWith('pid:')
|
||||
? Number.parseInt(notepadAppSelector.slice(4), 10)
|
||||
: notepadProcess?.pid
|
||||
if (notepadPid) {
|
||||
try {
|
||||
await execFileAsync('taskkill.exe', ['/PID', String(notepadProcess.pid), '/T', '/F'])
|
||||
await execFileAsync('taskkill.exe', ['/PID', String(notepadPid), '/T', '/F'])
|
||||
} catch {
|
||||
// The test-owned Notepad process may already be closed.
|
||||
}
|
||||
notepadProcess = null
|
||||
notepadAppSelector = null
|
||||
}
|
||||
if (windowsTempDir) {
|
||||
await rm(windowsTempDir, { force: true, recursive: true })
|
||||
|
|
@ -105,6 +120,13 @@ export async function killNotepad(): Promise<void> {
|
|||
}
|
||||
}
|
||||
|
||||
export function getNotepadAppSelector(): string {
|
||||
if (!notepadAppSelector) {
|
||||
throw new Error('Notepad has not been launched')
|
||||
}
|
||||
return notepadAppSelector
|
||||
}
|
||||
|
||||
export function findRoleIndex(treeText: string, role: string | RegExp): number {
|
||||
const matcher =
|
||||
typeof role === 'string'
|
||||
|
|
@ -122,6 +144,41 @@ function delay(ms: number): Promise<void> {
|
|||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
async function findNotepadWindowPid(filePath: string): Promise<number> {
|
||||
const targetName = filePath.split(/[\\/]/).at(-1) ?? filePath
|
||||
const script = [
|
||||
`$targetName = ${powerShellSingleQuoted(targetName)}`,
|
||||
'$deadline = (Get-Date).AddSeconds(15)',
|
||||
'$target = $null',
|
||||
'while ((Get-Date) -lt $deadline -and $null -eq $target) {',
|
||||
' Start-Sleep -Milliseconds 250',
|
||||
' $target = Get-Process Notepad -ErrorAction SilentlyContinue |',
|
||||
' Where-Object { $_.MainWindowHandle -ne 0 -and $_.MainWindowTitle -like "*$targetName*" } |',
|
||||
' Sort-Object StartTime -Descending |',
|
||||
' Select-Object -First 1',
|
||||
'}',
|
||||
'if ($null -eq $target) {',
|
||||
' $target = Get-Process Notepad -ErrorAction SilentlyContinue |',
|
||||
' Where-Object { $_.MainWindowHandle -ne 0 } |',
|
||||
' Sort-Object StartTime -Descending |',
|
||||
' Select-Object -First 1',
|
||||
'}',
|
||||
'if ($null -eq $target) { throw "No visible Notepad window found for $targetName" }',
|
||||
'Write-Output $target.Id'
|
||||
].join('\n')
|
||||
const result = await execFileAsync('powershell.exe', [
|
||||
'-NoProfile',
|
||||
'-NonInteractive',
|
||||
'-Command',
|
||||
script
|
||||
])
|
||||
return Number.parseInt(result.stdout.trim(), 10)
|
||||
}
|
||||
|
||||
function powerShellSingleQuoted(value: string): string {
|
||||
return `'${value.replaceAll("'", "''")}'`
|
||||
}
|
||||
|
||||
function escapeRegExp(input: string): string {
|
||||
return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue