* Revert "Enable accessibility tree (`ax`) command on iOS emulator sessions (#10007)"
This reverts commit 43ae014a64.
* fix(emulator): expose iOS accessibility tree
* fix(emulator): support device-only iOS AX
* fix(emulator): normalize iOS ax to 0..1 and heal missing axUrl
serve-sim's helper /ax reports element frames in absolute pixels, but
tap/gesture take normalized 0..1 coords. Normalize the raw AX node tree
into a compact nested shape whose frames are 0..1 over the device screen
(first root's frame), mirroring serve-sim's own normalizeAxTree, so agents
can feed ax output straight back into input commands.
Also heal sessions that were registered without an axUrl: #9924 only
derived /ax at parse time, so already-active sessions had no endpoint.
The bridge now derives it from the session's mjpeg stream URL, guarded to
the /stream.mjpeg suffix so a non-mjpeg URL never fabricates a bogus /ax.
* docs(emulator): mark ax working on iOS with correct raw-AX-tree shape
Both skill guides and the CLI summary described iOS ax as unsupported (or,
via the reverted #10007, as a normalized "screen + elements" shape that
never matched the endpoint). ax works on both backends: Android via
uiautomator, iOS via the serve-sim helper. Document the real iOS output —
a raw AX node tree (labels, roles, nested children) with frames normalized
to 0..1 — and regenerate the bundled skill guides.
* chore(skills): regenerate skill bundle manifests
CI verify failed because generated skill artifacts were stale after version/skill revision bumps.
* fix(emulator): read ax from explicit device without active session
Fall back to udid-keyed session lookup when a worktree has no active emulator,
allowing `--device` targeting to work the same way for ax as it does for tap/type.
Also clarify in docs that AX frames are normalized 0..1 with top-left origin,
and show how to tap an element at its frame center (x+width/2, y+height/2).
* fix(emulator): cap iOS AX tree at 500 nodes
Unbounded accessibility trees can flood agent output. Enforce a 500-node limit (matching serve-sim's snapshot cap) and mark truncated parents so consumers know the tree was cut.
---------
Co-authored-by: 5Hyeons <ohs2251@naver.com>
This commit is contained in:
Jinjing
GitHub
parent
ee6319ebe4
commit
4a9affd6e5
|
|
@ -118,8 +118,9 @@ Use `--json` for agent-friendly output. Coordinates are **normalized 0..1**
|
|||
- `gesture` is a straight swipe between the first and last point (adb limitation);
|
||||
fine for scroll/swipe, not for true multi-touch paths.
|
||||
- Capability verbs `install/launch/permissions/logcat` are **Android-only** and
|
||||
fail against an iOS device with `emulator_unsupported`. `ax` works on both,
|
||||
with backend-specific output (uiautomator tree vs serve-sim AX snapshot).
|
||||
fail against an iOS device with `emulator_unsupported`. `ax` works on **both**,
|
||||
with backend-specific output (Android: `uiautomator` node tree; iOS: serve-sim
|
||||
raw AX node tree with frames normalized to 0..1).
|
||||
- No camera/sensor injection yet.
|
||||
|
||||
## Targeting devices & worktrees
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ Use `--json` for agent-friendly output. Commands are workspace-scoped by default
|
|||
| Rotate device | `ORCA emulator rotate landscape_left` | Remembers orientation for subsequent gestures. |
|
||||
| Camera injection | `ORCA emulator camera com.acme.App --webcam` | Or --file, placeholder. Hot-swap with switch. May (re)launch app. |
|
||||
| Permissions | `ORCA emulator permissions grant camera com.acme.App` | grant/revoke/reset/list. See full subcommand help. |
|
||||
| Accessibility tree | `ORCA emulator ax [--device <id>]` | Raw serve-sim AX snapshot (screen + elements). Needs an active session. |
|
||||
| Accessibility tree | `ORCA emulator ax [--device <id>]` | Raw serve-sim AX node tree (labels, roles, nested children, capped at 500 nodes; frames normalized 0..1 with top-left origin — tap an element at its frame center: x+width/2, y+height/2). Needs an active session. |
|
||||
| Raw / advanced | `ORCA emulator exec --command "tap 0.5 0.7"` | Or "ca-debug blended on", "memory-warning", full serve-sim subcommands (no "serve-sim" prefix needed in the command string). Bridge injects active device context. |
|
||||
| Stop | `ORCA emulator kill [--device <id>]` | Or let pane close / Orca quit clean up. |
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -108,7 +108,7 @@ export const EMULATOR_COMMAND_SPECS: CommandSpec[] = [
|
|||
},
|
||||
{
|
||||
path: ['emulator', 'ax'],
|
||||
summary: 'Dump the device accessibility tree (uiautomator on Android, serve-sim AX on iOS)',
|
||||
summary: 'Dump the accessibility tree (Android uiautomator; iOS serve-sim AX, frames 0..1)',
|
||||
usage: 'orca emulator ax [--device <id>] [--worktree <selector>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'device', 'emulator', 'worktree']
|
||||
},
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import { spawn } from 'node:child_process'
|
|||
import { AndroidEmulatorBackend } from './android-emulator-backend'
|
||||
import type { AndroidCommandResult, AndroidCommandRunner } from '../android/android-command-runner'
|
||||
import type { AndroidSdkPaths } from '../android/android-sdk-discovery'
|
||||
import type { EmulatorBackend } from './emulator-backend'
|
||||
|
||||
// The AVD boot spawns the emulator detached (not via the command runner).
|
||||
vi.mock('node:child_process', async (importOriginal) => {
|
||||
|
|
@ -220,26 +219,6 @@ describe('AndroidEmulatorBackend', () => {
|
|||
expect(tree.children[0]).toMatchObject({ text: 'Hi' })
|
||||
})
|
||||
|
||||
it('ignores the ios ax endpoint argument and still dumps via adb', async () => {
|
||||
runner.mockImplementation(async (binary: string, args: readonly string[]) => {
|
||||
const a = args.join(' ')
|
||||
if (binary === SDK.adb && a === 'devices -l') {
|
||||
return ok(RUNNING_ADB)
|
||||
}
|
||||
if (binary === SDK.adb && a === '-s emulator-5554 shell cat /sdcard/window_dump.xml') {
|
||||
return ok('<hierarchy><node text="Hi"/></hierarchy>')
|
||||
}
|
||||
return ok('')
|
||||
})
|
||||
// The widened EmulatorBackend.accessibilityTree(deviceId, axUrl) hands Android an
|
||||
// iOS-only ax URL through the interface; Android must drop it and dump via adb.
|
||||
const iface: EmulatorBackend = backend(runner)
|
||||
const tree = (await iface.accessibilityTree!('emulator-5554', 'http://127.0.0.1:3100/ax')) as {
|
||||
children: { text?: string }[]
|
||||
}
|
||||
expect(tree.children[0]).toMatchObject({ text: 'Hi' })
|
||||
})
|
||||
|
||||
it('boots a shutdown AVD and waits for the new booted serial', async () => {
|
||||
let bootStarted = false
|
||||
vi.mocked(spawn).mockImplementation(() => {
|
||||
|
|
|
|||
|
|
@ -80,8 +80,8 @@ export type EmulatorBackend = {
|
|||
rotate(deviceId: string, orientation: string): Promise<void>
|
||||
exec(deviceId: string, command: string): Promise<unknown>
|
||||
|
||||
// Capability-gated verbs. The router checks `capabilities` before calling
|
||||
// these and rejects unsupported backends with emulator_unsupported.
|
||||
// Capability-gated verbs. The router checks `capabilities`
|
||||
// before calling these and rejects unsupported backends with emulator_unsupported.
|
||||
installApp?(deviceId: string, apkPath: string, options?: { reinstall?: boolean }): Promise<void>
|
||||
launchApp?(deviceId: string, packageName: string, activity?: string): Promise<void>
|
||||
setPermission?(
|
||||
|
|
@ -90,9 +90,7 @@ export type EmulatorBackend = {
|
|||
packageName: string,
|
||||
permission?: string
|
||||
): Promise<void>
|
||||
// axUrl is the session's serve-sim /ax endpoint from the registry; Android
|
||||
// dumps via adb from the device serial and ignores it.
|
||||
accessibilityTree?(deviceId: string, axUrl: string | null): Promise<unknown>
|
||||
accessibilityTree?(deviceId: string, axUrl?: string): Promise<unknown>
|
||||
logcat?(
|
||||
deviceId: string,
|
||||
options?: { lines?: number; filters?: readonly string[] }
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ const {
|
|||
listServeSimHelperProcessesForDeviceMock,
|
||||
shutdownSimulatorDeviceMock,
|
||||
sendEmulatorGestureSequenceMock,
|
||||
parseServeSimDetachedSessionMock
|
||||
parseServeSimDetachedSessionMock,
|
||||
netFetchMock
|
||||
} = vi.hoisted(() => ({
|
||||
ensureSimulatorBootedMock: vi.fn(async () => {}),
|
||||
execServeSimCommandMock: vi.fn(async (_executable?: unknown, _args?: string[]) => ({})),
|
||||
|
|
@ -21,9 +22,12 @@ const {
|
|||
listServeSimHelperProcessesForDeviceMock: vi.fn(async (): Promise<ServeSimHelperProcess[]> => []),
|
||||
shutdownSimulatorDeviceMock: vi.fn(async () => {}),
|
||||
sendEmulatorGestureSequenceMock: vi.fn(async () => {}),
|
||||
parseServeSimDetachedSessionMock: vi.fn()
|
||||
parseServeSimDetachedSessionMock: vi.fn(),
|
||||
netFetchMock: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('electron', () => ({ net: { fetch: netFetchMock } }))
|
||||
|
||||
vi.mock('../serve-sim-execution', () => ({
|
||||
execServeSimCommand: execServeSimCommandMock,
|
||||
parseServeSimCommandArgs: vi.fn((input: string) => input.split(' ').filter(Boolean)),
|
||||
|
|
@ -81,9 +85,10 @@ describe('IosEmulatorBackend', () => {
|
|||
sendEmulatorGestureSequenceMock.mockReset()
|
||||
sendEmulatorGestureSequenceMock.mockImplementation(async () => {})
|
||||
parseServeSimDetachedSessionMock.mockReset()
|
||||
netFetchMock.mockReset()
|
||||
})
|
||||
|
||||
it('declares ios kind, mjpeg codec, and only the ax explicit-verb capability', () => {
|
||||
it('advertises the iOS accessibility tree capability', () => {
|
||||
const backend = new IosEmulatorBackend()
|
||||
expect(backend.kind).toBe('ios')
|
||||
expect(backend.streamCodec).toBe('mjpeg')
|
||||
|
|
@ -96,16 +101,69 @@ describe('IosEmulatorBackend', () => {
|
|||
})
|
||||
})
|
||||
|
||||
it('fetches the accessibility tree from the session ax endpoint and rejects without one', async () => {
|
||||
const fetchAccessibilityTree = vi.fn(async () => ({ elements: [] }))
|
||||
const backend = new IosEmulatorBackend({ fetchAccessibilityTree })
|
||||
it('fetches and normalizes the serve-sim accessibility tree', async () => {
|
||||
const raw = [
|
||||
{
|
||||
type: 'Application',
|
||||
role_description: 'application',
|
||||
AXLabel: 'Demo',
|
||||
enabled: true,
|
||||
frame: { x: 0, y: 0, width: 400, height: 800 },
|
||||
children: [
|
||||
{
|
||||
type: 'Button',
|
||||
role_description: 'button',
|
||||
AXLabel: 'Continue',
|
||||
AXValue: '',
|
||||
enabled: true,
|
||||
frame: { x: 100, y: 400, width: 200, height: 50 },
|
||||
children: []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
netFetchMock.mockResolvedValue(new Response(JSON.stringify(raw), { status: 200 }))
|
||||
const backend = new IosEmulatorBackend()
|
||||
|
||||
await expect(
|
||||
backend.accessibilityTree('device-1', 'http://127.0.0.1:3100/ax')
|
||||
).resolves.toEqual({ elements: [] })
|
||||
expect(fetchAccessibilityTree).toHaveBeenCalledWith('http://127.0.0.1:3100/ax')
|
||||
await expect(backend.accessibilityTree('device-1', null)).rejects.toMatchObject({
|
||||
).resolves.toEqual([
|
||||
{
|
||||
role: 'application',
|
||||
type: 'Application',
|
||||
label: 'Demo',
|
||||
value: '',
|
||||
enabled: true,
|
||||
frame: { x: 0, y: 0, width: 1, height: 1 },
|
||||
children: [
|
||||
{
|
||||
role: 'button',
|
||||
type: 'Button',
|
||||
label: 'Continue',
|
||||
value: '',
|
||||
enabled: true,
|
||||
frame: { x: 0.25, y: 0.5, width: 0.5, height: 0.0625 },
|
||||
children: []
|
||||
}
|
||||
]
|
||||
}
|
||||
])
|
||||
expect(netFetchMock).toHaveBeenCalledWith(
|
||||
'http://127.0.0.1:3100/ax',
|
||||
expect.objectContaining({ signal: expect.any(AbortSignal) })
|
||||
)
|
||||
})
|
||||
|
||||
it('reports missing sessions and temporarily unavailable AX endpoints', async () => {
|
||||
const backend = new IosEmulatorBackend()
|
||||
await expect(backend.accessibilityTree('device-1')).rejects.toMatchObject({
|
||||
code: 'emulator_no_active'
|
||||
})
|
||||
|
||||
netFetchMock.mockResolvedValue(new Response('{"error":"ax_unavailable"}', { status: 503 }))
|
||||
await expect(
|
||||
backend.accessibilityTree('device-1', 'http://127.0.0.1:3100/ax')
|
||||
).rejects.toMatchObject({ code: 'emulator_helper_failed' })
|
||||
})
|
||||
|
||||
it('taps via serve-sim with the resolved device', async () => {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import {
|
|||
import type { EmulatorBridgeOptions } from '../emulator-bridge-types'
|
||||
import { sendEmulatorGestureSequence, type EmulatorGesturePoint } from '../emulator-gesture-sender'
|
||||
import { parseServeSimDetachedSession } from '../serve-sim-detached-session'
|
||||
import { fetchServeSimAccessibilityTree, type FetchAccessibilityTree } from '../serve-sim-ax-tree'
|
||||
import { requestServeSimAccessibilityTree } from '../serve-sim-accessibility-tree'
|
||||
import { hideNativeSimulatorApp } from '../simulator-app-visibility'
|
||||
import type {
|
||||
BackendAvailability,
|
||||
|
|
@ -38,8 +38,6 @@ import type {
|
|||
export class IosEmulatorBackend implements EmulatorBackend {
|
||||
readonly kind = 'ios' as const
|
||||
readonly streamCodec = 'mjpeg' as const
|
||||
// iOS exposes install/launch/permissions/logcat via `exec`; ax has an explicit
|
||||
// verb backed by the active session's serve-sim /ax endpoint.
|
||||
readonly capabilities: EmulatorBackendCapabilities = {
|
||||
install: false,
|
||||
launch: false,
|
||||
|
|
@ -50,11 +48,9 @@ export class IosEmulatorBackend implements EmulatorBackend {
|
|||
|
||||
private cachedServeSimExecutable: ServeSimExecutable | undefined
|
||||
private readonly waitForEndpointReady: (endpoint: string) => Promise<boolean>
|
||||
private readonly fetchAccessibilityTree: FetchAccessibilityTree
|
||||
|
||||
constructor(options: EmulatorBridgeOptions = {}) {
|
||||
this.waitForEndpointReady = options.waitForEndpointReady ?? waitForServeSimEndpointReady
|
||||
this.fetchAccessibilityTree = options.fetchAccessibilityTree ?? fetchServeSimAccessibilityTree
|
||||
}
|
||||
|
||||
// Why: resolving the executable can materialize the serve-sim runtime (a one-time
|
||||
|
|
@ -174,24 +170,22 @@ export class IosEmulatorBackend implements EmulatorBackend {
|
|||
await this.execServeSim(['rotate', orientation, '-d', udid])
|
||||
}
|
||||
|
||||
// Mirrors gesture: the tree comes from the active session's helper endpoint,
|
||||
// so without a session there is nothing to query.
|
||||
async accessibilityTree(_deviceId: string, axUrl: string | null): Promise<unknown> {
|
||||
if (!axUrl) {
|
||||
throw new EmulatorError(
|
||||
'emulator_no_active',
|
||||
'No active emulator session for the accessibility tree. Start one first.'
|
||||
)
|
||||
}
|
||||
return this.fetchAccessibilityTree(axUrl)
|
||||
}
|
||||
|
||||
async exec(deviceId: string, command: string): Promise<unknown> {
|
||||
const udid = await this.resolveDeviceId(deviceId)
|
||||
const rawArgs = stripEmulatorTargetArgs(parseServeSimCommandArgs(command.trim()))
|
||||
return this.execServeSim([...rawArgs, '-d', udid], { json: true })
|
||||
}
|
||||
|
||||
async accessibilityTree(_deviceId: string, axUrl?: string): Promise<unknown> {
|
||||
if (!axUrl) {
|
||||
throw new EmulatorError(
|
||||
'emulator_no_active',
|
||||
'No active iOS emulator AX endpoint — attach the simulator first.'
|
||||
)
|
||||
}
|
||||
return requestServeSimAccessibilityTree(axUrl)
|
||||
}
|
||||
|
||||
async startSession(deviceId: string): Promise<EmulatorSessionInfo> {
|
||||
const udid = await this.resolveDeviceId(deviceId)
|
||||
await ensureSimulatorBooted(udid)
|
||||
|
|
|
|||
|
|
@ -14,5 +14,4 @@ export type EmulatorSessionState = {
|
|||
|
||||
export type EmulatorBridgeOptions = {
|
||||
waitForEndpointReady?: (endpoint: string) => Promise<boolean>
|
||||
fetchAccessibilityTree?: (axUrl: string) => Promise<unknown>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,16 +9,20 @@ const {
|
|||
killServeSimHelperProcessesForDeviceMock,
|
||||
listSimulatorDevicesMock,
|
||||
listServeSimHelperProcessesForDeviceMock,
|
||||
shutdownSimulatorDeviceMock
|
||||
shutdownSimulatorDeviceMock,
|
||||
netFetchMock
|
||||
} = vi.hoisted(() => ({
|
||||
execServeSimCommandMock: vi.fn(async () => ({})),
|
||||
hideNativeSimulatorAppMock: vi.fn(async () => {}),
|
||||
killServeSimHelperProcessesForDeviceMock: vi.fn(async () => {}),
|
||||
listSimulatorDevicesMock: vi.fn(async (): Promise<SimulatorDevice[]> => []),
|
||||
listServeSimHelperProcessesForDeviceMock: vi.fn(async (): Promise<ServeSimHelperProcess[]> => []),
|
||||
shutdownSimulatorDeviceMock: vi.fn(async () => {})
|
||||
shutdownSimulatorDeviceMock: vi.fn(async () => {}),
|
||||
netFetchMock: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('electron', () => ({ net: { fetch: netFetchMock } }))
|
||||
|
||||
vi.mock('./serve-sim-execution', () => ({
|
||||
execServeSimCommand: execServeSimCommandMock,
|
||||
parseServeSimCommandArgs: vi.fn(() => []),
|
||||
|
|
@ -62,6 +66,7 @@ function session(deviceUdid: string): EmulatorSessionInfo {
|
|||
deviceUdid,
|
||||
streamUrl: `http://127.0.0.1:3100/${deviceUdid}`,
|
||||
wsUrl: `ws://127.0.0.1:3100/${deviceUdid}`,
|
||||
axUrl: `http://127.0.0.1:3100/${deviceUdid}/ax`,
|
||||
helperPid: 1234,
|
||||
// iOS serve-sim sessions round-trip through the registry as mjpeg.
|
||||
streamCodec: 'mjpeg'
|
||||
|
|
@ -84,6 +89,7 @@ describe('EmulatorBridge helper ownership', () => {
|
|||
hideNativeSimulatorAppMock.mockImplementation(async () => {})
|
||||
shutdownSimulatorDeviceMock.mockReset()
|
||||
shutdownSimulatorDeviceMock.mockImplementation(async () => {})
|
||||
netFetchMock.mockReset()
|
||||
})
|
||||
|
||||
it('stops the previous Orca-managed helper when a worktree switches devices', async () => {
|
||||
|
|
@ -184,51 +190,12 @@ describe('EmulatorBridge helper ownership', () => {
|
|||
|
||||
it('rejects a capability the resolved backend does not support', async () => {
|
||||
const bridge = new EmulatorBridge()
|
||||
// device-1 resolves to the iOS backend, which does not advertise install.
|
||||
// device-1 resolves to the iOS backend, which advertises no explicit-verb caps.
|
||||
await expect(
|
||||
bridge.runCapability('install', { device: 'device-1' }, async () => 'unused')
|
||||
).rejects.toMatchObject({ code: 'emulator_unsupported' })
|
||||
})
|
||||
|
||||
it('routes ax to the active session ax endpoint on iOS', async () => {
|
||||
const fetchAccessibilityTree = vi.fn(async () => ({ elements: [] }))
|
||||
const bridge = new EmulatorBridge({ fetchAccessibilityTree })
|
||||
bridge.registerActiveEmulator('wt-1', {
|
||||
...session('device-1'),
|
||||
axUrl: 'http://127.0.0.1:3100/device-1/ax'
|
||||
})
|
||||
|
||||
await expect(bridge.accessibilityTree({ worktreeId: 'wt-1' })).resolves.toEqual({
|
||||
elements: []
|
||||
})
|
||||
expect(fetchAccessibilityTree).toHaveBeenCalledWith('http://127.0.0.1:3100/device-1/ax')
|
||||
})
|
||||
|
||||
it('derives the ax endpoint for sessions registered without axUrl', async () => {
|
||||
const fetchAccessibilityTree = vi.fn(async () => ({ elements: [] }))
|
||||
const bridge = new EmulatorBridge({ fetchAccessibilityTree })
|
||||
// e.g. renderer-supplied session info that predates ax derivation.
|
||||
bridge.registerActiveEmulator('wt-1', {
|
||||
...session('device-1'),
|
||||
streamUrl: 'http://127.0.0.1:3100/stream.mjpeg'
|
||||
})
|
||||
|
||||
await expect(bridge.accessibilityTree({ worktreeId: 'wt-1' })).resolves.toEqual({
|
||||
elements: []
|
||||
})
|
||||
expect(fetchAccessibilityTree).toHaveBeenCalledWith('http://127.0.0.1:3100/ax')
|
||||
})
|
||||
|
||||
it('rejects ax when the session has no ax endpoint and none can be derived', async () => {
|
||||
const bridge = new EmulatorBridge()
|
||||
// session() streamUrl has no mjpeg suffix, so no /ax endpoint can be inferred.
|
||||
bridge.registerActiveEmulator('wt-1', session('device-1'))
|
||||
|
||||
await expect(bridge.accessibilityTree({ worktreeId: 'wt-1' })).rejects.toMatchObject({
|
||||
code: 'emulator_no_active'
|
||||
})
|
||||
})
|
||||
|
||||
it('kills the helper and shuts down the selected simulator', async () => {
|
||||
const bridge = new EmulatorBridge()
|
||||
bridge.registerActiveEmulator('wt-1', session('device-1'), { managed: true })
|
||||
|
|
@ -391,6 +358,190 @@ describe('RuntimeEmulatorCommands attach lifecycle', () => {
|
|||
hideNativeSimulatorAppMock.mockImplementation(async () => {})
|
||||
shutdownSimulatorDeviceMock.mockReset()
|
||||
shutdownSimulatorDeviceMock.mockImplementation(async () => {})
|
||||
netFetchMock.mockReset()
|
||||
})
|
||||
|
||||
it('reads iOS accessibility from the active worktree session', async () => {
|
||||
const tree = [{ type: 'Application', children: [] }]
|
||||
netFetchMock.mockResolvedValue(new Response(JSON.stringify(tree), { status: 200 }))
|
||||
const bridge = new EmulatorBridge()
|
||||
bridge.registerActiveEmulator('wt-1', session('device-1'), { managed: true })
|
||||
const commands = new RuntimeEmulatorCommands({
|
||||
getEmulatorBridge: () => bridge,
|
||||
resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-1' })),
|
||||
getAuthoritativeWindow: () => ({ webContents: { send: vi.fn() } }) as never,
|
||||
getSettings: () => ({
|
||||
mobileEmulatorEnabled: true,
|
||||
mobileEmulatorDefaultDeviceUdid: null
|
||||
})
|
||||
})
|
||||
|
||||
// Routing test: normalization is covered in serve-sim-ax-normalization.test.ts.
|
||||
await expect(commands.emulatorAx({ worktree: 'wt-1' })).resolves.toMatchObject([
|
||||
{ type: 'Application' }
|
||||
])
|
||||
expect(netFetchMock).toHaveBeenCalledWith(
|
||||
'http://127.0.0.1:3100/device-1/ax',
|
||||
expect.any(Object)
|
||||
)
|
||||
})
|
||||
|
||||
it('reads iOS accessibility from an attached device without a worktree', async () => {
|
||||
const tree = [{ type: 'Application', children: [] }]
|
||||
netFetchMock.mockResolvedValue(new Response(JSON.stringify(tree), { status: 200 }))
|
||||
listSimulatorDevicesMock.mockResolvedValue([
|
||||
{
|
||||
name: 'iPhone attached',
|
||||
udid: 'device-1',
|
||||
state: 'Booted',
|
||||
runtime: 'iOS 26.0'
|
||||
}
|
||||
])
|
||||
const bridge = new EmulatorBridge()
|
||||
bridge.registerActiveEmulator('wt-1', session('device-1'), { managed: true })
|
||||
const commands = new RuntimeEmulatorCommands({
|
||||
getEmulatorBridge: () => bridge,
|
||||
resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-1' })),
|
||||
getAuthoritativeWindow: () => ({ webContents: { send: vi.fn() } }) as never,
|
||||
getSettings: () => ({
|
||||
mobileEmulatorEnabled: true,
|
||||
mobileEmulatorDefaultDeviceUdid: null
|
||||
})
|
||||
})
|
||||
|
||||
await expect(commands.emulatorAx({ device: 'device-1' })).resolves.toMatchObject([
|
||||
{ type: 'Application' }
|
||||
])
|
||||
expect(netFetchMock).toHaveBeenCalledWith(
|
||||
'http://127.0.0.1:3100/device-1/ax',
|
||||
expect.any(Object)
|
||||
)
|
||||
})
|
||||
|
||||
it('reads ax for an explicit device when the worktree has no active session', async () => {
|
||||
const tree = [{ type: 'Application', children: [] }]
|
||||
netFetchMock.mockResolvedValue(new Response(JSON.stringify(tree), { status: 200 }))
|
||||
listSimulatorDevicesMock.mockResolvedValue([
|
||||
{
|
||||
name: 'iPhone elsewhere',
|
||||
udid: 'device-1',
|
||||
state: 'Booted',
|
||||
runtime: 'iOS 26.0'
|
||||
}
|
||||
])
|
||||
const bridge = new EmulatorBridge()
|
||||
// The session lives under another worktree; the CLI still resolves the
|
||||
// caller's cwd worktree, which has nothing attached.
|
||||
bridge.registerActiveEmulator('wt-other', session('device-1'), { managed: true })
|
||||
const commands = new RuntimeEmulatorCommands({
|
||||
getEmulatorBridge: () => bridge,
|
||||
resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-1' })),
|
||||
getAuthoritativeWindow: () => ({ webContents: { send: vi.fn() } }) as never,
|
||||
getSettings: () => ({
|
||||
mobileEmulatorEnabled: true,
|
||||
mobileEmulatorDefaultDeviceUdid: null
|
||||
})
|
||||
})
|
||||
|
||||
await expect(
|
||||
commands.emulatorAx({ device: 'device-1', worktree: 'wt-1' })
|
||||
).resolves.toMatchObject([{ type: 'Application' }])
|
||||
expect(netFetchMock).toHaveBeenCalledWith(
|
||||
'http://127.0.0.1:3100/device-1/ax',
|
||||
expect.any(Object)
|
||||
)
|
||||
})
|
||||
|
||||
it('reports when the requested iOS device differs from the active session', async () => {
|
||||
listSimulatorDevicesMock.mockResolvedValue([
|
||||
{
|
||||
name: 'iPhone requested',
|
||||
udid: 'device-requested',
|
||||
state: 'Booted',
|
||||
runtime: 'iOS 26.0'
|
||||
}
|
||||
])
|
||||
const bridge = new EmulatorBridge()
|
||||
bridge.registerActiveEmulator('wt-1', session('device-active'), { managed: true })
|
||||
const commands = new RuntimeEmulatorCommands({
|
||||
getEmulatorBridge: () => bridge,
|
||||
resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-1' })),
|
||||
getAuthoritativeWindow: () => ({ webContents: { send: vi.fn() } }) as never,
|
||||
getSettings: () => ({
|
||||
mobileEmulatorEnabled: true,
|
||||
mobileEmulatorDefaultDeviceUdid: null
|
||||
})
|
||||
})
|
||||
|
||||
await expect(
|
||||
commands.emulatorAx({ device: 'device-requested', worktree: 'wt-1' })
|
||||
).rejects.toMatchObject({
|
||||
code: 'emulator_no_active',
|
||||
message: expect.stringContaining('active: device-active')
|
||||
})
|
||||
expect(netFetchMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('heals a session registered without an axUrl by deriving it from the stream url', async () => {
|
||||
const tree = [{ type: 'Application', children: [] }]
|
||||
netFetchMock.mockResolvedValue(new Response(JSON.stringify(tree), { status: 200 }))
|
||||
const bridge = new EmulatorBridge()
|
||||
// No axUrl on the registered session (e.g. reattach path predating derivation).
|
||||
bridge.registerActiveEmulator(
|
||||
'wt-1',
|
||||
{
|
||||
deviceUdid: 'device-1',
|
||||
streamUrl: 'http://127.0.0.1:3100/helper/device-1/stream.mjpeg',
|
||||
wsUrl: 'ws://127.0.0.1:3100/helper/device-1/ws',
|
||||
streamCodec: 'mjpeg'
|
||||
},
|
||||
{ managed: true }
|
||||
)
|
||||
const commands = new RuntimeEmulatorCommands({
|
||||
getEmulatorBridge: () => bridge,
|
||||
resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-1' })),
|
||||
getAuthoritativeWindow: () => ({ webContents: { send: vi.fn() } }) as never,
|
||||
getSettings: () => ({
|
||||
mobileEmulatorEnabled: true,
|
||||
mobileEmulatorDefaultDeviceUdid: null
|
||||
})
|
||||
})
|
||||
|
||||
await expect(commands.emulatorAx({ worktree: 'wt-1' })).resolves.toMatchObject([
|
||||
{ type: 'Application' }
|
||||
])
|
||||
expect(netFetchMock).toHaveBeenCalledWith(
|
||||
'http://127.0.0.1:3100/helper/device-1/ax',
|
||||
expect.any(Object)
|
||||
)
|
||||
})
|
||||
|
||||
it('does not fabricate an /ax endpoint from a non-mjpeg stream url', async () => {
|
||||
const bridge = new EmulatorBridge()
|
||||
bridge.registerActiveEmulator(
|
||||
'wt-1',
|
||||
{
|
||||
deviceUdid: 'device-1',
|
||||
streamUrl: 'http://127.0.0.1:3100/helper/device-1/stream.h264',
|
||||
wsUrl: 'ws://127.0.0.1:3100/helper/device-1/ws',
|
||||
streamCodec: 'mjpeg'
|
||||
},
|
||||
{ managed: true }
|
||||
)
|
||||
const commands = new RuntimeEmulatorCommands({
|
||||
getEmulatorBridge: () => bridge,
|
||||
resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-1' })),
|
||||
getAuthoritativeWindow: () => ({ webContents: { send: vi.fn() } }) as never,
|
||||
getSettings: () => ({
|
||||
mobileEmulatorEnabled: true,
|
||||
mobileEmulatorDefaultDeviceUdid: null
|
||||
})
|
||||
})
|
||||
|
||||
await expect(commands.emulatorAx({ worktree: 'wt-1' })).rejects.toMatchObject({
|
||||
code: 'emulator_no_active'
|
||||
})
|
||||
expect(netFetchMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reconnects to an existing active helper instead of replacing it', async () => {
|
||||
|
|
@ -420,29 +571,6 @@ describe('RuntimeEmulatorCommands attach lifecycle', () => {
|
|||
})
|
||||
})
|
||||
|
||||
it('routes emulatorAx through the bridge to the active session ax endpoint', async () => {
|
||||
const fetchAccessibilityTree = vi.fn(async () => ({ elements: [{ label: 'Login' }] }))
|
||||
const bridge = new EmulatorBridge({ fetchAccessibilityTree })
|
||||
bridge.registerActiveEmulator('wt-1', {
|
||||
...session('device-1'),
|
||||
axUrl: 'http://127.0.0.1:3100/device-1/ax'
|
||||
})
|
||||
const commands = new RuntimeEmulatorCommands({
|
||||
getEmulatorBridge: () => bridge,
|
||||
resolveWorktreeSelector: vi.fn(async () => ({ id: 'wt-1' })),
|
||||
getAuthoritativeWindow: () => ({ webContents: { send: vi.fn() } }) as never,
|
||||
getSettings: () => ({
|
||||
mobileEmulatorEnabled: true,
|
||||
mobileEmulatorDefaultDeviceUdid: null
|
||||
})
|
||||
})
|
||||
|
||||
await expect(commands.emulatorAx({ worktree: 'wt-1' })).resolves.toEqual({
|
||||
elements: [{ label: 'Login' }]
|
||||
})
|
||||
expect(fetchAccessibilityTree).toHaveBeenCalledWith('http://127.0.0.1:3100/device-1/ax')
|
||||
})
|
||||
|
||||
it('rejects attach when mobile emulator is disabled', async () => {
|
||||
const bridge = new EmulatorBridge()
|
||||
const commands = new RuntimeEmulatorCommands({
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import type { SimulatorDevice } from './simctl-simulator-devices'
|
|||
import type { EmulatorBridgeOptions } from './emulator-bridge-types'
|
||||
import type { EmulatorGesturePoint } from './emulator-gesture-sender'
|
||||
import { EmulatorSessionRegistry } from './emulator-session-registry'
|
||||
import { deriveServeSimAxUrl } from './serve-sim-detached-session'
|
||||
import { deriveAxUrlFromStreamUrl } from './serve-sim-detached-session'
|
||||
import { IosEmulatorBackend } from './backends/ios-emulator-backend'
|
||||
import { AndroidEmulatorBackend } from './backends/android-emulator-backend'
|
||||
import type {
|
||||
|
|
@ -202,11 +202,26 @@ export class EmulatorBridge {
|
|||
|
||||
async accessibilityTree(opts?: EmulatorTargetOpts): Promise<unknown> {
|
||||
return this.runCapability('accessibilityTree', opts, async (backend, device) => {
|
||||
if (backend.kind !== 'ios') {
|
||||
return backend.accessibilityTree!(device)
|
||||
}
|
||||
const udid = await backend.resolveDeviceId(device)
|
||||
const session = this.sessionRegistry.getSession(udid)
|
||||
// Fallback heals sessions registered without axUrl (e.g. renderer-supplied
|
||||
// info that predates ax derivation); Android backends ignore the argument.
|
||||
const axUrl = session?.axUrl ?? deriveServeSimAxUrl(session?.streamUrl) ?? null
|
||||
const worktreeId = opts?.worktreeId
|
||||
// Fall back to the udid-keyed session so an explicit --device read works
|
||||
// from a worktree with no active emulator (matching tap/type reachability);
|
||||
// sessions are stored once per udid, so both lookups hit the same state.
|
||||
const session =
|
||||
(worktreeId ? this.getActiveForWorktree(worktreeId) : null) ??
|
||||
this.sessionRegistry.getSession(udid)
|
||||
if (worktreeId && session && session.deviceUdid !== udid) {
|
||||
throw new EmulatorError(
|
||||
'emulator_no_active',
|
||||
`iOS simulator ${udid} is not active for this worktree (active: ${session.deviceUdid}); attach the requested simulator first.`
|
||||
)
|
||||
}
|
||||
// Heal sessions registered without an axUrl (parse-time derivation only
|
||||
// covers fresh --detach output) by deriving it from the mjpeg stream URL.
|
||||
const axUrl = session?.axUrl ?? deriveAxUrlFromStreamUrl(session?.streamUrl)
|
||||
return backend.accessibilityTree!(udid, axUrl)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { netFetchMock } = vi.hoisted(() => ({ netFetchMock: vi.fn() }))
|
||||
|
||||
vi.mock('electron', () => ({ net: { fetch: netFetchMock } }))
|
||||
|
||||
import { requestServeSimAccessibilityTree } from './serve-sim-accessibility-tree'
|
||||
|
||||
const AX_URL = 'http://127.0.0.1:3100/ax'
|
||||
|
||||
describe('requestServeSimAccessibilityTree', () => {
|
||||
beforeEach(() => {
|
||||
netFetchMock.mockReset()
|
||||
})
|
||||
|
||||
it('fetches the one-shot JSON tree and returns it normalized to 0..1', async () => {
|
||||
const raw = [
|
||||
{
|
||||
type: 'Application',
|
||||
role_description: 'application',
|
||||
AXLabel: 'Root',
|
||||
enabled: true,
|
||||
frame: { x: 0, y: 0, width: 200, height: 400 },
|
||||
children: [
|
||||
{
|
||||
type: 'Button',
|
||||
role_description: 'button',
|
||||
AXLabel: 'OK',
|
||||
enabled: true,
|
||||
frame: { x: 50, y: 100, width: 100, height: 40 },
|
||||
children: []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
netFetchMock.mockResolvedValue(new Response(JSON.stringify(raw), { status: 200 }))
|
||||
|
||||
const tree = await requestServeSimAccessibilityTree(AX_URL)
|
||||
|
||||
expect(tree).toEqual([
|
||||
{
|
||||
role: 'application',
|
||||
type: 'Application',
|
||||
label: 'Root',
|
||||
value: '',
|
||||
enabled: true,
|
||||
frame: { x: 0, y: 0, width: 1, height: 1 },
|
||||
children: [
|
||||
{
|
||||
role: 'button',
|
||||
type: 'Button',
|
||||
label: 'OK',
|
||||
value: '',
|
||||
enabled: true,
|
||||
frame: { x: 0.25, y: 0.25, width: 0.5, height: 0.1 },
|
||||
children: []
|
||||
}
|
||||
]
|
||||
}
|
||||
])
|
||||
expect(netFetchMock).toHaveBeenCalledWith(
|
||||
AX_URL,
|
||||
expect.objectContaining({ signal: expect.any(AbortSignal) })
|
||||
)
|
||||
})
|
||||
|
||||
it('surfaces a retry hint when accessibility is temporarily unavailable (503)', async () => {
|
||||
netFetchMock.mockResolvedValue(new Response('{"error":"ax_unavailable"}', { status: 503 }))
|
||||
|
||||
await expect(requestServeSimAccessibilityTree(AX_URL)).rejects.toMatchObject({
|
||||
code: 'emulator_helper_failed',
|
||||
message: expect.stringContaining('retry')
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a non-array or unparseable payload', async () => {
|
||||
netFetchMock.mockResolvedValueOnce(new Response('{"not":"an array"}', { status: 200 }))
|
||||
await expect(requestServeSimAccessibilityTree(AX_URL)).rejects.toMatchObject({
|
||||
code: 'emulator_error'
|
||||
})
|
||||
|
||||
netFetchMock.mockResolvedValueOnce(new Response('not json', { status: 200 }))
|
||||
await expect(requestServeSimAccessibilityTree(AX_URL)).rejects.toMatchObject({
|
||||
code: 'emulator_error'
|
||||
})
|
||||
})
|
||||
|
||||
it('maps a network failure to a helper error', async () => {
|
||||
netFetchMock.mockRejectedValue(new Error('connect ECONNREFUSED'))
|
||||
|
||||
await expect(requestServeSimAccessibilityTree(AX_URL)).rejects.toMatchObject({
|
||||
code: 'emulator_helper_failed',
|
||||
message: expect.stringContaining('Unable to read serve-sim AX')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
import { net } from 'electron'
|
||||
import { EmulatorError } from './emulator-errors'
|
||||
import { normalizeServeSimAxTree, type NormalizedAxNode } from './serve-sim-ax-normalization'
|
||||
|
||||
const AX_REQUEST_TIMEOUT_MS = 5_000
|
||||
const MAX_ERROR_BODY_LENGTH = 512
|
||||
|
||||
export async function requestServeSimAccessibilityTree(axUrl: string): Promise<NormalizedAxNode[]> {
|
||||
try {
|
||||
const response = await net.fetch(axUrl, {
|
||||
signal: AbortSignal.timeout(AX_REQUEST_TIMEOUT_MS)
|
||||
})
|
||||
const body = await response.text()
|
||||
if (!response.ok) {
|
||||
const detail = body.slice(0, MAX_ERROR_BODY_LENGTH) || response.statusText
|
||||
const retry = response.status === 503 ? ' Accessibility may still be warming up; retry.' : ''
|
||||
throw new EmulatorError(
|
||||
'emulator_helper_failed',
|
||||
`serve-sim AX request failed (${response.status}): ${detail}.${retry}`
|
||||
)
|
||||
}
|
||||
|
||||
let tree: unknown
|
||||
try {
|
||||
tree = JSON.parse(body)
|
||||
} catch {
|
||||
throw new EmulatorError('emulator_error', 'serve-sim AX returned invalid JSON.')
|
||||
}
|
||||
if (
|
||||
!Array.isArray(tree) ||
|
||||
tree.some((node) => typeof node !== 'object' || node === null || Array.isArray(node))
|
||||
) {
|
||||
throw new EmulatorError('emulator_error', 'serve-sim AX returned an invalid tree.')
|
||||
}
|
||||
// serve-sim reports frames in absolute pixels; normalize to 0..1 so the
|
||||
// output feeds straight back into tap/gesture.
|
||||
return normalizeServeSimAxTree(tree)
|
||||
} catch (error) {
|
||||
if (error instanceof EmulatorError) {
|
||||
throw error
|
||||
}
|
||||
const detail =
|
||||
error instanceof Error && error.name === 'TimeoutError'
|
||||
? 'request timed out'
|
||||
: error instanceof Error
|
||||
? error.message
|
||||
: 'unknown request failure'
|
||||
throw new EmulatorError('emulator_helper_failed', `Unable to read serve-sim AX: ${detail}`)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import { normalizeServeSimAxTree } from './serve-sim-ax-normalization'
|
||||
|
||||
describe('normalizeServeSimAxTree', () => {
|
||||
it('normalizes frames to 0..1 over the first root screen frame and nests children', () => {
|
||||
const raw = [
|
||||
{
|
||||
type: 'Application',
|
||||
role_description: 'application',
|
||||
AXLabel: 'Demo',
|
||||
AXValue: '',
|
||||
AXUniqueId: null,
|
||||
enabled: true,
|
||||
frame: { x: 0, y: 0, width: 400, height: 800 },
|
||||
children: [
|
||||
{
|
||||
type: 'Button',
|
||||
role_description: 'button',
|
||||
AXLabel: 'Continue',
|
||||
AXValue: 'go',
|
||||
AXUniqueId: 'btn-1',
|
||||
enabled: true,
|
||||
frame: { x: 100, y: 400, width: 200, height: 50 },
|
||||
children: []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
expect(normalizeServeSimAxTree(raw)).toEqual([
|
||||
{
|
||||
role: 'application',
|
||||
type: 'Application',
|
||||
label: 'Demo',
|
||||
value: '',
|
||||
enabled: true,
|
||||
frame: { x: 0, y: 0, width: 1, height: 1 },
|
||||
children: [
|
||||
{
|
||||
role: 'button',
|
||||
type: 'Button',
|
||||
label: 'Continue',
|
||||
value: 'go',
|
||||
enabled: true,
|
||||
id: 'btn-1',
|
||||
frame: { x: 0.25, y: 0.5, width: 0.5, height: 0.0625 },
|
||||
children: []
|
||||
}
|
||||
]
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('normalizes relative to a screen frame with a non-zero origin', () => {
|
||||
const raw = [
|
||||
{
|
||||
type: 'Window',
|
||||
frame: { x: 10, y: 20, width: 200, height: 400 },
|
||||
children: [
|
||||
{ type: 'Cell', frame: { x: 60, y: 120, width: 100, height: 100 }, children: [] }
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
const [root] = normalizeServeSimAxTree(raw)
|
||||
expect(root.frame).toEqual({ x: 0, y: 0, width: 1, height: 1 })
|
||||
expect(root.children[0]!.frame).toEqual({ x: 0.25, y: 0.25, width: 0.5, height: 0.25 })
|
||||
})
|
||||
|
||||
it('marks a disabled element and defaults missing text fields to empty strings', () => {
|
||||
const raw = [
|
||||
{
|
||||
type: 'StaticText',
|
||||
enabled: false,
|
||||
frame: { x: 0, y: 0, width: 100, height: 100 },
|
||||
children: []
|
||||
}
|
||||
]
|
||||
|
||||
expect(normalizeServeSimAxTree(raw)[0]).toMatchObject({
|
||||
role: '',
|
||||
type: 'StaticText',
|
||||
label: '',
|
||||
value: '',
|
||||
enabled: false
|
||||
})
|
||||
})
|
||||
|
||||
it('caps the tree at 500 nodes and marks the parent whose children were cut', () => {
|
||||
const child = (label: string) => ({
|
||||
type: 'StaticText',
|
||||
AXLabel: label,
|
||||
frame: { x: 0, y: 0, width: 10, height: 10 },
|
||||
children: []
|
||||
})
|
||||
const raw = [
|
||||
{
|
||||
type: 'Application',
|
||||
frame: { x: 0, y: 0, width: 400, height: 800 },
|
||||
children: Array.from({ length: 600 }, (_, i) => child(`row-${i}`))
|
||||
}
|
||||
]
|
||||
|
||||
const [root] = normalizeServeSimAxTree(raw)
|
||||
// Root consumes one slot of the 500-node budget.
|
||||
expect(root.children).toHaveLength(499)
|
||||
expect(root.truncated).toBe(true)
|
||||
expect(root.children[0]!.truncated).toBeUndefined()
|
||||
})
|
||||
|
||||
it('falls back to a unit screen for malformed roots instead of dividing by zero', () => {
|
||||
const raw = [{ type: 'Application', children: [] }]
|
||||
expect(normalizeServeSimAxTree(raw)).toEqual([
|
||||
{
|
||||
role: '',
|
||||
type: 'Application',
|
||||
label: '',
|
||||
value: '',
|
||||
enabled: true,
|
||||
frame: { x: 0, y: 0, width: 0, height: 0 },
|
||||
children: []
|
||||
}
|
||||
])
|
||||
expect(normalizeServeSimAxTree([])).toEqual([])
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
// Normalizes serve-sim's raw /ax node tree into a compact nested tree whose
|
||||
// frames are in 0..1 device coordinates. serve-sim's helper reports frames in
|
||||
// absolute pixels; `tap`/`gesture` take normalized 0..1 — so we normalize here
|
||||
// to let agents feed element positions straight back into input commands.
|
||||
// Frame derivation mirrors normalizeAxTree in serve-sim/src/ax.ts: the first
|
||||
// root's frame is the device screen.
|
||||
|
||||
export type NormalizedAxFrame = { x: number; y: number; width: number; height: number }
|
||||
|
||||
// Matches serve-sim's own snapshot cap; an unbounded tree can flood agent output.
|
||||
const MAX_AX_NODES = 500
|
||||
|
||||
// One accessibility element, position normalized, children nested (raw tree shape).
|
||||
export type NormalizedAxNode = {
|
||||
role: string
|
||||
type: string
|
||||
label: string
|
||||
value: string
|
||||
enabled: boolean
|
||||
id?: string
|
||||
frame: NormalizedAxFrame
|
||||
children: NormalizedAxNode[]
|
||||
// Present when children were dropped by the node cap.
|
||||
truncated?: true
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null ? (value as Record<string, unknown>) : {}
|
||||
}
|
||||
|
||||
function numeric(value: unknown): number {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : 0
|
||||
}
|
||||
|
||||
function asString(value: unknown): string {
|
||||
return typeof value === 'string' ? value : ''
|
||||
}
|
||||
|
||||
function readFrame(value: unknown): NormalizedAxFrame {
|
||||
const frame = asRecord(value)
|
||||
return {
|
||||
x: numeric(frame.x),
|
||||
y: numeric(frame.y),
|
||||
width: numeric(frame.width),
|
||||
height: numeric(frame.height)
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to a unit screen so a malformed/empty root never divides by zero.
|
||||
function screenFrame(roots: unknown[]): NormalizedAxFrame {
|
||||
const first = readFrame(asRecord(roots[0]).frame)
|
||||
return first.width > 0 && first.height > 0 ? first : { x: 0, y: 0, width: 1, height: 1 }
|
||||
}
|
||||
|
||||
function round4(value: number): number {
|
||||
return Math.round(value * 10_000) / 10_000
|
||||
}
|
||||
|
||||
function normalizeFrame(frame: NormalizedAxFrame, screen: NormalizedAxFrame): NormalizedAxFrame {
|
||||
return {
|
||||
x: round4((frame.x - screen.x) / screen.width),
|
||||
y: round4((frame.y - screen.y) / screen.height),
|
||||
width: round4(frame.width / screen.width),
|
||||
height: round4(frame.height / screen.height)
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeNode(
|
||||
raw: unknown,
|
||||
screen: NormalizedAxFrame,
|
||||
budget: { remaining: number }
|
||||
): NormalizedAxNode {
|
||||
budget.remaining -= 1
|
||||
const node = asRecord(raw)
|
||||
const rawChildren = Array.isArray(node.children) ? node.children : []
|
||||
const children: NormalizedAxNode[] = []
|
||||
for (const child of rawChildren) {
|
||||
if (budget.remaining <= 0) {
|
||||
break
|
||||
}
|
||||
children.push(normalizeNode(child, screen, budget))
|
||||
}
|
||||
const normalized: NormalizedAxNode = {
|
||||
role: asString(node.role_description),
|
||||
type: asString(node.type),
|
||||
label: asString(node.AXLabel),
|
||||
value: asString(node.AXValue),
|
||||
enabled: node.enabled !== false,
|
||||
frame: normalizeFrame(readFrame(node.frame), screen),
|
||||
children
|
||||
}
|
||||
// AXUniqueId is often null; only surface it when the helper provides one.
|
||||
const uniqueId = asString(node.AXUniqueId)
|
||||
if (uniqueId) {
|
||||
normalized.id = uniqueId
|
||||
}
|
||||
if (children.length < rawChildren.length) {
|
||||
normalized.truncated = true
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
export function normalizeServeSimAxTree(roots: unknown[]): NormalizedAxNode[] {
|
||||
const screen = screenFrame(roots)
|
||||
const budget = { remaining: MAX_AX_NODES }
|
||||
const normalized: NormalizedAxNode[] = []
|
||||
for (const root of roots) {
|
||||
if (budget.remaining <= 0) {
|
||||
break
|
||||
}
|
||||
normalized.push(normalizeNode(root, screen, budget))
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
|
@ -1,175 +0,0 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { fetchServeSimAccessibilityTree } from './serve-sim-ax-tree'
|
||||
|
||||
function sseResponse(chunks: string[], init: { status?: number } = {}): Response {
|
||||
const encoder = new TextEncoder()
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
for (const chunk of chunks) {
|
||||
controller.enqueue(encoder.encode(chunk))
|
||||
}
|
||||
controller.close()
|
||||
}
|
||||
})
|
||||
return new Response(body, {
|
||||
status: init.status ?? 200,
|
||||
headers: { 'Content-Type': 'text/event-stream' }
|
||||
})
|
||||
}
|
||||
|
||||
describe('fetchServeSimAccessibilityTree', () => {
|
||||
it('returns the first data event and skips the SSE comment preamble', async () => {
|
||||
const tree = { screen: { width: 393, height: 852 }, elements: [{ label: 'Login' }], errors: [] }
|
||||
const fetchImpl = vi.fn(async () => sseResponse([':\n\n', `data: ${JSON.stringify(tree)}\n\n`]))
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl })
|
||||
).resolves.toEqual(tree)
|
||||
})
|
||||
|
||||
it('prefers a live update over the replayed cached tree', async () => {
|
||||
// The helper replays the cached tree to new clients, then polls the device
|
||||
// and writes a fresh event only if the tree changed — the fresh one must win.
|
||||
const stale = JSON.stringify({ elements: [{ label: 'Old' }] })
|
||||
const fresh = JSON.stringify({ elements: [{ label: 'New' }] })
|
||||
const fetchImpl = vi.fn(async () =>
|
||||
sseResponse([':\n\n', `data: ${stale}\n\n`, `data: ${fresh}\n\n`])
|
||||
)
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl })
|
||||
).resolves.toEqual({ elements: [{ label: 'New' }] })
|
||||
})
|
||||
|
||||
it('settles on the first event when no follow-up arrives within the window', async () => {
|
||||
const encoder = new TextEncoder()
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(':\n\n'))
|
||||
controller.enqueue(encoder.encode('data: {"elements":[]}\n\n'))
|
||||
// Never closes — an unchanged tree writes nothing, so the settle window must end the read.
|
||||
}
|
||||
})
|
||||
const fetchImpl = vi.fn(async () => new Response(body, { status: 200 }))
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl, settleMs: 30 })
|
||||
).resolves.toEqual({ elements: [] })
|
||||
})
|
||||
|
||||
it('handles a data event split across stream chunks', async () => {
|
||||
const payload = JSON.stringify({ elements: [] })
|
||||
const mid = Math.floor(payload.length / 2)
|
||||
const fetchImpl = vi.fn(async () =>
|
||||
sseResponse([':\n\n', `data: ${payload.slice(0, mid)}`, `${payload.slice(mid)}\n\n`])
|
||||
)
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl })
|
||||
).resolves.toEqual({ elements: [] })
|
||||
})
|
||||
|
||||
it('maps a non-200 response to an actionable stale-helper error', async () => {
|
||||
const fetchImpl = vi.fn(async () => sseResponse([], { status: 404 }))
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl })
|
||||
).rejects.toMatchObject({
|
||||
code: 'emulator_error',
|
||||
message: expect.stringContaining('Restart the emulator session')
|
||||
})
|
||||
})
|
||||
|
||||
it('maps a connection failure to emulator_no_active', async () => {
|
||||
const fetchImpl = vi.fn(async () => {
|
||||
throw new TypeError('fetch failed')
|
||||
})
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl })
|
||||
).rejects.toMatchObject({ code: 'emulator_no_active' })
|
||||
})
|
||||
|
||||
it('fails when the stream ends without a data event', async () => {
|
||||
const fetchImpl = vi.fn(async () => sseResponse([':\n\n', ':\n\n']))
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl })
|
||||
).rejects.toMatchObject({ code: 'emulator_helper_failed' })
|
||||
})
|
||||
|
||||
it('times out when no data event arrives', async () => {
|
||||
const fetchImpl = vi.fn(
|
||||
async () =>
|
||||
new Response(
|
||||
new ReadableStream<Uint8Array>({
|
||||
start() {
|
||||
// Never emits and never closes; the timeout abort must win.
|
||||
}
|
||||
}),
|
||||
{ status: 200 }
|
||||
)
|
||||
)
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl, timeoutMs: 50 })
|
||||
).rejects.toMatchObject({ code: 'emulator_error', message: expect.stringMatching(/Timed out/) })
|
||||
})
|
||||
|
||||
// Pull-based: controller.error() discards still-queued chunks, so the stream
|
||||
// must hand out each chunk on its own read before erroring on a later pull.
|
||||
function droppingSseResponse(chunks: string[]): Response {
|
||||
const encoder = new TextEncoder()
|
||||
let step = 0
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (step < chunks.length) {
|
||||
controller.enqueue(encoder.encode(chunks[step]))
|
||||
step += 1
|
||||
} else {
|
||||
controller.error(new TypeError('terminated'))
|
||||
}
|
||||
}
|
||||
})
|
||||
return new Response(body, { status: 200 })
|
||||
}
|
||||
|
||||
it('returns the last captured tree when the stream drops uncleanly mid-read', async () => {
|
||||
const tree = { elements: [{ label: 'Captured' }] }
|
||||
const fetchImpl = vi.fn(async () =>
|
||||
droppingSseResponse([':\n\n', `data: ${JSON.stringify(tree)}\n\n`])
|
||||
)
|
||||
// settleMs is long so the drop, not the settle window, ends the read.
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl, settleMs: 5000 })
|
||||
).resolves.toEqual(tree)
|
||||
})
|
||||
|
||||
it('maps an unclean mid-stream drop with no captured tree to emulator_helper_failed', async () => {
|
||||
const fetchImpl = vi.fn(async () => droppingSseResponse([':\n\n']))
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl })
|
||||
).rejects.toMatchObject({ code: 'emulator_helper_failed' })
|
||||
})
|
||||
|
||||
it('returns the captured tree when the timeout aborts mid-settle', async () => {
|
||||
const encoder = new TextEncoder()
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode(':\n\n'))
|
||||
controller.enqueue(encoder.encode('data: {"elements":[{"label":"Only"}]}\n\n'))
|
||||
// Never closes; the hard timeout must abort while the settle window is open.
|
||||
}
|
||||
})
|
||||
const fetchImpl = vi.fn(async () => new Response(body, { status: 200 }))
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', {
|
||||
fetchImpl,
|
||||
settleMs: 5000,
|
||||
timeoutMs: 40
|
||||
})
|
||||
).resolves.toEqual({ elements: [{ label: 'Only' }] })
|
||||
})
|
||||
|
||||
it('rejects an unparseable ax event with emulator_helper_failed', async () => {
|
||||
const fetchImpl = vi.fn(async () => sseResponse([':\n\n', 'data: not-json\n\n']))
|
||||
await expect(
|
||||
fetchServeSimAccessibilityTree('http://127.0.0.1:3100/ax', { fetchImpl })
|
||||
).rejects.toMatchObject({
|
||||
code: 'emulator_helper_failed',
|
||||
message: expect.stringContaining('unparseable')
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,167 +0,0 @@
|
|||
import { EmulatorError } from './emulator-errors'
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 15_000
|
||||
// Why: /ax replays the last *cached* tree to every new client before polling the
|
||||
// device, and polling pauses while no client is connected — so the first event
|
||||
// can be stale. A fresh event is only written if the tree changed, so we linger
|
||||
// briefly after the first event and return the last one seen: a follow-up means
|
||||
// the cache was stale; silence means the cache still matches the device.
|
||||
const DEFAULT_SETTLE_MS = 800
|
||||
|
||||
export type FetchAccessibilityTree = (axUrl: string) => Promise<unknown>
|
||||
|
||||
export async function fetchServeSimAccessibilityTree(
|
||||
axUrl: string,
|
||||
options: { timeoutMs?: number; settleMs?: number; fetchImpl?: typeof fetch } = {}
|
||||
): Promise<unknown> {
|
||||
const fetchImpl = options.fetchImpl ?? fetch
|
||||
const controller = new AbortController()
|
||||
const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? DEFAULT_TIMEOUT_MS)
|
||||
try {
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetchImpl(axUrl, {
|
||||
headers: { Accept: 'text/event-stream' },
|
||||
signal: controller.signal
|
||||
})
|
||||
} catch {
|
||||
if (controller.signal.aborted) {
|
||||
throw timeoutError(axUrl)
|
||||
}
|
||||
throw new EmulatorError(
|
||||
'emulator_no_active',
|
||||
`serve-sim accessibility endpoint is unreachable at ${axUrl}. Start the emulator session first.`
|
||||
)
|
||||
}
|
||||
if (!response.ok || !response.body) {
|
||||
// Why: a helper spawned by an older serve-sim build may predate /ax.
|
||||
throw new EmulatorError(
|
||||
'emulator_error',
|
||||
`serve-sim /ax endpoint returned HTTP ${response.status}. Restart the emulator session to refresh the helper.`
|
||||
)
|
||||
}
|
||||
const payload = await readSettledSseDataEvent(
|
||||
response.body,
|
||||
controller,
|
||||
options.settleMs ?? DEFAULT_SETTLE_MS
|
||||
)
|
||||
if (payload === null) {
|
||||
throw new EmulatorError(
|
||||
'emulator_helper_failed',
|
||||
'serve-sim /ax stream ended without an accessibility tree event.'
|
||||
)
|
||||
}
|
||||
try {
|
||||
return JSON.parse(payload)
|
||||
} catch {
|
||||
throw new EmulatorError(
|
||||
'emulator_helper_failed',
|
||||
'serve-sim /ax returned an unparseable accessibility tree event.'
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timeout)
|
||||
controller.abort()
|
||||
}
|
||||
}
|
||||
|
||||
function timeoutError(axUrl: string): EmulatorError {
|
||||
return new EmulatorError(
|
||||
'emulator_error',
|
||||
`Timed out waiting for the accessibility tree from ${axUrl}.`
|
||||
)
|
||||
}
|
||||
|
||||
const SETTLED = Symbol('settled')
|
||||
|
||||
// Returns the payload of the last data event seen up to settleMs after the
|
||||
// first one (see DEFAULT_SETTLE_MS), or null if the stream ends with none.
|
||||
async function readSettledSseDataEvent(
|
||||
body: ReadableStream<Uint8Array>,
|
||||
controller: AbortController,
|
||||
settleMs: number
|
||||
): Promise<string | null> {
|
||||
const reader = body.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
let buffer = ''
|
||||
let latest: string | null = null
|
||||
let settled: Promise<typeof SETTLED> | null = null
|
||||
let settleTimer: NodeJS.Timeout | undefined
|
||||
// Why: abort must also win over a body that never emits — reads on a stalled
|
||||
// stream do not observe the fetch signal on their own.
|
||||
const aborted = new Promise<never>((_, reject) => {
|
||||
const rejectAborted = (): void => reject(new Error('aborted'))
|
||||
if (controller.signal.aborted) {
|
||||
rejectAborted()
|
||||
return
|
||||
}
|
||||
controller.signal.addEventListener('abort', rejectAborted, { once: true })
|
||||
})
|
||||
aborted.catch(() => {})
|
||||
try {
|
||||
for (;;) {
|
||||
const result = await Promise.race(
|
||||
settled ? [reader.read(), aborted, settled] : [reader.read(), aborted]
|
||||
)
|
||||
if (result === SETTLED) {
|
||||
return latest
|
||||
}
|
||||
const { done, value } = result
|
||||
if (value) {
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const payload = extractLastDataPayload(buffer)
|
||||
if (payload !== null) {
|
||||
latest = payload
|
||||
settled ??= new Promise((resolve) => {
|
||||
settleTimer = setTimeout(() => resolve(SETTLED), settleMs)
|
||||
})
|
||||
}
|
||||
// Keep only the trailing partial event; complete ones are consumed.
|
||||
const lastBoundary = buffer.lastIndexOf('\n\n')
|
||||
if (lastBoundary !== -1) {
|
||||
buffer = buffer.slice(lastBoundary + 2)
|
||||
}
|
||||
}
|
||||
if (done) {
|
||||
return latest
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// A tree was captured before the stream aborted or dropped — return it rather
|
||||
// than discarding a valid result on an unclean close (helper crash / truncated
|
||||
// chunked stream), which also keeps a raw non-EmulatorError off the caller path.
|
||||
if (latest !== null) {
|
||||
return latest
|
||||
}
|
||||
if (controller.signal.aborted) {
|
||||
throw new EmulatorError(
|
||||
'emulator_error',
|
||||
'Timed out waiting for the accessibility tree event from serve-sim.'
|
||||
)
|
||||
}
|
||||
// Map an unclean mid-stream failure to the EmulatorError contract callers expect.
|
||||
throw new EmulatorError(
|
||||
'emulator_helper_failed',
|
||||
'serve-sim /ax stream ended unexpectedly. Restart the emulator session and retry.'
|
||||
)
|
||||
} finally {
|
||||
if (settleTimer) {
|
||||
clearTimeout(settleTimer)
|
||||
}
|
||||
reader.releaseLock()
|
||||
}
|
||||
}
|
||||
|
||||
function extractLastDataPayload(buffer: string): string | null {
|
||||
let payload: string | null = null
|
||||
for (const event of buffer.split('\n\n').slice(0, -1)) {
|
||||
const dataLines = event
|
||||
.split('\n')
|
||||
.filter((line) => line.startsWith('data:'))
|
||||
.map((line) => line.slice(5).replace(/^ /, ''))
|
||||
if (dataLines.length > 0) {
|
||||
payload = dataLines.join('\n')
|
||||
}
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
|
@ -1,23 +1,8 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import { deriveServeSimAxUrl, parseServeSimDetachedSession } from './serve-sim-detached-session'
|
||||
|
||||
describe('deriveServeSimAxUrl', () => {
|
||||
it('swaps the mjpeg suffix for /ax, preserving the path prefix', () => {
|
||||
expect(deriveServeSimAxUrl('http://127.0.0.1:3100/stream.mjpeg')).toBe(
|
||||
'http://127.0.0.1:3100/ax'
|
||||
)
|
||||
expect(deriveServeSimAxUrl('http://127.0.0.1:3100/device-1/stream.mjpeg')).toBe(
|
||||
'http://127.0.0.1:3100/device-1/ax'
|
||||
)
|
||||
})
|
||||
|
||||
it('does not derive from a non-mjpeg, query-tailed, or missing stream url', () => {
|
||||
// A query string defeats the suffix match, so no /ax is fabricated.
|
||||
expect(deriveServeSimAxUrl('http://127.0.0.1:3100/stream.mjpeg?token=x')).toBeUndefined()
|
||||
expect(deriveServeSimAxUrl('http://127.0.0.1:3100/custom-stream')).toBeUndefined()
|
||||
expect(deriveServeSimAxUrl(undefined)).toBeUndefined()
|
||||
})
|
||||
})
|
||||
import {
|
||||
deriveAxUrlFromStreamUrl,
|
||||
parseServeSimDetachedSession
|
||||
} from './serve-sim-detached-session'
|
||||
|
||||
describe('parseServeSimDetachedSession', () => {
|
||||
it('uses serve-sim streamUrl when present', () => {
|
||||
|
|
@ -33,10 +18,32 @@ describe('parseServeSimDetachedSession', () => {
|
|||
expect(info).toMatchObject({
|
||||
deviceUdid: 'device-1',
|
||||
streamUrl: 'http://127.0.0.1:3100/stream.mjpeg',
|
||||
wsUrl: 'ws://127.0.0.1:3100/ws'
|
||||
wsUrl: 'ws://127.0.0.1:3100/ws',
|
||||
axUrl: 'http://127.0.0.1:3100/ax'
|
||||
})
|
||||
})
|
||||
|
||||
it('derives the device-scoped AX endpoint and preserves an explicit one', () => {
|
||||
const derived = parseServeSimDetachedSession(
|
||||
{
|
||||
streamUrl: 'http://127.0.0.1:3200/helper/device-1/stream.mjpeg',
|
||||
wsUrl: 'ws://127.0.0.1:3200/helper/device-1/ws'
|
||||
},
|
||||
'device-1'
|
||||
)
|
||||
const explicit = parseServeSimDetachedSession(
|
||||
{
|
||||
streamUrl: 'http://127.0.0.1:3200/stream.mjpeg',
|
||||
wsUrl: 'ws://127.0.0.1:3200/ws',
|
||||
axUrl: 'http://127.0.0.1:3200/custom-ax'
|
||||
},
|
||||
'device-1'
|
||||
)
|
||||
|
||||
expect(derived.axUrl).toBe('http://127.0.0.1:3200/helper/device-1/ax')
|
||||
expect(explicit.axUrl).toBe('http://127.0.0.1:3200/custom-ax')
|
||||
})
|
||||
|
||||
it('derives the MJPEG stream endpoint from older serve-sim url output', () => {
|
||||
const info = parseServeSimDetachedSession(
|
||||
{
|
||||
|
|
@ -49,44 +56,21 @@ describe('parseServeSimDetachedSession', () => {
|
|||
|
||||
expect(info.streamUrl).toBe('http://127.0.0.1:3100/stream.mjpeg')
|
||||
})
|
||||
})
|
||||
|
||||
it('derives the ax endpoint when serve-sim omits axUrl', () => {
|
||||
const info = parseServeSimDetachedSession(
|
||||
{
|
||||
device: 'device-1',
|
||||
streamUrl: 'http://127.0.0.1:3100/stream.mjpeg',
|
||||
wsUrl: 'ws://127.0.0.1:3100/ws'
|
||||
},
|
||||
'device-1'
|
||||
describe('deriveAxUrlFromStreamUrl', () => {
|
||||
it('swaps the mjpeg stream suffix for /ax', () => {
|
||||
expect(deriveAxUrlFromStreamUrl('http://127.0.0.1:3100/stream.mjpeg')).toBe(
|
||||
'http://127.0.0.1:3100/ax'
|
||||
)
|
||||
expect(deriveAxUrlFromStreamUrl('http://127.0.0.1:3200/helper/device-1/stream.mjpeg')).toBe(
|
||||
'http://127.0.0.1:3200/helper/device-1/ax'
|
||||
)
|
||||
|
||||
expect(info.axUrl).toBe('http://127.0.0.1:3100/ax')
|
||||
})
|
||||
|
||||
it('does not fabricate an ax endpoint from a non-mjpeg stream url', () => {
|
||||
const info = parseServeSimDetachedSession(
|
||||
{
|
||||
device: 'device-1',
|
||||
streamUrl: 'http://127.0.0.1:3100/custom-stream',
|
||||
wsUrl: 'ws://127.0.0.1:3100/ws'
|
||||
},
|
||||
'device-1'
|
||||
)
|
||||
|
||||
expect(info.axUrl).toBeUndefined()
|
||||
})
|
||||
|
||||
it('keeps an explicit axUrl when serve-sim provides one', () => {
|
||||
const info = parseServeSimDetachedSession(
|
||||
{
|
||||
device: 'device-1',
|
||||
streamUrl: 'http://127.0.0.1:3100/stream.mjpeg',
|
||||
wsUrl: 'ws://127.0.0.1:3100/ws',
|
||||
axUrl: 'http://127.0.0.1:3100/custom-ax'
|
||||
},
|
||||
'device-1'
|
||||
)
|
||||
|
||||
expect(info.axUrl).toBe('http://127.0.0.1:3100/custom-ax')
|
||||
it('never fabricates an /ax endpoint from a non-mjpeg or missing url', () => {
|
||||
expect(deriveAxUrlFromStreamUrl('http://127.0.0.1:3100/stream.h264')).toBeUndefined()
|
||||
expect(deriveAxUrlFromStreamUrl('http://127.0.0.1:3100/')).toBeUndefined()
|
||||
expect(deriveAxUrlFromStreamUrl(undefined)).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -4,17 +4,19 @@ import { tmpdir } from 'node:os'
|
|||
import { EmulatorError } from './emulator-errors'
|
||||
import type { EmulatorSessionInfo } from './emulator-types'
|
||||
|
||||
const MJPEG_STREAM_SUFFIX = '/stream.mjpeg'
|
||||
|
||||
function streamUrlFromServeSimUrl(url: string): string {
|
||||
return url.endsWith('/stream.mjpeg') ? url : `${url.replace(/\/$/, '')}/stream.mjpeg`
|
||||
return url.endsWith(MJPEG_STREAM_SUFFIX) ? url : `${url.replace(/\/$/, '')}${MJPEG_STREAM_SUFFIX}`
|
||||
}
|
||||
|
||||
// Why: serve-sim serves /ax on the helper but omits it from --detach output.
|
||||
// Guarded on the mjpeg suffix so a foreign stream URL never masquerades as an
|
||||
// AX endpoint. Also used by the bridge to heal sessions registered without axUrl.
|
||||
export function deriveServeSimAxUrl(streamUrl: string | undefined): string | undefined {
|
||||
return streamUrl?.endsWith('/stream.mjpeg')
|
||||
? streamUrl.replace(/\/stream\.mjpeg$/, '/ax')
|
||||
: undefined
|
||||
// Derive the helper /ax endpoint by swapping the mjpeg stream suffix. Guarded to
|
||||
// that suffix so a non-mjpeg stream URL never fabricates a bogus /ax endpoint.
|
||||
export function deriveAxUrlFromStreamUrl(streamUrl: string | undefined): string | undefined {
|
||||
if (!streamUrl || !streamUrl.endsWith(MJPEG_STREAM_SUFFIX)) {
|
||||
return undefined
|
||||
}
|
||||
return `${streamUrl.slice(0, -MJPEG_STREAM_SUFFIX.length)}/ax`
|
||||
}
|
||||
|
||||
export function parseServeSimDetachedSession(raw: unknown, udid: string): EmulatorSessionInfo {
|
||||
|
|
@ -33,7 +35,7 @@ export function parseServeSimDetachedSession(raw: unknown, udid: string): Emulat
|
|||
deviceUdid: typeof json.device === 'string' ? json.device : udid,
|
||||
wsUrl: wsUrl ?? '',
|
||||
streamUrl: streamUrl ?? '',
|
||||
axUrl: typeof json.axUrl === 'string' ? json.axUrl : deriveServeSimAxUrl(streamUrl)
|
||||
axUrl: typeof json.axUrl === 'string' ? json.axUrl : deriveAxUrlFromStreamUrl(streamUrl)
|
||||
}
|
||||
if (!info.streamUrl || !info.wsUrl) {
|
||||
throw new EmulatorError('emulator_helper_failed', 'serve-sim did not return stream endpoints.')
|
||||
|
|
|
|||
|
|
@ -249,7 +249,6 @@ export class RuntimeEmulatorCommands {
|
|||
|
||||
async emulatorAx(params: EmulatorTargetParams): Promise<unknown> {
|
||||
const worktreeId = await this.resolveWorktreeId(params.worktree)
|
||||
// Via the bridge (not runCapability directly) so iOS gets the session's /ax endpoint.
|
||||
return this.requireEmulatorBridge().accessibilityTree({
|
||||
device: params.device ?? params.emulator,
|
||||
worktreeId
|
||||
|
|
|
|||
Loading…
Reference in New Issue