fix(emulator): recycle iOS simulators that wedge-boot without a display framebuffer (#7065)

* fix(emulator): recycle iOS simulators that wedge-boot without a display framebuffer CoreSimulator can report a device Booted while its display IO ports never came up (HID alive, no com.apple.framebuffer.display port), so ensureSimulatorBooted passes and serve-sim --detach dies with 'No framebuffer display descriptor found'. Reconnecting hits the same Booted early-return, so the pane could never recover without a manual simctl shutdown/boot. startSession now recognizes that helper failure signature, recycles the device once (shutdown + boot), and retries; if the display still fails to come up it surfaces an actionable erase/recreate message instead of the raw helper log dump. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * review: harden simulator framebuffer recovery --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Jinjing <6427696+AmethystLiang@users.noreply.github.com>
2026-07-02 18:13:33 -07:00 · 2026-07-02 18:13:33 -07:00 · dbd53ebb63
parent 11ca4955ae
commit dbd53ebb63
6 changed files with 253 additions and 8 deletions
--- a/src/cli/handlers/emulator.test.ts
+++ b/src/cli/handlers/emulator.test.ts
@ -73,6 +73,24 @@ describe('orca emulator CLI handlers', () => {
    })
  })

+  it('uses a wider client timeout for emulator attach recovery', async () => {
+    queueFixtures(
+      callMock,
+      okFixture('req_attach', {
+        attached: true,
+        info: { deviceUdid: 'device-1', streamUrl: 'http://127.0.0.1:3102/stream.mjpeg' }
+      })
+    )
+
+    await main(['emulator', 'attach', 'device-1', '--worktree', 'all'], '/repo/project')
+
+    expect(callMock).toHaveBeenCalledWith(
+      'emulator.attach',
+      { device: 'device-1', worktree: undefined, focus: false },
+      { timeoutMs: 180_000 }
+    )
+  })
+
  it('rejects relative APK paths for remote runtimes', async () => {
    remoteMock.mockReturnValue(true)

--- a/src/cli/handlers/emulator.ts
+++ b/src/cli/handlers/emulator.ts
@ -128,7 +128,13 @@ export const EMULATOR_HANDLERS: Record<string, CommandHandler> = {
    const target = await getEmulatorCommandTarget(flags, cwd, client)
    const device = getOptionalStringFlag(flags, 'device')
    const focus = flags.get('focus') === true
-    const res = await client.call('emulator.attach', { device, worktree: target.worktree, focus })
+    // Why: attach may cold-boot or recycle a wedged simulator (shutdown + boot +
+    // helper restart), which can legitimately exceed the 60s default budget.
+    const res = await client.call(
+      'emulator.attach',
+      { device, worktree: target.worktree, focus },
+      { timeoutMs: 180_000 }
+    )
    printResult(res, json, (r: unknown) => {
      const result = r as EmulatorAttachResult
      const info = result.info ?? result
--- a/src/main/emulator/backends/ios-emulator-backend.test.ts
+++ b/src/main/emulator/backends/ios-emulator-backend.test.ts
@ -3,6 +3,7 @@ import type { SimulatorDevice } from '../simctl-simulator-devices'
 import type { ServeSimHelperProcess } from '../serve-sim-helper-processes'

 const {
+  ensureSimulatorBootedMock,
  execServeSimCommandMock,
  hideNativeSimulatorAppMock,
  killServeSimHelperProcessesForDeviceMock,
@ -12,7 +13,8 @@ const {
  sendEmulatorGestureSequenceMock,
  parseServeSimDetachedSessionMock
 } = vi.hoisted(() => ({
-  execServeSimCommandMock: vi.fn(async () => ({})),
+  ensureSimulatorBootedMock: vi.fn(async () => {}),
+  execServeSimCommandMock: vi.fn(async (_executable?: unknown, _args?: string[]) => ({})),
  hideNativeSimulatorAppMock: vi.fn(async () => {}),
  killServeSimHelperProcessesForDeviceMock: vi.fn(async () => {}),
  listSimulatorDevicesMock: vi.fn(async (): Promise<SimulatorDevice[]> => []),
@ -30,7 +32,7 @@ vi.mock('../serve-sim-execution', () => ({
 }))

 vi.mock('../simctl-simulator-devices', () => ({
-  ensureSimulatorBooted: vi.fn(async () => {}),
+  ensureSimulatorBooted: ensureSimulatorBootedMock,
  listSimulatorDevices: listSimulatorDevicesMock,
  resolveSimulatorUdid: vi.fn(async (device: string) => device),
  shutdownSimulatorDevice: shutdownSimulatorDeviceMock
@ -53,12 +55,15 @@ vi.mock('../serve-sim-detached-session', () => ({
  parseServeSimDetachedSession: parseServeSimDetachedSessionMock
 }))

+import { EmulatorError } from '../emulator-errors'
 import { IosEmulatorBackend } from './ios-emulator-backend'

 const EXECUTABLE = { command: '/serve-sim', env: {} }

 describe('IosEmulatorBackend', () => {
  beforeEach(() => {
+    ensureSimulatorBootedMock.mockReset()
+    ensureSimulatorBootedMock.mockImplementation(async () => {})
    execServeSimCommandMock.mockReset()
    execServeSimCommandMock.mockImplementation(async () => ({}))
    listSimulatorDevicesMock.mockReset()
@ -186,6 +191,113 @@ describe('IosEmulatorBackend', () => {
    expect(hideNativeSimulatorAppMock).toHaveBeenCalledTimes(1)
  })

+  it('recycles the device and retries when the helper finds no framebuffer', async () => {
+    // The real-world failure: simctl reports Booted but the display IO ports
+    // never came up, so serve-sim --detach dies with the framebuffer error.
+    execServeSimCommandMock
+      .mockRejectedValueOnce(
+        new EmulatorError(
+          'emulator_error',
+          'Helper failed:\n[main] Starting serve-sim-bin\n[main] Failed to start capture: No framebuffer display descriptor found'
+        )
+      )
+      .mockResolvedValueOnce({})
+    parseServeSimDetachedSessionMock.mockReturnValue({
+      deviceUdid: 'device-1',
+      streamUrl: 'http://127.0.0.1:3102/stream.mjpeg',
+      wsUrl: 'ws://127.0.0.1:3102',
+      helperPid: 1234
+    })
+    const backend = new IosEmulatorBackend({ waitForEndpointReady: async () => true })
+    const info = await backend.startSession('device-1')
+    expect(info.deviceUdid).toBe('device-1')
+    expect(shutdownSimulatorDeviceMock).toHaveBeenCalledWith('device-1')
+    // Booted once up front, again after the recycle shutdown.
+    expect(ensureSimulatorBootedMock).toHaveBeenCalledTimes(2)
+    expect(execServeSimCommandMock).toHaveBeenCalledTimes(2)
+  })
+
+  it('does not recycle the device for unrelated helper start failures', async () => {
+    execServeSimCommandMock.mockRejectedValueOnce(
+      new EmulatorError('emulator_error', 'Helper failed:\n[main] Port 3100 already in use')
+    )
+    const backend = new IosEmulatorBackend({ waitForEndpointReady: async () => true })
+    await expect(backend.startSession('device-1')).rejects.toMatchObject({
+      message: expect.stringContaining('Port 3100 already in use')
+    })
+    expect(shutdownSimulatorDeviceMock).not.toHaveBeenCalled()
+    expect(execServeSimCommandMock).toHaveBeenCalledTimes(1)
+  })
+
+  it('propagates mid-recycle failures instead of masking them', async () => {
+    execServeSimCommandMock.mockRejectedValue(
+      new EmulatorError(
+        'emulator_error',
+        'Helper failed:\n[main] Failed to start capture: No framebuffer display descriptor found'
+      )
+    )
+    shutdownSimulatorDeviceMock.mockRejectedValueOnce(
+      new EmulatorError('emulator_error', 'xcrun simctl shutdown timed out')
+    )
+    const backend = new IosEmulatorBackend({ waitForEndpointReady: async () => true })
+    await expect(backend.startSession('device-1')).rejects.toMatchObject({
+      message: expect.stringContaining('timed out')
+    })
+    expect(execServeSimCommandMock).toHaveBeenCalledTimes(1)
+  })
+
+  it('surfaces an actionable error when the display stays wedged after one recycle', async () => {
+    execServeSimCommandMock.mockRejectedValue(
+      new EmulatorError(
+        'emulator_error',
+        'Helper failed:\n[main] Failed to start capture: No framebuffer display descriptor found'
+      )
+    )
+    const backend = new IosEmulatorBackend({ waitForEndpointReady: async () => true })
+    await expect(backend.startSession('device-1')).rejects.toMatchObject({
+      code: 'emulator_helper_failed',
+      // Actionable headline plus the raw helper log for diagnosis.
+      message: expect.stringMatching(/simctl erase[\s\S]*No framebuffer display descriptor found/)
+    })
+    // Exactly one recycle attempt; no shutdown/boot loop against a broken device.
+    expect(shutdownSimulatorDeviceMock).toHaveBeenCalledTimes(1)
+    expect(execServeSimCommandMock).toHaveBeenCalledTimes(2)
+  })
+
+  it('does not recycle more than once during one start attempt', async () => {
+    let detachCalls = 0
+    execServeSimCommandMock.mockImplementation(
+      async (_executable?: unknown, args: string[] = []) => {
+        if (args[0] !== '--detach') {
+          return {}
+        }
+        detachCalls += 1
+        if (detachCalls === 2) {
+          return {}
+        }
+        throw new EmulatorError(
+          'emulator_error',
+          'Helper failed:\n[main] Failed to start capture: No framebuffer display descriptor found'
+        )
+      }
+    )
+    parseServeSimDetachedSessionMock.mockReturnValue({
+      deviceUdid: 'device-1',
+      streamUrl: 'http://127.0.0.1:3102/stream.mjpeg',
+      wsUrl: 'ws://127.0.0.1:3102',
+      helperPid: 1234
+    })
+    const backend = new IosEmulatorBackend({ waitForEndpointReady: async () => false })
+
+    await expect(backend.startSession('device-1')).rejects.toMatchObject({
+      code: 'emulator_helper_failed',
+      message: expect.stringContaining('even after a reboot')
+    })
+    expect(detachCalls).toBe(3)
+    expect(shutdownSimulatorDeviceMock).toHaveBeenCalledTimes(1)
+    expect(ensureSimulatorBootedMock).toHaveBeenCalledTimes(2)
+  })
+
  it('stops a helper via serve-sim kill plus the orphan sweep', async () => {
    const backend = new IosEmulatorBackend()
    await backend.stopHelperForDevice('device-1', { helperPid: 1234, includeOrphaned: true })
--- a/src/main/emulator/backends/ios-emulator-backend.ts
+++ b/src/main/emulator/backends/ios-emulator-backend.ts
@ -190,9 +190,43 @@ export class IosEmulatorBackend implements EmulatorBackend {
      return false
    }

-    let info = await startDetachedHelper()
+    const throwPersistentMissingFramebuffer = (error: EmulatorError): never => {
+      throw new EmulatorError(
+        'emulator_helper_failed',
+        `Simulator ${udid} keeps booting without a working display (no framebuffer descriptor), even after a reboot. Erase it with \`xcrun simctl erase ${udid}\` or recreate it in Xcode > Window > Devices and Simulators.\n\n${error.message}`
+      )
+    }
+
+    // Why: CoreSimulator can report "Booted" with the display IO ports down
+    // (HID alive, no framebuffer); a shutdown/boot recycle is the only recovery.
+    let didRecycleWedgedBoot = false
+    const startHelperRecyclingWedgedBoot = async (): Promise<EmulatorSessionInfo> => {
+      try {
+        return await startDetachedHelper()
+      } catch (error) {
+        if (!isMissingFramebufferError(error)) {
+          throw error
+        }
+        if (didRecycleWedgedBoot) {
+          return throwPersistentMissingFramebuffer(error)
+        }
+        didRecycleWedgedBoot = true
+        await shutdownSimulatorDevice(udid)
+        await ensureSimulatorBooted(udid)
+        try {
+          return await startDetachedHelper()
+        } catch (retryError) {
+          if (!isMissingFramebufferError(retryError)) {
+            throw retryError
+          }
+          return throwPersistentMissingFramebuffer(retryError)
+        }
+      }
+    }
+
+    let info = await startHelperRecyclingWedgedBoot()
    if (!(await waitForReadyOrKill(info))) {
-      info = await startDetachedHelper()
+      info = await startHelperRecyclingWedgedBoot()
      if (!(await waitForReadyOrKill(info))) {
        throw new EmulatorError(
          'emulator_helper_failed',
@ -242,6 +276,18 @@ export class IosEmulatorBackend implements EmulatorBackend {
  }
 }

+// Why: serve-sim's capture helper prints exactly this when a booted device has
+// no com.apple.framebuffer.display IO port: the signature of a wedged boot.
+const MISSING_FRAMEBUFFER_RE = /No framebuffer display descriptor found/i
+
+function isMissingFramebufferError(error: unknown): error is EmulatorError {
+  return (
+    error instanceof EmulatorError &&
+    error.code === 'emulator_error' &&
+    MISSING_FRAMEBUFFER_RE.test(error.message)
+  )
+}
+
 function toEmulatorDevice(device: SimulatorDevice): EmulatorDevice {
  return {
    backend: 'ios',
--- a/src/main/emulator/simctl-simulator-devices.test.ts
+++ b/src/main/emulator/simctl-simulator-devices.test.ts
@ -22,7 +22,7 @@ vi.mock('./serve-sim-execution', () => ({
  execServeSimCommand: vi.fn()
 }))

-import { listSimulatorDevices } from './simctl-simulator-devices'
+import { listSimulatorDevices, shutdownSimulatorDevice } from './simctl-simulator-devices'

 describe('listSimulatorDevices', () => {
  beforeEach(() => {
@ -94,3 +94,64 @@ describe('listSimulatorDevices', () => {
    expect((error as Error).message).not.toContain('Command failed')
  })
 })
+
+describe('shutdownSimulatorDevice', () => {
+  beforeEach(() => {
+    execFileMock.mockReset()
+    platformMock.mockReset()
+    platformMock.mockReturnValue('darwin')
+  })
+
+  it('treats an already-shut-down device as success', async () => {
+    execFileMock.mockImplementation((_command, _args, _options, callback) => {
+      callback(
+        Object.assign(
+          new Error('Command failed: xcrun simctl shutdown AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE'),
+          { code: 164 }
+        ),
+        '',
+        'Unable to shutdown device in current state: Shutdown'
+      )
+    })
+
+    await expect(
+      shutdownSimulatorDevice('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')
+    ).resolves.toBeUndefined()
+  })
+
+  it('propagates real shutdown failures instead of swallowing them', async () => {
+    // execFile's message echoes the command line, which always contains
+    // "shutdown"; a hung device (timeout kill) must still reject.
+    execFileMock.mockImplementation((_command, _args, _options, callback) => {
+      callback(
+        Object.assign(
+          new Error('Command failed: xcrun simctl shutdown AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE'),
+          { killed: true, signal: 'SIGTERM' }
+        ),
+        '',
+        ''
+      )
+    })
+
+    await expect(
+      shutdownSimulatorDevice('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')
+    ).rejects.toMatchObject({ code: 'emulator_error' })
+  })
+
+  it('rejects current-state failures that are not already shut down', async () => {
+    execFileMock.mockImplementation((_command, _args, _options, callback) => {
+      callback(
+        Object.assign(
+          new Error('Command failed: xcrun simctl shutdown AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE'),
+          { code: 164 }
+        ),
+        '',
+        'Unable to shutdown device in current state: Booting'
+      )
+    })
+
+    await expect(
+      shutdownSimulatorDevice('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')
+    ).rejects.toMatchObject({ code: 'emulator_error' })
+  })
+})
--- a/src/main/emulator/simctl-simulator-devices.ts
+++ b/src/main/emulator/simctl-simulator-devices.ts
@ -191,8 +191,10 @@ export async function shutdownSimulatorDevice(udid: string): Promise<void> {
          resolve()
          return
        }
-        const message = error.message.toLowerCase()
-        if (message.includes('shutdown') || message.includes('current state')) {
+        // Why: execFile's message echoes the command line, so only the actual
+        // already-off state is idempotent; other current states are real failures.
+        const message = `${error.message}\n${stderr?.toString() ?? ''}`.toLowerCase()
+        if (/\bcurrent state:\s*shutdown\b/.test(message)) {
          resolve()
          return
        }