orca/tests/e2e/computer-windows.e2e.ts

import { afterAll, beforeAll, describe, expect, test } from 'vitest'
import type {
  ComputerActionResult,
  ComputerListAppsResult,
  ComputerListWindowsResult,
  ComputerSnapshotResult
} from '../../src/shared/runtime-types'
import {
  ensureNotepadLaunched,
  findRoleIndex,
  getNotepadAppSelector,
  killNotepad,
  parseJsonOutput,
  runOrcaCli
} from './helpers/computer-driver'

const isWindows = process.platform === 'win32'
const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'

describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', () => {
  beforeAll(async () => {
    await ensureNotepadLaunched()
  })

  afterAll(async () => {
    await killNotepad()
  })

  test('list-apps includes the test-owned Notepad process', async () => {
    const result = await runOrcaCli(['computer', 'list-apps', '--json'])
    const envelope = parseJsonOutput<{ result: ComputerListAppsResult }>(result.stdout)
    const pid = Number.parseInt(getNotepadAppSelector().slice(4), 10)

    expect(envelope.result.apps).toEqual(
      expect.arrayContaining([expect.objectContaining({ name: 'Notepad', pid })])
    )
  })

  test('list-windows returns a targetable Notepad window', async () => {
    const app = getNotepadAppSelector()
    const result = await runOrcaCli(['computer', 'list-windows', '--app', app, '--json'])
    const envelope = parseJsonOutput<{ result: ComputerListWindowsResult }>(result.stdout)

    expect(envelope.result.windows).toEqual([
      expect.objectContaining({
        index: 0,
        app: expect.objectContaining({ name: 'Notepad' }),
        id: expect.any(Number),
        title: expect.stringContaining('Notepad'),
        width: expect.any(Number),
        height: expect.any(Number)
      })
    ])
  })

  test('Notepad exposes a basic accessibility tree', async () => {
    const result = await runOrcaCli([
      'computer',
      'get-app-state',
      '--app',
      getNotepadAppSelector(),
      '--json'
    ])
    const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout)

    expect(envelope.result.snapshot.app.name).toBe('Notepad')
    expect(envelope.result.snapshot.window.title).toContain('Notepad')
    expect(envelope.result.snapshot.elementCount).toBeGreaterThan(0)
    expect(envelope.result.snapshot.coordinateSpace).toBe('window')
    expect(envelope.result.snapshot.truncation?.truncated).toBe(false)
    expect(envelope.result.screenshotStatus.state).toBe('captured')
    expect(envelope.result.screenshot?.format).toBe('png')
    expect(envelope.result.screenshot?.data).toBeUndefined()
    expect(envelope.result.screenshot?.dataOmitted).toBe(true)
    expect(envelope.result.screenshot?.path).toContain('orca-computer-use')
  })

  test('set-value mutates the document through UI Automation', async () => {
    const app = getNotepadAppSelector()
    const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
      (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
        .stdout
    )
    const documentIndex = findRoleIndex(before.result.snapshot.treeText, 'document')
    expect(documentIndex).toBeGreaterThanOrEqual(0)

    const marker = `orca-windows-set-${Date.now()}`
    const action = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'set-value',
          '--app',
          app,
          '--element-index',
          String(documentIndex),
          '--value',
          marker,
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(action.result.action?.path).toBe('accessibility')

    expect(action.result.snapshot.treeText).toContain(marker)
  })

  test('paste-text mutates the test-owned document', async () => {
    const app = getNotepadAppSelector()
    const marker = `orca-windows-paste-${Date.now()}`
    const action = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'paste-text',
          '--app',
          app,
          '--text',
          marker,
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(action.result.action?.path).toBe('clipboard')

    const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(
      (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
        .stdout
    )
    expect(after.result.snapshot.treeText).toContain(marker)
  })

  test('Unicode payloads survive set-value and paste-text', async () => {
    const app = getNotepadAppSelector()
    const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
      (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
        .stdout
    )
    const documentIndex = findRoleIndex(before.result.snapshot.treeText, 'document')
    expect(documentIndex).toBeGreaterThanOrEqual(0)

    const unicode = `orca unicode café Ω 漢字 ${Date.now()}`
    const set = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'set-value',
          '--app',
          app,
          '--element-index',
          String(documentIndex),
          '--value',
          unicode,
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(set.result.snapshot.treeText).toContain(unicode)

    const pasted = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'paste-text',
          '--app',
          app,
          '--text',
          unicode,
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(pasted.result.snapshot.treeText).toContain(unicode)
  })

  test('hotkey and paste-text can replace the document selection', async () => {
    const app = getNotepadAppSelector()
    const first = `orca-windows-first-${Date.now()}`
    await runOrcaCli([
      'computer',
      'paste-text',
      '--app',
      app,
      '--text',
      first,
      '--restore-window',
      '--no-screenshot',
      '--json'
    ])

    const selectAll = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'hotkey',
          '--app',
          app,
          '--key',
          'CmdOrCtrl+A',
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(selectAll.result.action?.actionName).toBe('hotkey')

    const marker = `orca-windows-replaced-${Date.now()}`
    const second = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'paste-text',
          '--app',
          app,
          '--text',
          marker,
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(second.result.snapshot.treeText).toContain(marker)
    expect(second.result.snapshot.treeText).not.toContain(first)
  })

  test('click and type-text send synthetic input to the document', async () => {
    const app = getNotepadAppSelector()
    const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
      (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
        .stdout
    )
    const documentIndex = findRoleIndex(before.result.snapshot.treeText, 'document')
    expect(documentIndex).toBeGreaterThanOrEqual(0)

    await runOrcaCli([
      'computer',
      'click',
      '--app',
      app,
      '--element-index',
      String(documentIndex),
      '--restore-window',
      '--no-screenshot',
      '--json'
    ])

    const marker = ` typed-${Date.now()}`
    const typed = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'type-text',
          '--app',
          app,
          '--text',
          marker,
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(typed.result.action?.path).toBe('synthetic')
    expect(typed.result.snapshot.treeText).toContain(marker.trim())
  })
})