From 69728c99047ad2f4e921f5bfb20f80fa6fa85b2e Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Fri, 22 May 2026 22:06:47 -0700 Subject: [PATCH] Reduce hidden terminal pressure during Codex typing (#2661) * Reduce hidden terminal pressure during Codex typing * Make terminal lag stress test tunable * Add real Codex terminal stress mode --- .../terminal-pane/pty-connection.test.ts | 95 +++ .../terminal-pane/pty-connection.ts | 102 +++- .../pane-terminal-output-scheduler.test.ts | 29 +- .../pane-terminal-output-scheduler.ts | 6 +- tests/e2e/terminal-codex-lag-stress.spec.ts | 561 ++++++++++++++++++ 5 files changed, 769 insertions(+), 24 deletions(-) create mode 100644 tests/e2e/terminal-codex-lag-stress.spec.ts diff --git a/src/renderer/src/components/terminal-pane/pty-connection.test.ts b/src/renderer/src/components/terminal-pane/pty-connection.test.ts index 79ccb01ac..4e55bce29 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.test.ts @@ -433,6 +433,101 @@ describe('connectPanePty', () => { delete (globalThis as Record).__ptyConnectDiag }) + it('coalesces same-class hidden title frames', async () => { + vi.useFakeTimers() + const { connectPanePty } = await import('./pty-connection') + transportFactoryQueue.push(createMockTransport('pty-1')) + const pane = createPane(1) + const manager = { + ...createManager(1), + getActivePane: vi.fn(() => ({ id: 1 })) + } + const setRuntimePaneTitle = vi.fn((tabId: string, paneId: number, title: string) => { + mockStoreState.runtimePaneTitlesByTabId[tabId] = { + ...mockStoreState.runtimePaneTitlesByTabId[tabId], + [paneId]: title + } + }) + const deps = createDeps({ + isVisibleRef: { current: false }, + setRuntimePaneTitle + }) + + const binding = connectPanePty(pane as never, manager as never, deps as never) + const options = createdTransportOptions[0] as { + onTitleChange: (title: string, rawTitle: string) => void + } + + options.onTitleChange('Codex working one', 'Codex working one') + options.onTitleChange('Codex working two', 'Codex working two') + + expect(setRuntimePaneTitle).toHaveBeenCalledTimes(1) + vi.advanceTimersByTime(99) + expect(setRuntimePaneTitle).toHaveBeenCalledTimes(1) + vi.advanceTimersByTime(1) + expect(setRuntimePaneTitle).toHaveBeenCalledTimes(2) + expect(setRuntimePaneTitle).toHaveBeenLastCalledWith('tab-1', 1, 'Codex working two') + + binding.dispose() + }) + + it('coalesces same-state hidden agent status pings', async () => { + vi.useFakeTimers() + const { connectPanePty } = await import('./pty-connection') + transportFactoryQueue.push(createMockTransport('pty-1')) + const deps = createDeps({ isVisibleRef: { current: false } }) + + const binding = connectPanePty(createPane(1) as never, createManager(1) as never, deps as never) + const options = createdTransportOptions[0] as { + onAgentStatus: (payload: { state: 'working'; prompt: string; toolInput?: string }) => void + } + + options.onAgentStatus({ state: 'working', prompt: 'p', toolInput: 'first' }) + options.onAgentStatus({ state: 'working', prompt: 'p', toolInput: 'second' }) + + expect(mockStoreState.setAgentStatus).toHaveBeenCalledTimes(1) + vi.advanceTimersByTime(100) + expect(mockStoreState.setAgentStatus).toHaveBeenCalledTimes(2) + expect(mockStoreState.setAgentStatus).toHaveBeenLastCalledWith( + makePaneKey('tab-1', LEAF_1), + { state: 'working', prompt: 'p', toolInput: 'second' }, + undefined + ) + + binding.dispose() + }) + + it('keeps hidden agent status transitions immediate', async () => { + vi.useFakeTimers() + const { connectPanePty } = await import('./pty-connection') + transportFactoryQueue.push(createMockTransport('pty-1')) + const deps = createDeps({ isVisibleRef: { current: false } }) + + const binding = connectPanePty(createPane(1) as never, createManager(1) as never, deps as never) + const options = createdTransportOptions[0] as { + onAgentStatus: (payload: { + state: 'working' | 'waiting' | 'done' + prompt: string + toolInput?: string + }) => void + } + + options.onAgentStatus({ state: 'working', prompt: 'p', toolInput: 'first' }) + options.onAgentStatus({ state: 'waiting', prompt: 'p', toolInput: 'blocked' }) + options.onAgentStatus({ state: 'done', prompt: 'p', toolInput: 'complete' }) + + expect(mockStoreState.setAgentStatus).toHaveBeenCalledTimes(3) + vi.advanceTimersByTime(100) + expect(mockStoreState.setAgentStatus).toHaveBeenCalledTimes(3) + expect(mockStoreState.setAgentStatus).toHaveBeenLastCalledWith( + makePaneKey('tab-1', LEAF_1), + { state: 'done', prompt: 'p', toolInput: 'complete' }, + undefined + ) + + binding.dispose() + }) + it('does not retain PTY connect diagnostics unless e2e debug state is enabled', async () => { const { connectPanePty } = await import('./pty-connection') const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) diff --git a/src/renderer/src/components/terminal-pane/pty-connection.ts b/src/renderer/src/components/terminal-pane/pty-connection.ts index 36cfbb94a..6dbc10667 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.ts @@ -33,7 +33,10 @@ import { recordTerminalOutput } from '@/lib/pane-manager/pane-scroll' import { makePaneKey } from '../../../../shared/stable-pane-id' import { createTerminalCommandLifecycle } from './terminal-command-lifecycle' import { e2eConfig } from '@/lib/e2e-config' -import type { AgentStatusEntry } from '../../../../shared/agent-status-types' +import type { + AgentStatusEntry, + ParsedAgentStatusPayload +} from '../../../../shared/agent-status-types' import { isWebTerminalSurfaceTabId } from '@/runtime/web-terminal-surface-id' import { createAgentInterruptInference, @@ -53,6 +56,7 @@ const PTY_CONNECT_DIAG_LIMIT = 200 const AGENT_TASK_COMPLETE_NOTIFICATION_GRACE_MS = 250 const AGENT_TASK_COMPLETE_NOTIFICATION_MAX_WAIT_MS = 1000 const AGENT_TASK_COMPLETE_NOTIFICATION_DETAIL_MAX_AGE_MS = 10_000 +const HIDDEN_TERMINAL_METADATA_FLUSH_MS = 100 let codexRestartNoticePresenceSource: Record< string, { previousAccountLabel: string; nextAccountLabel: string } @@ -476,6 +480,9 @@ export function connectPanePty( const onExit = (ptyId: string): void => { agentCompletionCoordinator.dispose() + clearHiddenMetadataFlushTimer() + pendingHiddenTitle = null + pendingHiddenAgentStatus = null deps.syncPanePtyLayoutBinding(pane.id, null) deps.clearRuntimePaneTitle(deps.tabId, pane.id) deps.clearTabPtyId(deps.tabId, ptyId) @@ -514,8 +521,11 @@ export function connectPanePty( // Claude launches also start idle, but they have no prompt cache yet. let hasConsideredInitialCacheTimerSeed = false let allowInitialIdleCacheSeed = false + let pendingHiddenTitle: { title: string; rawTitle: string } | null = null + let pendingHiddenAgentStatus: ParsedAgentStatusPayload | null = null + let hiddenMetadataFlushTimer: ReturnType | null = null - const onTitleChange = (title: string, rawTitle: string): void => { + const applyTitleChangeNow = (title: string, rawTitle: string): void => { manager.setPaneGpuRendering(pane.id, !isGeminiTerminalTitle(rawTitle)) deps.setRuntimePaneTitle(deps.tabId, pane.id, title) if (syncAgentTaskCompleteNotificationEnabled()) { @@ -546,6 +556,75 @@ export function connectPanePty( } } + const applyAgentStatusNow = (payload: ParsedAgentStatusPayload): void => { + // Why: capture the store snapshot once so the title lookup and the + // setAgentStatus call observe the same state. Re-reading getState() + // between the two lines opens a brief window where the title could + // shift (OSC title update landing in between) and the status would be + // stored against a title that was never paired with it. + const currentState = useAppStore.getState() + const title = currentState.runtimePaneTitlesByTabId?.[deps.tabId]?.[pane.id] + currentState.setAgentStatus(cacheKey, payload, title) + if (syncAgentTaskCompleteNotificationEnabled()) { + agentCompletionCoordinator.observeHookStatus(payload) + } + } + + const clearHiddenMetadataFlushTimer = (): void => { + if (hiddenMetadataFlushTimer !== null) { + clearTimeout(hiddenMetadataFlushTimer) + hiddenMetadataFlushTimer = null + } + } + + const flushHiddenMetadata = (): void => { + clearHiddenMetadataFlushTimer() + const title = pendingHiddenTitle + const agentStatus = pendingHiddenAgentStatus + pendingHiddenTitle = null + pendingHiddenAgentStatus = null + if (title) { + applyTitleChangeNow(title.title, title.rawTitle) + } + if (agentStatus) { + applyAgentStatusNow(agentStatus) + } + } + + const scheduleHiddenMetadataFlush = (): void => { + if (hiddenMetadataFlushTimer !== null) { + return + } + hiddenMetadataFlushTimer = setTimeout(() => { + hiddenMetadataFlushTimer = null + flushHiddenMetadata() + }, HIDDEN_TERMINAL_METADATA_FLUSH_MS) + } + + const hiddenTitleNeedsImmediateApply = (title: string): boolean => { + const currentTitle = useAppStore.getState().runtimePaneTitlesByTabId?.[deps.tabId]?.[pane.id] + return detectAgentStatusFromTitle(currentTitle ?? '') !== detectAgentStatusFromTitle(title) + } + + const hiddenAgentStatusNeedsImmediateApply = (payload: ParsedAgentStatusPayload): boolean => { + const existing = useAppStore.getState().agentStatusByPaneKey[cacheKey] + return !existing || existing.state !== payload.state || payload.state === 'done' + } + + const onTitleChange = (title: string, rawTitle: string): void => { + if (deps.isVisibleRef.current || hiddenTitleNeedsImmediateApply(title)) { + flushHiddenMetadata() + applyTitleChangeNow(title, rawTitle) + return + } + // Why: hidden Codex panes can emit decorative title frames faster than a + // user can observe them. Keep state transitions immediate, but coalesce + // same-class hidden frames so background agents don't steal the renderer + // thread from the focused xterm during typing. + pendingHiddenTitle = { title, rawTitle } + scheduleHiddenMetadataFlush() + } + const onPtySpawn = (ptyId: string): void => { bindPanePtyId(pane.id, ptyId, deps.tabId) pane.container.dataset.ptyId = ptyId @@ -824,17 +903,13 @@ export function connectPanePty( // Without this, the OSC parser in pty-transport strips sequences from xterm // output but the status never reaches the store or dashboard/hover UI. onAgentStatus: (payload) => { - // Why: capture the store snapshot once so the title lookup and the - // setAgentStatus call observe the same state. Re-reading getState() - // between the two lines opens a brief window where the title could - // shift (OSC title update landing in between) and the status would be - // stored against a title that was never paired with it. - const currentState = useAppStore.getState() - const title = currentState.runtimePaneTitlesByTabId?.[deps.tabId]?.[pane.id] - currentState.setAgentStatus(cacheKey, payload, title) - if (syncAgentTaskCompleteNotificationEnabled()) { - agentCompletionCoordinator.observeHookStatus(payload) + if (deps.isVisibleRef.current || hiddenAgentStatusNeedsImmediateApply(payload)) { + flushHiddenMetadata() + applyAgentStatusNow(payload) + return } + pendingHiddenAgentStatus = payload + scheduleHiddenMetadataFlush() } } const transport = runtimeEnvironmentId @@ -1777,6 +1852,9 @@ export function connectPanePty( pendingTerminalInputWrite = null interruptInference.dispose() clearTitleOnlyInterruptTimer() + clearHiddenMetadataFlushTimer() + pendingHiddenTitle = null + pendingHiddenAgentStatus = null // Why: actively resolve any in-flight passphrase-gate waits so their // zustand subscribers + async IIFEs don't hang for the rest of the // session when the pane is torn down before SSH state changes. diff --git a/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.test.ts b/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.test.ts index 6933f124e..fe71a2b9c 100644 --- a/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.test.ts +++ b/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.test.ts @@ -36,7 +36,7 @@ describe('pane terminal output scheduler', () => { writeTerminalOutput(terminal, 'b', { foreground: false }) expect(terminal.write).not.toHaveBeenCalled() - vi.advanceTimersByTime(50) + vi.advanceTimersByTime(100) expect(terminal.write).toHaveBeenCalledTimes(1) expect(terminal.write).toHaveBeenCalledWith('ab') @@ -52,7 +52,7 @@ describe('pane terminal output scheduler', () => { writeTerminalOutput(terminal, 'b', { foreground: false, beforeWrite }) expect(beforeWrite).not.toHaveBeenCalled() - vi.advanceTimersByTime(50) + vi.advanceTimersByTime(100) expect(beforeWrite).toHaveBeenCalledTimes(1) expect(beforeWrite).toHaveBeenCalledWith('ab') @@ -84,12 +84,16 @@ describe('pane terminal output scheduler', () => { writeTerminalOutput(terminal, `pane-${index}`, { foreground: false }) }) - vi.advanceTimersByTime(50) + vi.advanceTimersByTime(100) expect(terminals[0].write).toHaveBeenCalledWith('pane-0') + expect(terminals[1].write).not.toHaveBeenCalled() + expect(terminals[2].write).not.toHaveBeenCalled() + + vi.advanceTimersByTime(50) expect(terminals[1].write).toHaveBeenCalledWith('pane-1') expect(terminals[2].write).not.toHaveBeenCalled() - vi.advanceTimersByTime(16) + vi.advanceTimersByTime(50) expect(terminals[2].write).toHaveBeenCalledWith('pane-2') }) @@ -103,16 +107,23 @@ describe('pane terminal output scheduler', () => { writeTerminalOutput(terminals[1], 'pane-1', { foreground: false }) writeTerminalOutput(terminals[2], 'pane-2', { foreground: false }) - vi.advanceTimersByTime(50) + vi.advanceTimersByTime(100) expect(terminals[0].write).toHaveBeenCalledTimes(1) - expect(terminals[1].write).toHaveBeenCalledWith('pane-1') + expect(terminals[1].write).not.toHaveBeenCalled() expect(terminals[2].write).not.toHaveBeenCalled() // Why: a terminal with leftover bytes is deleted/re-set after each drain // chunk, moving it to the back of the Map so a big burst cannot starve // other queued panes. - vi.advanceTimersByTime(16) + vi.advanceTimersByTime(50) + expect(terminals[1].write).toHaveBeenCalledWith('pane-1') + expect(terminals[0].write).toHaveBeenCalledTimes(1) + + vi.advanceTimersByTime(50) expect(terminals[2].write).toHaveBeenCalledWith('pane-2') + expect(terminals[0].write).toHaveBeenCalledTimes(1) + + vi.advanceTimersByTime(50) expect(terminals[0].write).toHaveBeenCalledTimes(2) }) @@ -134,7 +145,7 @@ describe('pane terminal output scheduler', () => { writeTerminalOutput(terminal, 'stale', { foreground: false }) discardTerminalOutput(terminal) - vi.advanceTimersByTime(50) + vi.advanceTimersByTime(100) expect(terminal.write).not.toHaveBeenCalled() }) @@ -152,7 +163,7 @@ describe('pane terminal output scheduler', () => { // Why: drain runs inside setTimeout; if the throw escapes drainQueuedOutput // it would crash the timer callback and leave the scheduler poisoned. - expect(() => vi.advanceTimersByTime(50)).not.toThrow() + expect(() => vi.advanceTimersByTime(100)).not.toThrow() expect(throwing.write).toHaveBeenCalledTimes(1) // Advancing further must not rediscover the dead entry. diff --git a/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.ts b/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.ts index 9b7b43bc7..e278b446c 100644 --- a/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.ts +++ b/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.ts @@ -12,10 +12,10 @@ type QueueEntry = { beforeWrite?: TerminalOutputBeforeWrite } -const BACKGROUND_FLUSH_DELAY_MS = 50 -const BACKGROUND_DRAIN_INTERVAL_MS = 16 +const BACKGROUND_FLUSH_DELAY_MS = 100 +const BACKGROUND_DRAIN_INTERVAL_MS = 50 const BACKGROUND_CHUNK_CHARS = 16 * 1024 -const MAX_WRITES_PER_DRAIN = 2 +const MAX_WRITES_PER_DRAIN = 1 const PARSE_SETTLE_TIMEOUT_MS = 250 const queuedByTerminal = new Map() diff --git a/tests/e2e/terminal-codex-lag-stress.spec.ts b/tests/e2e/terminal-codex-lag-stress.spec.ts new file mode 100644 index 000000000..1f7e08d41 --- /dev/null +++ b/tests/e2e/terminal-codex-lag-stress.spec.ts @@ -0,0 +1,561 @@ +/* eslint-disable max-lines -- Why: this diagnostic stress test keeps setup, + * synthetic Codex scripts, renderer lag probing, and assertions together so the + * reproduction can run as one isolated e2e scenario. */ +import type { Page } from '@stablyai/playwright-test' +import { expect } from '@stablyai/playwright-test' +import { randomUUID } from 'node:crypto' +import { rmSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import { test } from './helpers/orca-app' +import { + getTerminalContent, + sendToTerminal, + waitForActivePanePtyId, + waitForActiveTerminalManager, + waitForTerminalOutput +} from './helpers/terminal' +import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' + +function readPositiveIntegerEnv(name: string, fallback: number): number { + const raw = process.env[name] + if (!raw) { + return fallback + } + const parsed = Number(raw) + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${name} must be a positive integer, received ${JSON.stringify(raw)}`) + } + return parsed +} + +const EXTRA_WORKTREE_COUNT = readPositiveIntegerEnv('ORCA_E2E_CODEX_LAG_WORKTREES', 36) +const BACKGROUND_CODEX_TERMINALS = readPositiveIntegerEnv( + 'ORCA_E2E_CODEX_LAG_BACKGROUND_TERMINALS', + 4 +) +const BACKGROUND_OUTPUT_INTERVAL_MS = readPositiveIntegerEnv( + 'ORCA_E2E_CODEX_LAG_BACKGROUND_INTERVAL_MS', + 10 +) +const BACKGROUND_OUTPUT_PAYLOAD_CHARS = readPositiveIntegerEnv( + 'ORCA_E2E_CODEX_LAG_BACKGROUND_PAYLOAD_CHARS', + 220 +) +const KEY_LATENCY_SAMPLES = + process.env.ORCA_E2E_CODEX_LAG_KEY_SAMPLES ?? 'abcdefghijklmnopqrstuvwxyz012345' +const BACKGROUND_MODE = process.env.ORCA_E2E_CODEX_LAG_BACKGROUND_MODE ?? 'synthetic' +if (BACKGROUND_MODE !== 'synthetic' && BACKGROUND_MODE !== 'real-codex') { + throw new Error( + `ORCA_E2E_CODEX_LAG_BACKGROUND_MODE must be "synthetic" or "real-codex", received ${JSON.stringify( + BACKGROUND_MODE + )}` + ) +} +const MAX_MEDIAN_KEY_LATENCY_MS = 250 +const MAX_WORST_KEY_LATENCY_MS = 1_000 +const MAX_RENDERER_FRAME_GAP_MS = 500 + +type LagProbeSnapshot = { + maxRafGapMs: number + rafGapsOver50Ms: number[] + longTasks: { duration: number; startTime: number; name: string }[] +} + +type TerminalOutputSchedulerDebugSnapshot = { + backgroundEnqueueCount: number + foregroundWriteCount: number + backgroundWriteCount: number + flushWriteCount: number + scheduledDrainCount: number + drainWrites: number[] +} + +type StressWorktree = { + id: string + path: string +} + +function interactivePromptScript(runId: string): string { + return ` +process.stdin.setEncoding('utf8') +if (process.stdin.isTTY) process.stdin.setRawMode(true) +process.stdin.resume() +let seq = 0 +const interrupt = String.fromCharCode(3) +process.stdout.write('\\x1b]0;Codex foreground typing benchmark\\x07') +process.stdout.write('TYPING_READY_${runId}\\n') +process.stdin.on('data', (chunk) => { + if (chunk.includes(interrupt)) { + process.exit(0) + } + for (const char of chunk) { + if (char === '\\r' || char === '\\n') continue + seq += 1 + process.stdout.write('\\r\\x1b[2KCodex prompt ' + seq + ': ' + char + ' TYPING_KEY_${runId}_' + seq + '\\n') + } +}) +` +} + +function backgroundCodexScript(runId: string, intervalMs: number, payloadChars: number): string { + return ` +const id = process.argv[2] ?? 'bg' +let seq = 0 +process.stdout.write('BG_READY_${runId}_' + id + '\\n') +const emit = () => { + seq += 1 + const spinner = ['|','/','-','\\\\'][seq % 4] + const state = seq % 40 === 0 ? 'waiting' : 'working' + const payload = { + state, + prompt: 'stress prompt ' + id, + agentType: 'codex', + toolName: seq % 7 === 0 ? 'Shell' : 'Read', + toolInput: 'background work item ' + seq, + lastAssistantMessage: 'synthetic codex progress ' + seq + } + process.stdout.write('\\x1b]0;' + spinner + ' Codex ' + id + ' ' + seq + '\\x07') + process.stdout.write('\\x1b]9999;' + JSON.stringify(payload) + '\\x07') + process.stdout.write('\\r\\x1b[2K' + spinner + ' codex ' + id + ' thinking ' + seq + ' ' + 'x'.repeat(${payloadChars}) + '\\n') +} +setTimeout(() => setInterval(emit, ${intervalMs}), 250) +` +} + +function realCodexBackgroundScript( + runId: string, + intervalMs: number, + payloadChars: number +): string { + return ` +import { spawn } from 'node:child_process' + +const id = process.argv[2] ?? 'bg' +process.stdout.write('BG_READY_${runId}_' + id + '\\n') + +let seq = 0 +const emit = () => { + seq += 1 + const spinner = ['|','/','-','\\\\'][seq % 4] + const payload = { + state: 'working', + prompt: 'real codex stress prompt ' + id, + agentType: 'codex', + toolName: 'Codex', + toolInput: 'real background codex process heartbeat ' + seq, + lastAssistantMessage: 'real codex process still active ' + seq + } + process.stdout.write('\\x1b]0;' + spinner + ' Real Codex ' + id + ' ' + seq + '\\x07') + process.stdout.write('\\x1b]9999;' + JSON.stringify(payload) + '\\x07') + process.stdout.write('\\r\\x1b[2K' + spinner + ' real codex ' + id + ' active ' + seq + ' ' + 'x'.repeat(${payloadChars}) + '\\n') +} +const heartbeat = setInterval(emit, ${intervalMs}) + +const progressPrefix = 'ORCA_REAL_CODEX_PROGRESS_${runId}_' + id +const progressProgram = + "let i=0; const t=setInterval(() => { i += 1; console.log('" + + progressPrefix + + " ' + i + ' ' + 'x'.repeat(180)); if (i >= 40) { clearInterval(t); } }, 250)" +const prompt = [ + 'This is an Orca terminal performance test.', + 'Before your final answer, run this exact read-only shell command:', + 'node -e ' + JSON.stringify(progressProgram), + 'After the command finishes, answer exactly: ORCA_REAL_CODEX_DONE_${runId}_' + id +].join(' ') + +const child = spawn( + 'codex', + ['-a', 'never', 'exec', '--sandbox', 'read-only', '--ephemeral', '--json', prompt], + { + stdio: ['ignore', 'inherit', 'inherit'], + env: process.env + } +) + +child.on('error', (error) => { + clearInterval(heartbeat) + console.error('BG_CODEX_ERROR_${runId}_' + id + ' ' + error.message) + process.exit(1) +}) + +child.on('exit', (code, signal) => { + clearInterval(heartbeat) + process.stdout.write( + 'BG_CODEX_EXIT_${runId}_' + id + ' ' + (code === null ? signal : code) + '\\n' + ) + process.exit(code ?? 0) +}) +` +} + +async function focusActiveTerminalInput(page: Page): Promise { + await page.evaluate(() => { + const state = window.__store?.getState() + const worktreeId = state?.activeWorktreeId + const tabId = + state?.activeTabType === 'terminal' + ? state.activeTabId + : worktreeId + ? (state?.activeTabIdByWorktree?.[worktreeId] ?? null) + : null + const manager = tabId ? window.__paneManagers?.get(tabId) : null + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + if (!pane) { + throw new Error('No active terminal pane to focus') + } + pane.terminal.focus() + const textarea = pane.container.querySelector( + '.xterm-helper-textarea' + ) as HTMLTextAreaElement | null + if (!textarea) { + throw new Error('Active terminal has no xterm helper textarea') + } + textarea.focus() + }) +} + +async function installRendererLagProbe(page: Page): Promise { + await page.evaluate(() => { + const target = window as unknown as { + __orcaTerminalLagProbe?: { + maxRafGapMs: number + rafGapsOver50Ms: number[] + longTasks: { duration: number; startTime: number; name: string }[] + stop: () => void + snapshot: () => LagProbeSnapshot + } + } + target.__orcaTerminalLagProbe?.stop() + + let stopped = false + let lastRaf = performance.now() + const rafGapsOver50Ms: number[] = [] + const longTasks: { duration: number; startTime: number; name: string }[] = [] + let maxRafGapMs = 0 + let observer: PerformanceObserver | null = null + + const tick = (): void => { + if (stopped) { + return + } + const now = performance.now() + const gap = now - lastRaf + lastRaf = now + maxRafGapMs = Math.max(maxRafGapMs, gap) + if (gap > 50) { + rafGapsOver50Ms.push(gap) + } + requestAnimationFrame(tick) + } + requestAnimationFrame(tick) + + if (typeof PerformanceObserver !== 'undefined') { + try { + observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + longTasks.push({ + duration: entry.duration, + startTime: entry.startTime, + name: entry.name + }) + } + }) + observer.observe({ entryTypes: ['longtask'] }) + } catch { + observer = null + } + } + + target.__orcaTerminalLagProbe = { + get maxRafGapMs() { + return maxRafGapMs + }, + get rafGapsOver50Ms() { + return rafGapsOver50Ms + }, + get longTasks() { + return longTasks + }, + stop: () => { + stopped = true + observer?.disconnect() + }, + snapshot: () => ({ + maxRafGapMs, + rafGapsOver50Ms: [...rafGapsOver50Ms], + longTasks: [...longTasks] + }) + } + }) +} + +async function readRendererLagProbe(page: Page): Promise { + return page.evaluate(() => { + const probe = ( + window as unknown as { + __orcaTerminalLagProbe?: { snapshot: () => LagProbeSnapshot } + } + ).__orcaTerminalLagProbe + if (!probe) { + throw new Error('Renderer lag probe was not installed') + } + return probe.snapshot() + }) +} + +async function resetTerminalOutputSchedulerDebug(page: Page): Promise { + await page.evaluate(() => { + const debugApi = ( + window as unknown as { + __terminalOutputSchedulerDebug?: { reset: () => void } + } + ).__terminalOutputSchedulerDebug + if (!debugApi) { + throw new Error('Terminal output scheduler debug API was not exposed') + } + debugApi.reset() + }) +} + +async function readTerminalOutputSchedulerDebug( + page: Page +): Promise { + return page.evaluate(() => { + const debugApi = ( + window as unknown as { + __terminalOutputSchedulerDebug?: { + snapshot: () => TerminalOutputSchedulerDebugSnapshot + } + } + ).__terminalOutputSchedulerDebug + if (!debugApi) { + throw new Error('Terminal output scheduler debug API was not exposed') + } + return debugApi.snapshot() + }) +} + +async function waitForMarkerLatency( + page: Page, + marker: string, + timeoutMs: number +): Promise { + const start = performance.now() + while (performance.now() - start < timeoutMs) { + if ((await getTerminalContent(page, 16_000)).includes(marker)) { + return performance.now() - start + } + await page.waitForTimeout(5) + } + throw new Error(`Timed out waiting for terminal marker ${marker}`) +} + +async function waitForShellCommandReady( + page: Page, + ptyId: string, + markerPrefix: string +): Promise { + for (let attempt = 0; attempt < 10; attempt++) { + const marker = `${markerPrefix}_${attempt}` + await sendToTerminal(page, ptyId, `printf '${marker}\\n'\r`) + try { + await waitForTerminalOutput(page, marker, 3_000) + return + } catch { + // Retry: a freshly spawned PTY can have an id before the login shell is + // ready to accept its first command, especially when many worktrees + // mount terminals in sequence. + } + } + throw new Error(`Timed out waiting for shell command readiness at ${markerPrefix}`) +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b) + return sorted[Math.floor(sorted.length / 2)] ?? 0 +} + +async function createStressWorktrees( + page: Page, + count: number, + runId: string +): Promise { + return page.evaluate( + async ({ count, runId }) => { + const store = window.__store + if (!store) { + throw new Error('window.__store is unavailable') + } + const state = store.getState() + const activeWorktree = Object.values(state.worktreesByRepo) + .flat() + .find((worktree) => worktree.id === state.activeWorktreeId) + if (!activeWorktree) { + throw new Error('No active worktree available for stress setup') + } + const worktrees: StressWorktree[] = [] + for (let index = 0; index < count; index++) { + const name = `e2e-lag-${runId.slice(0, 8)}-${index}` + const result = await state.createWorktree(activeWorktree.repoId, name, undefined, 'skip') + worktrees.push({ id: result.worktree.id, path: result.worktree.path }) + } + await state.fetchWorktrees(activeWorktree.repoId) + return worktrees + }, + { count, runId } + ) +} + +async function activateWorktreeTerminal(page: Page, worktreeId: string): Promise { + await page.evaluate((worktreeId) => { + const store = window.__store + if (!store) { + throw new Error('window.__store is unavailable') + } + const state = store.getState() + state.setActiveWorktree(worktreeId) + const existingTab = state.tabsByWorktree[worktreeId]?.[0] + const tab = existingTab ?? state.createTab(worktreeId) + state.setActiveTab(tab.id) + state.setActiveTabType('terminal') + }, worktreeId) + await waitForActiveTerminalManager(page, 30_000) + const ptyId = await waitForActivePanePtyId(page, 30_000) + await focusActiveTerminalInput(page) + await page.waitForTimeout(250) + return ptyId +} + +test.describe('Terminal Codex lag stress', () => { + test('foreground typing stays responsive with many worktrees and busy background codex panes', async ({ + orcaPage, + testRepoPath + }, testInfo) => { + test.setTimeout(300_000) + + await waitForSessionReady(orcaPage) + const foregroundWorktreeId = await waitForActiveWorktree(orcaPage) + await ensureTerminalVisible(orcaPage) + await waitForActiveTerminalManager(orcaPage, 30_000) + let foregroundPtyId = await waitForActivePanePtyId(orcaPage) + + const runId = randomUUID() + const foregroundScriptPath = path.join(testRepoPath, `.orca-typing-stress-${runId}.mjs`) + const backgroundScriptPath = path.join(testRepoPath, `.orca-bg-codex-stress-${runId}.mjs`) + writeFileSync(foregroundScriptPath, interactivePromptScript(runId)) + writeFileSync( + backgroundScriptPath, + BACKGROUND_MODE === 'real-codex' + ? realCodexBackgroundScript( + runId, + BACKGROUND_OUTPUT_INTERVAL_MS, + BACKGROUND_OUTPUT_PAYLOAD_CHARS + ) + : backgroundCodexScript( + runId, + BACKGROUND_OUTPUT_INTERVAL_MS, + BACKGROUND_OUTPUT_PAYLOAD_CHARS + ) + ) + + const backgroundPtyIds: string[] = [] + const createdWorktreeIds: string[] = [] + let foregroundCommandSent = false + try { + const syntheticWorktrees = await createStressWorktrees(orcaPage, EXTRA_WORKTREE_COUNT, runId) + createdWorktreeIds.push(...syntheticWorktrees.map((worktree) => worktree.id)) + testInfo.attachments.push({ + name: 'stress-worktrees', + contentType: 'text/plain', + body: Buffer.from(syntheticWorktrees.map((worktree) => worktree.path).join('\n')) + }) + + for (let index = 0; index < BACKGROUND_CODEX_TERMINALS; index++) { + const ptyId = await activateWorktreeTerminal(orcaPage, syntheticWorktrees[index].id) + backgroundPtyIds.push(ptyId) + await waitForShellCommandReady(orcaPage, ptyId, `SHELL_READY_${runId}_bg_${index}`) + await sendToTerminal( + orcaPage, + ptyId, + `node ${JSON.stringify(backgroundScriptPath)} ${JSON.stringify(`bg-${index}`)}\r` + ) + await waitForTerminalOutput(orcaPage, `BG_READY_${runId}_bg-${index}`, 30_000) + } + + foregroundPtyId = await activateWorktreeTerminal(orcaPage, foregroundWorktreeId) + await waitForShellCommandReady(orcaPage, foregroundPtyId, `SHELL_READY_${runId}_fg`) + await sendToTerminal( + orcaPage, + foregroundPtyId, + `node ${JSON.stringify(foregroundScriptPath)}\r` + ) + foregroundCommandSent = true + await waitForTerminalOutput(orcaPage, `TYPING_READY_${runId}`, 10_000) + await installRendererLagProbe(orcaPage) + await resetTerminalOutputSchedulerDebug(orcaPage) + await focusActiveTerminalInput(orcaPage) + + const latencies: number[] = [] + for (const [index, char] of [...KEY_LATENCY_SAMPLES].entries()) { + const seq = index + 1 + const marker = `TYPING_KEY_${runId}_${seq}` + const start = performance.now() + await orcaPage.keyboard.type(char) + await waitForMarkerLatency(orcaPage, marker, MAX_WORST_KEY_LATENCY_MS) + latencies.push(performance.now() - start) + } + + const probe = await readRendererLagProbe(orcaPage) + const schedulerDebug = await readTerminalOutputSchedulerDebug(orcaPage) + const medianLatency = median(latencies) + const worstLatency = Math.max(...latencies) + const worstLongTask = Math.max(0, ...probe.longTasks.map((entry) => entry.duration)) + const worstRafGap = probe.maxRafGapMs + + const summary = `worktrees=${EXTRA_WORKTREE_COUNT} backgroundTerminals=${BACKGROUND_CODEX_TERMINALS} backgroundMode=${BACKGROUND_MODE} backgroundIntervalMs=${BACKGROUND_OUTPUT_INTERVAL_MS} backgroundPayloadChars=${BACKGROUND_OUTPUT_PAYLOAD_CHARS} median=${medianLatency.toFixed(1)}ms worst=${worstLatency.toFixed( + 1 + )}ms worstRafGap=${worstRafGap.toFixed(1)}ms worstLongTask=${worstLongTask.toFixed( + 1 + )}ms rafGapsOver50=${probe.rafGapsOver50Ms + .map((value) => value.toFixed(1)) + .join(',')} scheduler=${JSON.stringify(schedulerDebug)} samples=${latencies + .map((value) => value.toFixed(1)) + .join(',')}` + testInfo.annotations.push({ type: 'terminal-codex-lag-stress', description: summary }) + console.log(`[terminal-codex-lag-stress] ${summary}`) + + expect(medianLatency).toBeLessThan(MAX_MEDIAN_KEY_LATENCY_MS) + expect(worstLatency).toBeLessThan(MAX_WORST_KEY_LATENCY_MS) + expect(worstRafGap).toBeLessThan(MAX_RENDERER_FRAME_GAP_MS) + expect(schedulerDebug.backgroundEnqueueCount).toBeGreaterThan(0) + expect(schedulerDebug.backgroundWriteCount).toBeGreaterThan(0) + expect(schedulerDebug.foregroundWriteCount).toBeGreaterThan(0) + } finally { + if (foregroundCommandSent) { + await sendToTerminal(orcaPage, foregroundPtyId, '\x03').catch(() => undefined) + } + for (const ptyId of backgroundPtyIds) { + await sendToTerminal(orcaPage, ptyId, '\x03').catch(() => undefined) + } + rmSync(foregroundScriptPath, { force: true }) + rmSync(backgroundScriptPath, { force: true }) + if (createdWorktreeIds.length > 0) { + await orcaPage + .evaluate(async (worktreeIds) => { + const store = window.__store + if (!store) { + return + } + for (const worktreeId of [...worktreeIds].reverse()) { + try { + await store.getState().removeWorktree(worktreeId, true) + } catch { + // best-effort cleanup + } + } + }, createdWorktreeIds) + .catch(() => undefined) + } + } + }) +})