Reduce hidden terminal pressure during Codex typing (#2661)
* Reduce hidden terminal pressure during Codex typing * Make terminal lag stress test tunable * Add real Codex terminal stress mode
This commit is contained in:
Neil
GitHub
parent
e8837275fe
commit
69728c9904
|
|
@ -433,6 +433,101 @@ describe('connectPanePty', () => {
|
|||
delete (globalThis as Record<string, unknown>).__ptyConnectDiag
|
||||
})
|
||||
|
||||
it('coalesces same-class hidden title frames', async () => {
|
||||
vi.useFakeTimers()
|
||||
const { connectPanePty } = await import('./pty-connection')
|
||||
transportFactoryQueue.push(createMockTransport('pty-1'))
|
||||
const pane = createPane(1)
|
||||
const manager = {
|
||||
...createManager(1),
|
||||
getActivePane: vi.fn(() => ({ id: 1 }))
|
||||
}
|
||||
const setRuntimePaneTitle = vi.fn((tabId: string, paneId: number, title: string) => {
|
||||
mockStoreState.runtimePaneTitlesByTabId[tabId] = {
|
||||
...mockStoreState.runtimePaneTitlesByTabId[tabId],
|
||||
[paneId]: title
|
||||
}
|
||||
})
|
||||
const deps = createDeps({
|
||||
isVisibleRef: { current: false },
|
||||
setRuntimePaneTitle
|
||||
})
|
||||
|
||||
const binding = connectPanePty(pane as never, manager as never, deps as never)
|
||||
const options = createdTransportOptions[0] as {
|
||||
onTitleChange: (title: string, rawTitle: string) => void
|
||||
}
|
||||
|
||||
options.onTitleChange('Codex working one', 'Codex working one')
|
||||
options.onTitleChange('Codex working two', 'Codex working two')
|
||||
|
||||
expect(setRuntimePaneTitle).toHaveBeenCalledTimes(1)
|
||||
vi.advanceTimersByTime(99)
|
||||
expect(setRuntimePaneTitle).toHaveBeenCalledTimes(1)
|
||||
vi.advanceTimersByTime(1)
|
||||
expect(setRuntimePaneTitle).toHaveBeenCalledTimes(2)
|
||||
expect(setRuntimePaneTitle).toHaveBeenLastCalledWith('tab-1', 1, 'Codex working two')
|
||||
|
||||
binding.dispose()
|
||||
})
|
||||
|
||||
it('coalesces same-state hidden agent status pings', async () => {
|
||||
vi.useFakeTimers()
|
||||
const { connectPanePty } = await import('./pty-connection')
|
||||
transportFactoryQueue.push(createMockTransport('pty-1'))
|
||||
const deps = createDeps({ isVisibleRef: { current: false } })
|
||||
|
||||
const binding = connectPanePty(createPane(1) as never, createManager(1) as never, deps as never)
|
||||
const options = createdTransportOptions[0] as {
|
||||
onAgentStatus: (payload: { state: 'working'; prompt: string; toolInput?: string }) => void
|
||||
}
|
||||
|
||||
options.onAgentStatus({ state: 'working', prompt: 'p', toolInput: 'first' })
|
||||
options.onAgentStatus({ state: 'working', prompt: 'p', toolInput: 'second' })
|
||||
|
||||
expect(mockStoreState.setAgentStatus).toHaveBeenCalledTimes(1)
|
||||
vi.advanceTimersByTime(100)
|
||||
expect(mockStoreState.setAgentStatus).toHaveBeenCalledTimes(2)
|
||||
expect(mockStoreState.setAgentStatus).toHaveBeenLastCalledWith(
|
||||
makePaneKey('tab-1', LEAF_1),
|
||||
{ state: 'working', prompt: 'p', toolInput: 'second' },
|
||||
undefined
|
||||
)
|
||||
|
||||
binding.dispose()
|
||||
})
|
||||
|
||||
it('keeps hidden agent status transitions immediate', async () => {
|
||||
vi.useFakeTimers()
|
||||
const { connectPanePty } = await import('./pty-connection')
|
||||
transportFactoryQueue.push(createMockTransport('pty-1'))
|
||||
const deps = createDeps({ isVisibleRef: { current: false } })
|
||||
|
||||
const binding = connectPanePty(createPane(1) as never, createManager(1) as never, deps as never)
|
||||
const options = createdTransportOptions[0] as {
|
||||
onAgentStatus: (payload: {
|
||||
state: 'working' | 'waiting' | 'done'
|
||||
prompt: string
|
||||
toolInput?: string
|
||||
}) => void
|
||||
}
|
||||
|
||||
options.onAgentStatus({ state: 'working', prompt: 'p', toolInput: 'first' })
|
||||
options.onAgentStatus({ state: 'waiting', prompt: 'p', toolInput: 'blocked' })
|
||||
options.onAgentStatus({ state: 'done', prompt: 'p', toolInput: 'complete' })
|
||||
|
||||
expect(mockStoreState.setAgentStatus).toHaveBeenCalledTimes(3)
|
||||
vi.advanceTimersByTime(100)
|
||||
expect(mockStoreState.setAgentStatus).toHaveBeenCalledTimes(3)
|
||||
expect(mockStoreState.setAgentStatus).toHaveBeenLastCalledWith(
|
||||
makePaneKey('tab-1', LEAF_1),
|
||||
{ state: 'done', prompt: 'p', toolInput: 'complete' },
|
||||
undefined
|
||||
)
|
||||
|
||||
binding.dispose()
|
||||
})
|
||||
|
||||
it('does not retain PTY connect diagnostics unless e2e debug state is enabled', async () => {
|
||||
const { connectPanePty } = await import('./pty-connection')
|
||||
const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {})
|
||||
|
|
|
|||
|
|
@ -33,7 +33,10 @@ import { recordTerminalOutput } from '@/lib/pane-manager/pane-scroll'
|
|||
import { makePaneKey } from '../../../../shared/stable-pane-id'
|
||||
import { createTerminalCommandLifecycle } from './terminal-command-lifecycle'
|
||||
import { e2eConfig } from '@/lib/e2e-config'
|
||||
import type { AgentStatusEntry } from '../../../../shared/agent-status-types'
|
||||
import type {
|
||||
AgentStatusEntry,
|
||||
ParsedAgentStatusPayload
|
||||
} from '../../../../shared/agent-status-types'
|
||||
import { isWebTerminalSurfaceTabId } from '@/runtime/web-terminal-surface-id'
|
||||
import {
|
||||
createAgentInterruptInference,
|
||||
|
|
@ -53,6 +56,7 @@ const PTY_CONNECT_DIAG_LIMIT = 200
|
|||
const AGENT_TASK_COMPLETE_NOTIFICATION_GRACE_MS = 250
|
||||
const AGENT_TASK_COMPLETE_NOTIFICATION_MAX_WAIT_MS = 1000
|
||||
const AGENT_TASK_COMPLETE_NOTIFICATION_DETAIL_MAX_AGE_MS = 10_000
|
||||
const HIDDEN_TERMINAL_METADATA_FLUSH_MS = 100
|
||||
let codexRestartNoticePresenceSource: Record<
|
||||
string,
|
||||
{ previousAccountLabel: string; nextAccountLabel: string }
|
||||
|
|
@ -476,6 +480,9 @@ export function connectPanePty(
|
|||
|
||||
const onExit = (ptyId: string): void => {
|
||||
agentCompletionCoordinator.dispose()
|
||||
clearHiddenMetadataFlushTimer()
|
||||
pendingHiddenTitle = null
|
||||
pendingHiddenAgentStatus = null
|
||||
deps.syncPanePtyLayoutBinding(pane.id, null)
|
||||
deps.clearRuntimePaneTitle(deps.tabId, pane.id)
|
||||
deps.clearTabPtyId(deps.tabId, ptyId)
|
||||
|
|
@ -514,8 +521,11 @@ export function connectPanePty(
|
|||
// Claude launches also start idle, but they have no prompt cache yet.
|
||||
let hasConsideredInitialCacheTimerSeed = false
|
||||
let allowInitialIdleCacheSeed = false
|
||||
let pendingHiddenTitle: { title: string; rawTitle: string } | null = null
|
||||
let pendingHiddenAgentStatus: ParsedAgentStatusPayload | null = null
|
||||
let hiddenMetadataFlushTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const onTitleChange = (title: string, rawTitle: string): void => {
|
||||
const applyTitleChangeNow = (title: string, rawTitle: string): void => {
|
||||
manager.setPaneGpuRendering(pane.id, !isGeminiTerminalTitle(rawTitle))
|
||||
deps.setRuntimePaneTitle(deps.tabId, pane.id, title)
|
||||
if (syncAgentTaskCompleteNotificationEnabled()) {
|
||||
|
|
@ -546,6 +556,75 @@ export function connectPanePty(
|
|||
}
|
||||
}
|
||||
|
||||
const applyAgentStatusNow = (payload: ParsedAgentStatusPayload): void => {
|
||||
// Why: capture the store snapshot once so the title lookup and the
|
||||
// setAgentStatus call observe the same state. Re-reading getState()
|
||||
// between the two lines opens a brief window where the title could
|
||||
// shift (OSC title update landing in between) and the status would be
|
||||
// stored against a title that was never paired with it.
|
||||
const currentState = useAppStore.getState()
|
||||
const title = currentState.runtimePaneTitlesByTabId?.[deps.tabId]?.[pane.id]
|
||||
currentState.setAgentStatus(cacheKey, payload, title)
|
||||
if (syncAgentTaskCompleteNotificationEnabled()) {
|
||||
agentCompletionCoordinator.observeHookStatus(payload)
|
||||
}
|
||||
}
|
||||
|
||||
const clearHiddenMetadataFlushTimer = (): void => {
|
||||
if (hiddenMetadataFlushTimer !== null) {
|
||||
clearTimeout(hiddenMetadataFlushTimer)
|
||||
hiddenMetadataFlushTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
const flushHiddenMetadata = (): void => {
|
||||
clearHiddenMetadataFlushTimer()
|
||||
const title = pendingHiddenTitle
|
||||
const agentStatus = pendingHiddenAgentStatus
|
||||
pendingHiddenTitle = null
|
||||
pendingHiddenAgentStatus = null
|
||||
if (title) {
|
||||
applyTitleChangeNow(title.title, title.rawTitle)
|
||||
}
|
||||
if (agentStatus) {
|
||||
applyAgentStatusNow(agentStatus)
|
||||
}
|
||||
}
|
||||
|
||||
const scheduleHiddenMetadataFlush = (): void => {
|
||||
if (hiddenMetadataFlushTimer !== null) {
|
||||
return
|
||||
}
|
||||
hiddenMetadataFlushTimer = setTimeout(() => {
|
||||
hiddenMetadataFlushTimer = null
|
||||
flushHiddenMetadata()
|
||||
}, HIDDEN_TERMINAL_METADATA_FLUSH_MS)
|
||||
}
|
||||
|
||||
const hiddenTitleNeedsImmediateApply = (title: string): boolean => {
|
||||
const currentTitle = useAppStore.getState().runtimePaneTitlesByTabId?.[deps.tabId]?.[pane.id]
|
||||
return detectAgentStatusFromTitle(currentTitle ?? '') !== detectAgentStatusFromTitle(title)
|
||||
}
|
||||
|
||||
const hiddenAgentStatusNeedsImmediateApply = (payload: ParsedAgentStatusPayload): boolean => {
|
||||
const existing = useAppStore.getState().agentStatusByPaneKey[cacheKey]
|
||||
return !existing || existing.state !== payload.state || payload.state === 'done'
|
||||
}
|
||||
|
||||
const onTitleChange = (title: string, rawTitle: string): void => {
|
||||
if (deps.isVisibleRef.current || hiddenTitleNeedsImmediateApply(title)) {
|
||||
flushHiddenMetadata()
|
||||
applyTitleChangeNow(title, rawTitle)
|
||||
return
|
||||
}
|
||||
// Why: hidden Codex panes can emit decorative title frames faster than a
|
||||
// user can observe them. Keep state transitions immediate, but coalesce
|
||||
// same-class hidden frames so background agents don't steal the renderer
|
||||
// thread from the focused xterm during typing.
|
||||
pendingHiddenTitle = { title, rawTitle }
|
||||
scheduleHiddenMetadataFlush()
|
||||
}
|
||||
|
||||
const onPtySpawn = (ptyId: string): void => {
|
||||
bindPanePtyId(pane.id, ptyId, deps.tabId)
|
||||
pane.container.dataset.ptyId = ptyId
|
||||
|
|
@ -824,17 +903,13 @@ export function connectPanePty(
|
|||
// Without this, the OSC parser in pty-transport strips sequences from xterm
|
||||
// output but the status never reaches the store or dashboard/hover UI.
|
||||
onAgentStatus: (payload) => {
|
||||
// Why: capture the store snapshot once so the title lookup and the
|
||||
// setAgentStatus call observe the same state. Re-reading getState()
|
||||
// between the two lines opens a brief window where the title could
|
||||
// shift (OSC title update landing in between) and the status would be
|
||||
// stored against a title that was never paired with it.
|
||||
const currentState = useAppStore.getState()
|
||||
const title = currentState.runtimePaneTitlesByTabId?.[deps.tabId]?.[pane.id]
|
||||
currentState.setAgentStatus(cacheKey, payload, title)
|
||||
if (syncAgentTaskCompleteNotificationEnabled()) {
|
||||
agentCompletionCoordinator.observeHookStatus(payload)
|
||||
if (deps.isVisibleRef.current || hiddenAgentStatusNeedsImmediateApply(payload)) {
|
||||
flushHiddenMetadata()
|
||||
applyAgentStatusNow(payload)
|
||||
return
|
||||
}
|
||||
pendingHiddenAgentStatus = payload
|
||||
scheduleHiddenMetadataFlush()
|
||||
}
|
||||
}
|
||||
const transport = runtimeEnvironmentId
|
||||
|
|
@ -1777,6 +1852,9 @@ export function connectPanePty(
|
|||
pendingTerminalInputWrite = null
|
||||
interruptInference.dispose()
|
||||
clearTitleOnlyInterruptTimer()
|
||||
clearHiddenMetadataFlushTimer()
|
||||
pendingHiddenTitle = null
|
||||
pendingHiddenAgentStatus = null
|
||||
// Why: actively resolve any in-flight passphrase-gate waits so their
|
||||
// zustand subscribers + async IIFEs don't hang for the rest of the
|
||||
// session when the pane is torn down before SSH state changes.
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ describe('pane terminal output scheduler', () => {
|
|||
writeTerminalOutput(terminal, 'b', { foreground: false })
|
||||
|
||||
expect(terminal.write).not.toHaveBeenCalled()
|
||||
vi.advanceTimersByTime(50)
|
||||
vi.advanceTimersByTime(100)
|
||||
|
||||
expect(terminal.write).toHaveBeenCalledTimes(1)
|
||||
expect(terminal.write).toHaveBeenCalledWith('ab')
|
||||
|
|
@ -52,7 +52,7 @@ describe('pane terminal output scheduler', () => {
|
|||
writeTerminalOutput(terminal, 'b', { foreground: false, beforeWrite })
|
||||
|
||||
expect(beforeWrite).not.toHaveBeenCalled()
|
||||
vi.advanceTimersByTime(50)
|
||||
vi.advanceTimersByTime(100)
|
||||
|
||||
expect(beforeWrite).toHaveBeenCalledTimes(1)
|
||||
expect(beforeWrite).toHaveBeenCalledWith('ab')
|
||||
|
|
@ -84,12 +84,16 @@ describe('pane terminal output scheduler', () => {
|
|||
writeTerminalOutput(terminal, `pane-${index}`, { foreground: false })
|
||||
})
|
||||
|
||||
vi.advanceTimersByTime(50)
|
||||
vi.advanceTimersByTime(100)
|
||||
expect(terminals[0].write).toHaveBeenCalledWith('pane-0')
|
||||
expect(terminals[1].write).not.toHaveBeenCalled()
|
||||
expect(terminals[2].write).not.toHaveBeenCalled()
|
||||
|
||||
vi.advanceTimersByTime(50)
|
||||
expect(terminals[1].write).toHaveBeenCalledWith('pane-1')
|
||||
expect(terminals[2].write).not.toHaveBeenCalled()
|
||||
|
||||
vi.advanceTimersByTime(16)
|
||||
vi.advanceTimersByTime(50)
|
||||
expect(terminals[2].write).toHaveBeenCalledWith('pane-2')
|
||||
})
|
||||
|
||||
|
|
@ -103,16 +107,23 @@ describe('pane terminal output scheduler', () => {
|
|||
writeTerminalOutput(terminals[1], 'pane-1', { foreground: false })
|
||||
writeTerminalOutput(terminals[2], 'pane-2', { foreground: false })
|
||||
|
||||
vi.advanceTimersByTime(50)
|
||||
vi.advanceTimersByTime(100)
|
||||
expect(terminals[0].write).toHaveBeenCalledTimes(1)
|
||||
expect(terminals[1].write).toHaveBeenCalledWith('pane-1')
|
||||
expect(terminals[1].write).not.toHaveBeenCalled()
|
||||
expect(terminals[2].write).not.toHaveBeenCalled()
|
||||
|
||||
// Why: a terminal with leftover bytes is deleted/re-set after each drain
|
||||
// chunk, moving it to the back of the Map so a big burst cannot starve
|
||||
// other queued panes.
|
||||
vi.advanceTimersByTime(16)
|
||||
vi.advanceTimersByTime(50)
|
||||
expect(terminals[1].write).toHaveBeenCalledWith('pane-1')
|
||||
expect(terminals[0].write).toHaveBeenCalledTimes(1)
|
||||
|
||||
vi.advanceTimersByTime(50)
|
||||
expect(terminals[2].write).toHaveBeenCalledWith('pane-2')
|
||||
expect(terminals[0].write).toHaveBeenCalledTimes(1)
|
||||
|
||||
vi.advanceTimersByTime(50)
|
||||
expect(terminals[0].write).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
|
|
@ -134,7 +145,7 @@ describe('pane terminal output scheduler', () => {
|
|||
|
||||
writeTerminalOutput(terminal, 'stale', { foreground: false })
|
||||
discardTerminalOutput(terminal)
|
||||
vi.advanceTimersByTime(50)
|
||||
vi.advanceTimersByTime(100)
|
||||
|
||||
expect(terminal.write).not.toHaveBeenCalled()
|
||||
})
|
||||
|
|
@ -152,7 +163,7 @@ describe('pane terminal output scheduler', () => {
|
|||
|
||||
// Why: drain runs inside setTimeout; if the throw escapes drainQueuedOutput
|
||||
// it would crash the timer callback and leave the scheduler poisoned.
|
||||
expect(() => vi.advanceTimersByTime(50)).not.toThrow()
|
||||
expect(() => vi.advanceTimersByTime(100)).not.toThrow()
|
||||
expect(throwing.write).toHaveBeenCalledTimes(1)
|
||||
|
||||
// Advancing further must not rediscover the dead entry.
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ type QueueEntry = {
|
|||
beforeWrite?: TerminalOutputBeforeWrite
|
||||
}
|
||||
|
||||
const BACKGROUND_FLUSH_DELAY_MS = 50
|
||||
const BACKGROUND_DRAIN_INTERVAL_MS = 16
|
||||
const BACKGROUND_FLUSH_DELAY_MS = 100
|
||||
const BACKGROUND_DRAIN_INTERVAL_MS = 50
|
||||
const BACKGROUND_CHUNK_CHARS = 16 * 1024
|
||||
const MAX_WRITES_PER_DRAIN = 2
|
||||
const MAX_WRITES_PER_DRAIN = 1
|
||||
const PARSE_SETTLE_TIMEOUT_MS = 250
|
||||
|
||||
const queuedByTerminal = new Map<TerminalOutputTarget, QueueEntry>()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,561 @@
|
|||
/* eslint-disable max-lines -- Why: this diagnostic stress test keeps setup,
|
||||
* synthetic Codex scripts, renderer lag probing, and assertions together so the
|
||||
* reproduction can run as one isolated e2e scenario. */
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { expect } from '@stablyai/playwright-test'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { rmSync, writeFileSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { test } from './helpers/orca-app'
|
||||
import {
|
||||
getTerminalContent,
|
||||
sendToTerminal,
|
||||
waitForActivePanePtyId,
|
||||
waitForActiveTerminalManager,
|
||||
waitForTerminalOutput
|
||||
} from './helpers/terminal'
|
||||
import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store'
|
||||
|
||||
function readPositiveIntegerEnv(name: string, fallback: number): number {
|
||||
const raw = process.env[name]
|
||||
if (!raw) {
|
||||
return fallback
|
||||
}
|
||||
const parsed = Number(raw)
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new Error(`${name} must be a positive integer, received ${JSON.stringify(raw)}`)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
const EXTRA_WORKTREE_COUNT = readPositiveIntegerEnv('ORCA_E2E_CODEX_LAG_WORKTREES', 36)
|
||||
const BACKGROUND_CODEX_TERMINALS = readPositiveIntegerEnv(
|
||||
'ORCA_E2E_CODEX_LAG_BACKGROUND_TERMINALS',
|
||||
4
|
||||
)
|
||||
const BACKGROUND_OUTPUT_INTERVAL_MS = readPositiveIntegerEnv(
|
||||
'ORCA_E2E_CODEX_LAG_BACKGROUND_INTERVAL_MS',
|
||||
10
|
||||
)
|
||||
const BACKGROUND_OUTPUT_PAYLOAD_CHARS = readPositiveIntegerEnv(
|
||||
'ORCA_E2E_CODEX_LAG_BACKGROUND_PAYLOAD_CHARS',
|
||||
220
|
||||
)
|
||||
const KEY_LATENCY_SAMPLES =
|
||||
process.env.ORCA_E2E_CODEX_LAG_KEY_SAMPLES ?? 'abcdefghijklmnopqrstuvwxyz012345'
|
||||
const BACKGROUND_MODE = process.env.ORCA_E2E_CODEX_LAG_BACKGROUND_MODE ?? 'synthetic'
|
||||
if (BACKGROUND_MODE !== 'synthetic' && BACKGROUND_MODE !== 'real-codex') {
|
||||
throw new Error(
|
||||
`ORCA_E2E_CODEX_LAG_BACKGROUND_MODE must be "synthetic" or "real-codex", received ${JSON.stringify(
|
||||
BACKGROUND_MODE
|
||||
)}`
|
||||
)
|
||||
}
|
||||
const MAX_MEDIAN_KEY_LATENCY_MS = 250
|
||||
const MAX_WORST_KEY_LATENCY_MS = 1_000
|
||||
const MAX_RENDERER_FRAME_GAP_MS = 500
|
||||
|
||||
type LagProbeSnapshot = {
|
||||
maxRafGapMs: number
|
||||
rafGapsOver50Ms: number[]
|
||||
longTasks: { duration: number; startTime: number; name: string }[]
|
||||
}
|
||||
|
||||
type TerminalOutputSchedulerDebugSnapshot = {
|
||||
backgroundEnqueueCount: number
|
||||
foregroundWriteCount: number
|
||||
backgroundWriteCount: number
|
||||
flushWriteCount: number
|
||||
scheduledDrainCount: number
|
||||
drainWrites: number[]
|
||||
}
|
||||
|
||||
type StressWorktree = {
|
||||
id: string
|
||||
path: string
|
||||
}
|
||||
|
||||
function interactivePromptScript(runId: string): string {
|
||||
return `
|
||||
process.stdin.setEncoding('utf8')
|
||||
if (process.stdin.isTTY) process.stdin.setRawMode(true)
|
||||
process.stdin.resume()
|
||||
let seq = 0
|
||||
const interrupt = String.fromCharCode(3)
|
||||
process.stdout.write('\\x1b]0;Codex foreground typing benchmark\\x07')
|
||||
process.stdout.write('TYPING_READY_${runId}\\n')
|
||||
process.stdin.on('data', (chunk) => {
|
||||
if (chunk.includes(interrupt)) {
|
||||
process.exit(0)
|
||||
}
|
||||
for (const char of chunk) {
|
||||
if (char === '\\r' || char === '\\n') continue
|
||||
seq += 1
|
||||
process.stdout.write('\\r\\x1b[2KCodex prompt ' + seq + ': ' + char + ' TYPING_KEY_${runId}_' + seq + '\\n')
|
||||
}
|
||||
})
|
||||
`
|
||||
}
|
||||
|
||||
function backgroundCodexScript(runId: string, intervalMs: number, payloadChars: number): string {
|
||||
return `
|
||||
const id = process.argv[2] ?? 'bg'
|
||||
let seq = 0
|
||||
process.stdout.write('BG_READY_${runId}_' + id + '\\n')
|
||||
const emit = () => {
|
||||
seq += 1
|
||||
const spinner = ['|','/','-','\\\\'][seq % 4]
|
||||
const state = seq % 40 === 0 ? 'waiting' : 'working'
|
||||
const payload = {
|
||||
state,
|
||||
prompt: 'stress prompt ' + id,
|
||||
agentType: 'codex',
|
||||
toolName: seq % 7 === 0 ? 'Shell' : 'Read',
|
||||
toolInput: 'background work item ' + seq,
|
||||
lastAssistantMessage: 'synthetic codex progress ' + seq
|
||||
}
|
||||
process.stdout.write('\\x1b]0;' + spinner + ' Codex ' + id + ' ' + seq + '\\x07')
|
||||
process.stdout.write('\\x1b]9999;' + JSON.stringify(payload) + '\\x07')
|
||||
process.stdout.write('\\r\\x1b[2K' + spinner + ' codex ' + id + ' thinking ' + seq + ' ' + 'x'.repeat(${payloadChars}) + '\\n')
|
||||
}
|
||||
setTimeout(() => setInterval(emit, ${intervalMs}), 250)
|
||||
`
|
||||
}
|
||||
|
||||
function realCodexBackgroundScript(
|
||||
runId: string,
|
||||
intervalMs: number,
|
||||
payloadChars: number
|
||||
): string {
|
||||
return `
|
||||
import { spawn } from 'node:child_process'
|
||||
|
||||
const id = process.argv[2] ?? 'bg'
|
||||
process.stdout.write('BG_READY_${runId}_' + id + '\\n')
|
||||
|
||||
let seq = 0
|
||||
const emit = () => {
|
||||
seq += 1
|
||||
const spinner = ['|','/','-','\\\\'][seq % 4]
|
||||
const payload = {
|
||||
state: 'working',
|
||||
prompt: 'real codex stress prompt ' + id,
|
||||
agentType: 'codex',
|
||||
toolName: 'Codex',
|
||||
toolInput: 'real background codex process heartbeat ' + seq,
|
||||
lastAssistantMessage: 'real codex process still active ' + seq
|
||||
}
|
||||
process.stdout.write('\\x1b]0;' + spinner + ' Real Codex ' + id + ' ' + seq + '\\x07')
|
||||
process.stdout.write('\\x1b]9999;' + JSON.stringify(payload) + '\\x07')
|
||||
process.stdout.write('\\r\\x1b[2K' + spinner + ' real codex ' + id + ' active ' + seq + ' ' + 'x'.repeat(${payloadChars}) + '\\n')
|
||||
}
|
||||
const heartbeat = setInterval(emit, ${intervalMs})
|
||||
|
||||
const progressPrefix = 'ORCA_REAL_CODEX_PROGRESS_${runId}_' + id
|
||||
const progressProgram =
|
||||
"let i=0; const t=setInterval(() => { i += 1; console.log('" +
|
||||
progressPrefix +
|
||||
" ' + i + ' ' + 'x'.repeat(180)); if (i >= 40) { clearInterval(t); } }, 250)"
|
||||
const prompt = [
|
||||
'This is an Orca terminal performance test.',
|
||||
'Before your final answer, run this exact read-only shell command:',
|
||||
'node -e ' + JSON.stringify(progressProgram),
|
||||
'After the command finishes, answer exactly: ORCA_REAL_CODEX_DONE_${runId}_' + id
|
||||
].join(' ')
|
||||
|
||||
const child = spawn(
|
||||
'codex',
|
||||
['-a', 'never', 'exec', '--sandbox', 'read-only', '--ephemeral', '--json', prompt],
|
||||
{
|
||||
stdio: ['ignore', 'inherit', 'inherit'],
|
||||
env: process.env
|
||||
}
|
||||
)
|
||||
|
||||
child.on('error', (error) => {
|
||||
clearInterval(heartbeat)
|
||||
console.error('BG_CODEX_ERROR_${runId}_' + id + ' ' + error.message)
|
||||
process.exit(1)
|
||||
})
|
||||
|
||||
child.on('exit', (code, signal) => {
|
||||
clearInterval(heartbeat)
|
||||
process.stdout.write(
|
||||
'BG_CODEX_EXIT_${runId}_' + id + ' ' + (code === null ? signal : code) + '\\n'
|
||||
)
|
||||
process.exit(code ?? 0)
|
||||
})
|
||||
`
|
||||
}
|
||||
|
||||
async function focusActiveTerminalInput(page: Page): Promise<void> {
|
||||
await page.evaluate(() => {
|
||||
const state = window.__store?.getState()
|
||||
const worktreeId = state?.activeWorktreeId
|
||||
const tabId =
|
||||
state?.activeTabType === 'terminal'
|
||||
? state.activeTabId
|
||||
: worktreeId
|
||||
? (state?.activeTabIdByWorktree?.[worktreeId] ?? null)
|
||||
: null
|
||||
const manager = tabId ? window.__paneManagers?.get(tabId) : null
|
||||
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null
|
||||
if (!pane) {
|
||||
throw new Error('No active terminal pane to focus')
|
||||
}
|
||||
pane.terminal.focus()
|
||||
const textarea = pane.container.querySelector(
|
||||
'.xterm-helper-textarea'
|
||||
) as HTMLTextAreaElement | null
|
||||
if (!textarea) {
|
||||
throw new Error('Active terminal has no xterm helper textarea')
|
||||
}
|
||||
textarea.focus()
|
||||
})
|
||||
}
|
||||
|
||||
async function installRendererLagProbe(page: Page): Promise<void> {
|
||||
await page.evaluate(() => {
|
||||
const target = window as unknown as {
|
||||
__orcaTerminalLagProbe?: {
|
||||
maxRafGapMs: number
|
||||
rafGapsOver50Ms: number[]
|
||||
longTasks: { duration: number; startTime: number; name: string }[]
|
||||
stop: () => void
|
||||
snapshot: () => LagProbeSnapshot
|
||||
}
|
||||
}
|
||||
target.__orcaTerminalLagProbe?.stop()
|
||||
|
||||
let stopped = false
|
||||
let lastRaf = performance.now()
|
||||
const rafGapsOver50Ms: number[] = []
|
||||
const longTasks: { duration: number; startTime: number; name: string }[] = []
|
||||
let maxRafGapMs = 0
|
||||
let observer: PerformanceObserver | null = null
|
||||
|
||||
const tick = (): void => {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
const now = performance.now()
|
||||
const gap = now - lastRaf
|
||||
lastRaf = now
|
||||
maxRafGapMs = Math.max(maxRafGapMs, gap)
|
||||
if (gap > 50) {
|
||||
rafGapsOver50Ms.push(gap)
|
||||
}
|
||||
requestAnimationFrame(tick)
|
||||
}
|
||||
requestAnimationFrame(tick)
|
||||
|
||||
if (typeof PerformanceObserver !== 'undefined') {
|
||||
try {
|
||||
observer = new PerformanceObserver((list) => {
|
||||
for (const entry of list.getEntries()) {
|
||||
longTasks.push({
|
||||
duration: entry.duration,
|
||||
startTime: entry.startTime,
|
||||
name: entry.name
|
||||
})
|
||||
}
|
||||
})
|
||||
observer.observe({ entryTypes: ['longtask'] })
|
||||
} catch {
|
||||
observer = null
|
||||
}
|
||||
}
|
||||
|
||||
target.__orcaTerminalLagProbe = {
|
||||
get maxRafGapMs() {
|
||||
return maxRafGapMs
|
||||
},
|
||||
get rafGapsOver50Ms() {
|
||||
return rafGapsOver50Ms
|
||||
},
|
||||
get longTasks() {
|
||||
return longTasks
|
||||
},
|
||||
stop: () => {
|
||||
stopped = true
|
||||
observer?.disconnect()
|
||||
},
|
||||
snapshot: () => ({
|
||||
maxRafGapMs,
|
||||
rafGapsOver50Ms: [...rafGapsOver50Ms],
|
||||
longTasks: [...longTasks]
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function readRendererLagProbe(page: Page): Promise<LagProbeSnapshot> {
|
||||
return page.evaluate(() => {
|
||||
const probe = (
|
||||
window as unknown as {
|
||||
__orcaTerminalLagProbe?: { snapshot: () => LagProbeSnapshot }
|
||||
}
|
||||
).__orcaTerminalLagProbe
|
||||
if (!probe) {
|
||||
throw new Error('Renderer lag probe was not installed')
|
||||
}
|
||||
return probe.snapshot()
|
||||
})
|
||||
}
|
||||
|
||||
async function resetTerminalOutputSchedulerDebug(page: Page): Promise<void> {
|
||||
await page.evaluate(() => {
|
||||
const debugApi = (
|
||||
window as unknown as {
|
||||
__terminalOutputSchedulerDebug?: { reset: () => void }
|
||||
}
|
||||
).__terminalOutputSchedulerDebug
|
||||
if (!debugApi) {
|
||||
throw new Error('Terminal output scheduler debug API was not exposed')
|
||||
}
|
||||
debugApi.reset()
|
||||
})
|
||||
}
|
||||
|
||||
async function readTerminalOutputSchedulerDebug(
|
||||
page: Page
|
||||
): Promise<TerminalOutputSchedulerDebugSnapshot> {
|
||||
return page.evaluate(() => {
|
||||
const debugApi = (
|
||||
window as unknown as {
|
||||
__terminalOutputSchedulerDebug?: {
|
||||
snapshot: () => TerminalOutputSchedulerDebugSnapshot
|
||||
}
|
||||
}
|
||||
).__terminalOutputSchedulerDebug
|
||||
if (!debugApi) {
|
||||
throw new Error('Terminal output scheduler debug API was not exposed')
|
||||
}
|
||||
return debugApi.snapshot()
|
||||
})
|
||||
}
|
||||
|
||||
async function waitForMarkerLatency(
|
||||
page: Page,
|
||||
marker: string,
|
||||
timeoutMs: number
|
||||
): Promise<number> {
|
||||
const start = performance.now()
|
||||
while (performance.now() - start < timeoutMs) {
|
||||
if ((await getTerminalContent(page, 16_000)).includes(marker)) {
|
||||
return performance.now() - start
|
||||
}
|
||||
await page.waitForTimeout(5)
|
||||
}
|
||||
throw new Error(`Timed out waiting for terminal marker ${marker}`)
|
||||
}
|
||||
|
||||
async function waitForShellCommandReady(
|
||||
page: Page,
|
||||
ptyId: string,
|
||||
markerPrefix: string
|
||||
): Promise<void> {
|
||||
for (let attempt = 0; attempt < 10; attempt++) {
|
||||
const marker = `${markerPrefix}_${attempt}`
|
||||
await sendToTerminal(page, ptyId, `printf '${marker}\\n'\r`)
|
||||
try {
|
||||
await waitForTerminalOutput(page, marker, 3_000)
|
||||
return
|
||||
} catch {
|
||||
// Retry: a freshly spawned PTY can have an id before the login shell is
|
||||
// ready to accept its first command, especially when many worktrees
|
||||
// mount terminals in sequence.
|
||||
}
|
||||
}
|
||||
throw new Error(`Timed out waiting for shell command readiness at ${markerPrefix}`)
|
||||
}
|
||||
|
||||
function median(values: number[]): number {
|
||||
const sorted = [...values].sort((a, b) => a - b)
|
||||
return sorted[Math.floor(sorted.length / 2)] ?? 0
|
||||
}
|
||||
|
||||
async function createStressWorktrees(
|
||||
page: Page,
|
||||
count: number,
|
||||
runId: string
|
||||
): Promise<StressWorktree[]> {
|
||||
return page.evaluate(
|
||||
async ({ count, runId }) => {
|
||||
const store = window.__store
|
||||
if (!store) {
|
||||
throw new Error('window.__store is unavailable')
|
||||
}
|
||||
const state = store.getState()
|
||||
const activeWorktree = Object.values(state.worktreesByRepo)
|
||||
.flat()
|
||||
.find((worktree) => worktree.id === state.activeWorktreeId)
|
||||
if (!activeWorktree) {
|
||||
throw new Error('No active worktree available for stress setup')
|
||||
}
|
||||
const worktrees: StressWorktree[] = []
|
||||
for (let index = 0; index < count; index++) {
|
||||
const name = `e2e-lag-${runId.slice(0, 8)}-${index}`
|
||||
const result = await state.createWorktree(activeWorktree.repoId, name, undefined, 'skip')
|
||||
worktrees.push({ id: result.worktree.id, path: result.worktree.path })
|
||||
}
|
||||
await state.fetchWorktrees(activeWorktree.repoId)
|
||||
return worktrees
|
||||
},
|
||||
{ count, runId }
|
||||
)
|
||||
}
|
||||
|
||||
async function activateWorktreeTerminal(page: Page, worktreeId: string): Promise<string> {
|
||||
await page.evaluate((worktreeId) => {
|
||||
const store = window.__store
|
||||
if (!store) {
|
||||
throw new Error('window.__store is unavailable')
|
||||
}
|
||||
const state = store.getState()
|
||||
state.setActiveWorktree(worktreeId)
|
||||
const existingTab = state.tabsByWorktree[worktreeId]?.[0]
|
||||
const tab = existingTab ?? state.createTab(worktreeId)
|
||||
state.setActiveTab(tab.id)
|
||||
state.setActiveTabType('terminal')
|
||||
}, worktreeId)
|
||||
await waitForActiveTerminalManager(page, 30_000)
|
||||
const ptyId = await waitForActivePanePtyId(page, 30_000)
|
||||
await focusActiveTerminalInput(page)
|
||||
await page.waitForTimeout(250)
|
||||
return ptyId
|
||||
}
|
||||
|
||||
test.describe('Terminal Codex lag stress', () => {
|
||||
test('foreground typing stays responsive with many worktrees and busy background codex panes', async ({
|
||||
orcaPage,
|
||||
testRepoPath
|
||||
}, testInfo) => {
|
||||
test.setTimeout(300_000)
|
||||
|
||||
await waitForSessionReady(orcaPage)
|
||||
const foregroundWorktreeId = await waitForActiveWorktree(orcaPage)
|
||||
await ensureTerminalVisible(orcaPage)
|
||||
await waitForActiveTerminalManager(orcaPage, 30_000)
|
||||
let foregroundPtyId = await waitForActivePanePtyId(orcaPage)
|
||||
|
||||
const runId = randomUUID()
|
||||
const foregroundScriptPath = path.join(testRepoPath, `.orca-typing-stress-${runId}.mjs`)
|
||||
const backgroundScriptPath = path.join(testRepoPath, `.orca-bg-codex-stress-${runId}.mjs`)
|
||||
writeFileSync(foregroundScriptPath, interactivePromptScript(runId))
|
||||
writeFileSync(
|
||||
backgroundScriptPath,
|
||||
BACKGROUND_MODE === 'real-codex'
|
||||
? realCodexBackgroundScript(
|
||||
runId,
|
||||
BACKGROUND_OUTPUT_INTERVAL_MS,
|
||||
BACKGROUND_OUTPUT_PAYLOAD_CHARS
|
||||
)
|
||||
: backgroundCodexScript(
|
||||
runId,
|
||||
BACKGROUND_OUTPUT_INTERVAL_MS,
|
||||
BACKGROUND_OUTPUT_PAYLOAD_CHARS
|
||||
)
|
||||
)
|
||||
|
||||
const backgroundPtyIds: string[] = []
|
||||
const createdWorktreeIds: string[] = []
|
||||
let foregroundCommandSent = false
|
||||
try {
|
||||
const syntheticWorktrees = await createStressWorktrees(orcaPage, EXTRA_WORKTREE_COUNT, runId)
|
||||
createdWorktreeIds.push(...syntheticWorktrees.map((worktree) => worktree.id))
|
||||
testInfo.attachments.push({
|
||||
name: 'stress-worktrees',
|
||||
contentType: 'text/plain',
|
||||
body: Buffer.from(syntheticWorktrees.map((worktree) => worktree.path).join('\n'))
|
||||
})
|
||||
|
||||
for (let index = 0; index < BACKGROUND_CODEX_TERMINALS; index++) {
|
||||
const ptyId = await activateWorktreeTerminal(orcaPage, syntheticWorktrees[index].id)
|
||||
backgroundPtyIds.push(ptyId)
|
||||
await waitForShellCommandReady(orcaPage, ptyId, `SHELL_READY_${runId}_bg_${index}`)
|
||||
await sendToTerminal(
|
||||
orcaPage,
|
||||
ptyId,
|
||||
`node ${JSON.stringify(backgroundScriptPath)} ${JSON.stringify(`bg-${index}`)}\r`
|
||||
)
|
||||
await waitForTerminalOutput(orcaPage, `BG_READY_${runId}_bg-${index}`, 30_000)
|
||||
}
|
||||
|
||||
foregroundPtyId = await activateWorktreeTerminal(orcaPage, foregroundWorktreeId)
|
||||
await waitForShellCommandReady(orcaPage, foregroundPtyId, `SHELL_READY_${runId}_fg`)
|
||||
await sendToTerminal(
|
||||
orcaPage,
|
||||
foregroundPtyId,
|
||||
`node ${JSON.stringify(foregroundScriptPath)}\r`
|
||||
)
|
||||
foregroundCommandSent = true
|
||||
await waitForTerminalOutput(orcaPage, `TYPING_READY_${runId}`, 10_000)
|
||||
await installRendererLagProbe(orcaPage)
|
||||
await resetTerminalOutputSchedulerDebug(orcaPage)
|
||||
await focusActiveTerminalInput(orcaPage)
|
||||
|
||||
const latencies: number[] = []
|
||||
for (const [index, char] of [...KEY_LATENCY_SAMPLES].entries()) {
|
||||
const seq = index + 1
|
||||
const marker = `TYPING_KEY_${runId}_${seq}`
|
||||
const start = performance.now()
|
||||
await orcaPage.keyboard.type(char)
|
||||
await waitForMarkerLatency(orcaPage, marker, MAX_WORST_KEY_LATENCY_MS)
|
||||
latencies.push(performance.now() - start)
|
||||
}
|
||||
|
||||
const probe = await readRendererLagProbe(orcaPage)
|
||||
const schedulerDebug = await readTerminalOutputSchedulerDebug(orcaPage)
|
||||
const medianLatency = median(latencies)
|
||||
const worstLatency = Math.max(...latencies)
|
||||
const worstLongTask = Math.max(0, ...probe.longTasks.map((entry) => entry.duration))
|
||||
const worstRafGap = probe.maxRafGapMs
|
||||
|
||||
const summary = `worktrees=${EXTRA_WORKTREE_COUNT} backgroundTerminals=${BACKGROUND_CODEX_TERMINALS} backgroundMode=${BACKGROUND_MODE} backgroundIntervalMs=${BACKGROUND_OUTPUT_INTERVAL_MS} backgroundPayloadChars=${BACKGROUND_OUTPUT_PAYLOAD_CHARS} median=${medianLatency.toFixed(1)}ms worst=${worstLatency.toFixed(
|
||||
1
|
||||
)}ms worstRafGap=${worstRafGap.toFixed(1)}ms worstLongTask=${worstLongTask.toFixed(
|
||||
1
|
||||
)}ms rafGapsOver50=${probe.rafGapsOver50Ms
|
||||
.map((value) => value.toFixed(1))
|
||||
.join(',')} scheduler=${JSON.stringify(schedulerDebug)} samples=${latencies
|
||||
.map((value) => value.toFixed(1))
|
||||
.join(',')}`
|
||||
testInfo.annotations.push({ type: 'terminal-codex-lag-stress', description: summary })
|
||||
console.log(`[terminal-codex-lag-stress] ${summary}`)
|
||||
|
||||
expect(medianLatency).toBeLessThan(MAX_MEDIAN_KEY_LATENCY_MS)
|
||||
expect(worstLatency).toBeLessThan(MAX_WORST_KEY_LATENCY_MS)
|
||||
expect(worstRafGap).toBeLessThan(MAX_RENDERER_FRAME_GAP_MS)
|
||||
expect(schedulerDebug.backgroundEnqueueCount).toBeGreaterThan(0)
|
||||
expect(schedulerDebug.backgroundWriteCount).toBeGreaterThan(0)
|
||||
expect(schedulerDebug.foregroundWriteCount).toBeGreaterThan(0)
|
||||
} finally {
|
||||
if (foregroundCommandSent) {
|
||||
await sendToTerminal(orcaPage, foregroundPtyId, '\x03').catch(() => undefined)
|
||||
}
|
||||
for (const ptyId of backgroundPtyIds) {
|
||||
await sendToTerminal(orcaPage, ptyId, '\x03').catch(() => undefined)
|
||||
}
|
||||
rmSync(foregroundScriptPath, { force: true })
|
||||
rmSync(backgroundScriptPath, { force: true })
|
||||
if (createdWorktreeIds.length > 0) {
|
||||
await orcaPage
|
||||
.evaluate(async (worktreeIds) => {
|
||||
const store = window.__store
|
||||
if (!store) {
|
||||
return
|
||||
}
|
||||
for (const worktreeId of [...worktreeIds].reverse()) {
|
||||
try {
|
||||
await store.getState().removeWorktree(worktreeId, true)
|
||||
} catch {
|
||||
// best-effort cleanup
|
||||
}
|
||||
}
|
||||
}, createdWorktreeIds)
|
||||
.catch(() => undefined)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
Loading…
Reference in New Issue