diff --git a/config/scripts/summarize-terminal-perf-report.mjs b/config/scripts/summarize-terminal-perf-report.mjs index a6e0654a9..33b43145d 100644 --- a/config/scripts/summarize-terminal-perf-report.mjs +++ b/config/scripts/summarize-terminal-perf-report.mjs @@ -92,7 +92,10 @@ function printMarkdownTable(rows) { ['Main Peak Max Pending', 'mainPeakMaxPendingChars'], ['Main Peak In-Flight', 'mainPeakInFlightChars'], ['Main Peak Max In-Flight', 'mainPeakMaxInFlightChars'], - ['Main ACK-Gated Skips', 'mainAckGatedFlushSkips'] + ['Main ACK-Gated Skips', 'mainAckGatedFlushSkips'], + ['Held ACK PTYs', 'heldAckPtys'], + ['Held ACK Chars', 'heldAckChars'], + ['Gated ACK PTYs', 'gatedAckPtys'] ] console.log(`| ${columns.map(([label]) => label).join(' | ')} |`) diff --git a/src/renderer/src/components/terminal-pane/pty-dispatcher.ts b/src/renderer/src/components/terminal-pane/pty-dispatcher.ts index 691b599c2..656657aab 100644 --- a/src/renderer/src/components/terminal-pane/pty-dispatcher.ts +++ b/src/renderer/src/components/terminal-pane/pty-dispatcher.ts @@ -7,6 +7,7 @@ */ import type { ParsedAgentStatusPayload } from '../../../../shared/agent-status-types' import type { EventProps } from '../../../../shared/telemetry-events' +import { ackPtyData, exposeE2eTerminalPtyAckGate } from './terminal-pty-ack-gate' // ── Singleton PTY event dispatcher ─────────────────────────────────── // One global IPC listener per channel, routes events to transports by @@ -93,6 +94,7 @@ export function ensurePtyDispatcher(): void { return } ptyDispatcherAttached = true + exposeE2eTerminalPtyAckGate() window.api.pty.onData((payload) => { try { let meta: PtyDataMeta | undefined @@ -123,7 +125,7 @@ export function ensurePtyDispatcher(): void { // Why: main budgets renderer-bound terminal output by bytes accepted // into this dispatcher. ACK in finally so a bad sidecar cannot leave // a PTY permanently backpressured. - window.api.pty.ackData?.(payload.id, payload.rawLength ?? payload.data.length) + ackPtyData(payload.id, payload.rawLength ?? payload.data.length) } }) window.api.pty.onReplay((payload) => { diff --git a/src/renderer/src/components/terminal-pane/terminal-pty-ack-gate.ts b/src/renderer/src/components/terminal-pane/terminal-pty-ack-gate.ts new file mode 100644 index 000000000..bea658434 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/terminal-pty-ack-gate.ts @@ -0,0 +1,66 @@ +import { e2eConfig } from '@/lib/e2e-config' + +type E2eTerminalPtyAckGateSnapshot = { + gatedPtyCount: number + heldAckCount: number + heldAckChars: number +} + +type E2eTerminalPtyAckGateApi = { + hold: (ptyIds: string[]) => void + release: () => void + snapshot: () => E2eTerminalPtyAckGateSnapshot +} + +type E2eTerminalPtyAckGateWindow = Window & { + __terminalPtyAckGate?: E2eTerminalPtyAckGateApi +} + +const e2eTerminalAckGatePtyIds = new Set() +const e2eTerminalAckGateHeldChars = new Map() + +function releaseE2eTerminalAckGate(): void { + const held = Array.from(e2eTerminalAckGateHeldChars.entries()) + e2eTerminalAckGatePtyIds.clear() + e2eTerminalAckGateHeldChars.clear() + for (const [ptyId, chars] of held) { + window.api.pty.ackData?.(ptyId, chars) + } +} + +export function exposeE2eTerminalPtyAckGate(): void { + if (!e2eConfig.exposeStore || typeof window === 'undefined') { + return + } + // Why: perf tests need to force main-process renderer-delivery pressure + // without changing production ACK behavior or dropping terminal output. + const target = window as E2eTerminalPtyAckGateWindow + target.__terminalPtyAckGate ??= { + hold: (ptyIds) => { + releaseE2eTerminalAckGate() + for (const ptyId of ptyIds) { + e2eTerminalAckGatePtyIds.add(ptyId) + } + }, + release: releaseE2eTerminalAckGate, + snapshot: () => { + let heldAckChars = 0 + for (const chars of e2eTerminalAckGateHeldChars.values()) { + heldAckChars += chars + } + return { + gatedPtyCount: e2eTerminalAckGatePtyIds.size, + heldAckCount: e2eTerminalAckGateHeldChars.size, + heldAckChars + } + } + } +} + +export function ackPtyData(ptyId: string, chars: number): void { + if (e2eTerminalAckGatePtyIds.has(ptyId)) { + e2eTerminalAckGateHeldChars.set(ptyId, (e2eTerminalAckGateHeldChars.get(ptyId) ?? 0) + chars) + return + } + window.api.pty.ackData?.(ptyId, chars) +} diff --git a/tests/e2e/artificial-opencode-terminal-load.spec.ts b/tests/e2e/artificial-opencode-terminal-load.spec.ts index 372f50c56..45373492e 100644 --- a/tests/e2e/artificial-opencode-terminal-load.spec.ts +++ b/tests/e2e/artificial-opencode-terminal-load.spec.ts @@ -38,6 +38,11 @@ type SyntheticOpenCodeWindow = Window & { __terminalPtyDataInjection?: { inject: (paneKey: string, data: string) => boolean } + __terminalPtyAckGate?: { + hold: (ptyIds: string[]) => void + release: () => void + snapshot: () => TerminalPtyAckGateSnapshot + } __terminalPtyOutputDebug?: { reset: () => void snapshot: () => TerminalPtyOutputDebugSnapshot @@ -65,6 +70,12 @@ type TerminalOutputSchedulerDebugSnapshot = { drainWrites: number[] } +type TerminalPtyAckGateSnapshot = { + gatedPtyCount: number + heldAckCount: number + heldAckChars: number +} + type MainPtyPressureDebugSnapshot = { pendingPtyCount: number pendingChars: number @@ -84,6 +95,8 @@ type MainPtyPressureDebugSnapshot = { const KEY_LATENCY_SAMPLES = 'abcdefghijklmnop' const DEFAULT_SAME_WORKSPACE_PANES = 5 const DEFAULT_CROSS_WORKSPACE_PANES_PER_WORKTREE = 3 +const DEFAULT_PRESSURE_BACKGROUND_PANES = 17 +const DEFAULT_PRESSURE_OUTPUT_CHARS = 768 * 1024 const DEFAULT_FRAME_COUNT = 180 const DEFAULT_FRAME_INTERVAL_MS = 6 const TIMER_SAMPLE_MS = 16 @@ -124,6 +137,14 @@ const CROSS_WORKSPACE_PANES_PER_WORKTREE = readPositiveInt( 'ORCA_E2E_OPENCODE_CROSS_WORKSPACE_PANES', DEFAULT_CROSS_WORKSPACE_PANES_PER_WORKTREE ) +const PRESSURE_BACKGROUND_PANES = readPositiveInt( + 'ORCA_E2E_OPENCODE_PRESSURE_BACKGROUND_PANES', + DEFAULT_PRESSURE_BACKGROUND_PANES +) +const PRESSURE_OUTPUT_CHARS = readPositiveInt( + 'ORCA_E2E_OPENCODE_PRESSURE_OUTPUT_CHARS', + DEFAULT_PRESSURE_OUTPUT_CHARS +) const FRAME_COUNT = readPositiveInt('ORCA_E2E_OPENCODE_FRAME_COUNT', DEFAULT_FRAME_COUNT) const FRAME_INTERVAL_MS = readPositiveInt( 'ORCA_E2E_OPENCODE_FRAME_INTERVAL_MS', @@ -163,6 +184,37 @@ function writeInteractivePromptScript(scriptPath: string, runId: string): void { writeFileSync(scriptPath, interactivePromptScript(runId)) } +function pressureOutputScript(runId: string): string { + return ` +const paneIndex = process.argv[2] ?? '0' +const targetChars = Number(process.argv[3] ?? '0') +const header = 'OPENCODE_PRESSURE_START_${runId}_' + paneIndex + '\\n' +const chunkBody = '#'.repeat(8192) +let written = 0 +process.stdout.write(header) +function writeMore() { + let canContinue = true + while (canContinue && written < targetChars) { + const frame = String(written).padStart(8, '0') + const chunk = '\\x1b[?2026h\\x1b[1;1Hpressure pane=' + paneIndex + ' frame=' + frame + ' ' + chunkBody + '\\x1b[?2026l\\n' + written += chunk.length + canContinue = process.stdout.write(chunk) + } + if (written < targetChars) { + process.stdout.once('drain', writeMore) + return + } + process.stdout.write('OPENCODE_PRESSURE_DONE_${runId}_' + paneIndex + '\\n') +} +writeMore() +` +} + +function writePressureOutputScript(scriptPath: string, runId: string): void { + mkdirSync(path.dirname(scriptPath), { recursive: true }) + writeFileSync(scriptPath, pressureOutputScript(runId)) +} + async function focusActiveTerminalInput(page: Page): Promise { await page.evaluate(() => { const store = window.__store @@ -377,6 +429,52 @@ async function readMainPtyPressureDebug(page: Page): Promise { + await page.evaluate((ids) => { + const gate = (window as SyntheticOpenCodeWindow).__terminalPtyAckGate + if (!gate) { + throw new Error('terminal PTY ACK gate is unavailable') + } + gate.hold(ids) + }, ptyIds) +} + +async function releaseTerminalAckGate(page: Page): Promise { + await page.evaluate(() => { + ;(window as SyntheticOpenCodeWindow).__terminalPtyAckGate?.release() + }) +} + +async function readTerminalAckGateDebug(page: Page): Promise { + return page.evaluate(() => { + return (window as SyntheticOpenCodeWindow).__terminalPtyAckGate?.snapshot() ?? null + }) +} + +async function waitForMainPtyPressureBacklog(page: Page): Promise { + let lastSnapshot: MainPtyPressureDebugSnapshot | null = null + await expect + .poll( + async () => { + lastSnapshot = await readMainPtyPressureDebug(page) + return ( + (lastSnapshot?.peakRendererInFlightChars ?? 0) >= 8 * 1024 * 1024 && + (lastSnapshot?.peakPendingChars ?? 0) > 0 && + (lastSnapshot?.ackGatedFlushSkipCount ?? 0) > 0 + ) + }, + { + timeout: 20_000, + message: 'Main PTY renderer delivery pressure did not build up' + } + ) + .toBe(true) + if (!lastSnapshot) { + throw new Error('Main PTY pressure snapshot unavailable') + } + return lastSnapshot +} + function annotateTypingMeasurement( testInfo: TestInfo, type: string, @@ -384,7 +482,8 @@ function annotateTypingMeasurement( measurement: TypingMeasurement, debug: TerminalPtyOutputDebugSnapshot | null = null, scheduler: TerminalOutputSchedulerDebugSnapshot | null = null, - mainPressure: MainPtyPressureDebugSnapshot | null = null + mainPressure: MainPtyPressureDebugSnapshot | null = null, + ackGate: TerminalPtyAckGateSnapshot | null = null ): void { const hiddenSkipSummary = debug ? ` hiddenSkips=${debug.hiddenRendererSkipCount} hiddenSkippedChars=${debug.hiddenRendererSkippedChars} mode2031Replies=${debug.hiddenRendererMode2031ReplyCount}` @@ -395,6 +494,9 @@ function annotateTypingMeasurement( const mainPressureSummary = mainPressure ? ` mainPendingPtys=${mainPressure.pendingPtyCount} mainPendingChars=${mainPressure.pendingChars} mainMaxPendingChars=${mainPressure.maxPendingCharsByPty} mainInFlightPtys=${mainPressure.rendererInFlightPtyCount} mainInFlightChars=${mainPressure.rendererInFlightChars} mainMaxInFlightChars=${mainPressure.maxRendererInFlightCharsByPty} mainActivePtys=${mainPressure.activeRendererPtyCount} mainFlushScheduled=${mainPressure.flushScheduled} mainPeakPendingChars=${mainPressure.peakPendingChars} mainPeakMaxPendingChars=${mainPressure.peakMaxPendingCharsByPty} mainPeakInFlightChars=${mainPressure.peakRendererInFlightChars} mainPeakMaxInFlightChars=${mainPressure.peakMaxRendererInFlightCharsByPty} mainAckGatedFlushSkips=${mainPressure.ackGatedFlushSkipCount}` : '' + const ackGateSummary = ackGate + ? ` heldAckPtys=${ackGate.heldAckCount} heldAckChars=${ackGate.heldAckChars} gatedAckPtys=${ackGate.gatedPtyCount}` + : '' testInfo.annotations.push({ type, description: `panes=${paneCount} frames=${measurement.frameCount} median=${measurement.medianLatencyMs.toFixed( @@ -403,7 +505,7 @@ function annotateTypingMeasurement( 1 )}ms maxTimerDrift=${measurement.maxTimerDriftMs.toFixed(1)}ms samples=${measurement.latencies .map((value) => value.toFixed(1)) - .join(',')}${hiddenSkipSummary}${schedulerSummary}${mainPressureSummary}` + .join(',')}${hiddenSkipSummary}${schedulerSummary}${mainPressureSummary}${ackGateSummary}` }) } @@ -556,6 +658,73 @@ test.describe('Artificial OpenCode terminal load', () => { } }) + test('keeps active typing responsive while background PTYs are ACK-backpressured', async ({ + orcaPage, + testRepoPath + }, testInfo) => { + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + const panes = await ensureActiveWorktreePaneLoad(orcaPage, PRESSURE_BACKGROUND_PANES + 1) + const [typingPane, ...loadPanes] = panes + await focusPane(orcaPage, typingPane.paneKey) + + const runId = randomUUID() + const typingScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-typing-${runId}.mjs`) + const pressureScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-load-${runId}.mjs`) + writeInteractivePromptScript(typingScriptPath, runId) + writePressureOutputScript(pressureScriptPath, runId) + await resetTerminalPtyOutputDebug(orcaPage) + await holdTerminalAckGate( + orcaPage, + loadPanes.map((pane) => pane.ptyId) + ) + try { + await Promise.all( + loadPanes.map((pane, paneIndex) => + sendToTerminal( + orcaPage, + pane.ptyId, + `node ${JSON.stringify(pressureScriptPath)} ${paneIndex} ${PRESSURE_OUTPUT_CHARS}\r` + ) + ) + ) + const pressureBeforeTyping = await waitForMainPtyPressureBacklog(orcaPage) + const measurement = await measureTypingDuringLoad( + orcaPage, + typingScriptPath, + typingPane.ptyId, + runId + ) + const mainPressure = await readMainPtyPressureDebug(orcaPage) + const ackGate = await readTerminalAckGateDebug(orcaPage) + annotateTypingMeasurement( + testInfo, + 'opencode-main-pressure-active-typing', + panes.length, + measurement, + await readTerminalPtyOutputDebug(orcaPage), + await readTerminalOutputSchedulerDebug(orcaPage), + mainPressure, + ackGate + ) + expect(pressureBeforeTyping.peakPendingChars).toBeGreaterThan(0) + expect(pressureBeforeTyping.ackGatedFlushSkipCount).toBeGreaterThan(0) + expect(mainPressure?.peakRendererInFlightChars ?? 0).toBeGreaterThanOrEqual(8 * 1024 * 1024) + expect(ackGate?.heldAckChars ?? 0).toBeGreaterThan(0) + expect(measurement.medianLatencyMs).toBeLessThan(MAX_MEDIAN_KEY_LATENCY_MS) + expect(measurement.worstLatencyMs).toBeLessThan(MAX_WORST_KEY_LATENCY_MS) + expect(measurement.maxTimerDriftMs).toBeLessThan(MAX_TIMER_DRIFT_MS) + } finally { + await releaseTerminalAckGate(orcaPage) + await sendToTerminal(orcaPage, typingPane.ptyId, '\x03').catch(() => undefined) + await Promise.all( + loadPanes.map((pane) => sendToTerminal(orcaPage, pane.ptyId, '\x03').catch(() => undefined)) + ) + rmSync(typingScriptPath, { force: true }) + rmSync(pressureScriptPath, { force: true }) + } + }) + for (const paneCount of SCALE_SAME_WORKSPACE_PANES) { test(`keeps typing responsive at ${paneCount} same-workspace OpenCode panes`, async ({ orcaPage,