diff --git a/config/scripts/run-terminal-scale-perf-e2e.mjs b/config/scripts/run-terminal-scale-perf-e2e.mjs index 5d8799d54..5c8ee1f9d 100644 --- a/config/scripts/run-terminal-scale-perf-e2e.mjs +++ b/config/scripts/run-terminal-scale-perf-e2e.mjs @@ -7,6 +7,8 @@ const env = { ORCA_E2E_OPENCODE_SCALE_PANES: process.env.ORCA_E2E_OPENCODE_SCALE_PANES ?? '10,25,50,100', ORCA_E2E_OPENCODE_SCALE_CROSS_WORKSPACE_PANES: process.env.ORCA_E2E_OPENCODE_SCALE_CROSS_WORKSPACE_PANES ?? '10,25,50,100', + ORCA_E2E_OPENCODE_SCALE_PRESSURE_PANES: + process.env.ORCA_E2E_OPENCODE_SCALE_PRESSURE_PANES ?? '25,50', ORCA_E2E_OPENCODE_FRAME_COUNT: process.env.ORCA_E2E_OPENCODE_FRAME_COUNT ?? '60' } const extraArgs = process.argv.slice(2) diff --git a/tests/e2e/artificial-opencode-main-pressure-scenario.ts b/tests/e2e/artificial-opencode-main-pressure-scenario.ts new file mode 100644 index 000000000..90b7b87b7 --- /dev/null +++ b/tests/e2e/artificial-opencode-main-pressure-scenario.ts @@ -0,0 +1,252 @@ +import type { Page, TestInfo } from '@stablyai/playwright-test' +import { expect } from '@stablyai/playwright-test' +import { randomUUID } from 'node:crypto' +import { rmSync } from 'node:fs' +import path from 'node:path' +import { sendToTerminal } from './helpers/terminal' +import { writePressureOutputScript } from './artificial-opencode-hidden-pressure-scenario' +import { + annotateScrollMeasurement, + measureActiveTerminalWheelScroll, + scrollActiveTerminalToBottom, + seedActiveTerminalScrollback +} from './artificial-opencode-scroll-scenario' + +type MainPressurePane = { + paneKey: string + ptyId: string +} + +type MainPressureMeasurement = { + medianLatencyMs: number + worstLatencyMs: number + maxTimerDriftMs: number +} + +type MainPressureSnapshot = { + peakPendingChars: number + peakRendererInFlightChars: number + ackGatedFlushSkipCount: number +} + +type MainPressureAckGate = { + heldAckChars: number +} + +type MainPressureDeps = { + annotateTypingMeasurement: ( + testInfo: TestInfo, + type: string, + paneCount: number, + measurement: TMeasurement, + debug: TDebug | null, + scheduler: TScheduler | null, + mainPressure: TMainPressure | null, + ackGate: TAckGate | null + ) => void + ensureActiveWorktreePaneLoad: (page: Page, paneCount: number) => Promise + focusPane: (page: Page, paneKey: string) => Promise + holdTerminalAckGate: (page: Page, ptyIds: string[]) => Promise + measureTypingDuringLoad: ( + page: Page, + scriptPath: string, + ptyId: string, + runId: string + ) => Promise + readMainPtyPressureDebug: (page: Page) => Promise + readTerminalAckGateDebug: (page: Page) => Promise + readTerminalOutputSchedulerDebug: (page: Page) => Promise + readTerminalPtyOutputDebug: (page: Page) => Promise + releaseTerminalAckGate: (page: Page) => Promise + resetTerminalPtyOutputDebug: (page: Page) => Promise + waitForActiveWorktree: (page: Page) => Promise + waitForMainPtyPressureBacklog: (page: Page) => Promise + waitForSessionReady: (page: Page) => Promise + writeInteractivePromptScript: (scriptPath: string, runId: string) => void +} + +export async function runMainPressureScenario< + TMeasurement extends MainPressureMeasurement, + TMainPressure extends MainPressureSnapshot, + TAckGate extends MainPressureAckGate, + TDebug, + TScheduler +>({ + annotationSuffix, + backgroundPaneCount, + deps, + maxMedianKeyLatencyMs, + maxScrollLatencyMs, + maxTimerDriftMs, + maxWorstKeyLatencyMs, + pressureOutputChars, + testInfo, + testRepoPath, + orcaPage +}: { + annotationSuffix: string + backgroundPaneCount: number + deps: MainPressureDeps + maxMedianKeyLatencyMs: number + maxScrollLatencyMs: number + maxTimerDriftMs: number + maxWorstKeyLatencyMs: number + pressureOutputChars: number + testInfo: TestInfo + testRepoPath: string + orcaPage: Page +}): Promise { + await deps.waitForSessionReady(orcaPage) + await deps.waitForActiveWorktree(orcaPage) + const panes = await deps.ensureActiveWorktreePaneLoad(orcaPage, backgroundPaneCount + 1) + const [typingPane, ...loadPanes] = panes + await deps.focusPane(orcaPage, typingPane.paneKey) + + const runId = randomUUID() + const scrollRunId = randomUUID() + const typingScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-typing-${runId}.mjs`) + const pressureScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-load-${runId}.mjs`) + await seedActiveTerminalScrollback(orcaPage, typingPane.ptyId, scrollRunId) + deps.writeInteractivePromptScript(typingScriptPath, runId) + writePressureOutputScript(pressureScriptPath, runId) + await deps.resetTerminalPtyOutputDebug(orcaPage) + await deps.holdTerminalAckGate( + orcaPage, + loadPanes.map((pane) => pane.ptyId) + ) + try { + await startPressureCommands({ + loadPanes, + orcaPage, + pressureOutputChars, + pressureScriptPath + }) + const pressureBeforeTyping = await deps.waitForMainPtyPressureBacklog(orcaPage) + await measureAndAnnotateScroll({ + annotationSuffix, + deps, + maxScrollLatencyMs, + maxTimerDriftMs, + orcaPage, + panes, + testInfo + }) + const measurement = await deps.measureTypingDuringLoad( + orcaPage, + typingScriptPath, + typingPane.ptyId, + runId + ) + const mainPressure = await deps.readMainPtyPressureDebug(orcaPage) + const ackGate = await deps.readTerminalAckGateDebug(orcaPage) + deps.annotateTypingMeasurement( + testInfo, + `opencode-main-pressure-active-typing${annotationSuffix}`, + panes.length, + measurement, + await deps.readTerminalPtyOutputDebug(orcaPage), + await deps.readTerminalOutputSchedulerDebug(orcaPage), + mainPressure, + ackGate + ) + expectMainPressureAndTyping({ + ackGate, + mainPressure, + maxMedianKeyLatencyMs, + maxTimerDriftMs, + maxWorstKeyLatencyMs, + measurement, + pressureBeforeTyping + }) + } finally { + await deps.releaseTerminalAckGate(orcaPage) + await sendToTerminal(orcaPage, typingPane.ptyId, '\x03').catch(() => undefined) + await Promise.all( + loadPanes.map((pane) => sendToTerminal(orcaPage, pane.ptyId, '\x03').catch(() => undefined)) + ) + rmSync(typingScriptPath, { force: true }) + rmSync(pressureScriptPath, { force: true }) + } +} + +async function startPressureCommands({ + loadPanes, + orcaPage, + pressureOutputChars, + pressureScriptPath +}: { + loadPanes: MainPressurePane[] + orcaPage: Page + pressureOutputChars: number + pressureScriptPath: string +}): Promise { + await Promise.all( + loadPanes.map((pane, paneIndex) => + sendToTerminal( + orcaPage, + pane.ptyId, + `node ${JSON.stringify(pressureScriptPath)} ${paneIndex} ${pressureOutputChars}\r` + ) + ) + ) +} + +async function measureAndAnnotateScroll({ + annotationSuffix, + deps, + maxScrollLatencyMs, + maxTimerDriftMs, + orcaPage, + panes, + testInfo +}: { + annotationSuffix: string + deps: MainPressureDeps + maxScrollLatencyMs: number + maxTimerDriftMs: number + orcaPage: Page + panes: MainPressurePane[] + testInfo: TestInfo +}): Promise { + const scrollMeasurement = await measureActiveTerminalWheelScroll(orcaPage) + const mainPressureAfterScroll = await deps.readMainPtyPressureDebug(orcaPage) + const ackGateAfterScroll = await deps.readTerminalAckGateDebug(orcaPage) + annotateScrollMeasurement( + testInfo, + `opencode-main-pressure-active-scroll${annotationSuffix}`, + panes.length, + scrollMeasurement, + mainPressureAfterScroll, + ackGateAfterScroll + ) + expect(scrollMeasurement.afterViewportY).toBeLessThan(scrollMeasurement.beforeViewportY) + expect(scrollMeasurement.scrollLatencyMs).toBeLessThan(maxScrollLatencyMs) + expect(scrollMeasurement.maxTimerDriftMs).toBeLessThan(maxTimerDriftMs) + await scrollActiveTerminalToBottom(orcaPage) +} + +function expectMainPressureAndTyping({ + ackGate, + mainPressure, + maxMedianKeyLatencyMs, + maxTimerDriftMs, + maxWorstKeyLatencyMs, + measurement, + pressureBeforeTyping +}: { + ackGate: MainPressureAckGate | null + mainPressure: MainPressureSnapshot | null + maxMedianKeyLatencyMs: number + maxTimerDriftMs: number + maxWorstKeyLatencyMs: number + measurement: TMeasurement + pressureBeforeTyping: MainPressureSnapshot +}): void { + expect(pressureBeforeTyping.peakPendingChars).toBeGreaterThan(0) + expect(pressureBeforeTyping.ackGatedFlushSkipCount).toBeGreaterThan(0) + expect(mainPressure?.peakRendererInFlightChars ?? 0).toBeGreaterThanOrEqual(8 * 1024 * 1024) + expect(ackGate?.heldAckChars ?? 0).toBeGreaterThan(0) + expect(measurement.medianLatencyMs).toBeLessThan(maxMedianKeyLatencyMs) + expect(measurement.worstLatencyMs).toBeLessThan(maxWorstKeyLatencyMs) + expect(measurement.maxTimerDriftMs).toBeLessThan(maxTimerDriftMs) +} diff --git a/tests/e2e/artificial-opencode-terminal-load.spec.ts b/tests/e2e/artificial-opencode-terminal-load.spec.ts index 15d704132..0cddb77d4 100644 --- a/tests/e2e/artificial-opencode-terminal-load.spec.ts +++ b/tests/e2e/artificial-opencode-terminal-load.spec.ts @@ -20,16 +20,8 @@ import { waitForPaneIdentitySnapshot, waitForTerminalOutput } from './helpers/terminal' -import { - runHiddenRealPtyPressureScenario, - writePressureOutputScript -} from './artificial-opencode-hidden-pressure-scenario' -import { - annotateScrollMeasurement, - measureActiveTerminalWheelScroll, - scrollActiveTerminalToBottom, - seedActiveTerminalScrollback -} from './artificial-opencode-scroll-scenario' +import { runHiddenRealPtyPressureScenario } from './artificial-opencode-hidden-pressure-scenario' +import { runMainPressureScenario } from './artificial-opencode-main-pressure-scenario' type TerminalLoadPane = { paneKey: string @@ -171,6 +163,7 @@ const SCALE_SAME_WORKSPACE_PANES = readPositiveIntList('ORCA_E2E_OPENCODE_SCALE_ const SCALE_CROSS_WORKSPACE_PANES = readPositiveIntList( 'ORCA_E2E_OPENCODE_SCALE_CROSS_WORKSPACE_PANES' ) +const SCALE_PRESSURE_PANES = readPositiveIntList('ORCA_E2E_OPENCODE_SCALE_PRESSURE_PANES') function interactivePromptScript(runId: string): string { return ` @@ -560,6 +553,50 @@ async function measureCrossWorkspaceTypingDuringHiddenLoad({ } } +async function runConfiguredMainPressureScenario({ + annotationSuffix, + backgroundPaneCount, + orcaPage, + testInfo, + testRepoPath +}: { + annotationSuffix: string + backgroundPaneCount: number + orcaPage: Page + testInfo: TestInfo + testRepoPath: string +}): Promise { + await runMainPressureScenario({ + annotationSuffix, + backgroundPaneCount, + orcaPage, + pressureOutputChars: PRESSURE_OUTPUT_CHARS, + testInfo, + testRepoPath, + maxMedianKeyLatencyMs: MAX_MEDIAN_KEY_LATENCY_MS, + maxScrollLatencyMs: MAX_SCROLL_LATENCY_MS, + maxTimerDriftMs: MAX_TIMER_DRIFT_MS, + maxWorstKeyLatencyMs: MAX_WORST_KEY_LATENCY_MS, + deps: { + annotateTypingMeasurement, + ensureActiveWorktreePaneLoad, + focusPane, + holdTerminalAckGate, + measureTypingDuringLoad, + readMainPtyPressureDebug, + readTerminalAckGateDebug, + readTerminalOutputSchedulerDebug, + readTerminalPtyOutputDebug, + releaseTerminalAckGate, + resetTerminalPtyOutputDebug, + waitForActiveWorktree, + waitForMainPtyPressureBacklog, + waitForSessionReady, + writeInteractivePromptScript + } + }) +} + test.describe('Artificial OpenCode terminal load', () => { test.describe.configure({ mode: 'serial' }) @@ -648,86 +685,30 @@ test.describe('Artificial OpenCode terminal load', () => { orcaPage, testRepoPath }, testInfo) => { - await waitForSessionReady(orcaPage) - await waitForActiveWorktree(orcaPage) - const panes = await ensureActiveWorktreePaneLoad(orcaPage, PRESSURE_BACKGROUND_PANES + 1) - const [typingPane, ...loadPanes] = panes - await focusPane(orcaPage, typingPane.paneKey) - - const runId = randomUUID() - const scrollRunId = randomUUID() - const typingScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-typing-${runId}.mjs`) - const pressureScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-load-${runId}.mjs`) - await seedActiveTerminalScrollback(orcaPage, typingPane.ptyId, scrollRunId) - writeInteractivePromptScript(typingScriptPath, runId) - writePressureOutputScript(pressureScriptPath, runId) - await resetTerminalPtyOutputDebug(orcaPage) - await holdTerminalAckGate( + await runConfiguredMainPressureScenario({ orcaPage, - loadPanes.map((pane) => pane.ptyId) - ) - try { - await Promise.all( - loadPanes.map((pane, paneIndex) => - sendToTerminal( - orcaPage, - pane.ptyId, - `node ${JSON.stringify(pressureScriptPath)} ${paneIndex} ${PRESSURE_OUTPUT_CHARS}\r` - ) - ) - ) - const pressureBeforeTyping = await waitForMainPtyPressureBacklog(orcaPage) - const scrollMeasurement = await measureActiveTerminalWheelScroll(orcaPage) - const mainPressureAfterScroll = await readMainPtyPressureDebug(orcaPage) - const ackGateAfterScroll = await readTerminalAckGateDebug(orcaPage) - annotateScrollMeasurement( - testInfo, - 'opencode-main-pressure-active-scroll', - panes.length, - scrollMeasurement, - mainPressureAfterScroll, - ackGateAfterScroll - ) - expect(scrollMeasurement.afterViewportY).toBeLessThan(scrollMeasurement.beforeViewportY) - expect(scrollMeasurement.scrollLatencyMs).toBeLessThan(MAX_SCROLL_LATENCY_MS) - expect(scrollMeasurement.maxTimerDriftMs).toBeLessThan(MAX_TIMER_DRIFT_MS) - await scrollActiveTerminalToBottom(orcaPage) - const measurement = await measureTypingDuringLoad( - orcaPage, - typingScriptPath, - typingPane.ptyId, - runId - ) - const mainPressure = await readMainPtyPressureDebug(orcaPage) - const ackGate = await readTerminalAckGateDebug(orcaPage) - annotateTypingMeasurement( - testInfo, - 'opencode-main-pressure-active-typing', - panes.length, - measurement, - await readTerminalPtyOutputDebug(orcaPage), - await readTerminalOutputSchedulerDebug(orcaPage), - mainPressure, - ackGate - ) - expect(pressureBeforeTyping.peakPendingChars).toBeGreaterThan(0) - expect(pressureBeforeTyping.ackGatedFlushSkipCount).toBeGreaterThan(0) - expect(mainPressure?.peakRendererInFlightChars ?? 0).toBeGreaterThanOrEqual(8 * 1024 * 1024) - expect(ackGate?.heldAckChars ?? 0).toBeGreaterThan(0) - expect(measurement.medianLatencyMs).toBeLessThan(MAX_MEDIAN_KEY_LATENCY_MS) - expect(measurement.worstLatencyMs).toBeLessThan(MAX_WORST_KEY_LATENCY_MS) - expect(measurement.maxTimerDriftMs).toBeLessThan(MAX_TIMER_DRIFT_MS) - } finally { - await releaseTerminalAckGate(orcaPage) - await sendToTerminal(orcaPage, typingPane.ptyId, '\x03').catch(() => undefined) - await Promise.all( - loadPanes.map((pane) => sendToTerminal(orcaPage, pane.ptyId, '\x03').catch(() => undefined)) - ) - rmSync(typingScriptPath, { force: true }) - rmSync(pressureScriptPath, { force: true }) - } + testRepoPath, + backgroundPaneCount: PRESSURE_BACKGROUND_PANES, + annotationSuffix: '', + testInfo + }) }) + for (const paneCount of SCALE_PRESSURE_PANES) { + test(`keeps active interactions responsive at ${paneCount} ACK-backpressured OpenCode PTYs`, async ({ + orcaPage, + testRepoPath + }, testInfo) => { + await runConfiguredMainPressureScenario({ + orcaPage, + testRepoPath, + backgroundPaneCount: paneCount, + annotationSuffix: `-${paneCount}`, + testInfo + }) + }) + } + for (const paneCount of SCALE_SAME_WORKSPACE_PANES) { test(`keeps typing responsive at ${paneCount} same-workspace OpenCode panes`, async ({ orcaPage,