Scale real PTY pressure benchmarks (#4808)
This commit is contained in:
Neil
GitHub
parent
9c92e3d47b
commit
7ac509ed62
|
|
@ -7,6 +7,8 @@ const env = {
|
|||
ORCA_E2E_OPENCODE_SCALE_PANES: process.env.ORCA_E2E_OPENCODE_SCALE_PANES ?? '10,25,50,100',
|
||||
ORCA_E2E_OPENCODE_SCALE_CROSS_WORKSPACE_PANES:
|
||||
process.env.ORCA_E2E_OPENCODE_SCALE_CROSS_WORKSPACE_PANES ?? '10,25,50,100',
|
||||
ORCA_E2E_OPENCODE_SCALE_PRESSURE_PANES:
|
||||
process.env.ORCA_E2E_OPENCODE_SCALE_PRESSURE_PANES ?? '25,50',
|
||||
ORCA_E2E_OPENCODE_FRAME_COUNT: process.env.ORCA_E2E_OPENCODE_FRAME_COUNT ?? '60'
|
||||
}
|
||||
const extraArgs = process.argv.slice(2)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,252 @@
|
|||
import type { Page, TestInfo } from '@stablyai/playwright-test'
|
||||
import { expect } from '@stablyai/playwright-test'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { rmSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { sendToTerminal } from './helpers/terminal'
|
||||
import { writePressureOutputScript } from './artificial-opencode-hidden-pressure-scenario'
|
||||
import {
|
||||
annotateScrollMeasurement,
|
||||
measureActiveTerminalWheelScroll,
|
||||
scrollActiveTerminalToBottom,
|
||||
seedActiveTerminalScrollback
|
||||
} from './artificial-opencode-scroll-scenario'
|
||||
|
||||
type MainPressurePane = {
|
||||
paneKey: string
|
||||
ptyId: string
|
||||
}
|
||||
|
||||
type MainPressureMeasurement = {
|
||||
medianLatencyMs: number
|
||||
worstLatencyMs: number
|
||||
maxTimerDriftMs: number
|
||||
}
|
||||
|
||||
type MainPressureSnapshot = {
|
||||
peakPendingChars: number
|
||||
peakRendererInFlightChars: number
|
||||
ackGatedFlushSkipCount: number
|
||||
}
|
||||
|
||||
type MainPressureAckGate = {
|
||||
heldAckChars: number
|
||||
}
|
||||
|
||||
type MainPressureDeps<TMeasurement, TDebug, TScheduler, TMainPressure, TAckGate> = {
|
||||
annotateTypingMeasurement: (
|
||||
testInfo: TestInfo,
|
||||
type: string,
|
||||
paneCount: number,
|
||||
measurement: TMeasurement,
|
||||
debug: TDebug | null,
|
||||
scheduler: TScheduler | null,
|
||||
mainPressure: TMainPressure | null,
|
||||
ackGate: TAckGate | null
|
||||
) => void
|
||||
ensureActiveWorktreePaneLoad: (page: Page, paneCount: number) => Promise<MainPressurePane[]>
|
||||
focusPane: (page: Page, paneKey: string) => Promise<void>
|
||||
holdTerminalAckGate: (page: Page, ptyIds: string[]) => Promise<void>
|
||||
measureTypingDuringLoad: (
|
||||
page: Page,
|
||||
scriptPath: string,
|
||||
ptyId: string,
|
||||
runId: string
|
||||
) => Promise<TMeasurement>
|
||||
readMainPtyPressureDebug: (page: Page) => Promise<TMainPressure | null>
|
||||
readTerminalAckGateDebug: (page: Page) => Promise<TAckGate | null>
|
||||
readTerminalOutputSchedulerDebug: (page: Page) => Promise<TScheduler | null>
|
||||
readTerminalPtyOutputDebug: (page: Page) => Promise<TDebug | null>
|
||||
releaseTerminalAckGate: (page: Page) => Promise<void>
|
||||
resetTerminalPtyOutputDebug: (page: Page) => Promise<void>
|
||||
waitForActiveWorktree: (page: Page) => Promise<string>
|
||||
waitForMainPtyPressureBacklog: (page: Page) => Promise<TMainPressure>
|
||||
waitForSessionReady: (page: Page) => Promise<void>
|
||||
writeInteractivePromptScript: (scriptPath: string, runId: string) => void
|
||||
}
|
||||
|
||||
export async function runMainPressureScenario<
|
||||
TMeasurement extends MainPressureMeasurement,
|
||||
TMainPressure extends MainPressureSnapshot,
|
||||
TAckGate extends MainPressureAckGate,
|
||||
TDebug,
|
||||
TScheduler
|
||||
>({
|
||||
annotationSuffix,
|
||||
backgroundPaneCount,
|
||||
deps,
|
||||
maxMedianKeyLatencyMs,
|
||||
maxScrollLatencyMs,
|
||||
maxTimerDriftMs,
|
||||
maxWorstKeyLatencyMs,
|
||||
pressureOutputChars,
|
||||
testInfo,
|
||||
testRepoPath,
|
||||
orcaPage
|
||||
}: {
|
||||
annotationSuffix: string
|
||||
backgroundPaneCount: number
|
||||
deps: MainPressureDeps<TMeasurement, TDebug, TScheduler, TMainPressure, TAckGate>
|
||||
maxMedianKeyLatencyMs: number
|
||||
maxScrollLatencyMs: number
|
||||
maxTimerDriftMs: number
|
||||
maxWorstKeyLatencyMs: number
|
||||
pressureOutputChars: number
|
||||
testInfo: TestInfo
|
||||
testRepoPath: string
|
||||
orcaPage: Page
|
||||
}): Promise<void> {
|
||||
await deps.waitForSessionReady(orcaPage)
|
||||
await deps.waitForActiveWorktree(orcaPage)
|
||||
const panes = await deps.ensureActiveWorktreePaneLoad(orcaPage, backgroundPaneCount + 1)
|
||||
const [typingPane, ...loadPanes] = panes
|
||||
await deps.focusPane(orcaPage, typingPane.paneKey)
|
||||
|
||||
const runId = randomUUID()
|
||||
const scrollRunId = randomUUID()
|
||||
const typingScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-typing-${runId}.mjs`)
|
||||
const pressureScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-load-${runId}.mjs`)
|
||||
await seedActiveTerminalScrollback(orcaPage, typingPane.ptyId, scrollRunId)
|
||||
deps.writeInteractivePromptScript(typingScriptPath, runId)
|
||||
writePressureOutputScript(pressureScriptPath, runId)
|
||||
await deps.resetTerminalPtyOutputDebug(orcaPage)
|
||||
await deps.holdTerminalAckGate(
|
||||
orcaPage,
|
||||
loadPanes.map((pane) => pane.ptyId)
|
||||
)
|
||||
try {
|
||||
await startPressureCommands({
|
||||
loadPanes,
|
||||
orcaPage,
|
||||
pressureOutputChars,
|
||||
pressureScriptPath
|
||||
})
|
||||
const pressureBeforeTyping = await deps.waitForMainPtyPressureBacklog(orcaPage)
|
||||
await measureAndAnnotateScroll({
|
||||
annotationSuffix,
|
||||
deps,
|
||||
maxScrollLatencyMs,
|
||||
maxTimerDriftMs,
|
||||
orcaPage,
|
||||
panes,
|
||||
testInfo
|
||||
})
|
||||
const measurement = await deps.measureTypingDuringLoad(
|
||||
orcaPage,
|
||||
typingScriptPath,
|
||||
typingPane.ptyId,
|
||||
runId
|
||||
)
|
||||
const mainPressure = await deps.readMainPtyPressureDebug(orcaPage)
|
||||
const ackGate = await deps.readTerminalAckGateDebug(orcaPage)
|
||||
deps.annotateTypingMeasurement(
|
||||
testInfo,
|
||||
`opencode-main-pressure-active-typing${annotationSuffix}`,
|
||||
panes.length,
|
||||
measurement,
|
||||
await deps.readTerminalPtyOutputDebug(orcaPage),
|
||||
await deps.readTerminalOutputSchedulerDebug(orcaPage),
|
||||
mainPressure,
|
||||
ackGate
|
||||
)
|
||||
expectMainPressureAndTyping({
|
||||
ackGate,
|
||||
mainPressure,
|
||||
maxMedianKeyLatencyMs,
|
||||
maxTimerDriftMs,
|
||||
maxWorstKeyLatencyMs,
|
||||
measurement,
|
||||
pressureBeforeTyping
|
||||
})
|
||||
} finally {
|
||||
await deps.releaseTerminalAckGate(orcaPage)
|
||||
await sendToTerminal(orcaPage, typingPane.ptyId, '\x03').catch(() => undefined)
|
||||
await Promise.all(
|
||||
loadPanes.map((pane) => sendToTerminal(orcaPage, pane.ptyId, '\x03').catch(() => undefined))
|
||||
)
|
||||
rmSync(typingScriptPath, { force: true })
|
||||
rmSync(pressureScriptPath, { force: true })
|
||||
}
|
||||
}
|
||||
|
||||
async function startPressureCommands({
|
||||
loadPanes,
|
||||
orcaPage,
|
||||
pressureOutputChars,
|
||||
pressureScriptPath
|
||||
}: {
|
||||
loadPanes: MainPressurePane[]
|
||||
orcaPage: Page
|
||||
pressureOutputChars: number
|
||||
pressureScriptPath: string
|
||||
}): Promise<void> {
|
||||
await Promise.all(
|
||||
loadPanes.map((pane, paneIndex) =>
|
||||
sendToTerminal(
|
||||
orcaPage,
|
||||
pane.ptyId,
|
||||
`node ${JSON.stringify(pressureScriptPath)} ${paneIndex} ${pressureOutputChars}\r`
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
async function measureAndAnnotateScroll<TMeasurement, TDebug, TScheduler, TMainPressure, TAckGate>({
|
||||
annotationSuffix,
|
||||
deps,
|
||||
maxScrollLatencyMs,
|
||||
maxTimerDriftMs,
|
||||
orcaPage,
|
||||
panes,
|
||||
testInfo
|
||||
}: {
|
||||
annotationSuffix: string
|
||||
deps: MainPressureDeps<TMeasurement, TDebug, TScheduler, TMainPressure, TAckGate>
|
||||
maxScrollLatencyMs: number
|
||||
maxTimerDriftMs: number
|
||||
orcaPage: Page
|
||||
panes: MainPressurePane[]
|
||||
testInfo: TestInfo
|
||||
}): Promise<void> {
|
||||
const scrollMeasurement = await measureActiveTerminalWheelScroll(orcaPage)
|
||||
const mainPressureAfterScroll = await deps.readMainPtyPressureDebug(orcaPage)
|
||||
const ackGateAfterScroll = await deps.readTerminalAckGateDebug(orcaPage)
|
||||
annotateScrollMeasurement(
|
||||
testInfo,
|
||||
`opencode-main-pressure-active-scroll${annotationSuffix}`,
|
||||
panes.length,
|
||||
scrollMeasurement,
|
||||
mainPressureAfterScroll,
|
||||
ackGateAfterScroll
|
||||
)
|
||||
expect(scrollMeasurement.afterViewportY).toBeLessThan(scrollMeasurement.beforeViewportY)
|
||||
expect(scrollMeasurement.scrollLatencyMs).toBeLessThan(maxScrollLatencyMs)
|
||||
expect(scrollMeasurement.maxTimerDriftMs).toBeLessThan(maxTimerDriftMs)
|
||||
await scrollActiveTerminalToBottom(orcaPage)
|
||||
}
|
||||
|
||||
function expectMainPressureAndTyping<TMeasurement extends MainPressureMeasurement>({
|
||||
ackGate,
|
||||
mainPressure,
|
||||
maxMedianKeyLatencyMs,
|
||||
maxTimerDriftMs,
|
||||
maxWorstKeyLatencyMs,
|
||||
measurement,
|
||||
pressureBeforeTyping
|
||||
}: {
|
||||
ackGate: MainPressureAckGate | null
|
||||
mainPressure: MainPressureSnapshot | null
|
||||
maxMedianKeyLatencyMs: number
|
||||
maxTimerDriftMs: number
|
||||
maxWorstKeyLatencyMs: number
|
||||
measurement: TMeasurement
|
||||
pressureBeforeTyping: MainPressureSnapshot
|
||||
}): void {
|
||||
expect(pressureBeforeTyping.peakPendingChars).toBeGreaterThan(0)
|
||||
expect(pressureBeforeTyping.ackGatedFlushSkipCount).toBeGreaterThan(0)
|
||||
expect(mainPressure?.peakRendererInFlightChars ?? 0).toBeGreaterThanOrEqual(8 * 1024 * 1024)
|
||||
expect(ackGate?.heldAckChars ?? 0).toBeGreaterThan(0)
|
||||
expect(measurement.medianLatencyMs).toBeLessThan(maxMedianKeyLatencyMs)
|
||||
expect(measurement.worstLatencyMs).toBeLessThan(maxWorstKeyLatencyMs)
|
||||
expect(measurement.maxTimerDriftMs).toBeLessThan(maxTimerDriftMs)
|
||||
}
|
||||
|
|
@ -20,16 +20,8 @@ import {
|
|||
waitForPaneIdentitySnapshot,
|
||||
waitForTerminalOutput
|
||||
} from './helpers/terminal'
|
||||
import {
|
||||
runHiddenRealPtyPressureScenario,
|
||||
writePressureOutputScript
|
||||
} from './artificial-opencode-hidden-pressure-scenario'
|
||||
import {
|
||||
annotateScrollMeasurement,
|
||||
measureActiveTerminalWheelScroll,
|
||||
scrollActiveTerminalToBottom,
|
||||
seedActiveTerminalScrollback
|
||||
} from './artificial-opencode-scroll-scenario'
|
||||
import { runHiddenRealPtyPressureScenario } from './artificial-opencode-hidden-pressure-scenario'
|
||||
import { runMainPressureScenario } from './artificial-opencode-main-pressure-scenario'
|
||||
|
||||
type TerminalLoadPane = {
|
||||
paneKey: string
|
||||
|
|
@ -171,6 +163,7 @@ const SCALE_SAME_WORKSPACE_PANES = readPositiveIntList('ORCA_E2E_OPENCODE_SCALE_
|
|||
const SCALE_CROSS_WORKSPACE_PANES = readPositiveIntList(
|
||||
'ORCA_E2E_OPENCODE_SCALE_CROSS_WORKSPACE_PANES'
|
||||
)
|
||||
const SCALE_PRESSURE_PANES = readPositiveIntList('ORCA_E2E_OPENCODE_SCALE_PRESSURE_PANES')
|
||||
|
||||
function interactivePromptScript(runId: string): string {
|
||||
return `
|
||||
|
|
@ -560,6 +553,50 @@ async function measureCrossWorkspaceTypingDuringHiddenLoad({
|
|||
}
|
||||
}
|
||||
|
||||
async function runConfiguredMainPressureScenario({
|
||||
annotationSuffix,
|
||||
backgroundPaneCount,
|
||||
orcaPage,
|
||||
testInfo,
|
||||
testRepoPath
|
||||
}: {
|
||||
annotationSuffix: string
|
||||
backgroundPaneCount: number
|
||||
orcaPage: Page
|
||||
testInfo: TestInfo
|
||||
testRepoPath: string
|
||||
}): Promise<void> {
|
||||
await runMainPressureScenario({
|
||||
annotationSuffix,
|
||||
backgroundPaneCount,
|
||||
orcaPage,
|
||||
pressureOutputChars: PRESSURE_OUTPUT_CHARS,
|
||||
testInfo,
|
||||
testRepoPath,
|
||||
maxMedianKeyLatencyMs: MAX_MEDIAN_KEY_LATENCY_MS,
|
||||
maxScrollLatencyMs: MAX_SCROLL_LATENCY_MS,
|
||||
maxTimerDriftMs: MAX_TIMER_DRIFT_MS,
|
||||
maxWorstKeyLatencyMs: MAX_WORST_KEY_LATENCY_MS,
|
||||
deps: {
|
||||
annotateTypingMeasurement,
|
||||
ensureActiveWorktreePaneLoad,
|
||||
focusPane,
|
||||
holdTerminalAckGate,
|
||||
measureTypingDuringLoad,
|
||||
readMainPtyPressureDebug,
|
||||
readTerminalAckGateDebug,
|
||||
readTerminalOutputSchedulerDebug,
|
||||
readTerminalPtyOutputDebug,
|
||||
releaseTerminalAckGate,
|
||||
resetTerminalPtyOutputDebug,
|
||||
waitForActiveWorktree,
|
||||
waitForMainPtyPressureBacklog,
|
||||
waitForSessionReady,
|
||||
writeInteractivePromptScript
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
test.describe('Artificial OpenCode terminal load', () => {
|
||||
test.describe.configure({ mode: 'serial' })
|
||||
|
||||
|
|
@ -648,86 +685,30 @@ test.describe('Artificial OpenCode terminal load', () => {
|
|||
orcaPage,
|
||||
testRepoPath
|
||||
}, testInfo) => {
|
||||
await waitForSessionReady(orcaPage)
|
||||
await waitForActiveWorktree(orcaPage)
|
||||
const panes = await ensureActiveWorktreePaneLoad(orcaPage, PRESSURE_BACKGROUND_PANES + 1)
|
||||
const [typingPane, ...loadPanes] = panes
|
||||
await focusPane(orcaPage, typingPane.paneKey)
|
||||
|
||||
const runId = randomUUID()
|
||||
const scrollRunId = randomUUID()
|
||||
const typingScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-typing-${runId}.mjs`)
|
||||
const pressureScriptPath = path.join(testRepoPath, `.orca-opencode-pressure-load-${runId}.mjs`)
|
||||
await seedActiveTerminalScrollback(orcaPage, typingPane.ptyId, scrollRunId)
|
||||
writeInteractivePromptScript(typingScriptPath, runId)
|
||||
writePressureOutputScript(pressureScriptPath, runId)
|
||||
await resetTerminalPtyOutputDebug(orcaPage)
|
||||
await holdTerminalAckGate(
|
||||
await runConfiguredMainPressureScenario({
|
||||
orcaPage,
|
||||
loadPanes.map((pane) => pane.ptyId)
|
||||
)
|
||||
try {
|
||||
await Promise.all(
|
||||
loadPanes.map((pane, paneIndex) =>
|
||||
sendToTerminal(
|
||||
orcaPage,
|
||||
pane.ptyId,
|
||||
`node ${JSON.stringify(pressureScriptPath)} ${paneIndex} ${PRESSURE_OUTPUT_CHARS}\r`
|
||||
)
|
||||
)
|
||||
)
|
||||
const pressureBeforeTyping = await waitForMainPtyPressureBacklog(orcaPage)
|
||||
const scrollMeasurement = await measureActiveTerminalWheelScroll(orcaPage)
|
||||
const mainPressureAfterScroll = await readMainPtyPressureDebug(orcaPage)
|
||||
const ackGateAfterScroll = await readTerminalAckGateDebug(orcaPage)
|
||||
annotateScrollMeasurement(
|
||||
testInfo,
|
||||
'opencode-main-pressure-active-scroll',
|
||||
panes.length,
|
||||
scrollMeasurement,
|
||||
mainPressureAfterScroll,
|
||||
ackGateAfterScroll
|
||||
)
|
||||
expect(scrollMeasurement.afterViewportY).toBeLessThan(scrollMeasurement.beforeViewportY)
|
||||
expect(scrollMeasurement.scrollLatencyMs).toBeLessThan(MAX_SCROLL_LATENCY_MS)
|
||||
expect(scrollMeasurement.maxTimerDriftMs).toBeLessThan(MAX_TIMER_DRIFT_MS)
|
||||
await scrollActiveTerminalToBottom(orcaPage)
|
||||
const measurement = await measureTypingDuringLoad(
|
||||
orcaPage,
|
||||
typingScriptPath,
|
||||
typingPane.ptyId,
|
||||
runId
|
||||
)
|
||||
const mainPressure = await readMainPtyPressureDebug(orcaPage)
|
||||
const ackGate = await readTerminalAckGateDebug(orcaPage)
|
||||
annotateTypingMeasurement(
|
||||
testInfo,
|
||||
'opencode-main-pressure-active-typing',
|
||||
panes.length,
|
||||
measurement,
|
||||
await readTerminalPtyOutputDebug(orcaPage),
|
||||
await readTerminalOutputSchedulerDebug(orcaPage),
|
||||
mainPressure,
|
||||
ackGate
|
||||
)
|
||||
expect(pressureBeforeTyping.peakPendingChars).toBeGreaterThan(0)
|
||||
expect(pressureBeforeTyping.ackGatedFlushSkipCount).toBeGreaterThan(0)
|
||||
expect(mainPressure?.peakRendererInFlightChars ?? 0).toBeGreaterThanOrEqual(8 * 1024 * 1024)
|
||||
expect(ackGate?.heldAckChars ?? 0).toBeGreaterThan(0)
|
||||
expect(measurement.medianLatencyMs).toBeLessThan(MAX_MEDIAN_KEY_LATENCY_MS)
|
||||
expect(measurement.worstLatencyMs).toBeLessThan(MAX_WORST_KEY_LATENCY_MS)
|
||||
expect(measurement.maxTimerDriftMs).toBeLessThan(MAX_TIMER_DRIFT_MS)
|
||||
} finally {
|
||||
await releaseTerminalAckGate(orcaPage)
|
||||
await sendToTerminal(orcaPage, typingPane.ptyId, '\x03').catch(() => undefined)
|
||||
await Promise.all(
|
||||
loadPanes.map((pane) => sendToTerminal(orcaPage, pane.ptyId, '\x03').catch(() => undefined))
|
||||
)
|
||||
rmSync(typingScriptPath, { force: true })
|
||||
rmSync(pressureScriptPath, { force: true })
|
||||
}
|
||||
testRepoPath,
|
||||
backgroundPaneCount: PRESSURE_BACKGROUND_PANES,
|
||||
annotationSuffix: '',
|
||||
testInfo
|
||||
})
|
||||
})
|
||||
|
||||
for (const paneCount of SCALE_PRESSURE_PANES) {
|
||||
test(`keeps active interactions responsive at ${paneCount} ACK-backpressured OpenCode PTYs`, async ({
|
||||
orcaPage,
|
||||
testRepoPath
|
||||
}, testInfo) => {
|
||||
await runConfiguredMainPressureScenario({
|
||||
orcaPage,
|
||||
testRepoPath,
|
||||
backgroundPaneCount: paneCount,
|
||||
annotationSuffix: `-${paneCount}`,
|
||||
testInfo
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
for (const paneCount of SCALE_SAME_WORKSPACE_PANES) {
|
||||
test(`keeps typing responsive at ${paneCount} same-workspace OpenCode panes`, async ({
|
||||
orcaPage,
|
||||
|
|
|
|||
Loading…
Reference in New Issue