orca/tools/benchmarks/terminal-cold-park-reveal-b...

619 lines
23 KiB
JavaScript

#!/usr/bin/env node
// Cold-park reveal-cost benchmark: measures the reveal latency a user pays when
// returning to a terminal whose hidden view was cold-parked (React subtree
// unmounted, xterm/buffers/WebGL released) versus a warm reveal where the tab
// was still in the hot-retain working set. The stock terminal-perf-bench
// workspace-switch scenario cycles 3 worktrees fast enough that parking never
// fires, so it only ever measures warm reveals. This bench drives the
// ORCA_E2E_TERMINAL_PARKING_DELAY_MS override to shrink the 30s hysteresis and
// window.__terminalParkingDebug.parkedTabIds() to *confirm* a tab parked before
// timing its reveal — so the cold vs warm delta is the real cost of parking.
//
// Output phases per reveal (ms from the switch-back click):
// activationMs store activeWorktree flips to the target terminal
// ptyBindMs the revealed pane has a bound ptyId (xterm remounted+attached)
// paintSettleMs two RAFs after ptyBind (first painted frame settled)
//
// Arms:
// cold — tab confirmed parked before reveal (park delay shrunk, wait to park)
// warm — tab revealed before the park delay elapses (hot-retain hit)
// off — parking disabled entirely (kill switch), reveal after the same idle
import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import process from 'node:process'
import { execFileSync } from 'node:child_process'
import { fileURLToPath } from 'node:url'
import {
collectRendererDiagnostics,
connectToApp,
installRendererProbe,
launchDevApp,
pickFreePort,
stopDevApp,
waitForStoreReady
} from '../../config/scripts/windows-apphang-repro/electron-dev-session.mjs'
import { createCompletedOnboardingProfile } from '../../config/scripts/windows-apphang-repro/wsl-workspace-fixture.mjs'
import {
pollUntil,
rendererActionTimeoutMs,
runWithTimeout,
setupTimeoutMs
} from '../../config/scripts/windows-apphang-repro/repro-timing.mjs'
import { safeRemoveLocalDirectory } from '../../config/scripts/windows-apphang-repro/wsl-workspace-fixture.mjs'
const rootDir = path.resolve(fileURLToPath(new URL('../..', import.meta.url)))
const scenarioTimeoutMs = 300_000
// Short enough that a tab parks within a few seconds of being hidden, long
// enough that the warm arm can reliably reveal before it elapses.
const PARK_DELAY_MS = 1_500
const WARM_REVEAL_AFTER_MS = 300
const COLD_PARK_CONFIRM_TIMEOUT_MS = 15_000
function parseArgs() {
const args = { label: 'run', cycles: 8, scrollbackLines: 0, reportPath: null, keep: false }
for (const arg of process.argv.slice(2)) {
if (arg === '--help' || arg === '-h') {
console.log(
'Usage: node tools/benchmarks/terminal-cold-park-reveal-bench.mjs [--label=name] [--cycles=N] [--scrollback-lines=N] [--report=path] [--keep]'
)
process.exit(0)
}
if (arg === '--keep') {
args.keep = true
continue
}
const [name, value] = arg.split('=', 2)
if (name === '--label') {
args.label = value?.trim() || 'run'
} else if (name === '--cycles') {
args.cycles = Math.max(1, Number(value) || 8)
} else if (name === '--scrollback-lines') {
args.scrollbackLines = Math.max(0, Number(value) || 0)
} else if (name === '--report') {
args.reportPath = value
}
}
return args
}
// Why: the daemon binds a Unix socket at <userData>/daemon/daemon-v20.sock.
// macOS sun_path caps at ~104 chars; the default os.tmpdir() (/var/folders/…)
// blows past it, the daemon fails EINVAL, terminals fall back to non-snapshot
// PTYs, and cold parking (snapshot-backed only) can never engage. Root the
// userData + fixture under a short ~/.ocpb path so the socket fits.
const shortRoot = path.join(os.homedir(), '.ocpb')
function createShortUserDataDirectory() {
mkdirSync(shortRoot, { recursive: true })
const userDataDir = mkdtempSync(path.join(shortRoot, 'ud-'))
createCompletedOnboardingProfile(userDataDir)
return userDataDir
}
function git(cwd, ...cmd) {
execFileSync('git', cmd, { cwd, stdio: 'pipe' })
}
function createLocalRepoFixture() {
mkdirSync(shortRoot, { recursive: true })
const baseDir = mkdtempSync(path.join(shortRoot, 'fx-'))
const repoPath = path.join(baseDir, 'repo')
mkdirSync(repoPath, { recursive: true })
git(repoPath, 'init', '--initial-branch=main')
git(repoPath, 'config', 'user.email', 'bench@orca.local')
git(repoPath, 'config', 'user.name', 'Orca Bench')
writeFileSync(path.join(repoPath, 'README.md'), '# cold-park fixture\n')
git(repoPath, 'add', '.')
git(repoPath, 'commit', '-m', 'init', '--no-gpg-sign')
const worktreePaths = []
for (const name of ['wt-one', 'wt-two']) {
const worktreePath = path.join(baseDir, name)
git(repoPath, 'worktree', 'add', worktreePath, '-b', name)
worktreePaths.push(worktreePath)
}
return { baseDir, repoPath, worktreePaths }
}
async function setupWorkspaces(page, fixture) {
return await runWithTimeout(
'fixture registration in Orca',
() =>
page.evaluate(
async ({ repoPath, importedWorktreePaths }) => {
const store = window.__store
if (!store) {
throw new Error('window.__store is unavailable.')
}
await store.getState().fetchSettings?.()
const addResult = await window.api.repos.add({ path: repoPath, kind: 'git' })
if ('error' in addResult) {
throw new Error(addResult.error)
}
await store.getState().fetchRepos()
const state = store.getState()
const repo = state.repos.find((c) => c.path === repoPath) ?? addResult.repo
await state.updateRepo(repo.id, {
externalWorktreeVisibility: 'show',
externalWorktreeVisibilityPromptDismissedAt: Date.now(),
importedExternalWorktreePaths: importedWorktreePaths,
externalWorktreeInboxBaselinePaths: importedWorktreePaths
})
await store.getState().fetchWorktrees(repo.id, { requireAuthoritative: true })
const nextState = store.getState()
nextState.setSidebarOpen(true)
nextState.setGroupBy('none')
nextState.setSortBy('recent')
nextState.setShowActiveOnly(false)
nextState.setActiveView('terminal')
const worktrees = nextState.worktreesByRepo[repo.id] ?? []
return {
repoId: repo.id,
worktrees: worktrees.map((w) => ({
id: w.id,
path: w.path,
displayName: w.displayName,
isMainWorktree: w.isMainWorktree
}))
}
},
{ repoPath: fixture.repoPath, importedWorktreePaths: fixture.worktreePaths }
),
setupTimeoutMs
)
}
async function clickWorktreeCard(page, worktreeId) {
const rect = await runWithTimeout(
`locate worktree card ${worktreeId}`,
() =>
page.evaluate((id) => {
const rows = Array.from(document.querySelectorAll('[data-worktree-id]'))
const row = rows.find((c) => c.getAttribute('data-worktree-id') === id)
if (!row) {
return null
}
row.scrollIntoView({ block: 'center', inline: 'nearest' })
const surface = row.querySelector('[data-worktree-card-surface="true"]') ?? row
const bounds = surface.getBoundingClientRect()
if (bounds.width <= 0 || bounds.height <= 0) {
return null
}
return { x: bounds.left + bounds.width / 2, y: bounds.top + bounds.height / 2 }
}, worktreeId),
rendererActionTimeoutMs
)
if (!rect) {
throw new Error(`Could not find rendered worktree card for ${worktreeId}`)
}
await runWithTimeout(
`click worktree card ${worktreeId}`,
() => page.mouse.click(rect.x, rect.y),
rendererActionTimeoutMs
)
}
async function activateWorktreeTerminal(page, worktreeId) {
await clickWorktreeCard(page, worktreeId)
await pollUntil(
`active worktree ${worktreeId}`,
() =>
page.evaluate((id) => {
const state = window.__store?.getState?.()
return state?.activeWorktreeId === id && state.activeTabType === 'terminal'
}, worktreeId),
Boolean,
30_000
)
}
/** Wait until the active terminal pane of `worktreeId` has a bound ptyId — i.e.
* its terminal actually exists and is warm before we start hiding it. */
async function waitForBoundTerminal(page, worktreeId) {
return await pollUntil(
`pty bound ${worktreeId}`,
() =>
page.evaluate((id) => {
const state = window.__store?.getState?.()
const tabId =
state?.activeWorktreeId === id && state.activeTabType === 'terminal'
? state.activeTabId
: (state?.activeTabIdByWorktree?.[id] ?? null)
const manager = tabId ? window.__paneManagers?.get(tabId) : null
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()?.[0] ?? null
return pane?.container?.dataset?.ptyId ?? null
}, worktreeId),
Boolean,
30_000,
10
)
}
/** Fill the active pane's terminal with ~lineCount lines of output so the
* parked snapshot (and thus the cold reveal replay) reflects a busy TUI-sized
* buffer rather than an empty shell. Writes to the pty and waits for the
* xterm buffer to grow. Returns the observed serialized length. */
async function primeScrollback(page, worktreeId, lineCount) {
const ptyId = await page.evaluate((id) => {
const state = window.__store?.getState?.()
const tabId =
state?.activeWorktreeId === id && state.activeTabType === 'terminal'
? state.activeTabId
: (state?.activeTabIdByWorktree?.[id] ?? null)
const manager = tabId ? window.__paneManagers?.get(tabId) : null
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()?.[0] ?? null
return pane?.container?.dataset?.ptyId ?? null
}, worktreeId)
if (!ptyId) {
throw new Error(`No pty to prime for ${worktreeId}`)
}
// A single command that emits many numbered 80-col lines. seq is POSIX-ish;
// fall back handled by awk for portability.
const cmd = `awk 'BEGIN{for(i=0;i<${lineCount};i++)printf "%04d %s\\n", i, "cold-park-scrollback-priming-line-padding-to-eighty-cols-000000"}'\n`
await page.evaluate(({ id, c }) => window.api.pty.write(id, c), { id: ptyId, c: cmd })
return await pollUntil(
`scrollback primed ${worktreeId}`,
() =>
page.evaluate((id) => {
const state = window.__store?.getState?.()
const tabId =
state?.activeWorktreeId === id && state.activeTabType === 'terminal'
? state.activeTabId
: (state?.activeTabIdByWorktree?.[id] ?? null)
const manager = tabId ? window.__paneManagers?.get(tabId) : null
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()?.[0] ?? null
const content = pane?.serializeAddon?.serialize?.() ?? ''
return content.length
}, worktreeId),
(len) => Number.isFinite(len) && len > lineCount * 40,
20_000,
100
).catch(() => 0)
}
function activeTabIdFor(page, worktreeId) {
return page.evaluate((id) => {
const state = window.__store?.getState?.()
return (
(state?.activeWorktreeId === id && state.activeTabType === 'terminal'
? state.activeTabId
: (state?.activeTabIdByWorktree?.[id] ?? null)) ?? null
)
}, worktreeId)
}
/** Times a reveal of `targetId` (must currently be hidden). Returns the three
* phase timings measured from the switch-back click. */
async function timeReveal(page, targetId) {
const t0 = Date.now()
await clickWorktreeCard(page, targetId)
await pollUntil(
`reveal activation ${targetId}`,
() =>
page.evaluate((id) => {
const state = window.__store?.getState?.()
return state?.activeWorktreeId === id && state.activeTabType === 'terminal'
}, targetId),
Boolean,
30_000,
5
)
const activationMs = Date.now() - t0
await pollUntil(
`reveal pty ${targetId}`,
() =>
page.evaluate((id) => {
const state = window.__store?.getState?.()
const tabId =
state?.activeWorktreeId === id && state.activeTabType === 'terminal'
? state.activeTabId
: (state?.activeTabIdByWorktree?.[id] ?? null)
const manager = tabId ? window.__paneManagers?.get(tabId) : null
const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()?.[0] ?? null
return pane?.container?.isConnected ? (pane?.container?.dataset?.ptyId ?? null) : null
}, targetId),
Boolean,
30_000,
5
)
const ptyBindMs = Date.now() - t0
await runWithTimeout(
'reveal paint settle',
() =>
page.evaluate(
() => new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r)))
),
rendererActionTimeoutMs
)
const paintSettleMs = Date.now() - t0
return { activationMs, ptyBindMs, paintSettleMs }
}
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
/** Parking signal that does not depend on the e2e debug handle: a parked tab's
* TerminalPane unmounts, so its manager leaves window.__paneManagers AND the
* overlay slot renders null (no [data-tab-id] container). Prefer the debug
* handle when present, fall back to manager/DOM absence. */
function isTabParked(page, tabId) {
return page.evaluate((id) => {
const handleIds = window.__terminalParkingDebug?.parkedTabIds?.()
if (Array.isArray(handleIds)) {
return { source: 'handle', parked: handleIds.includes(id) }
}
// Fallback (no debug handle): a parked tab's manager leaves __paneManagers.
const managerGone = !(window.__paneManagers && window.__paneManagers.has(id))
return { source: 'manager', parked: managerGone }
}, tabId)
}
async function debugSnapshot(page, tabId, worktreeId) {
return page.evaluate(
({ id, wtId }) => {
const state = window.__store?.getState?.()
const tabsByWorktree = state?.tabsByWorktree ?? {}
const tab = Object.values(tabsByWorktree)
.flat()
.find((t) => t?.id === id)
const wtTabs = tabsByWorktree[wtId] ?? []
return {
handlePresent: typeof window.__terminalParkingDebug?.parkedTabIds === 'function',
parkDelayMs: window.__terminalParkingDebug?.parkDelayMs ?? null,
parkedHandleIds: window.__terminalParkingDebug?.parkedTabIds?.() ?? null,
managerPresent: Boolean(window.__paneManagers?.has(id)),
managerKeys: window.__paneManagers ? Array.from(window.__paneManagers.keys()) : null,
ptyId: tab?.ptyId ?? null,
worktreeTabCount: wtTabs.length,
worktreeTabPtyIds: wtTabs.map((t) => t?.ptyId ?? null),
activeTabIdByWorktree: state?.activeTabIdByWorktree?.[wtId] ?? null,
parkingEnabled: state?.settings?.terminalHiddenViewParking !== false
}
},
{ id: tabId, wtId: worktreeId }
)
}
/** One cold cycle: focus target, hide it (switch to primary), wait until the
* tab parks (debug handle or manager-absence), then time the reveal. */
async function runColdCycle(page, primaryId, targetId, wantDebug, scrollbackLines) {
await activateWorktreeTerminal(page, targetId)
await waitForBoundTerminal(page, targetId)
if (scrollbackLines > 0) {
await primeScrollback(page, targetId, scrollbackLines)
}
const targetTabId = await activeTabIdFor(page, targetId)
await activateWorktreeTerminal(page, primaryId)
const parked = await pollUntil(
`cold-park ${targetTabId}`,
() => isTabParked(page, targetTabId),
(result) => result?.parked === true,
COLD_PARK_CONFIRM_TIMEOUT_MS,
50
).then(
() => true,
() => false
)
const diag = wantDebug ? await debugSnapshot(page, targetTabId, targetId) : null
const timings = await timeReveal(page, targetId)
return { ...timings, parkedConfirmed: parked, diag }
}
/** One warm cycle: focus target, hide it briefly (under the park delay), reveal
* before it can park. Asserts it did NOT park. */
async function runWarmCycle(page, primaryId, targetId, scrollbackLines) {
await activateWorktreeTerminal(page, targetId)
await waitForBoundTerminal(page, targetId)
if (scrollbackLines > 0) {
await primeScrollback(page, targetId, scrollbackLines)
}
const targetTabId = await activeTabIdFor(page, targetId)
await activateWorktreeTerminal(page, primaryId)
await sleep(WARM_REVEAL_AFTER_MS)
const parked = await isTabParked(page, targetTabId)
const timings = await timeReveal(page, targetId)
return { ...timings, parkedConfirmed: parked?.parked === true }
}
function summarize(values) {
const sorted = values.filter(Number.isFinite).sort((a, b) => a - b)
if (!sorted.length) {
return null
}
const at = (f) => sorted[Math.min(sorted.length - 1, Math.round(f * (sorted.length - 1)))]
return {
count: sorted.length,
median: Math.round(at(0.5)),
p95: Math.round(at(0.95)),
max: sorted.at(-1)
}
}
function summarizeArm(name, samples) {
const fields = {}
for (const key of ['activationMs', 'ptyBindMs', 'paintSettleMs']) {
fields[key] = summarize(samples.map((s) => s[key]))
}
return {
name,
samples: samples.length,
parkedConfirmed: samples.filter((s) => s.parkedConfirmed).length,
fields
}
}
async function main() {
const args = parseArgs()
const startedAt = Date.now()
const report = {
label: args.label,
startedAt: new Date(startedAt).toISOString(),
platform: `${process.platform} ${os.release()}`,
parkDelayMs: PARK_DELAY_MS,
args: { cycles: args.cycles, scrollbackLines: args.scrollbackLines },
arms: {},
summaries: [],
cleanupErrors: []
}
let fixture = null
let userDataDir = null
let launched = null
let browser = null
let page = null
try {
fixture = createLocalRepoFixture()
const cdpPort = await pickFreePort()
userDataDir = createShortUserDataDirectory()
// Shrink the 30s cold-park hysteresis + 5-min hot-retain so parking is
// observable in a benchmark run. Inherited by launchDevApp into the app.
process.env.ORCA_E2E_TERMINAL_PARKING_DELAY_MS = String(PARK_DELAY_MS)
console.log(
`[cold-park] fixture=${fixture.baseDir} userData=${userDataDir} cdp=${cdpPort} parkDelay=${PARK_DELAY_MS}ms`
)
launched = launchDevApp({ cdpPort, userDataDir })
const connected = await connectToApp(cdpPort)
browser = connected.browser
page = connected.page
await waitForStoreReady(page)
await installRendererProbe(page)
const setup = await setupWorkspaces(page, fixture)
let worktrees = setup.worktrees
// Why: worktree registration/fetch can lag the setup evaluate return; poll
// the store until the two external worktrees materialize before asserting.
if (worktrees.length < 2) {
// Why: the initial fetchWorktrees can return before the external-worktree
// scan surfaces the imported paths. Re-drive the fetch each poll (not just
// re-read the store) so a missed first scan self-heals.
worktrees = await pollUntil(
'worktrees registered',
() =>
page.evaluate(async (repoId) => {
await window.__store
?.getState?.()
?.fetchWorktrees?.(repoId, { requireAuthoritative: true })
.catch(() => undefined)
const state = window.__store?.getState?.()
const list = state?.worktreesByRepo?.[repoId] ?? []
return list.map((w) => ({
id: w.id,
path: w.path,
displayName: w.displayName,
isMainWorktree: w.isMainWorktree
}))
}, setup.repoId),
(list) => Array.isArray(list) && list.length >= 2,
30_000,
500
)
}
if (worktrees.length < 2) {
throw new Error(`Expected >=2 worktrees, got ${worktrees.length}`)
}
const primary = worktrees.find((w) => w.isMainWorktree) ?? worktrees[0]
const target = worktrees.find((w) => w.id !== primary.id)
await activateWorktreeTerminal(page, primary.id)
await waitForBoundTerminal(page, primary.id)
// Confirm the parking debug handle is present (exposeStore gate). It
// registers on first pane mount, so poll rather than read once.
const debugReady = await pollUntil(
'parking debug handle',
() => page.evaluate(() => typeof window.__terminalParkingDebug?.parkedTabIds === 'function'),
Boolean,
10_000,
100
).then(
() => true,
() => false
)
report.parkingDebugHandlePresent = debugReady
const cold = []
const warm = []
console.log(`[cold-park] running ${args.cycles} cold + ${args.cycles} warm cycles`)
for (let i = 0; i < args.cycles; i++) {
warm.push(
await runWithTimeout(
`warm cycle ${i}`,
() => runWarmCycle(page, primary.id, target.id, args.scrollbackLines),
scenarioTimeoutMs
)
)
const coldResult = await runWithTimeout(
`cold cycle ${i}`,
() => runColdCycle(page, primary.id, target.id, i === 0, args.scrollbackLines),
scenarioTimeoutMs
)
if (i === 0 && coldResult.diag) {
report.firstColdDiag = coldResult.diag
console.log(`[cold-park] first-cold diag: ${JSON.stringify(coldResult.diag)}`)
}
cold.push(coldResult)
}
report.arms.cold = cold
report.arms.warm = warm
report.summaries.push(summarizeArm('cold', cold))
report.summaries.push(summarizeArm('warm', warm))
report.finalDiagnostics = await collectRendererDiagnostics(page)
} finally {
report.elapsedMs = Date.now() - startedAt
if (browser) {
await browser.close().catch(() => undefined)
}
if (launched) {
try {
await stopDevApp(launched.child)
} catch (error) {
report.cleanupErrors.push(error instanceof Error ? error.message : String(error))
}
report.appLogsTail = launched.logs.slice(-60)
}
if (!args.keep) {
if (fixture) {
safeRemoveLocalDirectory(fixture.baseDir, report.cleanupErrors)
}
if (userDataDir) {
safeRemoveLocalDirectory(userDataDir, report.cleanupErrors)
}
}
const stamp = new Date(startedAt).toISOString().replace(/[:.]/g, '-')
const reportPath = path.resolve(
args.reportPath ??
path.join(
rootDir,
'tools',
'benchmarks',
'results',
`cold-park-${args.label}-${stamp}.json`
)
)
mkdirSync(path.dirname(reportPath), { recursive: true })
writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`)
console.log(`[cold-park] report=${reportPath}`)
const coldArm = report.summaries.find((s) => s.name === 'cold')
if (coldArm && coldArm.parkedConfirmed < coldArm.samples) {
console.log(
`[cold-park] WARNING: only ${coldArm.parkedConfirmed}/${coldArm.samples} cold reveals were confirmed parked — treat unconfirmed samples with caution`
)
}
for (const summary of report.summaries) {
console.log(
`[cold-park] ${summary.name}: samples=${summary.samples} parkedConfirmed=${summary.parkedConfirmed}/${summary.samples}`
)
for (const [field, stats] of Object.entries(summary.fields)) {
if (stats) {
console.log(
`[cold-park] ${field}: median=${stats.median}ms p95=${stats.p95}ms max=${stats.max}ms`
)
}
}
}
}
}
main().catch((error) => {
console.error(error instanceof Error ? error.stack || error.message : String(error))
process.exit(1)
})