From 3e6b93f0d2d6fb8b699481b126bff735560f1dee Mon Sep 17 00:00:00 2001 From: Jinjing <6427696+AmethystLiang@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:06:34 -0700 Subject: [PATCH] test(wsl): drive hook-relay reattach bench through real PTY spawn (#13260) * test(wsl): drive hook-relay reattach bench through real PTY spawn Follow-up to #13139: stop calling ensureWslHookRelayForReattach from the benchmark and instead reattach a surviving WSL PTY via main's registerPtyHandlers path, so a missing or misplaced integration in pty.ts fails the bench. * refactor(bench): isolate reattach relay refresh measurement and verify s - Wrap benchmark in try-finally for reliable cleanup - Add jiti module graph duplication detection to catch missing pty.ts integration - Track relay refreshes only during reattach phase to avoid false positives from earlier phases - Disable agent-status hooks during PTY spawn (reattach path doesn't gate on them) - Improve error messages and make cleanup safe with optional chaining --- ...relay-reattach-benchmark-electron-stub.mjs | 65 +++++ .../wsl-hook-relay-reattach-benchmark.mjs | 270 +++++++++++++----- 2 files changed, 270 insertions(+), 65 deletions(-) create mode 100644 config/scripts/wsl-hook-relay-reattach-benchmark-electron-stub.mjs diff --git a/config/scripts/wsl-hook-relay-reattach-benchmark-electron-stub.mjs b/config/scripts/wsl-hook-relay-reattach-benchmark-electron-stub.mjs new file mode 100644 index 000000000..1cd45f2ad --- /dev/null +++ b/config/scripts/wsl-hook-relay-reattach-benchmark-electron-stub.mjs @@ -0,0 +1,65 @@ +/** + * Minimal `electron` module for wsl-hook-relay-reattach-benchmark.mjs, which loads the real + * src/main/ipc/pty.ts under plain node. Only the surfaces that main's PTY graph touches are + * implemented; everything else stays undefined so an unexpected dependency fails loudly. + */ +import { homedir, tmpdir } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' + +const userDataPath = process.env.ORCA_USER_DATA_PATH ?? tmpdir() + +export const BrowserWindow = undefined + +export const app = { + isPackaged: false, + getVersion: () => '0.0.0-wsl-relay-benchmark', + getName: () => 'orca-wsl-relay-benchmark', + getPath: (name) => { + if (name === 'userData') { + return userDataPath + } + if (name === 'temp') { + return tmpdir() + } + if (name === 'home') { + return homedir() + } + return join(userDataPath, name) + }, + on: () => {}, + once: () => {}, + whenReady: () => Promise.resolve() +} + +export const ipcMain = { + handle: () => {}, + handleOnce: () => {}, + on: () => {}, + once: () => {}, + removeHandler: () => {}, + removeAllListeners: () => {} +} + +export const powerMonitor = { on: () => {}, once: () => {} } +export const nativeTheme = { shouldUseDarkColors: false, on: () => {} } +export const shell = { openPath: async () => '', openExternal: async () => {} } +export const dialog = { showMessageBox: async () => ({ response: 0 }) } +export const safeStorage = { isEncryptionAvailable: () => false } +export const clipboard = { readText: () => '', writeText: () => {} } +export const screen = { getPrimaryDisplay: () => ({ workAreaSize: { width: 0, height: 0 } }) } +export const session = { defaultSession: undefined } + +export default { + BrowserWindow, + app, + ipcMain, + powerMonitor, + nativeTheme, + shell, + dialog, + safeStorage, + clipboard, + screen, + session +} diff --git a/config/scripts/wsl-hook-relay-reattach-benchmark.mjs b/config/scripts/wsl-hook-relay-reattach-benchmark.mjs index 07ca9b3d6..7595c539b 100644 --- a/config/scripts/wsl-hook-relay-reattach-benchmark.mjs +++ b/config/scripts/wsl-hook-relay-reattach-benchmark.mjs @@ -1,14 +1,18 @@ import { spawn } from 'node:child_process' -import { existsSync, readFileSync } from 'node:fs' +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' import { join } from 'node:path' import { performance } from 'node:perf_hooks' import process from 'node:process' +import { fileURLToPath } from 'node:url' import { createJiti } from 'jiti' const DEFAULT_SAMPLES = 20 const PANE_KEY = 'wsl-relay-bench:11111111-1111-4111-8111-111111111111' const WSL_PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +const TERMINAL_COLS = 80 +const TERMINAL_ROWS = 24 function parseArgs(argv) { const options = { distro: null, samples: DEFAULT_SAMPLES } @@ -203,6 +207,41 @@ async function invokeHook(distro, scriptPath, endpointPath) { return { status: result.status, elapsedMs: performance.now() - startedAt } } +/** Captures the PTY controller that registerPtyHandlers installs; every other runtime + * call the spawn path makes is a no-op so main's real lifecycle ordering still runs. */ +function createRuntimeStub() { + let controller = null + const own = { + setPtyController: (next) => { + controller = next + } + } + // Why no `has: () => true`: feature detection (`'x' in runtime`, typeof checks) must + // not claim methods exist when they are only auto-stubbed getters. + const runtime = new Proxy(own, { + get: (target, property) => { + if (property in target) { + return target[property] + } + return typeof property === 'string' && property !== 'then' ? () => undefined : undefined + } + }) + return { runtime, getController: () => controller } +} + +function createRendererWindowStub() { + const webContents = { + id: 1, + on: () => {}, + once: () => {}, + removeListener: () => {}, + send: () => {}, + isDestroyed: () => false, + session: { on: () => {} } + } + return { webContents, on: () => {}, once: () => {}, isDestroyed: () => false } +} + function percentile(samples, percentileValue) { const sorted = [...samples].sort((left, right) => left - right) return sorted[Math.ceil((percentileValue / 100) * sorted.length) - 1] @@ -246,70 +285,145 @@ async function main() { }) } - const jiti = createJiti(import.meta.url) - const [{ WslHookRelayManager }, { ensureWslHookRelayForReattach }, { codexHookService }] = - await Promise.all([ - jiti.import('../../src/main/agent-hooks/wsl-hook-relay-manager.ts'), - jiti.import('../../src/main/agent-hooks/wsl-hook-relay-reattach.ts'), - jiti.import('../../src/main/codex/hook-service.ts') - ]) - const { MANAGED_AGENT_HOOK_TARGETS } = await jiti.import( - '../../src/shared/managed-agent-hook-targets.ts' - ) - const guestHome = ( - await run('wsl.exe', wslArgs(distro, ['/bin/sh', '-c', 'printf %s "$HOME"'])) - ).stdout.trim() - const instanceKey = `bench-${process.pid}-${Date.now().toString(36)}` - const benchmarkRoot = `${guestHome}/.orca-wsl/benchmarks/${instanceKey}` - const scriptPath = `${benchmarkRoot}/.orca/agent-hooks/codex-hook.sh` - const endpointPath = `${guestHome}/.orca-wsl/agent-hooks/instance-${instanceKey}/endpoint.env` - const cleanupPaths = [benchmarkRoot, `${guestHome}/.orca-wsl/agent-hooks/instance-${instanceKey}`] - if (cleanupPaths.some((cleanupPath) => !cleanupPath.startsWith(`${guestHome}/.orca-wsl/`))) { - throw new Error('Refusing to use an unexpected guest cleanup path') - } - const disabledTuiAgents = MANAGED_AGENT_HOOK_TARGETS.filter( - (target) => target.tuiAgent !== 'codex' - ).map((target) => target.tuiAgent) - const bundleVersion = readFileSync(versionPath, 'utf8').trim() - const warnings = [] - let delivered = 0 + // Why: main's PTY graph reads app paths at import time; keep it inside a disposable directory. + let userDataDir + let ptyIpc + let cleanupPaths = [] let manager = null let staller = null - - const createManager = async (token) => { - const preferredPort = await freeGuestPort(distro) - return new WslHookRelayManager({ - platform: () => 'win32', - remoteHooksEnabled: () => true, - hookCoordsEnv: () => ({ - ORCA_AGENT_HOOK_PORT: String(preferredPort), - ORCA_AGENT_HOOK_TOKEN: token, - ORCA_AGENT_HOOK_ENV: 'benchmark', - ORCA_AGENT_HOOK_VERSION: '1' - }), - instanceKey: () => instanceKey, - resolveBundle: () => ({ jsPath: bundlePath, version: bundleVersion }), - listDistros: async () => [distro], - ingest: () => { - delivered++ - }, - installHooks: async (sftp) => [ - await codexHookService.installRemote(sftp, benchmarkRoot, { - codexHomeDir: `${benchmarkRoot}/codex-home`, - deferTrustUntilConfigToml: true - }) - ], - managedHookSettings: () => ({ - agentCmdOverrides: { codex: '/bin/true' }, - disabledTuiAgents - }), - pluginSources: () => ({}), - warn: (message) => warnings.push(message), - transientRetryDelayMs: 100 - }) - } - try { + userDataDir = mkdtempSync(join(tmpdir(), 'orca-wsl-relay-bench-')) + process.env.ORCA_USER_DATA_PATH = userDataDir + const jiti = createJiti(import.meta.url, { + alias: { + electron: fileURLToPath( + new URL('./wsl-hook-relay-reattach-benchmark-electron-stub.mjs', import.meta.url) + ) + } + }) + // Why sequential: concurrent jiti.import calls can each instantiate the module graph, and two + // copies of wsl-hook-relay-manager.ts would leave pty.ts refreshing a singleton we never see. + ptyIpc = await jiti.import('../../src/main/ipc/pty.ts') + const { WslHookRelayManager, wslHookRelayManager } = await jiti.import( + '../../src/main/agent-hooks/wsl-hook-relay-manager.ts' + ) + const { ensureWslHookRelayForReattach } = await jiti.import( + '../../src/main/agent-hooks/wsl-hook-relay-reattach.ts' + ) + const { codexHookService } = await jiti.import('../../src/main/codex/hook-service.ts') + const { MANAGED_AGENT_HOOK_TARGETS } = await jiti.import( + '../../src/shared/managed-agent-hook-targets.ts' + ) + const { toWindowsWslPath } = await jiti.import('../../src/shared/wsl-paths.ts') + const { FLOATING_TERMINAL_WORKTREE_ID } = await jiti.import('../../src/shared/constants.ts') + + // Why: if jiti ever returns a second manager module, our monkey-patch would miss the singleton + // that ensureWslHookRelayForReattach (loaded via pty.ts) closes over — fail closed now. + const singletonProbe = [] + const previousEnsure = wslHookRelayManager.ensureForDistro.bind(wslHookRelayManager) + wslHookRelayManager.ensureForDistro = (probedDistro) => { + singletonProbe.push(probedDistro) + } + ensureWslHookRelayForReattach( + { isReattach: true, wslDistro: '__bench-singleton-probe__' }, + null + ) + wslHookRelayManager.ensureForDistro = previousEnsure + if (singletonProbe.length !== 1 || singletonProbe[0] !== '__bench-singleton-probe__') { + throw new Error( + 'jiti duplicated the WSL hook-relay manager graph; reattach patch would not observe pty.ts' + ) + } + + const guestHome = ( + await run('wsl.exe', wslArgs(distro, ['/bin/sh', '-c', 'printf %s "$HOME"'])) + ).stdout.trim() + const instanceKey = `bench-${process.pid}-${Date.now().toString(36)}` + const benchmarkRoot = `${guestHome}/.orca-wsl/benchmarks/${instanceKey}` + const scriptPath = `${benchmarkRoot}/.orca/agent-hooks/codex-hook.sh` + const endpointPath = `${guestHome}/.orca-wsl/agent-hooks/instance-${instanceKey}/endpoint.env` + cleanupPaths = [benchmarkRoot, `${guestHome}/.orca-wsl/agent-hooks/instance-${instanceKey}`] + if (cleanupPaths.some((cleanupPath) => !cleanupPath.startsWith(`${guestHome}/.orca-wsl/`))) { + throw new Error('Refusing to use an unexpected guest cleanup path') + } + const disabledTuiAgents = MANAGED_AGENT_HOOK_TARGETS.filter( + (target) => target.tuiAgent !== 'codex' + ).map((target) => target.tuiAgent) + const bundleVersion = readFileSync(versionPath, 'utf8').trim() + const warnings = [] + const relayRefreshes = [] + let delivered = 0 + + // Why: pty.ts refreshes through the production singleton, so route that singleton at the + // benchmark-scoped manager instead of calling the reattach helper from here — a removed or + // mislocated integration call in pty.ts must fail this benchmark. + wslHookRelayManager.ensureForDistro = (refreshedDistro) => { + relayRefreshes.push(refreshedDistro) + manager?.ensureForDistro(refreshedDistro) + } + + const { runtime, getController } = createRuntimeStub() + // Why hooks off: fresh WSL spawn still runs buildPtyHostEnv, which calls ensureForDistro when + // hooks are on. This bench isolates the reattach call site (ensureWslHookRelayForReattach), + // which does not gate on agentStatusHooksEnabled. + ptyIpc.registerPtyHandlers( + createRendererWindowStub(), + runtime, + undefined, + () => ({ agentStatusHooksEnabled: false }), + undefined, + undefined, + {} + ) + const ptyController = getController() + if (typeof ptyController?.spawn !== 'function') { + throw new Error('registerPtyHandlers did not install a runtime PTY controller') + } + const survivingSessionId = `orca-wsl-relay-bench-${instanceKey}` + const spawnSurvivingPty = () => + ptyController.spawn({ + cols: TERMINAL_COLS, + rows: TERMINAL_ROWS, + // Why floating: without a worktree root main drops the requested cwd, and the UNC path is + // what makes the local provider launch this PTY inside the distro under test. + worktreeId: FLOATING_TERMINAL_WORKTREE_ID, + cwd: toWindowsWslPath(guestHome, distro), + sessionId: survivingSessionId + }) + + const createManager = async (token) => { + const preferredPort = await freeGuestPort(distro) + return new WslHookRelayManager({ + platform: () => 'win32', + remoteHooksEnabled: () => true, + hookCoordsEnv: () => ({ + ORCA_AGENT_HOOK_PORT: String(preferredPort), + ORCA_AGENT_HOOK_TOKEN: token, + ORCA_AGENT_HOOK_ENV: 'benchmark', + ORCA_AGENT_HOOK_VERSION: '1' + }), + instanceKey: () => instanceKey, + resolveBundle: () => ({ jsPath: bundlePath, version: bundleVersion }), + listDistros: async () => [distro], + ingest: () => { + delivered++ + }, + installHooks: async (sftp) => [ + await codexHookService.installRemote(sftp, benchmarkRoot, { + codexHomeDir: `${benchmarkRoot}/codex-home`, + deferTrustUntilConfigToml: true + }) + ], + managedHookSettings: () => ({ + agentCmdOverrides: { codex: '/bin/true' }, + disabledTuiAgents + }), + pluginSources: () => ({}), + warn: (message) => warnings.push(message), + transientRetryDelayMs: 100 + }) + } + manager = await createManager('before-restart-token') manager.ensureForDistro(distro) await waitFor('initial relay and generated Codex hook', async () => { @@ -319,6 +433,12 @@ async function main() { ]) return endpoint?.includes('before-restart-token') && script?.includes('--max-time') }) + + const survivingPty = await spawnSurvivingPty() + if (relayRefreshes.length > 0) { + throw new Error(`Fresh WSL spawn refreshed the relay: ${relayRefreshes.join(', ')}`) + } + manager.disposeAll() manager = null @@ -336,9 +456,22 @@ async function main() { delivered = 0 manager = await createManager('after-restart-token') - ensureWslHookRelayForReattach({ isReattach: true, wslDistro: distro }, null, (ownedDistro) => - manager.ensureForDistro(ownedDistro) - ) + // Reattach the surviving WSL PTY through main's real spawn path — no direct helper call. + // Why delta: only the reattach phase should record ensureForDistro; length alone can false- + // green if an earlier phase already refreshed (or if a future hooks-on change reintroduces it). + const refreshesBeforeReattach = relayRefreshes.length + const reattachedPty = await spawnSurvivingPty() + if (reattachedPty.id !== survivingPty.id) { + throw new Error( + `Expected to reattach PTY ${survivingPty.id}, main spawned ${reattachedPty.id}` + ) + } + const reattachRefreshes = relayRefreshes.slice(refreshesBeforeReattach) + if (reattachRefreshes.length !== 1 || reattachRefreshes[0] !== distro) { + throw new Error( + `PTY reattach did not refresh the relay for '${distro}': before=${refreshesBeforeReattach} all=[${relayRefreshes.join(', ')}]` + ) + } const refreshedEndpoint = await waitFor('reattached relay endpoint rewrite', async () => { const contents = await readGuestFile(distro, endpointPath) const parsed = contents ? parseEndpoint(contents) : null @@ -362,6 +495,9 @@ async function main() { distro, endpointPath, endpointPortAfterReattach: refreshedEndpoint.port, + reattachedPtyId: reattachedPty.id, + relayRefreshes, + reattachRefreshes, stale, refreshed, medianSpeedup: Number((stale.medianMs / refreshed.medianMs).toFixed(1)), @@ -383,6 +519,10 @@ async function main() { } finally { manager?.disposeAll() await staller?.stop() + ptyIpc?.killAllPty?.() + if (userDataDir) { + rmSync(userDataDir, { recursive: true, force: true }) + } for (const cleanupPath of cleanupPaths) { await run('wsl.exe', wslArgs(distro, ['/bin/rm', '-rf', '--', cleanupPath]), { allowFailure: true