test(wsl): drive hook-relay reattach bench through real PTY spawn (#13260)
* test(wsl): drive hook-relay reattach bench through real PTY spawn Follow-up to #13139: stop calling ensureWslHookRelayForReattach from the benchmark and instead reattach a surviving WSL PTY via main's registerPtyHandlers path, so a missing or misplaced integration in pty.ts fails the bench. * refactor(bench): isolate reattach relay refresh measurement and verify s - Wrap benchmark in try-finally for reliable cleanup - Add jiti module graph duplication detection to catch missing pty.ts integration - Track relay refreshes only during reattach phase to avoid false positives from earlier phases - Disable agent-status hooks during PTY spawn (reattach path doesn't gate on them) - Improve error messages and make cleanup safe with optional chaining
This commit is contained in:
Jinjing
GitHub
parent
b11354aaa7
commit
3e6b93f0d2
|
|
@ -0,0 +1,65 @@
|
|||
/**
|
||||
* Minimal `electron` module for wsl-hook-relay-reattach-benchmark.mjs, which loads the real
|
||||
* src/main/ipc/pty.ts under plain node. Only the surfaces that main's PTY graph touches are
|
||||
* implemented; everything else stays undefined so an unexpected dependency fails loudly.
|
||||
*/
|
||||
import { homedir, tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import process from 'node:process'
|
||||
|
||||
const userDataPath = process.env.ORCA_USER_DATA_PATH ?? tmpdir()
|
||||
|
||||
export const BrowserWindow = undefined
|
||||
|
||||
export const app = {
|
||||
isPackaged: false,
|
||||
getVersion: () => '0.0.0-wsl-relay-benchmark',
|
||||
getName: () => 'orca-wsl-relay-benchmark',
|
||||
getPath: (name) => {
|
||||
if (name === 'userData') {
|
||||
return userDataPath
|
||||
}
|
||||
if (name === 'temp') {
|
||||
return tmpdir()
|
||||
}
|
||||
if (name === 'home') {
|
||||
return homedir()
|
||||
}
|
||||
return join(userDataPath, name)
|
||||
},
|
||||
on: () => {},
|
||||
once: () => {},
|
||||
whenReady: () => Promise.resolve()
|
||||
}
|
||||
|
||||
export const ipcMain = {
|
||||
handle: () => {},
|
||||
handleOnce: () => {},
|
||||
on: () => {},
|
||||
once: () => {},
|
||||
removeHandler: () => {},
|
||||
removeAllListeners: () => {}
|
||||
}
|
||||
|
||||
export const powerMonitor = { on: () => {}, once: () => {} }
|
||||
export const nativeTheme = { shouldUseDarkColors: false, on: () => {} }
|
||||
export const shell = { openPath: async () => '', openExternal: async () => {} }
|
||||
export const dialog = { showMessageBox: async () => ({ response: 0 }) }
|
||||
export const safeStorage = { isEncryptionAvailable: () => false }
|
||||
export const clipboard = { readText: () => '', writeText: () => {} }
|
||||
export const screen = { getPrimaryDisplay: () => ({ workAreaSize: { width: 0, height: 0 } }) }
|
||||
export const session = { defaultSession: undefined }
|
||||
|
||||
export default {
|
||||
BrowserWindow,
|
||||
app,
|
||||
ipcMain,
|
||||
powerMonitor,
|
||||
nativeTheme,
|
||||
shell,
|
||||
dialog,
|
||||
safeStorage,
|
||||
clipboard,
|
||||
screen,
|
||||
session
|
||||
}
|
||||
|
|
@ -1,14 +1,18 @@
|
|||
import { spawn } from 'node:child_process'
|
||||
import { existsSync, readFileSync } from 'node:fs'
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import process from 'node:process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { createJiti } from 'jiti'
|
||||
|
||||
const DEFAULT_SAMPLES = 20
|
||||
const PANE_KEY = 'wsl-relay-bench:11111111-1111-4111-8111-111111111111'
|
||||
const WSL_PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
|
||||
const TERMINAL_COLS = 80
|
||||
const TERMINAL_ROWS = 24
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = { distro: null, samples: DEFAULT_SAMPLES }
|
||||
|
|
@ -203,6 +207,41 @@ async function invokeHook(distro, scriptPath, endpointPath) {
|
|||
return { status: result.status, elapsedMs: performance.now() - startedAt }
|
||||
}
|
||||
|
||||
/** Captures the PTY controller that registerPtyHandlers installs; every other runtime
|
||||
* call the spawn path makes is a no-op so main's real lifecycle ordering still runs. */
|
||||
function createRuntimeStub() {
|
||||
let controller = null
|
||||
const own = {
|
||||
setPtyController: (next) => {
|
||||
controller = next
|
||||
}
|
||||
}
|
||||
// Why no `has: () => true`: feature detection (`'x' in runtime`, typeof checks) must
|
||||
// not claim methods exist when they are only auto-stubbed getters.
|
||||
const runtime = new Proxy(own, {
|
||||
get: (target, property) => {
|
||||
if (property in target) {
|
||||
return target[property]
|
||||
}
|
||||
return typeof property === 'string' && property !== 'then' ? () => undefined : undefined
|
||||
}
|
||||
})
|
||||
return { runtime, getController: () => controller }
|
||||
}
|
||||
|
||||
function createRendererWindowStub() {
|
||||
const webContents = {
|
||||
id: 1,
|
||||
on: () => {},
|
||||
once: () => {},
|
||||
removeListener: () => {},
|
||||
send: () => {},
|
||||
isDestroyed: () => false,
|
||||
session: { on: () => {} }
|
||||
}
|
||||
return { webContents, on: () => {}, once: () => {}, isDestroyed: () => false }
|
||||
}
|
||||
|
||||
function percentile(samples, percentileValue) {
|
||||
const sorted = [...samples].sort((left, right) => left - right)
|
||||
return sorted[Math.ceil((percentileValue / 100) * sorted.length) - 1]
|
||||
|
|
@ -246,70 +285,145 @@ async function main() {
|
|||
})
|
||||
}
|
||||
|
||||
const jiti = createJiti(import.meta.url)
|
||||
const [{ WslHookRelayManager }, { ensureWslHookRelayForReattach }, { codexHookService }] =
|
||||
await Promise.all([
|
||||
jiti.import('../../src/main/agent-hooks/wsl-hook-relay-manager.ts'),
|
||||
jiti.import('../../src/main/agent-hooks/wsl-hook-relay-reattach.ts'),
|
||||
jiti.import('../../src/main/codex/hook-service.ts')
|
||||
])
|
||||
const { MANAGED_AGENT_HOOK_TARGETS } = await jiti.import(
|
||||
'../../src/shared/managed-agent-hook-targets.ts'
|
||||
)
|
||||
const guestHome = (
|
||||
await run('wsl.exe', wslArgs(distro, ['/bin/sh', '-c', 'printf %s "$HOME"']))
|
||||
).stdout.trim()
|
||||
const instanceKey = `bench-${process.pid}-${Date.now().toString(36)}`
|
||||
const benchmarkRoot = `${guestHome}/.orca-wsl/benchmarks/${instanceKey}`
|
||||
const scriptPath = `${benchmarkRoot}/.orca/agent-hooks/codex-hook.sh`
|
||||
const endpointPath = `${guestHome}/.orca-wsl/agent-hooks/instance-${instanceKey}/endpoint.env`
|
||||
const cleanupPaths = [benchmarkRoot, `${guestHome}/.orca-wsl/agent-hooks/instance-${instanceKey}`]
|
||||
if (cleanupPaths.some((cleanupPath) => !cleanupPath.startsWith(`${guestHome}/.orca-wsl/`))) {
|
||||
throw new Error('Refusing to use an unexpected guest cleanup path')
|
||||
}
|
||||
const disabledTuiAgents = MANAGED_AGENT_HOOK_TARGETS.filter(
|
||||
(target) => target.tuiAgent !== 'codex'
|
||||
).map((target) => target.tuiAgent)
|
||||
const bundleVersion = readFileSync(versionPath, 'utf8').trim()
|
||||
const warnings = []
|
||||
let delivered = 0
|
||||
// Why: main's PTY graph reads app paths at import time; keep it inside a disposable directory.
|
||||
let userDataDir
|
||||
let ptyIpc
|
||||
let cleanupPaths = []
|
||||
let manager = null
|
||||
let staller = null
|
||||
|
||||
const createManager = async (token) => {
|
||||
const preferredPort = await freeGuestPort(distro)
|
||||
return new WslHookRelayManager({
|
||||
platform: () => 'win32',
|
||||
remoteHooksEnabled: () => true,
|
||||
hookCoordsEnv: () => ({
|
||||
ORCA_AGENT_HOOK_PORT: String(preferredPort),
|
||||
ORCA_AGENT_HOOK_TOKEN: token,
|
||||
ORCA_AGENT_HOOK_ENV: 'benchmark',
|
||||
ORCA_AGENT_HOOK_VERSION: '1'
|
||||
}),
|
||||
instanceKey: () => instanceKey,
|
||||
resolveBundle: () => ({ jsPath: bundlePath, version: bundleVersion }),
|
||||
listDistros: async () => [distro],
|
||||
ingest: () => {
|
||||
delivered++
|
||||
},
|
||||
installHooks: async (sftp) => [
|
||||
await codexHookService.installRemote(sftp, benchmarkRoot, {
|
||||
codexHomeDir: `${benchmarkRoot}/codex-home`,
|
||||
deferTrustUntilConfigToml: true
|
||||
})
|
||||
],
|
||||
managedHookSettings: () => ({
|
||||
agentCmdOverrides: { codex: '/bin/true' },
|
||||
disabledTuiAgents
|
||||
}),
|
||||
pluginSources: () => ({}),
|
||||
warn: (message) => warnings.push(message),
|
||||
transientRetryDelayMs: 100
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
userDataDir = mkdtempSync(join(tmpdir(), 'orca-wsl-relay-bench-'))
|
||||
process.env.ORCA_USER_DATA_PATH = userDataDir
|
||||
const jiti = createJiti(import.meta.url, {
|
||||
alias: {
|
||||
electron: fileURLToPath(
|
||||
new URL('./wsl-hook-relay-reattach-benchmark-electron-stub.mjs', import.meta.url)
|
||||
)
|
||||
}
|
||||
})
|
||||
// Why sequential: concurrent jiti.import calls can each instantiate the module graph, and two
|
||||
// copies of wsl-hook-relay-manager.ts would leave pty.ts refreshing a singleton we never see.
|
||||
ptyIpc = await jiti.import('../../src/main/ipc/pty.ts')
|
||||
const { WslHookRelayManager, wslHookRelayManager } = await jiti.import(
|
||||
'../../src/main/agent-hooks/wsl-hook-relay-manager.ts'
|
||||
)
|
||||
const { ensureWslHookRelayForReattach } = await jiti.import(
|
||||
'../../src/main/agent-hooks/wsl-hook-relay-reattach.ts'
|
||||
)
|
||||
const { codexHookService } = await jiti.import('../../src/main/codex/hook-service.ts')
|
||||
const { MANAGED_AGENT_HOOK_TARGETS } = await jiti.import(
|
||||
'../../src/shared/managed-agent-hook-targets.ts'
|
||||
)
|
||||
const { toWindowsWslPath } = await jiti.import('../../src/shared/wsl-paths.ts')
|
||||
const { FLOATING_TERMINAL_WORKTREE_ID } = await jiti.import('../../src/shared/constants.ts')
|
||||
|
||||
// Why: if jiti ever returns a second manager module, our monkey-patch would miss the singleton
|
||||
// that ensureWslHookRelayForReattach (loaded via pty.ts) closes over — fail closed now.
|
||||
const singletonProbe = []
|
||||
const previousEnsure = wslHookRelayManager.ensureForDistro.bind(wslHookRelayManager)
|
||||
wslHookRelayManager.ensureForDistro = (probedDistro) => {
|
||||
singletonProbe.push(probedDistro)
|
||||
}
|
||||
ensureWslHookRelayForReattach(
|
||||
{ isReattach: true, wslDistro: '__bench-singleton-probe__' },
|
||||
null
|
||||
)
|
||||
wslHookRelayManager.ensureForDistro = previousEnsure
|
||||
if (singletonProbe.length !== 1 || singletonProbe[0] !== '__bench-singleton-probe__') {
|
||||
throw new Error(
|
||||
'jiti duplicated the WSL hook-relay manager graph; reattach patch would not observe pty.ts'
|
||||
)
|
||||
}
|
||||
|
||||
const guestHome = (
|
||||
await run('wsl.exe', wslArgs(distro, ['/bin/sh', '-c', 'printf %s "$HOME"']))
|
||||
).stdout.trim()
|
||||
const instanceKey = `bench-${process.pid}-${Date.now().toString(36)}`
|
||||
const benchmarkRoot = `${guestHome}/.orca-wsl/benchmarks/${instanceKey}`
|
||||
const scriptPath = `${benchmarkRoot}/.orca/agent-hooks/codex-hook.sh`
|
||||
const endpointPath = `${guestHome}/.orca-wsl/agent-hooks/instance-${instanceKey}/endpoint.env`
|
||||
cleanupPaths = [benchmarkRoot, `${guestHome}/.orca-wsl/agent-hooks/instance-${instanceKey}`]
|
||||
if (cleanupPaths.some((cleanupPath) => !cleanupPath.startsWith(`${guestHome}/.orca-wsl/`))) {
|
||||
throw new Error('Refusing to use an unexpected guest cleanup path')
|
||||
}
|
||||
const disabledTuiAgents = MANAGED_AGENT_HOOK_TARGETS.filter(
|
||||
(target) => target.tuiAgent !== 'codex'
|
||||
).map((target) => target.tuiAgent)
|
||||
const bundleVersion = readFileSync(versionPath, 'utf8').trim()
|
||||
const warnings = []
|
||||
const relayRefreshes = []
|
||||
let delivered = 0
|
||||
|
||||
// Why: pty.ts refreshes through the production singleton, so route that singleton at the
|
||||
// benchmark-scoped manager instead of calling the reattach helper from here — a removed or
|
||||
// mislocated integration call in pty.ts must fail this benchmark.
|
||||
wslHookRelayManager.ensureForDistro = (refreshedDistro) => {
|
||||
relayRefreshes.push(refreshedDistro)
|
||||
manager?.ensureForDistro(refreshedDistro)
|
||||
}
|
||||
|
||||
const { runtime, getController } = createRuntimeStub()
|
||||
// Why hooks off: fresh WSL spawn still runs buildPtyHostEnv, which calls ensureForDistro when
|
||||
// hooks are on. This bench isolates the reattach call site (ensureWslHookRelayForReattach),
|
||||
// which does not gate on agentStatusHooksEnabled.
|
||||
ptyIpc.registerPtyHandlers(
|
||||
createRendererWindowStub(),
|
||||
runtime,
|
||||
undefined,
|
||||
() => ({ agentStatusHooksEnabled: false }),
|
||||
undefined,
|
||||
undefined,
|
||||
{}
|
||||
)
|
||||
const ptyController = getController()
|
||||
if (typeof ptyController?.spawn !== 'function') {
|
||||
throw new Error('registerPtyHandlers did not install a runtime PTY controller')
|
||||
}
|
||||
const survivingSessionId = `orca-wsl-relay-bench-${instanceKey}`
|
||||
const spawnSurvivingPty = () =>
|
||||
ptyController.spawn({
|
||||
cols: TERMINAL_COLS,
|
||||
rows: TERMINAL_ROWS,
|
||||
// Why floating: without a worktree root main drops the requested cwd, and the UNC path is
|
||||
// what makes the local provider launch this PTY inside the distro under test.
|
||||
worktreeId: FLOATING_TERMINAL_WORKTREE_ID,
|
||||
cwd: toWindowsWslPath(guestHome, distro),
|
||||
sessionId: survivingSessionId
|
||||
})
|
||||
|
||||
const createManager = async (token) => {
|
||||
const preferredPort = await freeGuestPort(distro)
|
||||
return new WslHookRelayManager({
|
||||
platform: () => 'win32',
|
||||
remoteHooksEnabled: () => true,
|
||||
hookCoordsEnv: () => ({
|
||||
ORCA_AGENT_HOOK_PORT: String(preferredPort),
|
||||
ORCA_AGENT_HOOK_TOKEN: token,
|
||||
ORCA_AGENT_HOOK_ENV: 'benchmark',
|
||||
ORCA_AGENT_HOOK_VERSION: '1'
|
||||
}),
|
||||
instanceKey: () => instanceKey,
|
||||
resolveBundle: () => ({ jsPath: bundlePath, version: bundleVersion }),
|
||||
listDistros: async () => [distro],
|
||||
ingest: () => {
|
||||
delivered++
|
||||
},
|
||||
installHooks: async (sftp) => [
|
||||
await codexHookService.installRemote(sftp, benchmarkRoot, {
|
||||
codexHomeDir: `${benchmarkRoot}/codex-home`,
|
||||
deferTrustUntilConfigToml: true
|
||||
})
|
||||
],
|
||||
managedHookSettings: () => ({
|
||||
agentCmdOverrides: { codex: '/bin/true' },
|
||||
disabledTuiAgents
|
||||
}),
|
||||
pluginSources: () => ({}),
|
||||
warn: (message) => warnings.push(message),
|
||||
transientRetryDelayMs: 100
|
||||
})
|
||||
}
|
||||
|
||||
manager = await createManager('before-restart-token')
|
||||
manager.ensureForDistro(distro)
|
||||
await waitFor('initial relay and generated Codex hook', async () => {
|
||||
|
|
@ -319,6 +433,12 @@ async function main() {
|
|||
])
|
||||
return endpoint?.includes('before-restart-token') && script?.includes('--max-time')
|
||||
})
|
||||
|
||||
const survivingPty = await spawnSurvivingPty()
|
||||
if (relayRefreshes.length > 0) {
|
||||
throw new Error(`Fresh WSL spawn refreshed the relay: ${relayRefreshes.join(', ')}`)
|
||||
}
|
||||
|
||||
manager.disposeAll()
|
||||
manager = null
|
||||
|
||||
|
|
@ -336,9 +456,22 @@ async function main() {
|
|||
|
||||
delivered = 0
|
||||
manager = await createManager('after-restart-token')
|
||||
ensureWslHookRelayForReattach({ isReattach: true, wslDistro: distro }, null, (ownedDistro) =>
|
||||
manager.ensureForDistro(ownedDistro)
|
||||
)
|
||||
// Reattach the surviving WSL PTY through main's real spawn path — no direct helper call.
|
||||
// Why delta: only the reattach phase should record ensureForDistro; length alone can false-
|
||||
// green if an earlier phase already refreshed (or if a future hooks-on change reintroduces it).
|
||||
const refreshesBeforeReattach = relayRefreshes.length
|
||||
const reattachedPty = await spawnSurvivingPty()
|
||||
if (reattachedPty.id !== survivingPty.id) {
|
||||
throw new Error(
|
||||
`Expected to reattach PTY ${survivingPty.id}, main spawned ${reattachedPty.id}`
|
||||
)
|
||||
}
|
||||
const reattachRefreshes = relayRefreshes.slice(refreshesBeforeReattach)
|
||||
if (reattachRefreshes.length !== 1 || reattachRefreshes[0] !== distro) {
|
||||
throw new Error(
|
||||
`PTY reattach did not refresh the relay for '${distro}': before=${refreshesBeforeReattach} all=[${relayRefreshes.join(', ')}]`
|
||||
)
|
||||
}
|
||||
const refreshedEndpoint = await waitFor('reattached relay endpoint rewrite', async () => {
|
||||
const contents = await readGuestFile(distro, endpointPath)
|
||||
const parsed = contents ? parseEndpoint(contents) : null
|
||||
|
|
@ -362,6 +495,9 @@ async function main() {
|
|||
distro,
|
||||
endpointPath,
|
||||
endpointPortAfterReattach: refreshedEndpoint.port,
|
||||
reattachedPtyId: reattachedPty.id,
|
||||
relayRefreshes,
|
||||
reattachRefreshes,
|
||||
stale,
|
||||
refreshed,
|
||||
medianSpeedup: Number((stale.medianMs / refreshed.medianMs).toFixed(1)),
|
||||
|
|
@ -383,6 +519,10 @@ async function main() {
|
|||
} finally {
|
||||
manager?.disposeAll()
|
||||
await staller?.stop()
|
||||
ptyIpc?.killAllPty?.()
|
||||
if (userDataDir) {
|
||||
rmSync(userDataDir, { recursive: true, force: true })
|
||||
}
|
||||
for (const cleanupPath of cleanupPaths) {
|
||||
await run('wsl.exe', wslArgs(distro, ['/bin/rm', '-rf', '--', cleanupPath]), {
|
||||
allowFailure: true
|
||||
|
|
|
|||
Loading…
Reference in New Issue