diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 2f0e19980..261c8d18a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -273,6 +273,9 @@ jobs: - name: Smoke packaged CLI run: node config/scripts/smoke-packaged-cli.mjs --app-dir=dist/linux-unpacked + - name: Smoke packaged hang watchdog worker + run: xvfb-run --auto-servernum node config/scripts/smoke-packaged-hang-watchdog-worker.mjs --app-dir=dist/linux-unpacked + package_windows: name: package (windows) runs-on: windows-2022 diff --git a/build-plugins/plain-node-entry-guard.ts b/build-plugins/plain-node-entry-guard.ts index 6638cca97..a7ad171a4 100644 --- a/build-plugins/plain-node-entry-guard.ts +++ b/build-plugins/plain-node-entry-guard.ts @@ -21,7 +21,6 @@ type OutputChunk = Rollup.OutputChunk const PLAIN_NODE_ENTRY_NAMES = [ 'daemon-entry', 'parcel-watcher-process-entry', - 'main-thread-hang-watchdog-entry', 'computer-sidecar', 'agent-hooks/managed-agent-hook-controls', 'codex/codex-app-server-grant-entry' diff --git a/config/electron-builder.config.cjs b/config/electron-builder.config.cjs index e7a80704c..da0f690ad 100644 --- a/config/electron-builder.config.cjs +++ b/config/electron-builder.config.cjs @@ -154,7 +154,6 @@ module.exports = { 'out/main/plugin-host-entry.js', 'out/main/computer-sidecar.js', 'out/main/parcel-watcher-process-entry.js', - 'out/main/main-thread-hang-watchdog-entry.js', 'out/main/chunks/**', 'resources/**', 'node_modules/ws/**', diff --git a/config/scripts/electron-builder-config.test.mjs b/config/scripts/electron-builder-config.test.mjs index 7d761c70a..1dd0cf78f 100644 --- a/config/scripts/electron-builder-config.test.mjs +++ b/config/scripts/electron-builder-config.test.mjs @@ -171,11 +171,9 @@ describe('electron-builder config', () => { ) }) - // Why: the watchdog only arms in packaged builds, and its ELECTRON_RUN_AS_NODE - // fork resolves the entry from app.asar.unpacked — inside the asar it never runs. - it('unpacks the forked main-thread hang-watchdog entry', () => { - expect(electronBuilderConfig.asarUnpack).toEqual( - expect.arrayContaining(['out/main/main-thread-hang-watchdog-entry.js']) + it('keeps the worker-thread hang watchdog inside app.asar', () => { + expect(electronBuilderConfig.asarUnpack).not.toContain( + 'out/main/main-thread-hang-watchdog-entry.js' ) }) diff --git a/config/scripts/hang-watchdog-memory-benchmark.mjs b/config/scripts/hang-watchdog-memory-benchmark.mjs new file mode 100644 index 000000000..49a2dd631 --- /dev/null +++ b/config/scripts/hang-watchdog-memory-benchmark.mjs @@ -0,0 +1,443 @@ +#!/usr/bin/env node +import { execFileSync, fork, spawnSync } from 'node:child_process' +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { Worker } from 'node:worker_threads' +import { + childRssBytes, + median, + physicalFootprintBytes, + sampleMemory, + sampleProductionPerformance +} from './hang-watchdog-process-metrics.mjs' + +const INTERNAL_ENV = 'ORCA_HANG_WATCHDOG_BENCH_INTERNAL' +const BOUNDARY_ENV = 'ORCA_HANG_WATCHDOG_BENCH_BOUNDARY' +const RESULT_PREFIX = 'ORCA_HANG_WATCHDOG_BENCH_RESULT=' +const DEFAULT_TRIALS = 7 +const SETTLE_MS = 2_000 +const SAMPLE_COUNT = 5 +const SAMPLE_INTERVAL_MS = 200 +const VERIFY_TIMEOUT_MS = 500 +const VERIFY_CHECK_INTERVAL_MS = 50 +const VERIFY_BLOCK_MS = 1_200 +const PRODUCTION_HEARTBEAT_INTERVAL_MS = 2_000 +const PRODUCTION_TIMEOUT_MS = 45_000 +const PRODUCTION_CHECK_INTERVAL_MS = 5_000 +const PRODUCTION_SAMPLE_MS = 30_000 +const MAX_LAUNCH_ATTEMPTS = 3 +const MIB = 1024 * 1024 +const scriptPath = import.meta.filename +const repoRoot = path.resolve(import.meta.dirname, '..', '..') +const entryPath = path.join(repoRoot, 'out', 'main', 'main-thread-hang-watchdog-entry.js') + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function forceGc() { + if (typeof global.gc !== 'function') { + throw new Error('Electron did not expose GC; keep --js-flags=--expose-gc in the harness') + } + global.gc() + global.gc() +} + +function blockMainThread(ms) { + const startedAt = Date.now() + while (Date.now() - startedAt < ms) { + // Intentional synchronous stall. + } + return { startedAt, endedAt: Date.now() } +} + +function readMarker(markerPath) { + try { + return JSON.parse(readFileSync(markerPath, 'utf8')) + } catch { + return null + } +} + +async function verifyBlockedMainDetection(markerPath, sendHeartbeat) { + const block = blockMainThread(VERIFY_BLOCK_MS) + const detected = readMarker(markerPath) + sendHeartbeat() + const deadline = Date.now() + VERIFY_TIMEOUT_MS + let resolved + do { + resolved = readMarker(markerPath) + if (resolved?.selfRecovered === true) { + break + } + await sleep(VERIFY_CHECK_INTERVAL_MS) + } while (Date.now() < deadline) + const verified = + detected?.detectedAt >= block.startedAt && + detected.detectedAt <= block.endedAt && + detected.selfRecovered === false && + resolved?.selfRecovered === true + if (!verified) { + throw new Error( + `Built watchdog failed blocked-main verification: ${JSON.stringify({ detected, resolved })}` + ) + } + return true +} + +function startChild(markerPath, timeoutMs, checkIntervalMs) { + const startedAt = process.hrtime.bigint() + const child = fork(entryPath, [], { + stdio: ['ignore', 'ignore', 'ignore', 'ipc'], + env: { + ...process.env, + ELECTRON_RUN_AS_NODE: '1', + ORCA_HANG_WATCHDOG_PARENT_PID: String(process.pid), + ORCA_HANG_WATCHDOG_MARKER_PATH: markerPath, + ORCA_HANG_WATCHDOG_TIMEOUT_MS: String(timeoutMs), + ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS: String(checkIntervalMs) + } + }) + const startupMs = Number(process.hrtime.bigint() - startedAt) / 1e6 + return { + pids: [process.pid, child.pid], + startupMs, + sendHeartbeat: () => child.send?.({ type: 'heartbeat' }), + shutdown: async () => { + if (child.exitCode !== null) { + return + } + const exitPromise = new Promise((resolve) => child.once('exit', resolve)) + child.send?.({ type: 'shutdown' }) + if (child.connected) { + child.disconnect() + } + await exitPromise + } + } +} + +function startWorker(markerPath, timeoutMs, checkIntervalMs) { + const startedAt = process.hrtime.bigint() + const worker = new Worker(entryPath, { + workerData: { + parentPid: process.pid, + markerPath, + timeoutMs, + checkIntervalMs + } + }) + const startupMs = Number(process.hrtime.bigint() - startedAt) / 1e6 + return { + pids: [process.pid], + startupMs, + sendHeartbeat: () => worker.postMessage({ type: 'heartbeat' }), + shutdown: async () => { + if (worker.threadId === -1) { + return + } + const exitPromise = new Promise((resolve) => worker.once('exit', resolve)) + worker.postMessage({ type: 'shutdown' }) + await exitPromise + } + } +} + +async function verifyBoundary(markerPath, startBoundary) { + const boundary = startBoundary(markerPath, VERIFY_TIMEOUT_MS, VERIFY_CHECK_INTERVAL_MS) + const heartbeat = setInterval(boundary.sendHeartbeat, 100) + try { + await sleep(SETTLE_MS) + return await verifyBlockedMainDetection(markerPath, boundary.sendHeartbeat) + } finally { + clearInterval(heartbeat) + await boundary.shutdown() + } +} + +async function measureChild(markerPath) { + forceGc() + await sleep(SETTLE_MS) + forceGc() + const before = await sampleMemory( + () => process.memoryUsage().rss, + () => physicalFootprintBytes([process.pid]), + { sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep } + ) + const child = startChild(markerPath, PRODUCTION_TIMEOUT_MS, PRODUCTION_CHECK_INTERVAL_MS) + let measurements + try { + await sleep(SETTLE_MS) + forceGc() + const childRss = await sampleMemory( + () => childRssBytes(child.pids[1]), + () => physicalFootprintBytes([child.pids[1]]), + { sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep } + ) + const total = await sampleMemory( + () => process.memoryUsage().rss + childRssBytes(child.pids[1]), + () => physicalFootprintBytes(child.pids), + { sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep } + ) + const performance = await sampleProductionPerformance(child, { + heartbeatIntervalMs: PRODUCTION_HEARTBEAT_INTERVAL_MS, + sampleMs: PRODUCTION_SAMPLE_MS, + sleep + }) + measurements = { + rssBytes: childRss.rssBytes, + summedProcessRssDeltaBytes: Math.max(0, total.rssBytes - before.rssBytes), + physicalFootprintDeltaBytes: Math.max( + 0, + total.physicalFootprintBytes - before.physicalFootprintBytes + ), + startupMs: child.startupMs, + ...performance + } + } finally { + await child.shutdown() + } + rmSync(markerPath, { force: true }) + return { + ...measurements, + blockedMainThreadVerified: await verifyBoundary(markerPath, startChild) + } +} + +async function measureWorker(markerPath) { + forceGc() + await sleep(SETTLE_MS) + forceGc() + const before = await sampleMemory( + () => process.memoryUsage().rss, + () => physicalFootprintBytes([process.pid]), + { sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep } + ) + const worker = startWorker(markerPath, PRODUCTION_TIMEOUT_MS, PRODUCTION_CHECK_INTERVAL_MS) + let measurements + try { + await sleep(SETTLE_MS) + forceGc() + const after = await sampleMemory( + () => process.memoryUsage().rss, + () => physicalFootprintBytes([process.pid]), + { sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep } + ) + const performance = await sampleProductionPerformance(worker, { + heartbeatIntervalMs: PRODUCTION_HEARTBEAT_INTERVAL_MS, + sampleMs: PRODUCTION_SAMPLE_MS, + sleep + }) + const rssBytes = Math.max(0, after.rssBytes - before.rssBytes) + measurements = { + rssBytes, + summedProcessRssDeltaBytes: rssBytes, + physicalFootprintDeltaBytes: Math.max( + 0, + after.physicalFootprintBytes - before.physicalFootprintBytes + ), + startupMs: worker.startupMs, + ...performance + } + } finally { + await worker.shutdown() + } + rmSync(markerPath, { force: true }) + return { + ...measurements, + blockedMainThreadVerified: await verifyBoundary(markerPath, startWorker) + } +} + +async function runInternal() { + if (process.platform !== 'darwin') { + throw new Error('The production watchdog is macOS-only; run this benchmark on macOS') + } + const { app } = await import('electron') + const boundary = process.env[BOUNDARY_ENV] + const profileDir = mkdtempSync(path.join(tmpdir(), 'orca-watchdog-bench-')) + app.setPath('userData', profileDir) + try { + await app.whenReady() + const markerPath = path.join(profileDir, 'main-thread-hang.json') + const result = + boundary === 'child' + ? await measureChild(markerPath) + : boundary === 'worker' + ? await measureWorker(markerPath) + : (() => { + throw new Error(`Unsupported boundary: ${boundary}`) + })() + process.stdout.write(`${RESULT_PREFIX}${JSON.stringify(result)}\n`) + } finally { + app.quit() + rmSync(profileDir, { recursive: true, force: true }) + } +} + +function parseArgs(argv) { + const options = { boundary: '', trials: DEFAULT_TRIALS, output: '' } + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] + const value = argv[index + 1] + if (arg === '--boundary' || arg === '--trials' || arg === '--output') { + if (!value) { + throw new Error(`Missing value for ${arg}`) + } + options[arg.slice(2)] = arg === '--trials' ? Number(value) : value + index += 1 + } else { + throw new Error(`Unknown argument: ${arg}`) + } + } + if (!['child', 'worker'].includes(options.boundary)) { + throw new Error('--boundary must be child or worker') + } + if (!Number.isInteger(options.trials) || options.trials < 1) { + throw new Error('--trials must be a positive integer') + } + return options +} + +function electronPath() { + const requirePath = import.meta.resolve('electron') + const electronModulePath = fileURLToPath(requirePath) + return execFileSync( + process.execPath, + ['-e', `process.stdout.write(require(${JSON.stringify(electronModulePath)}))`], + { + encoding: 'utf8' + } + ) +} + +function runTrial(executable, boundary) { + for (let attempt = 1; attempt <= MAX_LAUNCH_ATTEMPTS; attempt += 1) { + const env = { ...process.env, [INTERNAL_ENV]: '1', [BOUNDARY_ENV]: boundary } + delete env.ELECTRON_RUN_AS_NODE + const launcherDir = mkdtempSync(path.join(tmpdir(), 'orca-watchdog-bench-launcher-')) + writeFileSync( + path.join(launcherDir, 'package.json'), + JSON.stringify({ name: 'orca-watchdog-benchmark', main: 'main.cjs' }) + ) + writeFileSync( + path.join(launcherDir, 'main.cjs'), + `import(${JSON.stringify(pathToFileURL(scriptPath).href)}).catch((error) => { + console.error(error) + process.exitCode = 1 +})\n` + ) + let result + try { + result = spawnSync(executable, ['--js-flags=--expose-gc', launcherDir], { + cwd: repoRoot, + env, + encoding: 'utf8', + timeout: 90_000 + }) + } finally { + rmSync(launcherDir, { recursive: true, force: true }) + } + if (result.status !== 0) { + throw new Error( + `Electron trial failed (${result.error?.message ?? result.signal ?? result.status}):\n` + + `${result.stderr || result.stdout}` + ) + } + const line = result.stdout.split('\n').find((candidate) => candidate.startsWith(RESULT_PREFIX)) + if (line) { + return { ...JSON.parse(line.slice(RESULT_PREFIX.length)), launchAttempts: attempt } + } + if (attempt === MAX_LAUNCH_ATTEMPTS || result.stderr || result.stdout) { + throw new Error(`Electron trial did not report a result (status ${result.status})`) + } + } + throw new Error('Electron trial exhausted launcher attempts') +} + +function runBenchmark() { + if (process.platform !== 'darwin') { + throw new Error('The production watchdog is macOS-only; run this benchmark on macOS') + } + if (!existsSync(entryPath)) { + throw new Error(`Missing ${entryPath}; run pnpm exec electron-vite build first`) + } + const options = parseArgs(process.argv.slice(2)) + const builtEntry = readFileSync(entryPath, 'utf8') + const hasChildContract = builtEntry.includes('ORCA_HANG_WATCHDOG_PARENT_PID') + const hasWorkerContract = builtEntry.includes('workerData') && builtEntry.includes('parentPort') + if ( + (options.boundary === 'child' && !hasChildContract) || + (options.boundary === 'worker' && !hasWorkerContract) + ) { + throw new Error( + `Built watchdog does not implement the requested ${options.boundary} boundary; rebuild the matching revision` + ) + } + const executable = electronPath() + const results = Array.from({ length: options.trials }, () => + runTrial(executable, options.boundary) + ) + const rssBytes = results.map((result) => result.rssBytes) + const summedProcessRssDeltaBytes = results.map((result) => result.summedProcessRssDeltaBytes) + const physicalFootprintDeltaBytes = results.map((result) => result.physicalFootprintDeltaBytes) + const startupMs = results.map((result) => result.startupMs) + const cpuMs = results.map((result) => result.cpuMs) + const eventLoopDelayP95Ms = results.map((result) => result.eventLoopDelayP95Ms) + const eventLoopDelayP99Ms = results.map((result) => result.eventLoopDelayP99Ms) + const eventLoopDelayMaxMs = results.map((result) => result.eventLoopDelayMaxMs) + const report = { + benchmark: 'hang-watchdog-memory', + boundary: options.boundary, + revision: execFileSync('git', ['rev-parse', 'HEAD'], { + cwd: repoRoot, + encoding: 'utf8' + }).trim(), + electron: execFileSync(executable, ['-e', 'process.stdout.write(process.versions.electron)'], { + env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }, + encoding: 'utf8' + }).trim(), + settleMs: SETTLE_MS, + samplesPerTrial: SAMPLE_COUNT, + productionHeartbeatIntervalMs: PRODUCTION_HEARTBEAT_INTERVAL_MS, + productionCheckIntervalMs: PRODUCTION_CHECK_INTERVAL_MS, + productionSampleMs: PRODUCTION_SAMPLE_MS, + trials: options.trials, + rssMiB: rssBytes.map((value) => Number((value / MIB).toFixed(2))), + medianRssMiB: Number((median(rssBytes) / MIB).toFixed(2)), + summedProcessRssDeltaMiB: summedProcessRssDeltaBytes.map((value) => + Number((value / MIB).toFixed(2)) + ), + medianSummedProcessRssDeltaMiB: Number((median(summedProcessRssDeltaBytes) / MIB).toFixed(2)), + physicalFootprintDeltaMiB: physicalFootprintDeltaBytes.map((value) => + Number((value / MIB).toFixed(2)) + ), + medianPhysicalFootprintDeltaMiB: Number((median(physicalFootprintDeltaBytes) / MIB).toFixed(2)), + startupMs: startupMs.map((value) => Number(value.toFixed(3))), + medianStartupMs: Number(median(startupMs).toFixed(3)), + cpuMs: cpuMs.map((value) => Number(value.toFixed(2))), + medianCpuMs: Number(median(cpuMs).toFixed(2)), + eventLoopDelayP95Ms: eventLoopDelayP95Ms.map((value) => Number(value.toFixed(3))), + medianEventLoopDelayP95Ms: Number(median(eventLoopDelayP95Ms).toFixed(3)), + eventLoopDelayP99Ms: eventLoopDelayP99Ms.map((value) => Number(value.toFixed(3))), + medianEventLoopDelayP99Ms: Number(median(eventLoopDelayP99Ms).toFixed(3)), + eventLoopDelayMaxMs: eventLoopDelayMaxMs.map((value) => Number(value.toFixed(3))), + medianEventLoopDelayMaxMs: Number(median(eventLoopDelayMaxMs).toFixed(3)), + heartbeatCounts: results.map((result) => result.heartbeatCount), + launchAttempts: results.map((result) => result.launchAttempts), + blockedMainThreadVerified: results.every((result) => result.blockedMainThreadVerified) + } + const serialized = `${JSON.stringify(report, null, 2)}\n` + process.stdout.write(serialized) + if (options.output) { + writeFileSync(path.resolve(options.output), serialized) + } +} + +if (process.env[INTERNAL_ENV] === '1') { + await runInternal() +} else { + runBenchmark() +} diff --git a/config/scripts/hang-watchdog-process-metrics.mjs b/config/scripts/hang-watchdog-process-metrics.mjs new file mode 100644 index 000000000..7e5ee4de5 --- /dev/null +++ b/config/scripts/hang-watchdog-process-metrics.mjs @@ -0,0 +1,108 @@ +import { execFileSync } from 'node:child_process' +import { monitorEventLoopDelay } from 'node:perf_hooks' + +export function median(values) { + const sorted = [...values].sort((left, right) => left - right) + const middle = Math.floor(sorted.length / 2) + return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle] +} + +export async function sampleMemory(readRss, readPhysicalFootprint, options) { + const rssSamples = [] + const physicalFootprintSamples = [] + for (let index = 0; index < options.sampleCount; index += 1) { + rssSamples.push(readRss()) + physicalFootprintSamples.push(readPhysicalFootprint()) + await options.sleep(options.sampleIntervalMs) + } + return { + rssBytes: median(rssSamples), + physicalFootprintBytes: median(physicalFootprintSamples) + } +} + +export function childRssBytes(pid) { + const raw = execFileSync('ps', ['-o', 'rss=', '-p', String(pid)], { + encoding: 'utf8' + }).trim() + const rssKiB = Number(raw) + if (!Number.isFinite(rssKiB) || rssKiB <= 0) { + throw new Error(`Could not read watchdog child RSS for PID ${pid}`) + } + return rssKiB * 1024 +} + +export function parsePhysicalFootprintBytes(output, processCount) { + const match = + processCount > 1 + ? output.match(/^Summary Footprint:\s+(\d+) B$/m) + : output.match(/^[^\s].*\sFootprint:\s+(\d+) B/m) + const bytes = Number(match?.[1]) + return Number.isFinite(bytes) && bytes > 0 ? bytes : null +} + +export function physicalFootprintBytes(pids) { + const pidArgs = pids.flatMap((pid) => ['--pid', String(pid)]) + const output = execFileSync( + '/usr/bin/footprint', + [...pidArgs, '--format', 'bytes', '--noCategories'], + { encoding: 'utf8' } + ) + const bytes = parsePhysicalFootprintBytes(output, pids.length) + if (bytes === null) { + throw new Error(`Could not read physical footprint for PIDs ${pids.join(', ')}`) + } + return bytes +} + +export function parseProcessCpuTimeMs(raw) { + if (!raw.trim()) { + return null + } + const parts = raw.split(':').map(Number) + if (!parts.length || parts.some((part) => !Number.isFinite(part))) { + return null + } + const seconds = parts.reduce((total, part) => total * 60 + part, 0) + const milliseconds = seconds * 1_000 + return milliseconds >= 0 ? milliseconds : null +} + +function processCpuTimeMs(pid) { + const raw = execFileSync('ps', ['-o', 'time=', '-p', String(pid)], { + encoding: 'utf8' + }).trim() + const milliseconds = parseProcessCpuTimeMs(raw) + if (milliseconds === null) { + throw new Error(`Could not read CPU time for PID ${pid}`) + } + return milliseconds +} + +function combinedCpuTimeMs(pids) { + return pids.reduce((total, pid) => total + processCpuTimeMs(pid), 0) +} + +export async function sampleProductionPerformance(boundary, options) { + const loopDelay = monitorEventLoopDelay({ resolution: 10 }) + let heartbeatCount = 0 + const heartbeat = setInterval(() => { + heartbeatCount += 1 + boundary.sendHeartbeat() + }, options.heartbeatIntervalMs) + const cpuBefore = combinedCpuTimeMs(boundary.pids) + loopDelay.enable() + try { + await options.sleep(options.sampleMs) + } finally { + loopDelay.disable() + clearInterval(heartbeat) + } + return { + cpuMs: Math.max(0, combinedCpuTimeMs(boundary.pids) - cpuBefore), + heartbeatCount, + eventLoopDelayP95Ms: loopDelay.percentile(95) / 1e6, + eventLoopDelayP99Ms: loopDelay.percentile(99) / 1e6, + eventLoopDelayMaxMs: loopDelay.max / 1e6 + } +} diff --git a/config/scripts/hang-watchdog-process-metrics.test.mjs b/config/scripts/hang-watchdog-process-metrics.test.mjs new file mode 100644 index 000000000..2d54e691d --- /dev/null +++ b/config/scripts/hang-watchdog-process-metrics.test.mjs @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { + parsePhysicalFootprintBytes, + parseProcessCpuTimeMs +} from './hang-watchdog-process-metrics.mjs' + +describe('hang watchdog process metrics', () => { + it('uses the de-duplicated summary for multiple processes', () => { + const output = ` +Electron [101]: 64-bit Footprint: 5000000 B (16384 bytes per page) + phys_footprint: 5100000 B +Electron Helper [102]: 64-bit Footprint: 2000000 B (16384 bytes per page) + phys_footprint: 2100000 B +Summary Footprint: 6259264 B +` + expect(parsePhysicalFootprintBytes(output, 2)).toBe(6_259_264) + }) + + it('uses the process footprint rather than auxiliary accounting for one process', () => { + const output = ` +Electron [101]: 64-bit Footprint: 5000000 B (16384 bytes per page) + phys_footprint: 5100000 B +` + expect(parsePhysicalFootprintBytes(output, 1)).toBe(5_000_000) + }) + + it('rejects missing or zero footprint summaries', () => { + expect(parsePhysicalFootprintBytes('phys_footprint: 100 B', 2)).toBeNull() + expect(parsePhysicalFootprintBytes('Summary Footprint: 0 B', 2)).toBeNull() + }) + + it.each([ + ['0:00.04', 40], + ['1:02.50', 62_500], + ['2:01:02.50', 7_262_500] + ])('parses ps CPU time %s', (value, expected) => { + expect(parseProcessCpuTimeMs(value)).toBe(expected) + }) + + it('rejects invalid CPU times', () => { + expect(parseProcessCpuTimeMs('')).toBeNull() + expect(parseProcessCpuTimeMs('not-a-time')).toBeNull() + expect(parseProcessCpuTimeMs('-1:00')).toBeNull() + }) +}) diff --git a/config/scripts/packaged-hang-watchdog-worker-contract.test.mjs b/config/scripts/packaged-hang-watchdog-worker-contract.test.mjs new file mode 100644 index 000000000..1ae558c76 --- /dev/null +++ b/config/scripts/packaged-hang-watchdog-worker-contract.test.mjs @@ -0,0 +1,35 @@ +import { readFileSync } from 'node:fs' +import { parse } from 'yaml' +import { describe, expect, it } from 'vitest' + +describe('packaged hang watchdog worker contract', () => { + it('boots the worker from app.asar in PR checks', () => { + const workflow = parse(readFileSync('.github/workflows/pr.yml', 'utf8')) + const smokeSource = readFileSync( + 'config/scripts/smoke-packaged-hang-watchdog-worker.mjs', + 'utf8' + ) + const smokeStep = workflow.jobs.package.steps.find( + (step) => step.name === 'Smoke packaged hang watchdog worker' + ) + + expect(smokeStep.run).toBe( + 'xvfb-run --auto-servernum node config/scripts/smoke-packaged-hang-watchdog-worker.mjs --app-dir=dist/linux-unpacked' + ) + expect(smokeSource).toContain( + "process.platform === 'linux' ? ['--no-sandbox', launcherDir] : [launcherDir]" + ) + }) + + // Why: Electron ignores process.exitCode, so the gate needs app.exit plus a stdout assertion. + it('fails the smoke when the packaged worker never reports success', () => { + const smokeSource = readFileSync( + 'config/scripts/smoke-packaged-hang-watchdog-worker.mjs', + 'utf8' + ) + + expect(smokeSource).toContain('app.exit(1)') + expect(smokeSource).not.toContain('app.quit()') + expect(smokeSource).toContain('if (!result.stdout.includes(SUCCESS_LINE))') + }) +}) diff --git a/config/scripts/smoke-packaged-hang-watchdog-worker.mjs b/config/scripts/smoke-packaged-hang-watchdog-worker.mjs new file mode 100644 index 000000000..ae9b34078 --- /dev/null +++ b/config/scripts/smoke-packaged-hang-watchdog-worker.mjs @@ -0,0 +1,179 @@ +import { spawnSync } from 'node:child_process' +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { Worker } from 'node:worker_threads' + +const INTERNAL_ENV = 'ORCA_PACKAGED_WATCHDOG_SMOKE_INTERNAL' +const ASAR_ENV = 'ORCA_PACKAGED_WATCHDOG_SMOKE_ASAR' +const TIMEOUT_MS = 100 +const CHECK_INTERVAL_MS = 20 +const POLL_TIMEOUT_MS = 5_000 +const SUCCESS_LINE = '[packaged-watchdog-smoke] app.asar worker detected and recovered a stall' + +function sleep(ms) { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)) +} + +function readAppDirArg(argv) { + const explicit = argv.find((arg) => arg.startsWith('--app-dir=')) + if (explicit) { + return explicit.slice('--app-dir='.length) + } + if (process.platform === 'darwin') { + return 'dist/mac-arm64/Orca.app' + } + if (process.platform === 'win32') { + return 'dist/win-unpacked' + } + return 'dist/linux-unpacked' +} + +function getResourcesDir(appDir) { + return process.platform === 'darwin' || appDir.endsWith('.app') + ? join(appDir, 'Contents', 'Resources') + : join(appDir, 'resources') +} + +function readMarker(markerPath) { + try { + return JSON.parse(readFileSync(markerPath, 'utf8')) + } catch { + return null + } +} + +async function waitForMarker(markerPath, predicate, workerError) { + const deadline = Date.now() + POLL_TIMEOUT_MS + while (Date.now() < deadline) { + if (workerError.current) { + throw workerError.current + } + const marker = readMarker(markerPath) + if (predicate(marker)) { + return marker + } + await sleep(CHECK_INTERVAL_MS) + } + throw new Error(`Timed out waiting for packaged watchdog marker at ${markerPath}`) +} + +async function waitForExit(worker, workerError) { + const exitCode = await Promise.race([ + new Promise((resolveExit) => worker.once('exit', resolveExit)), + sleep(POLL_TIMEOUT_MS).then(() => { + throw new Error('Timed out waiting for packaged watchdog worker to exit') + }) + ]) + if (workerError.current) { + throw workerError.current + } + if (exitCode !== 0) { + throw new Error(`Packaged watchdog worker exited with code ${exitCode}`) + } +} + +async function runInternal() { + const appAsar = process.env[ASAR_ENV] + if (!appAsar) { + throw new Error(`Missing ${ASAR_ENV}`) + } + const { app } = await import('electron') + const tempRoot = mkdtempSync(join(tmpdir(), 'orca-packaged-watchdog-smoke-')) + const markerPath = join(tempRoot, 'main-thread-hang.json') + const entryPath = join(appAsar, 'out', 'main', 'main-thread-hang-watchdog-entry.js') + let worker + try { + await app.whenReady() + if (!existsSync(entryPath)) { + throw new Error(`Packaged watchdog entry is missing from app.asar: ${entryPath}`) + } + const workerError = { current: null } + worker = new Worker(entryPath, { + workerData: { + parentPid: process.pid, + markerPath, + timeoutMs: TIMEOUT_MS, + checkIntervalMs: CHECK_INTERVAL_MS + } + }) + worker.once('error', (error) => { + workerError.current = error + }) + await waitForMarker(markerPath, (marker) => marker?.selfRecovered === false, workerError) + worker.postMessage({ type: 'heartbeat' }) + await waitForMarker(markerPath, (marker) => marker?.selfRecovered === true, workerError) + worker.postMessage({ type: 'shutdown' }) + await waitForExit(worker, workerError) + worker = undefined + console.log(SUCCESS_LINE) + } finally { + await worker?.terminate() + rmSync(tempRoot, { recursive: true, force: true }) + } +} + +function runSmoke() { + const appDir = resolve(readAppDirArg(process.argv.slice(2))) + const appAsar = join(getResourcesDir(appDir), 'app.asar') + if (!existsSync(appAsar)) { + throw new Error(`Packaged app archive is missing: ${appAsar}`) + } + const require = createRequire(import.meta.url) + const executable = require('electron') + const launcherDir = mkdtempSync(join(tmpdir(), 'orca-packaged-watchdog-launcher-')) + const launcherPath = join(launcherDir, 'main.cjs') + writeFileSync( + join(launcherDir, 'package.json'), + JSON.stringify({ name: 'orca-packaged-watchdog-smoke', main: 'main.cjs' }) + ) + writeFileSync( + launcherPath, + `import(${JSON.stringify(pathToFileURL(import.meta.filename).href)}).catch((error) => { + console.error(error) + process.exitCode = 1 +})\n` + ) + const env = { ...process.env, [INTERNAL_ENV]: '1', [ASAR_ENV]: appAsar, NODE_PATH: '' } + delete env.ELECTRON_RUN_AS_NODE + try { + const electronArgs = + process.platform === 'linux' ? ['--no-sandbox', launcherDir] : [launcherDir] + const result = spawnSync(executable, electronArgs, { + env, + encoding: 'utf8', + timeout: 15_000 + }) + if (result.status !== 0) { + throw new Error( + `Packaged watchdog smoke failed (${result.error?.message ?? result.signal ?? result.status}):\n` + + `${result.stderr || result.stdout}` + ) + } + // Why: Electron discards process.exitCode, so status 0 alone can't prove the worker ran. + if (!result.stdout.includes(SUCCESS_LINE)) { + throw new Error( + `Packaged watchdog smoke did not report success:\n${result.stderr || result.stdout}` + ) + } + process.stdout.write(result.stdout) + } finally { + rmSync(launcherDir, { recursive: true, force: true }) + } +} + +if (process.env[INTERNAL_ENV] === '1') { + // Why: a graceful quit exits 0 regardless of process.exitCode; only app.exit propagates failure. + const { app } = await import('electron') + try { + await runInternal() + app.exit(0) + } catch (error) { + console.error(error) + app.exit(1) + } +} else { + runSmoke() +} diff --git a/electron.vite.config.ts b/electron.vite.config.ts index 4a4cd6c8e..957e3f60b 100644 --- a/electron.vite.config.ts +++ b/electron.vite.config.ts @@ -243,9 +243,8 @@ export const electronViteConfig: UserConfig = { // Why: forked with ELECTRON_RUN_AS_NODE so @parcel/watcher faults // can't take down the main process (issue #7547). 'parcel-watcher-process-entry': resolve('src/main/ipc/parcel-watcher-process-entry.ts'), - // Why: forked with ELECTRON_RUN_AS_NODE so it survives a deadlocked - // main thread (macOS 26 AppKit scene-update deadlock) and can record - // the stall for the next launch to report. + // Why: a worker thread survives the macOS 26 AppKit main-thread deadlock + // without paying for another Electron process. 'main-thread-hang-watchdog-entry': resolve( 'src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts' ), diff --git a/package.json b/package.json index cacfbe341..3c37bc18e 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,7 @@ "bench:macos-computer-helper-owner-loss": "node config/scripts/macos-computer-helper-owner-loss-benchmark.mjs", "bench:startup": "pnpm run ensure:electron-runtime && node tools/benchmarks/startup-time-bench.mjs", "bench:daemon-coldstart": "pnpm run ensure:electron-runtime && node tools/benchmarks/daemon-coldstart-bench.mjs", + "bench:hang-watchdog-memory": "pnpm run ensure:electron-runtime && node config/scripts/hang-watchdog-memory-benchmark.mjs", "bench:main-thread-jank": "pnpm run ensure:electron-runtime && node tools/benchmarks/main-thread-jank-bench.mjs", "bench:worktree-deletion": "node tools/benchmarks/worktree-deletion-dev-bench.mjs", "bench:zustand-selector-fanout": "node config/scripts/zustand-selector-fanout-benchmark.mjs", diff --git a/src/main/hang-watchdog/hang-detection-marker.ts b/src/main/hang-watchdog/hang-detection-marker.ts index a6e6fbddd..a32d872b8 100644 --- a/src/main/hang-watchdog/hang-detection-marker.ts +++ b/src/main/hang-watchdog/hang-detection-marker.ts @@ -1,7 +1,7 @@ import { readFileSync, rmSync, writeFileSync } from 'node:fs' import { join } from 'node:path' -// Why: written by the plain-Node watchdog child when main-thread heartbeats stop, and rewritten if +// Why: written by the watchdog worker when main-thread heartbeats stop, and rewritten if // they resume; consumed on the next launch to report how long the stall lasted and whether it ever // cleared. `selfRecovered` separates a real deadlock from a long-but-survivable stall — the two are // indistinguishable at detection time, and only the latter would have been a destructive kill. diff --git a/src/main/hang-watchdog/hang-watchdog-child-loop.test.ts b/src/main/hang-watchdog/hang-watchdog-detection-loop.test.ts similarity index 96% rename from src/main/hang-watchdog/hang-watchdog-child-loop.test.ts rename to src/main/hang-watchdog/hang-watchdog-detection-loop.test.ts index 8d390c04e..8d1c3ebfc 100644 --- a/src/main/hang-watchdog/hang-watchdog-child-loop.test.ts +++ b/src/main/hang-watchdog/hang-watchdog-detection-loop.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest' -import { createHangWatchdogChildLoop } from './hang-watchdog-child-loop' +import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop' const TIMEOUT_MS = 45_000 const CHECK_INTERVAL_MS = 5_000 @@ -8,7 +8,7 @@ function loopWithClock(startAt = 0) { let now = startAt const onHangDetected = vi.fn() const onHangResolved = vi.fn() - const loop = createHangWatchdogChildLoop({ + const loop = createHangWatchdogDetectionLoop({ timeoutMs: TIMEOUT_MS, checkIntervalMs: CHECK_INTERVAL_MS, now: () => now, @@ -18,7 +18,7 @@ function loopWithClock(startAt = 0) { return { loop, onHangDetected, onHangResolved, advance: (ms: number) => (now += ms) } } -describe('createHangWatchdogChildLoop', () => { +describe('createHangWatchdogDetectionLoop', () => { it('does not fire while heartbeats keep arriving', () => { const { loop, onHangDetected, advance } = loopWithClock() for (let i = 0; i < 100; i++) { diff --git a/src/main/hang-watchdog/hang-watchdog-child-loop.ts b/src/main/hang-watchdog/hang-watchdog-detection-loop.ts similarity index 86% rename from src/main/hang-watchdog/hang-watchdog-child-loop.ts rename to src/main/hang-watchdog/hang-watchdog-detection-loop.ts index 3a8ebb34f..9d45cba77 100644 --- a/src/main/hang-watchdog/hang-watchdog-child-loop.ts +++ b/src/main/hang-watchdog/hang-watchdog-detection-loop.ts @@ -1,4 +1,4 @@ -export type HangWatchdogChildLoopConfig = { +export type HangWatchdogDetectionLoopConfig = { timeoutMs: number checkIntervalMs: number now: () => number @@ -7,14 +7,14 @@ export type HangWatchdogChildLoopConfig = { onHangResolved: (unresponsiveMs: number) => void } -export type HangWatchdogChildLoop = { +export type HangWatchdogDetectionLoop = { recordHeartbeat: () => void tick: () => void } -export function createHangWatchdogChildLoop( - config: HangWatchdogChildLoopConfig -): HangWatchdogChildLoop { +export function createHangWatchdogDetectionLoop( + config: HangWatchdogDetectionLoopConfig +): HangWatchdogDetectionLoop { let lastHeartbeatAt = config.now() let lastTickAt = config.now() let detected = false diff --git a/src/main/hang-watchdog/hang-watchdog-entry-path.ts b/src/main/hang-watchdog/hang-watchdog-entry-path.ts deleted file mode 100644 index 74f8b1d00..000000000 --- a/src/main/hang-watchdog/hang-watchdog-entry-path.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { existsSync } from 'node:fs' -import { join } from 'node:path' - -export function resolveHangWatchdogEntryPath( - appPath: string, - isPackaged: boolean, - pathExists: (candidate: string) => boolean = existsSync -): string { - // Why: ELECTRON_RUN_AS_NODE bypasses asar integration, so the packaged entry must be forked from app.asar.unpacked. - const basePath = isPackaged ? appPath.replace('app.asar', 'app.asar.unpacked') : appPath - const adjacentBuildEntry = join(basePath, 'main-thread-hang-watchdog-entry.js') - // Why: electron-vite's unpackaged appPath is already out/main; appending out/main again would silently disable the watchdog in dev and E2E builds. - if (!isPackaged && pathExists(adjacentBuildEntry)) { - return adjacentBuildEntry - } - return join(basePath, 'out', 'main', 'main-thread-hang-watchdog-entry.js') -} diff --git a/src/main/hang-watchdog/hang-watchdog-worker-path.test.ts b/src/main/hang-watchdog/hang-watchdog-worker-path.test.ts new file mode 100644 index 000000000..3ade5b1cb --- /dev/null +++ b/src/main/hang-watchdog/hang-watchdog-worker-path.test.ts @@ -0,0 +1,26 @@ +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path' + +describe('resolveHangWatchdogWorkerPath', () => { + it('resolves packaged workers inside app.asar', () => { + const appPath = join('/apps', 'orca', 'app.asar') + expect(resolveHangWatchdogWorkerPath(appPath, true)).toBe( + join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js') + ) + }) + + it('uses an adjacent entry when the dev app path is out/main', () => { + const appPath = join('/repo', 'out', 'main') + const adjacent = join(appPath, 'main-thread-hang-watchdog-entry.js') + const pathExists = vi.fn((candidate: string) => candidate === adjacent) + expect(resolveHangWatchdogWorkerPath(appPath, false, pathExists)).toBe(adjacent) + }) + + it('resolves through out/main from a dev project root', () => { + const appPath = join('/repo', 'orca') + expect(resolveHangWatchdogWorkerPath(appPath, false, () => false)).toBe( + join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js') + ) + }) +}) diff --git a/src/main/hang-watchdog/hang-watchdog-worker-path.ts b/src/main/hang-watchdog/hang-watchdog-worker-path.ts new file mode 100644 index 000000000..7efc300cb --- /dev/null +++ b/src/main/hang-watchdog/hang-watchdog-worker-path.ts @@ -0,0 +1,14 @@ +import { existsSync } from 'node:fs' +import { join } from 'node:path' + +export function resolveHangWatchdogWorkerPath( + appPath: string, + isPackaged: boolean, + pathExists: (candidate: string) => boolean = existsSync +): string { + const adjacentBuildEntry = join(appPath, 'main-thread-hang-watchdog-entry.js') + if (!isPackaged && pathExists(adjacentBuildEntry)) { + return adjacentBuildEntry + } + return join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js') +} diff --git a/src/main/hang-watchdog/hang-watchdog-worker-protocol.ts b/src/main/hang-watchdog/hang-watchdog-worker-protocol.ts new file mode 100644 index 000000000..14e6d0077 --- /dev/null +++ b/src/main/hang-watchdog/hang-watchdog-worker-protocol.ts @@ -0,0 +1,12 @@ +export const HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS = 2_000 +export const HANG_WATCHDOG_TIMEOUT_MS = 45_000 +export const HANG_WATCHDOG_CHECK_INTERVAL_MS = 5_000 + +export type HangWatchdogWorkerData = { + parentPid: number + markerPath: string + timeoutMs: number + checkIntervalMs: number +} + +export type MainToHangWatchdogWorkerMessage = { type: 'heartbeat' } | { type: 'shutdown' } diff --git a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.test.ts b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.test.ts index 0f36e8aca..a1ececdad 100644 --- a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.test.ts +++ b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.test.ts @@ -12,7 +12,11 @@ vi.mock('node:child_process', () => ({ spawn: spawnMock })) -import { recordHangObservation } from './main-thread-hang-watchdog-entry' +import { + isHangWatchdogWorkerData, + recordHangObservation, + runWatchdog +} from './main-thread-hang-watchdog-entry' describe('recordHangObservation', () => { let dir: string @@ -67,10 +71,8 @@ describe('recordHangObservation', () => { }) }) - // Why: this is the safety contract of the whole PR. Observing a hang must never kill, relaunch, - // or exit — a false positive would SIGKILL a live main thread mid-write. If a future change - // reintroduces recovery, this test must fail loudly rather than ship silently. it('never spawns, signals, or exits', () => { + // Why: a false positive must never kill a live main thread while writes may be in flight. recordHangObservation({ parentPid: 4242, markerPath: join(dir, 'marker.json'), @@ -104,3 +106,75 @@ describe('recordHangObservation', () => { ).not.toThrow() }) }) + +describe('watchdog worker entry', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(0) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('accepts only complete positive worker data', () => { + expect( + isHangWatchdogWorkerData({ + parentPid: 42, + markerPath: '/tmp/marker', + timeoutMs: 100, + checkIntervalMs: 25 + }) + ).toBe(true) + for (const value of [ + null, + {}, + { parentPid: 0, markerPath: '/tmp/marker', timeoutMs: 100, checkIntervalMs: 25 }, + { parentPid: 42, markerPath: 7, timeoutMs: 100, checkIntervalMs: 25 }, + { parentPid: 42, markerPath: '/tmp/marker', timeoutMs: 0, checkIntervalMs: 25 }, + { parentPid: 42, markerPath: '/tmp/marker', timeoutMs: 100, checkIntervalMs: Infinity } + ]) { + expect(isHangWatchdogWorkerData(value)).toBe(false) + } + }) + + it('routes heartbeats and shuts down its timer and port', () => { + const markerPath = join(tmpdir(), `hang-watchdog-entry-${process.pid}.json`) + let onMessage: ((message: { type: 'heartbeat' | 'shutdown' }) => void) | undefined + const port = { + on: vi.fn( + (_event: 'message', listener: (message: { type: 'heartbeat' | 'shutdown' }) => void) => { + onMessage = listener + } + ), + close: vi.fn() + } + try { + runWatchdog( + { + parentPid: process.pid, + markerPath, + timeoutMs: 100, + checkIntervalMs: 25 + }, + port + ) + vi.advanceTimersByTime(75) + onMessage?.({ type: 'heartbeat' }) + vi.advanceTimersByTime(75) + expect(consumeHangDetectionMarker(markerPath)).toBeNull() + vi.advanceTimersByTime(50) + expect(consumeHangDetectionMarker(markerPath)).toMatchObject({ selfRecovered: false }) + + onMessage?.({ type: 'heartbeat' }) + expect(consumeHangDetectionMarker(markerPath)).toMatchObject({ selfRecovered: true }) + onMessage?.({ type: 'shutdown' }) + expect(port.close).toHaveBeenCalledOnce() + vi.advanceTimersByTime(1_000) + expect(port.close).toHaveBeenCalledOnce() + expect(consumeHangDetectionMarker(markerPath)).toBeNull() + } finally { + rmSync(markerPath, { force: true }) + } + }) +}) diff --git a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts index 8d45c725a..2c60b17e9 100644 --- a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts +++ b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts @@ -1,20 +1,17 @@ -// Forked with ELECTRON_RUN_AS_NODE from the main process. Watches heartbeats from the main -// thread; if they stop (e.g. the macOS 26 AppKit scene-update deadlock) it records a marker so -// the next launch can report the stall, and rewrites it if the heartbeats come back. -// -// Why no kill: a deadlocked main thread has already lost whatever sat in the persistence debounce -// window, so killing it recovers nothing the user could not recover by force-quitting. A false -// positive, though, would SIGKILL a live main thread that was merely blocked — most plausibly on -// I/O, i.e. exactly when writes are in flight. All of the downside sits in the misfire, so this -// measures first: `selfRecovered` counts the stalls a killer would have gotten wrong. -// -// Must never import electron. -import { createHangWatchdogChildLoop } from './hang-watchdog-child-loop' +import { isMainThread, parentPort, workerData } from 'node:worker_threads' +import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop' import { writeHangDetectionMarker } from './hang-detection-marker' +import type { + HangWatchdogWorkerData, + MainToHangWatchdogWorkerMessage +} from './hang-watchdog-worker-protocol' -const DEFAULT_TIMEOUT_MS = 45_000 -const DEFAULT_CHECK_INTERVAL_MS = 5_000 +type HangWatchdogPort = { + on: (event: 'message', listener: (message: MainToHangWatchdogWorkerMessage) => void) => unknown + close: () => void +} +// Observation only: a false positive must never kill a live main thread mid-write. export function recordHangObservation(options: { parentPid: number markerPath: string @@ -36,37 +33,65 @@ export function recordHangObservation(options: { } } -function runWatchdog(parentPid: number): void { - const markerPath = process.env.ORCA_HANG_WATCHDOG_MARKER_PATH ?? '' - const timeoutMs = Number(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS - const checkIntervalMs = - Number(process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS) || DEFAULT_CHECK_INTERVAL_MS - - const loop = createHangWatchdogChildLoop({ - timeoutMs, - checkIntervalMs, +export function runWatchdog( + config: HangWatchdogWorkerData, + port: HangWatchdogPort | null = parentPort +): void { + if (!port) { + return + } + const loop = createHangWatchdogDetectionLoop({ + timeoutMs: config.timeoutMs, + checkIntervalMs: config.checkIntervalMs, now: () => Date.now(), onHangDetected: (unresponsiveMs) => - recordHangObservation({ parentPid, markerPath, unresponsiveMs, selfRecovered: false }), + recordHangObservation({ + parentPid: config.parentPid, + markerPath: config.markerPath, + unresponsiveMs, + selfRecovered: false + }), // Why: rewriting the marker keeps one observation per stall rather than two rows to reconcile. onHangResolved: (unresponsiveMs) => - recordHangObservation({ parentPid, markerPath, unresponsiveMs, selfRecovered: true }) + recordHangObservation({ + parentPid: config.parentPid, + markerPath: config.markerPath, + unresponsiveMs, + selfRecovered: true + }) }) - process.on('message', (message) => { - const type = (message as { type?: string } | null)?.type - if (type === 'heartbeat') { + let checkTimer: ReturnType | null = setInterval( + () => loop.tick(), + config.checkIntervalMs + ) + port.on('message', (message: MainToHangWatchdogWorkerMessage) => { + if (message.type === 'heartbeat') { loop.recordHeartbeat() - } else if (type === 'shutdown') { - process.exit(0) + } else if (message.type === 'shutdown') { + if (checkTimer) { + clearInterval(checkTimer) + checkTimer = null + } + port.close() } }) - // Why: a normal parent exit closes the IPC channel; the watchdog must not outlive it and misfire. - process.on('disconnect', () => process.exit(0)) - setInterval(() => loop.tick(), checkIntervalMs) } -const configuredParentPid = Number(process.env.ORCA_HANG_WATCHDOG_PARENT_PID) -if (Number.isInteger(configuredParentPid) && configuredParentPid > 0) { - runWatchdog(configuredParentPid) +export function isHangWatchdogWorkerData(value: unknown): value is HangWatchdogWorkerData { + const data = value as Partial | null + return ( + !!data && + Number.isInteger(data.parentPid) && + (data.parentPid ?? 0) > 0 && + typeof data.markerPath === 'string' && + Number.isFinite(data.timeoutMs) && + (data.timeoutMs ?? 0) > 0 && + Number.isFinite(data.checkIntervalMs) && + (data.checkIntervalMs ?? 0) > 0 + ) +} + +if (!isMainThread && isHangWatchdogWorkerData(workerData)) { + runWatchdog(workerData) } diff --git a/src/main/hang-watchdog/main-thread-hang-watchdog.test.ts b/src/main/hang-watchdog/main-thread-hang-watchdog.test.ts index b5c1063a9..eaaea5422 100644 --- a/src/main/hang-watchdog/main-thread-hang-watchdog.test.ts +++ b/src/main/hang-watchdog/main-thread-hang-watchdog.test.ts @@ -1,7 +1,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { join } from 'node:path' -const { forkMock, appMock } = vi.hoisted(() => ({ - forkMock: vi.fn(), +const { workerState, appMock } = vi.hoisted(() => ({ + workerState: { + calls: [] as unknown[][], + instance: null as object | null, + error: null as Error | null + }, appMock: { isPackaged: true, getAppPath: vi.fn(() => '/apps/orca/app.asar'), @@ -9,8 +14,16 @@ const { forkMock, appMock } = vi.hoisted(() => ({ } })) -vi.mock('node:child_process', () => ({ - fork: forkMock +vi.mock('node:worker_threads', () => ({ + Worker: class WorkerMock { + constructor(...args: unknown[]) { + workerState.calls.push(args) + if (workerState.error) { + throw workerState.error + } + return workerState.instance as WorkerMock + } + } })) vi.mock('electron', () => ({ @@ -29,28 +42,33 @@ function withPlatform(platform: NodeJS.Platform, run: () => T): T { } } -function fakeChild() { +function fakeWorker() { return { - connected: true, - stderr: { on: vi.fn() }, + postMessage: vi.fn(), + unref: vi.fn(), on: vi.fn(), - send: vi.fn(), - disconnect: vi.fn(), - kill: vi.fn() + once: vi.fn() } } describe('installMainThreadHangWatchdog', () => { beforeEach(() => { vi.useFakeTimers() - forkMock.mockReset() + workerState.calls = [] + workerState.instance = null + workerState.error = null appMock.on.mockReset() appMock.isPackaged = true delete process.env.ORCA_HANG_WATCHDOG_FORCE + delete process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS + delete process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS }) afterEach(() => { vi.useRealTimers() + delete process.env.ORCA_HANG_WATCHDOG_FORCE + delete process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS + delete process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS }) it('is a no-op off macOS', () => { @@ -60,7 +78,7 @@ describe('installMainThreadHangWatchdog', () => { expect( withPlatform('linux', () => installMainThreadHangWatchdog({ userDataPath: '/ud' })) ).toBeNull() - expect(forkMock).not.toHaveBeenCalled() + expect(workerState.calls).toHaveLength(0) }) it('is a no-op in unpackaged builds unless forced', () => { @@ -69,62 +87,98 @@ describe('installMainThreadHangWatchdog', () => { withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' })) ).toBeNull() process.env.ORCA_HANG_WATCHDOG_FORCE = '1' - const child = fakeChild() - forkMock.mockReturnValue(child) + const worker = fakeWorker() + workerState.instance = worker expect( withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' })) ).not.toBeNull() delete process.env.ORCA_HANG_WATCHDOG_FORCE }) - it('forks the watchdog as plain Node with pid, bundle, and marker config', () => { - const child = fakeChild() - forkMock.mockReturnValue(child) + it('starts a worker with pid, marker, and timing config', () => { + const worker = fakeWorker() + workerState.instance = worker const handle = withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }) ) expect(handle).not.toBeNull() - const [, , options] = forkMock.mock.calls[0] - expect(options.env.ELECTRON_RUN_AS_NODE).toBe('1') - expect(options.env.ORCA_HANG_WATCHDOG_PARENT_PID).toBe(String(process.pid)) - expect(options.env.ORCA_HANG_WATCHDOG_MARKER_PATH).toContain('/ud') + const [workerPath, rawOptions] = workerState.calls[0] + const options = rawOptions as { + name: string + workerData: { + parentPid: number + markerPath: string + timeoutMs: number + checkIntervalMs: number + } + } + expect(workerPath).toBe( + join('/apps/orca/app.asar', 'out', 'main', 'main-thread-hang-watchdog-entry.js') + ) + expect(options.name).toBe('orca-main-thread-hang-watchdog') + expect(options.workerData).toMatchObject({ + parentPid: process.pid, + markerPath: join('/ud', 'main-thread-hang.json'), + timeoutMs: 45_000, + checkIntervalMs: 5_000 + }) + expect(worker.unref).toHaveBeenCalled() }) - it('sends heartbeats on an interval and shutdown+disconnect on stop', () => { - const child = fakeChild() - forkMock.mockReturnValue(child) + it('sends heartbeats on an interval and shutdown on stop', () => { + const worker = fakeWorker() + workerState.instance = worker const handle = withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }) ) vi.advanceTimersByTime(6_000) - const heartbeats = child.send.mock.calls.filter(([m]) => m.type === 'heartbeat') + const heartbeats = worker.postMessage.mock.calls.filter(([m]) => m.type === 'heartbeat') expect(heartbeats.length).toBe(3) handle?.stop() - expect(child.send.mock.calls.some(([m]) => m.type === 'shutdown')).toBe(true) - expect(child.disconnect).toHaveBeenCalled() + expect(worker.postMessage.mock.calls.some(([m]) => m.type === 'shutdown')).toBe(true) - // Why: quit fires will-quit twice; a second stop must not resend or throw. handle?.stop() - const shutdowns = child.send.mock.calls.filter(([m]) => m.type === 'shutdown') + const shutdowns = worker.postMessage.mock.calls.filter(([m]) => m.type === 'shutdown') expect(shutdowns.length).toBe(1) vi.advanceTimersByTime(10_000) - const heartbeatsAfterStop = child.send.mock.calls.filter(([m]) => m.type === 'heartbeat') + const heartbeatsAfterStop = worker.postMessage.mock.calls.filter( + ([m]) => m.type === 'heartbeat' + ) expect(heartbeatsAfterStop.length).toBe(3) }) + it('passes test timing overrides to the worker', () => { + process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS = '900' + process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS = '100' + workerState.instance = fakeWorker() + withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' })) + const options = workerState.calls[0][1] as { + workerData: { timeoutMs: number; checkIntervalMs: number } + } + expect(options.workerData).toMatchObject({ timeoutMs: 900, checkIntervalMs: 100 }) + }) + it('registers stop on will-quit', () => { - const child = fakeChild() - forkMock.mockReturnValue(child) + workerState.instance = fakeWorker() withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' })) expect(appMock.on).toHaveBeenCalledWith('will-quit', expect.any(Function)) }) - it('returns null and stays inert when the fork itself fails', () => { - forkMock.mockImplementation(() => { - throw new Error('spawn failure') - }) + it('stops heartbeat work when the worker exits', () => { + const worker = fakeWorker() + workerState.instance = worker + withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' })) + const exitListener = worker.once.mock.calls.find(([event]) => event === 'exit')?.[1] + expect(exitListener).toEqual(expect.any(Function)) + exitListener() + vi.advanceTimersByTime(6_000) + expect(worker.postMessage).not.toHaveBeenCalled() + }) + + it('returns null and stays inert when worker construction fails', () => { + workerState.error = new Error('worker failure') expect( withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' })) ).toBeNull() diff --git a/src/main/hang-watchdog/main-thread-hang-watchdog.ts b/src/main/hang-watchdog/main-thread-hang-watchdog.ts index f859f8b57..46454429f 100644 --- a/src/main/hang-watchdog/main-thread-hang-watchdog.ts +++ b/src/main/hang-watchdog/main-thread-hang-watchdog.ts @@ -1,19 +1,25 @@ -import { fork, type ChildProcess } from 'node:child_process' +import { Worker } from 'node:worker_threads' import { app } from 'electron' -import { resolveHangWatchdogEntryPath } from './hang-watchdog-entry-path' import { hangDetectionMarkerPath } from './hang-detection-marker' - -const HEARTBEAT_INTERVAL_MS = 2_000 +import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path' +import { + HANG_WATCHDOG_CHECK_INTERVAL_MS, + HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS, + HANG_WATCHDOG_TIMEOUT_MS, + type HangWatchdogWorkerData, + type MainToHangWatchdogWorkerMessage +} from './hang-watchdog-worker-protocol' export type MainThreadHangWatchdogHandle = { stop: () => void - child: ChildProcess + worker: Worker +} + +function positiveTiming(value: string | undefined, fallback: number): number { + const parsed = Number(value) + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback } -// Why: macOS 26 scene-backed AppKit windows can deadlock the main thread inside -// FrontBoardServices with no crash and no event loop left to self-report. A plain-Node sibling -// process watches heartbeats and records the stall so the next launch can report it. It observes -// only — see main-thread-hang-watchdog-entry.ts for why it does not kill the parent. export function installMainThreadHangWatchdog(options: { userDataPath: string }): MainThreadHangWatchdogHandle | null { @@ -24,58 +30,57 @@ export function installMainThreadHangWatchdog(options: { if (!app.isPackaged && process.env.ORCA_HANG_WATCHDOG_FORCE !== '1') { return null } - const entryPath = resolveHangWatchdogEntryPath(app.getAppPath(), app.isPackaged) - let child: ChildProcess + const workerPath = resolveHangWatchdogWorkerPath(app.getAppPath(), app.isPackaged) + const workerData: HangWatchdogWorkerData = { + parentPid: process.pid, + markerPath: hangDetectionMarkerPath(options.userDataPath), + timeoutMs: positiveTiming(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS, HANG_WATCHDOG_TIMEOUT_MS), + checkIntervalMs: positiveTiming( + process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS, + HANG_WATCHDOG_CHECK_INTERVAL_MS + ) + } + let worker: Worker try { - child = fork(entryPath, [], { - stdio: ['ignore', 'ignore', 'pipe', 'ipc'], - env: { - ...process.env, - ELECTRON_RUN_AS_NODE: '1', - ORCA_HANG_WATCHDOG_PARENT_PID: String(process.pid), - ORCA_HANG_WATCHDOG_MARKER_PATH: hangDetectionMarkerPath(options.userDataPath) - } + // Why: the worker survives an AppKit main-thread deadlock without another Electron process. + worker = new Worker(workerPath, { + name: 'orca-main-thread-hang-watchdog', + workerData }) } catch (error) { - console.error('[hang-watchdog] failed to fork watchdog process:', error) + console.error('[hang-watchdog] failed to start watchdog worker:', error) return null } - child.stderr?.on('data', (chunk: Buffer) => { - console.error('[hang-watchdog]', String(chunk).trimEnd()) + worker.on('error', (error) => { + console.error('[hang-watchdog] watchdog worker failed:', error) }) - // Why: the watchdog is a safety net — if it dies, run without it; a restart loop here must never affect the app. - child.on('error', () => {}) - const heartbeatTimer = setInterval(() => { - if (child.connected) { - try { - child.send({ type: 'heartbeat' }, () => {}) - } catch { - // Channel raced closed between the check and the send. - } - } - }, HEARTBEAT_INTERVAL_MS) let stopped = false + const postMessage = (message: MainToHangWatchdogWorkerMessage): void => { + if (stopped && message.type === 'heartbeat') { + return + } + try { + worker.postMessage(message) + } catch { + // The worker already exited. + } + } + const heartbeatTimer = setInterval(() => { + postMessage({ type: 'heartbeat' }) + }, HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS) const stop = (): void => { if (stopped) { return } stopped = true clearInterval(heartbeatTimer) - if (child.connected) { - try { - child.send({ type: 'shutdown' }, () => {}) - } catch { - // Already disconnecting. - } - } - // Why: disconnect guarantees the child observes parent shutdown and exits instead of misreading quit as a hang. - try { - child.disconnect() - } catch { - // Channel already closed. - } + postMessage({ type: 'shutdown' }) } - // Why: will-quit fires twice during quit; stop is idempotent. + worker.once('exit', () => { + stopped = true + clearInterval(heartbeatTimer) + }) + worker.unref() app.on('will-quit', stop) - return { stop, child } + return { stop, worker } }