perf(main): move hang watchdog into a worker thread (#11488)
* perf(main): add watchdog boundary memory benchmark Add a repeatable Electron 43 RSS harness that measures the production-built watchdog entry across the child-process and worker-thread boundaries. Record per-trial samples, the median, revision, runtime, and settling procedure for reproducible PR evidence. * perf(main): move hang watchdog into a worker thread Keep main-thread hang detection independent of the blocked Electron event loop without paying for a second ELECTRON_RUN_AS_NODE process. Preserve the marker and telemetry contract while moving timing configuration and heartbeats onto a bundled worker entry. * test(main): smoke packaged hang watchdog worker * fix(main): make packaged watchdog smoke able to fail The smoke reported failure only through process.exitCode, but its finally block quit Electron gracefully, and Electron takes its status from the browser exit code. Every failure mode — entry missing from app.asar, worker error, marker timeout, non-zero worker exit — exited 0 with the diagnostic discarded on stderr, so the required PR check could never go red. Propagate a real status via app.exit, assert the success line in stdout, and surface stderr. Verified against a packaged tree with the entry removed: exit 0 before, exit 1 after. Co-authored-by: Orca <help@stably.ai> --------- Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Neil
GitHub
parent
14de3fa14d
commit
cc078a5021
|
|
@ -273,6 +273,9 @@ jobs:
|
|||
- name: Smoke packaged CLI
|
||||
run: node config/scripts/smoke-packaged-cli.mjs --app-dir=dist/linux-unpacked
|
||||
|
||||
- name: Smoke packaged hang watchdog worker
|
||||
run: xvfb-run --auto-servernum node config/scripts/smoke-packaged-hang-watchdog-worker.mjs --app-dir=dist/linux-unpacked
|
||||
|
||||
package_windows:
|
||||
name: package (windows)
|
||||
runs-on: windows-2022
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ type OutputChunk = Rollup.OutputChunk
|
|||
const PLAIN_NODE_ENTRY_NAMES = [
|
||||
'daemon-entry',
|
||||
'parcel-watcher-process-entry',
|
||||
'main-thread-hang-watchdog-entry',
|
||||
'computer-sidecar',
|
||||
'agent-hooks/managed-agent-hook-controls',
|
||||
'codex/codex-app-server-grant-entry'
|
||||
|
|
|
|||
|
|
@ -154,7 +154,6 @@ module.exports = {
|
|||
'out/main/plugin-host-entry.js',
|
||||
'out/main/computer-sidecar.js',
|
||||
'out/main/parcel-watcher-process-entry.js',
|
||||
'out/main/main-thread-hang-watchdog-entry.js',
|
||||
'out/main/chunks/**',
|
||||
'resources/**',
|
||||
'node_modules/ws/**',
|
||||
|
|
|
|||
|
|
@ -171,11 +171,9 @@ describe('electron-builder config', () => {
|
|||
)
|
||||
})
|
||||
|
||||
// Why: the watchdog only arms in packaged builds, and its ELECTRON_RUN_AS_NODE
|
||||
// fork resolves the entry from app.asar.unpacked — inside the asar it never runs.
|
||||
it('unpacks the forked main-thread hang-watchdog entry', () => {
|
||||
expect(electronBuilderConfig.asarUnpack).toEqual(
|
||||
expect.arrayContaining(['out/main/main-thread-hang-watchdog-entry.js'])
|
||||
it('keeps the worker-thread hang watchdog inside app.asar', () => {
|
||||
expect(electronBuilderConfig.asarUnpack).not.toContain(
|
||||
'out/main/main-thread-hang-watchdog-entry.js'
|
||||
)
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,443 @@
|
|||
#!/usr/bin/env node
|
||||
import { execFileSync, fork, spawnSync } from 'node:child_process'
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url'
|
||||
import { Worker } from 'node:worker_threads'
|
||||
import {
|
||||
childRssBytes,
|
||||
median,
|
||||
physicalFootprintBytes,
|
||||
sampleMemory,
|
||||
sampleProductionPerformance
|
||||
} from './hang-watchdog-process-metrics.mjs'
|
||||
|
||||
const INTERNAL_ENV = 'ORCA_HANG_WATCHDOG_BENCH_INTERNAL'
|
||||
const BOUNDARY_ENV = 'ORCA_HANG_WATCHDOG_BENCH_BOUNDARY'
|
||||
const RESULT_PREFIX = 'ORCA_HANG_WATCHDOG_BENCH_RESULT='
|
||||
const DEFAULT_TRIALS = 7
|
||||
const SETTLE_MS = 2_000
|
||||
const SAMPLE_COUNT = 5
|
||||
const SAMPLE_INTERVAL_MS = 200
|
||||
const VERIFY_TIMEOUT_MS = 500
|
||||
const VERIFY_CHECK_INTERVAL_MS = 50
|
||||
const VERIFY_BLOCK_MS = 1_200
|
||||
const PRODUCTION_HEARTBEAT_INTERVAL_MS = 2_000
|
||||
const PRODUCTION_TIMEOUT_MS = 45_000
|
||||
const PRODUCTION_CHECK_INTERVAL_MS = 5_000
|
||||
const PRODUCTION_SAMPLE_MS = 30_000
|
||||
const MAX_LAUNCH_ATTEMPTS = 3
|
||||
const MIB = 1024 * 1024
|
||||
const scriptPath = import.meta.filename
|
||||
const repoRoot = path.resolve(import.meta.dirname, '..', '..')
|
||||
const entryPath = path.join(repoRoot, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
function forceGc() {
|
||||
if (typeof global.gc !== 'function') {
|
||||
throw new Error('Electron did not expose GC; keep --js-flags=--expose-gc in the harness')
|
||||
}
|
||||
global.gc()
|
||||
global.gc()
|
||||
}
|
||||
|
||||
function blockMainThread(ms) {
|
||||
const startedAt = Date.now()
|
||||
while (Date.now() - startedAt < ms) {
|
||||
// Intentional synchronous stall.
|
||||
}
|
||||
return { startedAt, endedAt: Date.now() }
|
||||
}
|
||||
|
||||
function readMarker(markerPath) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(markerPath, 'utf8'))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyBlockedMainDetection(markerPath, sendHeartbeat) {
|
||||
const block = blockMainThread(VERIFY_BLOCK_MS)
|
||||
const detected = readMarker(markerPath)
|
||||
sendHeartbeat()
|
||||
const deadline = Date.now() + VERIFY_TIMEOUT_MS
|
||||
let resolved
|
||||
do {
|
||||
resolved = readMarker(markerPath)
|
||||
if (resolved?.selfRecovered === true) {
|
||||
break
|
||||
}
|
||||
await sleep(VERIFY_CHECK_INTERVAL_MS)
|
||||
} while (Date.now() < deadline)
|
||||
const verified =
|
||||
detected?.detectedAt >= block.startedAt &&
|
||||
detected.detectedAt <= block.endedAt &&
|
||||
detected.selfRecovered === false &&
|
||||
resolved?.selfRecovered === true
|
||||
if (!verified) {
|
||||
throw new Error(
|
||||
`Built watchdog failed blocked-main verification: ${JSON.stringify({ detected, resolved })}`
|
||||
)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
function startChild(markerPath, timeoutMs, checkIntervalMs) {
|
||||
const startedAt = process.hrtime.bigint()
|
||||
const child = fork(entryPath, [], {
|
||||
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_RUN_AS_NODE: '1',
|
||||
ORCA_HANG_WATCHDOG_PARENT_PID: String(process.pid),
|
||||
ORCA_HANG_WATCHDOG_MARKER_PATH: markerPath,
|
||||
ORCA_HANG_WATCHDOG_TIMEOUT_MS: String(timeoutMs),
|
||||
ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS: String(checkIntervalMs)
|
||||
}
|
||||
})
|
||||
const startupMs = Number(process.hrtime.bigint() - startedAt) / 1e6
|
||||
return {
|
||||
pids: [process.pid, child.pid],
|
||||
startupMs,
|
||||
sendHeartbeat: () => child.send?.({ type: 'heartbeat' }),
|
||||
shutdown: async () => {
|
||||
if (child.exitCode !== null) {
|
||||
return
|
||||
}
|
||||
const exitPromise = new Promise((resolve) => child.once('exit', resolve))
|
||||
child.send?.({ type: 'shutdown' })
|
||||
if (child.connected) {
|
||||
child.disconnect()
|
||||
}
|
||||
await exitPromise
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function startWorker(markerPath, timeoutMs, checkIntervalMs) {
|
||||
const startedAt = process.hrtime.bigint()
|
||||
const worker = new Worker(entryPath, {
|
||||
workerData: {
|
||||
parentPid: process.pid,
|
||||
markerPath,
|
||||
timeoutMs,
|
||||
checkIntervalMs
|
||||
}
|
||||
})
|
||||
const startupMs = Number(process.hrtime.bigint() - startedAt) / 1e6
|
||||
return {
|
||||
pids: [process.pid],
|
||||
startupMs,
|
||||
sendHeartbeat: () => worker.postMessage({ type: 'heartbeat' }),
|
||||
shutdown: async () => {
|
||||
if (worker.threadId === -1) {
|
||||
return
|
||||
}
|
||||
const exitPromise = new Promise((resolve) => worker.once('exit', resolve))
|
||||
worker.postMessage({ type: 'shutdown' })
|
||||
await exitPromise
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyBoundary(markerPath, startBoundary) {
|
||||
const boundary = startBoundary(markerPath, VERIFY_TIMEOUT_MS, VERIFY_CHECK_INTERVAL_MS)
|
||||
const heartbeat = setInterval(boundary.sendHeartbeat, 100)
|
||||
try {
|
||||
await sleep(SETTLE_MS)
|
||||
return await verifyBlockedMainDetection(markerPath, boundary.sendHeartbeat)
|
||||
} finally {
|
||||
clearInterval(heartbeat)
|
||||
await boundary.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
async function measureChild(markerPath) {
|
||||
forceGc()
|
||||
await sleep(SETTLE_MS)
|
||||
forceGc()
|
||||
const before = await sampleMemory(
|
||||
() => process.memoryUsage().rss,
|
||||
() => physicalFootprintBytes([process.pid]),
|
||||
{ sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep }
|
||||
)
|
||||
const child = startChild(markerPath, PRODUCTION_TIMEOUT_MS, PRODUCTION_CHECK_INTERVAL_MS)
|
||||
let measurements
|
||||
try {
|
||||
await sleep(SETTLE_MS)
|
||||
forceGc()
|
||||
const childRss = await sampleMemory(
|
||||
() => childRssBytes(child.pids[1]),
|
||||
() => physicalFootprintBytes([child.pids[1]]),
|
||||
{ sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep }
|
||||
)
|
||||
const total = await sampleMemory(
|
||||
() => process.memoryUsage().rss + childRssBytes(child.pids[1]),
|
||||
() => physicalFootprintBytes(child.pids),
|
||||
{ sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep }
|
||||
)
|
||||
const performance = await sampleProductionPerformance(child, {
|
||||
heartbeatIntervalMs: PRODUCTION_HEARTBEAT_INTERVAL_MS,
|
||||
sampleMs: PRODUCTION_SAMPLE_MS,
|
||||
sleep
|
||||
})
|
||||
measurements = {
|
||||
rssBytes: childRss.rssBytes,
|
||||
summedProcessRssDeltaBytes: Math.max(0, total.rssBytes - before.rssBytes),
|
||||
physicalFootprintDeltaBytes: Math.max(
|
||||
0,
|
||||
total.physicalFootprintBytes - before.physicalFootprintBytes
|
||||
),
|
||||
startupMs: child.startupMs,
|
||||
...performance
|
||||
}
|
||||
} finally {
|
||||
await child.shutdown()
|
||||
}
|
||||
rmSync(markerPath, { force: true })
|
||||
return {
|
||||
...measurements,
|
||||
blockedMainThreadVerified: await verifyBoundary(markerPath, startChild)
|
||||
}
|
||||
}
|
||||
|
||||
async function measureWorker(markerPath) {
|
||||
forceGc()
|
||||
await sleep(SETTLE_MS)
|
||||
forceGc()
|
||||
const before = await sampleMemory(
|
||||
() => process.memoryUsage().rss,
|
||||
() => physicalFootprintBytes([process.pid]),
|
||||
{ sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep }
|
||||
)
|
||||
const worker = startWorker(markerPath, PRODUCTION_TIMEOUT_MS, PRODUCTION_CHECK_INTERVAL_MS)
|
||||
let measurements
|
||||
try {
|
||||
await sleep(SETTLE_MS)
|
||||
forceGc()
|
||||
const after = await sampleMemory(
|
||||
() => process.memoryUsage().rss,
|
||||
() => physicalFootprintBytes([process.pid]),
|
||||
{ sampleCount: SAMPLE_COUNT, sampleIntervalMs: SAMPLE_INTERVAL_MS, sleep }
|
||||
)
|
||||
const performance = await sampleProductionPerformance(worker, {
|
||||
heartbeatIntervalMs: PRODUCTION_HEARTBEAT_INTERVAL_MS,
|
||||
sampleMs: PRODUCTION_SAMPLE_MS,
|
||||
sleep
|
||||
})
|
||||
const rssBytes = Math.max(0, after.rssBytes - before.rssBytes)
|
||||
measurements = {
|
||||
rssBytes,
|
||||
summedProcessRssDeltaBytes: rssBytes,
|
||||
physicalFootprintDeltaBytes: Math.max(
|
||||
0,
|
||||
after.physicalFootprintBytes - before.physicalFootprintBytes
|
||||
),
|
||||
startupMs: worker.startupMs,
|
||||
...performance
|
||||
}
|
||||
} finally {
|
||||
await worker.shutdown()
|
||||
}
|
||||
rmSync(markerPath, { force: true })
|
||||
return {
|
||||
...measurements,
|
||||
blockedMainThreadVerified: await verifyBoundary(markerPath, startWorker)
|
||||
}
|
||||
}
|
||||
|
||||
async function runInternal() {
|
||||
if (process.platform !== 'darwin') {
|
||||
throw new Error('The production watchdog is macOS-only; run this benchmark on macOS')
|
||||
}
|
||||
const { app } = await import('electron')
|
||||
const boundary = process.env[BOUNDARY_ENV]
|
||||
const profileDir = mkdtempSync(path.join(tmpdir(), 'orca-watchdog-bench-'))
|
||||
app.setPath('userData', profileDir)
|
||||
try {
|
||||
await app.whenReady()
|
||||
const markerPath = path.join(profileDir, 'main-thread-hang.json')
|
||||
const result =
|
||||
boundary === 'child'
|
||||
? await measureChild(markerPath)
|
||||
: boundary === 'worker'
|
||||
? await measureWorker(markerPath)
|
||||
: (() => {
|
||||
throw new Error(`Unsupported boundary: ${boundary}`)
|
||||
})()
|
||||
process.stdout.write(`${RESULT_PREFIX}${JSON.stringify(result)}\n`)
|
||||
} finally {
|
||||
app.quit()
|
||||
rmSync(profileDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = { boundary: '', trials: DEFAULT_TRIALS, output: '' }
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index]
|
||||
const value = argv[index + 1]
|
||||
if (arg === '--boundary' || arg === '--trials' || arg === '--output') {
|
||||
if (!value) {
|
||||
throw new Error(`Missing value for ${arg}`)
|
||||
}
|
||||
options[arg.slice(2)] = arg === '--trials' ? Number(value) : value
|
||||
index += 1
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`)
|
||||
}
|
||||
}
|
||||
if (!['child', 'worker'].includes(options.boundary)) {
|
||||
throw new Error('--boundary must be child or worker')
|
||||
}
|
||||
if (!Number.isInteger(options.trials) || options.trials < 1) {
|
||||
throw new Error('--trials must be a positive integer')
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
function electronPath() {
|
||||
const requirePath = import.meta.resolve('electron')
|
||||
const electronModulePath = fileURLToPath(requirePath)
|
||||
return execFileSync(
|
||||
process.execPath,
|
||||
['-e', `process.stdout.write(require(${JSON.stringify(electronModulePath)}))`],
|
||||
{
|
||||
encoding: 'utf8'
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function runTrial(executable, boundary) {
|
||||
for (let attempt = 1; attempt <= MAX_LAUNCH_ATTEMPTS; attempt += 1) {
|
||||
const env = { ...process.env, [INTERNAL_ENV]: '1', [BOUNDARY_ENV]: boundary }
|
||||
delete env.ELECTRON_RUN_AS_NODE
|
||||
const launcherDir = mkdtempSync(path.join(tmpdir(), 'orca-watchdog-bench-launcher-'))
|
||||
writeFileSync(
|
||||
path.join(launcherDir, 'package.json'),
|
||||
JSON.stringify({ name: 'orca-watchdog-benchmark', main: 'main.cjs' })
|
||||
)
|
||||
writeFileSync(
|
||||
path.join(launcherDir, 'main.cjs'),
|
||||
`import(${JSON.stringify(pathToFileURL(scriptPath).href)}).catch((error) => {
|
||||
console.error(error)
|
||||
process.exitCode = 1
|
||||
})\n`
|
||||
)
|
||||
let result
|
||||
try {
|
||||
result = spawnSync(executable, ['--js-flags=--expose-gc', launcherDir], {
|
||||
cwd: repoRoot,
|
||||
env,
|
||||
encoding: 'utf8',
|
||||
timeout: 90_000
|
||||
})
|
||||
} finally {
|
||||
rmSync(launcherDir, { recursive: true, force: true })
|
||||
}
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
`Electron trial failed (${result.error?.message ?? result.signal ?? result.status}):\n` +
|
||||
`${result.stderr || result.stdout}`
|
||||
)
|
||||
}
|
||||
const line = result.stdout.split('\n').find((candidate) => candidate.startsWith(RESULT_PREFIX))
|
||||
if (line) {
|
||||
return { ...JSON.parse(line.slice(RESULT_PREFIX.length)), launchAttempts: attempt }
|
||||
}
|
||||
if (attempt === MAX_LAUNCH_ATTEMPTS || result.stderr || result.stdout) {
|
||||
throw new Error(`Electron trial did not report a result (status ${result.status})`)
|
||||
}
|
||||
}
|
||||
throw new Error('Electron trial exhausted launcher attempts')
|
||||
}
|
||||
|
||||
function runBenchmark() {
|
||||
if (process.platform !== 'darwin') {
|
||||
throw new Error('The production watchdog is macOS-only; run this benchmark on macOS')
|
||||
}
|
||||
if (!existsSync(entryPath)) {
|
||||
throw new Error(`Missing ${entryPath}; run pnpm exec electron-vite build first`)
|
||||
}
|
||||
const options = parseArgs(process.argv.slice(2))
|
||||
const builtEntry = readFileSync(entryPath, 'utf8')
|
||||
const hasChildContract = builtEntry.includes('ORCA_HANG_WATCHDOG_PARENT_PID')
|
||||
const hasWorkerContract = builtEntry.includes('workerData') && builtEntry.includes('parentPort')
|
||||
if (
|
||||
(options.boundary === 'child' && !hasChildContract) ||
|
||||
(options.boundary === 'worker' && !hasWorkerContract)
|
||||
) {
|
||||
throw new Error(
|
||||
`Built watchdog does not implement the requested ${options.boundary} boundary; rebuild the matching revision`
|
||||
)
|
||||
}
|
||||
const executable = electronPath()
|
||||
const results = Array.from({ length: options.trials }, () =>
|
||||
runTrial(executable, options.boundary)
|
||||
)
|
||||
const rssBytes = results.map((result) => result.rssBytes)
|
||||
const summedProcessRssDeltaBytes = results.map((result) => result.summedProcessRssDeltaBytes)
|
||||
const physicalFootprintDeltaBytes = results.map((result) => result.physicalFootprintDeltaBytes)
|
||||
const startupMs = results.map((result) => result.startupMs)
|
||||
const cpuMs = results.map((result) => result.cpuMs)
|
||||
const eventLoopDelayP95Ms = results.map((result) => result.eventLoopDelayP95Ms)
|
||||
const eventLoopDelayP99Ms = results.map((result) => result.eventLoopDelayP99Ms)
|
||||
const eventLoopDelayMaxMs = results.map((result) => result.eventLoopDelayMaxMs)
|
||||
const report = {
|
||||
benchmark: 'hang-watchdog-memory',
|
||||
boundary: options.boundary,
|
||||
revision: execFileSync('git', ['rev-parse', 'HEAD'], {
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf8'
|
||||
}).trim(),
|
||||
electron: execFileSync(executable, ['-e', 'process.stdout.write(process.versions.electron)'], {
|
||||
env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' },
|
||||
encoding: 'utf8'
|
||||
}).trim(),
|
||||
settleMs: SETTLE_MS,
|
||||
samplesPerTrial: SAMPLE_COUNT,
|
||||
productionHeartbeatIntervalMs: PRODUCTION_HEARTBEAT_INTERVAL_MS,
|
||||
productionCheckIntervalMs: PRODUCTION_CHECK_INTERVAL_MS,
|
||||
productionSampleMs: PRODUCTION_SAMPLE_MS,
|
||||
trials: options.trials,
|
||||
rssMiB: rssBytes.map((value) => Number((value / MIB).toFixed(2))),
|
||||
medianRssMiB: Number((median(rssBytes) / MIB).toFixed(2)),
|
||||
summedProcessRssDeltaMiB: summedProcessRssDeltaBytes.map((value) =>
|
||||
Number((value / MIB).toFixed(2))
|
||||
),
|
||||
medianSummedProcessRssDeltaMiB: Number((median(summedProcessRssDeltaBytes) / MIB).toFixed(2)),
|
||||
physicalFootprintDeltaMiB: physicalFootprintDeltaBytes.map((value) =>
|
||||
Number((value / MIB).toFixed(2))
|
||||
),
|
||||
medianPhysicalFootprintDeltaMiB: Number((median(physicalFootprintDeltaBytes) / MIB).toFixed(2)),
|
||||
startupMs: startupMs.map((value) => Number(value.toFixed(3))),
|
||||
medianStartupMs: Number(median(startupMs).toFixed(3)),
|
||||
cpuMs: cpuMs.map((value) => Number(value.toFixed(2))),
|
||||
medianCpuMs: Number(median(cpuMs).toFixed(2)),
|
||||
eventLoopDelayP95Ms: eventLoopDelayP95Ms.map((value) => Number(value.toFixed(3))),
|
||||
medianEventLoopDelayP95Ms: Number(median(eventLoopDelayP95Ms).toFixed(3)),
|
||||
eventLoopDelayP99Ms: eventLoopDelayP99Ms.map((value) => Number(value.toFixed(3))),
|
||||
medianEventLoopDelayP99Ms: Number(median(eventLoopDelayP99Ms).toFixed(3)),
|
||||
eventLoopDelayMaxMs: eventLoopDelayMaxMs.map((value) => Number(value.toFixed(3))),
|
||||
medianEventLoopDelayMaxMs: Number(median(eventLoopDelayMaxMs).toFixed(3)),
|
||||
heartbeatCounts: results.map((result) => result.heartbeatCount),
|
||||
launchAttempts: results.map((result) => result.launchAttempts),
|
||||
blockedMainThreadVerified: results.every((result) => result.blockedMainThreadVerified)
|
||||
}
|
||||
const serialized = `${JSON.stringify(report, null, 2)}\n`
|
||||
process.stdout.write(serialized)
|
||||
if (options.output) {
|
||||
writeFileSync(path.resolve(options.output), serialized)
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env[INTERNAL_ENV] === '1') {
|
||||
await runInternal()
|
||||
} else {
|
||||
runBenchmark()
|
||||
}
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
import { execFileSync } from 'node:child_process'
|
||||
import { monitorEventLoopDelay } from 'node:perf_hooks'
|
||||
|
||||
export function median(values) {
|
||||
const sorted = [...values].sort((left, right) => left - right)
|
||||
const middle = Math.floor(sorted.length / 2)
|
||||
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]
|
||||
}
|
||||
|
||||
export async function sampleMemory(readRss, readPhysicalFootprint, options) {
|
||||
const rssSamples = []
|
||||
const physicalFootprintSamples = []
|
||||
for (let index = 0; index < options.sampleCount; index += 1) {
|
||||
rssSamples.push(readRss())
|
||||
physicalFootprintSamples.push(readPhysicalFootprint())
|
||||
await options.sleep(options.sampleIntervalMs)
|
||||
}
|
||||
return {
|
||||
rssBytes: median(rssSamples),
|
||||
physicalFootprintBytes: median(physicalFootprintSamples)
|
||||
}
|
||||
}
|
||||
|
||||
export function childRssBytes(pid) {
|
||||
const raw = execFileSync('ps', ['-o', 'rss=', '-p', String(pid)], {
|
||||
encoding: 'utf8'
|
||||
}).trim()
|
||||
const rssKiB = Number(raw)
|
||||
if (!Number.isFinite(rssKiB) || rssKiB <= 0) {
|
||||
throw new Error(`Could not read watchdog child RSS for PID ${pid}`)
|
||||
}
|
||||
return rssKiB * 1024
|
||||
}
|
||||
|
||||
export function parsePhysicalFootprintBytes(output, processCount) {
|
||||
const match =
|
||||
processCount > 1
|
||||
? output.match(/^Summary Footprint:\s+(\d+) B$/m)
|
||||
: output.match(/^[^\s].*\sFootprint:\s+(\d+) B/m)
|
||||
const bytes = Number(match?.[1])
|
||||
return Number.isFinite(bytes) && bytes > 0 ? bytes : null
|
||||
}
|
||||
|
||||
export function physicalFootprintBytes(pids) {
|
||||
const pidArgs = pids.flatMap((pid) => ['--pid', String(pid)])
|
||||
const output = execFileSync(
|
||||
'/usr/bin/footprint',
|
||||
[...pidArgs, '--format', 'bytes', '--noCategories'],
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
const bytes = parsePhysicalFootprintBytes(output, pids.length)
|
||||
if (bytes === null) {
|
||||
throw new Error(`Could not read physical footprint for PIDs ${pids.join(', ')}`)
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
export function parseProcessCpuTimeMs(raw) {
|
||||
if (!raw.trim()) {
|
||||
return null
|
||||
}
|
||||
const parts = raw.split(':').map(Number)
|
||||
if (!parts.length || parts.some((part) => !Number.isFinite(part))) {
|
||||
return null
|
||||
}
|
||||
const seconds = parts.reduce((total, part) => total * 60 + part, 0)
|
||||
const milliseconds = seconds * 1_000
|
||||
return milliseconds >= 0 ? milliseconds : null
|
||||
}
|
||||
|
||||
function processCpuTimeMs(pid) {
|
||||
const raw = execFileSync('ps', ['-o', 'time=', '-p', String(pid)], {
|
||||
encoding: 'utf8'
|
||||
}).trim()
|
||||
const milliseconds = parseProcessCpuTimeMs(raw)
|
||||
if (milliseconds === null) {
|
||||
throw new Error(`Could not read CPU time for PID ${pid}`)
|
||||
}
|
||||
return milliseconds
|
||||
}
|
||||
|
||||
function combinedCpuTimeMs(pids) {
|
||||
return pids.reduce((total, pid) => total + processCpuTimeMs(pid), 0)
|
||||
}
|
||||
|
||||
export async function sampleProductionPerformance(boundary, options) {
|
||||
const loopDelay = monitorEventLoopDelay({ resolution: 10 })
|
||||
let heartbeatCount = 0
|
||||
const heartbeat = setInterval(() => {
|
||||
heartbeatCount += 1
|
||||
boundary.sendHeartbeat()
|
||||
}, options.heartbeatIntervalMs)
|
||||
const cpuBefore = combinedCpuTimeMs(boundary.pids)
|
||||
loopDelay.enable()
|
||||
try {
|
||||
await options.sleep(options.sampleMs)
|
||||
} finally {
|
||||
loopDelay.disable()
|
||||
clearInterval(heartbeat)
|
||||
}
|
||||
return {
|
||||
cpuMs: Math.max(0, combinedCpuTimeMs(boundary.pids) - cpuBefore),
|
||||
heartbeatCount,
|
||||
eventLoopDelayP95Ms: loopDelay.percentile(95) / 1e6,
|
||||
eventLoopDelayP99Ms: loopDelay.percentile(99) / 1e6,
|
||||
eventLoopDelayMaxMs: loopDelay.max / 1e6
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
parsePhysicalFootprintBytes,
|
||||
parseProcessCpuTimeMs
|
||||
} from './hang-watchdog-process-metrics.mjs'
|
||||
|
||||
describe('hang watchdog process metrics', () => {
|
||||
it('uses the de-duplicated summary for multiple processes', () => {
|
||||
const output = `
|
||||
Electron [101]: 64-bit Footprint: 5000000 B (16384 bytes per page)
|
||||
phys_footprint: 5100000 B
|
||||
Electron Helper [102]: 64-bit Footprint: 2000000 B (16384 bytes per page)
|
||||
phys_footprint: 2100000 B
|
||||
Summary Footprint: 6259264 B
|
||||
`
|
||||
expect(parsePhysicalFootprintBytes(output, 2)).toBe(6_259_264)
|
||||
})
|
||||
|
||||
it('uses the process footprint rather than auxiliary accounting for one process', () => {
|
||||
const output = `
|
||||
Electron [101]: 64-bit Footprint: 5000000 B (16384 bytes per page)
|
||||
phys_footprint: 5100000 B
|
||||
`
|
||||
expect(parsePhysicalFootprintBytes(output, 1)).toBe(5_000_000)
|
||||
})
|
||||
|
||||
it('rejects missing or zero footprint summaries', () => {
|
||||
expect(parsePhysicalFootprintBytes('phys_footprint: 100 B', 2)).toBeNull()
|
||||
expect(parsePhysicalFootprintBytes('Summary Footprint: 0 B', 2)).toBeNull()
|
||||
})
|
||||
|
||||
it.each([
|
||||
['0:00.04', 40],
|
||||
['1:02.50', 62_500],
|
||||
['2:01:02.50', 7_262_500]
|
||||
])('parses ps CPU time %s', (value, expected) => {
|
||||
expect(parseProcessCpuTimeMs(value)).toBe(expected)
|
||||
})
|
||||
|
||||
it('rejects invalid CPU times', () => {
|
||||
expect(parseProcessCpuTimeMs('')).toBeNull()
|
||||
expect(parseProcessCpuTimeMs('not-a-time')).toBeNull()
|
||||
expect(parseProcessCpuTimeMs('-1:00')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import { readFileSync } from 'node:fs'
|
||||
import { parse } from 'yaml'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
describe('packaged hang watchdog worker contract', () => {
|
||||
it('boots the worker from app.asar in PR checks', () => {
|
||||
const workflow = parse(readFileSync('.github/workflows/pr.yml', 'utf8'))
|
||||
const smokeSource = readFileSync(
|
||||
'config/scripts/smoke-packaged-hang-watchdog-worker.mjs',
|
||||
'utf8'
|
||||
)
|
||||
const smokeStep = workflow.jobs.package.steps.find(
|
||||
(step) => step.name === 'Smoke packaged hang watchdog worker'
|
||||
)
|
||||
|
||||
expect(smokeStep.run).toBe(
|
||||
'xvfb-run --auto-servernum node config/scripts/smoke-packaged-hang-watchdog-worker.mjs --app-dir=dist/linux-unpacked'
|
||||
)
|
||||
expect(smokeSource).toContain(
|
||||
"process.platform === 'linux' ? ['--no-sandbox', launcherDir] : [launcherDir]"
|
||||
)
|
||||
})
|
||||
|
||||
// Why: Electron ignores process.exitCode, so the gate needs app.exit plus a stdout assertion.
|
||||
it('fails the smoke when the packaged worker never reports success', () => {
|
||||
const smokeSource = readFileSync(
|
||||
'config/scripts/smoke-packaged-hang-watchdog-worker.mjs',
|
||||
'utf8'
|
||||
)
|
||||
|
||||
expect(smokeSource).toContain('app.exit(1)')
|
||||
expect(smokeSource).not.toContain('app.quit()')
|
||||
expect(smokeSource).toContain('if (!result.stdout.includes(SUCCESS_LINE))')
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
import { spawnSync } from 'node:child_process'
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { Worker } from 'node:worker_threads'
|
||||
|
||||
const INTERNAL_ENV = 'ORCA_PACKAGED_WATCHDOG_SMOKE_INTERNAL'
|
||||
const ASAR_ENV = 'ORCA_PACKAGED_WATCHDOG_SMOKE_ASAR'
|
||||
const TIMEOUT_MS = 100
|
||||
const CHECK_INTERVAL_MS = 20
|
||||
const POLL_TIMEOUT_MS = 5_000
|
||||
const SUCCESS_LINE = '[packaged-watchdog-smoke] app.asar worker detected and recovered a stall'
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms))
|
||||
}
|
||||
|
||||
function readAppDirArg(argv) {
|
||||
const explicit = argv.find((arg) => arg.startsWith('--app-dir='))
|
||||
if (explicit) {
|
||||
return explicit.slice('--app-dir='.length)
|
||||
}
|
||||
if (process.platform === 'darwin') {
|
||||
return 'dist/mac-arm64/Orca.app'
|
||||
}
|
||||
if (process.platform === 'win32') {
|
||||
return 'dist/win-unpacked'
|
||||
}
|
||||
return 'dist/linux-unpacked'
|
||||
}
|
||||
|
||||
function getResourcesDir(appDir) {
|
||||
return process.platform === 'darwin' || appDir.endsWith('.app')
|
||||
? join(appDir, 'Contents', 'Resources')
|
||||
: join(appDir, 'resources')
|
||||
}
|
||||
|
||||
function readMarker(markerPath) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(markerPath, 'utf8'))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForMarker(markerPath, predicate, workerError) {
|
||||
const deadline = Date.now() + POLL_TIMEOUT_MS
|
||||
while (Date.now() < deadline) {
|
||||
if (workerError.current) {
|
||||
throw workerError.current
|
||||
}
|
||||
const marker = readMarker(markerPath)
|
||||
if (predicate(marker)) {
|
||||
return marker
|
||||
}
|
||||
await sleep(CHECK_INTERVAL_MS)
|
||||
}
|
||||
throw new Error(`Timed out waiting for packaged watchdog marker at ${markerPath}`)
|
||||
}
|
||||
|
||||
async function waitForExit(worker, workerError) {
|
||||
const exitCode = await Promise.race([
|
||||
new Promise((resolveExit) => worker.once('exit', resolveExit)),
|
||||
sleep(POLL_TIMEOUT_MS).then(() => {
|
||||
throw new Error('Timed out waiting for packaged watchdog worker to exit')
|
||||
})
|
||||
])
|
||||
if (workerError.current) {
|
||||
throw workerError.current
|
||||
}
|
||||
if (exitCode !== 0) {
|
||||
throw new Error(`Packaged watchdog worker exited with code ${exitCode}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function runInternal() {
|
||||
const appAsar = process.env[ASAR_ENV]
|
||||
if (!appAsar) {
|
||||
throw new Error(`Missing ${ASAR_ENV}`)
|
||||
}
|
||||
const { app } = await import('electron')
|
||||
const tempRoot = mkdtempSync(join(tmpdir(), 'orca-packaged-watchdog-smoke-'))
|
||||
const markerPath = join(tempRoot, 'main-thread-hang.json')
|
||||
const entryPath = join(appAsar, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
let worker
|
||||
try {
|
||||
await app.whenReady()
|
||||
if (!existsSync(entryPath)) {
|
||||
throw new Error(`Packaged watchdog entry is missing from app.asar: ${entryPath}`)
|
||||
}
|
||||
const workerError = { current: null }
|
||||
worker = new Worker(entryPath, {
|
||||
workerData: {
|
||||
parentPid: process.pid,
|
||||
markerPath,
|
||||
timeoutMs: TIMEOUT_MS,
|
||||
checkIntervalMs: CHECK_INTERVAL_MS
|
||||
}
|
||||
})
|
||||
worker.once('error', (error) => {
|
||||
workerError.current = error
|
||||
})
|
||||
await waitForMarker(markerPath, (marker) => marker?.selfRecovered === false, workerError)
|
||||
worker.postMessage({ type: 'heartbeat' })
|
||||
await waitForMarker(markerPath, (marker) => marker?.selfRecovered === true, workerError)
|
||||
worker.postMessage({ type: 'shutdown' })
|
||||
await waitForExit(worker, workerError)
|
||||
worker = undefined
|
||||
console.log(SUCCESS_LINE)
|
||||
} finally {
|
||||
await worker?.terminate()
|
||||
rmSync(tempRoot, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
function runSmoke() {
|
||||
const appDir = resolve(readAppDirArg(process.argv.slice(2)))
|
||||
const appAsar = join(getResourcesDir(appDir), 'app.asar')
|
||||
if (!existsSync(appAsar)) {
|
||||
throw new Error(`Packaged app archive is missing: ${appAsar}`)
|
||||
}
|
||||
const require = createRequire(import.meta.url)
|
||||
const executable = require('electron')
|
||||
const launcherDir = mkdtempSync(join(tmpdir(), 'orca-packaged-watchdog-launcher-'))
|
||||
const launcherPath = join(launcherDir, 'main.cjs')
|
||||
writeFileSync(
|
||||
join(launcherDir, 'package.json'),
|
||||
JSON.stringify({ name: 'orca-packaged-watchdog-smoke', main: 'main.cjs' })
|
||||
)
|
||||
writeFileSync(
|
||||
launcherPath,
|
||||
`import(${JSON.stringify(pathToFileURL(import.meta.filename).href)}).catch((error) => {
|
||||
console.error(error)
|
||||
process.exitCode = 1
|
||||
})\n`
|
||||
)
|
||||
const env = { ...process.env, [INTERNAL_ENV]: '1', [ASAR_ENV]: appAsar, NODE_PATH: '' }
|
||||
delete env.ELECTRON_RUN_AS_NODE
|
||||
try {
|
||||
const electronArgs =
|
||||
process.platform === 'linux' ? ['--no-sandbox', launcherDir] : [launcherDir]
|
||||
const result = spawnSync(executable, electronArgs, {
|
||||
env,
|
||||
encoding: 'utf8',
|
||||
timeout: 15_000
|
||||
})
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
`Packaged watchdog smoke failed (${result.error?.message ?? result.signal ?? result.status}):\n` +
|
||||
`${result.stderr || result.stdout}`
|
||||
)
|
||||
}
|
||||
// Why: Electron discards process.exitCode, so status 0 alone can't prove the worker ran.
|
||||
if (!result.stdout.includes(SUCCESS_LINE)) {
|
||||
throw new Error(
|
||||
`Packaged watchdog smoke did not report success:\n${result.stderr || result.stdout}`
|
||||
)
|
||||
}
|
||||
process.stdout.write(result.stdout)
|
||||
} finally {
|
||||
rmSync(launcherDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
if (process.env[INTERNAL_ENV] === '1') {
|
||||
// Why: a graceful quit exits 0 regardless of process.exitCode; only app.exit propagates failure.
|
||||
const { app } = await import('electron')
|
||||
try {
|
||||
await runInternal()
|
||||
app.exit(0)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
app.exit(1)
|
||||
}
|
||||
} else {
|
||||
runSmoke()
|
||||
}
|
||||
|
|
@ -243,9 +243,8 @@ export const electronViteConfig: UserConfig = {
|
|||
// Why: forked with ELECTRON_RUN_AS_NODE so @parcel/watcher faults
|
||||
// can't take down the main process (issue #7547).
|
||||
'parcel-watcher-process-entry': resolve('src/main/ipc/parcel-watcher-process-entry.ts'),
|
||||
// Why: forked with ELECTRON_RUN_AS_NODE so it survives a deadlocked
|
||||
// main thread (macOS 26 AppKit scene-update deadlock) and can record
|
||||
// the stall for the next launch to report.
|
||||
// Why: a worker thread survives the macOS 26 AppKit main-thread deadlock
|
||||
// without paying for another Electron process.
|
||||
'main-thread-hang-watchdog-entry': resolve(
|
||||
'src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts'
|
||||
),
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@
|
|||
"bench:macos-computer-helper-owner-loss": "node config/scripts/macos-computer-helper-owner-loss-benchmark.mjs",
|
||||
"bench:startup": "pnpm run ensure:electron-runtime && node tools/benchmarks/startup-time-bench.mjs",
|
||||
"bench:daemon-coldstart": "pnpm run ensure:electron-runtime && node tools/benchmarks/daemon-coldstart-bench.mjs",
|
||||
"bench:hang-watchdog-memory": "pnpm run ensure:electron-runtime && node config/scripts/hang-watchdog-memory-benchmark.mjs",
|
||||
"bench:main-thread-jank": "pnpm run ensure:electron-runtime && node tools/benchmarks/main-thread-jank-bench.mjs",
|
||||
"bench:worktree-deletion": "node tools/benchmarks/worktree-deletion-dev-bench.mjs",
|
||||
"bench:zustand-selector-fanout": "node config/scripts/zustand-selector-fanout-benchmark.mjs",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
// Why: written by the plain-Node watchdog child when main-thread heartbeats stop, and rewritten if
|
||||
// Why: written by the watchdog worker when main-thread heartbeats stop, and rewritten if
|
||||
// they resume; consumed on the next launch to report how long the stall lasted and whether it ever
|
||||
// cleared. `selfRecovered` separates a real deadlock from a long-but-survivable stall — the two are
|
||||
// indistinguishable at detection time, and only the latter would have been a destructive kill.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { createHangWatchdogChildLoop } from './hang-watchdog-child-loop'
|
||||
import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop'
|
||||
|
||||
const TIMEOUT_MS = 45_000
|
||||
const CHECK_INTERVAL_MS = 5_000
|
||||
|
|
@ -8,7 +8,7 @@ function loopWithClock(startAt = 0) {
|
|||
let now = startAt
|
||||
const onHangDetected = vi.fn()
|
||||
const onHangResolved = vi.fn()
|
||||
const loop = createHangWatchdogChildLoop({
|
||||
const loop = createHangWatchdogDetectionLoop({
|
||||
timeoutMs: TIMEOUT_MS,
|
||||
checkIntervalMs: CHECK_INTERVAL_MS,
|
||||
now: () => now,
|
||||
|
|
@ -18,7 +18,7 @@ function loopWithClock(startAt = 0) {
|
|||
return { loop, onHangDetected, onHangResolved, advance: (ms: number) => (now += ms) }
|
||||
}
|
||||
|
||||
describe('createHangWatchdogChildLoop', () => {
|
||||
describe('createHangWatchdogDetectionLoop', () => {
|
||||
it('does not fire while heartbeats keep arriving', () => {
|
||||
const { loop, onHangDetected, advance } = loopWithClock()
|
||||
for (let i = 0; i < 100; i++) {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
export type HangWatchdogChildLoopConfig = {
|
||||
export type HangWatchdogDetectionLoopConfig = {
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
now: () => number
|
||||
|
|
@ -7,14 +7,14 @@ export type HangWatchdogChildLoopConfig = {
|
|||
onHangResolved: (unresponsiveMs: number) => void
|
||||
}
|
||||
|
||||
export type HangWatchdogChildLoop = {
|
||||
export type HangWatchdogDetectionLoop = {
|
||||
recordHeartbeat: () => void
|
||||
tick: () => void
|
||||
}
|
||||
|
||||
export function createHangWatchdogChildLoop(
|
||||
config: HangWatchdogChildLoopConfig
|
||||
): HangWatchdogChildLoop {
|
||||
export function createHangWatchdogDetectionLoop(
|
||||
config: HangWatchdogDetectionLoopConfig
|
||||
): HangWatchdogDetectionLoop {
|
||||
let lastHeartbeatAt = config.now()
|
||||
let lastTickAt = config.now()
|
||||
let detected = false
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
export function resolveHangWatchdogEntryPath(
|
||||
appPath: string,
|
||||
isPackaged: boolean,
|
||||
pathExists: (candidate: string) => boolean = existsSync
|
||||
): string {
|
||||
// Why: ELECTRON_RUN_AS_NODE bypasses asar integration, so the packaged entry must be forked from app.asar.unpacked.
|
||||
const basePath = isPackaged ? appPath.replace('app.asar', 'app.asar.unpacked') : appPath
|
||||
const adjacentBuildEntry = join(basePath, 'main-thread-hang-watchdog-entry.js')
|
||||
// Why: electron-vite's unpackaged appPath is already out/main; appending out/main again would silently disable the watchdog in dev and E2E builds.
|
||||
if (!isPackaged && pathExists(adjacentBuildEntry)) {
|
||||
return adjacentBuildEntry
|
||||
}
|
||||
return join(basePath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
import { join } from 'node:path'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path'
|
||||
|
||||
describe('resolveHangWatchdogWorkerPath', () => {
|
||||
it('resolves packaged workers inside app.asar', () => {
|
||||
const appPath = join('/apps', 'orca', 'app.asar')
|
||||
expect(resolveHangWatchdogWorkerPath(appPath, true)).toBe(
|
||||
join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
)
|
||||
})
|
||||
|
||||
it('uses an adjacent entry when the dev app path is out/main', () => {
|
||||
const appPath = join('/repo', 'out', 'main')
|
||||
const adjacent = join(appPath, 'main-thread-hang-watchdog-entry.js')
|
||||
const pathExists = vi.fn((candidate: string) => candidate === adjacent)
|
||||
expect(resolveHangWatchdogWorkerPath(appPath, false, pathExists)).toBe(adjacent)
|
||||
})
|
||||
|
||||
it('resolves through out/main from a dev project root', () => {
|
||||
const appPath = join('/repo', 'orca')
|
||||
expect(resolveHangWatchdogWorkerPath(appPath, false, () => false)).toBe(
|
||||
join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
export function resolveHangWatchdogWorkerPath(
|
||||
appPath: string,
|
||||
isPackaged: boolean,
|
||||
pathExists: (candidate: string) => boolean = existsSync
|
||||
): string {
|
||||
const adjacentBuildEntry = join(appPath, 'main-thread-hang-watchdog-entry.js')
|
||||
if (!isPackaged && pathExists(adjacentBuildEntry)) {
|
||||
return adjacentBuildEntry
|
||||
}
|
||||
return join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
export const HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS = 2_000
|
||||
export const HANG_WATCHDOG_TIMEOUT_MS = 45_000
|
||||
export const HANG_WATCHDOG_CHECK_INTERVAL_MS = 5_000
|
||||
|
||||
export type HangWatchdogWorkerData = {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
}
|
||||
|
||||
export type MainToHangWatchdogWorkerMessage = { type: 'heartbeat' } | { type: 'shutdown' }
|
||||
|
|
@ -12,7 +12,11 @@ vi.mock('node:child_process', () => ({
|
|||
spawn: spawnMock
|
||||
}))
|
||||
|
||||
import { recordHangObservation } from './main-thread-hang-watchdog-entry'
|
||||
import {
|
||||
isHangWatchdogWorkerData,
|
||||
recordHangObservation,
|
||||
runWatchdog
|
||||
} from './main-thread-hang-watchdog-entry'
|
||||
|
||||
describe('recordHangObservation', () => {
|
||||
let dir: string
|
||||
|
|
@ -67,10 +71,8 @@ describe('recordHangObservation', () => {
|
|||
})
|
||||
})
|
||||
|
||||
// Why: this is the safety contract of the whole PR. Observing a hang must never kill, relaunch,
|
||||
// or exit — a false positive would SIGKILL a live main thread mid-write. If a future change
|
||||
// reintroduces recovery, this test must fail loudly rather than ship silently.
|
||||
it('never spawns, signals, or exits', () => {
|
||||
// Why: a false positive must never kill a live main thread while writes may be in flight.
|
||||
recordHangObservation({
|
||||
parentPid: 4242,
|
||||
markerPath: join(dir, 'marker.json'),
|
||||
|
|
@ -104,3 +106,75 @@ describe('recordHangObservation', () => {
|
|||
).not.toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
describe('watchdog worker entry', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('accepts only complete positive worker data', () => {
|
||||
expect(
|
||||
isHangWatchdogWorkerData({
|
||||
parentPid: 42,
|
||||
markerPath: '/tmp/marker',
|
||||
timeoutMs: 100,
|
||||
checkIntervalMs: 25
|
||||
})
|
||||
).toBe(true)
|
||||
for (const value of [
|
||||
null,
|
||||
{},
|
||||
{ parentPid: 0, markerPath: '/tmp/marker', timeoutMs: 100, checkIntervalMs: 25 },
|
||||
{ parentPid: 42, markerPath: 7, timeoutMs: 100, checkIntervalMs: 25 },
|
||||
{ parentPid: 42, markerPath: '/tmp/marker', timeoutMs: 0, checkIntervalMs: 25 },
|
||||
{ parentPid: 42, markerPath: '/tmp/marker', timeoutMs: 100, checkIntervalMs: Infinity }
|
||||
]) {
|
||||
expect(isHangWatchdogWorkerData(value)).toBe(false)
|
||||
}
|
||||
})
|
||||
|
||||
it('routes heartbeats and shuts down its timer and port', () => {
|
||||
const markerPath = join(tmpdir(), `hang-watchdog-entry-${process.pid}.json`)
|
||||
let onMessage: ((message: { type: 'heartbeat' | 'shutdown' }) => void) | undefined
|
||||
const port = {
|
||||
on: vi.fn(
|
||||
(_event: 'message', listener: (message: { type: 'heartbeat' | 'shutdown' }) => void) => {
|
||||
onMessage = listener
|
||||
}
|
||||
),
|
||||
close: vi.fn()
|
||||
}
|
||||
try {
|
||||
runWatchdog(
|
||||
{
|
||||
parentPid: process.pid,
|
||||
markerPath,
|
||||
timeoutMs: 100,
|
||||
checkIntervalMs: 25
|
||||
},
|
||||
port
|
||||
)
|
||||
vi.advanceTimersByTime(75)
|
||||
onMessage?.({ type: 'heartbeat' })
|
||||
vi.advanceTimersByTime(75)
|
||||
expect(consumeHangDetectionMarker(markerPath)).toBeNull()
|
||||
vi.advanceTimersByTime(50)
|
||||
expect(consumeHangDetectionMarker(markerPath)).toMatchObject({ selfRecovered: false })
|
||||
|
||||
onMessage?.({ type: 'heartbeat' })
|
||||
expect(consumeHangDetectionMarker(markerPath)).toMatchObject({ selfRecovered: true })
|
||||
onMessage?.({ type: 'shutdown' })
|
||||
expect(port.close).toHaveBeenCalledOnce()
|
||||
vi.advanceTimersByTime(1_000)
|
||||
expect(port.close).toHaveBeenCalledOnce()
|
||||
expect(consumeHangDetectionMarker(markerPath)).toBeNull()
|
||||
} finally {
|
||||
rmSync(markerPath, { force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,20 +1,17 @@
|
|||
// Forked with ELECTRON_RUN_AS_NODE from the main process. Watches heartbeats from the main
|
||||
// thread; if they stop (e.g. the macOS 26 AppKit scene-update deadlock) it records a marker so
|
||||
// the next launch can report the stall, and rewrites it if the heartbeats come back.
|
||||
//
|
||||
// Why no kill: a deadlocked main thread has already lost whatever sat in the persistence debounce
|
||||
// window, so killing it recovers nothing the user could not recover by force-quitting. A false
|
||||
// positive, though, would SIGKILL a live main thread that was merely blocked — most plausibly on
|
||||
// I/O, i.e. exactly when writes are in flight. All of the downside sits in the misfire, so this
|
||||
// measures first: `selfRecovered` counts the stalls a killer would have gotten wrong.
|
||||
//
|
||||
// Must never import electron.
|
||||
import { createHangWatchdogChildLoop } from './hang-watchdog-child-loop'
|
||||
import { isMainThread, parentPort, workerData } from 'node:worker_threads'
|
||||
import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop'
|
||||
import { writeHangDetectionMarker } from './hang-detection-marker'
|
||||
import type {
|
||||
HangWatchdogWorkerData,
|
||||
MainToHangWatchdogWorkerMessage
|
||||
} from './hang-watchdog-worker-protocol'
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 45_000
|
||||
const DEFAULT_CHECK_INTERVAL_MS = 5_000
|
||||
type HangWatchdogPort = {
|
||||
on: (event: 'message', listener: (message: MainToHangWatchdogWorkerMessage) => void) => unknown
|
||||
close: () => void
|
||||
}
|
||||
|
||||
// Observation only: a false positive must never kill a live main thread mid-write.
|
||||
export function recordHangObservation(options: {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
|
|
@ -36,37 +33,65 @@ export function recordHangObservation(options: {
|
|||
}
|
||||
}
|
||||
|
||||
function runWatchdog(parentPid: number): void {
|
||||
const markerPath = process.env.ORCA_HANG_WATCHDOG_MARKER_PATH ?? ''
|
||||
const timeoutMs = Number(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS
|
||||
const checkIntervalMs =
|
||||
Number(process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS) || DEFAULT_CHECK_INTERVAL_MS
|
||||
|
||||
const loop = createHangWatchdogChildLoop({
|
||||
timeoutMs,
|
||||
checkIntervalMs,
|
||||
export function runWatchdog(
|
||||
config: HangWatchdogWorkerData,
|
||||
port: HangWatchdogPort | null = parentPort
|
||||
): void {
|
||||
if (!port) {
|
||||
return
|
||||
}
|
||||
const loop = createHangWatchdogDetectionLoop({
|
||||
timeoutMs: config.timeoutMs,
|
||||
checkIntervalMs: config.checkIntervalMs,
|
||||
now: () => Date.now(),
|
||||
onHangDetected: (unresponsiveMs) =>
|
||||
recordHangObservation({ parentPid, markerPath, unresponsiveMs, selfRecovered: false }),
|
||||
recordHangObservation({
|
||||
parentPid: config.parentPid,
|
||||
markerPath: config.markerPath,
|
||||
unresponsiveMs,
|
||||
selfRecovered: false
|
||||
}),
|
||||
// Why: rewriting the marker keeps one observation per stall rather than two rows to reconcile.
|
||||
onHangResolved: (unresponsiveMs) =>
|
||||
recordHangObservation({ parentPid, markerPath, unresponsiveMs, selfRecovered: true })
|
||||
recordHangObservation({
|
||||
parentPid: config.parentPid,
|
||||
markerPath: config.markerPath,
|
||||
unresponsiveMs,
|
||||
selfRecovered: true
|
||||
})
|
||||
})
|
||||
|
||||
process.on('message', (message) => {
|
||||
const type = (message as { type?: string } | null)?.type
|
||||
if (type === 'heartbeat') {
|
||||
let checkTimer: ReturnType<typeof setInterval> | null = setInterval(
|
||||
() => loop.tick(),
|
||||
config.checkIntervalMs
|
||||
)
|
||||
port.on('message', (message: MainToHangWatchdogWorkerMessage) => {
|
||||
if (message.type === 'heartbeat') {
|
||||
loop.recordHeartbeat()
|
||||
} else if (type === 'shutdown') {
|
||||
process.exit(0)
|
||||
} else if (message.type === 'shutdown') {
|
||||
if (checkTimer) {
|
||||
clearInterval(checkTimer)
|
||||
checkTimer = null
|
||||
}
|
||||
port.close()
|
||||
}
|
||||
})
|
||||
// Why: a normal parent exit closes the IPC channel; the watchdog must not outlive it and misfire.
|
||||
process.on('disconnect', () => process.exit(0))
|
||||
setInterval(() => loop.tick(), checkIntervalMs)
|
||||
}
|
||||
|
||||
const configuredParentPid = Number(process.env.ORCA_HANG_WATCHDOG_PARENT_PID)
|
||||
if (Number.isInteger(configuredParentPid) && configuredParentPid > 0) {
|
||||
runWatchdog(configuredParentPid)
|
||||
export function isHangWatchdogWorkerData(value: unknown): value is HangWatchdogWorkerData {
|
||||
const data = value as Partial<HangWatchdogWorkerData> | null
|
||||
return (
|
||||
!!data &&
|
||||
Number.isInteger(data.parentPid) &&
|
||||
(data.parentPid ?? 0) > 0 &&
|
||||
typeof data.markerPath === 'string' &&
|
||||
Number.isFinite(data.timeoutMs) &&
|
||||
(data.timeoutMs ?? 0) > 0 &&
|
||||
Number.isFinite(data.checkIntervalMs) &&
|
||||
(data.checkIntervalMs ?? 0) > 0
|
||||
)
|
||||
}
|
||||
|
||||
if (!isMainThread && isHangWatchdogWorkerData(workerData)) {
|
||||
runWatchdog(workerData)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const { forkMock, appMock } = vi.hoisted(() => ({
|
||||
forkMock: vi.fn(),
|
||||
const { workerState, appMock } = vi.hoisted(() => ({
|
||||
workerState: {
|
||||
calls: [] as unknown[][],
|
||||
instance: null as object | null,
|
||||
error: null as Error | null
|
||||
},
|
||||
appMock: {
|
||||
isPackaged: true,
|
||||
getAppPath: vi.fn(() => '/apps/orca/app.asar'),
|
||||
|
|
@ -9,8 +14,16 @@ const { forkMock, appMock } = vi.hoisted(() => ({
|
|||
}
|
||||
}))
|
||||
|
||||
vi.mock('node:child_process', () => ({
|
||||
fork: forkMock
|
||||
vi.mock('node:worker_threads', () => ({
|
||||
Worker: class WorkerMock {
|
||||
constructor(...args: unknown[]) {
|
||||
workerState.calls.push(args)
|
||||
if (workerState.error) {
|
||||
throw workerState.error
|
||||
}
|
||||
return workerState.instance as WorkerMock
|
||||
}
|
||||
}
|
||||
}))
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
|
|
@ -29,28 +42,33 @@ function withPlatform<T>(platform: NodeJS.Platform, run: () => T): T {
|
|||
}
|
||||
}
|
||||
|
||||
function fakeChild() {
|
||||
function fakeWorker() {
|
||||
return {
|
||||
connected: true,
|
||||
stderr: { on: vi.fn() },
|
||||
postMessage: vi.fn(),
|
||||
unref: vi.fn(),
|
||||
on: vi.fn(),
|
||||
send: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
kill: vi.fn()
|
||||
once: vi.fn()
|
||||
}
|
||||
}
|
||||
|
||||
describe('installMainThreadHangWatchdog', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
forkMock.mockReset()
|
||||
workerState.calls = []
|
||||
workerState.instance = null
|
||||
workerState.error = null
|
||||
appMock.on.mockReset()
|
||||
appMock.isPackaged = true
|
||||
delete process.env.ORCA_HANG_WATCHDOG_FORCE
|
||||
delete process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS
|
||||
delete process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
delete process.env.ORCA_HANG_WATCHDOG_FORCE
|
||||
delete process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS
|
||||
delete process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS
|
||||
})
|
||||
|
||||
it('is a no-op off macOS', () => {
|
||||
|
|
@ -60,7 +78,7 @@ describe('installMainThreadHangWatchdog', () => {
|
|||
expect(
|
||||
withPlatform('linux', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).toBeNull()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
expect(workerState.calls).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('is a no-op in unpackaged builds unless forced', () => {
|
||||
|
|
@ -69,62 +87,98 @@ describe('installMainThreadHangWatchdog', () => {
|
|||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).toBeNull()
|
||||
process.env.ORCA_HANG_WATCHDOG_FORCE = '1'
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
expect(
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).not.toBeNull()
|
||||
delete process.env.ORCA_HANG_WATCHDOG_FORCE
|
||||
})
|
||||
|
||||
it('forks the watchdog as plain Node with pid, bundle, and marker config', () => {
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
it('starts a worker with pid, marker, and timing config', () => {
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
const handle = withPlatform('darwin', () =>
|
||||
installMainThreadHangWatchdog({ userDataPath: '/ud' })
|
||||
)
|
||||
expect(handle).not.toBeNull()
|
||||
const [, , options] = forkMock.mock.calls[0]
|
||||
expect(options.env.ELECTRON_RUN_AS_NODE).toBe('1')
|
||||
expect(options.env.ORCA_HANG_WATCHDOG_PARENT_PID).toBe(String(process.pid))
|
||||
expect(options.env.ORCA_HANG_WATCHDOG_MARKER_PATH).toContain('/ud')
|
||||
const [workerPath, rawOptions] = workerState.calls[0]
|
||||
const options = rawOptions as {
|
||||
name: string
|
||||
workerData: {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
}
|
||||
}
|
||||
expect(workerPath).toBe(
|
||||
join('/apps/orca/app.asar', 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
)
|
||||
expect(options.name).toBe('orca-main-thread-hang-watchdog')
|
||||
expect(options.workerData).toMatchObject({
|
||||
parentPid: process.pid,
|
||||
markerPath: join('/ud', 'main-thread-hang.json'),
|
||||
timeoutMs: 45_000,
|
||||
checkIntervalMs: 5_000
|
||||
})
|
||||
expect(worker.unref).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('sends heartbeats on an interval and shutdown+disconnect on stop', () => {
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
it('sends heartbeats on an interval and shutdown on stop', () => {
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
const handle = withPlatform('darwin', () =>
|
||||
installMainThreadHangWatchdog({ userDataPath: '/ud' })
|
||||
)
|
||||
vi.advanceTimersByTime(6_000)
|
||||
const heartbeats = child.send.mock.calls.filter(([m]) => m.type === 'heartbeat')
|
||||
const heartbeats = worker.postMessage.mock.calls.filter(([m]) => m.type === 'heartbeat')
|
||||
expect(heartbeats.length).toBe(3)
|
||||
|
||||
handle?.stop()
|
||||
expect(child.send.mock.calls.some(([m]) => m.type === 'shutdown')).toBe(true)
|
||||
expect(child.disconnect).toHaveBeenCalled()
|
||||
expect(worker.postMessage.mock.calls.some(([m]) => m.type === 'shutdown')).toBe(true)
|
||||
|
||||
// Why: quit fires will-quit twice; a second stop must not resend or throw.
|
||||
handle?.stop()
|
||||
const shutdowns = child.send.mock.calls.filter(([m]) => m.type === 'shutdown')
|
||||
const shutdowns = worker.postMessage.mock.calls.filter(([m]) => m.type === 'shutdown')
|
||||
expect(shutdowns.length).toBe(1)
|
||||
|
||||
vi.advanceTimersByTime(10_000)
|
||||
const heartbeatsAfterStop = child.send.mock.calls.filter(([m]) => m.type === 'heartbeat')
|
||||
const heartbeatsAfterStop = worker.postMessage.mock.calls.filter(
|
||||
([m]) => m.type === 'heartbeat'
|
||||
)
|
||||
expect(heartbeatsAfterStop.length).toBe(3)
|
||||
})
|
||||
|
||||
it('passes test timing overrides to the worker', () => {
|
||||
process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS = '900'
|
||||
process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS = '100'
|
||||
workerState.instance = fakeWorker()
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
const options = workerState.calls[0][1] as {
|
||||
workerData: { timeoutMs: number; checkIntervalMs: number }
|
||||
}
|
||||
expect(options.workerData).toMatchObject({ timeoutMs: 900, checkIntervalMs: 100 })
|
||||
})
|
||||
|
||||
it('registers stop on will-quit', () => {
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
workerState.instance = fakeWorker()
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
expect(appMock.on).toHaveBeenCalledWith('will-quit', expect.any(Function))
|
||||
})
|
||||
|
||||
it('returns null and stays inert when the fork itself fails', () => {
|
||||
forkMock.mockImplementation(() => {
|
||||
throw new Error('spawn failure')
|
||||
})
|
||||
it('stops heartbeat work when the worker exits', () => {
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
const exitListener = worker.once.mock.calls.find(([event]) => event === 'exit')?.[1]
|
||||
expect(exitListener).toEqual(expect.any(Function))
|
||||
exitListener()
|
||||
vi.advanceTimersByTime(6_000)
|
||||
expect(worker.postMessage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns null and stays inert when worker construction fails', () => {
|
||||
workerState.error = new Error('worker failure')
|
||||
expect(
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).toBeNull()
|
||||
|
|
|
|||
|
|
@ -1,19 +1,25 @@
|
|||
import { fork, type ChildProcess } from 'node:child_process'
|
||||
import { Worker } from 'node:worker_threads'
|
||||
import { app } from 'electron'
|
||||
import { resolveHangWatchdogEntryPath } from './hang-watchdog-entry-path'
|
||||
import { hangDetectionMarkerPath } from './hang-detection-marker'
|
||||
|
||||
const HEARTBEAT_INTERVAL_MS = 2_000
|
||||
import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path'
|
||||
import {
|
||||
HANG_WATCHDOG_CHECK_INTERVAL_MS,
|
||||
HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS,
|
||||
HANG_WATCHDOG_TIMEOUT_MS,
|
||||
type HangWatchdogWorkerData,
|
||||
type MainToHangWatchdogWorkerMessage
|
||||
} from './hang-watchdog-worker-protocol'
|
||||
|
||||
export type MainThreadHangWatchdogHandle = {
|
||||
stop: () => void
|
||||
child: ChildProcess
|
||||
worker: Worker
|
||||
}
|
||||
|
||||
function positiveTiming(value: string | undefined, fallback: number): number {
|
||||
const parsed = Number(value)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
|
||||
}
|
||||
|
||||
// Why: macOS 26 scene-backed AppKit windows can deadlock the main thread inside
|
||||
// FrontBoardServices with no crash and no event loop left to self-report. A plain-Node sibling
|
||||
// process watches heartbeats and records the stall so the next launch can report it. It observes
|
||||
// only — see main-thread-hang-watchdog-entry.ts for why it does not kill the parent.
|
||||
export function installMainThreadHangWatchdog(options: {
|
||||
userDataPath: string
|
||||
}): MainThreadHangWatchdogHandle | null {
|
||||
|
|
@ -24,58 +30,57 @@ export function installMainThreadHangWatchdog(options: {
|
|||
if (!app.isPackaged && process.env.ORCA_HANG_WATCHDOG_FORCE !== '1') {
|
||||
return null
|
||||
}
|
||||
const entryPath = resolveHangWatchdogEntryPath(app.getAppPath(), app.isPackaged)
|
||||
let child: ChildProcess
|
||||
const workerPath = resolveHangWatchdogWorkerPath(app.getAppPath(), app.isPackaged)
|
||||
const workerData: HangWatchdogWorkerData = {
|
||||
parentPid: process.pid,
|
||||
markerPath: hangDetectionMarkerPath(options.userDataPath),
|
||||
timeoutMs: positiveTiming(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS, HANG_WATCHDOG_TIMEOUT_MS),
|
||||
checkIntervalMs: positiveTiming(
|
||||
process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS,
|
||||
HANG_WATCHDOG_CHECK_INTERVAL_MS
|
||||
)
|
||||
}
|
||||
let worker: Worker
|
||||
try {
|
||||
child = fork(entryPath, [], {
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_RUN_AS_NODE: '1',
|
||||
ORCA_HANG_WATCHDOG_PARENT_PID: String(process.pid),
|
||||
ORCA_HANG_WATCHDOG_MARKER_PATH: hangDetectionMarkerPath(options.userDataPath)
|
||||
}
|
||||
// Why: the worker survives an AppKit main-thread deadlock without another Electron process.
|
||||
worker = new Worker(workerPath, {
|
||||
name: 'orca-main-thread-hang-watchdog',
|
||||
workerData
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('[hang-watchdog] failed to fork watchdog process:', error)
|
||||
console.error('[hang-watchdog] failed to start watchdog worker:', error)
|
||||
return null
|
||||
}
|
||||
child.stderr?.on('data', (chunk: Buffer) => {
|
||||
console.error('[hang-watchdog]', String(chunk).trimEnd())
|
||||
worker.on('error', (error) => {
|
||||
console.error('[hang-watchdog] watchdog worker failed:', error)
|
||||
})
|
||||
// Why: the watchdog is a safety net — if it dies, run without it; a restart loop here must never affect the app.
|
||||
child.on('error', () => {})
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
if (child.connected) {
|
||||
try {
|
||||
child.send({ type: 'heartbeat' }, () => {})
|
||||
} catch {
|
||||
// Channel raced closed between the check and the send.
|
||||
}
|
||||
}
|
||||
}, HEARTBEAT_INTERVAL_MS)
|
||||
let stopped = false
|
||||
const postMessage = (message: MainToHangWatchdogWorkerMessage): void => {
|
||||
if (stopped && message.type === 'heartbeat') {
|
||||
return
|
||||
}
|
||||
try {
|
||||
worker.postMessage(message)
|
||||
} catch {
|
||||
// The worker already exited.
|
||||
}
|
||||
}
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
postMessage({ type: 'heartbeat' })
|
||||
}, HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS)
|
||||
const stop = (): void => {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
stopped = true
|
||||
clearInterval(heartbeatTimer)
|
||||
if (child.connected) {
|
||||
try {
|
||||
child.send({ type: 'shutdown' }, () => {})
|
||||
} catch {
|
||||
// Already disconnecting.
|
||||
}
|
||||
}
|
||||
// Why: disconnect guarantees the child observes parent shutdown and exits instead of misreading quit as a hang.
|
||||
try {
|
||||
child.disconnect()
|
||||
} catch {
|
||||
// Channel already closed.
|
||||
}
|
||||
postMessage({ type: 'shutdown' })
|
||||
}
|
||||
// Why: will-quit fires twice during quit; stop is idempotent.
|
||||
worker.once('exit', () => {
|
||||
stopped = true
|
||||
clearInterval(heartbeatTimer)
|
||||
})
|
||||
worker.unref()
|
||||
app.on('will-quit', stop)
|
||||
return { stop, child }
|
||||
return { stop, worker }
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue