perf(main): move hang watchdog into a worker thread (#11344)
Keep main-thread hang detection independent of the blocked Electron event loop while reducing watchdog memory from 47.1 MiB to 11.5 MiB. Preserve marker, recovery, and telemetry behavior with a bundled worker-thread entry.
This commit is contained in:
parent
dde72f85de
commit
3f37e32e72
|
|
@ -21,7 +21,6 @@ type OutputChunk = Rollup.OutputChunk
|
|||
const PLAIN_NODE_ENTRY_NAMES = [
|
||||
'daemon-entry',
|
||||
'parcel-watcher-process-entry',
|
||||
'main-thread-hang-watchdog-entry',
|
||||
'computer-sidecar',
|
||||
'agent-hooks/managed-agent-hook-controls',
|
||||
'codex/codex-app-server-grant-entry'
|
||||
|
|
|
|||
|
|
@ -153,7 +153,6 @@ module.exports = {
|
|||
'out/main/plugin-host-entry.js',
|
||||
'out/main/computer-sidecar.js',
|
||||
'out/main/parcel-watcher-process-entry.js',
|
||||
'out/main/main-thread-hang-watchdog-entry.js',
|
||||
'out/main/chunks/**',
|
||||
'resources/**',
|
||||
'node_modules/ws/**',
|
||||
|
|
|
|||
|
|
@ -166,11 +166,9 @@ describe('electron-builder config', () => {
|
|||
)
|
||||
})
|
||||
|
||||
// Why: the watchdog only arms in packaged builds, and its ELECTRON_RUN_AS_NODE
|
||||
// fork resolves the entry from app.asar.unpacked — inside the asar it never runs.
|
||||
it('unpacks the forked main-thread hang-watchdog entry', () => {
|
||||
expect(electronBuilderConfig.asarUnpack).toEqual(
|
||||
expect.arrayContaining(['out/main/main-thread-hang-watchdog-entry.js'])
|
||||
it('keeps the worker-thread hang watchdog inside app.asar', () => {
|
||||
expect(electronBuilderConfig.asarUnpack).not.toContain(
|
||||
'out/main/main-thread-hang-watchdog-entry.js'
|
||||
)
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -208,9 +208,8 @@ export const electronViteConfig: UserConfig = {
|
|||
// Why: forked with ELECTRON_RUN_AS_NODE so @parcel/watcher faults
|
||||
// can't take down the main process (issue #7547).
|
||||
'parcel-watcher-process-entry': resolve('src/main/ipc/parcel-watcher-process-entry.ts'),
|
||||
// Why: forked with ELECTRON_RUN_AS_NODE so it survives a deadlocked
|
||||
// main thread (macOS 26 AppKit scene-update deadlock) and can record
|
||||
// the stall for the next launch to report.
|
||||
// Why: a worker thread survives the macOS 26 AppKit main-thread deadlock
|
||||
// without paying for another Electron process.
|
||||
'main-thread-hang-watchdog-entry': resolve(
|
||||
'src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts'
|
||||
),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { createHangWatchdogChildLoop } from './hang-watchdog-child-loop'
|
||||
import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop'
|
||||
|
||||
const TIMEOUT_MS = 45_000
|
||||
const CHECK_INTERVAL_MS = 5_000
|
||||
|
|
@ -8,7 +8,7 @@ function loopWithClock(startAt = 0) {
|
|||
let now = startAt
|
||||
const onHangDetected = vi.fn()
|
||||
const onHangResolved = vi.fn()
|
||||
const loop = createHangWatchdogChildLoop({
|
||||
const loop = createHangWatchdogDetectionLoop({
|
||||
timeoutMs: TIMEOUT_MS,
|
||||
checkIntervalMs: CHECK_INTERVAL_MS,
|
||||
now: () => now,
|
||||
|
|
@ -18,7 +18,7 @@ function loopWithClock(startAt = 0) {
|
|||
return { loop, onHangDetected, onHangResolved, advance: (ms: number) => (now += ms) }
|
||||
}
|
||||
|
||||
describe('createHangWatchdogChildLoop', () => {
|
||||
describe('createHangWatchdogDetectionLoop', () => {
|
||||
it('does not fire while heartbeats keep arriving', () => {
|
||||
const { loop, onHangDetected, advance } = loopWithClock()
|
||||
for (let i = 0; i < 100; i++) {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
export type HangWatchdogChildLoopConfig = {
|
||||
export type HangWatchdogDetectionLoopConfig = {
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
now: () => number
|
||||
|
|
@ -7,14 +7,14 @@ export type HangWatchdogChildLoopConfig = {
|
|||
onHangResolved: (unresponsiveMs: number) => void
|
||||
}
|
||||
|
||||
export type HangWatchdogChildLoop = {
|
||||
export type HangWatchdogDetectionLoop = {
|
||||
recordHeartbeat: () => void
|
||||
tick: () => void
|
||||
}
|
||||
|
||||
export function createHangWatchdogChildLoop(
|
||||
config: HangWatchdogChildLoopConfig
|
||||
): HangWatchdogChildLoop {
|
||||
export function createHangWatchdogDetectionLoop(
|
||||
config: HangWatchdogDetectionLoopConfig
|
||||
): HangWatchdogDetectionLoop {
|
||||
let lastHeartbeatAt = config.now()
|
||||
let lastTickAt = config.now()
|
||||
let detected = false
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
export function resolveHangWatchdogEntryPath(
|
||||
appPath: string,
|
||||
isPackaged: boolean,
|
||||
pathExists: (candidate: string) => boolean = existsSync
|
||||
): string {
|
||||
// Why: ELECTRON_RUN_AS_NODE bypasses asar integration, so the packaged entry must be forked from app.asar.unpacked.
|
||||
const basePath = isPackaged ? appPath.replace('app.asar', 'app.asar.unpacked') : appPath
|
||||
const adjacentBuildEntry = join(basePath, 'main-thread-hang-watchdog-entry.js')
|
||||
// Why: electron-vite's unpackaged appPath is already out/main; appending out/main again would silently disable the watchdog in dev and E2E builds.
|
||||
if (!isPackaged && pathExists(adjacentBuildEntry)) {
|
||||
return adjacentBuildEntry
|
||||
}
|
||||
return join(basePath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
import { join } from 'node:path'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path'
|
||||
|
||||
describe('resolveHangWatchdogWorkerPath', () => {
|
||||
it('resolves packaged workers inside app.asar', () => {
|
||||
const appPath = join('/apps', 'orca', 'app.asar')
|
||||
expect(resolveHangWatchdogWorkerPath(appPath, true)).toBe(
|
||||
join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
)
|
||||
})
|
||||
|
||||
it('uses an adjacent entry when the dev app path is out/main', () => {
|
||||
const appPath = join('/repo', 'out', 'main')
|
||||
const adjacent = join(appPath, 'main-thread-hang-watchdog-entry.js')
|
||||
const pathExists = vi.fn((candidate: string) => candidate === adjacent)
|
||||
expect(resolveHangWatchdogWorkerPath(appPath, false, pathExists)).toBe(adjacent)
|
||||
})
|
||||
|
||||
it('resolves through out/main from a dev project root', () => {
|
||||
const appPath = join('/repo', 'orca')
|
||||
expect(resolveHangWatchdogWorkerPath(appPath, false, () => false)).toBe(
|
||||
join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
export function resolveHangWatchdogWorkerPath(
|
||||
appPath: string,
|
||||
isPackaged: boolean,
|
||||
pathExists: (candidate: string) => boolean = existsSync
|
||||
): string {
|
||||
const adjacentBuildEntry = join(appPath, 'main-thread-hang-watchdog-entry.js')
|
||||
if (!isPackaged && pathExists(adjacentBuildEntry)) {
|
||||
return adjacentBuildEntry
|
||||
}
|
||||
return join(appPath, 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
export const HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS = 2_000
|
||||
export const HANG_WATCHDOG_TIMEOUT_MS = 45_000
|
||||
export const HANG_WATCHDOG_CHECK_INTERVAL_MS = 5_000
|
||||
|
||||
export type HangWatchdogWorkerData = {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
}
|
||||
|
||||
export type MainToHangWatchdogWorkerMessage = { type: 'heartbeat' } | { type: 'shutdown' }
|
||||
|
|
@ -67,9 +67,6 @@ describe('recordHangObservation', () => {
|
|||
})
|
||||
})
|
||||
|
||||
// Why: this is the safety contract of the whole PR. Observing a hang must never kill, relaunch,
|
||||
// or exit — a false positive would SIGKILL a live main thread mid-write. If a future change
|
||||
// reintroduces recovery, this test must fail loudly rather than ship silently.
|
||||
it('never spawns, signals, or exits', () => {
|
||||
recordHangObservation({
|
||||
parentPid: 4242,
|
||||
|
|
|
|||
|
|
@ -1,20 +1,12 @@
|
|||
// Forked with ELECTRON_RUN_AS_NODE from the main process. Watches heartbeats from the main
|
||||
// thread; if they stop (e.g. the macOS 26 AppKit scene-update deadlock) it records a marker so
|
||||
// the next launch can report the stall, and rewrites it if the heartbeats come back.
|
||||
//
|
||||
// Why no kill: a deadlocked main thread has already lost whatever sat in the persistence debounce
|
||||
// window, so killing it recovers nothing the user could not recover by force-quitting. A false
|
||||
// positive, though, would SIGKILL a live main thread that was merely blocked — most plausibly on
|
||||
// I/O, i.e. exactly when writes are in flight. All of the downside sits in the misfire, so this
|
||||
// measures first: `selfRecovered` counts the stalls a killer would have gotten wrong.
|
||||
//
|
||||
// Must never import electron.
|
||||
import { createHangWatchdogChildLoop } from './hang-watchdog-child-loop'
|
||||
import { isMainThread, parentPort, workerData } from 'node:worker_threads'
|
||||
import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop'
|
||||
import { writeHangDetectionMarker } from './hang-detection-marker'
|
||||
import type {
|
||||
HangWatchdogWorkerData,
|
||||
MainToHangWatchdogWorkerMessage
|
||||
} from './hang-watchdog-worker-protocol'
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 45_000
|
||||
const DEFAULT_CHECK_INTERVAL_MS = 5_000
|
||||
|
||||
// Observation only: a false positive must never kill a live main thread mid-write.
|
||||
export function recordHangObservation(options: {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
|
|
@ -36,37 +28,63 @@ export function recordHangObservation(options: {
|
|||
}
|
||||
}
|
||||
|
||||
function runWatchdog(parentPid: number): void {
|
||||
const markerPath = process.env.ORCA_HANG_WATCHDOG_MARKER_PATH ?? ''
|
||||
const timeoutMs = Number(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS) || DEFAULT_TIMEOUT_MS
|
||||
const checkIntervalMs =
|
||||
Number(process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS) || DEFAULT_CHECK_INTERVAL_MS
|
||||
|
||||
const loop = createHangWatchdogChildLoop({
|
||||
timeoutMs,
|
||||
checkIntervalMs,
|
||||
function runWatchdog(config: HangWatchdogWorkerData): void {
|
||||
const port = parentPort
|
||||
if (!port) {
|
||||
return
|
||||
}
|
||||
const loop = createHangWatchdogDetectionLoop({
|
||||
timeoutMs: config.timeoutMs,
|
||||
checkIntervalMs: config.checkIntervalMs,
|
||||
now: () => Date.now(),
|
||||
onHangDetected: (unresponsiveMs) =>
|
||||
recordHangObservation({ parentPid, markerPath, unresponsiveMs, selfRecovered: false }),
|
||||
recordHangObservation({
|
||||
parentPid: config.parentPid,
|
||||
markerPath: config.markerPath,
|
||||
unresponsiveMs,
|
||||
selfRecovered: false
|
||||
}),
|
||||
// Why: rewriting the marker keeps one observation per stall rather than two rows to reconcile.
|
||||
onHangResolved: (unresponsiveMs) =>
|
||||
recordHangObservation({ parentPid, markerPath, unresponsiveMs, selfRecovered: true })
|
||||
recordHangObservation({
|
||||
parentPid: config.parentPid,
|
||||
markerPath: config.markerPath,
|
||||
unresponsiveMs,
|
||||
selfRecovered: true
|
||||
})
|
||||
})
|
||||
|
||||
process.on('message', (message) => {
|
||||
const type = (message as { type?: string } | null)?.type
|
||||
if (type === 'heartbeat') {
|
||||
let checkTimer: ReturnType<typeof setInterval> | null = setInterval(
|
||||
() => loop.tick(),
|
||||
config.checkIntervalMs
|
||||
)
|
||||
port.on('message', (message: MainToHangWatchdogWorkerMessage) => {
|
||||
if (message.type === 'heartbeat') {
|
||||
loop.recordHeartbeat()
|
||||
} else if (type === 'shutdown') {
|
||||
process.exit(0)
|
||||
} else if (message.type === 'shutdown') {
|
||||
if (checkTimer) {
|
||||
clearInterval(checkTimer)
|
||||
checkTimer = null
|
||||
}
|
||||
port.close()
|
||||
}
|
||||
})
|
||||
// Why: a normal parent exit closes the IPC channel; the watchdog must not outlive it and misfire.
|
||||
process.on('disconnect', () => process.exit(0))
|
||||
setInterval(() => loop.tick(), checkIntervalMs)
|
||||
}
|
||||
|
||||
const configuredParentPid = Number(process.env.ORCA_HANG_WATCHDOG_PARENT_PID)
|
||||
if (Number.isInteger(configuredParentPid) && configuredParentPid > 0) {
|
||||
runWatchdog(configuredParentPid)
|
||||
function isHangWatchdogWorkerData(value: unknown): value is HangWatchdogWorkerData {
|
||||
const data = value as Partial<HangWatchdogWorkerData> | null
|
||||
return (
|
||||
!!data &&
|
||||
Number.isInteger(data.parentPid) &&
|
||||
(data.parentPid ?? 0) > 0 &&
|
||||
typeof data.markerPath === 'string' &&
|
||||
Number.isFinite(data.timeoutMs) &&
|
||||
(data.timeoutMs ?? 0) > 0 &&
|
||||
Number.isFinite(data.checkIntervalMs) &&
|
||||
(data.checkIntervalMs ?? 0) > 0
|
||||
)
|
||||
}
|
||||
|
||||
if (!isMainThread && isHangWatchdogWorkerData(workerData)) {
|
||||
runWatchdog(workerData)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const { forkMock, appMock } = vi.hoisted(() => ({
|
||||
forkMock: vi.fn(),
|
||||
const { workerState, appMock } = vi.hoisted(() => ({
|
||||
workerState: {
|
||||
calls: [] as unknown[][],
|
||||
instance: null as object | null,
|
||||
error: null as Error | null
|
||||
},
|
||||
appMock: {
|
||||
isPackaged: true,
|
||||
getAppPath: vi.fn(() => '/apps/orca/app.asar'),
|
||||
|
|
@ -9,8 +14,16 @@ const { forkMock, appMock } = vi.hoisted(() => ({
|
|||
}
|
||||
}))
|
||||
|
||||
vi.mock('node:child_process', () => ({
|
||||
fork: forkMock
|
||||
vi.mock('node:worker_threads', () => ({
|
||||
Worker: class WorkerMock {
|
||||
constructor(...args: unknown[]) {
|
||||
workerState.calls.push(args)
|
||||
if (workerState.error) {
|
||||
throw workerState.error
|
||||
}
|
||||
return workerState.instance as WorkerMock
|
||||
}
|
||||
}
|
||||
}))
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
|
|
@ -29,28 +42,33 @@ function withPlatform<T>(platform: NodeJS.Platform, run: () => T): T {
|
|||
}
|
||||
}
|
||||
|
||||
function fakeChild() {
|
||||
function fakeWorker() {
|
||||
return {
|
||||
connected: true,
|
||||
stderr: { on: vi.fn() },
|
||||
postMessage: vi.fn(),
|
||||
unref: vi.fn(),
|
||||
on: vi.fn(),
|
||||
send: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
kill: vi.fn()
|
||||
once: vi.fn()
|
||||
}
|
||||
}
|
||||
|
||||
describe('installMainThreadHangWatchdog', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
forkMock.mockReset()
|
||||
workerState.calls = []
|
||||
workerState.instance = null
|
||||
workerState.error = null
|
||||
appMock.on.mockReset()
|
||||
appMock.isPackaged = true
|
||||
delete process.env.ORCA_HANG_WATCHDOG_FORCE
|
||||
delete process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS
|
||||
delete process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
delete process.env.ORCA_HANG_WATCHDOG_FORCE
|
||||
delete process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS
|
||||
delete process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS
|
||||
})
|
||||
|
||||
it('is a no-op off macOS', () => {
|
||||
|
|
@ -60,7 +78,7 @@ describe('installMainThreadHangWatchdog', () => {
|
|||
expect(
|
||||
withPlatform('linux', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).toBeNull()
|
||||
expect(forkMock).not.toHaveBeenCalled()
|
||||
expect(workerState.calls).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('is a no-op in unpackaged builds unless forced', () => {
|
||||
|
|
@ -69,62 +87,98 @@ describe('installMainThreadHangWatchdog', () => {
|
|||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).toBeNull()
|
||||
process.env.ORCA_HANG_WATCHDOG_FORCE = '1'
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
expect(
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).not.toBeNull()
|
||||
delete process.env.ORCA_HANG_WATCHDOG_FORCE
|
||||
})
|
||||
|
||||
it('forks the watchdog as plain Node with pid, bundle, and marker config', () => {
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
it('starts a worker with pid, marker, and timing config', () => {
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
const handle = withPlatform('darwin', () =>
|
||||
installMainThreadHangWatchdog({ userDataPath: '/ud' })
|
||||
)
|
||||
expect(handle).not.toBeNull()
|
||||
const [, , options] = forkMock.mock.calls[0]
|
||||
expect(options.env.ELECTRON_RUN_AS_NODE).toBe('1')
|
||||
expect(options.env.ORCA_HANG_WATCHDOG_PARENT_PID).toBe(String(process.pid))
|
||||
expect(options.env.ORCA_HANG_WATCHDOG_MARKER_PATH).toContain('/ud')
|
||||
const [workerPath, rawOptions] = workerState.calls[0]
|
||||
const options = rawOptions as {
|
||||
name: string
|
||||
workerData: {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
}
|
||||
}
|
||||
expect(workerPath).toBe(
|
||||
join('/apps/orca/app.asar', 'out', 'main', 'main-thread-hang-watchdog-entry.js')
|
||||
)
|
||||
expect(options.name).toBe('orca-main-thread-hang-watchdog')
|
||||
expect(options.workerData).toMatchObject({
|
||||
parentPid: process.pid,
|
||||
markerPath: join('/ud', 'main-thread-hang.json'),
|
||||
timeoutMs: 45_000,
|
||||
checkIntervalMs: 5_000
|
||||
})
|
||||
expect(worker.unref).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('sends heartbeats on an interval and shutdown+disconnect on stop', () => {
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
it('sends heartbeats on an interval and shutdown on stop', () => {
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
const handle = withPlatform('darwin', () =>
|
||||
installMainThreadHangWatchdog({ userDataPath: '/ud' })
|
||||
)
|
||||
vi.advanceTimersByTime(6_000)
|
||||
const heartbeats = child.send.mock.calls.filter(([m]) => m.type === 'heartbeat')
|
||||
const heartbeats = worker.postMessage.mock.calls.filter(([m]) => m.type === 'heartbeat')
|
||||
expect(heartbeats.length).toBe(3)
|
||||
|
||||
handle?.stop()
|
||||
expect(child.send.mock.calls.some(([m]) => m.type === 'shutdown')).toBe(true)
|
||||
expect(child.disconnect).toHaveBeenCalled()
|
||||
expect(worker.postMessage.mock.calls.some(([m]) => m.type === 'shutdown')).toBe(true)
|
||||
|
||||
// Why: quit fires will-quit twice; a second stop must not resend or throw.
|
||||
handle?.stop()
|
||||
const shutdowns = child.send.mock.calls.filter(([m]) => m.type === 'shutdown')
|
||||
const shutdowns = worker.postMessage.mock.calls.filter(([m]) => m.type === 'shutdown')
|
||||
expect(shutdowns.length).toBe(1)
|
||||
|
||||
vi.advanceTimersByTime(10_000)
|
||||
const heartbeatsAfterStop = child.send.mock.calls.filter(([m]) => m.type === 'heartbeat')
|
||||
const heartbeatsAfterStop = worker.postMessage.mock.calls.filter(
|
||||
([m]) => m.type === 'heartbeat'
|
||||
)
|
||||
expect(heartbeatsAfterStop.length).toBe(3)
|
||||
})
|
||||
|
||||
it('passes test timing overrides to the worker', () => {
|
||||
process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS = '900'
|
||||
process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS = '100'
|
||||
workerState.instance = fakeWorker()
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
const options = workerState.calls[0][1] as {
|
||||
workerData: { timeoutMs: number; checkIntervalMs: number }
|
||||
}
|
||||
expect(options.workerData).toMatchObject({ timeoutMs: 900, checkIntervalMs: 100 })
|
||||
})
|
||||
|
||||
it('registers stop on will-quit', () => {
|
||||
const child = fakeChild()
|
||||
forkMock.mockReturnValue(child)
|
||||
workerState.instance = fakeWorker()
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
expect(appMock.on).toHaveBeenCalledWith('will-quit', expect.any(Function))
|
||||
})
|
||||
|
||||
it('returns null and stays inert when the fork itself fails', () => {
|
||||
forkMock.mockImplementation(() => {
|
||||
throw new Error('spawn failure')
|
||||
})
|
||||
it('stops heartbeat work when the worker exits', () => {
|
||||
const worker = fakeWorker()
|
||||
workerState.instance = worker
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
const exitListener = worker.once.mock.calls.find(([event]) => event === 'exit')?.[1]
|
||||
expect(exitListener).toEqual(expect.any(Function))
|
||||
exitListener()
|
||||
vi.advanceTimersByTime(6_000)
|
||||
expect(worker.postMessage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns null and stays inert when worker construction fails', () => {
|
||||
workerState.error = new Error('worker failure')
|
||||
expect(
|
||||
withPlatform('darwin', () => installMainThreadHangWatchdog({ userDataPath: '/ud' }))
|
||||
).toBeNull()
|
||||
|
|
|
|||
|
|
@ -1,19 +1,25 @@
|
|||
import { fork, type ChildProcess } from 'node:child_process'
|
||||
import { Worker } from 'node:worker_threads'
|
||||
import { app } from 'electron'
|
||||
import { resolveHangWatchdogEntryPath } from './hang-watchdog-entry-path'
|
||||
import { hangDetectionMarkerPath } from './hang-detection-marker'
|
||||
|
||||
const HEARTBEAT_INTERVAL_MS = 2_000
|
||||
import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path'
|
||||
import {
|
||||
HANG_WATCHDOG_CHECK_INTERVAL_MS,
|
||||
HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS,
|
||||
HANG_WATCHDOG_TIMEOUT_MS,
|
||||
type HangWatchdogWorkerData,
|
||||
type MainToHangWatchdogWorkerMessage
|
||||
} from './hang-watchdog-worker-protocol'
|
||||
|
||||
export type MainThreadHangWatchdogHandle = {
|
||||
stop: () => void
|
||||
child: ChildProcess
|
||||
worker: Worker
|
||||
}
|
||||
|
||||
function positiveTiming(value: string | undefined, fallback: number): number {
|
||||
const parsed = Number(value)
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
|
||||
}
|
||||
|
||||
// Why: macOS 26 scene-backed AppKit windows can deadlock the main thread inside
|
||||
// FrontBoardServices with no crash and no event loop left to self-report. A plain-Node sibling
|
||||
// process watches heartbeats and records the stall so the next launch can report it. It observes
|
||||
// only — see main-thread-hang-watchdog-entry.ts for why it does not kill the parent.
|
||||
export function installMainThreadHangWatchdog(options: {
|
||||
userDataPath: string
|
||||
}): MainThreadHangWatchdogHandle | null {
|
||||
|
|
@ -24,58 +30,57 @@ export function installMainThreadHangWatchdog(options: {
|
|||
if (!app.isPackaged && process.env.ORCA_HANG_WATCHDOG_FORCE !== '1') {
|
||||
return null
|
||||
}
|
||||
const entryPath = resolveHangWatchdogEntryPath(app.getAppPath(), app.isPackaged)
|
||||
let child: ChildProcess
|
||||
const workerPath = resolveHangWatchdogWorkerPath(app.getAppPath(), app.isPackaged)
|
||||
const workerData: HangWatchdogWorkerData = {
|
||||
parentPid: process.pid,
|
||||
markerPath: hangDetectionMarkerPath(options.userDataPath),
|
||||
timeoutMs: positiveTiming(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS, HANG_WATCHDOG_TIMEOUT_MS),
|
||||
checkIntervalMs: positiveTiming(
|
||||
process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS,
|
||||
HANG_WATCHDOG_CHECK_INTERVAL_MS
|
||||
)
|
||||
}
|
||||
let worker: Worker
|
||||
try {
|
||||
child = fork(entryPath, [], {
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
|
||||
env: {
|
||||
...process.env,
|
||||
ELECTRON_RUN_AS_NODE: '1',
|
||||
ORCA_HANG_WATCHDOG_PARENT_PID: String(process.pid),
|
||||
ORCA_HANG_WATCHDOG_MARKER_PATH: hangDetectionMarkerPath(options.userDataPath)
|
||||
}
|
||||
// Why: the worker survives an AppKit main-thread deadlock without another Electron process.
|
||||
worker = new Worker(workerPath, {
|
||||
name: 'orca-main-thread-hang-watchdog',
|
||||
workerData
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('[hang-watchdog] failed to fork watchdog process:', error)
|
||||
console.error('[hang-watchdog] failed to start watchdog worker:', error)
|
||||
return null
|
||||
}
|
||||
child.stderr?.on('data', (chunk: Buffer) => {
|
||||
console.error('[hang-watchdog]', String(chunk).trimEnd())
|
||||
worker.on('error', (error) => {
|
||||
console.error('[hang-watchdog] watchdog worker failed:', error)
|
||||
})
|
||||
// Why: the watchdog is a safety net — if it dies, run without it; a restart loop here must never affect the app.
|
||||
child.on('error', () => {})
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
if (child.connected) {
|
||||
try {
|
||||
child.send({ type: 'heartbeat' }, () => {})
|
||||
} catch {
|
||||
// Channel raced closed between the check and the send.
|
||||
}
|
||||
}
|
||||
}, HEARTBEAT_INTERVAL_MS)
|
||||
let stopped = false
|
||||
const postMessage = (message: MainToHangWatchdogWorkerMessage): void => {
|
||||
if (stopped && message.type === 'heartbeat') {
|
||||
return
|
||||
}
|
||||
try {
|
||||
worker.postMessage(message)
|
||||
} catch {
|
||||
// The worker already exited.
|
||||
}
|
||||
}
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
postMessage({ type: 'heartbeat' })
|
||||
}, HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS)
|
||||
const stop = (): void => {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
stopped = true
|
||||
clearInterval(heartbeatTimer)
|
||||
if (child.connected) {
|
||||
try {
|
||||
child.send({ type: 'shutdown' }, () => {})
|
||||
} catch {
|
||||
// Already disconnecting.
|
||||
}
|
||||
}
|
||||
// Why: disconnect guarantees the child observes parent shutdown and exits instead of misreading quit as a hang.
|
||||
try {
|
||||
child.disconnect()
|
||||
} catch {
|
||||
// Channel already closed.
|
||||
}
|
||||
postMessage({ type: 'shutdown' })
|
||||
}
|
||||
// Why: will-quit fires twice during quit; stop is idempotent.
|
||||
worker.once('exit', () => {
|
||||
stopped = true
|
||||
clearInterval(heartbeatTimer)
|
||||
})
|
||||
worker.unref()
|
||||
app.on('will-quit', stop)
|
||||
return { stop, child }
|
||||
return { stop, worker }
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue