From 0b71f3bfbaf52c1ebb76510c1184cd801714cda8 Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Sun, 19 Jul 2026 11:27:29 -0700 Subject: [PATCH] test(e2e): prove the terminal daemon survives a main-process crash on Windows (#7742) (#9311) * test(e2e): prove the terminal daemon survives a main-process crash on Windows (#7742) Add a win-crash-survival e2e harness (sibling to win-update-e2e) that force-kills ONLY the packaged app's real Electron main (resolved via app.evaluate -> process.pid, /F no /T) and asserts the detached orca-terminal-daemon.exe plus its ConPTY shell survive with no pwsh 0xE9 FailFast, then that a relaunch re-adopts the SAME daemon and the reattached UI binds to the SAME survivor shell (proved via a per-shell env sentinel read back through the restored terminal). This guards the #7742 fix (standalone relocated daemon that outlives main death) against regression. A directional `--expect orphaned` profile fails on a fixed build, keeping the survival assertions honest. Windows-only; reuses win-update-e2e app-driver/daemon-process modules. * test(e2e): harden Windows crash-survival proof * test(ci): keep crash survival gate durable * test(e2e): tolerate restart hydration navigation * test(e2e): prove exact shell input after crash * perf(ci): avoid crash harness installer rebuilds * test(ci): harden crash survival evidence and cost * test(e2e): fail closed on authoritative crash target * test(e2e): fail closed on crash liveness evidence --------- Co-authored-by: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> --- .github/workflows/win-crash-survival-e2e.yml | 195 ++++++++ config/reliability-gates.jsonc | 94 +++- .../scripts/win-crash-survival-e2e.test.mjs | 227 +++++++++ package.json | 1 + tests/e2e/helpers/orca-restart.ts | 67 ++- .../e2e/orca-restart-navigation.unit.test.ts | 20 + tools/win-crash-survival-e2e/README.md | 114 +++++ tools/win-crash-survival-e2e/cli-args.mjs | 130 ++++++ .../crash-assertions.mjs | 101 ++++ tools/win-crash-survival-e2e/crash-step.mjs | 121 +++++ .../daemon-identity.mjs | 28 ++ .../win-crash-survival-e2e/reattach-proof.mjs | 19 + tools/win-crash-survival-e2e/run.mjs | 432 ++++++++++++++++++ tools/win-update-e2e/README.md | 6 + tools/win-update-e2e/app-driver.mjs | 137 +++++- tools/win-update-e2e/assertions.mjs | 7 +- tools/win-update-e2e/daemon-processes.mjs | 18 +- tools/win-update-e2e/powershell-runner.mjs | 5 + 18 files changed, 1665 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/win-crash-survival-e2e.yml create mode 100644 config/scripts/win-crash-survival-e2e.test.mjs create mode 100644 tests/e2e/orca-restart-navigation.unit.test.ts create mode 100644 tools/win-crash-survival-e2e/README.md create mode 100644 tools/win-crash-survival-e2e/cli-args.mjs create mode 100644 tools/win-crash-survival-e2e/crash-assertions.mjs create mode 100644 tools/win-crash-survival-e2e/crash-step.mjs create mode 100644 tools/win-crash-survival-e2e/daemon-identity.mjs create mode 100644 tools/win-crash-survival-e2e/reattach-proof.mjs create mode 100644 tools/win-crash-survival-e2e/run.mjs diff --git a/.github/workflows/win-crash-survival-e2e.yml b/.github/workflows/win-crash-survival-e2e.yml new file mode 100644 index 000000000..e4c388ceb --- /dev/null +++ b/.github/workflows/win-crash-survival-e2e.yml @@ -0,0 +1,195 @@ +name: Windows Crash-Survival E2E + +# Why: proves the detached daemon-host relocation makes terminal sessions SURVIVE +# a CRASH of Orca's main process (GitHub #7742), the companion guarantee to the +# update-survival harness. Builds an (unsigned) installer FROM THIS BRANCH, +# silent-installs it, opens a terminal, force-kills ONLY the app main (no +# tree-kill), and asserts the daemon + shell stay alive, a relaunch adopts the +# same daemon and shell, and post-crash input causes no pwsh FailFast. Targeted +# pull requests keep it as a durable regression gate. A CI runner is the only +# safe place to install — see the relocation post-mortem. + +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + paths: + - '.github/workflows/win-crash-survival-e2e.yml' + - 'package.json' + - 'pnpm-lock.yaml' + - 'pnpm-workspace.yaml' + - 'electron.vite.config.ts' + - 'build-plugins/**' + - 'config/electron-builder.config.cjs' + - 'config/patches/**' + - 'config/scripts/ensure-native-runtime.mjs' + - 'config/scripts/rebuild-native-deps.mjs' + - 'native/**' + - 'resources/win32/**' + - 'src/main/daemon/**' + - 'src/main/index.ts' + - 'src/main/ipc/pty*.ts' + - 'src/main/persistence.ts' + - 'src/main/providers/**' + - 'src/main/pty/**' + - 'src/main/startup/first-window-startup-services.ts' + - 'src/main/window/attach-main-window-services.ts' + - 'src/preload/**' + - 'src/renderer/src/App.tsx' + - 'src/renderer/src/components/terminal-pane/**' + - 'src/renderer/src/hooks/useIpcEvents.ts' + - 'src/renderer/src/lib/pane-manager/**' + - 'src/renderer/src/lib/session-write-subscriber.ts' + - 'src/renderer/src/lib/workspace-session-host-persistence.ts' + - 'src/renderer/src/store/slices/terminals.ts' + - 'src/shared/pty-session-id-format.ts' + - 'tools/win-crash-survival-e2e/**' + - 'tools/win-update-e2e/**' + # Why: source tests and benchmarks do not change the packaged artifact; + # their own verify jobs cover them without spending a Windows build slot. + - '!src/**/*.test.*' + - '!src/**/*.bench.*' + workflow_dispatch: + inputs: + expect: + description: Expected outcome profile + required: true + type: choice + default: survival + options: + - survival + - orphaned + +permissions: + contents: read + +concurrency: + group: win-crash-survival-e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + crash-survival: + name: crash-survival (packaged build) + runs-on: windows-2022 + timeout-minutes: 50 + env: + EXPECT: ${{ inputs.expect || 'survival' }} + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + # This job only builds and runs the harness locally; it never pushes. + persist-credentials: false + + # Why: setup-node can restore pnpm's content-addressed store only after + # the pnpm binary exists, avoiding repeat dependency downloads per run. + - name: Setup pnpm + uses: pnpm/action-setup@v6 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version-file: package.json + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Why: cache the built installer by production build inputs, excluding + # tests/reliability metadata so harness-only edits skip electron-builder. + # The daemon relocation lives under src/, so product changes still rebuild. + - name: Cache branch installer + id: cache-installer + uses: actions/cache@v4 + with: + path: dist/orca-windows-setup.exe + key: >- + crash-survival-installer-${{ hashFiles( + 'src/**', + '!src/**/*.test.*', + '!src/**/*.bench.*', + 'config/**', + '!config/**/*.test.*', + '!config/reliability-gates.jsonc', + '!config/max-lines-baseline.txt', + '!config/vitest.config.ts', + 'build-plugins/**', + 'native/**', + 'resources/**', + 'electron.vite.config.ts', + 'tsconfig.json', + 'vite.web.config.ts', + '.npmrc', + 'package.json', + 'pnpm-lock.yaml', + 'pnpm-workspace.yaml' + ) }} + + # Why: production edits miss the installer cache by design, but Electron + # and NSIS downloads are lockfile-owned and need not be fetched again. + - name: Cache electron-builder downloads + if: steps.cache-installer.outputs.cache-hit != 'true' + uses: actions/cache@v5 + with: + path: | + ~\AppData\Local\electron\Cache + ~\AppData\Local\electron-builder\Cache + key: crash-survival-electron-builder-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + crash-survival-electron-builder- + + - name: Build Windows installer (unsigned) + if: steps.cache-installer.outputs.cache-hit != 'true' + run: | + node config/scripts/ensure-native-runtime.mjs --runtime=electron + pnpm run build:desktop + pnpm exec electron-builder --config config/electron-builder.config.cjs --win --publish never + + # Why: silent-install the branch build so the crash harness has a packaged + # Orca.exe to drive. A clean CI runner installs to the default per-user + # location (%LOCALAPPDATA%\Programs\Orca), which the harness auto-locates. + - name: Silent-install branch build + shell: pwsh + run: | + $exe = "dist/orca-windows-setup.exe" + if (-not (Test-Path $exe)) { throw "Installer not found at $exe" } + Start-Process -FilePath $exe -ArgumentList '/S' -Wait + $installed = Join-Path $env:LOCALAPPDATA 'Programs/Orca/Orca.exe' + if (-not (Test-Path $installed)) { + $installed = Join-Path $env:LOCALAPPDATA 'Programs/orca/Orca.exe' + } + if (-not (Test-Path $installed)) { throw "Installed Orca.exe not found after silent install" } + "ORCA_EXE=$installed" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + + # Why: crash ONLY the app main and assert the relocated daemon + its shell + # survive, then that relaunch adopts them and input causes no pwsh FailFast. + - name: Run crash-survival harness + id: harness + shell: pwsh + env: + ORCA_E2E_DIAG_DIR: artifacts/diag + run: | + New-Item -ItemType Directory -Force artifacts | Out-Null + $log = "artifacts/crash-survival-output.log" + node tools/win-crash-survival-e2e/run.mjs ` + --expect "$env:EXPECT" ` + --exe-path "$env:ORCA_EXE" ` + --soak-seconds 8 2>&1 | Tee-Object -FilePath $log + exit $LASTEXITCODE + + - name: Upload crash-survival output + if: always() + uses: actions/upload-artifact@v7 + with: + name: win-crash-survival-output + path: | + artifacts/crash-survival-output.log + artifacts/diag/** + retention-days: 7 + if-no-files-found: warn diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 9846d201c..3c49afcb5 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "updatedAt": "2026-07-17", + "updatedAt": "2026-07-19", "policy": { "maturityLevels": [ "experimental", @@ -1693,6 +1693,98 @@ ], "demotionRule": "Cannot promote without deterministic user-visible oracle, failure artifacts, and stable runtime history." }, + { + "id": "terminal-session.windows-main-crash-survival", + "title": "A Windows main-process crash preserves and reattaches daemon terminals", + "maturity": "experimental", + "protection": "partial", + "owner": "terminal-platform", + "layer": "windows-packaged-electron-daemon", + "surfaces": [ + "Electron main-process crash", + "detached terminal daemon", + "ConPTY shell survival", + "packaged relaunch reattach" + ], + "platforms": ["windows"], + "providers": ["daemon"], + "coveredPlatforms": ["windows"], + "coveredProviders": ["daemon"], + "coverageNotes": "A packaged Windows CI run force-kills the launched instance's real Electron main without tree-killing, then proves the same scoped daemon and interactive PowerShell survive, no FailFast event is observed, and relaunch input reaches the same shell. Focused cross-platform unit tests fail closed on unavailable event-log or PID-liveness evidence, stale or ambiguous daemon identity, and incomplete reattach assertions.", + "motivatingLinks": [ + "https://github.com/stablyai/orca/issues/7742", + "https://github.com/stablyai/orca/pull/9311" + ], + "invariant": "On packaged Windows, abrupt death of Orca's Electron main process must not terminate or replace the userData-scoped terminal daemon or its live ConPTY shell, and a relaunch must adopt that exact daemon and route terminal input to that exact surviving shell without a PowerShell 0xE9 FailFast.", + "oracle": "Resolve the launched instance's real main PID from inside Electron, force-kill only that PID, require it to die, require one command-line-scoped daemon PID and the stamped interactive shell PID to remain live, relaunch with persisted state, require the daemon PID to remain identical, read the exact shell PID and a per-shell environment sentinel back through the exact restored tab, then require a successful Windows Application event-log query with zero matching pwsh FailFast events across the full crash-to-input window.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts config/scripts/win-crash-survival-e2e.test.mjs", + "node tools/win-crash-survival-e2e/run.mjs --expect survival --exe-path \"$env:ORCA_EXE\" --soak-seconds 8" + ], + "testFiles": [ + "config/scripts/win-crash-survival-e2e.test.mjs", + "tools/win-crash-survival-e2e/run.mjs" + ], + "assertionRefs": [ + { + "file": "config/scripts/win-crash-survival-e2e.test.mjs", + "assertions": [ + "survival requires the crash antecedent, daemon and shell liveness, unchanged daemon identity, zero FailFast events, and same-shell reattach", + "event-log query failure cannot be converted into zero FailFast events", + "malformed event-log and PID-liveness evidence fails closed", + "stale, missing, or ambiguous userData-scoped daemon identity fails closed", + "the packaged survival proof remains wired to targeted pull requests without a duplicate branch-push run" + ] + }, + { + "file": "tools/win-crash-survival-e2e/run.mjs", + "assertions": [ + "force-killing only the real Electron main leaves the exact scoped daemon and stamped interactive shell alive", + "packaged relaunch adopts the unchanged daemon and reads the survivor shell's environment sentinel through the restored terminal" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-07-18", + "runner": "ci", + "platform": "windows", + "command": "node tools/win-crash-survival-e2e/run.mjs --expect survival --exe-path \"$env:ORCA_EXE\" --soak-seconds 8", + "result": "passed", + "durationSeconds": 61, + "summary": "The packaged branch build's real main died; the same daemon and shell PIDs survived; the event-log scan found zero FailFast events; relaunch adopted the unchanged daemon; and terminal input read the survivor shell sentinel back." + } + ], + "runtimeBudget": { + "p95Seconds": 120, + "scope": "installed-app crash/relaunch harness, excluding installer build and dependency setup" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "One green packaged Windows CI run is recorded; the durable pull-request trigger must accumulate repeated history before promotion." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "The inverse orphaned profile failed against the fixed packaged build because the daemon stayed live and no FailFast occurred. A genuinely pre-relocation packaged build has not yet been retained as a CI red fixture." + }, + "performanceBudget": { + "required": true, + "evidence": "Production code is unchanged. The harness has bounded 500ms liveness polls, one 8-second crash soak, two scoped daemon identity scans plus teardown rediscovery, a 60-second cap on every synchronous PowerShell probe, a 5-second cap on Electron main-PID resolution, cleared and unreferenced close deadlines, a 50-minute job timeout, concurrency cancellation, and targeted production-path filtering that excludes source tests and benchmarks. The installer cache is keyed by every production build input; the job also restores the pnpm store and restores Electron/NSIS downloads only when an installer rebuild is required. The measured packaged harness completed in 61 seconds." + }, + "promotionCriteria": [ + "Collect at least 100 consecutive targeted Windows PR or soak passes over 14 days with zero unexplained flakes.", + "Retain a pre-relocation packaged red fixture or equivalent fault injection that makes the survival profile fail for daemon death and shell loss.", + "Keep the live oracle fail-closed for crash delivery, event-log access, daemon identity, and same-shell reattach." + ], + "knownGaps": [ + "The packaged journey proves one terminal end to end; concurrent restoration across the user's multi-terminal, multi-worktree layout is not exercised live.", + "The gate covers a Windows local daemon-backed PowerShell terminal; WSL-backed shells are not exercised.", + "SSH, remote-runtime, relay, mobile, macOS, and Linux paths are unaffected by the Windows relocated ConPTY host contract and are not exercised here.", + "The Application event-log query is machine-wide, so an unrelated PowerShell crash on a non-isolated runner can false-fail the gate.", + "Standalone daemon self-crash and renderer-only crash containment remain separate invariants." + ], + "demotionRule": "Keep experimental or quarantine with a linked harness defect if the Windows job flakes; never weaken the crash, identity, event-log, or same-shell reattach oracle to obtain a pass." + }, { "id": "terminal-platform.windows-conpty-liveness", "title": "Windows ConPTY terminals stay input-live, render-live, and geometry-live", diff --git a/config/scripts/win-crash-survival-e2e.test.mjs b/config/scripts/win-crash-survival-e2e.test.mjs new file mode 100644 index 000000000..c59f98ffb --- /dev/null +++ b/config/scripts/win-crash-survival-e2e.test.mjs @@ -0,0 +1,227 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it, vi } from 'vitest' +import { parseArgs } from '../../tools/win-crash-survival-e2e/cli-args.mjs' +import { buildCrashAssertions } from '../../tools/win-crash-survival-e2e/crash-assertions.mjs' +import { scanPwshFailFast } from '../../tools/win-crash-survival-e2e/crash-step.mjs' +import { selectScopedDaemon } from '../../tools/win-crash-survival-e2e/daemon-identity.mjs' +import { + reattachSentinelMatches, + selectCreatedTabId +} from '../../tools/win-crash-survival-e2e/reattach-proof.mjs' +import { quotePowerShellLiteral } from '../../tools/win-update-e2e/powershell-runner.mjs' +import { closeApp, resolveElectronMainPid } from '../../tools/win-update-e2e/app-driver.mjs' +import { isPidAlive } from '../../tools/win-update-e2e/daemon-processes.mjs' + +describe('win-crash-survival-e2e proof contracts', () => { + it('keeps the packaged proof wired as a targeted pull-request gate', () => { + const workflow = readFileSync('.github/workflows/win-crash-survival-e2e.yml', 'utf8') + expect(workflow).toMatch(/^ pull_request:/m) + expect(workflow).not.toMatch(/^ push:/m) + expect(workflow).toContain("- 'src/main/daemon/**'") + expect(workflow).toContain("- 'src/main/index.ts'") + expect(workflow).toContain("- 'src/main/ipc/pty*.ts'") + expect(workflow).toContain("- 'src/main/startup/first-window-startup-services.ts'") + expect(workflow).toContain("- 'src/main/window/attach-main-window-services.ts'") + expect(workflow).toContain("- 'src/preload/**'") + expect(workflow).toContain("- 'src/renderer/src/components/terminal-pane/**'") + expect(workflow).toContain("- 'src/renderer/src/store/slices/terminals.ts'") + expect(workflow).toContain("- '!src/**/*.test.*'") + expect(workflow).toContain("- '!src/**/*.bench.*'") + expect(workflow).toContain('--expect "$env:EXPECT"') + expect(workflow).toContain('exit $LASTEXITCODE') + expect(workflow).toContain("'!config/**/*.test.*'") + expect(workflow).toContain("'!src/**/*.test.*'") + expect(workflow).toContain("'!src/**/*.bench.*'") + expect(workflow).toContain("'!config/reliability-gates.jsonc'") + expect(workflow).toContain("'resources/**'") + expect(workflow).toContain('cache: pnpm') + expect(workflow.indexOf('- name: Setup Node.js')).toBeGreaterThan( + workflow.indexOf('- name: Setup pnpm') + ) + expect(workflow).toContain("if: steps.cache-installer.outputs.cache-hit != 'true'") + expect(workflow).toContain('crash-survival-electron-builder-') + }) + + it('requires the full survival oracle, including daemon identity and reattach', () => { + const base = { + profile: 'survival', + mainDied: true, + daemonAliveAfterCrash: true, + shellAliveAfterCrash: true, + failFastEvents: [], + preDaemonPid: 101, + postDaemonPid: 101, + postDaemonAlive: true, + reattachProven: true + } + expect(buildCrashAssertions(base).every((entry) => entry.pass)).toBe(true) + expect( + buildCrashAssertions({ ...base, postDaemonPid: 202 }).find((entry) => + entry.name.startsWith('relaunch adopts') + )?.pass + ).toBe(false) + expect( + buildCrashAssertions({ ...base, reattachProven: false }).find((entry) => + entry.name.startsWith('reattached UI') + )?.pass + ).toBe(false) + }) + + it('scans for FailFast only after the post-crash input probe', () => { + const harness = readFileSync('tools/win-crash-survival-e2e/run.mjs', 'utf8') + const scanIndex = harness.indexOf('const { events: failFastEvents }') + const probeIndex = harness.indexOf('reattachProven = await proveReattachedShell') + expect(scanIndex).not.toBe(-1) + expect(probeIndex).not.toBe(-1) + expect(scanIndex).toBeGreaterThan(probeIndex) + }) + + it('fails closed when the Windows event log query fails', () => { + let command = '' + expect(() => + scanPwshFailFast(1234, (received) => { + command = received + return { code: 1, stdout: '', stderr: 'access denied', error: null } + }) + ).toThrow('pwsh-failfast scan failed (exit 1): access denied') + expect(command).toContain('-ErrorAction Stop') + expect(command).toContain('NoMatchingEventsFound*') + }) + + it('accepts an empty event result only from the serialized evidence envelope', () => { + expect( + scanPwshFailFast(1234, () => ({ + code: 0, + stdout: '{"events":[]}', + stderr: '', + error: null + })) + ).toEqual({ events: [] }) + expect(() => + scanPwshFailFast(1234, () => ({ code: 0, stdout: '', stderr: '', error: null })) + ).toThrow('pwsh-failfast scan returned no JSON output') + expect(() => + scanPwshFailFast(1234, () => ({ code: 0, stdout: '{}', stderr: '', error: null })) + ).toThrow('without an events envelope') + }) + + it('fails closed when PID liveness evidence is unavailable', () => { + expect(isPidAlive(42, () => ({ code: 0, stdout: 'alive\n', stderr: '', error: null }))).toBe( + true + ) + expect(isPidAlive(42, () => ({ code: 0, stdout: 'dead\n', stderr: '', error: null }))).toBe( + false + ) + expect(() => + isPidAlive(42, () => ({ code: 1, stdout: '', stderr: 'access denied', error: null })) + ).toThrow('PID liveness probe failed (exit 1): access denied') + expect(() => isPidAlive(42, () => ({ code: 0, stdout: '', stderr: '', error: null }))).toThrow( + 'PID liveness probe returned an invalid state' + ) + }) + + it('uses the scoped live process as daemon authority', () => { + expect( + selectScopedDaemon( + [{ pid: 42, appVersion: '1.2.3' }], + [{ pid: 42, commandLine: 'daemon-entry.js --socket scoped' }] + ) + ).toEqual({ pid: 42, appVersion: '1.2.3' }) + expect(() => + selectScopedDaemon( + [{ pid: 41, appVersion: 'stale' }], + [{ pid: 42, commandLine: 'daemon-entry.js --socket scoped' }] + ) + ).toThrow('daemon PID file does not match scoped live daemon 42') + expect(() => selectScopedDaemon([], [])).toThrow('expected exactly one') + expect(() => selectScopedDaemon([], [{ pid: 1 }, { pid: 2 }])).toThrow('expected exactly one') + }) + + it('rejects CLI typos and duplicate value flags before launching', () => { + const baseArgs = ['--expect', 'survival', '--exe-path', process.execPath] + expect(parseArgs(baseArgs).errors).toEqual([]) + expect(parseArgs([...baseArgs, '--exe-pathh', process.execPath]).errors).toContain( + 'Unknown argument: --exe-pathh' + ) + expect(parseArgs([...baseArgs, '--expect', 'orphaned']).errors).toContain( + 'Duplicate argument: --expect' + ) + }) + + it('quotes apostrophes in generated PowerShell path literals', () => { + expect(quotePowerShellLiteral("C:\\Users\\O'Brien\\shell.pid")).toBe( + "'C:\\Users\\O''Brien\\shell.pid'" + ) + }) + + it('targets exactly the terminal tab created before the crash', () => { + expect(selectCreatedTabId(['agent-tab'], ['agent-tab', 'terminal-tab'])).toBe('terminal-tab') + expect(() => selectCreatedTabId(['agent-tab'], ['agent-tab'])).toThrow( + 'expected exactly one created terminal tab, found 0' + ) + expect(() => selectCreatedTabId([], ['first', 'second'])).toThrow( + 'expected exactly one created terminal tab, found 2' + ) + }) + + it('requires both the per-shell canary and exact survivor pid', () => { + expect(reattachSentinelMatches('1660|canary\r\n', 'canary', 1660)).toBe(true) + expect(reattachSentinelMatches('1770|canary', 'canary', 1660)).toBe(false) + expect(reattachSentinelMatches('1660|other', 'canary', 1660)).toBe(false) + expect(reattachSentinelMatches('1660|canary|extra', 'canary', 1660)).toBe(false) + }) + + it('requires the real packaged main for the crash proof but permits fallback cleanup', async () => { + const harness = readFileSync('tools/win-crash-survival-e2e/run.mjs', 'utf8') + expect(harness).toContain( + 'resolveElectronMainPid(session.app, { allowLauncherFallback: false })' + ) + expect( + await resolveElectronMainPid({ + evaluate: async () => 222, + process: () => ({ pid: 111 }) + }) + ).toBe(222) + const unavailableApp = { + evaluate: async () => { + throw new Error('main unavailable') + }, + process: () => ({ pid: 111 }) + } + expect( + await resolveElectronMainPid(unavailableApp, { allowLauncherFallback: false }) + ).toBeNull() + expect(await resolveElectronMainPid(unavailableApp)).toBe(111) + }) + + it('bounds main PID resolution when the Electron connection is wedged', async () => { + vi.useFakeTimers() + try { + const result = resolveElectronMainPid( + { + evaluate: () => new Promise(() => {}), + process: () => ({ pid: 333 }) + }, + { timeoutMs: 20 } + ) + await vi.advanceTimersByTimeAsync(20) + await expect(result).resolves.toBe(333) + } finally { + vi.useRealTimers() + } + }) + + it('releases the close deadline after a successful app close', async () => { + vi.useFakeTimers() + try { + await closeApp({ + evaluate: async () => 444, + process: () => ({ pid: 333 }), + close: async () => {} + }) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/package.json b/package.json index 170bfc995..5035c6213 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,7 @@ "test:e2e:ssh-docker-watcher-isolation": "node config/scripts/run-ssh-docker-watcher-isolation-e2e.mjs", "test:e2e:source-control-scale": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/source-control-large-file-count.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "win-update-e2e": "node tools/win-update-e2e/run.mjs", + "win-crash-survival-e2e": "node tools/win-crash-survival-e2e/run.mjs", "test:e2e:ssh-codex-artifacts-repro": "node config/scripts/run-ssh-codex-artifacts-repro-e2e.mjs", "test:e2e:headful": "pnpm run ensure:electron-runtime && npx playwright test --config tests/playwright.config.ts --project electron-headful", "test:e2e:computer": "vitest run --config tests/e2e/vitest.config.ts", diff --git a/tests/e2e/helpers/orca-restart.ts b/tests/e2e/helpers/orca-restart.ts index 742d34ffd..06d500066 100644 --- a/tests/e2e/helpers/orca-restart.ts +++ b/tests/e2e/helpers/orca-restart.ts @@ -141,23 +141,25 @@ export async function attachRepoAndOpenTerminal(page: Page, repoPath: string): P await expect .poll( () => - page.evaluate(async (repoId) => { - const store = window.__store - if (!store) { - return false - } - // Why: repos.add emits a concurrent refresh whose generation can - // supersede this fetch; poll until either refresh publishes the repo. - await store.getState().fetchRepos() - const repo = store.getState().repos.find((candidate) => candidate.id === repoId) - if (!repo) { - return false - } - // Why: this restart fixture uses the global e2e repo, whose seeded Git - // worktree is external to Orca's workspace root after the visibility rollout. - await store.getState().updateRepo(repo.id, { externalWorktreeVisibility: 'show' }) - return true - }, repoId), + readRestartRendererState(() => + page.evaluate(async (repoId) => { + const store = window.__store + if (!store) { + return false + } + // Why: repos.add emits a concurrent refresh whose generation can + // supersede this fetch; poll until either refresh publishes the repo. + await store.getState().fetchRepos() + const repo = store.getState().repos.find((candidate) => candidate.id === repoId) + if (!repo) { + return false + } + // Why: this restart fixture uses the global e2e repo, whose seeded Git + // worktree is external to Orca's workspace root after the visibility rollout. + await store.getState().updateRepo(repo.id, { externalWorktreeVisibility: 'show' }) + return true + }, repoId) + ), { timeout: 30_000, message: `attachRepoAndOpenTerminal: expected e2e repo to be loaded: ${repoPath}` @@ -178,14 +180,16 @@ export async function attachRepoAndOpenTerminal(page: Page, repoPath: string): P await expect .poll( async () => - page.evaluate(async (repoId) => { - const store = window.__store - if (!store) { - return false - } - await store.getState().fetchWorktrees(repoId) - return (store.getState().worktreesByRepo[repoId]?.length ?? 0) > 0 - }, repoId), + readRestartRendererState(() => + page.evaluate(async (repoId) => { + const store = window.__store + if (!store) { + return false + } + await store.getState().fetchWorktrees(repoId) + return (store.getState().worktreesByRepo[repoId]?.length ?? 0) > 0 + }, repoId) + ), { timeout: 15_000, message: 'attachRepoAndOpenTerminal: seeded worktree never surfaced in the store' @@ -217,6 +221,19 @@ export async function attachRepoAndOpenTerminal(page: Page, repoPath: string): P return worktreeId } +export async function readRestartRendererState(read: () => Promise): Promise { + try { + return await read() + } catch (error) { + // Why: initial hydration can replace the renderer document; the enclosing + // state poll must retry that transition without hiding other failures. + if (error instanceof Error && error.message.includes('Execution context was destroyed')) { + return null + } + throw error + } +} + function isValidGitRepo(repoPath: string): boolean { if (!repoPath || !existsSync(repoPath)) { return false diff --git a/tests/e2e/orca-restart-navigation.unit.test.ts b/tests/e2e/orca-restart-navigation.unit.test.ts new file mode 100644 index 000000000..5dcd02a4c --- /dev/null +++ b/tests/e2e/orca-restart-navigation.unit.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import { readRestartRendererState } from './helpers/orca-restart' + +describe('restart renderer state polling', () => { + it('treats document replacement as pending state', async () => { + await expect( + readRestartRendererState(async () => { + throw new Error('Execution context was destroyed, most likely because of a navigation.') + }) + ).resolves.toBeNull() + }) + + it('does not hide non-navigation renderer failures', async () => { + await expect( + readRestartRendererState(async () => { + throw new Error('fetchWorktrees failed') + }) + ).rejects.toThrow('fetchWorktrees failed') + }) +}) diff --git a/tools/win-crash-survival-e2e/README.md b/tools/win-crash-survival-e2e/README.md new file mode 100644 index 000000000..edfb57048 --- /dev/null +++ b/tools/win-crash-survival-e2e/README.md @@ -0,0 +1,114 @@ +# win-crash-survival-e2e — packaged crash-survival proof harness + +**Windows only.** Proves that a **crash of Orca's main process** does not orphan +open terminal PTYs — the regression behind +[GitHub #7742](https://github.com/stablyai/orca/issues/7742) — +with machine-checkable assertions against an **already-installed, packaged** +`Orca.exe`. + +## Why this exists + +On Windows, when Orca's main/renderer process crashed, open terminal PTYs were +orphaned and PowerShell hard-crashed with a `0xE9` "No process is on the other +end of the pipe" `FailFast`. Root cause: the terminal **daemon** (which hosts the +ConPTYs) died together with the main process, severing the console pipe. + +The fix re-architected the daemon into a standalone, relocated +`orca-terminal-daemon.exe` (see +[`src/main/daemon/daemon-host-relocation.ts`](../../src/main/daemon/daemon-host-relocation.ts)) +that is spawned **detached** and **survives main-process death**. + +There is already a harness proving the daemon survives a Windows **update** +([`tools/win-update-e2e`](../win-update-e2e/README.md)). This harness proves the +daemon survives a **crash** of the main process, so that guarantee can't silently +regress. It **reuses win-update-e2e's shared modules** (app driver, daemon +discovery, onboarding seed, PowerShell runner, platform guard, table renderer) +and adds only the crash step + its assertions. + +## What it does + +1. **Launch** the installed `Orca.exe` under an isolated `userData` dir + (`ORCA_E2E_USER_DATA_DIR`), seeded with a fresh profile (onboarding dismissed + plus one throwaway git repo), then open a plain terminal tab (the seeded + workspace opens an agent tab, not a bare shell). +2. **Stamp the interactive shell** — typing DIRECTLY into it (not a nested + `powershell`), set a per-shell env sentinel `ORCA_CRASH_SENTINEL=` and + record the shell's own `$PID`. The command finishes fast, leaving the shell + idle at a live PSReadLine prompt — the exact state that FailFasts with `0xE9` + on a broken build. +3. **Record** the daemon PID and the real Electron **main** PID (resolved via + `app.evaluate(() => process.pid)` — the launched instance's own main, not the + launcher stub `app.process()` returns, and not a machine-wide scan). +4. **Crash** — `taskkill /F /PID ` with **no `/T`** and **no + graceful close**. This kills ONLY the real main of the instance this harness + launched, never a scanned or image-named process, and never the process tree — + a real crash does not tree-kill the detached daemon. Then **prove the crash + landed** (poll the main PID until dead). +5. **Assert survival**: the daemon PID and the same interactive shell PID are + still alive after the crash soak. +6. **Relaunch** (same `userData`, no reseed) and assert the daemon PID is + **unchanged** (the new main **adopts** the surviving daemon instead of forking + a new one) and that the reattached UI is bound to the **same survivor shell** — + a bounded, readiness-aware command on the exact restored tab reads back both + `ORCA_CRASH_SENTINEL` and the shell's `$PID`, which a freshly re-spawned shell + would not carry. +7. **Scan the full crash-to-input window** and require the Windows **Application + event log** to contain **zero** pwsh `FailFast` / `0xE9` events (matched by + crash-reporter provider+id, not fragile Message text). Scanning after the + reattach keystroke catches shells that fail only on their next console read. +8. **Teardown** — close the relaunched app, then kill this run's scoped daemon + **tree** (re-discovered fresh via `findDaemonProcesses(userData)`, which the + surviving shell is a descendant of) and remove the temp profile. It never kills + a PID captured earlier in the run (a recycled PID could hit an innocent + process), never installs/uninstalls, and never touches any other Orca on the box. + +Exit code is `0` when every non-informational assertion passes, else `1` (`2` for +a CLI usage error). + +## Usage + +```powershell +pnpm win-crash-survival-e2e --expect survival +# or explicitly point at an installed exe: +node tools/win-crash-survival-e2e/run.mjs --expect survival --exe-path "C:\Users\\AppData\Local\Programs\orca\Orca.exe" +``` + +### Flags + +| Flag | Meaning | +| -------------------- | ---------------------------------------------------------------------------------------------- | +| `--expect ` | Assertion profile (required): `survival` or `orphaned` (see below) | +| `--exe-path ` | Installed `Orca.exe` to drive (default: per-user install under `%LOCALAPPDATA%\Programs\Orca`) | +| `--soak-seconds ` | Post-crash observation window before relaunch (default `8`) | +| `--keep-profile` | Skip temp-profile cleanup (debugging) | + +### Profiles + +- **`survival`** — the fixed behavior and the baseline that must keep passing: + the main crash actually lands, yet the daemon + the same interactive shell PID + survive, zero pwsh `FailFast` events fire, a relaunch **adopts** the same daemon + PID, and the reattached UI reads back the survivor shell's env sentinel. +- **`orphaned`** — the directional inverse describing the **old broken #7742** + behavior. Daemon death is the **primary** signal (deterministic); pwsh + `FailFast` / `0xE9` is **secondary** — faithful only because the shell is left + idle at a live PSReadLine prompt (which queries the severed console). On a fixed + build this profile is **expected to fail**, proving the survival assertions are + not vacuous. It is **not exercised in CI** (`workflow_dispatch` is unavailable on + a non-default branch) and the `0xE9` only reproduces on a genuinely broken build. + +## Safety + +- **Never installs, updates, or uninstalls anything** — it only launches an + existing exe against an isolated `userData` dir. +- The crash kills **only** the real Electron main of the instance this harness + launched — resolved via `app.evaluate(() => process.pid)` (not the launcher stub + `app.process()` returns) — `/F` with **no `/T`**. It never `taskkill`s by image + name or a scanned pid, so a developer's live Orca (a different `userData`, out of + scope) is untouched. +- **Teardown never kills a PID captured earlier in the run** (a recycled PID could + hit an innocent process): daemon cleanup re-discovers this run's daemon fresh via + a `userData`-scoped `findDaemonProcesses`, and the surviving shell is torn down + as a descendant of that daemon tree. +- Daemon discovery is **scoped** to this run's `userData` path, so it never + matches the many other daemons a dev box or CI runner can host. +- Windows-only (`assertWin32`); it no-ops with a clear error off win32. diff --git a/tools/win-crash-survival-e2e/cli-args.mjs b/tools/win-crash-survival-e2e/cli-args.mjs new file mode 100644 index 000000000..535d8b65c --- /dev/null +++ b/tools/win-crash-survival-e2e/cli-args.mjs @@ -0,0 +1,130 @@ +// Argument parsing for `node run.mjs` (crash-survival harness). +// +// Unlike win-update-e2e, this harness installs nothing: it drives an ALREADY +// installed, packaged Orca.exe. So its only source is `--exe-path` (defaulting +// to the per-user install location). Exactly one profile (--expect) is required. + +import { existsSync } from 'node:fs' +import path from 'node:path' +import { locateInstalledExe } from '../win-update-e2e/installer-steps.mjs' + +const VALID_PROFILES = new Set(['survival', 'orphaned']) +const VALUE_FLAGS = new Set(['--expect', '--exe-path', '--soak-seconds']) +const BOOLEAN_FLAGS = new Set(['--keep-profile']) + +const USAGE = ` +win-crash-survival-e2e — packaged crash-survival proof harness (Windows only) + +Proves that force-killing ONLY Orca's main process (a real crash, no tree-kill) +leaves the detached terminal daemon + its pwsh shell alive, with no pwsh FailFast +(0xE9 "No process is on the other end of the pipe"), and that a relaunch ADOPTS +the surviving daemon instead of forking a new one. See #7742. + +Usage: + node tools/win-crash-survival-e2e/run.mjs --expect [--exe-path ] [options] + +Required: + --expect Assertion profile: + survival = the fixed behavior (daemon + shell survive + the main crash, zero pwsh FailFast, relaunch adopts + the same daemon PID). This is what must keep passing. + orphaned = the OLD broken #7742 behavior (daemon dies + with main, pwsh FailFasts). Directional inverse used + to prove the harness actually catches a regression; + on a fixed build this profile is EXPECTED to fail. + +Options: + --exe-path Installed Orca.exe to drive (default: the per-user + install under %LOCALAPPDATA%\\Programs\\Orca). The + harness NEVER installs/uninstalls — it only launches + this exe against an isolated userData dir. + --soak-seconds Post-crash observation window before relaunch (default: 8) + --keep-profile Skip temp-profile cleanup at teardown (for debugging) + -h, --help Show this help +` + +export function parseArgs(argv) { + if (argv.includes('-h') || argv.includes('--help')) { + return { help: true, usage: USAGE } + } + + const exePathFlagPresent = argv.includes('--exe-path') + const opts = { + // Only auto-locate on win32: off-win32 this would needlessly spawn powershell, + // and run.mjs asserts win32 first so the platform message wins over any + // "no Orca.exe found" default-resolution error. + exePath: + takeValue(argv, '--exe-path') ?? + (process.platform === 'win32' ? locateInstalledExe() : undefined) ?? + undefined, + expect: takeValue(argv, '--expect'), + soakSeconds: Number(takeValue(argv, '--soak-seconds') ?? '8'), + keepProfile: argv.includes('--keep-profile'), + usage: USAGE + } + + const errors = validate(opts, exePathFlagPresent, argv) + return { ...opts, errors } +} + +function validate(opts, exePathFlagPresent, argv) { + const errors = [] + errors.push(...validateArgShape(argv)) + if (!opts.expect) { + errors.push('Missing --expect ') + } else if (!VALID_PROFILES.has(opts.expect)) { + errors.push(`Invalid --expect "${opts.expect}" (expected survival or orphaned)`) + } + // Distinguish "--exe-path omitted" (fall back to auto-locate) from + // "--exe-path with no value" (a mistake that must fail, not silently default). + if (exePathFlagPresent && takeValue(argv, '--exe-path') === undefined) { + errors.push('--exe-path requires a path value') + } else if (!opts.exePath) { + errors.push( + 'No installed Orca.exe found under %LOCALAPPDATA%\\Programs — pass --exe-path ' + ) + } else if (!existsSync(opts.exePath)) { + errors.push(`--exe-path does not exist: ${opts.exePath}`) + } else if (!path.isAbsolute(opts.exePath)) { + errors.push(`--exe-path must be an absolute path (got "${opts.exePath}")`) + } + if (!Number.isFinite(opts.soakSeconds) || opts.soakSeconds < 0) { + errors.push('--soak-seconds must be a non-negative number') + } + return errors +} + +function validateArgShape(argv) { + const errors = [] + const seen = new Set() + for (let index = 0; index < argv.length; index++) { + const arg = argv[index] + if (!VALUE_FLAGS.has(arg) && !BOOLEAN_FLAGS.has(arg)) { + errors.push(`Unknown argument: ${arg}`) + continue + } + if (seen.has(arg)) { + errors.push(`Duplicate argument: ${arg}`) + } + seen.add(arg) + if (VALUE_FLAGS.has(arg)) { + const value = argv[index + 1] + if (value !== undefined && !value.startsWith('--')) { + index++ + } + } + } + return errors +} + +function takeValue(argv, flag) { + const idx = argv.indexOf(flag) + if (idx < 0) { + return undefined + } + const value = argv[idx + 1] + if (value === undefined || value.startsWith('--')) { + return undefined + } + return value +} diff --git a/tools/win-crash-survival-e2e/crash-assertions.mjs b/tools/win-crash-survival-e2e/crash-assertions.mjs new file mode 100644 index 000000000..1e01938cb --- /dev/null +++ b/tools/win-crash-survival-e2e/crash-assertions.mjs @@ -0,0 +1,101 @@ +// Profile assertions for the crash-survival proof. +// +// Two directional profiles over the SAME observed evidence, so CI can prove the +// harness catches a regression (an --expect that disagrees with reality fails): +// survival — the fixed #7742 behavior. The main crash actually landed, yet the +// detached daemon and the same interactive shell PID survive, ZERO pwsh +// FailFast events fire, a relaunch ADOPTS the same daemon PID, and the +// reattached UI reads back the survivor shell's env sentinel (proving +// keystrokes reach the same PTY, not a re-spawn). +// orphaned — the OLD broken behavior. The daemon dies with main (primary) and, +// because the shell was left idle at a live PSReadLine prompt, pwsh FailFasts +// with 0xE9 (secondary). On a fixed build this profile is EXPECTED to fail. +// NOTE: the orphaned direction is NOT exercised in CI (workflow_dispatch is +// unavailable on a non-default branch) and the 0xE9 only reproduces on a +// genuinely broken build — it is the directional inverse, documented and +// locally runnable, not a gate. + +function assertion(name, pass, expected, actual, detail = '') { + return { name, pass, expected, actual, detail } +} + +/** Build the ordered assertion list for the run's profile. */ +export function buildCrashAssertions(ctx) { + // The crash actually landing is the shared antecedent for BOTH profiles: without + // it, every survival/orphan signal below would be meaningless. + const mainDied = assertion( + 'main process actually died (crash landed)', + Boolean(ctx.mainDied), + 'main pid dead after taskkill', + String(ctx.mainDied) + ) + const profileAssertions = + ctx.profile === 'orphaned' ? orphanedAssertions(ctx) : survivalAssertions(ctx) + return [mainDied, ...profileAssertions] +} + +function survivalAssertions(ctx) { + const failFastDetail = (ctx.failFastEvents ?? []) + .slice(0, 3) + .map((e) => `${e.provider}#${e.id}@${e.timeCreated}`) + .join('; ') + return [ + assertion( + 'daemon survives main crash (PID still alive)', + Boolean(ctx.daemonAliveAfterCrash), + `daemon pid ${ctx.preDaemonPid} alive after crash`, + String(ctx.daemonAliveAfterCrash) + ), + assertion( + 'interactive shell survives main crash (PID still alive)', + Boolean(ctx.shellAliveAfterCrash), + `shell pid ${ctx.shellPid} alive after crash`, + String(ctx.shellAliveAfterCrash) + ), + assertion( + 'zero pwsh FailFast / 0xE9 during crash window', + (ctx.failFastEvents ?? []).length === 0, + '0 pwsh FailFast events', + `${(ctx.failFastEvents ?? []).length} events`, + failFastDetail + ), + assertion( + 'relaunch adopts the same daemon (PID unchanged)', + ctx.preDaemonPid != null && + ctx.preDaemonPid === ctx.postDaemonPid && + Boolean(ctx.postDaemonAlive), + `daemon pid ${ctx.preDaemonPid} adopted (not re-forked)`, + `post pid ${ctx.postDaemonPid} (alive: ${ctx.postDaemonAlive})` + ), + assertion( + 'reattached UI is bound to the SAME survivor shell (env sentinel reads back)', + Boolean(ctx.reattachProven), + 'survivor shell env sentinel readable via reattached terminal', + String(ctx.reattachProven) + ) + ] +} + +function orphanedAssertions(ctx) { + // The directional inverse: this profile passes ONLY on a build that reproduces + // the #7742 orphaning. Daemon death is the PRIMARY inverse (deterministic); + // pwsh FailFast is SECONDARY — faithful here only because the shell is left idle + // at a live PSReadLine prompt, which is what queries the severed console and + // triggers the 0xE9 on a broken build. Running this profile against a fixed + // build makes these FAIL, which is the proof the survival assertions above are + // not vacuous. + return [ + assertion( + 'daemon dies with main crash (old #7742 behavior)', + ctx.daemonAliveAfterCrash === false, + `daemon pid ${ctx.preDaemonPid} dead after crash`, + `alive: ${ctx.daemonAliveAfterCrash}` + ), + assertion( + 'pwsh FailFast / 0xE9 fired during crash window', + (ctx.failFastEvents ?? []).length > 0, + '>=1 pwsh FailFast event', + `${(ctx.failFastEvents ?? []).length} events` + ) + ] +} diff --git a/tools/win-crash-survival-e2e/crash-step.mjs b/tools/win-crash-survival-e2e/crash-step.mjs new file mode 100644 index 000000000..43dc18f04 --- /dev/null +++ b/tools/win-crash-survival-e2e/crash-step.mjs @@ -0,0 +1,121 @@ +// The abrupt main-process crash + its Windows event-log forensics. +// +// GitHub #7742: when Orca's main/renderer process died on Windows, the terminal +// daemon (which hosts the ConPTYs) died with it, severing the console pipe, and +// PowerShell hard-crashed with a 0xE9 "No process is on the other end of the +// pipe" FailFast. The fix relocates the daemon into a standalone, detached +// orca-terminal-daemon.exe that SURVIVES main death (src/main/daemon/ +// daemon-host-relocation.ts). This module reproduces the crash and scans for the +// pwsh FailFast that must no longer occur. + +import { execFileSync } from 'node:child_process' +import { runCommandSync } from '../win-update-e2e/powershell-runner.mjs' + +/** + * Abruptly crash ONLY the app main process. `/F` (force) with NO `/T` (tree) is + * the single load-bearing detail: a real main crash does not tree-kill the + * detached daemon, so tree-killing here — or closing gracefully — would leave the + * daemon alive for the wrong reason and make the survival assertion pass + * vacuously. Kills exactly `pid` (the real Electron main of the instance the + * harness launched, resolved via app.evaluate -> process.pid), never a scanned + * or image-named process, so a live user Orca on the same box is never touched. + */ +export function crashMainProcess(pid) { + if (!Number.isInteger(pid) || pid <= 0) { + throw new Error(`crashMainProcess: refusing to kill invalid pid ${pid}`) + } + // NO '/T': tree-killing would take the detached daemon down with the main and + // defeat the entire point of the test. + execFileSync('taskkill', ['/F', '/PID', String(pid)], { stdio: 'ignore' }) +} + +/** + * Scan the Windows Application event log for pwsh/powershell FailFast crashes in + * the crash window (from `sinceMs`). The #7742 signature is a 0xE9 exit / "No + * process is on the other end of the pipe" / FailFast (0xc0000409 stack-overrun), + * reported as an Application Error (#1000), Windows Error Reporting (#1001), or + * .NET Runtime (#1026) event for pwsh.exe. Matching by provider+id (not Message + * text alone) matters because Message can be null when the provider's resource + * DLL does not render — a null-Message crash from those reporters is still counted + * unless its text positively rules out pwsh, so we never miss a real FailFast. + * + * The scan is MACHINE-WIDE (Get-WinEvent has no per-process filter): on an + * isolated CI runner nothing else crashes in the ~10s window, so it can only ever + * false-FAIL (an unrelated crash), never false-PASS. Zero events is the decisive + * proof the ConPTY pipe was NOT severed. Returns + * { events: [{ id, provider, timeCreated, message }] }. + */ +export function scanPwshFailFast(sinceMs, runCommand = runCommandSync) { + // Build the start boundary from Unix ms directly (no locale-dependent string + // parse) and back it off 5s for clock skew between Node's Date.now() and the + // event-log timestamps. + const startMs = Math.max(0, Math.floor(sinceMs) - 5000) + const command = [ + `$start = [System.DateTimeOffset]::FromUnixTimeMilliseconds(${startMs}).LocalDateTime`, + `$crashProviders = @('Application Error','Windows Error Reporting','.NET Runtime')`, + `$crashIds = @(1000,1001,1026)`, + `$sig = '0xe9|other end of the pipe|FailFast|0xc0000409|Faulting application name: pwsh|Faulting application name: powershell'`, + // @() guards the PS 5.1 single-item unwrap: one match must still serialize as + // an array, or the JS side sees an object and .length explodes. + `try {`, + ` $sourceEvents = @(Get-WinEvent -FilterHashtable @{ LogName='Application'; StartTime=$start } -ErrorAction Stop)`, + `} catch {`, + // Get-WinEvent reports an empty window as an error; that is valid zero-event + // evidence, while permissions/service/query failures must still fail closed. + ` if ($_.FullyQualifiedErrorId -like 'NoMatchingEventsFound*') { $sourceEvents = @() } else { throw }`, + `}`, + `$events = @($sourceEvents |`, + ` Where-Object {`, + // A crash-reporter event referencing pwsh/powershell (or whose Message failed + // to render at all) counts; otherwise fall back to explicit signature text. + ` (($crashProviders -contains $_.ProviderName) -and ($crashIds -contains $_.Id) -and`, + ` ((-not $_.Message) -or ($_.Message -match 'pwsh|powershell'))) -or`, + ` ($_.Message -and ($_.Message -match 'pwsh|powershell') -and ($_.Message -match $sig)) })`, + `$out = @($events | ForEach-Object {`, + ` $msg = if ($_.Message) { $_.Message.Substring(0, [Math]::Min(400, $_.Message.Length)) } else { '' }`, + ` [pscustomobject]@{ id = $_.Id; provider = $_.ProviderName; timeCreated = $_.TimeCreated.ToString('o'); message = $msg } })`, + `ConvertTo-Json -InputObject @{ events = $out } -Depth 4 -Compress` + ].join('\n') + + const { stdout, stderr, code, error } = runCommand(command) + if (error) { + throw new Error(`pwsh-failfast scan spawn failed: ${error.message}`) + } + // Why: unavailable event-log evidence cannot count as proof that no FailFast + // occurred; fail the harness instead of converting an empty error into zero. + if (code !== 0) { + throw new Error(`pwsh-failfast scan failed (exit ${code}): ${stderr.trim()}`) + } + const trimmed = stdout.trim() + if (!trimmed) { + // Why: the script always serializes an events envelope, including for zero + // matches; empty stdout means the load-bearing evidence never arrived. + throw new Error('pwsh-failfast scan returned no JSON output') + } + let parsed + try { + parsed = JSON.parse(trimmed) + } catch (parseError) { + throw new Error( + `pwsh-failfast scan returned non-JSON (exit ${code}): ${parseError.message}\n` + + `stdout:\n${trimmed}\nstderr:\n${stderr}` + ) + } + if ( + !parsed || + typeof parsed !== 'object' || + Array.isArray(parsed) || + !Object.hasOwn(parsed, 'events') || + parsed.events == null + ) { + // Why: only the explicit envelope proves the query completed; malformed + // JSON must not be indistinguishable from an authoritative zero-event result. + throw new Error('pwsh-failfast scan returned JSON without an events envelope') + } + const raw = parsed.events + if (!Array.isArray(raw) && typeof raw !== 'object') { + throw new Error('pwsh-failfast scan returned an invalid events envelope') + } + const events = Array.isArray(raw) ? raw : raw ? [raw] : [] + return { events } +} diff --git a/tools/win-crash-survival-e2e/daemon-identity.mjs b/tools/win-crash-survival-e2e/daemon-identity.mjs new file mode 100644 index 000000000..268b23be8 --- /dev/null +++ b/tools/win-crash-survival-e2e/daemon-identity.mjs @@ -0,0 +1,28 @@ +// Resolve one authoritative daemon identity from the userData-scoped process +// scan and its advisory PID file metadata. + +/** + * Require exactly one live, scoped daemon and cross-check any PID-file record. + * The process command line owns identity; a stale PID file must never turn an + * unrelated recycled PID into false crash-survival evidence. + */ +export function selectScopedDaemon(pidFiles, scannedProcesses) { + if (scannedProcesses.length !== 1) { + throw new Error(`expected exactly one userData-scoped daemon, found ${scannedProcesses.length}`) + } + + const processRecord = scannedProcesses[0] + const numericPidFiles = pidFiles.filter((record) => Number.isInteger(record.pid)) + const matchingPidFile = numericPidFiles.find((record) => record.pid === processRecord.pid) + if (numericPidFiles.length > 0 && !matchingPidFile) { + throw new Error( + `daemon PID file does not match scoped live daemon ${processRecord.pid} ` + + `(recorded: ${numericPidFiles.map((record) => record.pid).join(', ')})` + ) + } + + return { + pid: processRecord.pid, + appVersion: matchingPidFile?.appVersion ?? null + } +} diff --git a/tools/win-crash-survival-e2e/reattach-proof.mjs b/tools/win-crash-survival-e2e/reattach-proof.mjs new file mode 100644 index 000000000..9ecec5ba2 --- /dev/null +++ b/tools/win-crash-survival-e2e/reattach-proof.mjs @@ -0,0 +1,19 @@ +/** Resolve the one tab introduced by the harness. Ambiguity would make the + * post-crash input oracle capable of targeting an unrelated restored tab. */ +export function selectCreatedTabId(beforeTabIds, afterTabIds) { + const before = new Set(beforeTabIds) + const created = afterTabIds.filter((tabId) => !before.has(tabId)) + if (created.length !== 1) { + throw new Error(`expected exactly one created terminal tab, found ${created.length}`) + } + return created[0] +} + +/** The canary rejects a respawned shell; the PID additionally proves that input + * reached the exact survivor whose liveness was checked during the crash. */ +export function reattachSentinelMatches(raw, expectedCanary, expectedShellPid) { + const [pidPart, canaryPart, extraPart] = raw.trim().split('|') + return ( + extraPart === undefined && canaryPart === expectedCanary && Number(pidPart) === expectedShellPid + ) +} diff --git a/tools/win-crash-survival-e2e/run.mjs b/tools/win-crash-survival-e2e/run.mjs new file mode 100644 index 000000000..4f9d8b242 --- /dev/null +++ b/tools/win-crash-survival-e2e/run.mjs @@ -0,0 +1,432 @@ +// win-crash-survival-e2e — packaged crash-survival proof harness. +// +// GitHub #7742: on Windows, when Orca's main/renderer process crashed, open +// terminal PTYs were orphaned and PowerShell hard-crashed with a 0xE9 "No +// process is on the other end of the pipe" FailFast, because the terminal daemon +// (hosting the ConPTYs) died together with the main process and severed the +// console pipe. The fix relocates the daemon into a standalone, detached +// orca-terminal-daemon.exe that survives main death (src/main/daemon/ +// daemon-host-relocation.ts). win-update-e2e proves the daemon survives a +// Windows UPDATE; this harness proves it survives a CRASH of the main process. +// +// Flow: launch the installed app (isolated userData) → open a plain terminal and, +// typing DIRECTLY into the interactive shell, stamp a per-shell env sentinel plus +// that shell's own $PID (leaving it idle at a live PSReadLine prompt, the faithful +// #7742 crash condition) → force-kill ONLY the real app main (no tree-kill, no +// graceful close) → prove the main actually died, then that the daemon + that same +// shell PID stay alive with no pwsh FailFast → relaunch, adopt the surviving +// daemon, and prove the reattached UI is bound to the SAME survivor shell by +// reading back its env sentinel (a re-spawned shell would not have it). +// Windows-only. See README.md. + +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { execFileSync } from 'node:child_process' +import { assertWin32 } from '../win-update-e2e/platform-guard.mjs' +import { + launchInstalledApp, + ensureTerminal, + dismissOverlays, + createTerminalTab, + listTabIds, + typeLine, + sendCtrlC, + waitForTerminalReady, + closeApp, + captureFailureDiagnostics, + resolveElectronMainPid +} from '../win-update-e2e/app-driver.mjs' +import { + findDaemonProcesses, + isPidAlive, + readDaemonPidFiles +} from '../win-update-e2e/daemon-processes.mjs' +import { createSeededRepo, buildFreshProfile } from '../win-update-e2e/onboarding-profile.mjs' +import { renderTable, allPassed } from '../win-update-e2e/assertions.mjs' +import { quotePowerShellLiteral } from '../win-update-e2e/powershell-runner.mjs' +import { parseArgs } from './cli-args.mjs' +import { crashMainProcess, scanPwshFailFast } from './crash-step.mjs' +import { buildCrashAssertions } from './crash-assertions.mjs' +import { selectScopedDaemon } from './daemon-identity.mjs' +import { reattachSentinelMatches, selectCreatedTabId } from './reattach-proof.mjs' + +const SORTABLE_TAB = '[data-testid="sortable-tab"]' +// The per-shell env var stamped into the interactive shell; reading it back after +// relaunch proves keystrokes reach the SAME survivor shell (a fresh re-spawn lacks it). +const SENTINEL_ENV = 'ORCA_CRASH_SENTINEL' + +function log(step, msg) { + console.log(`[win-crash-survival-e2e] ${step}: ${msg}`) +} + +async function main() { + const opts = parseArgs(process.argv.slice(2)) + if (opts.help) { + console.log(opts.usage) + return 0 + } + // Assert win32 BEFORE surfacing arg errors so an off-win32 invocation gets the + // clear platform message, not a confusing "no Orca.exe found" default-resolution + // failure. + assertWin32('win-crash-survival-e2e') + if (opts.errors?.length) { + console.error(`Argument errors:\n - ${opts.errors.join('\n - ')}\n${opts.usage}`) + return 2 + } + + const runId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}` + const canary = `ORCA-CRASH-SENTINEL-${runId}` + const runDir = mkdtempSync(path.join(tmpdir(), `orca-win-crash-e2e-${runId}-`)) + const userDataDir = path.join(runDir, 'userData') + const shellPidFile = path.join(runDir, 'shell.pid') + const reattachFile = path.join(runDir, 'reattach.txt') + + log('setup', `runId=${runId} runDir=${runDir} profile=${opts.expect} exe=${opts.exePath}`) + + const ctx = { session: null } + const diagDir = process.env.ORCA_E2E_DIAG_DIR || path.join(runDir, 'diag') + let passed = false + try { + passed = await runProof(ctx, { opts, canary, runDir, userDataDir, shellPidFile, reattachFile }) + if (!passed && ctx.session?.page) { + const diag = await captureFailureDiagnostics(ctx.session.page, diagDir, 'assertion-failure') + log('diag', `captured -> ${diagDir} (store=${diag.info?.hasStore ?? 'n/a'})`) + } + } catch (err) { + console.error(`[win-crash-survival-e2e] FATAL: ${err.stack || err.message}`) + if (ctx.session?.page) { + const diag = await captureFailureDiagnostics(ctx.session.page, diagDir, 'driving-failure') + log('diag', `captured -> ${diagDir} (store=${diag.info?.hasStore ?? 'n/a'})`) + } + passed = false + } finally { + await teardown({ app: ctx.session?.app, userDataDir, keepProfile: opts.keepProfile, runDir }) + } + return passed ? 0 : 1 +} + +/** + * Run the launch → crash → survive → relaunch → assert proof. `ctx.session` is + * assigned as each app launches so a caller's finally can tear down a partial + * session. Returns whether every assertion passed. + */ +async function runProof(ctx, args) { + const { opts, canary, runDir, userDataDir, shellPidFile, reattachFile } = args + + // Seed a fresh profile (onboarding dismissed + one throwaway repo) ONLY before + // the first launch. The relaunch must use the app's own persisted state so the + // reattach/adoption assertions are meaningful. + const seededRepo = createSeededRepo(path.join(runDir, 'fixture-repo')) + const seedProfile = buildFreshProfile({ repo: seededRepo }) + + // --- First launch: open a plain terminal and stamp the interactive shell --- + let session = await launchInstalledApp({ exePath: opts.exePath, userDataDir, seedProfile }) + ctx.session = session + await ensureTerminal(session.page, { allowCreate: true }) + await dismissOverlays(session.page) + // Opening the seeded workspace lands on its default tab (an agent, not a bare + // shell). Add an explicit plain-terminal tab so the sentinel commands run in a + // real pwsh prompt — typing shell commands into an agent TUI would never run. + const initialTabIds = await listTabIds(session.page) + await createTerminalTab(session.page) + await dismissOverlays(session.page) + const tabIds = await listTabIds(session.page) + const terminalTabId = selectCreatedTabId(initialTabIds, tabIds) + await waitForTerminalReady(session.page, 60_000, terminalTabId) + log('sessions', `terminal ready; created=${terminalTabId}; tab ids: ${tabIds.join(', ')}`) + + // Type DIRECTLY into the interactive shell (not a nested powershell) so $env and + // $PID belong to THIS shell: stamp the env sentinel and record the shell's own + // PID. The command completes fast, leaving the shell idle at a live PSReadLine + // prompt — the exact state that FailFasts with 0xE9 on a broken build. The pid + // file appearing also proves keystrokes reached and ran in the shell. + await typeLine( + session.page, + `$env:${SENTINEL_ENV}='${canary}'; Set-Content -LiteralPath ${quotePowerShellLiteral(shellPidFile)} -Value $PID`, + terminalTabId + ) + const shellPid = await waitForIntFile(shellPidFile, 15_000) + log('shell', `interactive shell pid=${shellPid} (sentinel ${SENTINEL_ENV}=${canary})`) + + const preDaemon = resolveScopedDaemon(userDataDir) + log('daemon', `pre-crash daemon pid=${preDaemon.pid} appVersion=${preDaemon.appVersion}`) + + // Resolve the REAL Electron main pid from INSIDE the main process. On this + // packaged build app.process().pid is a launcher stub that immediately re-execs + // the actual browser process; killing the stub would leave the real main (and + // its single-instance lock) alive and make survival vacuously true. app.evaluate + // runs in the main process, so process.pid there is the exact main of the + // instance this harness launched — authoritative, not a machine-wide scan. + const mainPid = await resolveElectronMainPid(session.app, { allowLauncherFallback: false }) + if (!Number.isInteger(mainPid) || mainPid <= 0) { + throw new Error(`could not resolve app main pid (got ${mainPid})`) + } + + // --- CRASH: force-kill ONLY the real main (no /T tree-kill, no graceful close) --- + const crashStartMs = Date.now() + log('crash', `taskkill /F /PID ${mainPid} (real main, no /T) — abrupt main-process death`) + crashMainProcess(mainPid) + // The crashed app's driver is dead; drop it so teardown never re-closes it. + ctx.session = null + + // Prove the crash actually LANDED before trusting any survival signal — an + // assertion that never fires would make the whole proof vacuous. + const mainDied = await waitForPidDead(mainPid, 15_000) + log('crash', `main pid ${mainPid} dead: ${mainDied}`) + + // Observe the survival window: the daemon and the SAME shell PID must keep running. + await delay(opts.soakSeconds * 1000) + const daemonAliveAfterCrash = preDaemon.pid != null && isPidAlive(preDaemon.pid) + const shellAliveAfterCrash = shellPid != null && isPidAlive(shellPid) + log( + 'crash', + `after crash: daemonAlive=${daemonAliveAfterCrash} shellAlive=${shellAliveAfterCrash}` + ) + + // --- Relaunch: adopt the surviving daemon and prove the reattached UI is the + // same survivor shell (env sentinel reads back) --- + clearSingletonLocks(userDataDir) + session = await launchInstalledApp({ exePath: opts.exePath, userDataDir }) + ctx.session = session + let reattachProven = false + try { + // No create on relaunch: the terminal must be RESTORED, not freshly made. + await ensureTerminal(session.page, { allowCreate: false }) + await dismissOverlays(session.page) + reattachProven = await proveReattachedShell(session.page, { + file: reattachFile, + expectedCanary: canary, + expectedShellPid: shellPid, + terminalTabId + }) + } catch (err) { + log('relaunch', `reattach proof did not complete: ${err.message}`) + } + log('relaunch', `reattached UI bound to survivor shell: ${reattachProven}`) + + const postDaemon = resolveScopedDaemon(userDataDir) + const postDaemonAlive = postDaemon.pid != null && isPidAlive(postDaemon.pid) + log('daemon', `post-relaunch daemon pid=${postDaemon.pid} alive=${postDaemonAlive}`) + + // Why: PowerShell can stay alive on a severed ConPTY until the next console + // read. Scan after the reattach keystroke so the user-visible 0xE9 is covered. + const { events: failFastEvents } = scanPwshFailFast(crashStartMs) + log('event-log', `pwsh FailFast/0xE9 events since crash: ${failFastEvents.length}`) + for (const e of failFastEvents.slice(0, 3)) { + log('event-log', ` ${e.provider}#${e.id}@${e.timeCreated}`) + } + + const assertions = buildCrashAssertions({ + profile: opts.expect, + shellPid, + preDaemonPid: preDaemon.pid, + postDaemonPid: postDaemon.pid, + postDaemonAlive, + mainDied, + daemonAliveAfterCrash, + shellAliveAfterCrash, + reattachProven, + failFastEvents + }) + const passed = allPassed(assertions) + console.log(renderTable(assertions, 'win-crash-survival-e2e')) + log('result', passed ? 'PASS' : 'FAIL') + return passed +} + +/** + * Prove the reattached UI is bound to the SAME survivor shell: type a command that + * writes the shell's own $PID plus the persisted env sentinel to a file, then + * confirm the sentinel (and PID) match. A freshly re-spawned shell would not carry + * the env var. Targets the exact pre-crash tab id so the probe cannot type shell + * commands into an unrelated agent tab. Repeats the idempotent command while the + * restored pane transport converges; a filesystem match, not elapsed time, wins. + */ +async function proveReattachedShell( + page, + { file, expectedCanary, expectedShellPid, terminalTabId } +) { + const restoredTabIds = await listTabIds(page) + log('relaunch', `restored tab ids: ${restoredTabIds.join(', ')}; target=${terminalTabId}`) + const targetTab = page.locator(`${SORTABLE_TAB}[data-tab-id="${terminalTabId}"]`).first() + await targetTab.waitFor({ state: 'attached', timeout: 15_000 }) + + const deadline = Date.now() + 30_000 + let attempt = 0 + while (Date.now() < deadline) { + attempt++ + const readinessBudgetMs = Math.max(deadline - Date.now(), 1) + await targetTab.click({ force: true, timeout: readinessBudgetMs }) + await waitForTerminalReady(page, readinessBudgetMs, terminalTabId) + // Why: a partially forwarded earlier attempt can leave text at PSReadLine; + // clear it before replaying the complete idempotent proof command. + await sendCtrlC(page, terminalTabId) + await typeLine( + page, + `Set-Content -LiteralPath ${quotePowerShellLiteral(file)} -Value "$($PID)|$($env:${SENTINEL_ENV})"`, + terminalTabId + ) + const remainingMs = deadline - Date.now() + const hit = await waitForSentinel( + file, + expectedCanary, + expectedShellPid, + Math.min(3_000, Math.max(remainingMs, 0)) + ) + if (hit) { + log('relaunch', `same-shell sentinel read back on attempt ${attempt}`) + return true + } + log('relaunch', `same-shell probe attempt ${attempt} produced no matching sentinel`) + } + return false +} + +/** Poll for the reattach file and require both the per-shell canary and exact + * survivor PID. Either check alone is weaker than the asserted shell identity. */ +async function waitForSentinel(file, expectedCanary, expectedShellPid, timeoutMs) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + try { + if (reattachSentinelMatches(readFileSync(file, 'utf8'), expectedCanary, expectedShellPid)) { + return true + } + } catch { + /* not written yet */ + } + await delay(500) + } + return false +} + +/** + * Resolve THIS run's daemon, scoped to its isolated userData dir so unrelated + * daemons on the machine (including the developer's live Orca) are ignored. + * The scoped live process scan is authoritative; PID files only contribute + * metadata after their PID matches that process. + */ +function resolveScopedDaemon(userDataDir) { + const pidFiles = readDaemonPidFiles(userDataDir) + const scan = findDaemonProcesses(userDataDir) + return selectScopedDaemon(pidFiles, scan) +} + +/** + * Remove Electron/Chromium single-instance lock files a crashed main can leave + * behind in the isolated profile, so the relaunch is not refused/redirected by a + * stale lock. Best-effort — absent files are normal. + */ +function clearSingletonLocks(userDataDir) { + let entries = [] + try { + entries = readdirSync(userDataDir) + } catch { + return + } + for (const entry of entries) { + if (entry.startsWith('Singleton')) { + try { + rmSync(path.join(userDataDir, entry), { recursive: true, force: true }) + } catch { + /* leave it; relaunch may still succeed */ + } + } + } +} + +/** + * Tear down only what THIS harness created. Kills are re-scoped at teardown time + * via a FRESH findDaemonProcesses(userDataDir): the interactive shell is a + * descendant of this run's daemon, so a /T tree-kill of the freshly-discovered + * daemon removes the daemon + OpenConsole + shell together. We deliberately do NOT + * kill any pid captured earlier in the run — a captured pid can be recycled by the + * OS onto an innocent process, so only pids re-verified as this run's daemon (by + * scoped command-line match) are ever killed. Never installs/uninstalls and never + * touches any other Orca on the box (a live user instance uses a different + * userData and is out of scope by construction). + */ +async function teardown({ app, userDataDir, keepProfile, runDir }) { + try { + await closeApp(app) + } catch { + /* already closed / never launched */ + } + for (const proc of findDaemonProcesses(userDataDir)) { + killPidTree(proc.pid) + } + if (keepProfile) { + log('teardown', `--keep-profile set; leaving ${runDir}`) + return + } + // Best-effort: a just-killed daemon/child can briefly hold file handles under + // the profile, so a locked rmSync must not turn cleanup into a FATAL. + try { + rmSync(runDir, { recursive: true, force: true }) + } catch (err) { + log('teardown', `could not remove ${runDir} (${err.code || err.message}); leaving it`) + } +} + +function killPidTree(pid) { + if (!Number.isInteger(pid) || pid <= 0) { + return + } + try { + execFileSync('taskkill', ['/pid', String(pid), '/T', '/F'], { stdio: 'ignore' }) + } catch { + /* already dead */ + } +} + +/** Poll until a pid is no longer alive (the crash landed), or timeout. */ +async function waitForPidDead(pid, timeoutMs) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (!isPidAlive(pid)) { + return true + } + await delay(500) + } + return false +} + +function readIntFile(filePath) { + try { + const n = Number(readFileSync(filePath, 'utf8').trim()) + return Number.isInteger(n) ? n : null + } catch { + return null + } +} + +/** Poll for an int-valued file (the shell writes its PID asynchronously once the + * typed command runs), returning the int or null after timeoutMs. */ +async function waitForIntFile(filePath, timeoutMs) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + const n = readIntFile(filePath) + if (n != null) { + return n + } + await delay(500) + } + return null +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +if (process.argv[1] && path.resolve(process.argv[1]) === import.meta.filename) { + main() + .then((code) => { + // Force-exit: a launched Electron app can keep libuv handles open, which + // would otherwise pin Node alive until the CI job timeout. + process.exit(code) + }) + .catch((err) => { + console.error('[win-crash-survival-e2e] FATAL:', err.stack || err.message) + process.exit(1) + }) +} diff --git a/tools/win-update-e2e/README.md b/tools/win-update-e2e/README.md index 973958a8b..6c771bb11 100644 --- a/tools/win-update-e2e/README.md +++ b/tools/win-update-e2e/README.md @@ -5,6 +5,12 @@ harness performs a real silent update and proves, with machine-checkable assertions, what happens to the terminal **daemon** and its **sessions** across the update — and whether any console/terminal window flashes. +> **Companion harness:** proving the daemon survives a **crash** of the main +> process (GitHub #7742), rather than an update, lives in +> [`tools/win-crash-survival-e2e`](../win-crash-survival-e2e/README.md). It reuses +> the shared modules in this directory (app driver, daemon discovery, PowerShell +> runner, platform guard, table renderer). + It is the Phase 0 "proof harness" deliverable from [`docs/windows-terminal-update-survival-plan.md`](../../docs/windows-terminal-update-survival-plan.md). It exists specifically because the July 2026 attempt shipped four broken RCs diff --git a/tools/win-update-e2e/app-driver.mjs b/tools/win-update-e2e/app-driver.mjs index 8f16a4464..71baba465 100644 --- a/tools/win-update-e2e/app-driver.mjs +++ b/tools/win-update-e2e/app-driver.mjs @@ -61,11 +61,62 @@ export async function launchInstalledApp({ ...extraEnv } }) - const page = await app.firstWindow({ timeout: 120_000 }) - await page.waitForLoadState('domcontentloaded') + // If firstWindow times out (the launched main never shows a window), the + // Electron process is still running — force-kill its tree before rethrowing so + // a driving failure never leaks an orphaned main to the CI job timeout. + let page + try { + page = await app.firstWindow({ timeout: 120_000 }) + await page.waitForLoadState('domcontentloaded') + } catch (err) { + const pid = await resolveElectronMainPid(app) + if (pid) { + try { + execFileSync('taskkill', ['/pid', String(pid), '/T', '/F'], { stdio: 'ignore' }) + } catch { + /* already gone */ + } + } + throw err + } return { app, page } } +/** Resolve the packaged Electron main, optionally falling back to Playwright's child PID. */ +export async function resolveElectronMainPid( + app, + { allowLauncherFallback = true, timeoutMs = 5_000 } = {} +) { + let timeout + try { + // Why: packaged launchers can re-exec, leaving app.process() pointing at a + // dead stub while evaluate runs in the authoritative Electron main. + const pid = await Promise.race([ + app.evaluate(() => process.pid), + new Promise((_, reject) => { + // Why: a wedged main connection is common on cleanup paths; resolving + // its authoritative PID must not consume the entire CI job timeout. + timeout = setTimeout(() => reject(new Error('main PID resolution timed out')), timeoutMs) + timeout.unref?.() + }) + ]) + if (Number.isInteger(pid) && pid > 0) { + return pid + } + } catch { + /* the main connection may already be unavailable */ + } finally { + clearTimeout(timeout) + } + // Why: crash proofs must fail closed rather than kill a packaged launcher stub + // and mistake its death for the authoritative Electron main crashing. + if (!allowLauncherFallback) { + return null + } + const fallbackPid = app.process()?.pid + return Number.isInteger(fallbackPid) && fallbackPid > 0 ? fallbackPid : null +} + /** * Best-effort diagnostics dump when driving fails: a screenshot, the visible * body text, and whether the e2e store is exposed (it is not in production @@ -102,7 +153,22 @@ export async function captureFailureDiagnostics(page, dir, label) { buttons: Array.from(document.querySelectorAll('button,[role="button"]')) .map((el) => (el.getAttribute('aria-label') || el.textContent || '').trim()) .filter((v, i, a) => v && a.indexOf(v) === i) - .slice(0, 60) + .slice(0, 60), + tabs: Array.from(document.querySelectorAll('[data-testid="sortable-tab"]')) + .map((el) => ({ + id: el.getAttribute('data-tab-id'), + title: el.getAttribute('data-tab-title'), + ariaLabel: el.getAttribute('aria-label'), + selected: el.getAttribute('aria-selected') + })) + .slice(0, 40), + activeElement: document.activeElement + ? { + tag: document.activeElement.tagName, + className: document.activeElement.getAttribute('class'), + ariaLabel: document.activeElement.getAttribute('aria-label') + } + : null })) writeFileSync(path.join(dir, `${label}.json`), JSON.stringify(info, null, 2)) out.info = info @@ -112,16 +178,18 @@ export async function captureFailureDiagnostics(page, dir, label) { return out } -/** Wait until the visible terminal surface and its xterm container are mounted. */ -export async function waitForTerminalReady(page, timeoutMs = 60_000) { - await page - .locator(TERMINAL_SURFACE_VISIBLE) - .first() - .waitFor({ state: 'visible', timeout: timeoutMs }) - await page - .locator(XTERM_CONTAINER_VISIBLE) - .first() - .waitFor({ state: 'visible', timeout: timeoutMs }) +/** Wait until the visible terminal surface and its xterm container are mounted. + * An expected tab id prevents post-restore probes from accepting another tab. */ +export async function waitForTerminalReady(page, timeoutMs = 60_000, terminalTabId = null) { + const selector = terminalTabId + ? `[data-terminal-tab-id="${terminalTabId}"]:visible` + : TERMINAL_SURFACE_VISIBLE + const surface = page.locator(selector).first() + await surface.waitFor({ state: 'visible', timeout: timeoutMs }) + await surface.locator(XTERM_CONTAINER_VISIBLE).first().waitFor({ + state: 'visible', + timeout: timeoutMs + }) } /** @@ -245,7 +313,7 @@ export async function listTabIds(page) { * off-screen helper textarea alone does not, which is why typed input was being * dropped. Click the pane, then focus the helper textarea as a belt-and-braces. */ -export async function focusActiveTerminal(page) { +export async function focusActiveTerminal(page, terminalTabId = null) { // A feature-tip modal can appear late and swallow keystrokes; clear any before // focusing so typed commands actually reach the shell. for (const name of OVERLAY_DISMISS_LABELS) { @@ -254,25 +322,32 @@ export async function focusActiveTerminal(page) { await btn.click({ timeout: 2_000 }).catch(() => {}) } } - const surface = page.locator(TERMINAL_SURFACE_VISIBLE).first() - await surface.click({ position: { x: 24, y: 24 }, timeout: 15_000 }).catch(() => {}) + const selector = terminalTabId + ? `[data-terminal-tab-id="${terminalTabId}"]:visible` + : TERMINAL_SURFACE_VISIBLE + const surface = page.locator(selector).first() + const click = surface.click({ position: { x: 24, y: 24 }, timeout: 15_000 }) + // Why: an exact-tab proof must fail closed if that restored surface vanishes; + // typing into whichever element retained focus could falsely target another tab. + await (terminalTabId ? click : click.catch(() => {})) // Scope the helper textarea to the visible surface so focus can't land on a // hidden duplicate pane's textarea (which would silently swallow keystrokes). const input = surface.locator(XTERM_INPUT).last() - await input.focus().catch(() => {}) + const focus = input.focus() + await (terminalTabId ? focus : focus.catch(() => {})) return input } /** Type a line and submit it (Enter → \r submits in the shell). */ -export async function typeLine(page, text) { - await focusActiveTerminal(page) +export async function typeLine(page, text, terminalTabId = null) { + await focusActiveTerminal(page, terminalTabId) await page.keyboard.type(text) await page.keyboard.press('Enter') } /** Send Ctrl+C to the active terminal. */ -export async function sendCtrlC(page) { - await focusActiveTerminal(page) +export async function sendCtrlC(page, terminalTabId = null) { + await focusActiveTerminal(page, terminalTabId) await page.keyboard.press('Control+C') } @@ -339,19 +414,31 @@ export async function readTerminalTextBestEffort(page) { * left alive exactly as a normal quit would. */ export async function closeApp(app, timeoutMs = 10_000) { - const proc = app.process() + // A partially-created session (launch failed before assignment) passes undefined. + if (!app) { + return + } + const mainPid = await resolveElectronMainPid(app) + let closeTimeout try { await Promise.race([ app.close(), - new Promise((_, reject) => setTimeout(() => reject(new Error('close timeout')), timeoutMs)) + new Promise((_, reject) => { + closeTimeout = setTimeout(() => reject(new Error('close timeout')), timeoutMs) + closeTimeout.unref?.() + }) ]) } catch { - if (proc?.pid) { + if (mainPid) { try { - execFileSync('taskkill', ['/pid', String(proc.pid), '/T', '/F'], { stdio: 'ignore' }) + execFileSync('taskkill', ['/pid', String(mainPid), '/T', '/F'], { stdio: 'ignore' }) } catch { /* already gone */ } } + } finally { + // Why: successful closes must not retain a timeout closure or keep a shared + // harness process alive until the failure deadline expires. + clearTimeout(closeTimeout) } } diff --git a/tools/win-update-e2e/assertions.mjs b/tools/win-update-e2e/assertions.mjs index 20db79e48..3d3efd77c 100644 --- a/tools/win-update-e2e/assertions.mjs +++ b/tools/win-update-e2e/assertions.mjs @@ -172,8 +172,9 @@ export function allPassed(assertions) { return assertions.every((a) => a.pass === true || a.pass === null) } -/** Render the assertion list as an aligned PASS/FAIL/INFO table. */ -export function renderTable(assertions) { +/** Render the assertion list as an aligned PASS/FAIL/INFO table. `label` names + * the harness in the header (shared with the crash-survival harness). */ +export function renderTable(assertions, label = 'win-update-e2e') { const symbol = (pass) => (pass === true ? 'PASS' : pass === false ? 'FAIL' : 'INFO') const nameWidth = Math.max(...assertions.map((a) => a.name.length), 10) const lines = assertions.map((a) => { @@ -181,6 +182,6 @@ export function renderTable(assertions) { return ` [${symbol(a.pass)}] ${a.name.padEnd(nameWidth)} expected: ${a.expected}; actual: ${a.actual}${detail}` }) const failed = assertions.filter((a) => a.pass === false).length - const header = `\n===== win-update-e2e assertions (${failed === 0 ? 'ALL PASS' : `${failed} FAILED`}) =====` + const header = `\n===== ${label} assertions (${failed === 0 ? 'ALL PASS' : `${failed} FAILED`}) =====` return [header, ...lines, ''].join('\n') } diff --git a/tools/win-update-e2e/daemon-processes.mjs b/tools/win-update-e2e/daemon-processes.mjs index cfd9c5d41..f7d43cc9b 100644 --- a/tools/win-update-e2e/daemon-processes.mjs +++ b/tools/win-update-e2e/daemon-processes.mjs @@ -97,14 +97,26 @@ export function readDaemonPidFiles(userDataDir = defaultUserDataDir()) { } /** True if a PID currently maps to a live process. */ -export function isPidAlive(pid) { +export function isPidAlive(pid, runCommand = runCommandSync) { if (!Number.isInteger(pid) || pid <= 0) { return false } - const { stdout } = runCommandSync( + const { stdout, stderr, code, error } = runCommand( `if (Get-Process -Id ${pid} -ErrorAction SilentlyContinue) { 'alive' } else { 'dead' }` ) - return stdout.trim() === 'alive' + if (error) { + throw new Error(`PID liveness probe failed to spawn: ${error.message}`) + } + if (code !== 0) { + throw new Error(`PID liveness probe failed (exit ${code}): ${stderr.trim()}`) + } + const state = stdout.trim() + if (state !== 'alive' && state !== 'dead') { + // Why: blank or unexpected output is unavailable evidence, not proof that + // a process died; crash-survival assertions must fail closed. + throw new Error(`PID liveness probe returned an invalid state: ${JSON.stringify(state)}`) + } + return state === 'alive' } function runJsonCommand(command) { diff --git a/tools/win-update-e2e/powershell-runner.mjs b/tools/win-update-e2e/powershell-runner.mjs index 68561b261..9b401d1ff 100644 --- a/tools/win-update-e2e/powershell-runner.mjs +++ b/tools/win-update-e2e/powershell-runner.mjs @@ -13,6 +13,11 @@ const BASE_ARGS = ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass' // whole CI job times out. Callers can override via opts.timeout. const DEFAULT_SYNC_TIMEOUT_MS = 60_000 +/** Quote a value as a PowerShell single-quoted literal. */ +export function quotePowerShellLiteral(value) { + return `'${String(value).replaceAll("'", "''")}'` +} + /** * Run a .ps1 file synchronously and return { code, stdout, stderr }. * scriptArgs is an array of string arguments passed after -File.