diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index de0afcf8a..802526681 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -1031,6 +1031,126 @@ ], "demotionRule": "Demote if any probe can run before lifecycle ownership, if more than one shared heartbeat timer is armed, if first-socket cleanup exceeds one interval, if later-socket cleanup exceeds two intervals, or if close/error cleanup retains listeners or timers." }, + { + "id": "runtime.streaming-subscription-close-delivery", + "title": "Retiring a runtime transport always tells the renderer its streams closed", + "maturity": "experimental", + "protection": "partial", + "owner": "runtime-platform", + "layer": "runtime-subscription-ipc-contract", + "surfaces": [ + "paired remote server", + "runtime environment disconnect and re-pair", + "terminal.multiplex streaming", + "browser screencast streaming", + "parked remote terminal reveal" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["remote-runtime"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["remote-runtime"], + "coverageNotes": "Deterministic main-IPC contract tests cover disconnect-driven close delivery, exactly-once close, per-subscription teardown isolation against a failing socket close and a throwing liveness probe, containment of a throwing renderer send on the unguarded host-close path, and continued suppression of stale payloads from a retired transport. A headed paired-server journey (real Orca host plus a separate paired Orca desktop client) covers hidden-but-mounted reveal, cold-parked reveal, and cold-parked reveal across a disconnect/reconnect. Live Linux and Windows paired-server evidence and real sleep/wake transport loss remain uncollected.", + "motivatingLinks": [ + "tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts", + "docs/reference/headless-linux-server.md" + ], + "invariant": "Every renderer-held runtime subscription receives exactly one terminal close event when its transport is retired, including when the retirement advanced the transport generation first, and a single failing teardown never abandons that environment's remaining subscriptions nor escapes into the transport that reported the close. Payload frames from a retired transport stay suppressed. A revealed remote terminal therefore reattaches over a live multiplex connection: its buffer restores, typed input reaches the host PTY, the echo paints without a tab flip, and the PTY converges on the revealed pane grid.", + "oracle": "The main IPC contract test subscribes terminal.multiplex through the real handler, disconnects the environment, and asserts the renderer received exactly one {type: close} subscription event. Two isolation tests subscribe a second stream to the same environment and make the first one fail -- in its socket close, and in the liveness probe inside notifyClosed -- then assert the disconnect does not throw, both transports closed, and every close the renderer could still receive was delivered. A third drives a host-initiated close through the transport callback, which is the one notifyClosed call site with no surrounding guard, with a renderer send that throws, and asserts it cannot escape into the WebSocket close handler. A fourth test asserts that after retirement a late response frame is not forwarded and a late transport close does not re-send. The paired-server journey runs three reveal scenarios against one real host and one real paired desktop client, and for each records buffer restore, host-side receipt of the typed marker through an out-of-band host sink file, live paint without a tab flip, and PTY-versus-pane grid convergence.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/runtime-environments.test.ts", + "pnpm exec playwright test tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" + ], + "testFiles": [ + "src/main/ipc/runtime-environments.test.ts", + "tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts" + ], + "assertionRefs": [ + { + "file": "src/main/ipc/runtime-environments.test.ts", + "assertions": [ + "tells the renderer when retiring the transport closes its streaming subscription (the load-bearing repro; red with the fix reverted)", + "retires an environment's remaining subscriptions when one teardown throws (load-bearing; red without per-subscription isolation)", + "contains a throwing renderer send on a host-initiated close (load-bearing; red without the guarded send, and the only coverage of the unguarded notifyClosed call site)", + "retires remaining subscriptions when a liveness probe inside notifyClosed throws (load-bearing; keeps the isolation structural rather than comment-asserted)", + "suppresses stale payloads from a retired transport but never re-sends its close (forward guard on the retained generation gate, not a repro)" + ] + }, + { + "file": "tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts", + "assertions": [ + "a revealed hidden-but-mounted remote terminal restores, accepts input, and paints live", + "a revealed cold-parked remote terminal restores, accepts input, and paints live", + "a cold-parked remote terminal revealed after a runtime disconnect and reconnect restores, accepts input, and paints live", + "the host PTY grid converges on the revealed pane grid in every scenario" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-08-03", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/runtime-environments.test.ts", + "result": "passed", + "durationSeconds": 0.81, + "summary": "53 runtime environment IPC tests passed, including the new close-delivery, both teardown-isolation, transport-path containment, and stale-payload-suppression contracts." + }, + { + "date": "2026-08-03", + "runner": "local", + "platform": "macos", + "command": "pnpm exec playwright test tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 15.5, + "summary": "All three reveal scenarios restored their buffer, delivered typed input to the host PTY, painted live without a tab flip, and converged the PTY on the 135x60 pane grid. Before the fix the reconnect-parked scenario stayed blank at recoveryState connecting with the PTY stranded at the host 128x60 grid." + }, + { + "date": "2026-08-03", + "runner": "local", + "platform": "macos", + "command": "pnpm exec playwright test tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 15.5, + "summary": "Repeat run on a clean uninstrumented build; all three scenarios green with identical grids." + }, + { + "date": "2026-08-03", + "runner": "local", + "platform": "macos", + "command": "pnpm exec playwright test tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 14.7, + "summary": "Third consecutive pass after the review follow-ups (env save/restore and the hidden-mounted stayed-mounted assertion)." + } + ], + "runtimeBudget": { + "p95Seconds": 60, + "scope": "focused runtime environment IPC tests plus one headed paired-server reveal journey" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "Deterministic IPC coverage plus the three recorded paired-server journey runs, which passed consecutively on macOS at 14.7-15.5s; the spec also passed once on a shared CI runner. Longer soak history is not yet available." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "On main the paired-server journey reproduced a blank, non-interactive pane after revealing a cold-parked remote terminal across a reconnect: recoveryState stayed connecting, the xterm buffer stayed empty, typed input never reached the host sink, a tab flip did not recover it, and the PTY stayed at the host 128x60 grid. Renderer instrumentation showed the multiplexer reusing a dead subscription (ensureConnected reuse-ready) and never receiving a subscribed event. Reverting the main-side fix also turns the close-delivery IPC test red; with the fix the multiplexer observes handleClose, reconnects fresh, and every scenario passes." + }, + "performanceBudget": { + "required": true, + "evidence": "The fix adds one boolean latch and at most one extra IPC send per retired subscription, on a path that already tears the subscription down. It introduces no timer, poll, retry, subprocess, or per-frame work, and the close is deduplicated so a transport-driven close after an environment-wide retirement sends nothing." + }, + "promotionCriteria": [ + "Collect at least 100 consecutive CI or soak passes or 14 days without an unexplained flake.", + "Collect live Linux and Windows paired-server reveal evidence.", + "Add real sleep/wake and network-loss transport drops alongside the explicit disconnect trigger." + ], + "knownGaps": [ + "Live paired-server validation currently covers macOS host and client only.", + "The transport drop is an explicit runtime disconnect; real sleep/wake and network partitions are not yet exercised by this gate.", + "Browser screencast subscribers share the fixed contract but have no dedicated reveal journey; the web and mobile clients run parallel transports and are unaffected." + ], + "demotionRule": "Demote if a retired runtime transport can leave a renderer subscription without a close event, if a close is delivered more than once, if one failing teardown strands its sibling subscriptions, if payload frames from a retired transport reach the renderer, or if a revealed remote terminal can stay blank or reject input after a reconnect." + }, { "id": "editor.live-log-append-stability", "title": "Long live session logs retain their Monaco viewport while appending", diff --git a/src/main/ipc/runtime-environments.test.ts b/src/main/ipc/runtime-environments.test.ts index d0101ee39..0e3739d6c 100644 --- a/src/main/ipc/runtime-environments.test.ts +++ b/src/main/ipc/runtime-environments.test.ts @@ -1689,6 +1689,363 @@ describe('registerRuntimeEnvironmentHandlers', () => { }) }) + it('tells the renderer when retiring the transport closes its streaming subscription', async () => { + // Why: disconnect advances the transport generation before closing sockets. + // Gating the terminal 'close' on that generation stranded the renderer with a + // handle it believed was open, so every later subscribe wrote into a socket + // main no longer owned — blank, wedged remote terminals after a reconnect. + registerRuntimeEnvironmentHandlers(store as never) + let transportCallbacks: { + onResponse: (response: Record) => void + onClose: () => void + } | null = null + const close = vi.fn(() => { + transportCallbacks?.onClose() + }) + subscribeRemoteRuntimeRequestMock.mockImplementation( + async ( + _environment: unknown, + _method: string, + _params: unknown, + _timeoutMs: number, + callbacks: NonNullable + ) => { + transportCallbacks = callbacks + return { requestId: 'multiplex-1', close, sendBinary: vi.fn() } + } + ) + + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + const added = await add(null, { name: 'desk', pairingCode: pairingCode() }) + + const senderSend = vi.fn() + const subscribe = handler< + { selector: string; method: string; params?: unknown; subscriptionId?: string }, + { subscriptionId: string; requestId: string } + >('runtimeEnvironments:subscribe') + const subscribed = await subscribe( + { + sender: { + id: 1, + isDestroyed: () => false, + send: senderSend, + once: vi.fn(), + removeListener: vi.fn() + } + }, + { + selector: added.environment.id, + method: 'terminal.multiplex', + params: {}, + subscriptionId: 'multiplex-sub' + } + ) + + const disconnect = handler<{ selector: string }, { disconnected: { id: string } }>( + 'runtimeEnvironments:disconnect' + ) + disconnect(null, { selector: added.environment.id }) + + const closeEvents = senderSend.mock.calls.filter( + (call) => + call[0] === 'runtimeEnvironments:subscriptionEvent' && + (call[1] as { type?: string }).type === 'close' + ) + expect(closeEvents).toEqual([ + [ + 'runtimeEnvironments:subscriptionEvent', + { subscriptionId: subscribed.subscriptionId, type: 'close' } + ] + ]) + }) + + it("retires an environment's remaining subscriptions when one teardown throws", async () => { + // Why: the sweep exists to retire dead handles, so a single failing teardown + // must not strand the very sockets it was called to close. + registerRuntimeEnvironmentHandlers(store as never) + const closeCalls: string[] = [] + let streamCount = 0 + subscribeRemoteRuntimeRequestMock.mockImplementation(async () => { + streamCount += 1 + const requestId = `stream-${streamCount}` + return { + requestId, + close: () => { + closeCalls.push(requestId) + if (requestId === 'stream-1') { + throw new Error('socket teardown exploded') + } + }, + sendBinary: vi.fn() + } + }) + + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + const added = await add(null, { name: 'desk', pairingCode: pairingCode() }) + + const senderSend = vi.fn() + const subscribe = handler< + { selector: string; method: string; params?: unknown; subscriptionId?: string }, + { subscriptionId: string; requestId: string } + >('runtimeEnvironments:subscribe') + const sender = { + sender: { + id: 1, + isDestroyed: () => false, + send: senderSend, + once: vi.fn(), + removeListener: vi.fn() + } + } + await subscribe(sender, { + selector: added.environment.id, + method: 'terminal.multiplex', + params: {}, + subscriptionId: 'doomed-sub' + }) + await subscribe(sender, { + selector: added.environment.id, + method: 'browser.screencast', + params: {}, + subscriptionId: 'sibling-sub' + }) + + const disconnect = handler<{ selector: string }, { disconnected: { id: string } }>( + 'runtimeEnvironments:disconnect' + ) + expect(() => disconnect(null, { selector: added.environment.id })).not.toThrow() + + expect(closeCalls).toEqual(['stream-1', 'stream-2']) + expect( + senderSend.mock.calls + .filter((call) => (call[1] as { type?: string }).type === 'close') + .map((call) => (call[1] as { subscriptionId: string }).subscriptionId) + ).toEqual(['doomed-sub', 'sibling-sub']) + + // Both entries are gone, so a later unsubscribe finds nothing to release. + const unsubscribe = handler<{ subscriptionId: string }, { unsubscribed: boolean }>( + 'runtimeEnvironments:unsubscribe' + ) + expect(await unsubscribe({ sender: { id: 1 } }, { subscriptionId: 'sibling-sub' })).toEqual({ + unsubscribed: false + }) + }) + + it('contains a throwing renderer send on a host-initiated close', async () => { + // Why: this is the notifyClosed call site with no surrounding guard. A host + // close arriving on a disposed render frame would otherwise throw out through + // the transport's onClose and into the WebSocket close handler. + registerRuntimeEnvironmentHandlers(store as never) + let transportCallbacks: { + onResponse: (response: Record) => void + onClose: () => void + } | null = null + subscribeRemoteRuntimeRequestMock.mockImplementation( + async ( + _environment: unknown, + _method: string, + _params: unknown, + _timeoutMs: number, + callbacks: NonNullable + ) => { + transportCallbacks = callbacks + return { requestId: 'host-closed-stream', close: vi.fn(), sendBinary: vi.fn() } + } + ) + + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + const added = await add(null, { name: 'desk', pairingCode: pairingCode() }) + + const senderSend = vi.fn((_channel: string, payload: { type: string }) => { + if (payload.type === 'close') { + throw new Error('Render frame was disposed') + } + }) + const subscribe = handler< + { selector: string; method: string; params?: unknown; subscriptionId?: string }, + { subscriptionId: string; requestId: string } + >('runtimeEnvironments:subscribe') + await subscribe( + { + sender: { + id: 1, + isDestroyed: () => false, + send: senderSend, + once: vi.fn(), + removeListener: vi.fn() + } + }, + { + selector: added.environment.id, + method: 'terminal.multiplex', + params: {}, + subscriptionId: 'host-closed-sub' + } + ) + + // The host closes the stream on its own; nothing wraps this call site. + expect(() => transportCallbacks!.onClose()).not.toThrow() + expect( + senderSend.mock.calls.filter((call) => (call[1] as { type?: string }).type === 'close') + ).toHaveLength(1) + + // The entry is still released, so a later unsubscribe finds nothing. + const unsubscribe = handler<{ subscriptionId: string }, { unsubscribed: boolean }>( + 'runtimeEnvironments:unsubscribe' + ) + expect(await unsubscribe({ sender: { id: 1 } }, { subscriptionId: 'host-closed-sub' })).toEqual( + { + unsubscribed: false + } + ) + }) + + it('retires remaining subscriptions when a liveness probe inside notifyClosed throws', async () => { + // Why: notifyClosed guards its own send, but the sweep must not depend on + // everything else inside it staying throw-free -- that is how the abandoned + // -siblings defect comes back the next time a line is added there. + registerRuntimeEnvironmentHandlers(store as never) + const closedStreams: string[] = [] + let streamCount = 0 + subscribeRemoteRuntimeRequestMock.mockImplementation(async () => { + streamCount += 1 + const requestId = `stream-${streamCount}` + return { + requestId, + close: () => closedStreams.push(requestId), + sendBinary: vi.fn() + } + }) + + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + const added = await add(null, { name: 'desk', pairingCode: pairingCode() }) + + const deliveredCloses: string[] = [] + const senderSend = vi.fn( + (_channel: string, payload: { subscriptionId: string; type: string }) => { + if (payload.type === 'close') { + deliveredCloses.push(payload.subscriptionId) + } + } + ) + let probeShouldThrow = false + const subscribe = handler< + { selector: string; method: string; params?: unknown; subscriptionId?: string }, + { subscriptionId: string; requestId: string } + >('runtimeEnvironments:subscribe') + const sender = { + sender: { + id: 1, + isDestroyed: () => { + if (probeShouldThrow) { + throw new Error('WebContents liveness probe exploded') + } + return false + }, + send: senderSend, + once: vi.fn(), + removeListener: vi.fn() + } + } + await subscribe(sender, { + selector: added.environment.id, + method: 'terminal.multiplex', + params: {}, + subscriptionId: 'probe-throws-sub' + }) + await subscribe(sender, { + selector: added.environment.id, + method: 'browser.screencast', + params: {}, + subscriptionId: 'surviving-sub' + }) + // Why: arm only after subscribe, whose own isDestroyed checks must succeed. + probeShouldThrow = true + + const disconnect = handler<{ selector: string }, { disconnected: { id: string } }>( + 'runtimeEnvironments:disconnect' + ) + expect(() => disconnect(null, { selector: added.environment.id })).not.toThrow() + + // Both transports still closed even though every notifyClosed probe threw. + expect(closedStreams).toEqual(['stream-1', 'stream-2']) + expect(deliveredCloses).toEqual([]) + }) + + it('suppresses stale payloads from a retired transport but never re-sends its close', async () => { + registerRuntimeEnvironmentHandlers(store as never) + let transportCallbacks: { + onResponse: (response: Record) => void + onClose: () => void + } | null = null + subscribeRemoteRuntimeRequestMock.mockImplementation( + async ( + _environment: unknown, + _method: string, + _params: unknown, + _timeoutMs: number, + callbacks: NonNullable + ) => { + transportCallbacks = callbacks + return { requestId: 'multiplex-2', close: vi.fn(), sendBinary: vi.fn() } + } + ) + + const add = handler< + { name: string; pairingCode: string }, + { environment: { id: string; name: string } } + >('runtimeEnvironments:addFromPairingCode') + const added = await add(null, { name: 'desk', pairingCode: pairingCode() }) + + const senderSend = vi.fn() + const subscribe = handler< + { selector: string; method: string; params?: unknown; subscriptionId?: string }, + { subscriptionId: string; requestId: string } + >('runtimeEnvironments:subscribe') + await subscribe( + { + sender: { + id: 1, + isDestroyed: () => false, + send: senderSend, + once: vi.fn(), + removeListener: vi.fn() + } + }, + { + selector: added.environment.id, + method: 'terminal.multiplex', + params: {}, + subscriptionId: 'multiplex-stale' + } + ) + + invalidateRuntimeEnvironmentTransport(added.environment.id) + senderSend.mockClear() + // A late frame from the retired socket must not reach the renderer... + transportCallbacks!.onResponse({ + id: 'r1', + ok: true, + result: {}, + _meta: { runtimeId: 'runtime-a' } + }) + // ...and its late close must not re-fire after the retirement already sent one. + transportCallbacks!.onClose() + expect(senderSend).not.toHaveBeenCalled() + }) + it('rejects cross-window streaming subscription control', async () => { registerRuntimeEnvironmentHandlers(store as never) const close = vi.fn() diff --git a/src/main/ipc/runtime-environments.ts b/src/main/ipc/runtime-environments.ts index 6bf71edd6..e31f8569d 100644 --- a/src/main/ipc/runtime-environments.ts +++ b/src/main/ipc/runtime-environments.ts @@ -25,6 +25,7 @@ type RetainedRemoteRuntimeSubscription = RemoteRuntimeSubscription & { environmentId: string ownerWebContentsId: number removeDestroyedListener: () => void + notifyClosed: () => void } const remoteRuntimeSubscriptions = new Map() const getUserDataPath = (): string => app.getPath('userData') @@ -36,7 +37,22 @@ function closeSubscriptionsForEnvironment(environmentId: string): void { continue } remoteRuntimeSubscriptions.delete(subscriptionId) - subscription.close() + // Why: one failing teardown must not abandon this environment's other + // sockets -- that strands exactly the dead handles this sweep exists to + // retire. Guard the two steps independently so neither can skip the other, + // and so the isolation stays structural rather than resting on a claim that + // nothing inside notifyClosed will ever throw. + try { + subscription.close() + } catch (error) { + console.warn('[runtime-environments] subscription close failed during retirement:', error) + } + try { + // Why: a shared-control logical close never calls back, so notify directly. + subscription.notifyClosed() + } catch (error) { + console.warn('[runtime-environments] subscription close notice failed:', error) + } } } export function invalidateRuntimeEnvironmentTransport(environmentId: string): void { @@ -120,6 +136,21 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { removeDestroyedListener() subscription?.close() } + // Why: the renderer treats close as terminal and drops its handle, so send it once. + // Latch before sending so a re-entrant call cannot duplicate it, and never + // throw: a dying renderer must not abort its siblings' retirement. + let closeNotified = false + const notifyClosed = (): void => { + if (closeNotified || sender.isDestroyed()) { + return + } + closeNotified = true + try { + sender.send('runtimeEnvironments:subscriptionEvent', { subscriptionId, type: 'close' }) + } catch { + // The renderer is gone; there is no one left to tell. + } + } sender.once('destroyed', closeSubscription) destroyedListenerAttached = true try { @@ -131,6 +162,12 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { args.timeoutMs, { onEvent: (payload) => { + if (payload.type === 'close') { + // Why: retirement advances the generation before closing, so gating + // close on it stranded the renderer with a dead subscription. + notifyClosed() + return + } if (transportIsCurrent() && !sender.isDestroyed()) { sender.send('runtimeEnvironments:subscriptionEvent', { subscriptionId, @@ -172,6 +209,7 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { environmentId: environment.id, ownerWebContentsId, removeDestroyedListener, + notifyClosed, sendBinary: (bytes) => subscription?.sendBinary(bytes) ?? false, close: () => { removeDestroyedListener() diff --git a/tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts b/tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts new file mode 100644 index 000000000..acabed065 --- /dev/null +++ b/tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts @@ -0,0 +1,484 @@ +/** + * Paired remote server: a revealed remote terminal must stay interactive + * without a tab flip. + * + * Topology: headed Orca desktop host (remote server) + a separate paired Orca + * desktop client — the "connect to Windows 2, open an old workspace" shape. + * + * Oracle (the reported symptom verbatim): type into the revealed pane and see + * the echo paint live. "Paints only after switching to another terminal and + * back" is the failure, so each scenario records both. + * + * The decoy tabs are load-bearing, not scenery. Pre-fix, handleClose never runs, + * so closeIfIdle is the only remaining release path and it needs zero streams — + * a sibling stream is what keeps the wedged multiplexer alive and the pre-fix + * state red. Each scenario asserts its flip decoy is still mounted at the reveal + * so that invariant cannot quietly lapse and turn this green. + * + * Run: + * pnpm exec playwright test \ + * tests/e2e/paired-remote-terminal-parked-reveal-interactivity.spec.ts \ + * --config tests/playwright.config.ts --project electron-headless --workers=1 + */ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { randomUUID } from 'node:crypto' +import os from 'node:os' +import path from 'node:path' +import type { Page } from '@stablyai/playwright-test' +import { + HOST_TERMINAL_SURFACE_SEPARATOR, + toWebTerminalSurfaceTabId +} from '../../src/shared/terminal-surface-id' +import { expect, test } from './helpers/orca-app' +import { + createRuntimeDesktopPairingOffer, + launchPairedElectronClient, + type PairedElectronClient +} from './helpers/paired-electron-client' +import { focusActiveTerminalInput } from './helpers/terminal' +import { waitForTabParked } from './helpers/terminal-hidden-parking' + +const PARK_DELAY_MS = 2_000 +const LIVE_PAINT_BUDGET_MS = 12_000 +const REVEAL_BUDGET_MS = 20_000 +const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-parked-reveal-')) +const fixturePath = path.join(scratch, 'parked-reveal-terminal.mjs') +writeFileSync( + fixturePath, + [ + "import { appendFileSync } from 'node:fs'", + 'const sink = process.argv[2]', + 'const size = () => `${process.stdout.columns}x${process.stdout.rows}`', + 'const record = (line) => appendFileSync(sink, `${line}\\n`)', + 'record(`READY:${size()}`)', + 'process.stdout.write(`READY:${size()}\\r\\n`)', + "process.stdout.on('resize', () => {", + ' record(`SIZE:${size()}`)', + ' process.stdout.write(`SIZE:${size()}\\r\\n`)', + '})', + "process.stdin.setEncoding('utf8')", + "let pending = ''", + "process.stdin.on('data', (data) => {", + ' pending += data', + ' const lines = pending.split(/\\r\\n|\\r|\\n/)', + " pending = lines.pop() ?? ''", + ' for (const line of lines) {', + ' record(`LINE:${line}`)', + ' process.stdout.write(`LINE:${line}\\r\\n`)', + ' }', + '})', + 'process.stdin.resume()' + ].join('\n') +) + +test.afterAll(() => { + rmSync(scratch, { recursive: true, force: true }) +}) + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(sinkPath: string): string { + const command = [process.execPath, fixturePath, sinkPath] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +function readSink(sinkPath: string): string { + try { + return readFileSync(sinkPath, 'utf8') + } catch { + return '' + } +} + +async function callEnvironment( + page: Page, + environmentId: string, + method: string, + params: unknown +): Promise { + return page.evaluate( + async ({ environmentId, method, params }) => { + const response = await window.api.runtimeEnvironments.call({ + selector: environmentId, + method, + params + }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { environmentId, method, params } + ) as Promise +} + +type HostTerminal = { + hostTabId: string + sinkPath: string + terminal: string + webTabId: string +} + +async function createHostTerminal( + page: Page, + environmentId: string, + worktreeId: string +): Promise { + const sinkPath = path.join(scratch, `sink-${randomUUID()}.log`) + const result = await callEnvironment<{ tab: { id: string; terminal: string | null } }>( + page, + environmentId, + 'session.tabs.createTerminal', + { + worktree: `id:${worktreeId}`, + command: fixtureCommand(sinkPath), + activate: false, + select: false, + navigation: 'caller' + } + ) + if (!result.tab.terminal) { + throw new Error('host session terminal was not created') + } + // Why: the host answers with a `tabId::leafId` surface id; client tabs mirror the parent tab. + const hostTabId = result.tab.id.split(HOST_TERMINAL_SURFACE_SEPARATOR)[0] + return { + hostTabId, + sinkPath, + terminal: result.tab.terminal, + webTabId: toWebTerminalSurfaceTabId(hostTabId) + } +} + +async function openClientTab(page: Page, worktreeId: string, webTabId: string): Promise { + await expect + .poll( + () => + page.evaluate( + (id) => (window.__store?.getState().tabsByWorktree[id] ?? []).map((tab) => tab.id), + worktreeId + ), + { timeout: 60_000, message: `client never mirrored host tab ${webTabId}` } + ) + .toContain(webTabId) + await page.evaluate( + ({ webTabId, worktreeId }) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(worktreeId) + state?.setActiveTab(webTabId) + state?.setActiveTabType('terminal') + }, + { webTabId, worktreeId } + ) + await expect + .poll(() => page.evaluate((id) => window.__paneManagers?.has(id) ?? false, webTabId), { + timeout: 60_000, + message: `client pane for ${webTabId} did not mount` + }) + .toBe(true) +} + +async function readActivePaneGrid( + page: Page, + webTabId: string +): Promise<{ cols: number; rows: number } | null> { + return page.evaluate((id) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return pane ? { cols: pane.terminal.cols, rows: pane.terminal.rows } : null + }, webTabId) +} + +/** Reads the target tab's own buffer. `getTerminalContent` resolves whatever + * tab the store thinks is active, which hides per-tab reveal failures. */ +async function readPaneContent(page: Page, webTabId: string): Promise { + return page.evaluate((id) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return pane?.serializeAddon?.serialize?.() ?? '' + }, webTabId) +} + +async function readPaneDiagnostics( + page: Page, + worktreeId: string, + webTabId: string +): Promise { + return page.evaluate( + ({ webTabId, worktreeId }) => { + const manager = window.__paneManagers?.get(webTabId) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + const state = window.__store?.getState() + const tab = (state?.tabsByWorktree[worktreeId] ?? []).find((entry) => entry.id === webTabId) + return { + mounted: Boolean(manager), + ptyId: pane?.container?.dataset?.ptyId ?? null, + recoveryState: pane?.container?.dataset?.ptyRecoveryState ?? null, + cols: pane?.terminal?.cols ?? null, + rows: pane?.terminal?.rows ?? null, + bufferLength: pane?.serializeAddon?.serialize?.()?.length ?? null, + paneLeafIds: manager?.getPanes?.().map((entry) => entry.leafId ?? null) ?? null, + storeTabPtyId: tab?.ptyId ?? null, + storeTabLayout: tab?.paneLayout ? JSON.stringify(tab.paneLayout) : null, + storePtyIdsByTab: state?.ptyIdsByTabId?.[webTabId] ?? null + } + }, + { webTabId, worktreeId } + ) +} + +async function waitForPaneMarker( + page: Page, + webTabId: string, + marker: string, + budgetMs: number +): Promise { + const deadline = Date.now() + budgetMs + while (Date.now() < deadline) { + if ((await readPaneContent(page, webTabId)).includes(marker)) { + return true + } + await new Promise((resolve) => setTimeout(resolve, 250)) + } + return false +} + +function readPtyGridFromContent(content: string): { cols: number; rows: number } | null { + const sizes = [...content.matchAll(/(?:READY|SIZE):(\d+)x(\d+)/g)] + const last = sizes.at(-1) + return last ? { cols: Number(last[1]), rows: Number(last[2]) } : null +} + +type ScenarioResult = { + name: string + restoredBuffer: boolean + hostReceivedInput: boolean + paintedLive: boolean + paintedAfterFlip: boolean + paneGrid: { cols: number; rows: number } | null + ptyGrid: { cols: number; rows: number } | null + diagnostics: unknown +} + +/** Types a unique marker into the revealed pane and records whether the host + * received it, whether it painted live, and (if not) whether the reported + * tab-flip workaround reveals it. */ +async function probeInteractivity( + page: Page, + worktreeId: string, + target: HostTerminal, + flipTo: HostTerminal, + name: string +): Promise { + const token = `probe-${name}` + // Why: a human types once the pane looks restored; typing earlier would race the reattach. + const restoredBuffer = await waitForPaneMarker(page, target.webTabId, 'READY:', REVEAL_BUDGET_MS) + await focusActiveTerminalInput(page) + await page.keyboard.type(token) + await page.keyboard.press('Enter') + const paintedLive = await waitForPaneMarker( + page, + target.webTabId, + `LINE:${token}`, + LIVE_PAINT_BUDGET_MS + ) + const paneGrid = await readActivePaneGrid(page, target.webTabId) + const diagnostics = await readPaneDiagnostics(page, worktreeId, target.webTabId) + let paintedAfterFlip = paintedLive + if (!paintedLive) { + await openClientTab(page, worktreeId, flipTo.webTabId) + await openClientTab(page, worktreeId, target.webTabId) + paintedAfterFlip = await waitForPaneMarker( + page, + target.webTabId, + `LINE:${token}`, + LIVE_PAINT_BUDGET_MS + ) + } + const sink = readSink(target.sinkPath) + return { + name, + restoredBuffer, + hostReceivedInput: sink.includes(`LINE:${token}`), + paintedLive, + paintedAfterFlip, + paneGrid, + ptyGrid: readPtyGridFromContent(sink), + diagnostics + } +} + +/** The wedged-multiplexer repro needs a sibling stream alive at reveal time + * (see the header), and S1 additionally needs its target never to have parked. */ +async function expectStillMounted(page: Page, webTabId: string, label: string): Promise { + expect( + await page.evaluate((id) => window.__paneManagers?.has(id) ?? false, webTabId), + `${label} parked before the reveal it is supposed to survive` + ).toBe(true) +} + +/** Logged unconditionally: on failure this line is the whole diagnosis. */ +function logResult(result: ScenarioResult): ScenarioResult { + console.log(`[paired-reveal] ${JSON.stringify(result)}`) + return result +} + +async function seedScenario( + client: PairedElectronClient, + worktreeId: string +): Promise<{ target: HostTerminal; decoys: HostTerminal[] }> { + const target = await createHostTerminal(client.page, client.environmentId, worktreeId) + const decoys = [ + await createHostTerminal(client.page, client.environmentId, worktreeId), + await createHostTerminal(client.page, client.environmentId, worktreeId) + ] + await openClientTab(client.page, worktreeId, target.webTabId) + await expect + .poll(() => readPaneContent(client.page, target.webTabId), { + timeout: 60_000, + message: 'target terminal never painted its READY marker' + }) + .toContain('READY:') + return { target, decoys } +} + +test('paired client keeps revealed remote terminals interactive', async ({ + orcaPage +}, testInfo) => { + test.setTimeout(600_000) + const offer = await createRuntimeDesktopPairingOffer(orcaPage) + // Why: the paired client inherits this from the launching process; a reused + // Playwright worker would otherwise leak the shortened delay into later specs. + const previousParkDelay = process.env.ORCA_E2E_TERMINAL_PARKING_DELAY_MS + process.env.ORCA_E2E_TERMINAL_PARKING_DELAY_MS = String(PARK_DELAY_MS) + const client = await launchPairedElectronClient(offer, testInfo, 'parked-reveal') + const createdTerminals: string[] = [] + const results: ScenarioResult[] = [] + try { + const worktreeId = await orcaPage.evaluate(() => { + const id = window.__store?.getState().activeWorktreeId + if (!id) { + throw new Error('headed host has no active worktree') + } + return id + }) + await expect + .poll( + () => + client.page.evaluate( + (id) => + window.__store + ?.getState() + .allWorktrees() + .some((worktree) => worktree.id === id) ?? false, + worktreeId + ), + { timeout: 60_000, message: 'paired client never saw the host worktree' } + ) + .toBe(true) + await client.page.evaluate((id) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(id) + }, worktreeId) + + // S1 — hidden tab that stays mounted, then revealed. + { + const { target, decoys } = await seedScenario(client, worktreeId) + createdTerminals.push(target.terminal, ...decoys.map((decoy) => decoy.terminal)) + await openClientTab(client.page, worktreeId, decoys[0].webTabId) + // Pins the scenario label: this reveal must not have gone through a park. + await expectStillMounted(client.page, target.webTabId, 'hidden-mounted target') + await openClientTab(client.page, worktreeId, target.webTabId) + results.push( + logResult( + await probeInteractivity(client.page, worktreeId, target, decoys[1], 'hidden-mounted') + ) + ) + } + + // S2 — cold-parked tab (renderer unmounted), then revealed. + { + const { target, decoys } = await seedScenario(client, worktreeId) + createdTerminals.push(target.terminal, ...decoys.map((decoy) => decoy.terminal)) + await openClientTab(client.page, worktreeId, decoys[0].webTabId) + await openClientTab(client.page, worktreeId, decoys[1].webTabId) + await waitForTabParked(client.page, target.webTabId, { parkDelayMs: PARK_DELAY_MS }) + await expectStillMounted(client.page, decoys[1].webTabId, 'cold-parked flip decoy') + await openClientTab(client.page, worktreeId, target.webTabId) + results.push( + logResult( + await probeInteractivity(client.page, worktreeId, target, decoys[1], 'cold-parked') + ) + ) + } + + // S3 — cold-parked tab whose runtime connection dropped and came back + // while parked (the "returned after a while" report). + { + const { target, decoys } = await seedScenario(client, worktreeId) + createdTerminals.push(target.terminal, ...decoys.map((decoy) => decoy.terminal)) + await openClientTab(client.page, worktreeId, decoys[0].webTabId) + await openClientTab(client.page, worktreeId, decoys[1].webTabId) + await waitForTabParked(client.page, target.webTabId, { parkDelayMs: PARK_DELAY_MS }) + await client.page.evaluate(async (selector) => { + await window.api.runtimeEnvironments.disconnect({ selector }) + }, client.environmentId) + await expect + .poll( + async () => + client.page.evaluate(async (selector) => { + const response = await window.api.runtimeEnvironments.connect({ selector }) + return response.ok + }, client.environmentId), + { timeout: 60_000, message: 'paired client never reconnected to the host runtime' } + ) + .toBe(true) + await expectStillMounted(client.page, decoys[1].webTabId, 'reconnect-parked flip decoy') + await openClientTab(client.page, worktreeId, target.webTabId) + results.push( + logResult( + await probeInteractivity(client.page, worktreeId, target, decoys[1], 'reconnect-parked') + ) + ) + } + + for (const result of results) { + expect( + { + scenario: result.name, + restoredBuffer: result.restoredBuffer, + hostReceivedInput: result.hostReceivedInput, + paintedLive: result.paintedLive + }, + `${result.name}: revealed pane was not interactive (painted after tab flip: ${result.paintedAfterFlip})` + ).toEqual({ + scenario: result.name, + restoredBuffer: true, + hostReceivedInput: true, + paintedLive: true + }) + expect( + { scenario: result.name, ptyGrid: result.ptyGrid }, + `${result.name}: host PTY geometry never converged on the revealed pane grid` + ).toEqual({ scenario: result.name, ptyGrid: result.paneGrid }) + } + } finally { + if (previousParkDelay === undefined) { + delete process.env.ORCA_E2E_TERMINAL_PARKING_DELAY_MS + } else { + process.env.ORCA_E2E_TERMINAL_PARKING_DELAY_MS = previousParkDelay + } + for (const terminal of createdTerminals) { + await callEnvironment(client.page, client.environmentId, 'terminal.closeTab', { + terminal + }).catch(() => undefined) + } + await client.dispose() + } +})