From ddf58d6d6a60e8fb4e34f96f4414dfd171dc1388 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Fri, 7 Aug 2026 01:25:18 -0700 Subject: [PATCH] fix(terminal): restore preserved remote PTYs after host relaunch (#12990) * fix(terminal): foreground preserved daemon PTYs * fix(terminal): keep snapshot sequence domains distinct * test(terminal): use active reconnect control * test(terminal): await reconnect control activation * test(terminal): validate reconnect with fresh control * test(terminal): tighten host restart evidence * fix(terminal): retry preserved PTY attach after inventory * fix(terminal): retry attach after overlapping inventory --------- Co-authored-by: Jinwoo-H --- config/reliability-gates.jsonc | 84 +++- src/main/ipc/pty.test.ts | 47 ++ src/main/ipc/pty.ts | 2 +- src/main/runtime/orca-runtime.ts | 33 +- src/main/runtime/rpc/methods/terminal.ts | 9 +- .../runtime/rpc/terminal-multiplex.test.ts | 52 ++ .../rpc/terminal-subscribe-buffer.test.ts | 12 +- ...al-subscriber-driven-daemon-attach.test.ts | 65 +++ ...minal-host-restart-background-sync.spec.ts | 447 ++++++++++++++++++ 9 files changed, 728 insertions(+), 23 deletions(-) create mode 100644 tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 33945cfe8..f7935dbfc 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -4196,23 +4196,28 @@ "snapshot-probe sequence gap recovery", "hidden remote worktree retention", "remote terminal reveal and input", + "paired host relaunch with preserved daemon PTYs", "manual server disconnect" ], "platforms": ["macos", "linux", "windows"], "providers": ["paired-runtime", "ssh"], "coveredPlatforms": ["macos"], "coveredProviders": ["paired-runtime"], - "coverageNotes": "Deterministic headed macOS runs launch an isolated Orca desktop server and a separate paired web client. A byte-identical headless run uses an isolated `orca serve` host. Both create six real paired host PTYs with bounded high-output scrollback, prove sustained output while all six client xterms are warm-mounted but hidden causes zero renderer scheduler work, ordinary-park five xterms without enabling the lossy retention budget, assert bounded cells/heap/timer lag, then restore the bounded authoritative tail exactly once on the original PTY including output produced while parked and continued input/output. The headed oracle additionally keeps every remote target workspace and terminal unmounted in the host renderer before and after client recovery. Separate headed/headless ACK-starvation tests recover one stalled stream without replacing its PTY. A headed output-drop oracle proves a successful snapshot probe cannot certify a stale live stream when the authoritative PTY sequence advanced beyond the client high-water. Unit coverage requires an unsequenced client to establish a first-probe baseline, remain attached at the same sequence, and recover only after a later probe advances. A capability-disabled run proves legacy hosts retain the prior lossy limit/TTL fallback. Unit tests cover exact-owner capability routing, raw-stream release, singleton side-effect facts with timed handoff cleanup, 64 simultaneous synthetic-title sources with zero decorative client events after convergence, per-client late-subscribe recovery, semantic bell/title transitions, local animation-frame preservation, legacy no-output-pause delivery gating with exact snapshot restore, provider-authoritative snapshots, 128 active streams plus retry after a 129th-stream capacity rejection, capacity-pressure backoff, full split-leaf remint reconciliation, truncation, and manual-disconnect queue fencing. Linux, Windows, live SSH, and production-scale paired hosts remain gaps.", + "coverageNotes": "Deterministic headed macOS runs launch an isolated Orca desktop server and a separate paired web client. A byte-identical headless run uses an isolated `orca serve` host. Both create six real paired host PTYs with bounded high-output scrollback, prove sustained output while all six client xterms are warm-mounted but hidden causes zero renderer scheduler work, ordinary-park five xterms without enabling the lossy retention budget, assert bounded cells/heap/timer lag, then restore the bounded authoritative tail exactly once on the original PTY including output produced while parked and continued input/output. The headed oracle additionally keeps every remote target workspace and terminal unmounted in the host renderer before and after client recovery. A separate headed desktop-client journey relaunches the host main process while retaining the daemon and a daemon-backgrounded PTY, then requires first-observation foreground sync and continued input/output on the same PTY. Unit coverage keeps layout versions out of snapshot output high-waters when a restored snapshot has no output sequence and retries an existing subscriber only after authoritative provider inventory proves the retained PTY exists. Separate headed/headless ACK-starvation tests recover one stalled stream without replacing its PTY. A headed output-drop oracle proves a successful snapshot probe cannot certify a stale live stream when the authoritative PTY sequence advanced beyond the client high-water. Unit coverage requires an unsequenced client to establish a first-probe baseline, remain attached at the same sequence, and recover only after a later probe advances. A capability-disabled run proves legacy hosts retain the prior lossy limit/TTL fallback. Unit tests cover exact-owner capability routing, raw-stream release, singleton side-effect facts with timed handoff cleanup, 64 simultaneous synthetic-title sources with zero decorative client events after convergence, per-client late-subscribe recovery, semantic bell/title transitions, local animation-frame preservation, legacy no-output-pause delivery gating with exact snapshot restore, provider-authoritative snapshots, 128 active streams plus retry after a 129th-stream capacity rejection, capacity-pressure backoff, full split-leaf remint reconciliation, truncation, and manual-disconnect queue fencing. Linux, Windows, live SSH, and production-scale paired hosts remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/issues/8652", "https://github.com/stablyai/orca/pull/10625" ], - "invariant": "A host advertising terminal.paired-parking.v1 keeps the PTY and bounded authoritative history alive while an ordinary hidden-view park destroys the client xterm and releases its raw per-PTY stream. Reveal must restore up to the requested 5,000 rows, parked-time side effects/output, the same PTY identity, and continued input/output. Hosts without the capability must gate hidden raw output before xterm scheduling and repaint from the authoritative snapshot on reveal while retaining the existing limit/TTL force-parking fallback. After a paired client's first semantic title state, decorative spinner frequency must add zero encrypted client-event frames; local title animation and semantic title, status, bell, completion, and query facts remain intact. A paired terminal stream whose delivery credits stop progressing must replace only that stream; command silence first probes authoritative state and replaces the stream if the probe times out or proves the PTY advanced beyond the client's delivered output sequence. When no comparable delivery high-water exists, the first sequenced probe establishes a baseline and only later advancement proves staleness. A same-sequence snapshot remains valid proof of a responsive silent command. A successful status probe may replace a pre-ready shared-control socket without rejecting or duplicating calls already waiting for that transport. Manual disconnect must retain pairing while preventing queued or passive calls and subscriptions from recreating transport until explicit Connect.", + "invariant": "A host advertising terminal.paired-parking.v1 keeps the PTY and bounded authoritative history alive while an ordinary hidden-view park destroys the client xterm and releases its raw per-PTY stream. Reveal must restore up to the requested 5,000 rows, parked-time side effects/output, the same PTY identity, and continued input/output. After a host main-process relaunch, unknown local delivery-sync state must not be treated as proof that a surviving daemon PTY is already foregrounded. If subscription precedes provider readiness, authoritative inventory must reconsider that existing subscriber without spawning, resizing, or changing reconnect state. A snapshot without an output high-water must keep it unknown rather than borrowing a layout version that can suppress newer live output. Hosts without the capability must gate hidden raw output before xterm scheduling and repaint from the authoritative snapshot on reveal while retaining the existing limit/TTL force-parking fallback. After a paired client's first semantic title state, decorative spinner frequency must add zero encrypted client-event frames; local title animation and semantic title, status, bell, completion, and query facts remain intact. A paired terminal stream whose delivery credits stop progressing must replace only that stream; command silence first probes authoritative state and replaces the stream if the probe times out or proves the PTY advanced beyond the client's delivered output sequence. When no comparable delivery high-water exists, the first sequenced probe establishes a baseline and only later advancement proves staleness. A same-sequence snapshot remains valid proof of a responsive silent command. A successful status probe may replace a pre-ready shared-control socket without rejecting or duplicating calls already waiting for that transport. Manual disconnect must retain pairing while preventing queued or passive calls and subscriptions from recreating transport until explicit Connect.", "oracle": "Run one byte-identical six-terminal oracle against an isolated headed desktop host and an isolated headless `orca serve` host. Stage at least 1,000,000 xterm cells, enable ordinary parking with the lossy retention budget disabled, require exactly one mounted manager and five parked tabs, at most 45% retained cells, no more than 16 MiB heap growth, and under 500 ms timer drift. In headed mode, require the host renderer to remain on its original workspace with zero target terminal managers mounted throughout client park, reveal, and live I/O. While a tab is parked, require authoritative terminal.read to observe new PTY output; reveal it and require the original PTY, a marker within the requested 5,000-row history, the parked marker, and post-reveal input/output. Drive 64 host PTYs through ten 80 ms synthetic title frames, require zero paired client events after the first frame per PTY while all 640 local frames remain observable, attach a late client and require one current frame per PTY, then require semantic bell and idle transitions on both clients. Reconstruct a legacy subscribed stream with no outputPause capability, hide a chatty pane, require no hidden xterm writes, reveal it, and require one authoritative snapshot plus continued live output. Drop and acknowledge output only for one original paired-client stream, prove the fixture process consumed input while the host model advanced and the client stayed stale, then require the command snapshot probe to replace that stream, repaint exact fixture output, preserve authoritative PTY identity and target-tab cardinality, and resume live I/O. For a client without a delivered sequence, require the first numeric probe to establish a baseline without replacement, a same-sequence probe to remain attached, and a later advanced probe to replace only that stream. Withhold the first encrypted shared-control ready frame, start one RPC, trigger a status-probe refresh, and require two connections, one host delivery, successful response, and zero retained request bytes. Admit 128 active streams, reject the 129th as retryable, release one stream, then require the retry to attach and publish a snapshot without multiplying retained subscribers. Disable terminal.paired-parking.v1 and require the same oracle to fail before parking, while the legacy limit-one fallback separately passes. Also preserve truncated first paint, ACK-starved same-PTY recovery, responsive silent-command snapshot probes, dead-stream probe timeout recovery, and queued manual-disconnect fencing.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/web/web-preload-api.test.ts src/main/ipc/runtime-environments.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/terminal-subscriber-driven-daemon-attach.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/shared/remote-runtime-shared-control-connection.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty.test.ts", + "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "ORCA_E2E_FORWARD_APP_LOGS=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1 --repeat-each=10 --max-failures=1", "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", "ORCA_E2E_WEB_CLIENT=1 ORCA_E2E_DISABLE_PAIRED_TERMINAL_PARKING=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", @@ -4228,6 +4233,8 @@ "src/main/ipc/runtime-environments.test.ts", "src/main/runtime/rpc/terminal-multiplex.test.ts", "src/main/runtime/rpc/terminal-subscribe-buffer.test.ts", + "src/main/runtime/terminal-subscriber-driven-daemon-attach.test.ts", + "src/main/ipc/pty.test.ts", "src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts", "src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts", "src/renderer/src/components/terminal-pane/pty-connection.test.ts", @@ -4248,9 +4255,38 @@ "tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts", "tests/e2e/paired-remote-terminal-retention-memory.spec.ts", "tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts", - "tests/e2e/terminal-parked-memory.spec.ts" + "tests/e2e/terminal-parked-memory.spec.ts", + "tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts" ], "assertionRefs": [ + { + "file": "src/main/runtime/terminal-subscriber-driven-daemon-attach.test.ts", + "assertions": ["retries an existing subscriber when provider inventory becomes ready"] + }, + { + "file": "src/main/ipc/pty.test.ts", + "assertions": [ + "foregrounds a preserved daemon PTY after handler recreation loses sync memory exactly once" + ] + }, + { + "file": "src/main/runtime/rpc/terminal-multiplex.test.ts", + "assertions": ["keeps layout versions out of the output sequence domain"] + }, + { + "file": "src/main/runtime/rpc/terminal-subscribe-buffer.test.ts", + "assertions": [ + "omits an unknown snapshot output sequence instead of publishing the layout version" + ] + }, + { + "file": "tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts", + "assertions": [ + "preserves daemon PID and target PTY identity across the paired host relaunch", + "foregrounds the daemon-backgrounded target on its first post-relaunch remote observation", + "delivers target and post-relaunch control terminal input and live output" + ] + }, { "file": "tests/e2e/paired-remote-terminal-probe-gap-recovery.spec.ts", "assertions": ["replaces a stale paired stream when the PTY snapshot advanced"] @@ -4379,6 +4415,42 @@ } ], "evidenceRuns": [ + { + "date": "2026-08-07", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/terminal-subscriber-driven-daemon-attach.test.ts", + "result": "passed", + "durationSeconds": 4.29, + "summary": "All 10 subscriber-driven attach contracts passed. With two authoritative inventories overlapping one refused in-flight attach, removing only the post-settlement retry made the byte-identical focused assertion fail with one attach instead of two; restoring it retained one bounded waiter, retried once, and resumed live output on the existing stream." + }, + { + "date": "2026-08-07", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/pty.test.ts", + "result": "passed", + "durationSeconds": 5.84, + "summary": "The full PTY IPC file passed 449 tests, including one first-observation foreground sync after handler recreation and deduplication of the repeated presence signal." + }, + { + "date": "2026-08-07", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_FORWARD_APP_LOGS=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1 --repeat-each=10 --max-failures=1", + "result": "passed", + "durationSeconds": 126, + "summary": "Ten consecutive isolated host-relaunch journeys preserved the daemon and PTY identities and resumed terminal input plus live output after the paired transport recovered." + }, + { + "date": "2026-08-07", + "runner": "local", + "platform": "macos", + "command": "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "result": "passed", + "durationSeconds": 12.5, + "summary": "With the parking override captured by both host launches, the isolated headed host and separate paired desktop client retained the daemon PID and PTY ID across host relaunch, emitted one target foreground transition, and delivered target plus post-relaunch control input/output. The byte-identical run with only the unknown-state fix reverted failed at the missing target foreground transition." + }, { "date": "2026-08-02", "runner": "local", @@ -4503,15 +4575,15 @@ }, "flakeHistory": { "status": "unknown", - "evidence": "The paired headed oracle is deterministic locally but has no CI or soak history yet." + "evidence": "The host-relaunch oracle passed a fresh 10-run local series. One earlier exploratory repeat timed out in transport recovery before terminal input/output assertions; it did not recur with forwarded app logs. A later Linux CI run reproduced missing live output when inventory overlapped the refused in-flight attach, which now has a deterministic unit oracle and is pending cross-platform rerun. Broader headed/headless parking coverage has no aggregated CI or soak history yet." }, "redGreenEvidence": { "status": "complete", - "evidence": "The six-PTY ordinary-parking oracle was red in two independently observed stages: the parked paired remount first used detached attach and restored only rows 5,963–5,999, then the requested snapshot still lost to the current-screen replay until park reveals explicitly entered the capability-gated reattach coordinator. The candidate is green on headed and headless hosts, while disabling terminal.paired-parking.v1 makes the byte-identical oracle fail at its capability precondition and leaves the separately tested legacy lossy fallback green. The 64-PTY decorative-title control delivered 576 client events after convergence on unmodified main; the candidate delivers zero while preserving all 640 local frames, one current frame to a late client, and bell/idle transitions to both clients. The legacy no-output-pause control failed to request a reveal snapshot and kept hidden bytes on the xterm path; the candidate gates those bytes locally and restores one authoritative snapshot before live output. With stalled-stream recovery disabled, the host cursor advanced but the client remained frozen; enabling it repainted the marker with the same PTY and tab. The sequence-gap unit and headed paired oracle are red on current main and with only the new probe decision reverted: the authoritative host advances while the client stays stale after a successful probe. The candidate replaces the stream, repaints the marker, preserves PTY identity, and resumes I/O. Reverting the active-stream limit to 64 makes the exact 128-stream contract fail at 64; restoring 128 admits every intended stream, rejects 129, and reattaches it after one slot releases." + "evidence": "The six-PTY ordinary-parking oracle was red in two independently observed stages: the parked paired remount first used detached attach and restored only rows 5,963–5,999, then the requested snapshot still lost to the current-screen replay until park reveals explicitly entered the capability-gated reattach coordinator. The candidate is green on headed and headless hosts, while disabling terminal.paired-parking.v1 makes the byte-identical oracle fail at its capability precondition and leaves the separately tested legacy lossy fallback green. The 64-PTY decorative-title control delivered 576 client events after convergence on unmodified main; the candidate delivers zero while preserving all 640 local frames, one current frame to a late client, and bell/idle transitions to both clients. The legacy no-output-pause control failed to request a reveal snapshot and kept hidden bytes on the xterm path; the candidate gates those bytes locally and restores one authoritative snapshot before live output. With stalled-stream recovery disabled, the host cursor advanced but the client remained frozen; enabling it repainted the marker with the same PTY and tab. The sequence-gap unit and headed paired oracle are red on current main and with only the new probe decision reverted: the authoritative host advances while the client stays stale after a successful probe. The candidate replaces the stream, repaints the marker, preserves PTY identity, and resumes I/O. Reverting the active-stream limit to 64 makes the exact 128-stream contract fail at 64; restoring 128 admits every intended stream, rejects 129, and reattaches it after one slot releases. The host-relaunch oracle fails when unknown delivery state is coerced to foregrounded or a layout version becomes an output high-water; the candidate foregrounds the preserved daemon PTY once and resumes same-PTY input/output. When authoritative inventory overlaps a refused in-flight attach, removing only the post-settlement retry leaves the existing subscriber without daemon output; the candidate retains one waiter across repeated inventory publications, retries exactly once, and restores live output." }, "performanceBudget": { "required": true, - "evidence": "Before ordinary parking, six warm-mounted hidden paired xterms receive zero raw frames and schedule zero renderer output drains while host models continue ingesting sustained output. Ordinary parking then retains the host PTY and bounded 5,000-row provider model while destroying five client xterms. Decorative title ticks are collapsed per paired subscriber before JSON serialization and encryption; semantic facts remain on the singleton client-event stream. Headed and headless oracles require exact reduction from six managers to one, at least 55% staged xterm-cell release, no more than 16 MiB heap growth, and under 500 ms timer drift. Legacy hosts retain the prior 12-worktree/45-minute lossy force-parking policy but now gate hidden raw output before xterm scheduling and restore from a bounded snapshot on reveal. Stream recovery adds no polling and is scoped to one stream." + "evidence": "Before ordinary parking, six warm-mounted hidden paired xterms receive zero raw frames and schedule zero renderer output drains while host models continue ingesting sustained output. Ordinary parking then retains the host PTY and bounded 5,000-row provider model while destroying five client xterms. Decorative title ticks are collapsed per paired subscriber before JSON serialization and encryption; semantic facts remain on the singleton client-event stream. Headed and headless oracles require exact reduction from six managers to one, at least 55% staged xterm-cell release, no more than 16 MiB heap growth, and under 500 ms timer drift. Legacy hosts retain the prior 12-worktree/45-minute lossy force-parking policy but now gate hidden raw output before xterm scheduling and restore from a bounded snapshot on reveal. Stream recovery adds no polling and is scoped to one stream. Provider-ready reconciliation runs only for PTYs returned by a successful controller inventory, requires an existing remote subscriber, and retains at most one post-settlement waiter per PTY while attach deduplication remains active." }, "promotionCriteria": [ "Collect 100 consecutive CI passes or 14 days of soak history for the headed and headless parking commands.", diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 877d7d886..678cc8774 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -637,6 +637,7 @@ describe('registerPtyHandlers', () => { const write = vi.fn() const pauseProducer = vi.fn() const resumeProducer = vi.fn() + const setPtyBackgrounded = vi.fn() const shutdown = vi.fn() let dataHandler: ((payload: { id: string; data: string }) => void) | null = null let exitHandler: ((payload: { id: string; code: number }) => void) | null = null @@ -650,6 +651,7 @@ describe('registerPtyHandlers', () => { resize: vi.fn(), pauseProducer, resumeProducer, + setPtyBackgrounded, kill: vi.fn(), shutdown, sendSignal: vi.fn(), @@ -688,6 +690,7 @@ describe('registerPtyHandlers', () => { write, pauseProducer, resumeProducer, + setPtyBackgrounded, shutdown, getBufferSnapshot, emitData: (id: string, data: string) => dataHandler?.({ id, data }), @@ -15608,6 +15611,50 @@ describe('registerPtyHandlers', () => { }) describe('hidden renderer delivery gate', () => { + it('foregrounds a preserved daemon PTY after handler recreation loses sync memory', async () => { + const daemon = installObservableDaemonTestProvider() + const firstRuntime = { + setPtyController: vi.fn(), + hasRawTerminalViewSubscriber: vi.fn(() => false), + createPreAllocatedTerminalHandle: vi.fn(() => null), + registerPty: vi.fn(), + noteTerminalSpawnCommand: vi.fn(), + onPtySpawned: vi.fn(), + onPtyExit: vi.fn(), + onPtyData: vi.fn() + } + registerPtyHandlers(mainWindow as never, firstRuntime as never) + const result = (await handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + sessionId: 'daemon-session' + })) as { id: string } + + getPtySetRendererPtyVisibleListener()(null, { id: result.id, visible: false }) + expect(daemon.setPtyBackgrounded).toHaveBeenLastCalledWith(result.id, true) + + daemon.setPtyBackgrounded.mockClear() + handlers.clear() + let rawSubscriberPresent = false + const nextRuntime = { + setPtyController: vi.fn(), + hasRawTerminalViewSubscriber: vi.fn(() => rawSubscriberPresent), + onRemoteTerminalViewPresenceChanged: null as ((id: string) => void) | null, + registerRawTerminalViewSubscriber(id: string): void { + rawSubscriberPresent = true + this.onRemoteTerminalViewPresenceChanged?.(id) + } + } + registerPtyHandlers(mainWindow as never, nextRuntime as never) + + nextRuntime.registerRawTerminalViewSubscriber(result.id) + // Repeated presence signals must remain deduplicated. + nextRuntime.onRemoteTerminalViewPresenceChanged?.(result.id) + + expect(daemon.setPtyBackgrounded).toHaveBeenCalledOnce() + expect(daemon.setPtyBackgrounded).toHaveBeenCalledWith(result.id, false) + }) + it('drops hidden PTY data after model ingestion and emits one out-of-band restore marker', async () => { vi.useFakeTimers() const runtime = { diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 5d858cac7..46498fdf0 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -2516,7 +2516,7 @@ export function registerPtyHandlers( function syncPtyBackgroundedDelivery(id: string, caller: string): void { const background = rendererPtyIsKnownHidden(id) && !(runtime?.hasRawTerminalViewSubscriber?.(id) ?? false) - if ((backgroundedDeliverySyncByPty.get(id) ?? false) === background) { + if (backgroundedDeliverySyncByPty.get(id) === background) { return } const provider = tryGetProviderForPty(id) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 7983ebacb..72bd2b0c1 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -3111,6 +3111,7 @@ export class OrcaRuntimeService { // concurrent first-subscribes and keeps later subscribes no-ops; it is // cleared per lifecycle generation (exit/respawn) and on failed attempts. private subscriberDrivenProviderAttachesByPtyId = new Map>() + private subscriberDrivenProviderAttachInventoryWaiters = new Set() // Why: a spawn through this app already attaches its provider stream, so // subscriber-driven attach and the never-attached read fallback must target // only inventory-discovered sessions no local spawn published this @@ -10597,6 +10598,7 @@ export class OrcaRuntimeService { this.legacyWorkerRecoveredPtys.delete(ptyId) // Why: a respawn under the same session id needs its own subscriber-driven attach. this.subscriberDrivenProviderAttachesByPtyId.delete(ptyId) + this.subscriberDrivenProviderAttachInventoryWaiters.delete(ptyId) this.spawnPublishedPtys.delete(ptyId) // Why: a provider response belongs to the process generation that issued // it; a respawn must neither reuse its frame nor join its in-flight call. @@ -10806,10 +10808,7 @@ export class OrcaRuntimeService { // A spawn published (or admission pending) this generation already // attaches the provider stream; a replacement under a reused id must not // read as the discovered never-attached session it replaced. - if ( - this.spawnPublishedPtys.has(ptyId) || - this.pendingPtyRegistrationIncarnations.has(ptyId) - ) { + if (this.spawnPublishedPtys.has(ptyId) || this.pendingPtyRegistrationIncarnations.has(ptyId)) { return false } // SSH panes have their own lease/reattach machinery. @@ -10846,6 +10845,31 @@ export class OrcaRuntimeService { }) } + private reconcileSubscriberDrivenProviderAttach(ptyId: string): void { + if (!this.hasRemoteTerminalViewSubscriber(ptyId)) { + return + } + const pending = this.subscriberDrivenProviderAttachesByPtyId.get(ptyId) + if (!pending) { + this.ensureSubscriberDrivenProviderAttach(ptyId) + return + } + if (this.subscriberDrivenProviderAttachInventoryWaiters.has(ptyId)) { + return + } + this.subscriberDrivenProviderAttachInventoryWaiters.add(ptyId) + void pending.then((attached) => { + this.subscriberDrivenProviderAttachInventoryWaiters.delete(ptyId) + if (attached || !this.hasRemoteTerminalViewSubscriber(ptyId)) { + return + } + if (this.subscriberDrivenProviderAttachesByPtyId.get(ptyId) === pending) { + this.subscriberDrivenProviderAttachesByPtyId.delete(ptyId) + } + this.ensureSubscriberDrivenProviderAttach(ptyId) + }) + } + /** Mark a raw-output viewer without transferring terminal query authority. */ registerRawTerminalViewSubscriber(ptyId: string): () => void { this.rawTerminalViewSubscriberCounts.set( @@ -29268,6 +29292,7 @@ export class OrcaRuntimeService { this.restoredOrchestrationAuthorityByPtyId.delete(session.id) } pty.controllerTitle = session.title?.trim() || null + this.reconcileSubscriberDrivenProviderAttach(session.id) } // Why: fire-and-forget so this listing hot path doesn't serialize a relay round-trip per session and a throw can't abort the sweep below. this.refreshPtyForegroundAgent(session.id) diff --git a/src/main/runtime/rpc/methods/terminal.ts b/src/main/runtime/rpc/methods/terminal.ts index 05f94f2e6..1cdec021e 100644 --- a/src/main/runtime/rpc/methods/terminal.ts +++ b/src/main/runtime/rpc/methods/terminal.ts @@ -2623,7 +2623,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ const size = runtime.getTerminalSize(ptyId) const displayMode = runtime.getMobileDisplayMode(ptyId) const layoutSeq = runtime.getLayout(ptyId)?.seq - const snapshotFrameSeq = serialized?.seq ?? layoutSeq + // Why: layout versions and output offsets are different sequence domains. const snapshotOutputSeq = serialized?.seq emit({ type: 'subscribed', @@ -2664,7 +2664,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ cols: serialized?.cols ?? size?.cols ?? 80, rows: serialized?.rows ?? size?.rows ?? 24, displayMode, - seq: snapshotFrameSeq, + seq: snapshotOutputSeq, cwd: serialized?.cwd, truncated: initialOutputOverflowed, truncatedByByteBudget: serialized?.truncatedByByteBudget, @@ -3424,8 +3424,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ const displayMode = runtime.getMobileDisplayMode(ptyId) // Why: layout seq is the mobile stale-event filter's high-water mark (undefined pre-transition is fail-open). See docs/mobile-terminal-layout-state-machine.md. const layoutSeq = runtime.getLayout(ptyId)?.seq - const snapshotFrameSeq = serialized?.seq ?? layoutSeq - // Why: track the seq that actually covered the buffered chunks (recovery snapshots advance it) or an absorbed query gets zero replies. + // Why: only an output offset can cover buffered chunks; layout versions are a separate sequence domain. let snapshotOutputSeq = serialized?.seq emit({ type: 'subscribed', @@ -3445,7 +3444,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ cols: serialized?.cols ?? size?.cols ?? params.viewport?.cols ?? 80, rows: serialized?.rows ?? size?.rows ?? params.viewport?.rows ?? 24, displayMode, - seq: snapshotFrameSeq, + seq: snapshotOutputSeq, cwd: serialized?.cwd, truncated: initialOutputOverflowed, truncatedByByteBudget: serialized?.truncatedByByteBudget, diff --git a/src/main/runtime/rpc/terminal-multiplex.test.ts b/src/main/runtime/rpc/terminal-multiplex.test.ts index dd46b3665..95ad62ec1 100644 --- a/src/main/runtime/rpc/terminal-multiplex.test.ts +++ b/src/main/runtime/rpc/terminal-multiplex.test.ts @@ -444,6 +444,58 @@ function startSourceRangeOverflowHarness(options: { } describe('terminal multiplex RPC', () => { + it('keeps layout versions out of the output sequence domain', async () => { + let dataListener: ((data: string, meta?: RuntimeTerminalDataMeta) => void) | undefined + const harness = startDesktopMultiplexSubscribe({ + getLayout: vi.fn().mockReturnValue({ seq: 675 }), + serializeTerminalBuffer: vi.fn().mockResolvedValue({ + data: 'restored terminal', + cols: 120, + rows: 40, + source: 'renderer' + }), + subscribeToTerminalData: vi.fn((_ptyId, listener) => { + dataListener = listener + return vi.fn() + }) + }) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + sendDesktopMultiplexSubscribe(harness.handlers) + await vi.waitFor(() => + expect( + harness.messages.some((message) => JSON.parse(message).result?.type === 'subscribed') + ).toBe(true) + ) + + const subscribed = harness.messages + .map((message) => JSON.parse(message).result) + .find((event) => event?.type === 'subscribed') + const snapshotStart = harness.binaryFrames + .map(decodeTerminalStreamFrame) + .find((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart) + expect(subscribed.seq).toBe(675) + if (!snapshotStart) { + throw new Error('Missing multiplex snapshot start frame') + } + const snapshotPayload = decodeTerminalStreamJson(snapshotStart.payload) + expect(snapshotPayload).toMatchObject({ kind: 'scrollback' }) + expect(snapshotPayload).not.toHaveProperty('seq') + + harness.binaryFrames.splice(0) + dataListener?.('live', { seq: 4, rawLength: 4 }) + await vi.waitFor(() => + expect( + harness.binaryFrames.some((bytes) => { + const frame = decodeTerminalStreamFrame(bytes) + return frame?.opcode === TerminalStreamOpcode.Output && frame.seq === 4 + }) + ).toBe(true) + ) + + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + it.each(['headless', 'renderer'] as const)( 'commits a source-range replacement only after the %s snapshot publishes', async (source) => { diff --git a/src/main/runtime/rpc/terminal-subscribe-buffer.test.ts b/src/main/runtime/rpc/terminal-subscribe-buffer.test.ts index 837e49eff..8265e713f 100644 --- a/src/main/runtime/rpc/terminal-subscribe-buffer.test.ts +++ b/src/main/runtime/rpc/terminal-subscribe-buffer.test.ts @@ -833,15 +833,13 @@ describe('terminal subscribe buffering', () => { const decodedFrames = binaryFrames .map((frame) => decodeTerminalStreamFrame(frame)) .filter((frame): frame is NonNullable => frame !== null) - const snapshotStarts = decodedFrames.filter( + const snapshotStart = decodedFrames.find( (frame) => frame.opcode === TerminalStreamOpcode.SnapshotStart ) - // Why seq 1 (layout seq): a no-output-seq snapshot falls back to the - // layout seq on the wire; the recovered data still ships as the first - // and only scrollback snapshot. - expect(snapshotStarts.map((frame) => decodeTerminalStreamJson(frame.payload))).toEqual([ - expect.objectContaining({ kind: 'scrollback', seq: 1 }) - ]) + // Why: layout versions and output offsets are different sequence domains. + const snapshotInfo = decodeTerminalStreamJson(snapshotStart!.payload) + expect(snapshotInfo).toMatchObject({ kind: 'scrollback' }) + expect(snapshotInfo).not.toHaveProperty('seq') const snapshotText = decodedFrames .filter((frame) => frame.opcode === TerminalStreamOpcode.SnapshotChunk) .map((frame) => decodeTerminalStreamText(frame.payload)) diff --git a/src/main/runtime/terminal-subscriber-driven-daemon-attach.test.ts b/src/main/runtime/terminal-subscriber-driven-daemon-attach.test.ts index 7fd6c42b5..09c39f201 100644 --- a/src/main/runtime/terminal-subscriber-driven-daemon-attach.test.ts +++ b/src/main/runtime/terminal-subscriber-driven-daemon-attach.test.ts @@ -37,6 +37,12 @@ type RuntimeInternals = { ) => unknown issuePtyHandle: (pty: unknown) => string headlessTerminals: Map + subscriberDrivenProviderAttachesByPtyId: Map> + subscriberDrivenProviderAttachInventoryWaiters: Set + refreshPtyWorktreeRecordsWithControllerInventory: ( + worktrees: [], + targetWorktreeId: string | null + ) => Promise } function internals(runtime: OrcaRuntimeService): RuntimeInternals { @@ -56,11 +62,21 @@ function createDaemonProviderModel(opts: { snapshotCapable: boolean }) { const sessions = new Map() const attachCalls: string[] = [] const resizeCalls: [string, number, number][] = [] + let nextAttachBarrier: { promise: Promise; result: boolean } | null = null let runtime: OrcaRuntimeService | null = null const controller = { write: () => true, kill: () => true, getForegroundProcess: async () => null, + listProcesses: async () => + [...sessions.entries()].map(([id, session]) => ({ + id, + cwd: '/tmp/wt', + worktreeId: WORKTREE_ID, + title: '', + cols: session.cols, + rows: session.rows + })), hasRendererSerializer: () => false, getSize: (ptyId: string) => { const session = sessions.get(ptyId) @@ -72,6 +88,14 @@ function createDaemonProviderModel(opts: { snapshotCapable: boolean }) { }, attach: async (ptyId: string) => { attachCalls.push(ptyId) + const barrier = nextAttachBarrier + nextAttachBarrier = null + if (barrier) { + await barrier.promise + if (!barrier.result) { + return false + } + } const session = sessions.get(ptyId) // Attach-only: an absent session is refused, never created. if (!session) { @@ -100,6 +124,14 @@ function createDaemonProviderModel(opts: { snapshotCapable: boolean }) { sessions, attachCalls, resizeCalls, + deferNextAttach(result: boolean): () => void { + let release!: () => void + const promise = new Promise((resolve) => { + release = resolve + }) + nextAttachBarrier = { promise, result } + return release + }, bind(target: OrcaRuntimeService) { runtime = target }, @@ -376,6 +408,39 @@ describe('subscriber-driven daemon attach (never-activated tab)', () => { await vi.waitFor(() => expect(outputText(harness)).toContain('late daemon hello')) }) + it('retries an existing subscriber when provider inventory becomes ready', async () => { + const { runtime, model, handle } = setupNeverAttachedDaemonSession({ + snapshotCapable: false + }) + model.sessions.delete(PTY_ID) + const refuseFirstAttach = model.deferNextAttach(false) + const harness = startMultiplex(runtime) + await vi.waitFor(() => expect(harness.handlers.has(0)).toBe(true)) + + sendSubscribe(harness, 1, handle) + await waitForSubscribed(harness, 1) + await vi.waitFor(() => expect(model.attachCalls).toEqual([PTY_ID])) + expect(internals(runtime).subscriberDrivenProviderAttachesByPtyId.has(PTY_ID)).toBe(true) + expect(runtime.hasRemoteTerminalViewSubscriber(PTY_ID)).toBe(true) + expect(model.emitData(PTY_ID, 'before inventory\r\n')).toBe(false) + + model.sessions.set(PTY_ID, { cols: 100, rows: 30, attached: false, screen: '' }) + await internals(runtime).refreshPtyWorktreeRecordsWithControllerInventory([], null) + await internals(runtime).refreshPtyWorktreeRecordsWithControllerInventory([], null) + expect(model.attachCalls).toEqual([PTY_ID]) + expect([...internals(runtime).subscriberDrivenProviderAttachInventoryWaiters]).toEqual([PTY_ID]) + refuseFirstAttach() + + await vi.waitFor(() => expect(model.attachCalls).toEqual([PTY_ID, PTY_ID])) + await vi.waitFor(() => + expect(internals(runtime).subscriberDrivenProviderAttachInventoryWaiters.has(PTY_ID)).toBe( + false + ) + ) + expect(model.emitData(PTY_ID, 'after inventory\r\n')).toBe(true) + await vi.waitFor(() => expect(outputText(harness)).toContain('after inventory')) + }) + it('does not subscriber-attach or provider-read a session this app already spawned', async () => { const { runtime, model, handle } = setupNeverAttachedDaemonSession({ snapshotCapable: true, diff --git a/tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts b/tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts new file mode 100644 index 000000000..2d0eb5a9a --- /dev/null +++ b/tests/e2e/paired-remote-terminal-host-restart-background-sync.spec.ts @@ -0,0 +1,447 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { ElectronApplication, Page } from '@stablyai/playwright-test' +import { PROTOCOL_VERSION } from '../../src/main/daemon/types' +import { + HOST_TERMINAL_SURFACE_SEPARATOR, + toWebTerminalSurfaceTabId +} from '../../src/shared/terminal-surface-id' +import { expect, test } from './helpers/orca-app' +import { TEST_REPO_PATH_FILE } from './global-setup' +import { + createRuntimeDesktopPairingOffer, + launchPairedElectronClient, + type PairedElectronClient +} from './helpers/paired-electron-client' +import { attachRepoAndOpenTerminal, createRestartSession } from './helpers/orca-restart' +import { waitForTabParked } from './helpers/terminal-hidden-parking' + +const PARK_DELAY_MS = 2_000 +const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-host-restart-background-')) +const fixturePath = path.join(scratch, 'paired-host-restart-terminal.mjs') +const backlogPath = path.join(scratch, 'daemon-stream-backlog.jsonl') + +writeFileSync( + fixturePath, + [ + "import { appendFileSync } from 'node:fs'", + 'const sink = process.argv[2]', + "process.stdout.write('READY\\r\\n')", + "process.stdin.setEncoding('utf8')", + "let pending = ''", + "process.stdin.on('data', (data) => {", + ' pending += data', + ' const lines = pending.split(/\\r\\n|\\r|\\n/)', + " pending = lines.pop() ?? ''", + ' for (const line of lines) {', + ' appendFileSync(sink, `${line}\\n`)', + ' process.stdout.write(`LIVE:${line}\\r\\n`)', + ' }', + '})', + 'process.stdin.resume()' + ].join('\n') +) + +test.afterAll(() => { + rmSync(scratch, { recursive: true, force: true }) +}) + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(sinkPath: string): string { + const command = [process.execPath, fixturePath, sinkPath] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +function seededRepoPathOrSkip(): string { + const repoPath = existsSync(TEST_REPO_PATH_FILE) + ? readFileSync(TEST_REPO_PATH_FILE, 'utf8').trim() + : '' + test.skip(!repoPath || !existsSync(repoPath), 'Global setup did not produce a seeded test repo') + return repoPath +} + +function readDaemonPid(userDataDir: string): number { + const value = JSON.parse( + readFileSync(path.join(userDataDir, 'daemon', `daemon-v${PROTOCOL_VERSION}.pid`), 'utf8') + ) as { pid?: unknown } + if (typeof value.pid !== 'number') { + throw new Error('Daemon pid file did not contain a numeric pid') + } + return value.pid +} + +type BacklogEntry = { + atMs?: number + background?: boolean + backgroundedSessionIdSuffixes?: string[] + event?: string + sessionIdSuffix?: string +} + +function readBacklogEntries(): BacklogEntry[] { + try { + return readFileSync(backlogPath, 'utf8') + .split(/\r?\n/) + .filter(Boolean) + .flatMap((line) => { + try { + return [JSON.parse(line) as BacklogEntry] + } catch { + return [] + } + }) + } catch { + return [] + } +} + +function readText(filePath: string): string { + try { + return readFileSync(filePath, 'utf8') + } catch { + return '' + } +} + +async function callRuntime( + page: Page, + environmentId: string, + method: string, + params: unknown +): Promise { + return page.evaluate( + async ({ environmentId, method, params }) => { + const response = await window.api.runtimeEnvironments.call({ + selector: environmentId, + method, + params + }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { environmentId, method, params } + ) as Promise +} + +type HostTerminal = { + handle: string + parentTabId: string + ptyId: string + sinkPath: string + webTabId: string +} + +async function createHostTerminal( + client: PairedElectronClient, + worktreeId: string, + name: string +): Promise { + const sinkPath = path.join(scratch, `${name}.log`) + const created = await callRuntime<{ + tab: { id: string; parentTabId: string; terminal: string | null } + }>(client.page, client.environmentId, 'session.tabs.createTerminal', { + worktree: `id:${worktreeId}`, + command: fixtureCommand(sinkPath), + activate: false, + select: false, + navigation: 'caller' + }) + if (!created.tab.terminal) { + throw new Error('Host did not publish the fixture terminal') + } + const shown = await callRuntime<{ terminal: { ptyId: string | null } }>( + client.page, + client.environmentId, + 'terminal.show', + { terminal: created.tab.terminal } + ) + if (!shown.terminal.ptyId) { + throw new Error('Host fixture terminal has no PTY') + } + const parentTabId = + created.tab.parentTabId || created.tab.id.split(HOST_TERMINAL_SURFACE_SEPARATOR)[0] + return { + handle: created.tab.terminal, + parentTabId, + ptyId: shown.terminal.ptyId, + sinkPath, + webTabId: toWebTerminalSurfaceTabId(parentTabId) + } +} + +async function findTerminalHandle( + client: PairedElectronClient, + worktreeId: string, + parentTabId: string +): Promise { + let terminal: string | null = null + await expect + .poll( + async () => { + const snapshot = await callRuntime<{ + tabs: { type: string; parentTabId?: string; terminal?: string | null }[] + }>(client.page, client.environmentId, 'session.tabs.list', { + worktree: `id:${worktreeId}` + }) + terminal = + snapshot.tabs.find((tab) => tab.type === 'terminal' && tab.parentTabId === parentTabId) + ?.terminal ?? null + return terminal + }, + { timeout: 30_000, message: `Host did not republish terminal tab ${parentTabId}` } + ) + .not.toBeNull() + if (!terminal) { + throw new Error(`Host did not republish terminal tab ${parentTabId}`) + } + return terminal +} + +async function openClientTab(page: Page, worktreeId: string, webTabId: string): Promise { + await expect + .poll( + () => + page.evaluate( + ({ worktreeId, webTabId }) => + (window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some( + (tab) => tab.id === webTabId + ), + { worktreeId, webTabId } + ), + { timeout: 60_000, message: `Client never mirrored host tab ${webTabId}` } + ) + .toBe(true) + await page.evaluate( + ({ worktreeId, webTabId }) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(worktreeId) + state?.setActiveTab(webTabId) + state?.setActiveTabType('terminal') + }, + { worktreeId, webTabId } + ) + await expect + .poll(() => page.evaluate((id) => window.__paneManagers?.has(id) ?? false, webTabId), { + timeout: 60_000, + message: `Client pane for ${webTabId} did not mount` + }) + .toBe(true) +} + +async function readPaneContent(page: Page, webTabId: string): Promise { + return page.evaluate((id) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return pane?.serializeAddon?.serialize?.() ?? '' + }, webTabId) +} + +async function waitForPaneConnected(page: Page, webTabId: string): Promise { + await expect + .poll( + () => + page.evaluate((id) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return pane?.container.dataset.ptyRecoveryState ?? null + }, webTabId), + { timeout: 30_000, message: `Pane ${webTabId} never completed transport recovery` } + ) + .toBe('connected') +} + +async function expectTerminalInteractive( + client: PairedElectronClient, + target: HostTerminal, + marker: string +): Promise { + await client.page.evaluate((id) => { + const manager = window.__paneManagers?.get(id) + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + if (!pane) { + throw new Error(`No pane mounted for ${id}`) + } + pane.terminal.focus() + const textarea = pane.container.querySelector('.xterm-helper-textarea') as HTMLTextAreaElement + textarea.focus() + }, target.webTabId) + await client.page.keyboard.type(marker) + await client.page.keyboard.press('Enter') + await expect.poll(() => readText(target.sinkPath), { timeout: 15_000 }).toContain(marker) + await expect + .poll(() => readPaneContent(client.page, target.webTabId), { timeout: 15_000 }) + .toContain(`LIVE:${marker}`) +} + +async function moveHostAwayFromWorktree(page: Page, targetWorktreeId: string): Promise { + const alternateWorktreeId = await page.evaluate((targetId) => { + const state = window.__store?.getState() + const alternate = state?.allWorktrees().find((worktree) => worktree.id !== targetId) + if (!state || !alternate) { + return null + } + state.setActiveView('editor') + state.setActiveWorktree(alternate.id) + return alternate.id + }, targetWorktreeId) + if (!alternateWorktreeId) { + throw new Error('Host fixture needs a second worktree for inactive-workspace restart coverage') + } + await expect + .poll(() => page.evaluate(() => window.__store?.getState().activeWorktreeId ?? null)) + .toBe(alternateWorktreeId) + return alternateWorktreeId +} + +test('foregrounds a preserved daemon PTY after the paired host relaunches', async (// oxlint-disable-next-line no-empty-pattern -- This lifecycle test owns both host launches. +{}, testInfo) => { + test.setTimeout(360_000) + const repoPath = seededRepoPathOrSkip() + writeFileSync(backlogPath, '') + const previousParkDelay = process.env.ORCA_E2E_TERMINAL_PARKING_DELAY_MS + process.env.ORCA_E2E_TERMINAL_PARKING_DELAY_MS = String(PARK_DELAY_MS) + const session = createRestartSession(testInfo, { + ORCA_DAEMON_STREAM_BACKLOG_FILE: backlogPath + }) + let firstHost: ElectronApplication | null = null + let secondHost: ElectronApplication | null = null + let client: PairedElectronClient | null = null + const terminals: HostTerminal[] = [] + try { + const first = await session.launch() + firstHost = first.app + const worktreeId = await attachRepoAndOpenTerminal(first.page, repoPath) + const daemonPid = readDaemonPid(session.userDataDir) + client = await launchPairedElectronClient( + await createRuntimeDesktopPairingOffer(first.page), + testInfo, + 'host-restart-background-sync' + ) + await expect + .poll( + () => + client!.page.evaluate( + (id) => + window.__store + ?.getState() + .allWorktrees() + .some((worktree) => worktree.id === id) ?? false, + worktreeId + ), + { timeout: 60_000, message: 'Paired client never saw the host worktree' } + ) + .toBe(true) + + const target = await createHostTerminal(client, worktreeId, 'target') + const firstDecoy = await createHostTerminal(client, worktreeId, 'first-decoy') + const parkingDecoy = await createHostTerminal(client, worktreeId, 'parking-decoy') + terminals.push(target, firstDecoy, parkingDecoy) + await openClientTab(client.page, worktreeId, target.webTabId) + await expect + .poll(() => readPaneContent(client!.page, target.webTabId), { timeout: 30_000 }) + .toContain('READY') + await openClientTab(client.page, worktreeId, firstDecoy.webTabId) + await openClientTab(client.page, worktreeId, parkingDecoy.webTabId) + await waitForTabParked(client.page, target.webTabId, { parkDelayMs: PARK_DELAY_MS }) + + const targetSuffix = target.ptyId.slice(-10) + await expect + .poll( + () => + readBacklogEntries() + .findLast((entry) => entry.backgroundedSessionIdSuffixes) + ?.backgroundedSessionIdSuffixes?.includes(targetSuffix) ?? false, + { timeout: 15_000, message: 'Daemon never retained the target background hint' } + ) + .toBe(true) + + const alternateHostWorktreeId = await moveHostAwayFromWorktree(first.page, worktreeId) + await session.close(firstHost) + firstHost = null + const foregroundEventStartMs = Date.now() + const second = await session.launch() + secondHost = second.app + await second.page.waitForFunction( + (expectedId) => + window.__store?.getState().workspaceSessionReady === true && + window.__store?.getState().activeWorktreeId === expectedId, + alternateHostWorktreeId, + { timeout: 30_000 } + ) + expect(readDaemonPid(session.userDataDir), 'daemon must survive the host relaunch').toBe( + daemonPid + ) + await expect + .poll( + () => + client!.page.evaluate(async (selector) => { + const response = await window.api.runtimeEnvironments.connect({ selector }) + return response.ok + }, client!.environmentId), + { timeout: 60_000, message: 'Paired client never reconnected to the relaunched host' } + ) + .toBe(true) + + target.handle = await findTerminalHandle(client, worktreeId, target.parentTabId) + const shown = await callRuntime<{ terminal: { ptyId: string | null } }>( + client.page, + client.environmentId, + 'terminal.show', + { terminal: target.handle } + ) + expect(shown.terminal.ptyId).toBe(target.ptyId) + await openClientTab(client.page, worktreeId, target.webTabId) + await waitForPaneConnected(client.page, target.webTabId) + await expect + .poll( + () => + readBacklogEntries().some( + (entry) => + (entry.atMs ?? 0) >= foregroundEventStartMs && + entry.event === 'setSessionBackground' && + entry.sessionIdSuffix === targetSuffix && + entry.background === false + ), + { timeout: 15_000, message: 'Relaunched host never foregrounded the preserved PTY' } + ) + .toBe(true) + await expectTerminalInteractive(client, target, 'x') + + const reconnectControl = await createHostTerminal(client, worktreeId, 'reconnect-control') + terminals.push(reconnectControl) + expect(reconnectControl.ptyId).not.toBe(target.ptyId) + await openClientTab(client.page, worktreeId, reconnectControl.webTabId) + await waitForPaneConnected(client.page, reconnectControl.webTabId) + await expectTerminalInteractive(client, reconnectControl, 'y') + } finally { + if (client) { + for (const terminal of terminals) { + await callRuntime(client.page, client.environmentId, 'terminal.closeTab', { + terminal: terminal.handle + }).catch(() => undefined) + } + await client.dispose() + } + if (secondHost) { + await session.close(secondHost) + } + if (firstHost) { + await session.close(firstHost) + } + await session.dispose() + if (previousParkDelay === undefined) { + delete process.env.ORCA_E2E_TERMINAL_PARKING_DELAY_MS + } else { + process.env.ORCA_E2E_TERMINAL_PARKING_DELAY_MS = previousParkDelay + } + } +})