From bf894ef150e0c8da557d1518cf5e82cc8625cf5d Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Wed, 29 Jul 2026 20:04:55 -0700 Subject: [PATCH] fix(remote): recover and safely park paired terminals (#11416) --- config/reliability-gates.jsonc | 325 +++++++++++- ...ntime-environment-connectivity-handlers.ts | 173 +++++++ src/main/ipc/runtime-environments.test.ts | 44 ++ src/main/ipc/runtime-environments.ts | 109 +--- src/main/runtime/orca-runtime.test.ts | 156 ++++++ src/main/runtime/orca-runtime.ts | 108 ++-- src/main/runtime/rpc/methods/terminal.ts | 51 +- ...minal-multiplex-resync-replay-trim.test.ts | 20 +- .../runtime/rpc/terminal-multiplex.test.ts | 259 +++++++++- .../rpc/terminal-source-range-registry.ts | 4 +- .../rpc/terminal-subscribe-buffer.test.ts | 10 +- src/preload/api-types.ts | 4 + src/preload/index.ts | 5 + src/renderer/src/components/Terminal.tsx | 47 +- .../settings/RuntimeEnvironmentsPane.tsx | 2 +- .../status-bar/SshStatusSegment.tsx | 6 +- .../runtime-environment-explicit-connect.ts | 22 + .../parked-terminal-byte-watcher.test.ts | 26 + .../parked-terminal-byte-watcher.ts | 58 ++- .../terminal-pane/pty-connection.test.ts | 56 ++ .../terminal-pane/pty-connection.ts | 77 ++- .../remote-runtime-pty-transport.test.ts | 120 ++++- .../remote-runtime-pty-transport.ts | 93 +++- .../terminal-pane/replay-guard.test.ts | 37 ++ .../components/terminal-pane/replay-guard.ts | 52 +- .../terminal-hidden-view-parking.test.ts | 63 ++- .../terminal-hidden-view-parking.ts | 40 +- ...terminal-hidden-worktree-retention.test.ts | 34 ++ .../terminal-hidden-worktree-retention.ts | 13 + .../terminal-parked-pty-watcher.ts | 131 +++++ .../terminal-parked-tab-watchers.test.ts | 26 + .../terminal-parked-tab-watchers.ts | 306 +++++------ ...ked-watcher-partial-reconciliation.test.ts | 111 ++++ ...inal-parked-watcher-reconciliation.test.ts | 87 ++++ .../terminal-parked-watcher-reconciliation.ts | 91 ++++ .../terminal-parking-e2e-overrides.ts | 26 +- ...terminal-side-effect-facts-handler.test.ts | 2 + .../terminal-side-effect-facts-handler.ts | 51 +- ...background-terminal-worktree-mount.test.ts | 27 + .../background-terminal-worktree-mount.ts | 10 + src/renderer/src/env.d.ts | 3 + src/renderer/src/hooks/useIpcEvents.ts | 9 + src/renderer/src/i18n/locales/en.json | 3 +- src/renderer/src/i18n/locales/es.json | 3 +- src/renderer/src/i18n/locales/ja.json | 3 +- src/renderer/src/i18n/locales/ko.json | 3 +- src/renderer/src/i18n/locales/zh.json | 3 +- src/renderer/src/lib/e2e-config.ts | 21 +- .../remote-runtime-terminal-multiplexer.ts | 159 +++++- ...te-runtime-terminal-stall-recovery.test.ts | 277 ++++++++++ .../remote-terminal-stream-watchdog.ts | 108 ++++ .../src/runtime/runtime-client-events.test.ts | 13 +- .../src/runtime/runtime-client-events.ts | 1 + .../src/runtime/runtime-terminal-stream.ts | 31 +- src/renderer/src/web/web-preload-api.test.ts | 172 +++++++ src/renderer/src/web/web-preload-api.ts | 87 +++- src/shared/protocol-version.ts | 4 + src/shared/runtime-client-events.ts | 2 + src/shared/terminal-multiplex-flow-control.ts | 5 +- ...d-remote-terminal-retention-memory.spec.ts | 40 ++ ...red-remote-terminal-stall-recovery.spec.ts | 226 ++++++++ .../helpers/headless-paired-runtime-host.ts | 149 ++++++ tests/e2e/helpers/paired-electron-client.ts | 63 ++- .../paired-terminal-parking-fixture.ts | 48 ++ .../helpers/paired-terminal-parking-oracle.ts | 284 ++++++++++ tests/e2e/helpers/paired-web-client-url.ts | 23 + ...d-remote-terminal-retention-memory.spec.ts | 34 ++ ...red-remote-terminal-stall-recovery.spec.ts | 251 +++++++++ ...erminal-truncated-tail-first-paint.spec.ts | 483 ++++++++++++++++++ tests/e2e/paired-runtime-retention-metrics.ts | 67 +++ 70 files changed, 4916 insertions(+), 541 deletions(-) create mode 100644 src/main/ipc/runtime-environment-connectivity-handlers.ts create mode 100644 src/renderer/src/components/status-bar/runtime-environment-explicit-connect.ts create mode 100644 src/renderer/src/components/terminal-pane/terminal-parked-pty-watcher.ts create mode 100644 src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts create mode 100644 src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts create mode 100644 src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.ts create mode 100644 src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts create mode 100644 src/renderer/src/runtime/remote-terminal-stream-watchdog.ts create mode 100644 tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts create mode 100644 tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts create mode 100644 tests/e2e/helpers/headless-paired-runtime-host.ts create mode 100644 tests/e2e/helpers/paired-terminal-parking-fixture.ts create mode 100644 tests/e2e/helpers/paired-terminal-parking-oracle.ts create mode 100644 tests/e2e/helpers/paired-web-client-url.ts create mode 100644 tests/e2e/paired-remote-terminal-retention-memory.spec.ts create mode 100644 tests/e2e/paired-remote-terminal-stall-recovery.spec.ts create mode 100644 tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts create mode 100644 tests/e2e/paired-runtime-retention-metrics.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 38693c5b3..40e20fb26 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -3067,6 +3067,277 @@ ], "demotionRule": "Keep experimental or demote to protection none if output differs across replay slices, header-only restores consume a replay slot, retained payload bytes exceed the cap, or the focused gate flakes." }, + { + "id": "terminal-performance.remote-hidden-retention-budget", + "title": "Paired terminals park client renderers while host PTYs preserve bounded history", + "maturity": "experimental", + "protection": "partial", + "owner": "terminal-runtime", + "layer": "paired-headed-and-headless-runtime", + "surfaces": [ + "paired remote terminal first paint", + "paired remote terminal ordinary parking", + "paired remote terminal bounded scrollback restore", + "stalled paired terminal stream recovery", + "hidden remote worktree retention", + "remote terminal reveal and input", + "manual server disconnect" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["paired-runtime", "ssh"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["paired-runtime"], + "coverageNotes": "Deterministic headed macOS runs launch an isolated Orca desktop server and a separate paired web client. A byte-identical headless run uses an isolated `orca serve` host. Both create six real paired host PTYs with bounded high-output scrollback, ordinary-park five client xterms without enabling the lossy retention budget, assert bounded cells/heap/timer lag, then restore 5,000 rows on the original PTY including output produced while parked and continued input/output. Separate headed/headless ACK-starvation tests recover one stalled stream without replacing its PTY. A capability-disabled run proves legacy hosts retain the prior lossy limit/TTL fallback. Unit tests cover exact-owner capability routing, raw-stream release, singleton side-effect facts with timed handoff cleanup, provider-authoritative snapshots, 128 active streams plus retry after a 129th-stream capacity rejection, capacity-pressure backoff, full split-leaf remint reconciliation, truncation, and manual-disconnect queue fencing. Linux, Windows, mixed-version paired hosts, live SSH, and production-scale paired hosts remain gaps.", + "motivatingLinks": [ + "https://github.com/stablyai/orca/issues/8652", + "https://github.com/stablyai/orca/pull/10625" + ], + "invariant": "A host advertising terminal.paired-parking.v1 keeps the PTY and bounded authoritative history alive while an ordinary hidden-view park destroys the client xterm and releases its raw per-PTY stream. Reveal must restore up to the requested 5,000 rows, parked-time side effects/output, the same PTY identity, and continued input/output. Hosts without the capability must retain the existing limit/TTL force-parking fallback. A paired terminal stream whose delivery credits stop progressing must replace only that stream; command silence must first probe authoritative state and replace only if that probe times out. Manual disconnect must retain pairing while preventing queued or passive calls and subscriptions from recreating transport until explicit Connect.", + "oracle": "Run one byte-identical six-terminal oracle against an isolated headed desktop host and an isolated headless `orca serve` host. Stage at least 1,000,000 xterm cells, enable ordinary parking with the lossy retention budget disabled, require exactly one mounted manager and five parked tabs, at most 45% retained cells, no more than 16 MiB heap growth, and under 500 ms timer drift. While a tab is parked, require authoritative terminal.read to observe new PTY output; reveal it and require the original PTY, a marker within the requested 5,000-row history, the parked marker, and post-reveal input/output. Admit 128 active streams, reject the 129th as retryable, release one stream, then require the retry to attach and publish a snapshot without multiplying retained subscribers. Disable terminal.paired-parking.v1 and require the same oracle to fail before parking, while the legacy limit-one fallback separately passes. Also preserve truncated first paint, ACK-starved same-PTY recovery, responsive silent-command snapshot probes, dead-stream probe timeout recovery, and queued manual-disconnect fencing.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/web/web-preload-api.test.ts src/main/ipc/runtime-environments.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts", + "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "ORCA_E2E_WEB_CLIENT=1 ORCA_E2E_DISABLE_PAIRED_TERMINAL_PARKING=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-retention-memory.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/terminal-parked-memory.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1" + ], + "testFiles": [ + "src/main/runtime/orca-runtime.test.ts", + "src/renderer/src/web/web-preload-api.test.ts", + "src/main/ipc/runtime-environments.test.ts", + "src/main/runtime/rpc/terminal-multiplex.test.ts", + "src/main/runtime/rpc/terminal-subscribe-buffer.test.ts", + "src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts", + "src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts", + "src/renderer/src/components/terminal-pane/pty-connection.test.ts", + "src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts", + "src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts", + "src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts", + "src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts", + "src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts", + "src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts", + "src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts", + "src/renderer/src/runtime/remote-runtime-terminal-parse-backpressure.test.ts", + "src/renderer/src/runtime/runtime-client-events.test.ts", + "src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts", + "tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts", + "tests/e2e/paired-remote-terminal-stall-recovery.spec.ts", + "tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts", + "tests/e2e/paired-remote-terminal-retention-memory.spec.ts", + "tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts", + "tests/e2e/terminal-parked-memory.spec.ts" + ], + "assertionRefs": [ + { + "file": "tests/e2e/paired-remote-terminal-stall-recovery.spec.ts", + "assertions": [ + "restarts one ACK-starved paired terminal stream without replacing its PTY" + ] + }, + { + "file": "tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts", + "assertions": ["recovers an ACK-starved stream from an isolated headless Orca host"] + }, + { + "file": "tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts", + "assertions": [ + "paints a paired remote terminal when only its retained text tail overflowed", + "legacy paired hosts retain the lossy hidden-manager budget fallback" + ] + }, + { + "file": "tests/e2e/paired-remote-terminal-retention-memory.spec.ts", + "assertions": [ + "ordinary-parks paired terminals and restores authoritative host scrollback" + ] + }, + { + "file": "tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts", + "assertions": ["ordinary-parks paired terminals against an isolated headless Orca host"] + }, + { + "file": "tests/e2e/terminal-parked-memory.spec.ts", + "assertions": [ + "releases un-parkable hidden worktree buffers only once the retention budget engages" + ] + }, + { + "file": "src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts", + "assertions": [ + "treats a host-backed paired PTY as settled despite activation residue", + "preserves real startup work and non-paired activation guards", + "force-parks the least-recently-hidden candidates beyond the retention limit" + ] + }, + { + "file": "src/main/runtime/orca-runtime.test.ts", + "assertions": [ + "forwards facts over the shared client-event stream without a desktop renderer", + "prefers provider history over a partial headless mirror for requested snapshots", + "falls back to the available mirror when authoritative provider history is unavailable", + "bounds a hung authoritative provider acquisition and reuses its fallback" + ] + }, + { + "file": "src/main/runtime/rpc/terminal-multiplex.test.ts", + "assertions": [ + "binary first paint remains valid when only retained history was truncated", + "admits 128 active streams, rejects the 129th, and reuses released capacity", + "reserves PTY wait capacity independently from active streams" + ] + }, + { + "file": "src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts", + "assertions": ["consumes host facts without a raw terminal stream for paired PTYs"] + }, + { + "file": "src/renderer/src/components/terminal-pane/pty-connection.test.ts", + "assertions": ["restores configured paired scrollback after an ordinary park reveal"] + }, + { + "file": "src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts", + "assertions": [ + "selects only reachable hosts advertising the paired parking contract", + "accepts paired ptys only for the exact snapshot-capable owner", + "rejects paired, fail-open, foreign, and null ptys without capability evidence" + ] + }, + { + "file": "src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts", + "assertions": [ + "keeps a mounted HUB mirror alive when the old stream ends before the replacement snapshot" + ] + }, + { + "file": "src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts", + "assertions": ["starts a fact watcher for snapshot-capable paired PTYs"] + }, + { + "file": "src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts", + "assertions": [ + "retains a continuing watcher and title while reconciling a reminted split leaf" + ] + }, + { + "file": "src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts", + "assertions": [ + "restarts only the stream whose renderer delivery credit never settles", + "probes then restarts a stream when an entered command receives no frames", + "keeps a silent responsive stream after its authoritative snapshot probe", + "classifies a capacity rejection followed by end as recoverable transport pressure" + ] + }, + { + "file": "src/renderer/src/web/web-preload-api.test.ts", + "assertions": [ + "keeps pairing while manual disconnect fences passive reconnects", + "fences a web runtime response that completes after manual disconnect", + "returns a disconnect envelope when a queued active runtime call disconnects", + "returns a disconnect envelope when a queued selected environment call disconnects" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/rpc/terminal-multiplex.test.ts src/main/runtime/rpc/terminal-subscribe-buffer.test.ts src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.test.ts src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/web/web-preload-api.test.ts src/main/ipc/runtime-environments.test.ts", + "result": "passed", + "durationSeconds": 19.54, + "summary": "Seventeen focused files passed 1,938 tests with one existing skip, including 128 active streams, retryable rejection at 129, released-slot snapshot recovery, bounded liveness probes, and balanced subscriber cleanup." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "result": "passed", + "durationSeconds": 11.4, + "summary": "The headed paired-server scenario painted a truncated retained history after reload; the capability-disabled legacy fallback scenario is skipped unless explicitly selected." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "result": "passed", + "durationSeconds": 18, + "summary": "The byte-identical headed oracle exhausted one paired stream, proved host/client divergence, then repainted the marker while preserving the original PTY and tab." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-retention-memory.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "result": "passed", + "durationSeconds": 14.5, + "summary": "Six real paired worktrees and host PTYs ordinary-parked five client xterms with the lossy budget disabled, released at least 55% of staged cells within heap and timer-lag budgets, then restored requested row 4,000, parked-time output, and live I/O on the original PTY." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "result": "passed", + "durationSeconds": 14.4, + "summary": "An isolated `orca serve` host and paired web renderer exhausted one terminal stream, recovered it, and preserved the original PTY and tab without exposing readiness or pairing material." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_WEB_CLIENT=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "result": "passed", + "durationSeconds": 14.2, + "summary": "The isolated headless `orca serve` host passed the byte-identical six-PTY ordinary-parking, memory/lag, deep-history, same-PTY, parked-output, and post-reveal I/O oracle." + }, + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "ORCA_E2E_WEB_CLIENT=1 ORCA_E2E_DISABLE_PAIRED_TERMINAL_PARKING=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1", + "result": "passed", + "durationSeconds": 18.6, + "summary": "A host without terminal.paired-parking.v1 preserved the existing lossy limit-one fallback, restored retained output, and continued PTY input/output." + } + ], + "runtimeBudget": { + "p95Seconds": 360, + "scope": "focused units plus isolated headed paired-runtime and renderer-memory scenarios" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "The paired headed oracle is deterministic locally but has no CI or soak history yet." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "The six-PTY ordinary-parking oracle was red in two independently observed stages: the parked paired remount first used detached attach and restored only rows 5,963–5,999, then the requested snapshot still lost to the current-screen replay until park reveals explicitly entered the capability-gated reattach coordinator. The candidate is green on headed and headless hosts, while disabling terminal.paired-parking.v1 makes the byte-identical oracle fail at its capability precondition and leaves the separately tested legacy lossy fallback green. With stalled-stream recovery disabled, the host cursor advanced but the client remained frozen; enabling it repainted the marker with the same PTY and tab. Reverting the active-stream limit to 64 makes the exact 128-stream contract fail at 64; restoring 128 admits every intended stream, rejects 129, and reattaches it after one slot releases." + }, + "performanceBudget": { + "required": true, + "evidence": "Capable paired hosts use ordinary parking: the host PTY and bounded 5,000-row provider model remain, while five hidden client xterms and their raw per-PTY multiplex streams are destroyed. Parked side effects share the singleton runtime client-event stream. Headed and headless oracles require exact reduction from six managers to one, at least 55% staged xterm-cell release, no more than 16 MiB heap growth, and under 500 ms timer drift. Legacy hosts retain the prior 12-worktree/45-minute lossy force-parking policy. Stream recovery adds no polling and is scoped to one stream." + }, + "promotionCriteria": [ + "Collect 100 consecutive CI passes or 14 days of soak history for the headed and headless parking commands.", + "Run the paired topology on Windows and Linux and add one live SSH retention run.", + "Run the stream-stall oracle with a v1.4.160-rc.3 host and v1.4.160-rc.4 client.", + "Add a production-scale paired run with dozens of worktrees and explicit event-loop and renderer-memory budgets.", + "Keep the budget-off control, exact manager count, row-4,000 reveal marker, parked-time output, and post-reveal PTY input in both topology runs." + ], + "knownGaps": [ + "Paired reveal restores at most the requested 5,000 rows; older history is intentionally unavailable.", + "The scaled paired scenarios use six worktrees and ordinary-park five; they prove real host-backed eviction and buffer release, not a 100-worktree soak.", + "Display-off reveal latency is a separate atlas and viewport-reflow class and is not covered by this gate.", + "Historical field trace archives are still required to order replay-wedge events against renderer heartbeat loss and deduplicate archived records.", + "The live windows-issues incident proves a healthy host PTY and dead existing client stream, but lacks per-stream ACK counters; ACK starvation is reproduced, not yet proven as that field incident's unique boundary." + ], + "demotionRule": "Keep experimental or demote to protection none if paired first paint is blank, passive work reconnects after manual disconnect, a capable hidden paired manager survives ordinary parking, a legacy host bypasses its fallback, reveal loses requested bounded history or live PTY identity, retained buffer cells do not fall, or either headed/headless scenario flakes." + }, { "id": "terminal-performance.no-hot-list-sessions", "title": "Hot terminal interactions do not call global PTY session listing", @@ -3181,46 +3452,67 @@ "id": "terminal-observability.lifecycle-breadcrumbs", "title": "Terminal lifecycle anomalies enter crash diagnostics as compact breadcrumbs", "maturity": "experimental", - "protection": "none", + "protection": "partial", "owner": "terminal-runtime", "layer": "renderer-observability", "surfaces": [ "terminal lifecycle", "reattach", "restore", + "replay-wedge identity", "provider ownership", "diagnostics bundle" ], "platforms": ["macos", "linux", "windows"], "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], - "coveredPlatforms": [], + "coveredPlatforms": ["macos"], "coveredProviders": [], - "coverageNotes": "Registered gap on main. The crash-breadcrumb recording and its test exist only on the pending reliability stack. It registers here with its owning split PR.", + "coverageNotes": "Renderer unit coverage proves replay-guard lost-completion and certified-wedge events carry correlatable tab, worktree, durable leaf, pane, and redacted PTY identity without exposing path-bearing values. Full provider lifecycle attribution and diagnostics-bundle artifact proof remain gaps.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/6800", "https://github.com/stablyai/orca/issues/6773" ], - "invariant": "Terminal lifecycle anomalies around reattach, restore, provider ownership, stale liveness, and fallback routing must leave compact, deduped, privacy-safe breadcrumbs in crash diagnostics so future reports can be attributed from evidence.", - "oracle": "The current executable slice calls warnTerminalLifecycleAnomaly with terminal identity, provider, PTY id, binding epoch, and reason, then asserts the existing console warning is preserved and a compact terminal_lifecycle_anomaly crash breadcrumb is recorded once per lifecycle identity. Full pane transition traces and diagnostics-bundle artifact proof remain follow-ups.", - "commands": [], - "testFiles": [], - "assertionRefs": [], - "evidenceRuns": [], + "invariant": "Terminal lifecycle anomalies around reattach, restore, replay wedges, provider ownership, stale liveness, and fallback routing must leave compact, deduped, privacy-safe breadcrumbs in crash diagnostics so future reports can be attributed from evidence. Replay anomalies must distinguish pane managers and PTYs without recording path-bearing worktree or session identities.", + "oracle": "Drop a replay write completion while allowing its FIFO probe to parse, then require the lost-completion breadcrumb to include pane ID, stable hashes for tab/worktree/leaf identity, and a path-redacted PTY ID. Existing wedge tests require both lost-completion and certified-dead paths to record their distinct event names. Full pane transition traces and diagnostics-bundle artifact proof remain follow-ups.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/replay-guard.test.ts" + ], + "testFiles": ["src/renderer/src/components/terminal-pane/replay-guard.test.ts"], + "assertionRefs": [ + { + "file": "src/renderer/src/components/terminal-pane/replay-guard.test.ts", + "assertions": [ + "records correlatable replay identity without exposing worktree or PTY paths", + "releases after the probe itself never parses (wedged pipeline) and reports it" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-07-29", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/replay-guard.test.ts", + "result": "passed", + "durationSeconds": 0.201, + "summary": "The focused file passed 29 tests; replay anomaly breadcrumbs preserved event classification while adding hashed tab/worktree/leaf correlation and a path-redacted PTY identity." + } + ], "runtimeBudget": { "p95Seconds": 10, "scope": "renderer observability unit test" }, "flakeHistory": { "status": "unknown", - "evidence": "Focused unit slice passed locally once; no CI soak history yet." + "evidence": "The replay-guard unit slice passed locally once; no CI soak history yet." }, "redGreenEvidence": { "status": "partial", - "evidence": "Tests would fail if lifecycle anomalies stopped recording crash breadcrumbs or stopped deduping repeated identities. Needs diagnostics-bundle artifact proof and full transition-trace evidence before promotion." + "evidence": "Tests fail if replay anomalies stop recording their event name, omit pane/terminal correlation, or expose the fixture's path-bearing worktree and PTY prefixes. Needs diagnostics-bundle artifact proof and full transition-trace evidence before promotion." }, "performanceBudget": { "required": true, - "evidence": "Breadcrumb recording is deduped and capped by the existing lifecycle anomaly guard; full trace buffers must include size and event-count caps before promotion." + "evidence": "Replay identity hashing is synchronous and only runs when replay writes are queued; it adds no polling or provider calls. Breadcrumb storage remains bounded by the existing crash reporter. Full trace buffers must include size and event-count caps before promotion." }, "promotionCriteria": [ "Add full compact pane lifecycle trace buffer with event-count caps.", @@ -3228,10 +3520,10 @@ "Add forbidden-transition tests for stale close, unknown owner fallback, and stuck zero-size panes." ], "knownGaps": [ - "No executable coverage on main yet; the slice lives on the pending fix-terminal-reliability stack.", - "Current command records anomaly breadcrumbs only, not a full pane lifecycle state-machine trace.", - "Current command does not prove crash/diagnostics bundle export includes the breadcrumb.", - "Current command does not assert forbidden transitions across live Electron/provider flows." + "The replay identity schema has not yet been exercised in a live Electron/provider failure.", + "Current coverage records anomaly breadcrumbs only, not a full pane lifecycle state-machine trace.", + "Current coverage does not prove crash/diagnostics bundle export includes the breadcrumb.", + "Current coverage does not assert forbidden transitions across live Electron/provider flows." ], "demotionRule": "Cannot promote if diagnostics are console-only, unbounded, or missing from support artifacts." }, @@ -4091,6 +4383,7 @@ { "file": "src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts", "assertions": [ + "startup waits for hydration before mounting terminal panes while degraded mode remains interactive", "only an explicit local execution host can defer cold activation", "SSH, remote-runtime, and unresolved owners remain eager" ] diff --git a/src/main/ipc/runtime-environment-connectivity-handlers.ts b/src/main/ipc/runtime-environment-connectivity-handlers.ts new file mode 100644 index 000000000..7f2f49e84 --- /dev/null +++ b/src/main/ipc/runtime-environment-connectivity-handlers.ts @@ -0,0 +1,173 @@ +import { ipcMain } from 'electron' +import { + addEnvironmentFromPairingCode, + listEnvironments, + removeEnvironment, + resolveEnvironment +} from '../../shared/runtime-environment-store' +import { + redactRuntimeEnvironment, + type PublicKnownRuntimeEnvironment +} from '../../shared/runtime-environments' +import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' +import type { RuntimeStatus } from '../../shared/runtime-types' +import type { Store } from '../persistence' +import { closeRemoteRuntimeRequestConnection } from './runtime-environment-request-connections' +import { + callRuntimeEnvironment, + clearSharedControlSupport, + getRuntimeEnvironmentStatus +} from './runtime-environment-transport-routing' + +const manuallyDisconnectedEnvironmentIds = new Set() + +function manuallyDisconnectedResponse( + environment: ReturnType +): RuntimeRpcResponse { + return { + id: 'runtime.manualDisconnect', + ok: false, + error: { + code: 'runtime_manually_disconnected', + message: 'Runtime environment is manually disconnected.' + }, + _meta: { runtimeId: environment.runtimeId } + } +} + +export function isRuntimeEnvironmentManuallyDisconnected(environmentId: string): boolean { + return manuallyDisconnectedEnvironmentIds.has(environmentId) +} + +type ConnectivityHandlerOptions = { + store: Store + getUserDataPath: () => string + invalidateTransport: (environmentId: string) => void +} + +export function registerRuntimeEnvironmentConnectivityHandlers({ + store, + getUserDataPath, + invalidateTransport +}: ConnectivityHandlerOptions): void { + ipcMain.handle('runtimeEnvironments:list', () => + listEnvironments(getUserDataPath()).map(redactRuntimeEnvironment) + ) + ipcMain.handle( + 'runtimeEnvironments:addFromPairingCode', + ( + _event, + args: { name: string; pairingCode: string } + ): { environment: PublicKnownRuntimeEnvironment } => { + const environment = addEnvironmentFromPairingCode(getUserDataPath(), args) + manuallyDisconnectedEnvironmentIds.delete(environment.id) + return { environment: redactRuntimeEnvironment(environment) } + } + ) + ipcMain.handle('runtimeEnvironments:resolve', (_event, args: { selector: string }) => + redactRuntimeEnvironment(resolveEnvironment(getUserDataPath(), args.selector)) + ) + ipcMain.handle( + 'runtimeEnvironments:remove', + (_event, args: { selector: string }): { removed: PublicKnownRuntimeEnvironment } => { + const environment = resolveEnvironment(getUserDataPath(), args.selector) + if (store.getSettings().activeRuntimeEnvironmentId === environment.id) { + throw new Error('Choose another Active Server in Advanced before removing this server.') + } + const removed = removeEnvironment(getUserDataPath(), args.selector) + manuallyDisconnectedEnvironmentIds.delete(removed.id) + invalidateTransport(removed.id) + closeLegacySelectorTransport(args.selector, removed.id) + return { removed: redactRuntimeEnvironment(removed) } + } + ) + ipcMain.handle( + 'runtimeEnvironments:disconnect', + (_event, args: { selector: string }): { disconnected: PublicKnownRuntimeEnvironment } => { + const environment = resolveEnvironment(getUserDataPath(), args.selector) + manuallyDisconnectedEnvironmentIds.add(environment.id) + invalidateTransport(environment.id) + closeLegacySelectorTransport(args.selector, environment.id) + return { disconnected: redactRuntimeEnvironment(environment) } + } + ) + ipcMain.handle( + 'runtimeEnvironments:connect', + async ( + _event, + args: { selector: string; timeoutMs?: number } + ): Promise> => { + const environment = resolveEnvironment(getUserDataPath(), args.selector) + manuallyDisconnectedEnvironmentIds.delete(environment.id) + return getRuntimeEnvironmentStatus(getUserDataPath(), environment.id, args.timeoutMs) + } + ) +} + +export function registerRuntimeEnvironmentPassiveHandlers(getUserDataPath: () => string): void { + registerPassiveStatusHandler(getUserDataPath) + registerPassiveCallHandler(getUserDataPath) +} + +function closeLegacySelectorTransport(selector: string, environmentId: string): void { + if (selector === environmentId) { + return + } + closeRemoteRuntimeRequestConnection(selector) + clearSharedControlSupport(selector) +} + +function registerPassiveStatusHandler(getUserDataPath: () => string): void { + ipcMain.handle( + 'runtimeEnvironments:getStatus', + async ( + _event, + args: { selector: string; timeoutMs?: number } + ): Promise> => { + const environment = resolveEnvironment(getUserDataPath(), args.selector) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + return manuallyDisconnectedResponse(environment) + } + const response = await getRuntimeEnvironmentStatus( + getUserDataPath(), + environment.id, + args.timeoutMs + ) + return isRuntimeEnvironmentManuallyDisconnected(environment.id) + ? manuallyDisconnectedResponse(environment) + : response + } + ) +} + +function registerPassiveCallHandler(getUserDataPath: () => string): void { + ipcMain.handle( + 'runtimeEnvironments:call', + async ( + _event, + args: { + selector: string + method: string + params?: unknown + timeoutMs?: number + expectedEnvironmentPairingRevision?: number + } + ): Promise> => { + const environment = resolveEnvironment(getUserDataPath(), args.selector) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + return manuallyDisconnectedResponse(environment) + } + const response = await callRuntimeEnvironment( + getUserDataPath(), + environment.id, + args.method, + args.params, + args.timeoutMs, + args.expectedEnvironmentPairingRevision + ) + return isRuntimeEnvironmentManuallyDisconnected(environment.id) + ? manuallyDisconnectedResponse(environment) + : response + } + ) +} diff --git a/src/main/ipc/runtime-environments.test.ts b/src/main/ipc/runtime-environments.test.ts index a261625a6..833265705 100644 --- a/src/main/ipc/runtime-environments.test.ts +++ b/src/main/ipc/runtime-environments.test.ts @@ -135,6 +135,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:resolve', 'runtimeEnvironments:remove', 'runtimeEnvironments:disconnect', + 'runtimeEnvironments:connect', 'runtimeEnvironments:retryConnectionsNow', 'runtimeEnvironments:getStatus', 'runtimeEnvironments:call', @@ -155,6 +156,7 @@ describe('registerRuntimeEnvironmentHandlers', () => { 'runtimeEnvironments:resolve', 'runtimeEnvironments:remove', 'runtimeEnvironments:disconnect', + 'runtimeEnvironments:connect', 'runtimeEnvironments:getStatus', 'runtimeEnvironments:call', 'runtimeEnvironments:subscribe', @@ -230,6 +232,12 @@ describe('registerRuntimeEnvironmentHandlers', () => { it('disconnects a saved runtime without removing it', async () => { registerRuntimeEnvironmentHandlers(store as never) + sendRemoteRuntimeRequestMock.mockResolvedValue({ + id: 'status', + ok: true, + result: { runtimeId: 'runtime-remote' }, + _meta: { runtimeId: 'runtime-remote' } + }) const add = handler< { name: string; pairingCode: string }, @@ -250,6 +258,39 @@ describe('registerRuntimeEnvironmentHandlers', () => { const list = handler('runtimeEnvironments:list') expect(await list(null, undefined)).toMatchObject([{ id: added.environment.id, name: 'desk' }]) + + const getStatus = handler<{ selector: string }, { ok: boolean; error?: { code: string } }>( + 'runtimeEnvironments:getStatus' + ) + await expect(getStatus(null, { selector: 'desk' })).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_manually_disconnected' } + }) + const call = handler< + { selector: string; method: string }, + { ok: boolean; error?: { code: string } } + >('runtimeEnvironments:call') + await expect(call(null, { selector: 'desk', method: 'repo.list' })).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_manually_disconnected' } + }) + const subscribe = handler<{ selector: string; method: string }, { subscriptionId: string }>( + 'runtimeEnvironments:subscribe' + ) + await expect( + subscribe(null, { selector: 'desk', method: 'terminal.multiplex' }) + ).rejects.toThrow('runtime_manually_disconnected') + expect(sendRemoteRuntimeRequestMock).not.toHaveBeenCalled() + expect(subscribeRemoteRuntimeRequestMock).not.toHaveBeenCalled() + + const connect = handler<{ selector: string }, { ok: boolean; result?: { runtimeId: string } }>( + 'runtimeEnvironments:connect' + ) + await expect(connect(null, { selector: 'desk' })).resolves.toMatchObject({ + ok: true, + result: { runtimeId: 'runtime-remote' } + }) + expect(sendRemoteRuntimeRequestMock).toHaveBeenCalledOnce() }) it('marks environments owned by ephemeral VM runtimes in the public list', async () => { @@ -1155,9 +1196,12 @@ describe('registerRuntimeEnvironmentHandlers', () => { { disconnected: { id: string; name: string } } >('runtimeEnvironments:disconnect') await disconnect(null, { selector: 'desk' }) + const connect = handler<{ selector: string }, { ok: boolean }>('runtimeEnvironments:connect') + await connect(null, { selector: 'desk' }) await call(null, { selector: 'desk', method: 'repo.list' }) expect(sendRemoteRuntimeRequestMock.mock.calls.map((call) => call[1])).toEqual([ + 'status.get', 'status.get', 'status.get' ]) diff --git a/src/main/ipc/runtime-environments.ts b/src/main/ipc/runtime-environments.ts index 2bd0c1e18..e01a5d9ae 100644 --- a/src/main/ipc/runtime-environments.ts +++ b/src/main/ipc/runtime-environments.ts @@ -1,19 +1,13 @@ import { app, ipcMain } from 'electron' import { randomUUID } from 'node:crypto' -import { - addEnvironmentFromPairingCode, - listEnvironments, - removeEnvironment, - resolveEnvironment -} from '../../shared/runtime-environment-store' -import { - redactRuntimeEnvironment, - type PublicKnownRuntimeEnvironment -} from '../../shared/runtime-environments' -import type { RuntimeStatus } from '../../shared/runtime-types' -import type { RuntimeRpcResponse } from '../../shared/runtime-rpc-envelope' +import { resolveEnvironment } from '../../shared/runtime-environment-store' import type { RemoteRuntimeSubscription } from '../../shared/remote-runtime-client' import type { Store } from '../persistence' +import { + isRuntimeEnvironmentManuallyDisconnected, + registerRuntimeEnvironmentConnectivityHandlers, + registerRuntimeEnvironmentPassiveHandlers +} from './runtime-environment-connectivity-handlers' import { closeRemoteRuntimeRequestConnection } from './runtime-environment-request-connections' import { registerRuntimeEnvironmentRecoveryHandler } from './runtime-environment-recovery-handler' import { @@ -21,9 +15,7 @@ import { getRuntimeEnvironmentTransportGeneration } from './runtime-environment-transport-generation' import { - callRuntimeEnvironment, clearSharedControlSupport, - getRuntimeEnvironmentStatus, resetSharedControlSupport, subscribeRuntimeEnvironment } from './runtime-environment-transport-routing' @@ -34,6 +26,7 @@ const RUNTIME_ENVIRONMENT_HANDLER_CHANNELS = [ 'runtimeEnvironments:resolve', 'runtimeEnvironments:remove', 'runtimeEnvironments:disconnect', + 'runtimeEnvironments:connect', 'runtimeEnvironments:getStatus', 'runtimeEnvironments:call', 'runtimeEnvironments:subscribe', @@ -66,11 +59,6 @@ export function invalidateRuntimeEnvironmentTransport(environmentId: string): vo closeSubscriptionsForEnvironment(environmentId) } -function listPublicRuntimeEnvironments(): PublicKnownRuntimeEnvironment[] { - // Why: a corrupt VM store must not break persisted environment listing. - return listEnvironments(getUserDataPath()).map(redactRuntimeEnvironment) -} - export function registerRuntimeEnvironmentHandlers(store: Store): void { // Why: keep direct re-registration safe even though register-core-handlers // normally guards this path; otherwise the binary send listener can stack. @@ -80,81 +68,13 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { } ipcMain.removeAllListeners('runtimeEnvironments:subscriptionBinary') - ipcMain.handle('runtimeEnvironments:list', listPublicRuntimeEnvironments) - ipcMain.handle( - 'runtimeEnvironments:addFromPairingCode', - ( - _event, - args: { name: string; pairingCode: string } - ): { environment: PublicKnownRuntimeEnvironment } => ({ - environment: redactRuntimeEnvironment(addEnvironmentFromPairingCode(getUserDataPath(), args)) - }) - ) - ipcMain.handle('runtimeEnvironments:resolve', (_event, args: { selector: string }) => - redactRuntimeEnvironment(resolveEnvironment(getUserDataPath(), args.selector)) - ) - ipcMain.handle( - 'runtimeEnvironments:remove', - (_event, args: { selector: string }): { removed: PublicKnownRuntimeEnvironment } => { - const environment = resolveEnvironment(getUserDataPath(), args.selector) - if (store.getSettings().activeRuntimeEnvironmentId === environment.id) { - throw new Error('Choose another Active Server in Advanced before removing this server.') - } - const removed = removeEnvironment(getUserDataPath(), args.selector) - invalidateRuntimeEnvironmentTransport(removed.id) - if (args.selector !== removed.id) { - closeRemoteRuntimeRequestConnection(args.selector) - clearSharedControlSupport(args.selector) - } - return { removed: redactRuntimeEnvironment(removed) } - } - ) - ipcMain.handle( - 'runtimeEnvironments:disconnect', - (_event, args: { selector: string }): { disconnected: PublicKnownRuntimeEnvironment } => { - const environment = resolveEnvironment(getUserDataPath(), args.selector) - // Why: disconnect is intentionally non-destructive; it drops live - // transport state while keeping the paired server available for later. - invalidateRuntimeEnvironmentTransport(environment.id) - if (args.selector !== environment.id) { - closeRemoteRuntimeRequestConnection(args.selector) - clearSharedControlSupport(args.selector) - } - return { disconnected: redactRuntimeEnvironment(environment) } - } - ) + registerRuntimeEnvironmentConnectivityHandlers({ + store, + getUserDataPath, + invalidateTransport: invalidateRuntimeEnvironmentTransport + }) registerRuntimeEnvironmentRecoveryHandler() - ipcMain.handle( - 'runtimeEnvironments:getStatus', - async ( - _event, - args: { selector: string; timeoutMs?: number } - ): Promise> => { - return getRuntimeEnvironmentStatus(getUserDataPath(), args.selector, args.timeoutMs) - } - ) - ipcMain.handle( - 'runtimeEnvironments:call', - async ( - _event, - args: { - selector: string - method: string - params?: unknown - timeoutMs?: number - expectedEnvironmentPairingRevision?: number - } - ): Promise> => { - return callRuntimeEnvironment( - getUserDataPath(), - args.selector, - args.method, - args.params, - args.timeoutMs, - args.expectedEnvironmentPairingRevision - ) - } - ) + registerRuntimeEnvironmentPassiveHandlers(getUserDataPath) ipcMain.handle( 'runtimeEnvironments:subscribe', async ( @@ -176,6 +96,9 @@ export function registerRuntimeEnvironmentHandlers(store: Store): void { throw new Error('Runtime environment subscription id already exists') } const environment = resolveEnvironment(getUserDataPath(), args.selector) + if (isRuntimeEnvironmentManuallyDisconnected(environment.id)) { + throw new Error('runtime_manually_disconnected') + } const pairingRevision = environment.pairingRevision ?? environment.createdAt if ( args.expectedEnvironmentPairingRevision !== undefined && diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index 5daddfb70..8fe931323 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -58,6 +58,7 @@ import { getBaseRefDefault, getBranchConflictKind } from '../git/repo' import { OrchestrationDb } from './orchestration/db' import type { MessagePriority, MessageRow, MessageType } from './orchestration/types' import { + AUTHORITATIVE_TERMINAL_SNAPSHOT_TIMEOUT_MS, appendNormalizedToTailBuffer, appendRecentPtyPathCandidates, buildPreview, @@ -8231,6 +8232,38 @@ describe('OrcaRuntimeService', () => { expect(trackerEntries.get('pty-1')?.commandCodeDetector).toBeNull() }) + it('forwards facts over the shared client-event stream without a desktop renderer', () => { + const runtime = new OrcaRuntimeService(store) + const events: RuntimeClientEvent[] = [] + runtime.syncWindowGraph(HEADLESS_RUNTIME_WINDOW_ID, { tabs: [], leaves: [] }) + const unsubscribe = runtime.onClientEvent((event) => events.push(event)) + + runtime.onPtyData('pty-remote', '\x1b]0;Codex working\x07\x07', 100) + + expect(events).toEqual([ + { + type: 'terminalSideEffects', + batch: { + ptyId: 'pty-remote', + seq: 19, + facts: [ + { + kind: 'title', + normalizedTitle: 'Codex working', + rawTitle: 'Codex working' + }, + { kind: 'agent-working' }, + { kind: 'bell' } + ] + } + } + ]) + + unsubscribe() + runtime.onPtyData('pty-remote', '\x07', 101) + expect(events).toHaveLength(1) + }) + it('emits one batched event per chunk with facts in byte order and attribution', () => { const { runtime, batches } = createSideEffectRuntime() syncSinglePty(runtime) @@ -8619,6 +8652,129 @@ describe('OrcaRuntimeService', () => { }) }) + it('prefers provider history over a partial headless mirror for requested snapshots', async () => { + const { runtime } = createSideEffectRuntime() + const serializeProviderBuffer = vi.fn().mockResolvedValue({ + data: 'authoritative screen\r\n', + scrollbackAnsi: 'deep provider history\r\n', + cols: 120, + rows: 40, + seq: 900, + source: 'headless' + }) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeProviderBuffer, + hasRendererSerializer: () => false + }) + syncSinglePty(runtime) + runtime.onPtyData('pty-1', 'partial current screen\r\n', 100) + + await expect( + runtime.serializeAuthoritativeTerminalBuffer('pty-1', { scrollbackRows: 5000 }) + ).resolves.toMatchObject({ + data: 'authoritative screen\r\n', + scrollbackAnsi: 'deep provider history\r\n', + seq: 900 + }) + expect(serializeProviderBuffer).toHaveBeenCalledWith('pty-1', { + scrollbackRows: 5000 + }) + }) + + it('falls back to the available mirror when authoritative provider history is unavailable', async () => { + const { runtime } = createSideEffectRuntime() + const serializeProviderBuffer = vi.fn().mockResolvedValue(null) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeProviderBuffer, + hasRendererSerializer: () => false + }) + syncSinglePty(runtime) + runtime.onPtyData('pty-1', 'partial current screen\r\n', 100) + + await expect( + runtime.serializeAuthoritativeTerminalBuffer('pty-1', { scrollbackRows: 5000 }) + ).resolves.toMatchObject({ + data: expect.stringContaining('partial current screen'), + source: 'headless' + }) + expect(serializeProviderBuffer).toHaveBeenCalledWith('pty-1', { + scrollbackRows: 5000 + }) + }) + + it('bounds a hung authoritative provider acquisition and reuses its fallback', async () => { + vi.useFakeTimers() + try { + let releaseProvider: (value: null) => void = () => {} + const hungProvider = new Promise((resolve) => { + releaseProvider = resolve + }) + const serializeProviderBuffer = vi + .fn() + .mockReturnValueOnce(hungProvider) + .mockResolvedValueOnce({ + data: 'provider recovered\r\n', + cols: 100, + rows: 30, + seq: 200, + source: 'headless' + }) + const { runtime } = createSideEffectRuntime() + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null, + serializeProviderBuffer, + hasRendererSerializer: () => false + }) + syncSinglePty(runtime) + runtime.onPtyData('pty-1', 'available mirror\r\n', 100) + + const firstSnapshot = runtime.serializeAuthoritativeTerminalBuffer('pty-1', { + scrollbackRows: 5000 + }) + const concurrentSnapshot = runtime.serializeAuthoritativeTerminalBuffer('pty-1', { + scrollbackRows: 5000 + }) + expect(serializeProviderBuffer).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(AUTHORITATIVE_TERMINAL_SNAPSHOT_TIMEOUT_MS) + await expect(firstSnapshot).resolves.toMatchObject({ + data: expect.stringContaining('available mirror'), + source: 'headless' + }) + await expect(concurrentSnapshot).resolves.toMatchObject({ + data: expect.stringContaining('available mirror'), + source: 'headless' + }) + + await expect( + runtime.serializeAuthoritativeTerminalBuffer('pty-1', { scrollbackRows: 5000 }) + ).resolves.toMatchObject({ + data: expect.stringContaining('available mirror'), + source: 'headless' + }) + expect(serializeProviderBuffer).toHaveBeenCalledOnce() + + releaseProvider(null) + await vi.advanceTimersByTimeAsync(0) + await expect( + runtime.serializeAuthoritativeTerminalBuffer('pty-1', { scrollbackRows: 5000 }) + ).resolves.toMatchObject({ + data: 'provider recovered\r\n', + source: 'headless' + }) + expect(serializeProviderBuffer).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + } + }) + it('falls back to provider history when a mounted renderer has not hydrated yet', async () => { const { runtime } = createSideEffectRuntime() const serializeBuffer = vi.fn().mockResolvedValue({ diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 5f66d893b..8376ca52f 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -448,6 +448,7 @@ import { REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY, RUNTIME_CAPABILITIES, RUNTIME_PROTOCOL_VERSION, + TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY, type RuntimeCapability } from '../../shared/protocol-version' import { @@ -1472,6 +1473,21 @@ type ProviderBufferAcquisition = { generation: number scrollbackRows: number promise: Promise + timedOut: boolean +} + +type RuntimeTerminalBufferSnapshot = { + data: string + cols: number + rows: number + seq?: number + cwd?: string | null + lastTitle?: string + source?: 'headless' | 'renderer' + oscLinks?: TerminalOscLinkRange[] + alternateScreen?: boolean + scrollbackAnsi?: string + pendingEscapeTailAnsi?: string } type HeadlessSeedMetadata = { @@ -3056,6 +3072,7 @@ export class OrcaRuntimeService { private readonly onPtyStopped: ((ptyId: string) => void) | null private readonly onTerminalAgentStatus: ((event: RuntimeTerminalAgentStatusEvent) => void) | null private readonly onTerminalSideEffects: ((batch: TerminalSideEffectBatch) => void) | null + private terminalSideEffectLocalConsumerAvailable = false private terminalSideEffectConsumerAvailable = false private readonly getAgentStatusSnapshotFn: (() => AgentStatusIpcPayload[]) | null private readonly getAgentProviderSessionSnapshotFn: (() => AgentStatusIpcPayload[]) | null @@ -4531,7 +4548,9 @@ export class OrcaRuntimeService { (capability !== 'browser.screencast.v1' || canBrowse) && // Why: the nested-runtime E2E needs a real legacy transport without maintaining an old binary fixture. (process.env.ORCA_E2E_DISABLE_RUNTIME_SHARED_CONTROL !== '1' || - capability !== REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) + capability !== REMOTE_RUNTIME_SHARED_CONTROL_CAPABILITY) && + (process.env.ORCA_E2E_DISABLE_PAIRED_TERMINAL_PARKING !== '1' || + capability !== TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY) ) if (hasOffscreen) { capabilities.push(BROWSER_HEADLESS_RUNTIME_CAPABILITY) @@ -4597,8 +4616,10 @@ export class OrcaRuntimeService { onClientEvent(listener: (event: RuntimeClientEvent) => void): () => void { this.clientEventListeners.add(listener) + this.refreshTerminalSideEffectConsumerAvailability() return () => { this.clientEventListeners.delete(listener) + this.refreshTerminalSideEffectConsumerAvailability() } } @@ -9157,7 +9178,7 @@ export class OrcaRuntimeService { /** Record one derived side-effect fact: batched per chunk while applying * bytes, emitted immediately for between-chunk facts (stale-title timer). */ private recordTerminalSideEffectFact(ptyId: string, fact: TerminalSideEffectFact): void { - if (!this.onTerminalSideEffects || !this.terminalSideEffectConsumerAvailable) { + if (!this.terminalSideEffectConsumerAvailable) { return } const entry = this.ptyTitleTrackersByPtyId.get(ptyId) @@ -9173,11 +9194,7 @@ export class OrcaRuntimeService { facts: TerminalSideEffectFact[], options: { replay?: boolean } = {} ): void { - if ( - !this.onTerminalSideEffects || - !this.terminalSideEffectConsumerAvailable || - facts.length === 0 - ) { + if (!this.terminalSideEffectConsumerAvailable || facts.length === 0) { return } const batch: TerminalSideEffectBatch = { @@ -9187,10 +9204,15 @@ export class OrcaRuntimeService { ...(options.replay ? { replay: true } : {}), ...this.resolveTerminalSideEffectAttribution(ptyId) } - try { - this.onTerminalSideEffects(batch) - } catch (err) { - console.error('[runtime] terminal side-effect listener threw', { ptyId, err }) + if (this.terminalSideEffectLocalConsumerAvailable) { + try { + this.onTerminalSideEffects?.(batch) + } catch (err) { + console.error('[runtime] terminal side-effect listener threw', { ptyId, err }) + } + } + if (this.clientEventListeners.size > 0) { + this.emitClientEvent({ type: 'terminalSideEffects', batch }) } } @@ -9518,7 +9540,13 @@ export class OrcaRuntimeService { } private setTerminalSideEffectConsumerAvailable(available: boolean): void { - const nextAvailable = available && this.onTerminalSideEffects !== null + this.terminalSideEffectLocalConsumerAvailable = available && this.onTerminalSideEffects !== null + this.refreshTerminalSideEffectConsumerAvailability() + } + + private refreshTerminalSideEffectConsumerAvailability(): void { + const nextAvailable = + this.terminalSideEffectLocalConsumerAvailable || this.clientEventListeners.size > 0 if (nextAvailable === this.terminalSideEffectConsumerAvailable) { return } @@ -10013,19 +10041,21 @@ export class OrcaRuntimeService { serializeTerminalBuffer( ptyId: string, opts: { scrollbackRows?: number } = {} - ): Promise<{ - data: string - cols: number - rows: number - seq?: number - cwd?: string | null - lastTitle?: string - source?: 'headless' | 'renderer' - oscLinks?: TerminalOscLinkRange[] - alternateScreen?: boolean - scrollbackAnsi?: string - pendingEscapeTailAnsi?: string - } | null> { + ): Promise { + return this.serializeTerminalBufferFromAvailableState(ptyId, opts) + } + + async serializeAuthoritativeTerminalBuffer( + ptyId: string, + opts: { scrollbackRows?: number } = {} + ): Promise { + const providerSnapshot = await this.serializeProviderTerminalBuffer(ptyId, opts, { + timeoutMs: AUTHORITATIVE_TERMINAL_SNAPSHOT_TIMEOUT_MS, + retireOnTimeout: true + }) + if (providerSnapshot) { + return providerSnapshot + } return this.serializeTerminalBufferFromAvailableState(ptyId, opts) } @@ -10589,7 +10619,7 @@ export class OrcaRuntimeService { private async serializeProviderTerminalBuffer( ptyId: string, opts: { scrollbackRows?: number } = {}, - wait: { timeoutMs?: number } = {} + wait: { timeoutMs?: number; retireOnTimeout?: boolean } = {} ): Promise { const generation = this.getPtyLifecycleGeneration(ptyId) const scrollbackRows = Math.max(0, Math.floor(opts.scrollbackRows ?? 0)) @@ -10600,7 +10630,7 @@ export class OrcaRuntimeService { acquisition.scrollbackRows < scrollbackRows ) { const promise = this.captureProviderTerminalBuffer(ptyId, opts, generation) - acquisition = { generation, scrollbackRows, promise } + acquisition = { generation, scrollbackRows, promise, timedOut: false } this.providerBufferAcquisitionsByPtyId.set(ptyId, acquisition) void promise.finally(() => { if (this.providerBufferAcquisitionsByPtyId.get(ptyId) === acquisition) { @@ -10608,9 +10638,26 @@ export class OrcaRuntimeService { } }) } - return typeof wait.timeoutMs === 'number' - ? withTimeout(acquisition.promise, wait.timeoutMs, null) - : acquisition.promise + if (acquisition.timedOut) { + return null + } + if (typeof wait.timeoutMs !== 'number') { + return acquisition.promise + } + const result = await withTimeout< + { settled: true; value: PtyProviderBufferSnapshot | null } | { settled: false } + >( + acquisition.promise.then((value) => ({ settled: true as const, value })), + wait.timeoutMs, + { settled: false as const } + ) + if (!result.settled) { + if (wait.retireOnTimeout) { + acquisition.timedOut = true + } + return null + } + return result.value } private async captureProviderTerminalBuffer( @@ -32593,6 +32640,7 @@ const MAX_TAIL_PENDING_ANSI_CHARS = 4096 const DEFAULT_TERMINAL_READ_LIMIT = 120 const MAX_TERMINAL_READ_LIMIT = 2000 const MAX_TERMINAL_PREVIEW_CHARS = 32 * 1024 +export const AUTHORITATIVE_TERMINAL_SNAPSHOT_TIMEOUT_MS = 8_000 const VISIBLE_TERMINAL_SNAPSHOT_TIMEOUT_MS = 750 const VISIBLE_TERMINAL_SNAPSHOT_RETRY_MS = 1_000 const MAX_PREVIEW_LINES = 6 diff --git a/src/main/runtime/rpc/methods/terminal.ts b/src/main/runtime/rpc/methods/terminal.ts index 2b5d65a5d..2f874ea9f 100644 --- a/src/main/runtime/rpc/methods/terminal.ts +++ b/src/main/runtime/rpc/methods/terminal.ts @@ -58,8 +58,10 @@ import { TERMINAL_MULTIPLEX_ACK_STREAM_MAX_WINDOW_BYTES, TERMINAL_MULTIPLEX_ACK_TOTAL_INITIAL_WINDOW_BYTES, TERMINAL_MULTIPLEX_ACK_TOTAL_MAX_WINDOW_BYTES, - TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION, + TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION, + TERMINAL_MULTIPLEX_MAX_PENDING_PTY_WAITS_PER_CONNECTION, TERMINAL_MULTIPLEX_PENDING_MAX_BYTES, + TERMINAL_MULTIPLEX_STREAM_LIMIT_ERROR, TERMINAL_OUTPUT_BATCH_MAX_BYTES } from '../../../../shared/terminal-multiplex-flow-control' import { drainTerminalMultiplexRoundRobin } from '../terminal-multiplex-round-robin' @@ -582,11 +584,17 @@ async function serializeBudgetedRequestedSnapshot( ): Promise { const requestedRows = scrollbackRows ?? 0 for (const rows of requestedSnapshotScrollbackCandidates(scrollbackRows)) { - const serialized = await runtime.serializeTerminalBuffer(ptyId, { scrollbackRows: rows }) + const serialized = await runtime.serializeAuthoritativeTerminalBuffer(ptyId, { + scrollbackRows: rows + }) if (!serialized) { return null } - const data = (serialized.scrollbackAnsi ?? '') + serialized.data + const scrollbackAnsi = + 'scrollbackAnsi' in serialized && typeof serialized.scrollbackAnsi === 'string' + ? serialized.scrollbackAnsi + : '' + const data = scrollbackAnsi + serialized.data const overByteBudget = terminalStreamByteLengthExceeds(data, REQUESTED_SNAPSHOT_BYTE_BUDGET) if (!overByteBudget || rows === 0) { return { @@ -2299,14 +2307,6 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ const request = parsed.data detachStream(request.streamId, false) cancelPendingPtyWaits(request.streamId) - if ( - streams.size + pendingPtyWaitControllers.size >= - TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION - ) { - sendStreamError(request.streamId, 'terminal_stream_limit_exceeded') - emit({ type: 'end', streamId: request.streamId }) - return - } const isMobile = request.client?.type === 'mobile' let leaf: { ptyId: string | null } | null @@ -2319,6 +2319,14 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ return } if (!leaf?.ptyId && request.client) { + if ( + pendingPtyWaitControllers.size >= + TERMINAL_MULTIPLEX_MAX_PENDING_PTY_WAITS_PER_CONNECTION + ) { + sendStreamError(request.streamId, TERMINAL_MULTIPLEX_STREAM_LIMIT_ERROR) + emit({ type: 'end', streamId: request.streamId }) + return + } // Why: a never-mounted tab has no graph leaf to await; mounting the exact tab attaches its PTY without activating the worktree. runtime.requestRendererTerminalTabMount(request.terminal) const waitController = new AbortController() @@ -2369,6 +2377,11 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ } // Why: a competing subscribe may own this streamId after the PTY await; detach it so an orphaned view subscriber can't silence the model responder (terminal-query-authority.md). detachStream(request.streamId, false) + if (streams.size >= TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION) { + sendStreamError(request.streamId, TERMINAL_MULTIPLEX_STREAM_LIMIT_ERROR) + emit({ type: 'end', streamId: request.streamId }) + return + } const ptyId = leaf.ptyId const remoteDesktopSubscriptionKey = `multiplex:${connectionId}:${request.streamId}` @@ -2532,9 +2545,8 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ capabilities: { ackOutputSourceRanges: 1 as const } } : {}), - truncated: - initialOutputOverflowed || - (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)) + // Why: retained-tail truncation loses history, not the authoritative latest-screen fallback. + truncated: initialOutputOverflowed }) stream.sourceRangeReplacement = stream.ackOutputSourceRanges && @@ -2559,9 +2571,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ displayMode, seq: snapshotFrameSeq, cwd: serialized?.cwd, - truncated: - initialOutputOverflowed || - (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)), + truncated: initialOutputOverflowed, truncatedByByteBudget: serialized?.truncatedByByteBudget, source: serialized?.source, oscLinks: serialized?.oscLinks, @@ -3321,8 +3331,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ streamId, lines: read.tail, truncated: - initialOutputOverflowed || - (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)), + initialOutputOverflowed || (!sendBinary && isTerminalReadPayloadIncomplete(read)), cols: serialized?.cols ?? size?.cols, rows: serialized?.rows ?? size?.rows, displayMode, @@ -3335,9 +3344,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ displayMode, seq: snapshotFrameSeq, cwd: serialized?.cwd, - truncated: - initialOutputOverflowed || - (serialized ? read.truncated : isTerminalReadPayloadIncomplete(read)), + truncated: initialOutputOverflowed, truncatedByByteBudget: serialized?.truncatedByByteBudget, oscLinks: serialized?.oscLinks, data: serialized?.data ?? '' diff --git a/src/main/runtime/rpc/terminal-multiplex-resync-replay-trim.test.ts b/src/main/runtime/rpc/terminal-multiplex-resync-replay-trim.test.ts index 99c386c84..a39f63572 100644 --- a/src/main/runtime/rpc/terminal-multiplex-resync-replay-trim.test.ts +++ b/src/main/runtime/rpc/terminal-multiplex-resync-replay-trim.test.ts @@ -42,20 +42,22 @@ async function setupMultiplexStream(): Promise<{ let snapshot: { data: string; seq?: number } = { data: 'INITIAL', seq: 0 } let deferSerialize = false let releaseDeferredSerialize: (() => void) | null = null + const serializeSnapshot = vi.fn(async () => { + if (deferSerialize) { + deferSerialize = false + await new Promise((resolve) => { + releaseDeferredSerialize = resolve + }) + } + return { data: snapshot.data, cols: 80, rows: 24, seq: snapshot.seq } + }) const runtime = { getRuntimeId: () => 'test-runtime', resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }), readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false }), - serializeTerminalBuffer: vi.fn(async () => { - if (deferSerialize) { - deferSerialize = false - await new Promise((resolve) => { - releaseDeferredSerialize = resolve - }) - } - return { data: snapshot.data, cols: 80, rows: 24, seq: snapshot.seq } - }), + serializeTerminalBuffer: serializeSnapshot, + serializeAuthoritativeTerminalBuffer: serializeSnapshot, getTerminalSize: vi.fn().mockReturnValue({ cols: 80, rows: 24 }), getMobileDisplayMode: vi.fn().mockReturnValue('auto'), getLayout: vi.fn().mockReturnValue({ seq: 1 }), diff --git a/src/main/runtime/rpc/terminal-multiplex.test.ts b/src/main/runtime/rpc/terminal-multiplex.test.ts index 0779ea370..17e31eb58 100644 --- a/src/main/runtime/rpc/terminal-multiplex.test.ts +++ b/src/main/runtime/rpc/terminal-multiplex.test.ts @@ -14,10 +14,18 @@ import { encodeTerminalStreamJson, encodeTerminalStreamText } from '../../../shared/terminal-stream-protocol' -import { TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES } from '../../../shared/terminal-multiplex-flow-control' +import { + TERMINAL_MULTIPLEX_ACK_STREAM_INITIAL_WINDOW_BYTES, + TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION, + TERMINAL_MULTIPLEX_MAX_PENDING_PTY_WAITS_PER_CONNECTION +} from '../../../shared/terminal-multiplex-flow-control' import { SshPtyOutputIntake, type SshPtyOutputDataEvent } from '../../ipc/ssh-pty-output-intake' function stubRuntime(overrides: Partial = {}): OrcaRuntimeService { + const serializeAuthoritativeTerminalBuffer = + overrides.serializeAuthoritativeTerminalBuffer ?? + ((ptyId: string, opts?: { scrollbackRows?: number }) => + overrides.serializeTerminalBuffer?.(ptyId, opts)) return { getRuntimeId: () => 'test-runtime', // Why: every multiplex stream registers as a remote view subscriber for @@ -35,6 +43,7 @@ function stubRuntime(overrides: Partial = {}): OrcaRuntimeSe isPtyResizeDrivenRemotely: vi.fn().mockReturnValue(false), getRemoteDesktopFitHold: vi.fn().mockReturnValue({ mode: 'desktop-fit', cols: 120, rows: 40 }), isRemoteDesktopViewerOwner: vi.fn().mockReturnValue(false), + serializeAuthoritativeTerminalBuffer, getPtyOutputSequence: vi.fn().mockReturnValue(0), ...overrides } as OrcaRuntimeService @@ -935,6 +944,11 @@ describe('terminal multiplex RPC', () => { cols: 120, rows: 40 }), + serializeAuthoritativeTerminalBuffer: vi.fn().mockResolvedValue({ + data: 'authoritative snapshot', + cols: 120, + rows: 40 + }), getTerminalSize: vi.fn().mockReturnValue({ cols: 120, rows: 40 }), getMobileDisplayMode: vi.fn().mockReturnValue('auto'), getLayout: vi.fn().mockReturnValue({ seq: 1 }), @@ -1225,7 +1239,10 @@ describe('terminal multiplex RPC', () => { ).toMatchObject({ requestId: 7 }) - expect(runtime.serializeTerminalBuffer).toHaveBeenLastCalledWith('pty-1', { + expect(runtime.serializeTerminalBuffer).toHaveBeenCalledWith('pty-1', { + scrollbackRows: 0 + }) + expect(runtime.serializeAuthoritativeTerminalBuffer).toHaveBeenLastCalledWith('pty-1', { scrollbackRows: 5000 }) expect( @@ -1233,7 +1250,7 @@ describe('terminal multiplex RPC', () => { .filter((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotChunk) .map((frame) => (frame ? decodeTerminalStreamText(frame.payload) : '')) .join('') - ).toBe('snapshot') + ).toBe('authoritative snapshot') // A viewport-less stream is passive: it must neither register nor later // release the active stream's width floor when the connection closes. @@ -2894,7 +2911,7 @@ describe('terminal multiplex RPC', () => { await dispatchPromise }) - it('marks multiplex fallback snapshots truncated when the uncursored read is limited', async () => { + it('keeps a limited retained-tail fallback usable for multiplex first paint', async () => { const messages: string[] = [] const binaryFrames: Uint8Array[] = [] const handlers = new Map< @@ -2977,7 +2994,7 @@ describe('terminal multiplex RPC', () => { expect(subscribed).toMatchObject({ type: 'subscribed', streamId: 11, - truncated: true + truncated: false }) const decodedFrames = binaryFrames.map((frame) => decodeTerminalStreamFrame(frame)) @@ -2985,7 +3002,7 @@ describe('terminal multiplex RPC', () => { (frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart && frame.streamId === 11 ) expect(snapshotStart && decodeTerminalStreamJson(snapshotStart.payload)).toMatchObject({ - truncated: true + truncated: false }) const snapshotData = decodedFrames .filter((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotChunk) @@ -2997,6 +3014,50 @@ describe('terminal multiplex RPC', () => { await dispatchPromise }) + it('does not mark a serialized multiplex snapshot truncated from an overflowed read', async () => { + const harness = startDesktopMultiplexSubscribe({ + readTerminal: vi.fn().mockResolvedValue({ + tail: ['old retained line'], + truncated: true, + limited: true + }), + serializeTerminalBuffer: vi.fn().mockResolvedValue({ + data: 'authoritative current screen\r\n', + cols: 120, + rows: 40 + }) + }) + + await vi.waitFor(() => + expect(harness.messages.some((message) => JSON.parse(message).result?.type === 'ready')).toBe( + true + ) + ) + sendDesktopMultiplexSubscribe(harness.handlers) + await vi.waitFor(() => + expect( + harness.messages.some((message) => JSON.parse(message).result?.type === 'subscribed') + ).toBe(true) + ) + + const snapshotStart = harness.binaryFrames + .map((bytes) => decodeTerminalStreamFrame(bytes)) + .find((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart && frame.streamId === 7) + expect(snapshotStart && decodeTerminalStreamJson(snapshotStart.payload)).toMatchObject({ + truncated: false + }) + expect( + harness.binaryFrames + .map((bytes) => decodeTerminalStreamFrame(bytes)) + .filter((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotChunk) + .map((frame) => (frame ? decodeTerminalStreamText(frame.payload) : '')) + .join('') + ).toBe('authoritative current screen\r\n') + + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + }) + it('falls back to smaller requested snapshots when serialized data exceeds the send budget', async () => { const messages: string[] = [] const binaryFrames: Uint8Array[] = [] @@ -4055,14 +4116,37 @@ describe('terminal multiplex RPC', () => { await harness.dispatchPromise }) - it('caps multiplex stream slots so aggregate pending output stays bounded', async () => { - const harness = startDesktopMultiplexSubscribe() + it('admits 128 active streams, rejects the 129th, and reuses released capacity', async () => { + let dataSubscriberCount = 0 + let viewSubscriberCount = 0 + const harness = startDesktopMultiplexSubscribe({ + subscribeToTerminalData: vi.fn(() => { + dataSubscriberCount += 1 + let released = false + return () => { + if (!released) { + released = true + dataSubscriberCount -= 1 + } + } + }), + registerRemoteTerminalViewSubscriber: vi.fn(() => { + viewSubscriberCount += 1 + let released = false + return () => { + if (!released) { + released = true + viewSubscriberCount -= 1 + } + } + }) + }) await vi.waitFor(() => expect(harness.messages.some((message) => JSON.parse(message).result?.type === 'ready')).toBe( true ) ) - for (let streamId = 1; streamId <= 33; streamId += 1) { + const sendSubscribe = (streamId: number): void => { harness.handlers.get(0)?.( decodeTerminalStreamFrame( encodeTerminalStreamFrame({ @@ -4079,16 +4163,167 @@ describe('terminal multiplex RPC', () => { )! ) } + expect(TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION).toBe(128) + for ( + let streamId = 1; + streamId <= TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION + 1; + streamId += 1 + ) { + sendSubscribe(streamId) + } await vi.waitFor(() => { const results = harness.messages.map((message) => JSON.parse(message).result) - expect(results.filter((result) => result?.type === 'subscribed')).toHaveLength(32) + const subscribedStreamIds = results + .filter((result) => result?.type === 'subscribed') + .map((result) => result.streamId) + expect(subscribedStreamIds).toHaveLength(TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION) + expect(subscribedStreamIds).toContain(44) expect(results).toContainEqual({ type: 'error', - streamId: 33, + streamId: TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION + 1, message: 'terminal_stream_limit_exceeded' }) - expect(results).toContainEqual({ type: 'end', streamId: 33 }) + expect(results).toContainEqual({ + type: 'end', + streamId: TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION + 1 + }) + }) + expect(dataSubscriberCount).toBe(TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION) + expect(viewSubscriberCount).toBe(TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION) + + harness.handlers.get(1)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Unsubscribe, + streamId: 1, + seq: TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION + 2, + payload: new Uint8Array() + }) + )! + ) + await vi.waitFor(() => { + expect(dataSubscriberCount).toBe(TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION - 1) + expect(viewSubscriberCount).toBe(TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION - 1) + expect(harness.handlers.has(1)).toBe(false) + }) + + const retriedStreamId = TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION + 1 + sendSubscribe(retriedStreamId) + await vi.waitFor(() => { + const subscribedStreamIds = harness.messages + .map((message) => JSON.parse(message).result) + .filter((result) => result?.type === 'subscribed') + .map((result) => result.streamId) + expect(subscribedStreamIds).toContain(retriedStreamId) + expect( + harness.binaryFrames.some((bytes) => { + const frame = decodeTerminalStreamFrame(bytes) + return ( + frame?.streamId === retriedStreamId && frame.opcode === TerminalStreamOpcode.SnapshotEnd + ) + }) + ).toBe(true) + }) + expect(dataSubscriberCount).toBe(TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION) + expect(viewSubscriberCount).toBe(TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION) + + harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() + await harness.dispatchPromise + expect(dataSubscriberCount).toBe(0) + expect(viewSubscriberCount).toBe(0) + }) + + it('reserves PTY wait capacity independently from active streams', async () => { + const activeStreamCount = 44 + const waitSignals: AbortSignal[] = [] + const resolveWaits: ((ptyId: string) => void)[] = [] + const runtime = stubRuntime({ + resolveLiveLeafForHandle: vi.fn((terminal: string) => + terminal.startsWith('pending-') ? { ptyId: null } : { ptyId: `pty-${terminal}` } + ), + waitForLeafPtyId: vi.fn( + (_handle: string, _timeoutMs?: number, signal?: AbortSignal) => + new Promise((resolve, reject) => { + if (signal) { + waitSignals.push(signal) + } + resolveWaits.push(resolve) + signal?.addEventListener('abort', () => reject(new Error('request_aborted')), { + once: true + }) + }) + ) + }) + const harness = startDesktopMultiplexSubscribe(runtime) + await vi.waitFor(() => + expect(harness.messages.some((message) => JSON.parse(message).result?.type === 'ready')).toBe( + true + ) + ) + const sendSubscribe = (streamId: number, terminal: string): void => { + harness.handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: streamId, + payload: encodeTerminalStreamJson({ + streamId, + terminal, + client: { id: 'desktop-1', type: 'desktop' }, + capabilities: { ackOutput: 1 } + }) + }) + )! + ) + } + + for (let streamId = 1; streamId <= activeStreamCount; streamId += 1) { + sendSubscribe(streamId, `active-${streamId}`) + } + await vi.waitFor(() => + expect( + harness.messages.filter((message) => JSON.parse(message).result?.type === 'subscribed') + ).toHaveLength(activeStreamCount) + ) + + for ( + let offset = 1; + offset <= TERMINAL_MULTIPLEX_MAX_PENDING_PTY_WAITS_PER_CONNECTION + 1; + offset += 1 + ) { + sendSubscribe(activeStreamCount + offset, `pending-${offset}`) + } + await vi.waitFor(() => + expect(waitSignals).toHaveLength(TERMINAL_MULTIPLEX_MAX_PENDING_PTY_WAITS_PER_CONNECTION) + ) + const rejectedStreamId = + activeStreamCount + TERMINAL_MULTIPLEX_MAX_PENDING_PTY_WAITS_PER_CONNECTION + 1 + await vi.waitFor(() => { + const results = harness.messages.map((message) => JSON.parse(message).result) + expect(results).toContainEqual({ + type: 'error', + streamId: rejectedStreamId, + message: 'terminal_stream_limit_exceeded' + }) + expect(results).toContainEqual({ type: 'end', streamId: rejectedStreamId }) + }) + + for (const [index, resolve] of resolveWaits.entries()) { + resolve(`pty-pending-${index + 1}`) + } + await vi.waitFor(() => { + const results = harness.messages.map((message) => JSON.parse(message).result) + expect(results.filter((result) => result?.type === 'subscribed')).toHaveLength( + activeStreamCount + TERMINAL_MULTIPLEX_MAX_PENDING_PTY_WAITS_PER_CONNECTION + ) + expect( + results.filter( + (result) => + result?.type === 'error' && result.message === 'terminal_stream_limit_exceeded' + ) + ).toHaveLength(1) }) harness.cleanups.get('terminal-multiplex:conn-desktop-first-paint')?.() diff --git a/src/main/runtime/rpc/terminal-source-range-registry.ts b/src/main/runtime/rpc/terminal-source-range-registry.ts index c2192183a..0538b0b4d 100644 --- a/src/main/runtime/rpc/terminal-source-range-registry.ts +++ b/src/main/runtime/rpc/terminal-source-range-registry.ts @@ -1,4 +1,4 @@ -import { TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION } from '../../../shared/terminal-multiplex-flow-control' +import { TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION } from '../../../shared/terminal-multiplex-flow-control' import { TerminalSourceRangeLedger, type TerminalSourceRangeBudget @@ -11,7 +11,7 @@ export class TerminalSourceRangeRegistry { private retainedBytes = 0 open(streamGeneration: string): TerminalSourceRangeLedger | null { - if (this.ledgers.size >= TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION) { + if (this.ledgers.size >= TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION) { return null } let ledger: TerminalSourceRangeLedger diff --git a/src/main/runtime/rpc/terminal-subscribe-buffer.test.ts b/src/main/runtime/rpc/terminal-subscribe-buffer.test.ts index 5c02a2532..837e49eff 100644 --- a/src/main/runtime/rpc/terminal-subscribe-buffer.test.ts +++ b/src/main/runtime/rpc/terminal-subscribe-buffer.test.ts @@ -316,7 +316,7 @@ describe('terminal subscribe buffering', () => { } }) - it('marks binary subscribed previews truncated when the uncursored read is limited', async () => { + it('keeps a limited retained-tail fallback usable for binary first paint', async () => { const messages: string[] = [] const binaryFrames: Uint8Array[] = [] const cleanups = new Map void>() @@ -374,20 +374,20 @@ describe('terminal subscribe buffering', () => { expect(subscribed).toMatchObject({ type: 'subscribed', lines: ['line 120'], - truncated: true + truncated: false }) const snapshotStart = binaryFrames .map((frame) => decodeTerminalStreamFrame(frame)) .find((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart) expect(snapshotStart && decodeTerminalStreamJson(snapshotStart.payload)).toMatchObject({ - truncated: true + truncated: false }) runtime.cleanupSubscription('terminal-1:desktop-1') await dispatchPromise }) - it('does not mark binary snapshot frames truncated from a limited read when serialized data is available', async () => { + it('does not mark binary snapshot frames truncated from an overflowed read when serialized data is available', async () => { const messages: string[] = [] const binaryFrames: Uint8Array[] = [] const cleanups = new Map void>() @@ -395,7 +395,7 @@ describe('terminal subscribe buffering', () => { resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }), readTerminal: vi.fn().mockResolvedValue({ tail: ['line 120'], - truncated: false, + truncated: true, limited: true }), serializeTerminalBuffer: vi.fn().mockResolvedValue({ diff --git a/src/preload/api-types.ts b/src/preload/api-types.ts index 7a6d8199b..e0f86d437 100644 --- a/src/preload/api-types.ts +++ b/src/preload/api-types.ts @@ -3319,6 +3319,10 @@ export type PreloadApi = { disconnect: (args: { selector: string }) => Promise<{ disconnected: PublicKnownRuntimeEnvironment }> + connect: (args: { + selector: string + timeoutMs?: number + }) => Promise> getStatus: (args: { selector: string timeoutMs?: number diff --git a/src/preload/index.ts b/src/preload/index.ts index cce974476..63db42aed 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -4281,6 +4281,11 @@ const api = { selector: string }): Promise<{ disconnected: PublicKnownRuntimeEnvironment }> => ipcRenderer.invoke('runtimeEnvironments:disconnect', args), + connect: (args: { + selector: string + timeoutMs?: number + }): Promise> => + ipcRenderer.invoke('runtimeEnvironments:connect', args), getStatus: (args: { selector: string timeoutMs?: number diff --git a/src/renderer/src/components/Terminal.tsx b/src/renderer/src/components/Terminal.tsx index 4fc153c01..5f9c333ea 100644 --- a/src/renderer/src/components/Terminal.tsx +++ b/src/renderer/src/components/Terminal.tsx @@ -69,6 +69,7 @@ import { scheduleBackgroundTerminalWorktreeMeasure } from './terminal/background import { applyBackgroundMountTabRestriction, canDeferColdActivationTabsForHost, + canMountTerminalWorkspaceForStartup, planColdActivationTabDeferral, pruneClosedBackgroundMountTabs, revealActivationDeferredTabs, @@ -88,18 +89,23 @@ import { getTerminalWorktreeColdParkRecheckDelayMs } from './terminal-pane/termi import { TERMINAL_WORKTREE_COLD_PARK_DELAY_MS, canParkTerminalWorktreeRenderers, + selectPairedRuntimeParkingEnvironmentIds, selectColdParkedTerminalWorktrees, type TerminalWorktreeColdParkCandidate } from './terminal-pane/terminal-hidden-view-parking' import { TERMINAL_HIDDEN_WORKTREE_RETENTION_TTL_MS, + hasPendingRetentionSpawnWork, selectForceParkEvictableTabIds, selectRetentionForceParkedTerminalWorktrees, type TerminalWorktreeRetentionCandidate } from './terminal-pane/terminal-hidden-worktree-retention' import { captureForceParkedWorktreeBuffers } from './terminal-pane/force-park-buffer-capture' import { warnTerminalLifecycleAnomaly } from './terminal-pane/terminal-lifecycle-diagnostics' -import { getTerminalParkingPolicyOverrides } from './terminal-pane/terminal-parking-e2e-overrides' +import { + getTerminalParkingPolicyOverrides, + recordTerminalWorktreeParkingDebugVerdicts +} from './terminal-pane/terminal-parking-e2e-overrides' import { selectEvictionExemptTerminalTabIds } from './terminal-pane/terminal-eviction-exempt-tabs' import { canWatcherCoverParkedTerminalTab, @@ -296,6 +302,11 @@ function Terminal(): React.JSX.Element | null { const pendingStartupByTabId = useAppStore((s) => s.pendingStartupByTabId) const terminalParkingEnabled = useAppStore((s) => s.settings?.terminalHiddenViewParking !== false) const terminalSshParkingEnabled = useAppStore((s) => s.settings?.terminalSshViewParking !== false) + const runtimeStatusByEnvironmentId = useAppStore((s) => s.runtimeStatusByEnvironmentId) + const pairedRuntimeParkingEnvironmentIds = useMemo( + () => selectPairedRuntimeParkingEnvironmentIds(runtimeStatusByEnvironmentId), + [runtimeStatusByEnvironmentId] + ) const terminalRetentionBudgetEnabled = useAppStore( (s) => s.settings?.terminalHiddenWorktreeRetentionBudget !== false ) @@ -317,6 +328,7 @@ function Terminal(): React.JSX.Element | null { const expandedPaneByTabId = useAppStore((s) => s.expandedPaneByTabId) const workspaceSessionReady = useAppStore((s) => s.workspaceSessionReady) const hydrationSucceeded = useAppStore((s) => s.hydrationSucceeded) + const startupWorktreeRefreshCompleted = useAppStore((s) => s.startupWorktreeRefreshCompleted) const openFiles = useAppStore((s) => s.openFiles) const activeFileId = useAppStore((s) => s.activeFileId) const activeBrowserTabId = useAppStore((s) => s.activeBrowserTabId) @@ -949,7 +961,10 @@ function Terminal(): React.JSX.Element | null { }) } - const restorePolicy = { sshParkingEnabled: terminalSshParkingEnabled } + const restorePolicy = { + sshParkingEnabled: terminalSshParkingEnabled, + pairedRuntimeParkingEnvironmentIds + } const nextParkedTerminalWorktreeIds = selectColdParkedTerminalWorktrees({ worktrees: retentionCandidates, pendingStartupByTabId, @@ -1006,14 +1021,11 @@ function Terminal(): React.JSX.Element | null { isVisible: candidate.isVisible, shouldMeasureHiddenWorktree: candidate.shouldMeasureHiddenWorktree, hasActivityTerminalPortal: candidate.hasActivityTerminalPortal, - parkCooldownUntilMs: candidate.parkCooldownUntilMs, + parkCooldownUntilMs: candidate.parkCooldownUntilMs ?? null, ordinaryParkingCovers: parkEligible && worktreeTabsAreWatcherCovered(candidate.worktreeId, tabs), - hasPendingSpawnWork: tabs.some( - (tab) => - pendingStartupByTabId[tab.id] !== undefined || - tab.pendingActivationSpawn === true || - (typeof tab.pendingActivationSpawn === 'number' && tab.pendingActivationSpawn > 0) + hasPendingSpawnWork: tabs.some((tab) => + hasPendingRetentionSpawnWork(tab, pendingStartupByTabId) ) } } @@ -1025,6 +1037,13 @@ function Terminal(): React.JSX.Element | null { nowMs, ...overrides }) + recordTerminalWorktreeParkingDebugVerdicts( + retentionBudgetCandidates.map((candidate) => ({ + ...candidate, + parkCooldownUntilMs: candidate.parkCooldownUntilMs ?? null, + forceParked: forceParkedWorktreeIds.has(candidate.worktreeId) + })) + ) const capturedForceParked = forceParkedCaptureDoneRef.current for (const id of Array.from(capturedForceParked)) { if (!forceParkedWorktreeIds.has(id)) { @@ -1126,6 +1145,7 @@ function Terminal(): React.JSX.Element | null { activityTerminalPortals, backgroundMountRevision, pendingStartupByTabId, + pairedRuntimeParkingEnvironmentIds, renderedActiveWorktreeId, tabsByWorktree, terminalParkingEnabled, @@ -1134,8 +1154,15 @@ function Terminal(): React.JSX.Element | null { terminalSshParkingEnabled, workspaceSurfaces ]) - // Why: gate on workspaceSessionReady so TerminalPane doesn't mount and spawn a duplicate PTY before reconnectPersistedTerminals() finishes. - if (renderedActiveWorktreeId && workspaceSessionReady) { + // Why: a slow post-reconnect step exposes workspaceSessionReady before hydration can populate snapshot capabilities. + if ( + renderedActiveWorktreeId && + canMountTerminalWorkspaceForStartup({ + workspaceSessionReady, + hydrationSucceeded, + startupWorktreeRefreshCompleted + }) + ) { // Why: mounting every saved tab at once (scrollback replay + WebGL + sync-IPC snapshot per pane) freezes the renderer, so hidden tabs defer and mount on first reveal. const worktreeTabs = tabsByWorktree[renderedActiveWorktreeId] ?? [] const coldActivationDeferralEnabled = diff --git a/src/renderer/src/components/settings/RuntimeEnvironmentsPane.tsx b/src/renderer/src/components/settings/RuntimeEnvironmentsPane.tsx index 5012fedae..d22ce1fad 100644 --- a/src/renderer/src/components/settings/RuntimeEnvironmentsPane.tsx +++ b/src/renderer/src/components/settings/RuntimeEnvironmentsPane.tsx @@ -600,7 +600,7 @@ export function RuntimeEnvironmentsPane({ setConnectingId(environment.id) setSwitchError(null) try { - const response = await window.api.runtimeEnvironments.getStatus({ + const response = await window.api.runtimeEnvironments.connect({ selector: environment.id, timeoutMs: 15_000 }) diff --git a/src/renderer/src/components/status-bar/SshStatusSegment.tsx b/src/renderer/src/components/status-bar/SshStatusSegment.tsx index e6785870c..d67fb82cf 100644 --- a/src/renderer/src/components/status-bar/SshStatusSegment.tsx +++ b/src/renderer/src/components/status-bar/SshStatusSegment.tsx @@ -20,6 +20,7 @@ import { isUserManagedRuntimeEnvironment } from '../../../../shared/runtime-envi import { RuntimeHostStatusRow, type RuntimeHostConnectionState } from './RuntimeHostStatusRow' import { SshTargetStatusRow } from './SshTargetStatusRow' import type { RemoteRuntimeSharedConnectionDiagnostics } from '../../../../shared/remote-runtime-shared-control-types' +import { connectRuntimeEnvironmentAndRecordStatus } from './runtime-environment-explicit-connect' function isConnecting(status: SshConnectionStatus): boolean { return ['connecting', 'deploying-relay', 'reconnecting'].includes(status) @@ -177,7 +178,6 @@ export function SshStatusSegment({ const runtimeStatusByEnvironmentId = useAppStore((s) => s.runtimeStatusByEnvironmentId) const setRuntimeEnvironmentStatus = useAppStore((s) => s.setRuntimeEnvironmentStatus) const hydrateRuntimeEnvironmentStatuses = useAppStore((s) => s.hydrateRuntimeEnvironmentStatuses) - const refreshRuntimeEnvironmentStatus = useAppStore((s) => s.refreshRuntimeEnvironmentStatus) const remoteWorkspaceSyncStatusByTargetId = useAppStore( (s) => s.remoteWorkspaceSyncStatusByTargetId ) @@ -232,7 +232,7 @@ export function SshStatusSegment({ const store = useAppStore.getState() const reachable = await connectRuntimeHostForNavigation({ environmentId, - refreshStatus: refreshRuntimeEnvironmentStatus, + refreshStatus: connectRuntimeEnvironmentAndRecordStatus, fetchRepos: store.fetchRuntimeEnvironmentRepos, fetchWorktrees: store.fetchWorktrees, fetchLineage: store.fetchWorktreeLineage @@ -248,7 +248,7 @@ export function SshStatusSegment({ } recordFeatureInteraction('ssh') }, - [recordFeatureInteraction, refreshRuntimeEnvironmentStatus] + [recordFeatureInteraction] ) const disconnectRuntimeHost = useCallback( async (environmentId: string): Promise => { diff --git a/src/renderer/src/components/status-bar/runtime-environment-explicit-connect.ts b/src/renderer/src/components/status-bar/runtime-environment-explicit-connect.ts new file mode 100644 index 000000000..1e0f13496 --- /dev/null +++ b/src/renderer/src/components/status-bar/runtime-environment-explicit-connect.ts @@ -0,0 +1,22 @@ +import type { RuntimeStatus } from '../../../../shared/runtime-types' +import { unwrapRuntimeRpcResult } from '@/runtime/runtime-rpc-client' +import { useAppStore } from '../../store' + +export async function connectRuntimeEnvironmentAndRecordStatus( + environmentId: string, + timeoutMs: number +): Promise { + const setStatus = useAppStore.getState().setRuntimeEnvironmentStatus + try { + const response = await window.api.runtimeEnvironments.connect({ + selector: environmentId, + timeoutMs + }) + const status = unwrapRuntimeRpcResult(response) + setStatus(environmentId, { status, checkedAt: Date.now() }) + return true + } catch { + setStatus(environmentId, { status: null, checkedAt: Date.now() }) + return false + } +} diff --git a/src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts b/src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts index 96bfd22b9..21d912dd7 100644 --- a/src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts +++ b/src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.test.ts @@ -554,6 +554,32 @@ describe('startParkedTerminalByteWatcher', () => { second.dispose() }) + it('consumes host facts without a raw terminal stream for paired PTYs', async () => { + const remotePtyId = 'remote:env-1@@terminal-1' + const { dispose } = await startWatcher({ ptyId: remotePtyId }) + const handler = await import('./terminal-side-effect-facts-handler') + + expect(onData).toBeNull() + handler._dispatchTerminalSideEffectBatchForTest({ + ptyId: remotePtyId, + seq: 10, + facts: [ + { + kind: 'title', + normalizedTitle: '⠋ Remote build', + rawTitle: '⠋ Remote build' + } + ] + }) + + expect(mockStoreState.setRuntimePaneTitle).toHaveBeenCalledWith( + TAB_ID, + PANE_ID, + '⠋ Remote build' + ) + dispose() + }) + // ─── Main side-effect authority (terminal-side-effect-authority.md) ──── // // With the kill switch on, the watcher must not register byte parsers — diff --git a/src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.ts b/src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.ts index c6cad81c6..faa9601cb 100644 --- a/src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.ts +++ b/src/renderer/src/components/terminal-pane/parked-terminal-byte-watcher.ts @@ -32,6 +32,7 @@ import { } from './terminal-side-effect-facts-handler' import { dispatchTerminalNotification } from './use-notification-dispatch' import { acquireHiddenRendererPtyDeliveryClaim } from './pty-renderer-delivery-claims' +import { isRemoteRuntimePtyId } from '@/runtime/runtime-terminal-inspection' // Why: keep the live path's BEL-vs-completion race window so notification behavior is identical whether a tab is parked or mounted. const PARKED_NOTIFICATION_GRACE_MS = AGENT_TASK_COMPLETE_NOTIFICATION_GRACE_MS @@ -70,6 +71,7 @@ export function startParkedTerminalByteWatcher( options: ParkedTerminalByteWatcherOptions ): () => void { const { ptyId, tabId, worktreeId, paneId, sendInput } = options + const remoteRuntimePty = isRemoteRuntimePtyId(ptyId) const drivesTabTitle = options.drivesTabTitle ?? true const paneKey = makePaneKey(tabId, options.leafId) @@ -204,14 +206,18 @@ export function startParkedTerminalByteWatcher( }) // Why: with the authority switch on, the fact consumer is the single policy consumer — registering byte parsers too would double-fire bells. - const mainSideEffectAuthority = isMainTerminalSideEffectAuthorityForPty({ - settings: useAppStore.getState().settings, - runtimeEnvironmentId: null - }) + const mainSideEffectAuthority = + !remoteRuntimePty && + isMainTerminalSideEffectAuthorityForPty({ + settings: useAppStore.getState().settings, + runtimeEnvironmentId: null + }) + const factSideEffectAuthority = mainSideEffectAuthority || remoteRuntimePty // Why: decided once at watcher start — it picks which 2031 responder (byte sidecar vs fact reply) exists, so it must never flip per chunk. const hiddenDeliveryGateActive = mainSideEffectAuthority && isRendererHiddenPtyDeliveryGateEnabled(useAppStore.getState().settings) + const factOwnsMode2031 = hiddenDeliveryGateActive || remoteRuntimePty const sendMode2031Reply = (): void => { const settings = useAppStore.getState().settings @@ -220,32 +226,32 @@ export function startParkedTerminalByteWatcher( // Why (byte-parser mode only): reuse the transport's output processor to keep exact live-path parsing semantics. // initialAgentTitle: an agent already working at park time still produces a working→idle transition. - const processor = mainSideEffectAuthority + const processor = factSideEffectAuthority ? null : createPtyOutputProcessor({ ...(options.initialTitle !== undefined ? { initialAgentTitle: options.initialTitle } : {}), ...sideEffectCallbacks }) // Why (byte-parser mode only): under main authority, byte-scanning PR links too would observe every link twice (facts already carry them). - const observeTerminalGitHubPRLink = mainSideEffectAuthority + const observeTerminalGitHubPRLink = factSideEffectAuthority ? null : createTerminalGitHubPRLinkDetector() // Why (byte-parser mode only): mode parity — main's tracker emits these as facts; the byte // path scans the same shared parsers the mounted kill-switch-off pane uses. - const commandFinishedScanner = mainSideEffectAuthority + const commandFinishedScanner = factSideEffectAuthority ? null : createOsc133CommandFinishedScanner(commandStatusPolicy.onCommandFinished) // Why the seed: this detector is recreated per park cycle with no startup command // to fast-arm it, and a Command Code TUI parked mid-turn is long past its banner — // unseeded it would never scrape the turn's return to the idle composer. - const commandCodeOutputStatusDetector = mainSideEffectAuthority + const commandCodeOutputStatusDetector = factSideEffectAuthority ? null : createCommandCodeOutputStatusDetector({ inFlightTurn: readInFlightCommandCodeTurn(paneKey), onWorking: commandStatusPolicy.onCommandCodeWorking, onDone: commandStatusPolicy.onCommandCodeDone }) - const unregisterFactConsumer = mainSideEffectAuthority + const unregisterFactConsumer = factSideEffectAuthority ? registerTerminalSideEffectFactConsumer({ ptyId, // Why: ordinary park already has a pane-owned title; the flag below requests a snapshot only when no pane did. @@ -257,7 +263,7 @@ export function startParkedTerminalByteWatcher( onPrLink: (link) => useAppStore.getState().observeTerminalGitHubPullRequestLink(worktreeId, link), // Why (gate mode only): the 2031 subscribe arrives as a fact, but the reply stays here — query authority stays with the view/watcher (invariant 6). - ...(hiddenDeliveryGateActive ? { onMode2031Subscribe: sendMode2031Reply } : {}) + ...(factOwnsMode2031 ? { onMode2031Subscribe: sendMode2031Reply } : {}) }, // Why: activation-deferred tabs can start a watcher before any pane restored the title; ordinary parked tabs avoid this IPC. restoreTitleOnRegister: options.restoreTitleOnRegister === true @@ -265,7 +271,7 @@ export function startParkedTerminalByteWatcher( : null // Why: no xterm answers DECSET 2031 while parked; with the gate ON, the responder's sidecar would force-feed bytes to the gated PTY, so skip it. - const stopMode2031Responder = hiddenDeliveryGateActive + const stopMode2031Responder = factOwnsMode2031 ? null : startParkedTerminalMode2031Responder({ ptyId, sendInput }) @@ -274,21 +280,23 @@ export function startParkedTerminalByteWatcher( ? acquireHiddenRendererPtyDeliveryClaim(ptyId) : null - // Why (byte-parser mode only): under main authority, registering byte parsers here would double-fire policy already carried by facts. + const processLiveData = (data: string): void => { + if (!processor) { + return + } + processor.processData(data, {}) + commandFinishedScanner?.scan(data) + commandCodeOutputStatusDetector?.observe(data) + if (observeTerminalGitHubPRLink) { + for (const link of observeTerminalGitHubPRLink(data)) { + useAppStore.getState().observeTerminalGitHubPullRequestLink(worktreeId, link) + } + } + } + // Why: paired hosts forward derived facts over the one environment event + // stream; parked PTYs never consume terminal multiplex slots or raw bytes. const unsubscribeByteParsers = - processor === null - ? null - : subscribeToPtyData(ptyId, (data) => { - // Why: empty pane callbacks — no xterm to deliver bytes to, the watcher wants only the parser side effects. - processor.processData(data, {}) - commandFinishedScanner?.scan(data) - commandCodeOutputStatusDetector?.observe(data) - if (observeTerminalGitHubPRLink) { - for (const link of observeTerminalGitHubPRLink(data)) { - useAppStore.getState().observeTerminalGitHubPullRequestLink(worktreeId, link) - } - } - }) + processor === null ? null : subscribeToPtyData(ptyId, processLiveData) const dispose = (): void => { if (disposed) { diff --git a/src/renderer/src/components/terminal-pane/pty-connection.test.ts b/src/renderer/src/components/terminal-pane/pty-connection.test.ts index d1978d9bc..db6030d00 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.test.ts @@ -165,6 +165,13 @@ type StoreState = { }[] > runtimeEnvironments?: { id: string }[] + runtimeStatusByEnvironmentId: Map< + string, + { + checkedAt: number + status: { capabilities?: string[] } | null + } + > runtimeEnvironmentCatalogHydrated?: boolean repos: { id: string @@ -894,6 +901,7 @@ describe('connectPanePty', () => { worktreesByRepo: { repo1: [{ id: 'wt-1', repoId: 'repo1', path: '/tmp/wt-1', displayName: 'feat/notis' }] }, + runtimeStatusByEnvironmentId: new Map(), repos: [{ id: 'repo1', connectionId: null, displayName: 'orca' }], projects: [], sshConnectionStates: new Map(), @@ -20047,6 +20055,54 @@ describe('connectPanePty', () => { expect(api.pty.signal).toHaveBeenCalledWith('leaf-session', 'SIGWINCH') }) + it('restores configured paired scrollback after an ordinary park reveal', async () => { + const { connectPanePty } = await import('./pty-connection') + const remotePtyId = 'remote:env-1@@terminal-1' + const transport = createMockTransport(remotePtyId) + transport.connect.mockImplementation(async ({ callbacks }: { callbacks: ConnectCallbacks }) => { + transport.getPtyId.mockReturnValue(remotePtyId) + callbacks.onReplayData?.('current screen from initial subscribe\r\n') + return { id: remotePtyId, isReattach: true, replay: '' } + }) + transport.serializeBuffer = vi.fn().mockResolvedValue({ + data: 'DEEP_PAIRED_SCROLLBACK\r\ncurrent screen\r\n', + cols: 100, + rows: 30, + seq: 4_096, + source: 'headless' + }) + transportFactoryQueue.push(transport) + await parkTabForReveal('tab-1', remotePtyId) + mockStoreState = { + ...mockStoreState, + runtimeStatusByEnvironmentId: new Map([ + [ + 'env-1', + { + checkedAt: Date.now(), + status: { capabilities: ['terminal.paired-parking.v1'] } + } + ] + ]) + } + + const pane = createPane(1) + const { parseCallbacks, writes } = captureCallbackTerminalWrites(pane) + const deps = createDeps() + + connectPanePty(pane as never, createManager(1) as never, deps as never) + for (let step = 0; step < 30; step += 1) { + parseCallbacks.shift()?.() + await flushAsyncTicks(2) + } + + expect(transport.serializeBuffer).toHaveBeenCalledWith({ scrollbackRows: 5000 }) + expect(transport.attach).not.toHaveBeenCalled() + expect(writes.join('')).toContain('DEEP_PAIRED_SCROLLBACK') + expect(writes.join('')).toContain('current screen from initial subscribe') + expect(transport.getPtyId).toHaveReturnedWith(remotePtyId) + }) + it('falls back to relay replay when the SSH model snapshot stalls', async () => { const { connectPanePty } = await import('./pty-connection') const { SSH_REATTACH_MODEL_SNAPSHOT_TIMEOUT_MS } = await import('./ssh-reattach-model-restore') diff --git a/src/renderer/src/components/terminal-pane/pty-connection.ts b/src/renderer/src/components/terminal-pane/pty-connection.ts index 68dc330f6..188fdf11c 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.ts @@ -14,6 +14,7 @@ import { getWorktreeMapFromState } from '@/store/selectors' import { parseWorkspaceKey } from '../../../../shared/workspace-scope' import { FLOATING_TERMINAL_WORKTREE_ID } from '../../../../shared/constants' import { isEphemeralSetupTerminalWorktreeId } from '../../../../shared/ephemeral-setup-terminal-worktree-id' +import { TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' import { TerminalKittyKeyboardModeTracker } from '../../../../shared/terminal-kitty-keyboard-mode-tracker' import { isRuntimeOwnedSshTargetId, parseExecutionHostId } from '../../../../shared/execution-host' import { createTerminalZeroDimensionsMessage } from '../../../../shared/terminal-zero-dimensions-diagnostic' @@ -769,6 +770,17 @@ function isRemoteRuntimePtyId(ptyId: string | null | undefined): boolean { return typeof ptyId === 'string' && ptyId.startsWith(REMOTE_PTY_ID_PREFIX) } +function canRestorePairedParkedTerminal(ptyId: string): boolean { + const environmentId = getRemoteRuntimePtyEnvironmentId(ptyId) + return ( + environmentId !== null && + useAppStore + .getState() + .runtimeStatusByEnvironmentId.get(environmentId) + ?.status?.capabilities?.includes(TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY) === true + ) +} + function consumeInactiveForegroundImmediateBudget(dataLength: number): boolean { const now = performance.now() if (now - inactiveForegroundImmediateBudgetWindowStart > FOREGROUND_BUDGET_WINDOW_MS) { @@ -2051,6 +2063,11 @@ export function connectPanePty( // surviving shell then receives pointer moves as typed SGR reports; the // replay guard keeps xterm's auto-replies from leaking to the shell. replayIntoTerminal(pane, deps.replayingPanesRef, POST_REPLAY_REATTACH_RESET, { + breadcrumbIdentity: { + tabId: deps.tabId, + worktreeId: deps.worktreeId, + ptyId: transport.getPtyId() + }, shouldRefreshViewportSynchronously: shouldRefreshForegroundSynchronously }) if (reason === 'visible-pty') { @@ -2580,6 +2597,11 @@ export function connectPanePty( // eats every click and keystroke against a dead transport — disarm the // modes now and arm the reveal-time wake. replayIntoTerminal(pane, deps.replayingPanesRef, POST_REPLAY_MODE_RESET, { + breadcrumbIdentity: { + tabId: deps.tabId, + worktreeId: deps.worktreeId, + ptyId + }, shouldRefreshViewportSynchronously: shouldRefreshForegroundSynchronously }) hibernatedWakeTarget = { ptyId, record: sleepingRecordEntry.record } @@ -5339,6 +5361,11 @@ export function connectPanePty( // scheduler's deferred drain cannot land older bytes on top of the replay. flushTerminalOutput(pane.terminal) replayIntoTerminal(pane, deps.replayingPanesRef, data, { + breadcrumbIdentity: { + tabId: deps.tabId, + worktreeId: deps.worktreeId, + ptyId: transport.getPtyId() + }, shouldRefreshViewportSynchronously: shouldRefreshForegroundSynchronously, shouldReleaseRenderPause: () => deps.isVisibleRef.current }) @@ -5349,6 +5376,11 @@ export function connectPanePty( // merely after the write was queued. flushTerminalOutput(pane.terminal) return replayIntoTerminalAsync(pane, deps.replayingPanesRef, data, { + breadcrumbIdentity: { + tabId: deps.tabId, + worktreeId: deps.worktreeId, + ptyId: transport.getPtyId() + }, shouldRefreshViewportSynchronously: shouldRefreshForegroundSynchronously, shouldReleaseRenderPause: () => deps.isVisibleRef.current }) @@ -7677,20 +7709,30 @@ export function connectPanePty( // the probe; a later in-place reconnect on this same mount must not buy a // second timeout before the relay paint. const revealFollowsTerminalPark = - mountFollowsTerminalPark && connectResult?.isReattach === true + mountFollowsTerminalPark && + (connectResult?.isReattach === true || isRemoteRuntimePtyId(ptyId)) mountFollowsTerminalPark = false - // Why: a relay restart empties the replay buffer, but main's model may - // still hold the session — a park-reveal probes it even with no replay - // so the reveal is never blank when main has content. Prefetched (before - // the payload task) so the coordinator route covers the paint. - let prefetchedSshModelSnapshot: PtyBufferSnapshot | null = null - if (revealFollowsTerminalPark && !hasStructuralReplay) { - prefetchedSshModelSnapshot = await fetchSshMainModelReattachSnapshot() + // Why: ordinary parking destroys xterm. Rebuild from the authoritative + // host snapshot before releasing queued live bytes; null falls back to + // the subscribe screen without keeping the old xterm mounted. + let prefetchedParkModelSnapshot: PtyBufferSnapshot | null = null + if (revealFollowsTerminalPark && (!hasStructuralReplay || isRemoteRuntimePtyId(ptyId))) { + if (isRemoteRuntimePtyId(ptyId)) { + try { + prefetchedParkModelSnapshot = await serializeHiddenOutputSnapshot(ptyId, { + scrollbackRows: resolveHiddenRestoreScrollbackRows(pane.terminal.options.scrollback) + }) + } catch { + prefetchedParkModelSnapshot = null + } + } else { + prefetchedParkModelSnapshot = await fetchSshMainModelReattachSnapshot() + } if (!isCurrentReattachPayload()) { return false } } - let reattachPayloadApplied = !hasStructuralReplay && prefetchedSshModelSnapshot === null + let reattachPayloadApplied = !hasStructuralReplay && prefetchedParkModelSnapshot === null const applyReattachPayload = async (): Promise => { if (!isCurrentReattachPayload()) { return @@ -7731,13 +7773,14 @@ export function connectPanePty( window.api.pty.ackColdRestore(ptyId) } } - } else if (connectResult?.replay || prefetchedSshModelSnapshot) { + } else if (connectResult?.replay || prefetchedParkModelSnapshot) { // Why scoped to a park-reveal: the 100KiB relay tail loses scrollback the // model still holds, but an in-place reattach (network reconnect, wake, // reload) already has that replay in hand, so probing would only delay its // paint by the timeout. Memoized, so this is never a second probe. const modelSnapshot = revealFollowsTerminalPark - ? (prefetchedSshModelSnapshot ?? (await fetchSshMainModelReattachSnapshot())) + ? (prefetchedParkModelSnapshot ?? + (isRemoteRuntimePtyId(ptyId) ? null : await fetchSshMainModelReattachSnapshot())) : null if (!isCurrentReattachPayload()) { return @@ -7863,7 +7906,7 @@ export function connectPanePty( schedulePendingStartupCommandDelivery() } } - if (hasStructuralReplay || prefetchedSshModelSnapshot) { + if (hasStructuralReplay || prefetchedParkModelSnapshot) { await waitForTerminalReplayWritesParsed(pane.terminal) if (!isCurrentReattachPayload()) { return @@ -7917,7 +7960,7 @@ export function connectPanePty( window.api.pty.signal(reattachPtyId, 'SIGWINCH') } } - if (hasStructuralReplay || prefetchedSshModelSnapshot) { + if (hasStructuralReplay || prefetchedParkModelSnapshot) { await structuralReplayCoordinator.run(applyReattachPayload, { shouldRestore: isCurrentReattachPayload, afterRestore: fitAfterReattachRestore @@ -8332,9 +8375,17 @@ export function connectPanePty( const legacyAttachOnlyPtyId = isLegacyWorkerAutomaticResumeBlocked() ? candidateReattachSessionId : null + const pairedParkedReattachSessionId = + mountFollowsTerminalPark && + candidateReattachSessionId && + isRemoteRuntimePtyId(candidateReattachSessionId) && + canRestorePairedParkedTerminal(candidateReattachSessionId) + ? candidateReattachSessionId + : null const deferredReattachSessionId = legacyAttachOnlyPtyId ? null : (runtimeHostPtyWakeHint ?? + pairedParkedReattachSessionId ?? (candidateReattachSessionId && !isRemoteRuntimePtyId(candidateReattachSessionId) && !candidateHasEagerBuffer && diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts index 08044fed3..1a7ca3d71 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts @@ -737,7 +737,11 @@ describe('createRemoteRuntimePtyTransport', () => { callbacks: {} }) - expect(result).toEqual({ id: 'remote:hub-env@@hub-terminal-1', replay: '' }) + expect(result).toEqual({ + id: 'remote:hub-env@@hub-terminal-1', + replay: '', + isReattach: true + }) expect(transport.getPtyId()).toBe('remote:hub-env@@hub-terminal-1') expect(transport.getExecutionHostId?.()).toBe('ssh:hub-private') expect(transport.getRemotePlatform?.()).toBe('win32') @@ -1504,6 +1508,8 @@ describe('createRemoteRuntimePtyTransport', () => { }) await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) const oldStreamId = latestSubscribePayload().streamId + emitSnapshot(oldStreamId, 'before restart') + expect(transport.isConnected()).toBe(true) runtimeCall.mockImplementation(async (args: { method: string }) => args.method === 'session.tabs.list' ? new Promise(() => {}) : { ok: true, result: {} } @@ -1512,10 +1518,17 @@ describe('createRemoteRuntimePtyTransport', () => { ok: true, result: { type: 'end', streamId: oldStreamId, code: 0 } }) + const replacementSnapshot = transport.serializeBuffer?.({ scrollbackRows: 5000 }) + let snapshotSettled = false + void replacementSnapshot?.then(() => { + snapshotSettled = true + }) + await Promise.resolve() expect(onExit).not.toHaveBeenCalled() expect(onPtyExit).not.toHaveBeenCalled() expect(transport.getPtyId()).toBe('remote:hub-env@@terminal-1') + expect(snapshotSettled).toBe(false) expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1) handleEvents.queueAcceptedWebSessionTerminalSnapshot( @@ -1553,6 +1566,39 @@ describe('createRemoteRuntimePtyTransport', () => { expect(onPtySpawn).not.toHaveBeenCalled() expect(onPtyExit).not.toHaveBeenCalled() expect(onExit).not.toHaveBeenCalled() + emitSnapshot(latestSubscribePayload().streamId, 'replacement initial state') + await vi.waitFor(() => + expect(latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest)).toBeDefined() + ) + const requestFrame = latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest) + const request = requestFrame + ? decodeTerminalStreamJson<{ requestId?: number }>(requestFrame.payload) + : null + emitSnapshotFrame( + latestSubscribePayload().streamId, + TerminalStreamOpcode.SnapshotStart, + encodeTerminalStreamJson({ + kind: 'scrollback', + requestId: request?.requestId, + cols: 100, + rows: 30 + }) + ) + emitSnapshotFrame( + latestSubscribePayload().streamId, + TerminalStreamOpcode.SnapshotChunk, + encodeTerminalStreamText('replacement authoritative state') + ) + emitSnapshotFrame( + latestSubscribePayload().streamId, + TerminalStreamOpcode.SnapshotEnd, + new Uint8Array() + ) + await expect(replacementSnapshot).resolves.toMatchObject({ + data: 'replacement authoritative state', + cols: 100, + rows: 30 + }) }) it('coalesces concurrent stale errors for the handle that was replaced', async () => { @@ -2486,7 +2532,11 @@ describe('createRemoteRuntimePtyTransport', () => { const result = await transport.connect({ url: '', callbacks: {} }) - expect(result).toEqual({ id: 'remote:env-1@@terminal-1', replay: '' }) + expect(result).toEqual({ + id: 'remote:env-1@@terminal-1', + replay: '', + isReattach: true + }) expect(runtimeCall).toHaveBeenCalledWith( expect.objectContaining({ method: 'session.tabs.activate', @@ -2633,7 +2683,11 @@ describe('createRemoteRuntimePtyTransport', () => { const result = await transport.connect({ url: '', callbacks: {} }) - expect(result).toEqual({ id: 'remote:env-1@@terminal-2', replay: '' }) + expect(result).toEqual({ + id: 'remote:env-1@@terminal-2', + replay: '', + isReattach: true + }) expect(runtimeCall).toHaveBeenCalledWith( expect.objectContaining({ method: 'session.tabs.activate', @@ -2738,7 +2792,11 @@ describe('createRemoteRuntimePtyTransport', () => { const result = await transport.connect({ url: '', callbacks: {} }) - expect(result).toEqual({ id: 'remote:env-1@@terminal-2', replay: '' }) + expect(result).toEqual({ + id: 'remote:env-1@@terminal-2', + replay: '', + isReattach: true + }) expect(latestSubscribePayload()).toMatchObject({ terminal: 'terminal-2' }) }) @@ -3113,6 +3171,44 @@ describe('createRemoteRuntimePtyTransport', () => { await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2)) }) + it('backs off before retrying a capacity-rejected terminal stream', async () => { + vi.useFakeTimers() + try { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'tab-1', + leafId: 'pane:1' + }) + + await transport.connect({ url: '', callbacks: {} }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + const { streamId } = latestSubscribePayload() + subscriptionCallbacks?.onResponse({ + ok: true, + result: { + type: 'error', + streamId, + message: 'terminal_stream_limit_exceeded' + } + }) + subscriptionCallbacks?.onResponse({ + ok: true, + result: { type: 'end', streamId } + }) + + expect(transport.getRecoveryState?.().phase).toBe('backoff') + expect(runtimeSubscribe).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(249) + expect(runtimeSubscribe).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2)) + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + it('keeps retrying when the first post-partition terminal reattach fails', async () => { let subscribeAttempt = 0 const recoveryPhases: string[] = [] @@ -4052,6 +4148,9 @@ describe('createRemoteRuntimePtyTransport', () => { expect(onConnect).toHaveBeenCalled() const snapshotPromise = transport.serializeBuffer?.({ scrollbackRows: 5000 }) + await vi.waitFor(() => + expect(latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest)).toBeDefined() + ) const snapshotRequestFrame = latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest) const snapshotRequestPayload = snapshotRequestFrame ? decodeTerminalStreamJson<{ requestId?: number; scrollbackRows?: number }>( @@ -4104,16 +4203,21 @@ describe('createRemoteRuntimePtyTransport', () => { const { streamId } = latestSubscribePayload() const snapshotPromise = transport.serializeBuffer?.({ scrollbackRows: 5000 }) + expect(latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest)).toBeUndefined() + + emitSnapshot(streamId, 'initial replay') + expect(onReplayData).toHaveBeenCalledWith('initial replay') + expect(onConnect).toHaveBeenCalled() + + await vi.waitFor(() => + expect(latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest)).toBeDefined() + ) const snapshotRequestFrame = latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest) const snapshotRequestPayload = snapshotRequestFrame ? decodeTerminalStreamJson<{ requestId?: number }>(snapshotRequestFrame.payload) : null expect(snapshotRequestPayload?.requestId).toBe(1) - emitSnapshot(streamId, 'initial replay') - expect(onReplayData).toHaveBeenCalledWith('initial replay') - expect(onConnect).toHaveBeenCalled() - emitSnapshotFrame( streamId, TerminalStreamOpcode.SnapshotStart, diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index 7e857588c..7048e324e 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -146,6 +146,7 @@ export function createRemoteRuntimePtyTransport( let destroyed = false let terminalEnded = false let connecting = false + const attachmentReadyWaiters = new Set<(ready: boolean) => void>() // Why: transport methods overlap during remounts; only the latest pane lifecycle may install a returned PTY. let lifecycleEpoch = 0 let handle: string | null = null @@ -165,6 +166,46 @@ export function createRemoteRuntimePtyTransport( let stopWaitingForPublishedHandle: (() => void) | null = null let attachGeneration = 0 let subscriptionGeneration = 0 + + function setAttachmentReady(ready: boolean): void { + attachmentReady = ready + if (!ready) { + return + } + for (const resolve of attachmentReadyWaiters) { + resolve(true) + } + attachmentReadyWaiters.clear() + } + + function setAttachmentUnavailable(): void { + attachmentReady = false + for (const resolve of attachmentReadyWaiters) { + resolve(false) + } + attachmentReadyWaiters.clear() + } + + function waitForAttachmentReady(): Promise { + if (attachmentReady) { + return Promise.resolve(true) + } + if (destroyed || terminalEnded || !connected || !handle) { + return Promise.resolve(false) + } + return new Promise((resolve) => { + const timer = setTimeout(() => { + attachmentReadyWaiters.delete(settle) + resolve(false) + }, HOST_SESSION_ATTACH_TIMEOUT_MS) + const settle = (ready: boolean): void => { + clearTimeout(timer) + resolve(ready) + } + attachmentReadyWaiters.add(settle) + }) + } + const recovery = new RemoteRuntimePtyRecoveryState(() => { if (recovery.currentPhase === 'disconnected') { clearPublishedHandleWait() @@ -562,7 +603,8 @@ export function createRemoteRuntimePtyTransport( return { id: remotePtyId, - replay: '' + replay: '', + isReattach: true } satisfies PtyConnectResult } @@ -846,7 +888,7 @@ export function createRemoteRuntimePtyTransport( ) { return undefined } - return { id: remotePtyId, replay: '' } + return { id: remotePtyId, replay: '', isReattach: true } } function recoverExpiredHostPane(): void { @@ -1033,7 +1075,7 @@ export function createRemoteRuntimePtyTransport( multiplexedStream?.close() multiplexedStream = null multiplexedStreamHandle = null - attachmentReady = false + setAttachmentReady(false) } function clearPublishedHandleWait(): void { @@ -1064,6 +1106,7 @@ export function createRemoteRuntimePtyTransport( handle = null remotePtyId = null closeMultiplexedStream() + setAttachmentUnavailable() emitRecoveryState() if (stalePtyId) { onPtyExit?.(stalePtyId) @@ -1077,7 +1120,7 @@ export function createRemoteRuntimePtyTransport( handle = nextHandle remotePtyId = toRemoteRuntimePtyId(nextHandle, currentRuntimeEnvironmentId) registerShutdownHandlers(remotePtyId) - attachmentReady = false + setAttachmentReady(false) // Why: host handle rotation preserves the pane generation; only the store identity changes, not spawn/exit semantics. if (replacedPtyId) { replaceFitOverridePtyId(replacedPtyId, remotePtyId) @@ -1306,6 +1349,22 @@ export function createRemoteRuntimePtyTransport( }) } + function scheduleCapacityPressureRetry(): void { + if (destroyed || !connected || !handle) { + return + } + const recoveryWasActive = recovery.isActive + const recoveryEpoch = recovery.begin() + if (!recoveryWasActive) { + inputBatcher.clear() + viewportBatcher.clear() + clearPendingViewportClaim() + } + recovery.schedule(recoveryEpoch, (nextEpoch) => { + scheduleResubscribeAfterTransportClose(false, nextEpoch) + }) + } + async function subscribeToHandle(expectedRecoveryEpoch?: number): Promise { if (!handle) { return @@ -1313,7 +1372,7 @@ export function createRemoteRuntimePtyTransport( const subscribedHandle = handle const subscribedPtyId = remotePtyId const generation = ++subscriptionGeneration - attachmentReady = false + setAttachmentReady(false) let transportClosed = false let subscriptionAttached = false // Why: viewport handed to subscribe; a resize during the round-trip falls back to the refresh-only one-shot RPC, replayed through the stream below once current. @@ -1358,7 +1417,7 @@ export function createRemoteRuntimePtyTransport( return } subscriptionAttached = true - attachmentReady = true + setAttachmentReady(true) connecting = false recoveryRequiresReplacement = false recovery.markHealthy() @@ -1372,6 +1431,7 @@ export function createRemoteRuntimePtyTransport( } outputProcessor.clearAccumulatedState() if (tabId && isWebTerminalSurfaceTabId(tabId)) { + setAttachmentReady(false) multiplexedStream = null multiplexedStreamHandle = null clearPendingViewportClaim() @@ -1386,7 +1446,7 @@ export function createRemoteRuntimePtyTransport( remotePtyId = null multiplexedStream = null multiplexedStreamHandle = null - attachmentReady = false + setAttachmentUnavailable() terminalEnded = true clearPendingViewportClaim() emitRecoveryState() @@ -1411,7 +1471,7 @@ export function createRemoteRuntimePtyTransport( setDriverForPty(subscribedPtyId, driver) } }, - onTransportClose: ({ recoverable }) => { + onTransportClose: ({ recoverable, retryWithBackoff }) => { transportClosed = true if (generation !== subscriptionGeneration) { return @@ -1424,12 +1484,17 @@ export function createRemoteRuntimePtyTransport( } multiplexedStream = null multiplexedStreamHandle = null - attachmentReady = false + setAttachmentReady(false) if (recoverable) { - scheduleResubscribeAfterTransportClose() + if (retryWithBackoff) { + scheduleCapacityPressureRetry() + } else { + scheduleResubscribeAfterTransportClose() + } } else { connecting = false recovery.cancel() + setAttachmentUnavailable() emitRecoveryState() } } @@ -1450,7 +1515,7 @@ export function createRemoteRuntimePtyTransport( closeMultiplexedStream() multiplexedStream = nextStream multiplexedStreamHandle = subscribedHandle - attachmentReady = subscriptionAttached + setAttachmentReady(subscriptionAttached) if (subscriptionAttached) { recoveryRequiresReplacement = false recovery.markHealthy() @@ -1809,6 +1874,7 @@ export function createRemoteRuntimePtyTransport( const id = remotePtyId unregisterShutdownHandlers(id) closeMultiplexedStream() + setAttachmentUnavailable() handle = null remotePtyId = null emitRecoveryState() @@ -1834,6 +1900,7 @@ export function createRemoteRuntimePtyTransport( connecting = false clearPendingViewportClaim() closeMultiplexedStream() + setAttachmentUnavailable() emitRecoveryState() storedCallbacks = {} }, @@ -1982,11 +2049,15 @@ export function createRemoteRuntimePtyTransport( if (!connected || !handle) { return null } + if (!(await waitForAttachmentReady()) || !handle) { + return null + } return getCurrentMultiplexedStream(handle)?.serializeBuffer(opts) ?? null }, destroy() { destroyed = true + setAttachmentUnavailable() this.disconnect() recovery.dispose() inputBatcher.clear() diff --git a/src/renderer/src/components/terminal-pane/replay-guard.test.ts b/src/renderer/src/components/terminal-pane/replay-guard.test.ts index beddf35e1..9db02cc33 100644 --- a/src/renderer/src/components/terminal-pane/replay-guard.test.ts +++ b/src/renderer/src/components/terminal-pane/replay-guard.test.ts @@ -495,6 +495,43 @@ describe('replay-guard stall handling (probe-certified release)', () => { } }) + it('records correlatable replay identity without exposing worktree or PTY paths', () => { + vi.useFakeTimers() + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + try { + const ref = makeRef() + const { pane, terminal } = makeFakePane(1) + pane.leafId = 'leaf-private-identity' as ManagedPane['leafId'] + + replayIntoTerminal(pane, ref, 'restored bytes', { + breadcrumbIdentity: { + tabId: 'tab-private-identity', + worktreeId: 'repo::/Users/alice/private-worktree', + ptyId: '/Users/alice/private-worktree@@ab12cd34' + }, + stallCheckMs: 1_000 + }) + terminal.pendingCallbacks.shift() + vi.advanceTimersByTime(1_000) + terminal.flush() + + const breadcrumbData = mocks.recordRendererCrashBreadcrumb.mock.calls[0]?.[1] + expect(mocks.recordRendererCrashBreadcrumb).toHaveBeenCalledWith( + 'terminal_replay_guard_lost_completion', + { + paneId: 1, + leafIdHash: expect.stringMatching(/^[0-9a-f]{8}$/), + tabIdHash: expect.stringMatching(/^[0-9a-f]{8}$/), + worktreeIdHash: expect.stringMatching(/^[0-9a-f]{8}$/), + ptyId: '…@@ab12cd34' + } + ) + expect(JSON.stringify(breadcrumbData)).not.toContain('/Users/alice') + } finally { + errorSpy.mockRestore() + } + }) + it('releases after the probe itself never parses (wedged pipeline) and reports it', () => { vi.useFakeTimers() const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) diff --git a/src/renderer/src/components/terminal-pane/replay-guard.ts b/src/renderer/src/components/terminal-pane/replay-guard.ts index 24ad07e42..5b3d5d984 100644 --- a/src/renderer/src/components/terminal-pane/replay-guard.ts +++ b/src/renderer/src/components/terminal-pane/replay-guard.ts @@ -9,6 +9,7 @@ import { notifyUndeliverableWrite, recordTerminalParseProgress } from '@/lib/pane-manager/terminal-write-pipeline-health' +import { redactPtyIdForDiagnostics } from '../../../../shared/pty-delivery-diagnostics' // Why this guard exists: xterm auto-replies to query sequences (DA1/DECRQM/OSC 10-11/CPR) via onData → shell stdin, so replaying recorded PTY bytes leaks stray replies onto the new shell's prompt. // No wasUserInput flag distinguishes replay replies from real keystrokes, so a per-pane in-flight counter gates onData; bounded by xterm's parse completion (not a timer), only auto-replies from replayed bytes are dropped. @@ -20,11 +21,53 @@ export type ReplayingPanesRef = React.RefObject> const REPLAY_GUARD_STALL_CHECK_MS = 10_000 type ReplayTerminalOptions = { + breadcrumbIdentity?: { + tabId?: string + worktreeId?: string + ptyId?: string | null + } shouldRefreshViewportSynchronously?: () => boolean shouldReleaseRenderPause?: () => boolean stallCheckMs?: number } +type ReplayGuardBreadcrumbData = { + paneId: number + tabIdHash?: string + worktreeIdHash?: string + leafIdHash?: string + ptyId?: string +} + +function hashReplayIdentity(value: string): string { + let hash = 0x811c9dc5 + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index) + hash = Math.imul(hash, 0x01000193) + } + return (hash >>> 0).toString(16).padStart(8, '0') +} + +function replayGuardBreadcrumbData( + pane: ManagedPane, + identity: ReplayTerminalOptions['breadcrumbIdentity'] +): ReplayGuardBreadcrumbData { + const data: ReplayGuardBreadcrumbData = { paneId: pane.id } + if (pane.leafId) { + data.leafIdHash = hashReplayIdentity(pane.leafId) + } + if (identity?.tabId) { + data.tabIdHash = hashReplayIdentity(identity.tabId) + } + if (identity?.worktreeId) { + data.worktreeIdHash = hashReplayIdentity(identity.worktreeId) + } + if (identity?.ptyId) { + data.ptyId = redactPtyIdForDiagnostics(identity.ptyId) + } + return data +} + export function isPaneReplaying(ref: ReplayingPanesRef, paneId: number): boolean { return (ref.current.get(paneId) ?? 0) > 0 } @@ -45,6 +88,7 @@ function engageReplayGuard( paneId: number, terminal: ReplayGuardWriteTarget, stallCheckMs: number, + breadcrumbData: ReplayGuardBreadcrumbData, onRelease?: () => void ): ReplayGuardWriteCallbacks { map.set(paneId, (map.get(paneId) ?? 0) + 1) @@ -69,12 +113,12 @@ function engageReplayGuard( console.error( `[terminal] replay guard released for pane ${paneId} — the probe write parsed but the replay completion never arrived (lost write callback)` ) - recordRendererCrashBreadcrumb('terminal_replay_guard_lost_completion', { paneId }) + recordRendererCrashBreadcrumb('terminal_replay_guard_lost_completion', breadcrumbData) } else if (reason === 'wedged') { console.error( `[terminal] replay guard released for pane ${paneId} — xterm rejected the replay write or its probe never parsed (undeliverable write pipeline; pane likely needs recovery)` ) - recordRendererCrashBreadcrumb('terminal_replay_guard_wedged_release', { paneId }) + recordRendererCrashBreadcrumb('terminal_replay_guard_wedged_release', breadcrumbData) // Why: a rejected replay or silent probe makes the pipeline undeliverable; recover instead of a fossil that eats input. notifyUndeliverableWrite(terminal, 'replay-wedged') } @@ -144,7 +188,8 @@ export function replayIntoTerminal( replayingPanesRef.current, pane.id, pane.terminal, - options.stallCheckMs ?? REPLAY_GUARD_STALL_CHECK_MS + options.stallCheckMs ?? REPLAY_GUARD_STALL_CHECK_MS, + replayGuardBreadcrumbData(pane, options.breadcrumbIdentity) ) // Why: hidden/snapshot replay skips the foreground path; WebGL/canvas still need a post-parse repaint to drop stale cells. writeForegroundTerminalChunk(pane.terminal, data, { @@ -178,6 +223,7 @@ export function replayIntoTerminalAsync( pane.id, pane.terminal, options.stallCheckMs ?? REPLAY_GUARD_STALL_CHECK_MS, + replayGuardBreadcrumbData(pane, options.breadcrumbIdentity), resolve ) writeForegroundTerminalChunk(pane.terminal, data, { diff --git a/src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts b/src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts index e518829b1..aaf22cab5 100644 --- a/src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts +++ b/src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.test.ts @@ -11,10 +11,26 @@ import { canParkTerminalWorktreeRenderers, isParkRestorableTerminalPty, isSnapshotBackedTerminalPty, + selectPairedRuntimeParkingEnvironmentIds, selectColdParkedTerminalTabs, selectColdParkedTerminalWorktrees } from './terminal-hidden-view-parking' +describe('selectPairedRuntimeParkingEnvironmentIds', () => { + it('selects only reachable hosts advertising the paired parking contract', () => { + const statuses = new Map([ + [ + 'capable', + { status: { capabilities: ['terminal.paired-parking.v1'] }, checkedAt: Date.now() } + ], + ['legacy', { status: { capabilities: ['terminal.multiplex.v1'] }, checkedAt: Date.now() }], + ['offline', { status: null, checkedAt: Date.now() }] + ]) + + expect(selectPairedRuntimeParkingEnvironmentIds(statuses)).toEqual(new Set(['capable'])) + }) +}) + describe('isSnapshotBackedTerminalPty', () => { it('allows local daemon sessions owned by the worktree', () => { expect(isSnapshotBackedTerminalPty('repo::/worktree@@session-1', 'repo::/worktree')).toBe(true) @@ -76,7 +92,22 @@ describe('isParkRestorableTerminalPty', () => { ).toBe(false) }) - it('rejects remote-runtime, fail-open, foreign, and null ptys under every policy', () => { + it('accepts paired ptys only for the exact snapshot-capable owner', () => { + const pairedPolicy = { + ...sshPolicy, + pairedRuntimeParkingEnvironmentIds: new Set(['env-1']) + } + + expect(isParkRestorableTerminalPty('remote:env-1@@terminal-1', worktreeId, pairedPolicy)).toBe( + true + ) + expect(isParkRestorableTerminalPty('remote:env-2@@terminal-1', worktreeId, pairedPolicy)).toBe( + false + ) + expect(isParkRestorableTerminalPty('remote:terminal-1', worktreeId, pairedPolicy)).toBe(false) + }) + + it('rejects paired, fail-open, foreign, and null ptys without capability evidence', () => { for (const ptyId of ['remote:env-1@@terminal-1', 'pty-local-detached', 'other@@s-1', null]) { expect(isParkRestorableTerminalPty(ptyId, worktreeId, sshPolicy)).toBe(false) } @@ -214,6 +245,22 @@ describe('canParkTerminalWorktreeRenderers', () => { }) ).toBe(false) }) + + it('ignores mirrored activation residue after a capable paired PTY exists', () => { + expect( + canParkTerminalWorktreeRenderers({ + ...base, + terminalTabs: [ + { + id: 'tab-1', + ptyId: 'remote:env-1@@terminal-1', + pendingActivationSpawn: true + } + ], + restorePolicy: { pairedRuntimeParkingEnvironmentIds: new Set(['env-1']) } + }) + ).toBe(true) + }) }) describe('canParkTerminalTabRenderer', () => { @@ -237,6 +284,20 @@ describe('canParkTerminalTabRenderer', () => { expect(canParkTerminalTabRenderer({ ...base, parkingEnabled: false })).toBe(false) }) + it('ignores mirrored activation residue after a capable paired PTY exists', () => { + expect( + canParkTerminalTabRenderer({ + ...base, + terminalTab: { + ...base.terminalTab, + ptyId: 'remote:env-1@@terminal-1', + pendingActivationSpawn: true + }, + restorePolicy: { pairedRuntimeParkingEnvironmentIds: new Set(['env-1']) } + }) + ).toBe(true) + }) + it('honors a per-call cold-park delay override', () => { expect( canParkTerminalTabRenderer({ ...base, coldParkDelayMs: 100, nowMs: hiddenSinceMs + 99 }) diff --git a/src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.ts b/src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.ts index 92dcfd48b..a1d4690f4 100644 --- a/src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.ts +++ b/src/renderer/src/components/terminal-pane/terminal-hidden-view-parking.ts @@ -1,5 +1,7 @@ import { isRemoteRuntimePtyId } from '@/runtime/runtime-terminal-inspection' +import { getRemoteRuntimePtyEnvironmentId } from '@/runtime/runtime-terminal-stream' import { PTY_SESSION_ID_SEPARATOR } from '../../../../shared/pty-session-id-format' +import { TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version' import { parseAppSshPtyId } from '../../../../shared/ssh-pty-id' import type { TerminalTab } from '../../../../shared/types' @@ -56,6 +58,13 @@ function getPendingActivationSpawnCount(value: boolean | number | undefined): nu return typeof value === 'number' && value > 0 ? value : 0 } +function hasPendingActivationSpawn(tab: ColdParkableTerminalTab): boolean { + return ( + getPendingActivationSpawnCount(tab.pendingActivationSpawn) > 0 && + (!tab.ptyId || !isRemoteRuntimePtyId(tab.ptyId)) + ) +} + // Why: snapshot-backed = local daemon session owned by this worktree (foreign // ids reattach through a path parking cannot replay). SSH is restorable too, // via isParkRestorableTerminalPty + main's headless model; only remote-runtime @@ -77,12 +86,24 @@ export function isSnapshotBackedTerminalPty(ptyId: string | null, worktreeId: st export type TerminalParkRestorePolicy = { /** settings.terminalSshViewParking !== false — the C1 SSH-parking kill switch. */ sshParkingEnabled?: boolean + /** Exact paired environments whose host advertises bounded snapshot restore. */ + pairedRuntimeParkingEnvironmentIds?: ReadonlySet } -// Why: SSH bytes transit local main, so main's headless model (served over -// pty:getMainBufferSnapshot) can re-hydrate a parked SSH reveal, with the -// relay's replay buffer as fallback — fact-mode watchers cover side effects -// either way. Remote-runtime ptys never transit main; they stay un-parkable. +export function selectPairedRuntimeParkingEnvironmentIds( + statuses: ReadonlyMap +): Set { + const capable = new Set() + for (const [environmentId, entry] of statuses) { + if (entry.status?.capabilities?.includes(TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY)) { + capable.add(environmentId) + } + } + return capable +} + +// Why: SSH uses local main's model; paired PTYs are eligible only when their +// exact host advertises authoritative bounded restore. export function isParkRestorableTerminalPty( ptyId: string | null, worktreeId: string, @@ -91,6 +112,13 @@ export function isParkRestorableTerminalPty( if (isSnapshotBackedTerminalPty(ptyId, worktreeId)) { return true } + if (ptyId && isRemoteRuntimePtyId(ptyId)) { + const environmentId = getRemoteRuntimePtyEnvironmentId(ptyId) + return ( + environmentId !== null && + policy?.pairedRuntimeParkingEnvironmentIds?.has(environmentId) === true + ) + } return policy?.sshParkingEnabled === true && ptyId !== null && parseAppSshPtyId(ptyId) !== null } @@ -130,7 +158,7 @@ export function canParkTerminalWorktreeRenderers(args: { if (args.pendingStartupByTabId[tab.id] !== undefined) { return false } - if (getPendingActivationSpawnCount(tab.pendingActivationSpawn) > 0) { + if (hasPendingActivationSpawn(tab)) { return false } return isParkRestorableTerminalPty(tab.ptyId, args.worktreeId, args.restorePolicy) @@ -164,7 +192,7 @@ export function canParkTerminalTabRenderer(args: { if (args.pendingStartupByTabId[tab.id] !== undefined) { return false } - if (getPendingActivationSpawnCount(tab.pendingActivationSpawn) > 0) { + if (hasPendingActivationSpawn(tab)) { return false } return isParkRestorableTerminalPty(tab.ptyId, args.worktreeId, args.restorePolicy) diff --git a/src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts b/src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts index 3b1621b3f..11a3bb516 100644 --- a/src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts +++ b/src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.test.ts @@ -2,12 +2,46 @@ import { describe, expect, it } from 'vitest' import { TERMINAL_WORKTREE_PARK_DELAY_MS } from './terminal-hidden-view-parking' import { TERMINAL_HIDDEN_WORKTREE_RETENTION_TTL_MS, + hasPendingRetentionSpawnWork, isEvictionExemptTerminalPty, selectForceParkEvictableTabIds, selectRetentionForceParkedTerminalWorktrees, type TerminalWorktreeRetentionCandidate } from './terminal-hidden-worktree-retention' +describe('hasPendingRetentionSpawnWork', () => { + const remoteTab = { + id: 'tab-remote', + ptyId: 'remote:env-1@@terminal-1', + pendingActivationSpawn: true as const + } + + it('treats a host-backed paired PTY as settled despite activation residue', () => { + expect(hasPendingRetentionSpawnWork(remoteTab, {})).toBe(false) + expect(hasPendingRetentionSpawnWork({ ...remoteTab, pendingActivationSpawn: 2 }, {})).toBe( + false + ) + }) + + it('preserves real startup work and non-paired activation guards', () => { + expect(hasPendingRetentionSpawnWork(remoteTab, { [remoteTab.id]: ['echo', 'pending'] })).toBe( + true + ) + expect( + hasPendingRetentionSpawnWork( + { id: 'tab-local', ptyId: 'pty-local', pendingActivationSpawn: true }, + {} + ) + ).toBe(true) + expect( + hasPendingRetentionSpawnWork( + { id: 'tab-unbound', ptyId: null, pendingActivationSpawn: true }, + {} + ) + ).toBe(true) + }) +}) + describe('isEvictionExemptTerminalPty', () => { const worktreeId = 'repo::/worktree' diff --git a/src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.ts b/src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.ts index 86291b62c..9cfc4e374 100644 --- a/src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.ts +++ b/src/renderer/src/components/terminal-pane/terminal-hidden-worktree-retention.ts @@ -7,6 +7,7 @@ import { type ColdParkRetainCandidate, type TerminalColdParkPolicyOverrides } from './terminal-hidden-view-parking' +import type { TerminalTab } from '../../../../shared/types' // Why these sizes: a retained hidden pane costs a measured ~2.5MB of V8 heap // at the 5k-row default scrollback and ~19MB at 50k (plus per-pane queues), @@ -28,6 +29,18 @@ import { export const TERMINAL_HIDDEN_WORKTREE_RETENTION_LIMIT = 12 export const TERMINAL_HIDDEN_WORKTREE_RETENTION_TTL_MS = 45 * 60_000 +export function hasPendingRetentionSpawnWork( + tab: Pick, + pendingStartupByTabId: Readonly> +): boolean { + if (pendingStartupByTabId[tab.id] !== undefined) { + return true + } + // Why: paired mirrors never spawn locally; their host-backed PTY id proves + // activation's sort-suppression residue cannot represent unfinished work. + return Boolean(tab.pendingActivationSpawn && (!tab.ptyId || !isRemoteRuntimePtyId(tab.ptyId))) +} + // Why: an eviction-exempt pty is a live local one a remount could not reattach // (daemon-fail-open separator-less ids, ptys minted under another worktree) — a // fresh spawn would orphan the live shell. Its TAB keeps its mounted pane when diff --git a/src/renderer/src/components/terminal-pane/terminal-parked-pty-watcher.ts b/src/renderer/src/components/terminal-pane/terminal-parked-pty-watcher.ts new file mode 100644 index 000000000..4c091456e --- /dev/null +++ b/src/renderer/src/components/terminal-pane/terminal-parked-pty-watcher.ts @@ -0,0 +1,131 @@ +import { isTerminalLeafId } from '../../../../shared/stable-pane-id' +import { isRemoteRuntimePtyId, sendRuntimePtyInput } from '@/runtime/runtime-terminal-inspection' +import { useAppStore } from '@/store' +import { closeTerminalTab } from '../terminal/terminal-tab-actions' +import { startParkedTerminalByteWatcher } from './parked-terminal-byte-watcher' +import { subscribeToPtyExit } from './pty-dispatcher' +import { discardPreHandlerPtyState } from './pty-pre-handler-buffer' +import { detachTerminalLayoutLeaf } from './terminal-layout-leaf-detach' +import { + isParkRestorableTerminalPty, + type TerminalParkRestorePolicy +} from './terminal-hidden-view-parking' +import type { ParkableTerminalTabModel } from './terminal-parked-watcher-reconciliation' +import { + resolveTabTitleAfterPaneClose, + shouldClearLaunchAgentForClosedPane +} from './terminal-pane-close-identity' +import { + capturedPanesByTabId, + parkedWatchersByTabId, + type ParkedTabWatcherEntry, + type ParkedTerminalPaneCapture +} from './terminal-parked-watcher-registry' + +export function startParkedPtyWatcher(args: { + worktreeId: string + tab: ParkableTerminalTabModel + pane: ParkedTerminalPaneCapture + entry: ParkedTabWatcherEntry + restoreTitleOnRegister: boolean + restorePolicy: TerminalParkRestorePolicy +}): void { + const { worktreeId, tab, pane, entry, restoreTitleOnRegister, restorePolicy } = args + const state = useAppStore.getState() + const ptyId = pane.ptyId + // Why: the tab model can change after the park decision, and legacy leaf ids make pane keys throw. + if ( + !ptyId || + entry.disposersByPtyId.has(ptyId) || + !isTerminalLeafId(pane.leafId) || + !isParkRestorableTerminalPty(ptyId, worktreeId, restorePolicy) + ) { + return + } + const handlePtyExit = (_code: number, { hadPrimary }: { hadPrimary: boolean }): void => { + useAppStore.getState().clearRuntimePaneTitle(tab.id, pane.paneId) + if (entry.disposersByPtyId.size > 1) { + discardPreHandlerPtyState(ptyId) + collapseParkedExitedLeaf(tab.id, ptyId) + entry.disposersByPtyId.get(ptyId)?.() + entry.disposersByPtyId.delete(ptyId) + return + } + if (hadPrimary) { + entry.disposersByPtyId.get(ptyId)?.() + entry.disposersByPtyId.delete(ptyId) + return + } + + // Why: the empty entry prevents a pending pinned-close confirmation from restarting the dead PTY. + entry.disposersByPtyId.get(ptyId)?.() + entry.disposersByPtyId.delete(ptyId) + closeTerminalTab(tab.id, { + captureRecentlyClosed: false, + hostCloseReason: 'pty-exit', + lifecyclePtyId: ptyId, + onClosed: () => { + discardPreHandlerPtyState(ptyId) + if (parkedWatchersByTabId.get(tab.id) === entry) { + parkedWatchersByTabId.delete(tab.id) + } + }, + onCancel: () => {} + }) + } + const initialTitle = state.runtimePaneTitlesByTabId[tab.id]?.[pane.paneId] + const disposeWatcher = startParkedTerminalByteWatcher({ + ptyId, + tabId: tab.id, + worktreeId, + leafId: pane.leafId, + paneId: pane.paneId, + drivesTabTitle: pane.drivesTabTitle, + ...(initialTitle !== undefined ? { initialTitle } : {}), + ...(restoreTitleOnRegister ? { restoreTitleOnRegister: true } : {}), + sendInput: (data) => { + sendRuntimePtyInput(useAppStore.getState().settings, ptyId, data) + } + }) + const unsubscribeExit = isRemoteRuntimePtyId(ptyId) + ? () => {} + : subscribeToPtyExit(ptyId, handlePtyExit) + entry.paneIdByPtyId.set(ptyId, pane.paneId) + entry.disposersByPtyId.set(ptyId, () => { + unsubscribeExit() + disposeWatcher() + }) +} + +export function collapseParkedExitedLeaf(tabId: string, ptyId: string): void { + const state = useAppStore.getState() + const layout = state.terminalLayoutsByTabId[tabId] + const leafId = + capturedPanesByTabId.get(tabId)?.panes.find((pane) => pane.ptyId === ptyId)?.leafId ?? + Object.entries(layout?.ptyIdsByLeafId ?? {}).find(([, boundPtyId]) => boundPtyId === ptyId)?.[0] + if (!leafId) { + return + } + const detached = detachTerminalLayoutLeaf(layout, leafId) + if (!detached) { + return + } + const terminalTab = Object.values(state.tabsByWorktree) + .flat() + .find((candidate) => candidate.id === tabId) + if (shouldClearLaunchAgentForClosedPane(terminalTab, ptyId)) { + state.clearTabLaunchAgent(tabId) + } + state.setTabLayout(tabId, detached.sourceLayout) + const activeLeafId = detached.sourceLayout.activeLeafId + const activePtyId = activeLeafId + ? detached.sourceLayout.ptyIdsByLeafId?.[activeLeafId] + : undefined + const activePaneId = activePtyId + ? (parkedWatchersByTabId.get(tabId)?.paneIdByPtyId.get(activePtyId) ?? null) + : null + state.updateTabTitle( + tabId, + resolveTabTitleAfterPaneClose(state.runtimePaneTitlesByTabId[tabId] ?? {}, activePaneId) + ) +} diff --git a/src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts b/src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts index 5242ec9d9..21c292751 100644 --- a/src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts +++ b/src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.test.ts @@ -80,8 +80,13 @@ type MockStoreState = { > runtimePaneTitlesByTabId: Record> settings: { terminalSshViewParking?: boolean } | null + runtimeStatusByEnvironmentId: Map< + string, + { status: { capabilities?: string[] } | null; checkedAt: number } + > clearTabLaunchAgent: ReturnType clearRuntimePaneTitle: ReturnType + setRuntimePaneTitle: ReturnType setTabLayout: ReturnType updateTabTitle: ReturnType } @@ -141,8 +146,10 @@ describe('terminal-parked-tab-watchers', () => { terminalLayoutsByTabId: {}, runtimePaneTitlesByTabId: {}, settings: null, + runtimeStatusByEnvironmentId: new Map(), clearTabLaunchAgent: vi.fn(), clearRuntimePaneTitle: vi.fn(), + setRuntimePaneTitle: vi.fn(), setTabLayout: vi.fn(), updateTabTitle: vi.fn() } @@ -222,6 +229,24 @@ describe('terminal-parked-tab-watchers', () => { expect(getParkedTerminalWatcherTabIds()).toEqual([TAB_ID]) }) + it('starts a fact watcher for snapshot-capable paired PTYs', () => { + mockStoreState.runtimeStatusByEnvironmentId.set('env-1', { + status: { capabilities: ['terminal.paired-parking.v1'] }, + checkedAt: Date.now() + }) + capturePanes([ + { ptyId: 'remote:env-1@@terminal-1', paneId: 1, leafId: LEAF_ID, drivesTabTitle: true } + ]) + syncParked({ tabs: [{ id: TAB_ID, ptyId: 'remote:env-1@@terminal-1' }] }) + + expect(startParkedTerminalByteWatcher).toHaveBeenCalledTimes(1) + expect(startedWatchers[0].options).toMatchObject({ + ptyId: 'remote:env-1@@terminal-1' + }) + expect(subscribeToPtyExit).not.toHaveBeenCalled() + expect(exitSubscriptions).toEqual([]) + }) + it('starts watchers for SSH PTYs (C1 SSH parking, default on)', () => { capturePanes([{ ptyId: 'ssh:conn-1@@pty-1', paneId: 1, leafId: LEAF_ID, drivesTabTitle: true }]) syncParked({ tabs: [{ id: TAB_ID, ptyId: 'ssh:conn-1@@pty-1' }] }) @@ -263,6 +288,7 @@ describe('terminal-parked-tab-watchers', () => { syncParked({ tabs: [], parkedTabIds: [TAB_ID] }) expect(startedWatchers[0].dispose).toHaveBeenCalledTimes(1) + expect(mockStoreState.clearRuntimePaneTitle).toHaveBeenCalledWith(TAB_ID, 1) expect(getParkedTerminalWatcherTabIds()).toEqual([]) }) diff --git a/src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.ts b/src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.ts index 0078705e8..04456bd9f 100644 --- a/src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.ts +++ b/src/renderer/src/components/terminal-pane/terminal-parked-tab-watchers.ts @@ -6,29 +6,24 @@ * disposes them on reveal, tab close, PTY exit, or worktree teardown. */ import { isTerminalLeafId } from '../../../../shared/stable-pane-id' -import type { TerminalTab } from '../../../../shared/types' import { useAppStore } from '@/store' -import { closeTerminalTab } from '../terminal/terminal-tab-actions' -import { - collectLeafIdsInOrder, - resolveRootlessTerminalLayoutLeafId -} from './terminal-layout-leaf-ids' -import { detachTerminalLayoutLeaf } from './terminal-layout-leaf-detach' -import { subscribeToPtyExit } from './pty-dispatcher' import { discardPreHandlerPtyState } from './pty-pre-handler-buffer' -import { startParkedTerminalByteWatcher } from './parked-terminal-byte-watcher' import { isParkRestorableTerminalPty, + selectPairedRuntimeParkingEnvironmentIds, type TerminalParkRestorePolicy } from './terminal-hidden-view-parking' import { - resolveTabTitleAfterPaneClose, - shouldClearLaunchAgentForClosedPane -} from './terminal-pane-close-identity' + reconcileParkedWatcherPtyIds, + resolveParkedTerminalPaneCandidates, + type ParkableTerminalTabModel +} from './terminal-parked-watcher-reconciliation' +import { collapseParkedExitedLeaf, startParkedPtyWatcher } from './terminal-parked-pty-watcher' import { capturedPanesByTabId, disposeParkedTabWatchers, parkedWatchersByTabId, + type ParkedTabWatcherEntry, type ParkedTerminalPaneCapture } from './terminal-parked-watcher-registry' @@ -43,67 +38,31 @@ export { pruneParkedTerminalWatchers } from './terminal-parked-watcher-registry' export type { ParkedTerminalPaneCapture } from './terminal-parked-watcher-registry' - -export type ParkableTerminalTabModel = Pick +export { + fallbackParkedPaneCandidates, + resolveParkedTerminalPaneCandidates +} from './terminal-parked-watcher-reconciliation' +export type { ParkableTerminalTabModel } from './terminal-parked-watcher-reconciliation' export type ParkedTerminalPtyEligibility = (ptyId: string) => boolean const allowSnapshotBackedPty = (): boolean => true -type ParkedPaneFallbackState = { - terminalLayoutsByTabId: ReturnType['terminalLayoutsByTabId'] - runtimePaneTitlesByTabId: ReturnType['runtimePaneTitlesByTabId'] -} - -// Why: pane ids are unknown in this layout fallback; reuse the sole runtime-title slot when unambiguous to overwrite a stale title, else negative slots that can't collide with real PaneManager ids. -export function fallbackParkedPaneCandidates( - tab: ParkableTerminalTabModel, - state: ParkedPaneFallbackState -): ParkedTerminalPaneCapture[] { - const layout = state.terminalLayoutsByTabId[tab.id] - // Why: a tab that never mounted a pane persists a rootless layout, so there is - // no root to walk. replayTerminalLayout resolves the single leaf for that same - // shape — match it here, or such a tab is permanently uncoverable and can - // never park. - const rootLeafIds = collectLeafIdsInOrder(layout?.root) - const rootlessLeafId = layout ? resolveRootlessTerminalLayoutLeafId(layout) : null - const leafIds = - rootLeafIds.length > 0 ? rootLeafIds : rootlessLeafId !== null ? [rootlessLeafId] : [] - if (leafIds.length === 0) { - return [] - } - const ptyIdsByLeafId = layout?.ptyIdsByLeafId ?? {} - const titleSlots = Object.keys(state.runtimePaneTitlesByTabId[tab.id] ?? {}) - const reusableSlot = - leafIds.length === 1 && titleSlots.length === 1 ? Number(titleSlots[0]) : null - return leafIds.map((leafId, index) => ({ - ptyId: ptyIdsByLeafId[leafId] ?? (leafIds.length === 1 ? tab.ptyId : null), - paneId: reusableSlot ?? -(index + 1), - leafId, - drivesTabTitle: layout?.activeLeafId ? leafId === layout.activeLeafId : index === 0 - })) -} - -// Why: start path and eligibility check must resolve identical candidates, or a tab passes the check then starts uncoverable. -export function resolveParkedTerminalPaneCandidates( - tab: ParkableTerminalTabModel, - state: ParkedPaneFallbackState -): ParkedTerminalPaneCapture[] { - const captured = capturedPanesByTabId.get(tab.id) - // Why: a capture missing the tab's current PTY is stale (PTY re-minted since unmount); fall back to the layout. - const capturedIsCurrent = - captured !== undefined && - captured.panes.length > 0 && - (tab.ptyId === null || captured.panes.some((pane) => pane.ptyId === tab.ptyId)) - return capturedIsCurrent ? captured.panes : fallbackParkedPaneCandidates(tab, state) -} - // Why: fact-mode watchers work for any pty whose bytes transit local main — // SSH included — so watcher coverage follows the park-restore policy, not the // stricter daemon-snapshot predicate. function parkRestorePolicyFromState(state: { settings: { terminalSshViewParking?: boolean } | null + runtimeStatusByEnvironmentId: ReadonlyMap< + string, + { status: { capabilities?: readonly string[] } | null | undefined } + > }): TerminalParkRestorePolicy { - return { sshParkingEnabled: state.settings?.terminalSshViewParking !== false } + return { + sshParkingEnabled: state.settings?.terminalSshViewParking !== false, + pairedRuntimeParkingEnvironmentIds: selectPairedRuntimeParkingEnvironmentIds( + state.runtimeStatusByEnvironmentId + ) + } } /** @@ -137,90 +96,99 @@ function startParkedTabWatchers( tab: ParkableTerminalTabModel, restoreTitleOnRegister: boolean ): void { - const state = useAppStore.getState() - const panes = resolveParkedTerminalPaneCandidates(tab, state) - const restorePolicy = parkRestorePolicyFromState(state) - const disposersByPtyId = new Map void>() - const paneIdByPtyId = new Map() - for (const pane of panes) { - const ptyId = pane.ptyId - // Why: re-guard — the tab model can change after the park decision, and legacy non-UUID leaf ids make makePaneKey throw. - if ( - !ptyId || - disposersByPtyId.has(ptyId) || - !isTerminalLeafId(pane.leafId) || - !isParkRestorableTerminalPty(ptyId, worktreeId, restorePolicy) - ) { - continue - } - const initialTitle = state.runtimePaneTitlesByTabId[tab.id]?.[pane.paneId] - const disposeWatcher = startParkedTerminalByteWatcher({ - ptyId, - tabId: tab.id, - worktreeId, - leafId: pane.leafId, - paneId: pane.paneId, - drivesTabTitle: pane.drivesTabTitle, - // Why: seed the agent tracker with the last title so an agent working at park time still notifies on finish. - ...(initialTitle !== undefined ? { initialTitle } : {}), - ...(restoreTitleOnRegister ? { restoreTitleOnRegister: true } : {}), - // Why: no pane transport while parked, so write straight to the PTY (same channel as background launches). - sendInput: (data) => window.api.pty.write(ptyId, data) - }) - // Why: a PTY exiting while parked has no pane for cleanup, so its watcher must not outlive it. - const unsubscribeExit = subscribeToPtyExit(ptyId, (_code, { hadPrimary }) => { - // Why: while parked this sidecar is the only exit observer, so teardown must run here or dead leaves resurrect on reveal. - useAppStore.getState().clearRuntimePaneTitle(tab.id, pane.paneId) - if (disposersByPtyId.size > 1) { - // Why: a parked PaneManager is gone, so its retained primary cannot remove a dead split leaf from persisted layout. - discardPreHandlerPtyState(ptyId) - collapseParkedExitedLeaf(tab.id, ptyId) - disposersByPtyId.get(ptyId)?.() - disposersByPtyId.delete(ptyId) - return - } - if (hadPrimary) { - // Why: the sole pane's primary owner closes its tab; retire the sidecar to avoid duplicate confirmation. - disposersByPtyId.get(ptyId)?.() - disposersByPtyId.delete(ptyId) - return - } - - // Why: keep the empty entry so a pending pinned-close confirm can't let parking restart a watcher on the dead PTY. - disposersByPtyId.get(ptyId)?.() - disposersByPtyId.delete(ptyId) - closeTerminalTab(tab.id, { - // Why: autonomous PTY exit still needs pinned-tab confirmation but must not enter reopen history. - captureRecentlyClosed: false, - // Why: same lifecycle echo as the mounted pty-exit handlers — tag the - // wire so the host can refuse it while its PTY is live, without - // `reason: 'pty-exit'` skipping the pinned confirmation above. - hostCloseReason: 'pty-exit', - lifecyclePtyId: ptyId, - onClosed: () => { - discardPreHandlerPtyState(ptyId) - const entry = parkedWatchersByTabId.get(tab.id) - if (entry?.disposersByPtyId === disposersByPtyId) { - parkedWatchersByTabId.delete(tab.id) - } - }, - // Why: cancellation keeps the buffered final frame/exit for the reveal-mounted pane. - onCancel: () => {} - }) - }) - paneIdByPtyId.set(ptyId, pane.paneId) - disposersByPtyId.set(ptyId, () => { - unsubscribeExit() - disposeWatcher() - }) - } - // Why: track even with zero watchers so window.__terminalParkingDebug reflects every parked tab. - parkedWatchersByTabId.set(tab.id, { + const entry: ParkedTabWatcherEntry = { worktreeId, tabPtyId: tab.ptyId, - paneIdByPtyId, - disposersByPtyId + paneIdByPtyId: new Map(), + disposersByPtyId: new Map() + } + parkedWatchersByTabId.set(tab.id, entry) + const restorePolicy = parkRestorePolicyFromState(useAppStore.getState()) + for (const pane of resolveParkedTerminalPaneCandidates(tab, useAppStore.getState())) { + startParkedPtyWatcher({ + worktreeId, + tab, + pane, + entry, + restoreTitleOnRegister, + restorePolicy + }) + } +} + +function reconcileParkedTabWatchers( + worktreeId: string, + tab: ParkableTerminalTabModel, + entry: ParkedTabWatcherEntry, + restoreTitleOnRegister: boolean +): void { + const state = useAppStore.getState() + const expectedPanes = watchablePanes(worktreeId, tab) + const expectedPtyIds = new Set(expectedPanes.keys()) + const reconciliation = reconcileParkedWatcherPtyIds({ + currentTabPtyId: tab.ptyId, + entryTabPtyId: entry.tabPtyId, + paneIdByPtyId: entry.paneIdByPtyId, + expectedPtyIds }) + if (reconciliation.restartAll) { + const retainedTitles = reconciliation.retainedPtyIds.flatMap((ptyId) => { + const paneId = entry.paneIdByPtyId.get(ptyId) + const title = + paneId === undefined ? undefined : state.runtimePaneTitlesByTabId[tab.id]?.[paneId] + return paneId !== undefined && title !== undefined ? [{ paneId, title }] : [] + }) + for (const paneId of reconciliation.retiredPaneIds) { + state.clearRuntimePaneTitle(tab.id, paneId) + } + disposeParkedTabWatchers(tab.id) + for (const { paneId, title } of retainedTitles) { + useAppStore.getState().setRuntimePaneTitle(tab.id, paneId, title) + } + startParkedTabWatchers(worktreeId, tab, restoreTitleOnRegister) + return + } + for (const [ptyId, paneId] of Array.from(entry.paneIdByPtyId)) { + if (expectedPtyIds.has(ptyId)) { + continue + } + entry.paneIdByPtyId.delete(ptyId) + const dispose = entry.disposersByPtyId.get(ptyId) + entry.disposersByPtyId.delete(ptyId) + dispose?.() + state.clearRuntimePaneTitle(tab.id, paneId) + } + const restorePolicy = parkRestorePolicyFromState(useAppStore.getState()) + for (const ptyId of reconciliation.addedPtyIds) { + const pane = expectedPanes.get(ptyId) + if (pane) { + startParkedPtyWatcher({ + worktreeId, + tab, + pane, + entry, + restoreTitleOnRegister, + restorePolicy + }) + } + } +} + +function watchablePanes( + worktreeId: string, + tab: ParkableTerminalTabModel +): Map { + const state = useAppStore.getState() + const restorePolicy = parkRestorePolicyFromState(state) + return new Map( + resolveParkedTerminalPaneCandidates(tab, state).flatMap((pane) => + pane.ptyId && + isTerminalLeafId(pane.leafId) && + isParkRestorableTerminalPty(pane.ptyId, worktreeId, restorePolicy) + ? [[pane.ptyId, pane] as const] + : [] + ) + ) } /** @@ -256,39 +224,6 @@ export function shouldDeferParkedPtyExitTabClose(tabId: string, ptyId: string): return defer } -// Why: collapse the leaf from the stored layout so reveal can't reattach and resurrect the exited shell. -function collapseParkedExitedLeaf(tabId: string, ptyId: string): void { - const state = useAppStore.getState() - const layout = state.terminalLayoutsByTabId[tabId] - const leafId = - capturedPanesByTabId.get(tabId)?.panes.find((pane) => pane.ptyId === ptyId)?.leafId ?? - Object.entries(layout?.ptyIdsByLeafId ?? {}).find(([, boundPtyId]) => boundPtyId === ptyId)?.[0] - if (!leafId) { - return - } - const detached = detachTerminalLayoutLeaf(layout, leafId) - if (detached) { - const terminalTab = Object.values(state.tabsByWorktree) - .flat() - .find((candidate) => candidate.id === tabId) - if (shouldClearLaunchAgentForClosedPane(terminalTab, ptyId)) { - state.clearTabLaunchAgent(tabId) - } - state.setTabLayout(tabId, detached.sourceLayout) - const activeLeafId = detached.sourceLayout.activeLeafId - const activePtyId = activeLeafId - ? detached.sourceLayout.ptyIdsByLeafId?.[activeLeafId] - : undefined - const activePaneId = activePtyId - ? (parkedWatchersByTabId.get(tabId)?.paneIdByPtyId.get(activePtyId) ?? null) - : null - state.updateTabTitle( - tabId, - resolveTabTitleAfterPaneClose(state.runtimePaneTitlesByTabId[tabId] ?? {}, activePaneId) - ) - } -} - function disposeClosedParkedTabWatchers( tabId: string, entry: { paneIdByPtyId: ReadonlyMap } @@ -297,6 +232,9 @@ function disposeClosedParkedTabWatchers( for (const ptyId of entry.paneIdByPtyId.keys()) { discardPreHandlerPtyState(ptyId) } + for (const paneId of entry.paneIdByPtyId.values()) { + useAppStore.getState().clearRuntimePaneTitle(tabId, paneId) + } disposeParkedTabWatchers(tabId) } @@ -336,15 +274,11 @@ export function syncParkedTerminalTabWatchers(args: { continue } const entry = parkedWatchersByTabId.get(tab.id) - if (entry && entry.tabPtyId !== tab.ptyId) { - disposeParkedTabWatchers(tab.id) - } - if (!parkedWatchersByTabId.has(tab.id)) { - startParkedTabWatchers( - args.worktreeId, - tab, - args.restoreTitleOnStartTabIds?.has(tab.id) === true - ) + const restoreTitleOnRegister = args.restoreTitleOnStartTabIds?.has(tab.id) === true + if (entry) { + reconcileParkedTabWatchers(args.worktreeId, tab, entry, restoreTitleOnRegister) + } else { + startParkedTabWatchers(args.worktreeId, tab, restoreTitleOnRegister) } } } diff --git a/src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts b/src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts new file mode 100644 index 000000000..d5fd24eca --- /dev/null +++ b/src/renderer/src/components/terminal-pane/terminal-parked-watcher-partial-reconciliation.test.ts @@ -0,0 +1,111 @@ +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { + ParkedTabWatcherEntry, + ParkedTerminalPaneCapture +} from './terminal-parked-watcher-registry' + +const WORKTREE_ID = 'repo::/worktree' +const TAB_ID = 'tab-1' +const FIRST_PTY_ID = `${WORKTREE_ID}@@session-1` +const OLD_SECOND_PTY_ID = `${WORKTREE_ID}@@session-2` +const NEW_SECOND_PTY_ID = `${WORKTREE_ID}@@session-3` +const FIRST_LEAF_ID = '11111111-1111-4111-8111-111111111111' +const SECOND_LEAF_ID = '22222222-2222-4222-8222-222222222222' + +const startedWatchers: { + pane: ParkedTerminalPaneCapture + dispose: ReturnType +}[] = [] + +vi.mock('./terminal-parked-pty-watcher', () => ({ + collapseParkedExitedLeaf: vi.fn(), + startParkedPtyWatcher: (args: { + pane: ParkedTerminalPaneCapture + entry: ParkedTabWatcherEntry + }) => { + if (!args.pane.ptyId) { + return + } + const dispose = vi.fn() + startedWatchers.push({ pane: args.pane, dispose }) + args.entry.paneIdByPtyId.set(args.pane.ptyId, args.pane.paneId) + args.entry.disposersByPtyId.set(args.pane.ptyId, dispose) + } +})) + +vi.mock('./pty-pre-handler-buffer', () => ({ + discardPreHandlerPtyState: vi.fn() +})) + +const state = { + tabsByWorktree: {}, + terminalLayoutsByTabId: {} as Record, + runtimePaneTitlesByTabId: {} as Record>, + settings: null, + runtimeStatusByEnvironmentId: new Map(), + clearRuntimePaneTitle: vi.fn(), + setRuntimePaneTitle: vi.fn() +} + +vi.mock('@/store', () => ({ + useAppStore: { getState: () => state } +})) + +import { + captureParkedTerminalPaneCandidates, + pruneParkedTerminalWatchers, + syncParkedTerminalTabWatchers +} from './terminal-parked-tab-watchers' + +function sync(): void { + syncParkedTerminalTabWatchers({ + worktreeId: WORKTREE_ID, + tabs: [{ id: TAB_ID, ptyId: FIRST_PTY_ID }], + parkedTabIds: new Set([TAB_ID]) + }) +} + +beforeEach(() => { + state.terminalLayoutsByTabId = {} + state.runtimePaneTitlesByTabId = { [TAB_ID]: { 1: '⠋ Continuing agent', 2: 'Retired shell' } } + startedWatchers.length = 0 + vi.clearAllMocks() +}) + +afterEach(() => { + pruneParkedTerminalWatchers(new Set()) +}) + +it('retains a continuing watcher and title while reconciling a reminted split leaf', () => { + captureParkedTerminalPaneCandidates(TAB_ID, WORKTREE_ID, [ + { ptyId: FIRST_PTY_ID, paneId: 1, leafId: FIRST_LEAF_ID, drivesTabTitle: true }, + { ptyId: OLD_SECOND_PTY_ID, paneId: 2, leafId: SECOND_LEAF_ID, drivesTabTitle: false } + ]) + sync() + + state.terminalLayoutsByTabId[TAB_ID] = { + root: { + type: 'split', + direction: 'vertical', + first: { type: 'leaf', leafId: FIRST_LEAF_ID }, + second: { type: 'leaf', leafId: SECOND_LEAF_ID } + }, + activeLeafId: FIRST_LEAF_ID, + expandedLeafId: null, + ptyIdsByLeafId: { + [FIRST_LEAF_ID]: FIRST_PTY_ID, + [SECOND_LEAF_ID]: NEW_SECOND_PTY_ID + } + } + sync() + + expect(startedWatchers[0].dispose).not.toHaveBeenCalled() + expect(startedWatchers[1].dispose).toHaveBeenCalledOnce() + expect(startedWatchers[2].pane).toMatchObject({ + ptyId: NEW_SECOND_PTY_ID, + paneId: 2, + drivesTabTitle: false + }) + expect(state.clearRuntimePaneTitle).toHaveBeenCalledWith(TAB_ID, 2) + expect(state.clearRuntimePaneTitle).not.toHaveBeenCalledWith(TAB_ID, 1) +}) diff --git a/src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts b/src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts new file mode 100644 index 000000000..18e8f41ca --- /dev/null +++ b/src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.test.ts @@ -0,0 +1,87 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { + captureParkedTerminalPaneCandidates, + retireParkedTerminalTab +} from './terminal-parked-watcher-registry' +import { + reconcileParkedWatcherPtyIds, + resolveParkedTerminalPaneCandidates +} from './terminal-parked-watcher-reconciliation' + +const TAB_ID = 'tab-1' +const WORKTREE_ID = 'repo::/worktree' +const FIRST_LEAF_ID = '11111111-1111-4111-8111-111111111111' +const SECOND_LEAF_ID = '22222222-2222-4222-8222-222222222222' +const FIRST_PTY_ID = 'remote:env-1@@terminal-1' +const OLD_SECOND_PTY_ID = 'remote:env-1@@terminal-2' +const NEW_SECOND_PTY_ID = 'remote:env-1@@terminal-3' + +afterEach(() => { + retireParkedTerminalTab(TAB_ID) +}) + +describe('paired parked-watcher reconciliation', () => { + it('prefers an authoritative inactive split-leaf remint over the unmount capture', () => { + captureParkedTerminalPaneCandidates(TAB_ID, WORKTREE_ID, [ + { ptyId: FIRST_PTY_ID, paneId: 1, leafId: FIRST_LEAF_ID, drivesTabTitle: true }, + { + ptyId: OLD_SECOND_PTY_ID, + paneId: 2, + leafId: SECOND_LEAF_ID, + drivesTabTitle: false + } + ]) + + const panes = resolveParkedTerminalPaneCandidates( + { id: TAB_ID, ptyId: FIRST_PTY_ID }, + { + runtimePaneTitlesByTabId: {}, + terminalLayoutsByTabId: { + [TAB_ID]: { + root: { + type: 'split', + direction: 'vertical', + first: { type: 'leaf', leafId: FIRST_LEAF_ID }, + second: { type: 'leaf', leafId: SECOND_LEAF_ID } + }, + activeLeafId: FIRST_LEAF_ID, + expandedLeafId: null, + ptyIdsByLeafId: { + [FIRST_LEAF_ID]: FIRST_PTY_ID, + [SECOND_LEAF_ID]: NEW_SECOND_PTY_ID + } + } + } + } + ) + + expect(panes).toEqual([ + { ptyId: FIRST_PTY_ID, paneId: 1, leafId: FIRST_LEAF_ID, drivesTabTitle: true }, + { + ptyId: NEW_SECOND_PTY_ID, + paneId: 2, + leafId: SECOND_LEAF_ID, + drivesTabTitle: false + } + ]) + }) + + it('surgically reconciles a reminted split leaf without restarting its sibling', () => { + expect( + reconcileParkedWatcherPtyIds({ + currentTabPtyId: FIRST_PTY_ID, + entryTabPtyId: FIRST_PTY_ID, + paneIdByPtyId: new Map([ + [FIRST_PTY_ID, 1], + [OLD_SECOND_PTY_ID, 2] + ]), + expectedPtyIds: new Set([FIRST_PTY_ID, NEW_SECOND_PTY_ID]) + }) + ).toEqual({ + restartAll: false, + addedPtyIds: [NEW_SECOND_PTY_ID], + retainedPtyIds: [FIRST_PTY_ID], + retiredPaneIds: [2] + }) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.ts b/src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.ts new file mode 100644 index 000000000..5376cb9ed --- /dev/null +++ b/src/renderer/src/components/terminal-pane/terminal-parked-watcher-reconciliation.ts @@ -0,0 +1,91 @@ +import type { TerminalTab } from '../../../../shared/types' +import type { useAppStore } from '@/store' +import { + collectLeafIdsInOrder, + resolveRootlessTerminalLayoutLeafId +} from './terminal-layout-leaf-ids' +import { + capturedPanesByTabId, + type ParkedTerminalPaneCapture +} from './terminal-parked-watcher-registry' + +export type ParkableTerminalTabModel = Pick + +type ParkedPaneFallbackState = { + terminalLayoutsByTabId: ReturnType['terminalLayoutsByTabId'] + runtimePaneTitlesByTabId: ReturnType['runtimePaneTitlesByTabId'] +} + +export function fallbackParkedPaneCandidates( + tab: ParkableTerminalTabModel, + state: ParkedPaneFallbackState +): ParkedTerminalPaneCapture[] { + const layout = state.terminalLayoutsByTabId[tab.id] + const rootLeafIds = collectLeafIdsInOrder(layout?.root) + const rootlessLeafId = layout ? resolveRootlessTerminalLayoutLeafId(layout) : null + const leafIds = + rootLeafIds.length > 0 ? rootLeafIds : rootlessLeafId !== null ? [rootlessLeafId] : [] + if (leafIds.length === 0) { + return [] + } + const ptyIdsByLeafId = layout?.ptyIdsByLeafId ?? {} + const titleSlots = Object.keys(state.runtimePaneTitlesByTabId[tab.id] ?? {}) + const reusableSlot = + leafIds.length === 1 && titleSlots.length === 1 ? Number(titleSlots[0]) : null + return leafIds.map((leafId, index) => ({ + ptyId: ptyIdsByLeafId[leafId] ?? (leafIds.length === 1 ? tab.ptyId : null), + paneId: reusableSlot ?? -(index + 1), + leafId, + drivesTabTitle: layout?.activeLeafId ? leafId === layout.activeLeafId : index === 0 + })) +} + +export function resolveParkedTerminalPaneCandidates( + tab: ParkableTerminalTabModel, + state: ParkedPaneFallbackState +): ParkedTerminalPaneCapture[] { + const captured = capturedPanesByTabId.get(tab.id) + const fallback = fallbackParkedPaneCandidates(tab, state) + const capturedIsCurrent = + captured !== undefined && + captured.panes.length > 0 && + (tab.ptyId === null || captured.panes.some((pane) => pane.ptyId === tab.ptyId)) && + (fallback.length === 0 || + (captured.panes.length === fallback.length && + fallback.every((pane) => + captured.panes.some( + (candidate) => candidate.leafId === pane.leafId && candidate.ptyId === pane.ptyId + ) + ))) + if (capturedIsCurrent) { + return captured.panes + } + return fallback.map((pane) => { + const prior = captured?.panes.find((candidate) => candidate.leafId === pane.leafId) + return prior ? { ...pane, paneId: prior.paneId, drivesTabTitle: prior.drivesTabTitle } : pane + }) +} + +export function reconcileParkedWatcherPtyIds(args: { + currentTabPtyId: string | null + entryTabPtyId: string | null + paneIdByPtyId: ReadonlyMap + expectedPtyIds: ReadonlySet +}): { + restartAll: boolean + addedPtyIds: string[] + retainedPtyIds: string[] + retiredPaneIds: number[] +} { + const retainedPtyIds = Array.from(args.paneIdByPtyId.keys()).filter((ptyId) => + args.expectedPtyIds.has(ptyId) + ) + return { + restartAll: args.entryTabPtyId !== args.currentTabPtyId, + addedPtyIds: Array.from(args.expectedPtyIds).filter((ptyId) => !args.paneIdByPtyId.has(ptyId)), + retainedPtyIds, + retiredPaneIds: Array.from(args.paneIdByPtyId) + .filter(([ptyId]) => !args.expectedPtyIds.has(ptyId)) + .map(([, paneId]) => paneId) + } +} diff --git a/src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.ts b/src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.ts index cbef3cebf..9fca8a2ca 100644 --- a/src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.ts +++ b/src/renderer/src/components/terminal-pane/terminal-parking-e2e-overrides.ts @@ -5,6 +5,20 @@ import { } from './terminal-hidden-view-parking' import { getParkedTerminalWatcherTabIds } from './terminal-parked-tab-watchers' +export type TerminalWorktreeParkingDebugVerdict = { + worktreeId: string + forceParked: boolean + hasActivityTerminalPortal: boolean + hasPendingSpawnWork: boolean + hiddenSinceMs: number | null + isVisible: boolean + ordinaryParkingCovers: boolean + parkCooldownUntilMs: number | null + shouldMeasureHiddenWorktree: boolean +} + +let worktreeVerdicts: TerminalWorktreeParkingDebugVerdict[] = [] + // Why: ORCA_E2E_TERMINAL_PARKING_DELAY_MS must shrink BOTH the cold-park // hysteresis and the hot-retain window — recently hidden tabs otherwise sit // in the hot-retain working set for 5 minutes and never park within a test @@ -36,7 +50,17 @@ export function registerTerminalParkingDebugHandle(): void { window.__terminalParkingDebug = { parkDelayMs: getTerminalParkingPolicyOverrides().coldParkDelayMs ?? TERMINAL_TAB_COLD_PARK_DELAY_MS, - parkedTabIds: () => getParkedTerminalWatcherTabIds() + parkedTabIds: () => getParkedTerminalWatcherTabIds(), + retentionLimit: getTerminalParkingPolicyOverrides().retentionLimit ?? null, + worktreeVerdicts: () => worktreeVerdicts + } +} + +export function recordTerminalWorktreeParkingDebugVerdicts( + verdicts: TerminalWorktreeParkingDebugVerdict[] +): void { + if (e2eConfig.exposeStore) { + worktreeVerdicts = verdicts } } diff --git a/src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts b/src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts index c5f2b696f..b73f690e5 100644 --- a/src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts +++ b/src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.test.ts @@ -547,8 +547,10 @@ describe('registerTerminalSideEffectFactConsumer', () => { }) dispose() _dispatchTerminalSideEffectBatchForTest(batch([{ kind: 'bell' }])) + expect(vi.getTimerCount()).toBe(1) vi.advanceTimersByTime(15_001) + expect(vi.getTimerCount()).toBe(0) const { callbacks, events } = createCallbackRecorder() registerTerminalSideEffectFactConsumer({ ptyId: PTY_ID, callbacks }) diff --git a/src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.ts b/src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.ts index 95374ae9f..e7f4a54ec 100644 --- a/src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.ts +++ b/src/renderer/src/components/terminal-pane/terminal-side-effect-facts-handler.ts @@ -177,29 +177,46 @@ const HANDOFF_FACT_BUFFER_TTL_MS = 15_000 const MAX_HANDOFF_FACT_BATCHES = 64 const MAX_HANDOFF_FACT_PTYS = 32 -type HandoffFactBuffer = { batches: TerminalSideEffectBatch[]; expiresAtMs: number } +type HandoffFactBuffer = { + batches: TerminalSideEffectBatch[] + expiresAtMs: number + expiryTimer: ReturnType +} const handoffFactBuffersByPtyId = new Map() +function deleteHandoffFactBuffer(ptyId: string): void { + const buffer = handoffFactBuffersByPtyId.get(ptyId) + if (buffer) { + clearTimeout(buffer.expiryTimer) + handoffFactBuffersByPtyId.delete(ptyId) + } +} + function openHandoffFactBuffer(ptyId: string): void { const nowMs = Date.now() for (const [bufferedPtyId, buffer] of handoffFactBuffersByPtyId) { if (buffer.expiresAtMs <= nowMs) { - handoffFactBuffersByPtyId.delete(bufferedPtyId) + deleteHandoffFactBuffer(bufferedPtyId) } } - if ( - !handoffFactBuffersByPtyId.has(ptyId) && - handoffFactBuffersByPtyId.size >= MAX_HANDOFF_FACT_PTYS - ) { + deleteHandoffFactBuffer(ptyId) + if (handoffFactBuffersByPtyId.size >= MAX_HANDOFF_FACT_PTYS) { const oldestPtyId = handoffFactBuffersByPtyId.keys().next().value if (typeof oldestPtyId === 'string') { - handoffFactBuffersByPtyId.delete(oldestPtyId) + deleteHandoffFactBuffer(oldestPtyId) } } - handoffFactBuffersByPtyId.set(ptyId, { + const buffer: HandoffFactBuffer = { batches: [], - expiresAtMs: nowMs + HANDOFF_FACT_BUFFER_TTL_MS - }) + expiresAtMs: nowMs + HANDOFF_FACT_BUFFER_TTL_MS, + expiryTimer: setTimeout(() => { + if (handoffFactBuffersByPtyId.get(ptyId) === buffer) { + handoffFactBuffersByPtyId.delete(ptyId) + } + }, HANDOFF_FACT_BUFFER_TTL_MS) + } + buffer.expiryTimer.unref?.() + handoffFactBuffersByPtyId.set(ptyId, buffer) } function bufferHandoffFactBatch(batch: TerminalSideEffectBatch): void { @@ -208,7 +225,7 @@ function bufferHandoffFactBatch(batch: TerminalSideEffectBatch): void { return } if (buffer.expiresAtMs <= Date.now()) { - handoffFactBuffersByPtyId.delete(batch.ptyId) + deleteHandoffFactBuffer(batch.ptyId) return } // Why: replay batches are snapshots the next consumer requests for itself. @@ -226,7 +243,7 @@ function drainHandoffFactBuffer(ptyId: string, entry: ConsumerEntry): void { if (!buffer) { return } - handoffFactBuffersByPtyId.delete(ptyId) + deleteHandoffFactBuffer(ptyId) if (buffer.expiresAtMs <= Date.now()) { return } @@ -235,7 +252,7 @@ function drainHandoffFactBuffer(ptyId: string, entry: ConsumerEntry): void { } } -function handleSideEffectBatch(batch: TerminalSideEffectBatch): void { +export function dispatchTerminalSideEffectBatch(batch: TerminalSideEffectBatch): void { const entry = consumersByPtyId.get(batch.ptyId) if (!entry) { bufferHandoffFactBatch(batch) @@ -254,7 +271,7 @@ function ensureSideEffectChannelSubscription(): void { if (typeof onSideEffect !== 'function') { return } - channelUnsubscribe = onSideEffect(handleSideEffectBatch) + channelUnsubscribe = onSideEffect(dispatchTerminalSideEffectBatch) } export type TerminalSideEffectFactConsumerOptions = { @@ -310,13 +327,15 @@ export function registerTerminalSideEffectFactConsumer( /** Test seam: deliver a batch as if it arrived on the channel. */ export function _dispatchTerminalSideEffectBatchForTest(batch: TerminalSideEffectBatch): void { - handleSideEffectBatch(batch) + dispatchTerminalSideEffectBatch(batch) } /** Test seam: reset module state between tests. */ export function _resetTerminalSideEffectFactConsumersForTest(): void { consumersByPtyId.clear() - handoffFactBuffersByPtyId.clear() + for (const ptyId of Array.from(handoffFactBuffersByPtyId.keys())) { + deleteHandoffFactBuffer(ptyId) + } channelUnsubscribe?.() channelUnsubscribe = null persistedAuthorityFlagCache = undefined diff --git a/src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts b/src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts index 85209b8c6..f98e39cc3 100644 --- a/src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts +++ b/src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts @@ -11,6 +11,7 @@ import { addBackgroundMountedTerminalWorktree, applyBackgroundMountTabRestriction, canDeferColdActivationTabsForHost, + canMountTerminalWorkspaceForStartup, collectDeferredMountTabIds, hasRequestedBackgroundTerminalWorktreeMount, planColdActivationTabDeferral, @@ -22,6 +23,32 @@ import { shouldMountBackgroundWorktreeTab } from './background-terminal-worktree-mount' +describe('terminal workspace startup mount gate', () => { + it('waits for hydration unless startup entered degraded mode', () => { + expect( + canMountTerminalWorkspaceForStartup({ + workspaceSessionReady: true, + hydrationSucceeded: false, + startupWorktreeRefreshCompleted: false + }) + ).toBe(false) + expect( + canMountTerminalWorkspaceForStartup({ + workspaceSessionReady: true, + hydrationSucceeded: true, + startupWorktreeRefreshCompleted: false + }) + ).toBe(true) + expect( + canMountTerminalWorkspaceForStartup({ + workspaceSessionReady: true, + hydrationSucceeded: false, + startupWorktreeRefreshCompleted: true + }) + ).toBe(true) + }) +}) + describe('background terminal mount request registry', () => { it('replays a request made before the Terminal listener mounts', () => { takeAllPendingBackgroundTerminalWorktreeMounts() diff --git a/src/renderer/src/components/terminal/background-terminal-worktree-mount.ts b/src/renderer/src/components/terminal/background-terminal-worktree-mount.ts index f0428baa0..ae8c677c2 100644 --- a/src/renderer/src/components/terminal/background-terminal-worktree-mount.ts +++ b/src/renderer/src/components/terminal/background-terminal-worktree-mount.ts @@ -141,6 +141,16 @@ export function shouldMountBackgroundWorktreeTab( // until first reveal, parked byte watchers own their side effects meanwhile. export const COLD_ACTIVATION_TAB_DEFER_THRESHOLD = 4 +export function canMountTerminalWorkspaceForStartup(args: { + workspaceSessionReady: boolean + hydrationSucceeded: boolean + startupWorktreeRefreshCompleted: boolean +}): boolean { + return ( + args.workspaceSessionReady && (args.hydrationSucceeded || args.startupWorktreeRefreshCompleted) + ) +} + export function canDeferColdActivationTabsForHost(args: { executionHostId: string | null }): boolean { diff --git a/src/renderer/src/env.d.ts b/src/renderer/src/env.d.ts index 390ebfe36..5ecc70a5d 100644 --- a/src/renderer/src/env.d.ts +++ b/src/renderer/src/env.d.ts @@ -4,6 +4,7 @@ import type { PaneManager } from '@/lib/pane-manager/pane-manager' import type { OnboardingFeatureSetupDeps } from '@/components/onboarding/onboarding-feature-setup' import type { languages } from 'monaco-editor' import type { MonacoE2EProbe } from './components/editor/monaco-e2e-probe' +import type { TerminalWorktreeParkingDebugVerdict } from './components/terminal-pane/terminal-parking-e2e-overrides' declare module 'monaco-editor/esm/vs/basic-languages/python/python.js' { export const conf: languages.LanguageConfiguration @@ -71,6 +72,8 @@ declare global { __terminalParkingDebug?: { parkDelayMs: number parkedTabIds: () => string[] + retentionLimit: number | null + worktreeVerdicts: () => TerminalWorktreeParkingDebugVerdict[] } __monacoEditorE2E?: MonacoE2EProbe __e2ePtyAppliedSizeReadDelayMs?: number diff --git a/src/renderer/src/hooks/useIpcEvents.ts b/src/renderer/src/hooks/useIpcEvents.ts index 9d9acf188..03d3a64b7 100644 --- a/src/renderer/src/hooks/useIpcEvents.ts +++ b/src/renderer/src/hooks/useIpcEvents.ts @@ -88,6 +88,8 @@ import { attachMobileMarkdownBridge } from '@/runtime/mobile-markdown-bridge' import { closeMobileSessionTabInStore } from '@/runtime/mobile-session-tab-close' import { createWorktreeChangeRefreshQueue } from './worktree-change-refresh-queue' import { subscribeRuntimeClientEvents } from '@/runtime/runtime-client-events' +import { toRemoteRuntimePtyId } from '@/runtime/runtime-terminal-stream' +import { dispatchTerminalSideEffectBatch } from '@/components/terminal-pane/terminal-side-effect-facts-handler' import { subscribeToUnpairedDeviceAuthNotification } from './unpaired-device-auth-notification' import { applyRuntimeEnvironmentSshStateChanged, @@ -927,6 +929,13 @@ export function useIpcEvents(): void { applyHostWorktreeTerminalSleepState(environmentId, event) return } + if (event.type === 'terminalSideEffects') { + dispatchTerminalSideEffectBatch({ + ...event.batch, + ptyId: toRemoteRuntimePtyId(event.batch.ptyId, environmentId) + }) + return + } if (event.type === 'reposChanged') { runtimeProjectRefreshScheduler.request(environmentId) return diff --git a/src/renderer/src/i18n/locales/en.json b/src/renderer/src/i18n/locales/en.json index c20c32966..e6f913198 100644 --- a/src/renderer/src/i18n/locales/en.json +++ b/src/renderer/src/i18n/locales/en.json @@ -171,7 +171,8 @@ "mobileScopeRejected": "This QR code grants limited (mobile) access. To use the full web app, open the browser access link from Settings → Runtime Environments → Share this Orca server → New Link." }, "webPreloadApi": { - "aiVaultUnavailableForHost": "Agent Session History is not available for this execution host." + "aiVaultUnavailableForHost": "Agent Session History is not available for this execution host.", + "runtimeEnvironmentManuallyDisconnected": "Runtime environment is manually disconnected." }, "web": { "preload": { diff --git a/src/renderer/src/i18n/locales/es.json b/src/renderer/src/i18n/locales/es.json index a0ef85e94..b925b8c47 100644 --- a/src/renderer/src/i18n/locales/es.json +++ b/src/renderer/src/i18n/locales/es.json @@ -148,7 +148,8 @@ "mobileScopeRejected": "Este código QR solo da acceso limitado para móvil. Para usar la app web completa, abre el enlace para navegador desde Ajustes → Servidores remotos de Orca → Comparte este servidor Orca → Nuevo enlace." }, "webPreloadApi": { - "aiVaultUnavailableForHost": "El historial de sesiones de Agents no está disponible para este host de ejecución." + "aiVaultUnavailableForHost": "El historial de sesiones de Agents no está disponible para este host de ejecución.", + "runtimeEnvironmentManuallyDisconnected": "El entorno de ejecución se ha desconectado manualmente." }, "web": { "preload": { diff --git a/src/renderer/src/i18n/locales/ja.json b/src/renderer/src/i18n/locales/ja.json index b5c473513..3e3711874 100644 --- a/src/renderer/src/i18n/locales/ja.json +++ b/src/renderer/src/i18n/locales/ja.json @@ -148,7 +148,8 @@ "mobileScopeRejected": "この QR コードは制限付き（モバイル）アクセスを許可します。完全な Web アプリを使うには、設定 → リモート Orca サーバー → この Orca サーバーを共有する → 新規リンク からブラウザー用アクセスリンクを開いてください。" }, "webPreloadApi": { - "aiVaultUnavailableForHost": "この実行ホストでは Agent セッション履歴を使用できません。" + "aiVaultUnavailableForHost": "この実行ホストでは Agent セッション履歴を使用できません。", + "runtimeEnvironmentManuallyDisconnected": "ランタイム環境は手動で切断されています。" }, "web": { "preload": { diff --git a/src/renderer/src/i18n/locales/ko.json b/src/renderer/src/i18n/locales/ko.json index 2d5d50d44..5310100e4 100644 --- a/src/renderer/src/i18n/locales/ko.json +++ b/src/renderer/src/i18n/locales/ko.json @@ -148,7 +148,8 @@ "mobileScopeRejected": "이 QR 코드는 제한된(모바일) 액세스 권한만 제공합니다. 전체 웹 앱을 사용하려면 설정 → 원격 Orca 서버 → 이 Orca 서버 공유 → 새 링크에서 브라우저 액세스 링크를 여세요." }, "webPreloadApi": { - "aiVaultUnavailableForHost": "이 실행 호스트에서는 Agent 세션 기록을 사용할 수 없습니다." + "aiVaultUnavailableForHost": "이 실행 호스트에서는 Agent 세션 기록을 사용할 수 없습니다.", + "runtimeEnvironmentManuallyDisconnected": "런타임 환경이 수동으로 연결 해제되었습니다." }, "web": { "preload": { diff --git a/src/renderer/src/i18n/locales/zh.json b/src/renderer/src/i18n/locales/zh.json index 5dc222ce6..3a948e98e 100644 --- a/src/renderer/src/i18n/locales/zh.json +++ b/src/renderer/src/i18n/locales/zh.json @@ -148,7 +148,8 @@ "mobileScopeRejected": "此二维码仅授予受限（移动端）访问权限。若要使用完整 Web 应用，请从设置 → 远程 Orca 服务器 → 分享此 Orca 服务器 → 新链接打开浏览器访问链接。" }, "webPreloadApi": { - "aiVaultUnavailableForHost": "此执行主机不支持 Agent 会话历史记录。" + "aiVaultUnavailableForHost": "此执行主机不支持 Agent 会话历史记录。", + "runtimeEnvironmentManuallyDisconnected": "运行时环境已手动断开连接。" }, "web": { "preload": { diff --git a/src/renderer/src/lib/e2e-config.ts b/src/renderer/src/lib/e2e-config.ts index 2c7572f55..5ed719bdb 100644 --- a/src/renderer/src/lib/e2e-config.ts +++ b/src/renderer/src/lib/e2e-config.ts @@ -1,11 +1,24 @@ import { createE2EConfig } from '../../../shared/e2e-config' +const rendererE2EExposeStore = String(import.meta.env.VITE_EXPOSE_STORE) === 'true' +// Why: the paired web API installs after static modules initialize, so its +// build-gated fallback must read the test URL before caching this config. +const rendererE2EQuery = + rendererE2EExposeStore && typeof window !== 'undefined' + ? new URLSearchParams(window.location.search) + : null +export const e2eDisableRemoteTerminalStallRecovery = + rendererE2EExposeStore && + rendererE2EQuery?.get('orcaE2EDisableRemoteTerminalStallRecovery') === '1' +const rendererFallbackE2EConfig = createE2EConfig({ + exposeStore: rendererE2EExposeStore, + terminalParkingDelayMs: Number(rendererE2EQuery?.get('orcaE2ETerminalParkingDelayMs')) || null, + terminalRetentionLimit: Number(rendererE2EQuery?.get('orcaE2ETerminalRetentionLimit')) || null +}) + // Why: preload owns the Electron startup contract, so renderer code should // consume the bridged E2E config from window.api instead of reading env vars. export const e2eConfig = typeof window !== 'undefined' && window.api?.e2e ? window.api.e2e.getConfig() - : createE2EConfig({ - // Why: paired browser E2E has no preload bridge, so its build flag is the only safe test-hook signal. - exposeStore: String(import.meta.env.VITE_EXPOSE_STORE) === 'true' - }) + : rendererFallbackE2EConfig diff --git a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts index 3e9d67fbc..5d828b36c 100644 --- a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts +++ b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts @@ -10,14 +10,20 @@ import { encodeTerminalStreamJson, encodeTerminalStreamText } from '../../../shared/terminal-stream-protocol' -import { e2eConfig } from '@/lib/e2e-config' +import { e2eConfig, e2eDisableRemoteTerminalStallRecovery } from '@/lib/e2e-config' +import { recordRendererCrashBreadcrumb } from '@/lib/crash-breadcrumb-recorder' import { deliverTerminalDataWithDeferredCredit } from '@/lib/pane-manager/terminal-delivery-credit' import { unwrapRuntimeRpcResult } from './runtime-rpc-client' import { getRuntimeEnvironmentRevision } from './runtime-environment-revision' import { TERMINAL_MULTIPLEX_ACK_BATCH_BYTES, - TERMINAL_MULTIPLEX_ACK_FLUSH_MS + TERMINAL_MULTIPLEX_ACK_FLUSH_MS, + TERMINAL_MULTIPLEX_STREAM_LIMIT_ERROR } from '../../../shared/terminal-multiplex-flow-control' +import { + createRemoteTerminalStreamWatchdog, + type RemoteTerminalStreamWatchdog +} from './remote-terminal-stream-watchdog' type RuntimeEnvironmentSubscriptionHandle = { unsubscribe: () => void @@ -62,7 +68,7 @@ export type RemoteRuntimeMultiplexedTerminalCallbacks = { onDriverChanged?: ( driver: { kind: 'idle' } | { kind: 'desktop' } | { kind: 'mobile'; clientId: string } ) => void - onTransportClose?: (event: { recoverable: boolean }) => void + onTransportClose?: (event: { recoverable: boolean; retryWithBackoff?: boolean }) => void } export type RemoteRuntimeMultiplexedTerminal = { @@ -109,6 +115,8 @@ type RemoteRuntimeMultiplexedTerminalState = { resyncPendingSend: boolean resyncTimer: ReturnType | null resyncAttempts: number + capacityRejected: boolean + watchdog: RemoteTerminalStreamWatchdog } type RemoteRuntimeSnapshotInfo = { @@ -142,7 +150,7 @@ type RemoteRuntimeSnapshotRequest = { const CONTROL_STREAM_ID = 0 const MAX_REMOTE_TERMINAL_SNAPSHOT_BYTES = 2 * 1024 * 1024 -const REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS = 10_000 +export const REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS = 10_000 const REMOTE_TERMINAL_RESYNC_TIMEOUT_MS = 10_000 // Why: a truncated recovery means the server is too flooded to serialize; // retrying once per incoming chunk would stampede it, so back off instead. @@ -163,6 +171,7 @@ type E2eRemoteTerminalMultiplexAckGateSnapshot = { type E2eRemoteTerminalMultiplexAckGateApi = { hold: (terminals: string[]) => void release: () => void + sendInput: (terminal: string, text: string) => number snapshot: () => E2eRemoteTerminalMultiplexAckGateSnapshot } @@ -216,6 +225,13 @@ function exposeE2eRemoteTerminalMultiplexAckGate(): void { } }, release: releaseE2eRemoteTerminalAcks, + sendInput: (terminal, value) => { + let sent = 0 + for (const multiplexer of multiplexers.values()) { + sent += multiplexer.sendInputForE2e(terminal, value) + } + return sent + }, snapshot: getE2eRemoteAckSnapshot } } @@ -278,14 +294,37 @@ class RemoteRuntimeTerminalMultiplexer { resyncInFlight: false, resyncPendingSend: false, resyncTimer: null, - resyncAttempts: 0 + resyncAttempts: 0, + capacityRejected: false, + watchdog: createRemoteTerminalStreamWatchdog((stall) => { + if (e2eDisableRemoteTerminalStallRecovery) { + state.watchdog.completeCommandResponseProbe() + return + } + recordRendererCrashBreadcrumb('remote_terminal_stream_stall_recovery', { + environmentId: this.environmentId, + expectedSeq: state.expectedSeq ?? null, + inactiveForMs: stall.inactiveForMs, + outstandingDeliveryBytes: stall.outstandingDeliveryBytes, + pendingAckBytes: state.pendingAckBytes, + reason: stall.reason, + resyncAttempts: state.resyncAttempts, + snapshotPending: state.pendingSnapshotRequest !== null, + streamId: state.streamId, + terminal: state.terminal + }) + if (stall.reason === 'command-response-timeout') { + this.probeCommandResponse(state) + } else { + this.recoverStalledStream(state) + } + }) } this.streams.set(streamId, state) const stream: RemoteRuntimeMultiplexedTerminal = { streamId, - sendInput: (text) => - this.sendFrame(streamId, TerminalStreamOpcode.Input, encodeTerminalStreamText(text)), + sendInput: (text) => this.sendInput(state, text), resize: (cols, rows) => this.sendFrame( streamId, @@ -312,6 +351,7 @@ class RemoteRuntimeTerminalMultiplexer { close: () => { if (this.streams.get(streamId) === state) { discardOutputAcknowledgements(state) + state.watchdog.dispose() this.sendFrame(streamId, TerminalStreamOpcode.Unsubscribe) clearResyncTimer(state) rejectPendingSnapshotRequest(state, 'Remote terminal stream closed.') @@ -451,6 +491,7 @@ class RemoteRuntimeTerminalMultiplexer { if (!stream) { return } + stream.watchdog.recordInbound() if (event.type === 'subscribed') { const capabilities = typeof event.capabilities === 'object' && event.capabilities !== null @@ -466,18 +507,30 @@ class RemoteRuntimeTerminalMultiplexer { } } else if (event.type === 'end') { discardOutputAcknowledgements(stream) + stream.watchdog.dispose() clearSnapshot(stream) clearResyncTimer(stream) rejectPendingSnapshotRequest(stream, 'Remote terminal stream ended.') this.streams.delete(event.streamId) - stream.callbacks.onEnd?.() + if (stream.capacityRejected) { + if (stream.callbacks.onTransportClose) { + stream.callbacks.onTransportClose({ recoverable: true, retryWithBackoff: true }) + } else { + stream.callbacks.onError?.(TERMINAL_MULTIPLEX_STREAM_LIMIT_ERROR) + } + } else { + stream.callbacks.onEnd?.() + } this.closeIfIdle() } else if (event.type === 'error') { - clearSnapshot(stream) - rejectPendingSnapshotRequest( - stream, + const message = typeof event.message === 'string' ? event.message : 'Remote terminal stream failed.' - ) + if (message === TERMINAL_MULTIPLEX_STREAM_LIMIT_ERROR) { + stream.capacityRejected = true + return + } + clearSnapshot(stream) + rejectPendingSnapshotRequest(stream, message) // Why: the paired binary Error frame can be dropped under backpressure; // this reliable event must also dispatch or release the resync gate, and // must never disarm the watchdog while leaving the gate shut. @@ -487,9 +540,7 @@ class RemoteRuntimeTerminalMultiplexer { clearResyncTimer(stream) stream.resyncInFlight = false } - stream.callbacks.onError?.( - typeof event.message === 'string' ? event.message : 'Remote terminal stream failed.' - ) + stream.callbacks.onError?.(message) } else if (event.type === 'fit-override-changed') { if ( (event.mode !== 'mobile-fit' && @@ -535,6 +586,7 @@ class RemoteRuntimeTerminalMultiplexer { } return } + stream.watchdog.recordInbound() if ( frame.opcode === TerminalStreamOpcode.Output || frame.opcode === TerminalStreamOpcode.OutputSpan @@ -604,7 +656,9 @@ class RemoteRuntimeTerminalMultiplexer { return } try { + const settleWatchdog = stream.watchdog.beginOutputDelivery(frame.payload.byteLength) deliverTerminalDataWithDeferredCredit(() => { + settleWatchdog() if (shouldHoldE2eRemoteTerminalAck(stream.terminal)) { stream.heldAckBytes += frame.payload.byteLength } else { @@ -720,11 +774,16 @@ class RemoteRuntimeTerminalMultiplexer { return } if (frame.opcode === TerminalStreamOpcode.Error) { + const message = decodeTerminalStreamText(frame.payload) + if (message === TERMINAL_MULTIPLEX_STREAM_LIMIT_ERROR) { + stream.capacityRejected = true + return + } clearSnapshot(stream) const pendingSnapshotRequest = stream.pendingSnapshotRequest if (pendingSnapshotRequest) { clearPendingSnapshotRequest(stream) - pendingSnapshotRequest.reject(new Error(decodeTerminalStreamText(frame.payload))) + pendingSnapshotRequest.reject(new Error(message)) this.sendDeferredResyncSnapshot(stream) return } @@ -732,7 +791,7 @@ class RemoteRuntimeTerminalMultiplexer { clearResyncTimer(stream) stream.resyncInFlight = false stream.resyncPendingSend = false - stream.callbacks.onError?.(decodeTerminalStreamText(frame.payload)) + stream.callbacks.onError?.(message) } } @@ -873,7 +932,7 @@ class RemoteRuntimeTerminalMultiplexer { if (stream.pendingSnapshotRequest?.timer === timer) { clearPendingSnapshotRequest(stream) reject(new Error('Remote terminal snapshot timed out.')) - this.sendDeferredResyncSnapshot(stream) + this.recoverStalledStream(stream) } }, REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS) if (typeof timer.unref === 'function') { @@ -923,6 +982,59 @@ class RemoteRuntimeTerminalMultiplexer { ) } + private sendInput(stream: RemoteRuntimeMultiplexedTerminalState, text: string): boolean { + const sent = this.sendFrame( + stream.streamId, + TerminalStreamOpcode.Input, + encodeTerminalStreamText(text) + ) + if (sent) { + stream.watchdog.recordCommandInput(text) + } + return sent + } + + private probeCommandResponse(stream: RemoteRuntimeMultiplexedTerminalState): void { + void this.requestSnapshot(stream).then( + () => { + if (this.streams.get(stream.streamId) !== stream) { + return + } + stream.watchdog.completeCommandResponseProbe() + recordRendererCrashBreadcrumb('remote_terminal_stream_stall_probe_succeeded', { + environmentId: this.environmentId, + streamId: stream.streamId, + terminal: stream.terminal + }) + }, + () => { + // Snapshot timeout owns recovery; an explicit host error already proves liveness. + if (this.streams.get(stream.streamId) === stream) { + stream.watchdog.completeCommandResponseProbe() + } + } + ) + } + + private recoverStalledStream(stream: RemoteRuntimeMultiplexedTerminalState): void { + if (this.streams.get(stream.streamId) !== stream) { + return + } + stream.watchdog.dispose() + discardOutputAcknowledgements(stream) + clearSnapshot(stream) + clearResyncTimer(stream) + rejectPendingSnapshotRequest(stream, 'Remote terminal stream stopped responding.') + this.streams.delete(stream.streamId) + this.sendFrame(stream.streamId, TerminalStreamOpcode.Unsubscribe) + if (stream.callbacks.onTransportClose) { + stream.callbacks.onTransportClose({ recoverable: true }) + } else { + stream.callbacks.onError?.('Remote terminal stream stopped responding.') + } + this.closeIfIdle() + } + private queueOutputAcknowledgement( stream: RemoteRuntimeMultiplexedTerminalState, bytes: number @@ -969,6 +1081,16 @@ class RemoteRuntimeTerminalMultiplexer { return released } + sendInputForE2e(terminal: string, text: string): number { + let sent = 0 + for (const stream of this.streams.values()) { + if (stream.terminal === terminal && this.sendInput(stream, text)) { + sent += 1 + } + } + return sent + } + private sendFrame( streamId: number, opcode: TerminalStreamOpcode, @@ -1025,6 +1147,7 @@ class RemoteRuntimeTerminalMultiplexer { this.releaseIfCurrent(this.environmentId, this) for (const stream of streams) { discardOutputAcknowledgements(stream) + stream.watchdog.dispose() clearSnapshot(stream) clearResyncTimer(stream) rejectPendingSnapshotRequest(stream, message ?? 'Remote runtime connection closed.') diff --git a/src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts b/src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts new file mode 100644 index 000000000..921eb0d3a --- /dev/null +++ b/src/renderer/src/runtime/remote-runtime-terminal-stall-recovery.test.ts @@ -0,0 +1,277 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson, + encodeTerminalStreamFrame, + encodeTerminalStreamJson, + encodeTerminalStreamText +} from '../../../shared/terminal-stream-protocol' +import { + REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS, + REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS +} from './remote-terminal-stream-watchdog' + +describe('remote terminal stalled stream recovery', () => { + const sendBinary = vi.fn() + const unsubscribe = vi.fn() + const recordBreadcrumb = vi.fn() + let callbacks: { + onResponse: (response: unknown) => void + onBinary: (bytes: Uint8Array) => void + } | null = null + + beforeEach(() => { + vi.useFakeTimers() + vi.resetModules() + sendBinary.mockReset() + unsubscribe.mockReset() + recordBreadcrumb.mockReset() + callbacks = null + vi.stubGlobal('window', { + api: { + crashReports: { + recordBreadcrumb + }, + runtimeEnvironments: { + subscribe: vi.fn(async (_args, nextCallbacks) => { + callbacks = nextCallbacks + queueMicrotask(() => { + callbacks?.onResponse({ ok: true, result: { type: 'ready' } }) + }) + return { unsubscribe, sendBinary } + }) + } + } + }) + }) + + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + it('restarts only the stream whose renderer delivery credit never settles', async () => { + const { getRemoteRuntimeTerminalMultiplexer } = + await import('./remote-runtime-terminal-multiplexer') + const { takeCurrentTerminalDeliveryCredit } = + await import('../lib/pane-manager/terminal-delivery-credit') + const stalledCredits: (() => void)[] = [] + const onTransportClose = vi.fn() + const multiplexer = getRemoteRuntimeTerminalMultiplexer('windows-test') + const stalled = await multiplexer.subscribeTerminal({ + terminal: 'term-stalled', + client: { id: 'mac-viewer', type: 'desktop' }, + callbacks: { + onData: () => { + const credit = takeCurrentTerminalDeliveryCredit() + if (credit) { + stalledCredits.push(credit) + } + }, + onSnapshot: vi.fn(), + onTransportClose + } + }) + const healthy = await multiplexer.subscribeTerminal({ + terminal: 'term-healthy', + client: { id: 'mac-viewer', type: 'desktop' }, + callbacks: { onData: vi.fn(), onSnapshot: vi.fn() } + }) + sendBinary.mockClear() + + emitOutput(stalled.streamId, 'host output that xterm never parses') + expect(stalledCredits).toHaveLength(1) + + await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS) + + expect(onTransportClose).toHaveBeenCalledWith({ recoverable: true }) + expect(sentUnsubscribeStreamIds()).toEqual([stalled.streamId]) + expect(unsubscribe).not.toHaveBeenCalled() + expect(recordBreadcrumb).toHaveBeenCalledWith({ + name: 'remote_terminal_stream_stall_recovery', + data: expect.objectContaining({ + inactiveForMs: REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS, + outstandingDeliveryBytes: 'host output that xterm never parses'.length, + reason: 'delivery-credit-timeout', + streamId: stalled.streamId, + terminal: 'term-stalled' + }) + }) + healthy.close() + }) + + it('probes then restarts a stream when an entered command receives no frames', async () => { + const { getRemoteRuntimeTerminalMultiplexer, REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS } = + await import('./remote-runtime-terminal-multiplexer') + const onTransportClose = vi.fn() + const stream = await getRemoteRuntimeTerminalMultiplexer('windows-test').subscribeTerminal({ + terminal: 'term-silent', + client: { id: 'mac-viewer', type: 'desktop' }, + callbacks: { onData: vi.fn(), onSnapshot: vi.fn(), onTransportClose } + }) + sendBinary.mockClear() + + expect(stream.sendInput('ls\r')).toBe(true) + await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS) + + expect(onTransportClose).not.toHaveBeenCalled() + expect(sentFrames(TerminalStreamOpcode.SnapshotRequest)).toHaveLength(1) + await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS) + + expect(onTransportClose).toHaveBeenCalledWith({ recoverable: true }) + expect(sentUnsubscribeStreamIds()).toEqual([stream.streamId]) + expect(recordBreadcrumb).toHaveBeenCalledWith({ + name: 'remote_terminal_stream_stall_recovery', + data: expect.objectContaining({ + inactiveForMs: REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS, + outstandingDeliveryBytes: 0, + reason: 'command-response-timeout', + streamId: stream.streamId, + terminal: 'term-silent' + }) + }) + }) + + it('keeps a silent responsive stream after its authoritative snapshot probe', async () => { + const { getRemoteRuntimeTerminalMultiplexer } = + await import('./remote-runtime-terminal-multiplexer') + const onTransportClose = vi.fn() + const stream = await getRemoteRuntimeTerminalMultiplexer('windows-test').subscribeTerminal({ + terminal: 'term-password', + client: { id: 'mac-viewer', type: 'desktop' }, + callbacks: { onData: vi.fn(), onSnapshot: vi.fn(), onTransportClose } + }) + sendBinary.mockClear() + + expect(stream.sendInput('secret\r')).toBe(true) + await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS) + const firstRequest = sentFrames(TerminalStreamOpcode.SnapshotRequest)[0] + const firstRequestPayload = firstRequest + ? decodeTerminalStreamJson<{ requestId: number }>(firstRequest.payload) + : null + expect(firstRequestPayload?.requestId).toBeTypeOf('number') + + emitRequestedSnapshot(stream.streamId, firstRequestPayload?.requestId ?? 0) + await vi.advanceTimersByTimeAsync(0) + + expect(onTransportClose).not.toHaveBeenCalled() + expect(sentUnsubscribeStreamIds()).toEqual([]) + expect(stream.sendInput('silent-command\r')).toBe(true) + await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS) + expect(sentFrames(TerminalStreamOpcode.SnapshotRequest)).toHaveLength(2) + stream.close() + }) + + it('keeps a command stream when any host frame proves it is responsive', async () => { + const { getRemoteRuntimeTerminalMultiplexer } = + await import('./remote-runtime-terminal-multiplexer') + const onTransportClose = vi.fn() + const onData = vi.fn() + const stream = await getRemoteRuntimeTerminalMultiplexer('windows-test').subscribeTerminal({ + terminal: 'term-responsive', + client: { id: 'mac-viewer', type: 'desktop' }, + callbacks: { onData, onSnapshot: vi.fn(), onTransportClose } + }) + sendBinary.mockClear() + + expect(stream.sendInput('ls\r')).toBe(true) + emitOutput(stream.streamId, 'responsive output') + await vi.advanceTimersByTimeAsync(REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS) + + expect(onData).toHaveBeenCalledWith('responsive output', { + seq: 'responsive output'.length, + rawLength: 'responsive output'.length + }) + expect(onTransportClose).not.toHaveBeenCalled() + expect(sentUnsubscribeStreamIds()).toEqual([]) + stream.close() + }) + + it('classifies a capacity rejection followed by end as recoverable transport pressure', async () => { + const { getRemoteRuntimeTerminalMultiplexer } = + await import('./remote-runtime-terminal-multiplexer') + const onEnd = vi.fn() + const onError = vi.fn() + const onTransportClose = vi.fn() + const stream = await getRemoteRuntimeTerminalMultiplexer('windows-test').subscribeTerminal({ + terminal: 'term-over-capacity', + client: { id: 'mac-viewer', type: 'desktop' }, + callbacks: { + onData: vi.fn(), + onSnapshot: vi.fn(), + onEnd, + onError, + onTransportClose + } + }) + + callbacks?.onResponse({ + ok: true, + result: { + type: 'error', + streamId: stream.streamId, + message: 'terminal_stream_limit_exceeded' + } + }) + callbacks?.onResponse({ + ok: true, + result: { type: 'end', streamId: stream.streamId } + }) + + expect(onTransportClose).toHaveBeenCalledOnce() + expect(onTransportClose).toHaveBeenCalledWith({ + recoverable: true, + retryWithBackoff: true + }) + expect(onEnd).not.toHaveBeenCalled() + expect(onError).not.toHaveBeenCalled() + expect(unsubscribe).toHaveBeenCalledOnce() + }) + + function emitOutput(streamId: number, text: string): void { + callbacks?.onBinary( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Output, + streamId, + seq: text.length, + payload: encodeTerminalStreamText(text) + }) + ) + } + + function emitRequestedSnapshot(streamId: number, requestId: number): void { + callbacks?.onBinary( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.SnapshotStart, + streamId, + seq: 1, + payload: encodeTerminalStreamJson({ + kind: 'scrollback', + requestId, + cols: 80, + rows: 24 + }) + }) + ) + callbacks?.onBinary( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.SnapshotEnd, + streamId, + seq: 2, + payload: new Uint8Array() + }) + ) + } + + function sentFrames(opcode: TerminalStreamOpcode) { + return sendBinary.mock.calls.flatMap(([bytes]) => { + const frame = decodeTerminalStreamFrame(bytes) + return frame?.opcode === opcode ? [frame] : [] + }) + } + + function sentUnsubscribeStreamIds(): number[] { + return sentFrames(TerminalStreamOpcode.Unsubscribe).map((frame) => frame.streamId) + } +}) diff --git a/src/renderer/src/runtime/remote-terminal-stream-watchdog.ts b/src/renderer/src/runtime/remote-terminal-stream-watchdog.ts new file mode 100644 index 000000000..4cb6b081c --- /dev/null +++ b/src/renderer/src/runtime/remote-terminal-stream-watchdog.ts @@ -0,0 +1,108 @@ +export const REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS = 10_000 +export const REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS = 30_000 + +export type RemoteTerminalStreamStall = { + inactiveForMs: number + outstandingDeliveryBytes: number + reason: 'command-response-timeout' | 'delivery-credit-timeout' +} + +export type RemoteTerminalStreamWatchdog = { + beginOutputDelivery: (bytes: number) => () => void + completeCommandResponseProbe: () => void + recordCommandInput: (text: string) => void + recordInbound: () => void + dispose: () => void +} + +export function createRemoteTerminalStreamWatchdog( + onStall: (stall: RemoteTerminalStreamStall) => void +): RemoteTerminalStreamWatchdog { + let responseTimer: ReturnType | null = null + let deliveryTimer: ReturnType | null = null + let outstandingDeliveryBytes = 0 + let lastInboundAtMs = Date.now() + let commandResponseProbePending = false + let disposed = false + + const clearResponseTimer = (): void => { + if (responseTimer) { + clearTimeout(responseTimer) + responseTimer = null + } + } + const clearDeliveryTimer = (): void => { + if (deliveryTimer) { + clearTimeout(deliveryTimer) + deliveryTimer = null + } + } + const trip = (reason: RemoteTerminalStreamStall['reason']): void => { + if (disposed) { + return + } + clearResponseTimer() + if (reason === 'command-response-timeout') { + commandResponseProbePending = true + } else { + disposed = true + clearDeliveryTimer() + } + onStall({ + inactiveForMs: Math.max(0, Date.now() - lastInboundAtMs), + outstandingDeliveryBytes, + reason + }) + } + const armDeliveryTimer = (): void => { + clearDeliveryTimer() + if (outstandingDeliveryBytes <= 0 || disposed) { + return + } + deliveryTimer = setTimeout( + () => trip('delivery-credit-timeout'), + REMOTE_TERMINAL_DELIVERY_STALL_TIMEOUT_MS + ) + } + + return { + beginOutputDelivery(bytes) { + outstandingDeliveryBytes += bytes + if (!deliveryTimer) { + armDeliveryTimer() + } + let settled = false + return () => { + if (settled || disposed) { + return + } + settled = true + outstandingDeliveryBytes = Math.max(0, outstandingDeliveryBytes - bytes) + armDeliveryTimer() + } + }, + completeCommandResponseProbe() { + commandResponseProbePending = false + }, + recordCommandInput(text) { + if (disposed || commandResponseProbePending || responseTimer || !/[\r\n]/u.test(text)) { + return + } + responseTimer = setTimeout( + () => trip('command-response-timeout'), + REMOTE_TERMINAL_COMMAND_RESPONSE_TIMEOUT_MS + ) + }, + recordInbound() { + lastInboundAtMs = Date.now() + clearResponseTimer() + }, + dispose() { + disposed = true + commandResponseProbePending = false + clearResponseTimer() + clearDeliveryTimer() + outstandingDeliveryBytes = 0 + } + } +} diff --git a/src/renderer/src/runtime/runtime-client-events.test.ts b/src/renderer/src/runtime/runtime-client-events.test.ts index b0da9b8b1..4f61c56b6 100644 --- a/src/renderer/src/runtime/runtime-client-events.test.ts +++ b/src/renderer/src/runtime/runtime-client-events.test.ts @@ -47,13 +47,24 @@ describe('subscribeRuntimeClientEvents', () => { ok: true, result: { type: 'worktreesChanged', repoId: 'repo-1' } }) + capturedOnResponse({ + ok: true, + result: { + type: 'terminalSideEffects', + batch: { ptyId: 'pty-1', seq: 7, facts: [{ kind: 'bell' }] } + } + }) capturedOnResponse({ ok: false, error: { code: 'method_not_found', message: 'missing' } }) - expect(onEvent).toHaveBeenCalledTimes(1) + expect(onEvent).toHaveBeenCalledTimes(2) expect(onEvent).toHaveBeenCalledWith({ type: 'worktreesChanged', repoId: 'repo-1' }) + expect(onEvent).toHaveBeenCalledWith({ + type: 'terminalSideEffects', + batch: { ptyId: 'pty-1', seq: 7, facts: [{ kind: 'bell' }] } + }) expect(onError).toHaveBeenCalledWith({ code: 'method_not_found', message: 'missing' }) subscription.unsubscribe() diff --git a/src/renderer/src/runtime/runtime-client-events.ts b/src/renderer/src/runtime/runtime-client-events.ts index 0f1b72371..2a41dbb1c 100644 --- a/src/renderer/src/runtime/runtime-client-events.ts +++ b/src/renderer/src/runtime/runtime-client-events.ts @@ -86,6 +86,7 @@ function isRuntimeClientEvent( return ( message.type === 'reposChanged' || message.type === 'worktreesChanged' || + message.type === 'terminalSideEffects' || message.type === 'sshStateChanged' || message.type === 'linearLinkedIssueUpdated' || message.type === 'activateWorktree' || diff --git a/src/renderer/src/runtime/runtime-terminal-stream.ts b/src/renderer/src/runtime/runtime-terminal-stream.ts index 314aa01b9..af5ffdd6f 100644 --- a/src/renderer/src/runtime/runtime-terminal-stream.ts +++ b/src/renderer/src/runtime/runtime-terminal-stream.ts @@ -11,6 +11,14 @@ export type RemoteRuntimePtyIdParts = { handle: string } +export type RuntimeTerminalDataSubscriptionOptions = { + startAtLiveTail?: boolean + onSnapshot?: (data: string, meta?: { pendingEscapeTailAnsi?: string }) => void + onEnd?: () => void + onError?: (message: string) => void + onTransportClose?: (event: { recoverable: boolean; retryWithBackoff?: boolean }) => void +} + export function toRemoteRuntimePtyId(handle: string, environmentId?: string | null): string { const owner = environmentId?.trim() if (!owner) { @@ -58,7 +66,7 @@ export async function subscribeToRuntimeTerminalData( ptyId: string, clientId: string, watcher: (data: string) => void, - options?: { startAtLiveTail?: boolean } + options?: RuntimeTerminalDataSubscriptionOptions ): Promise<() => void> { const terminal = getRemoteRuntimeTerminalHandle(ptyId) const ownerEnvironmentId = getRemoteRuntimePtyEnvironmentId(ptyId) @@ -88,9 +96,12 @@ export async function subscribeToRuntimeTerminalData( client: { id: clientId, type: 'desktop' }, callbacks: { onData: (data) => watcher(data), - onSnapshot: (data) => { + onSnapshot: (data, meta) => { + options?.onSnapshot?.(data, meta) if (!options?.startAtLiveTail) { - watcher(data) + if (!options?.onSnapshot) { + watcher(data) + } } }, onSubscribed: () => { @@ -98,10 +109,18 @@ export async function subscribeToRuntimeTerminalData( resolveLiveTail = null rejectLiveTail = null }, - onEnd: () => rejectPendingLiveTail('Remote terminal ended before live output was ready.'), - onError: (message) => rejectPendingLiveTail(message), - onTransportClose: () => + onEnd: () => { + rejectPendingLiveTail('Remote terminal ended before live output was ready.') + options?.onEnd?.() + }, + onError: (message) => { + rejectPendingLiveTail(message) + options?.onError?.(message) + }, + onTransportClose: (event) => { rejectPendingLiveTail('Remote terminal closed before live output was ready.') + options?.onTransportClose?.(event) + } } }) diff --git a/src/renderer/src/web/web-preload-api.test.ts b/src/renderer/src/web/web-preload-api.test.ts index f97872592..a8636b50e 100644 --- a/src/renderer/src/web/web-preload-api.test.ts +++ b/src/renderer/src/web/web-preload-api.test.ts @@ -235,6 +235,7 @@ describe('web runtime environment identity', () => { afterEach(() => { vi.unstubAllGlobals() + vi.doUnmock('./web-runtime-client') }) it('does not resolve an old server selector through a differently keyed server', async () => { @@ -359,6 +360,177 @@ describe('web runtime environment identity', () => { globals.window.api.runtimeEnvironments.resolve({ selector: 'web-server-old' }) ).rejects.toThrow('Unknown Orca runtime environment: web-server-old') }) + + it('keeps pairing while manual disconnect fences passive reconnects', async () => { + const calls: string[] = [] + const close = vi.fn() + let clientCount = 0 + vi.doMock('./web-runtime-client', () => ({ + WebRuntimeClient: class { + constructor() { + clientCount += 1 + } + + call(method: string): Promise> { + calls.push(method) + return Promise.resolve({ + id: method, + ok: true, + result: { runtimeId: 'runtime-1' }, + _meta: { runtimeId: 'runtime-1' } + }) + } + + close(): void { + close() + } + } + })) + const globals = installBrowserGlobals('Linux') + writeStoredRuntimeEnvironment(globals.storage, 'web-server-a') + const { installWebPreloadApi } = await import('./web-preload-api') + installWebPreloadApi() + + await expect( + globals.window.api.runtimeEnvironments.getStatus({ selector: 'web-server-a' }) + ).resolves.toMatchObject({ ok: true }) + await globals.window.api.runtimeEnvironments.disconnect({ selector: 'web-server-a' }) + + await expect(globals.window.api.runtimeEnvironments.list()).resolves.toMatchObject([ + { id: 'web-server-a' } + ]) + await expect( + globals.window.api.runtimeEnvironments.getStatus({ selector: 'web-server-a' }) + ).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_manually_disconnected' } + }) + await expect( + globals.window.api.runtimeEnvironments.call({ + selector: 'web-server-a', + method: 'repos.list' + }) + ).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_manually_disconnected' } + }) + await expect( + globals.window.api.runtimeEnvironments.subscribe( + { selector: 'web-server-a', method: 'terminal.subscribe' }, + { onResponse: vi.fn() } + ) + ).rejects.toThrow('runtime_manually_disconnected') + expect(clientCount).toBe(1) + expect(calls).toEqual(['status.get']) + expect(close).toHaveBeenCalledOnce() + + await expect( + globals.window.api.runtimeEnvironments.connect({ selector: 'web-server-a' }) + ).resolves.toMatchObject({ ok: true }) + expect(clientCount).toBe(2) + expect(calls).toEqual(['status.get', 'status.get']) + }) + + it('fences a web runtime response that completes after manual disconnect', async () => { + let resolveCall!: (response: RuntimeRpcResponse) => void + const pendingCall = new Promise>((resolve) => { + resolveCall = resolve + }) + const call = vi.fn(() => pendingCall) + vi.doMock('./web-runtime-client', () => ({ + WebRuntimeClient: class { + call = call + close(): void {} + } + })) + const globals = installBrowserGlobals('Linux') + writeStoredRuntimeEnvironment(globals.storage, 'web-server-a') + const { installWebPreloadApi } = await import('./web-preload-api') + installWebPreloadApi() + + const status = globals.window.api.runtimeEnvironments.getStatus({ + selector: 'web-server-a' + }) + await vi.waitFor(() => expect(call).toHaveBeenCalledOnce()) + await globals.window.api.runtimeEnvironments.disconnect({ selector: 'web-server-a' }) + resolveCall({ + id: 'status.get', + ok: true, + result: { runtimeId: 'runtime-1' }, + _meta: { runtimeId: 'runtime-1' } + }) + + await expect(status).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_manually_disconnected' } + }) + }) + + it.each(['active runtime', 'selected environment'] as const)( + 'returns a disconnect envelope when a queued %s call disconnects', + async (route) => { + const pending: ((response: RuntimeRpcResponse) => void)[] = [] + const call = vi.fn( + (method: string) => + new Promise>((resolve) => { + pending.push((response) => resolve({ ...response, id: method })) + }) + ) + vi.doMock('./web-runtime-client', () => ({ + WebRuntimeClient: class { + call = call + close(): void {} + } + })) + const globals = installBrowserGlobals('Linux') + writeStoredRuntimeEnvironment(globals.storage, 'web-server-a') + const { installWebPreloadApi } = await import('./web-preload-api') + installWebPreloadApi() + const invoke = (): Promise> => + route === 'active runtime' + ? globals.window.api.runtime.call({ method: 'repos.list' }) + : globals.window.api.runtimeEnvironments.call({ + selector: 'web-server-a', + method: 'repos.list' + }) + + const activeCalls = Array.from({ length: 8 }, invoke) + await vi.waitFor(() => expect(call).toHaveBeenCalledTimes(8)) + const queuedCall = invoke() + expect(call).toHaveBeenCalledTimes(8) + + await globals.window.api.runtimeEnvironments.disconnect({ selector: 'web-server-a' }) + pending[0]?.({ + id: 'repos.list', + ok: true, + result: {}, + _meta: { runtimeId: 'runtime-1' } + }) + + await expect(queuedCall).resolves.toMatchObject({ + ok: false, + error: { code: 'runtime_manually_disconnected' } + }) + expect(call).toHaveBeenCalledTimes(8) + + for (const resolve of pending.slice(1)) { + resolve({ + id: 'repos.list', + ok: true, + result: {}, + _meta: { runtimeId: 'runtime-1' } + }) + } + await expect(Promise.all(activeCalls)).resolves.toEqual( + Array.from({ length: 8 }, () => + expect.objectContaining({ + ok: false, + error: expect.objectContaining({ code: 'runtime_manually_disconnected' }) + }) + ) + ) + } + ) }) describe('web browser-local port capability', () => { diff --git a/src/renderer/src/web/web-preload-api.ts b/src/renderer/src/web/web-preload-api.ts index 8ce018f4f..d8e3ef432 100644 --- a/src/renderer/src/web/web-preload-api.ts +++ b/src/renderer/src/web/web-preload-api.ts @@ -148,6 +148,14 @@ const SESSION_STORAGE_KEY = 'orca.web.workspaceSession.v1' const ONBOARDING_STORAGE_KEY = 'orca.web.onboarding.v1' const GITHUB_CACHE_STORAGE_KEY = 'orca.web.githubCache.v1' const KEYBINDINGS_STORAGE_KEY = 'orca.web.keybindings.v1' +// Why: paired web clients lack Electron env/preload state; the E2E build gate keeps URL overrides out of releases. +const webE2EExposeStore = String(import.meta.env.VITE_EXPOSE_STORE) === 'true' +const webE2EQuery = webE2EExposeStore ? new URLSearchParams(window.location.search) : null +const webE2EConfig = createE2EConfig({ + exposeStore: webE2EExposeStore, + terminalParkingDelayMs: Number(webE2EQuery?.get('orcaE2ETerminalParkingDelayMs')) || null, + terminalRetentionLimit: Number(webE2EQuery?.get('orcaE2ETerminalRetentionLimit')) || null +}) // Why: paired clients need parity for large dev sessions; the runtime default stays capped for lower-level RPC callers. const WEB_RUNTIME_WORKTREE_LIST_LIMIT = 10_000 const MAX_CLIPBOARD_IMAGE_BASE64_CHARS = CLIPBOARD_IMAGE_MAX_BASE64_CHARS @@ -160,6 +168,7 @@ const CLIPBOARD_IMAGE_SAVE_TIMEOUT_MS = 30_000 let activeEnvironment: StoredWebRuntimeEnvironment | null = readStoredWebRuntimeEnvironment() let activeClient: WebRuntimeClient | null = null let activeClientEnvironmentId: string | null = null +const manuallyDisconnectedEnvironmentIds = new Set() let cachedWorktrees: { loadedAt: number; worktrees: Worktree[] } | null = null let cachedDetectedWorktrees: { loadedAt: number; worktrees: Worktree[] } | null = null const runtimeCallQueuePool = new RuntimeRpcCallQueuePool() @@ -648,7 +657,7 @@ function createWebPreloadApi(): Partial { orgMemberRemove: async () => ({ status: 'unconfigured' }) }, e2e: { - getConfig: () => createE2EConfig({}) + getConfig: () => webE2EConfig }, settings: { get: async () => getRuntimeBackedStoredSettings(), @@ -1348,6 +1357,7 @@ function createRuntimeEnvironmentsApi(): NonNullable['runtim const previousEnvironment = activeEnvironment closeActiveRuntimeClients() activeEnvironment = createStoredWebRuntimeEnvironment({ name, offer, previousEnvironment }) + manuallyDisconnectedEnvironmentIds.clear() saveStoredWebRuntimeEnvironment(activeEnvironment) return { environment: redactStoredWebRuntimeEnvironment(activeEnvironment) } }, @@ -1356,17 +1366,29 @@ function createRuntimeEnvironmentsApi(): NonNullable['runtim remove: async ({ selector }) => { const environment = resolveEnvironment(selector) if (activeEnvironment?.id === environment.id) { - disconnectActiveRuntimeEnvironment() + removeActiveRuntimeEnvironment() } + manuallyDisconnectedEnvironmentIds.delete(environment.id) return { removed: redactStoredWebRuntimeEnvironment(environment) } }, disconnect: async ({ selector }) => { const environment = resolveEnvironment(selector) if (activeEnvironment?.id === environment.id) { + manuallyDisconnectedEnvironmentIds.add(environment.id) disconnectActiveRuntimeEnvironment() } return { disconnected: redactStoredWebRuntimeEnvironment(environment) } }, + connect: ({ selector, timeoutMs }) => { + const environment = resolveEnvironment(selector) + manuallyDisconnectedEnvironmentIds.delete(environment.id) + return callEnvironmentEnvelope( + environment.id, + 'status.get', + undefined, + timeoutMs + ) + }, getStatus: ({ selector, timeoutMs }) => callEnvironmentEnvelope(selector, 'status.get', undefined, timeoutMs), call: ({ selector, method, params, timeoutMs }) => @@ -1374,7 +1396,12 @@ function createRuntimeEnvironmentsApi(): NonNullable['runtim subscribe: async ({ selector, method, params, timeoutMs }, callbacks) => { const environment = resolveEnvironment(selector) const client = getClientForEnvironment(environment) - return client.subscribe(method, params, callbacks, { timeoutMs }) + const subscription = await client.subscribe(method, params, callbacks, { timeoutMs }) + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + subscription.unsubscribe() + throw new Error('runtime_manually_disconnected') + } + return subscription } } } @@ -3139,9 +3166,18 @@ async function callRuntimeEnvelope( timeoutMs?: number ): Promise> { const environment = requireActiveEnvironment() - const response = await runtimeCallQueuePool.enqueue(environment.id, method, () => - getClientForEnvironment(environment).call(method, params, { timeoutMs }) - ) + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + return manuallyDisconnectedResponse(environment) + } + const response = await runtimeCallQueuePool.enqueue(environment.id, method, () => { + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + return Promise.resolve(manuallyDisconnectedResponse(environment)) + } + return getClientForEnvironment(environment).call(method, params, { timeoutMs }) + }) + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + return manuallyDisconnectedResponse(environment) + } updateEnvironmentFromResponse(environment, response) return response as RuntimeRpcResponse } @@ -3153,9 +3189,18 @@ async function callEnvironmentEnvelope( timeoutMs?: number ): Promise> { const environment = resolveEnvironment(selector) - const response = await runtimeCallQueuePool.enqueue(environment.id, method, () => - getClientForEnvironment(environment).call(method, params, { timeoutMs }) - ) + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + return manuallyDisconnectedResponse(environment) + } + const response = await runtimeCallQueuePool.enqueue(environment.id, method, () => { + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + return Promise.resolve(manuallyDisconnectedResponse(environment)) + } + return getClientForEnvironment(environment).call(method, params, { timeoutMs }) + }) + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + return manuallyDisconnectedResponse(environment) + } updateEnvironmentFromResponse(environment, response) return response as RuntimeRpcResponse } @@ -3332,6 +3377,9 @@ async function getRemoteRuntimeStatus(): Promise { } function getClientForEnvironment(environment: StoredWebRuntimeEnvironment): WebRuntimeClient { + if (manuallyDisconnectedEnvironmentIds.has(environment.id)) { + throw new Error('runtime_manually_disconnected') + } if (!activeClient || activeClientEnvironmentId !== environment.id) { activeClient?.close() activeClient = new WebRuntimeClient(getPreferredWebPairingOffer(environment)) @@ -3349,10 +3397,31 @@ function closeActiveRuntimeClients(): void { function disconnectActiveRuntimeEnvironment(): void { closeActiveRuntimeClients() +} + +function removeActiveRuntimeEnvironment(): void { + disconnectActiveRuntimeEnvironment() clearStoredWebRuntimeEnvironment() activeEnvironment = null } +function manuallyDisconnectedResponse( + environment: StoredWebRuntimeEnvironment +): RuntimeRpcResponse { + return { + id: 'runtime.manualDisconnect', + ok: false, + error: { + code: 'runtime_manually_disconnected', + message: translate( + 'auto.web.webPreloadApi.runtimeEnvironmentManuallyDisconnected', + 'Runtime environment is manually disconnected.' + ) + }, + _meta: { runtimeId: environment.runtimeId } + } +} + function resolveEnvironment(selector: string): StoredWebRuntimeEnvironment { const environment = requireActiveEnvironment() if (selector === environment.id || selector === environment.name || selector === 'active') { diff --git a/src/shared/protocol-version.ts b/src/shared/protocol-version.ts index c5915977f..f68fc7b86 100644 --- a/src/shared/protocol-version.ts +++ b/src/shared/protocol-version.ts @@ -54,6 +54,9 @@ export const BROWSER_CERTIFICATE_TRUST_RUNTIME_CAPABILITY = 'browser.certificate // floor-taking input. Mobile must not forward replies unless advertised. export const TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY = 'terminal.query-reply-input.v1' as const +// Why: paired clients may unmount xterm only when the host can return a +// bounded, sequenced scrollback snapshot for lossless reveal. +export const TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY = 'terminal.paired-parking.v1' as const // Why: older hosts lack the targeted settings RPCs and strip agentPrompt from // terminal creation, so mobile must hide Quick Commands unless both are present. export const TERMINAL_QUICK_COMMANDS_RUNTIME_CAPABILITY = 'terminal.quick-commands.v1' as const @@ -96,6 +99,7 @@ export const RUNTIME_CAPABILITIES = [ LINEAR_ISSUE_ATTRIBUTE_FILTER_RUNTIME_CAPABILITY, AI_VAULT_RUNTIME_CAPABILITY, TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY, + TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY, TERMINAL_QUICK_COMMANDS_RUNTIME_CAPABILITY, WORKTREE_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY, TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY, diff --git a/src/shared/runtime-client-events.ts b/src/shared/runtime-client-events.ts index 0c2603781..3e0b1a255 100644 --- a/src/shared/runtime-client-events.ts +++ b/src/shared/runtime-client-events.ts @@ -5,10 +5,12 @@ import type { WorktreeStartupLaunch } from './types' import type { SshConnectionState } from './ssh-types' +import type { TerminalSideEffectBatch } from './terminal-side-effect-facts' export type RuntimeClientEvent = | { type: 'reposChanged' } | { type: 'worktreesChanged'; repoId: string } + | { type: 'terminalSideEffects'; batch: TerminalSideEffectBatch } // Why: SSH connections live on the runtime host; paired clients have no IPC // channel for ssh:state-changed, so without this event their reconnect // overlays never learn the host connected (STA-1468). diff --git a/src/shared/terminal-multiplex-flow-control.ts b/src/shared/terminal-multiplex-flow-control.ts index a14f45028..8500bee3b 100644 --- a/src/shared/terminal-multiplex-flow-control.ts +++ b/src/shared/terminal-multiplex-flow-control.ts @@ -7,4 +7,7 @@ export const TERMINAL_MULTIPLEX_ACK_TOTAL_MAX_WINDOW_BYTES = 8 * 1024 * 1024 export const TERMINAL_MULTIPLEX_PENDING_MAX_BYTES = 256 * 1024 export const TERMINAL_MULTIPLEX_ACK_BATCH_BYTES = 192 * 1024 export const TERMINAL_MULTIPLEX_ACK_FLUSH_MS = 4 -export const TERMINAL_MULTIPLEX_MAX_STREAMS_PER_CONNECTION = 32 +// 128 covers large paired clients while fixed ACK windows and per-stream queues bound pressure. +export const TERMINAL_MULTIPLEX_MAX_ACTIVE_STREAMS_PER_CONNECTION = 128 +export const TERMINAL_MULTIPLEX_MAX_PENDING_PTY_WAITS_PER_CONNECTION = 32 +export const TERMINAL_MULTIPLEX_STREAM_LIMIT_ERROR = 'terminal_stream_limit_exceeded' diff --git a/tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts b/tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts new file mode 100644 index 000000000..fcb462d4c --- /dev/null +++ b/tests/e2e/headless-paired-remote-terminal-retention-memory.spec.ts @@ -0,0 +1,40 @@ +import { expect, test } from './helpers/orca-app' +import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host' +import { launchPairedWebClient, type PairedWebClient } from './helpers/paired-electron-client' +import { runPairedTerminalParkingOracle } from './helpers/paired-terminal-parking-oracle' + +test('ordinary-parks paired terminals against an isolated headless Orca host @headful', async ({ + testRepoPath +}) => { + test.setTimeout(240_000) + const host = await launchHeadlessPairedRuntimeHost() + let client: PairedWebClient | null = null + try { + await host.client.call('repo.add', { path: testRepoPath, kind: 'git' }) + client = await launchPairedWebClient(host.app, host.offer, { + terminalParkingDelayMs: 100 + }) + await expect + .poll( + () => + client?.page.evaluate(() => { + const state = window.__store?.getState() + const worktree = state?.allWorktrees()[0] + return worktree ? { id: worktree.id, repoId: worktree.repoId } : null + }) ?? null, + { timeout: 30_000 } + ) + .not.toBeNull() + const seed = await client.page.evaluate(() => { + const worktree = window.__store?.getState().allWorktrees()[0] + if (!worktree) { + throw new Error('Headless paired client did not receive the host worktree') + } + return { fallbackWorktreeId: worktree.id, repoId: worktree.repoId } + }) + await runPairedTerminalParkingOracle(client.page, seed) + } finally { + await client?.dispose() + await host.dispose() + } +}) diff --git a/tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts b/tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts new file mode 100644 index 000000000..4aa41b7ee --- /dev/null +++ b/tests/e2e/headless-paired-remote-terminal-stall-recovery.spec.ts @@ -0,0 +1,226 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { Page } from '@stablyai/playwright-test' +import type { RuntimeTerminalRead } from '../../src/shared/runtime-types' +import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id' +import { expect, test } from './helpers/orca-app' +import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host' +import { launchPairedWebClient } from './helpers/paired-electron-client' +import { getTerminalContent, waitForActivePanePtyId } from './helpers/terminal' + +const MIN_EXHAUSTED_ACK_BYTES = 400 * 1024 +const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-headless-stalled-stream-')) +const fixturePath = path.join(scratch, 'headless-stalled-stream.mjs') + +writeFileSync( + fixturePath, + [ + "process.stdout.write('HEADLESS_STALL_READY\\r\\n')", + "process.stdin.setEncoding('utf8')", + "let pending = ''", + "process.stdin.on('data', (data) => {", + ' pending += data', + ' const commands = pending.split(/\\r\\n|\\r|\\n/)', + ' pending = commands.pop() ?? ""', + ' for (const input of commands) {', + " if (input === 'GO') {", + " for (let row = 0; row < 16_000; row += 1) process.stdout.write(`headless-${row}-${'x'.repeat(80)}\\r\\n`)", + " process.stdout.write('HEADLESS_FLOOD_COMPLETE\\r\\n')", + ' continue', + ' }', + ' process.stdout.write(`LIVE:${input}\\r\\n`)', + ' }', + '})', + 'process.stdin.resume()' + ].join('\n') +) + +test.afterAll(() => { + rmSync(scratch, { recursive: true, force: true }) +}) + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(): string { + const command = [process.execPath, fixturePath] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +async function callRuntime(page: Page, method: string, params: unknown): Promise { + return page.evaluate( + async ({ method, params }) => { + const response = await window.api.runtime.call({ method, params }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { method, params } + ) as Promise +} + +test('recovers an ACK-starved stream from an isolated headless Orca host @headful', async ({ + testRepoPath +}) => { + test.setTimeout(180_000) + const host = await launchHeadlessPairedRuntimeHost() + const client = await launchPairedWebClient(host.app, host.offer, { + waitForWorkspace: false + }).catch(async (error) => { + await host.dispose() + throw error + }) + let terminal: string | null = null + try { + await host.client.call('repo.add', { path: testRepoPath, kind: 'git' }) + try { + await client.page.locator('[data-worktree-sidebar]').waitFor({ + state: 'visible', + timeout: 30_000 + }) + } catch { + const boot = await client.page.evaluate(() => ({ + bodyChildren: document.body?.children.length ?? 0, + bodyTextLength: document.body?.innerText.length ?? 0, + hasApi: Boolean(window.api), + hasRoot: Boolean(document.querySelector('#root')), + hasStore: Boolean(window.__store), + readyState: document.readyState, + title: document.title + })) + throw new Error(`Headless paired web client did not boot: ${JSON.stringify(boot)}`) + } + await expect + .poll( + () => + client.page.evaluate(() => { + const worktrees = window.__store?.getState().allWorktrees() ?? [] + return worktrees[0]?.id ?? null + }), + { timeout: 30_000 } + ) + .not.toBeNull() + const worktreeId = await client.page.evaluate( + () => window.__store?.getState().allWorktrees()[0]?.id ?? null + ) + if (!worktreeId) { + throw new Error('Headless paired client did not receive the host worktree') + } + const created = await callRuntime<{ + tab: { parentTabId: string; terminal: string | null } + }>(client.page, 'session.tabs.createTerminal', { + worktree: `id:${worktreeId}`, + command: fixtureCommand(), + activate: false, + select: false, + navigation: 'caller' + }) + terminal = created.tab.terminal + if (!terminal) { + throw new Error('Headless paired host did not publish the fixture terminal') + } + const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId) + await client.page.evaluate((id) => window.__store?.getState().setActiveWorktree(id), worktreeId) + const tab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`) + await expect(tab).toBeVisible({ timeout: 30_000 }) + await tab.click() + const originalPtyId = await waitForActivePanePtyId(client.page, 30_000) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain('HEADLESS_STALL_READY') + + await client.page.evaluate((target) => { + const gate = ( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { hold: (terminals: string[]) => void } + } + ).__remoteTerminalMultiplexAckGate + if (!gate) { + throw new Error('Remote terminal multiplex ACK gate is unavailable') + } + gate.hold([target]) + }, terminal) + const textarea = client.page.locator('.xterm-helper-textarea:visible').first() + await textarea.focus() + await client.page.keyboard.type('GO') + await client.page.keyboard.press('Enter') + await expect + .poll( + () => + client.page.evaluate(() => { + const gate = ( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { + snapshot: () => { heldAckChars: number } + } + } + ).__remoteTerminalMultiplexAckGate + return gate?.snapshot().heldAckChars ?? 0 + }), + { timeout: 30_000 } + ) + .toBeGreaterThan(MIN_EXHAUSTED_ACK_BYTES) + await expect + .poll( + async () => { + const result = await callRuntime<{ terminal: RuntimeTerminalRead }>( + client.page, + 'terminal.read', + { terminal } + ) + return result.terminal.tail.join('\n') + }, + { timeout: 30_000 } + ) + .toContain('HEADLESS_FLOOD_COMPLETE') + + const marker = `HEADLESS_RECOVERED_${Date.now()}` + await callRuntime(client.page, 'terminal.send', { + terminal, + text: marker, + enter: true, + client: { id: 'headless-stalled-stream-e2e', type: 'desktop' } + }) + expect(await getTerminalContent(client.page)).not.toContain(marker) + expect( + await client.page.evaluate( + ({ target }) => { + const gate = ( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { + sendInput: (terminal: string, text: string) => number + } + } + ).__remoteTerminalMultiplexAckGate + return gate?.sendInput(target, '\r') ?? 0 + }, + { target: terminal } + ) + ).toBe(1) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain(`LIVE:${marker}`) + expect(await waitForActivePanePtyId(client.page, 30_000)).toBe(originalPtyId) + await expect(tab).toHaveAttribute('data-active', 'true') + } finally { + await client.page + .evaluate(() => { + ;( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { release: () => void } + } + ).__remoteTerminalMultiplexAckGate?.release() + }) + .catch(() => undefined) + if (terminal) { + await callRuntime(client.page, 'terminal.closeTab', { terminal }).catch(() => undefined) + } + await client.dispose() + await host.dispose() + } +}) diff --git a/tests/e2e/helpers/headless-paired-runtime-host.ts b/tests/e2e/helpers/headless-paired-runtime-host.ts new file mode 100644 index 000000000..926c4bf5f --- /dev/null +++ b/tests/e2e/helpers/headless-paired-runtime-host.ts @@ -0,0 +1,149 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { _electron as electron, type ElectronApplication } from '@stablyai/playwright-test' +import { RuntimeClient } from '../../../src/cli/runtime/client' +import { getE2ECompletedOnboardingProfile } from './e2e-completed-onboarding-profile' +import { getOrcaElectronLaunchArgs } from './electron-launch-args' +import { cleanupE2EDaemons, closeElectronAppForE2E } from './electron-process-shutdown' +import { + assertElectronResolvedIsolatedHome, + createElectronHomeIsolation +} from './electron-home-isolation' +import type { RuntimeDesktopPairingOffer } from './paired-electron-client' + +type ServeReady = { + type?: unknown + pairing?: { + available?: unknown + url?: unknown + webClientUrl?: unknown + } +} + +export type HeadlessPairedRuntimeHost = { + app: ElectronApplication + client: RuntimeClient + dispose: () => Promise + offer: RuntimeDesktopPairingOffer +} + +async function readPairingOffer(app: ElectronApplication): Promise { + const child = app.process() + const stdout = child.stdout + if (!stdout) { + throw new Error('Headless runtime stdout is unavailable') + } + return new Promise((resolve, reject) => { + let buffered = '' + const timeout = setTimeout(() => { + cleanup() + reject(new Error('Headless runtime did not publish pairing readiness')) + }, 60_000) + const cleanup = (): void => { + clearTimeout(timeout) + stdout.off('data', onData) + child.off('close', onClose) + } + const onClose = (code: number | null, signal: NodeJS.Signals | null): void => { + cleanup() + reject( + new Error( + `Headless runtime exited before pairing readiness (code=${code ?? 'none'}, signal=${signal ?? 'none'})` + ) + ) + } + const onData = (chunk: Buffer): void => { + buffered += chunk.toString() + const lines = buffered.split(/\r?\n/) + buffered = lines.pop() ?? '' + for (const line of lines) { + let readiness: ServeReady + try { + readiness = JSON.parse(line) as ServeReady + } catch { + continue + } + const pairing = readiness.pairing + if ( + readiness.type !== 'orca_server_ready' || + pairing?.available !== true || + typeof pairing.url !== 'string' || + typeof pairing.webClientUrl !== 'string' + ) { + continue + } + cleanup() + resolve({ pairingUrl: pairing.url, webClientUrl: pairing.webClientUrl }) + return + } + } + stdout.on('data', onData) + child.on('close', onClose) + if (child.exitCode !== null || child.signalCode !== null) { + onClose(child.exitCode, child.signalCode) + } + }) +} + +export async function launchHeadlessPairedRuntimeHost(): Promise { + const userDataDir = mkdtempSync(path.join(os.tmpdir(), 'orca-e2e-headless-paired-')) + let app: ElectronApplication | undefined + try { + writeFileSync( + path.join(userDataDir, 'orca-data.json'), + `${JSON.stringify(getE2ECompletedOnboardingProfile(), null, 2)}\n` + ) + const { ELECTRON_RUN_AS_NODE: _unused, ...cleanEnv } = process.env + void _unused + const isolation = createElectronHomeIsolation({ + inheritedEnv: cleanEnv, + launchEnv: { + NODE_ENV: 'development', + ORCA_E2E_ENFORCE_SINGLE_INSTANCE_LOCK: '1', + ORCA_E2E_HEADLESS: '1' + }, + extraEnv: {}, + userDataDir, + codexRealHomeEnabled: false + }) + const mainPath = path.join(process.cwd(), 'out', 'main', 'index.js') + app = await electron.launch({ + args: [ + ...getOrcaElectronLaunchArgs(mainPath, false), + '--serve', + '--serve-json', + '--serve-port', + '0', + '--serve-pairing-address', + '127.0.0.1' + ], + env: isolation.env + }) + assertElectronResolvedIsolatedHome( + await app.evaluate(({ app: electronApp }) => electronApp.getPath('home')), + isolation + ) + const offer = await readPairingOffer(app) + return { + app, + client: new RuntimeClient(userDataDir, 5_000), + offer, + dispose: async () => { + await closeElectronAppForE2E(app) + await cleanupE2EDaemons(userDataDir) + rmSync(userDataDir, { recursive: true, force: true }) + } + } + } catch (error) { + try { + if (app) { + await closeElectronAppForE2E(app) + } + await cleanupE2EDaemons(userDataDir) + } finally { + rmSync(userDataDir, { recursive: true, force: true }) + } + throw error + } +} diff --git a/tests/e2e/helpers/paired-electron-client.ts b/tests/e2e/helpers/paired-electron-client.ts index 67427f3b3..c33764a24 100644 --- a/tests/e2e/helpers/paired-electron-client.ts +++ b/tests/e2e/helpers/paired-electron-client.ts @@ -21,6 +21,7 @@ import { replaceRuntimePairingInPlace, type SameIdPairingReplacement } from './nested-runtime-same-id-pairing' +import { createPairedWebClientUrl, type PairedWebClientOptions } from './paired-web-client-url' export type { SameIdPairingReplacement } from './nested-runtime-same-id-pairing' @@ -92,35 +93,49 @@ export async function createRuntimeDesktopPairingOffer( export async function launchPairedWebClient( hubApp: ElectronApplication, - offer: RuntimeDesktopPairingOffer + offer: RuntimeDesktopPairingOffer, + options: PairedWebClientOptions = {} ): Promise { if (!offer.webClientUrl) { throw new Error('HUB runtime did not provide a paired web client URL') } - const pagePromise = hubApp.waitForEvent('window') - await hubApp.evaluate( - async ({ BrowserWindow }, { partition, url }) => { - const clientWindow = new BrowserWindow({ - height: 1200, - show: false, - width: 1440, - webPreferences: { - contextIsolation: true, - nodeIntegration: false, - partition, - sandbox: true - } - }) - await clientWindow.loadURL(url) - }, - { - partition: `e2e-nested-runtime-web-${randomUUID()}`, - url: offer.webClientUrl + const clientUrl = createPairedWebClientUrl(offer.webClientUrl, options) + let page: Page | undefined + const pagePromise = hubApp.waitForEvent('window').then((candidate) => (page = candidate)) + try { + await hubApp.evaluate( + async ({ BrowserWindow }, { partition, url }) => { + const clientWindow = new BrowserWindow({ + height: 1200, + show: false, + width: 1440, + webPreferences: { + contextIsolation: true, + nodeIntegration: false, + partition, + sandbox: true + } + }) + await clientWindow.loadURL(url).catch((error) => { + clientWindow.destroy() + throw error + }) + }, + { + partition: `e2e-nested-runtime-web-${randomUUID()}`, + url: clientUrl + } + ) + page = await pagePromise + if (options.waitForWorkspace !== false) { + await page.locator('[data-worktree-sidebar]').waitFor({ state: 'visible', timeout: 30_000 }) } - ) - const page = await pagePromise - await page.locator('[data-worktree-sidebar]').waitFor({ state: 'visible', timeout: 30_000 }) - return { page, dispose: () => page.close() } + return { page, dispose: () => page?.close() ?? Promise.resolve() } + } catch (error) { + void pagePromise.catch(() => undefined) + await page?.close().catch(() => undefined) + throw error + } } export async function launchPairedElectronClient( diff --git a/tests/e2e/helpers/paired-terminal-parking-fixture.ts b/tests/e2e/helpers/paired-terminal-parking-fixture.ts new file mode 100644 index 000000000..48444b79e --- /dev/null +++ b/tests/e2e/helpers/paired-terminal-parking-fixture.ts @@ -0,0 +1,48 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +const FILL_ROWS = 6_000 + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(fixturePath: string, marker: string): string { + const command = [process.execPath, fixturePath, marker] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +export function createPairedTerminalParkingFixture(): { + command: (marker: string) => string + dispose: () => void +} { + const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-retention-memory-')) + const fixturePath = path.join(scratch, 'paired-retention-memory.mjs') + writeFileSync( + fixturePath, + [ + 'const marker = process.argv[2]', + 'process.stdout.write(`READY:${marker}\\r\\n`)', + 'process.stdin.setRawMode?.(true)', + "process.stdin.setEncoding('utf8')", + "process.stdin.on('data', (data) => {", + ' for (const command of data.split(/\\r\\n|\\r|\\n/).filter(Boolean)) {', + " if (command === 'FILL') {", + ` for (let row = 0; row < ${FILL_ROWS}; row += 1) process.stdout.write(\`fill-${'${marker}'}-${'${row}'}-${'x'.repeat(80)}\\r\\n\`)`, + ' process.stdout.write(`FILLED:${marker}\\r\\n`)', + ' continue', + ' }', + ' process.stdout.write(`LIVE:${command}\\r\\n`)', + ' }', + '})', + 'process.stdin.resume()' + ].join('\n') + ) + return { + command: (marker) => fixtureCommand(fixturePath, marker), + dispose: () => rmSync(scratch, { recursive: true, force: true }) + } +} diff --git a/tests/e2e/helpers/paired-terminal-parking-oracle.ts b/tests/e2e/helpers/paired-terminal-parking-oracle.ts new file mode 100644 index 000000000..30970e610 --- /dev/null +++ b/tests/e2e/helpers/paired-terminal-parking-oracle.ts @@ -0,0 +1,284 @@ +import type { Page } from '@stablyai/playwright-test' +import type { RuntimeTerminalRead } from '../../../src/shared/runtime-types' +import { TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY } from '../../../src/shared/protocol-version' +import { toWebTerminalSurfaceTabId } from '../../../src/shared/terminal-surface-id' +import { + readPairedRetentionSample, + startRendererLagProbe +} from '../paired-runtime-retention-metrics' +import { expect } from './orca-app' +import { createPairedTerminalParkingFixture } from './paired-terminal-parking-fixture' +import { getTerminalContent, waitForActivePanePtyId } from './terminal' + +const TARGET_WORKTREE_COUNT = 6 +const MIN_STAGED_BUFFER_CELLS = 1_000_000 +const MAX_RETAINED_CELL_FRACTION = 0.45 +const MAX_EVICTION_LAG_MS = 500 +const MAX_HEAP_GROWTH_BYTES = 16 * 1024 * 1024 + +type PairedTerminalParkingSeed = { + fallbackWorktreeId: string + repoId: string +} + +type RemoteTab = { + marker: string + originalPtyId: string + tabId: string + terminal: string + worktreeId: string +} + +async function callRuntime(page: Page, method: string, params: unknown): Promise { + return page.evaluate( + async ({ method, params }) => { + const response = await window.api.runtime.call({ method, params }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { method, params } + ) as Promise +} + +export async function runPairedTerminalParkingOracle( + page: Page, + seed: PairedTerminalParkingSeed +): Promise { + const fixture = createPairedTerminalParkingFixture() + const createdWorktreeIds: string[] = [] + const remoteTabs: RemoteTab[] = [] + try { + await expect + .poll( + () => + page.evaluate((capability) => { + const statuses = window.__store?.getState().runtimeStatusByEnvironmentId.values() ?? [] + return Array.from(statuses).some((entry) => + entry.status?.capabilities?.includes(capability) + ) + }, TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY), + { timeout: 30_000 } + ) + .toBe(true) + await page.evaluate(async () => { + await window.__store?.getState().updateSettings({ + terminalHiddenViewParking: false, + terminalHiddenWorktreeRetentionBudget: false + }) + }) + const createdTerminals: Omit[] = [] + while (createdTerminals.length < TARGET_WORKTREE_COUNT) { + const index = createdTerminals.length + const marker = `PAIR_RETENTION_${index}` + const suffix = `${Date.now()}-${index}` + const created = await callRuntime<{ + startupTerminal?: { handle?: string; tabId?: string } + worktree: { id: string } + }>(page, 'worktree.create', { + repo: seed.repoId, + name: `paired-retention-${suffix}`, + setupDecision: 'skip', + activate: false, + noParent: true, + startupCommand: fixture.command(marker) + }) + if (!created.startupTerminal?.handle || !created.startupTerminal.tabId) { + throw new Error(`Paired retention startup terminal ${index} was not created`) + } + createdWorktreeIds.push(created.worktree.id) + createdTerminals.push({ + marker, + tabId: toWebTerminalSurfaceTabId(created.startupTerminal.tabId), + terminal: created.startupTerminal.handle, + worktreeId: created.worktree.id + }) + } + await expect + .poll( + () => + page.evaluate( + (ids) => + ids.every((id) => + window.__store + ?.getState() + .allWorktrees() + .some((worktree) => worktree.id === id) + ), + createdWorktreeIds + ), + { timeout: 30_000 } + ) + .toBe(true) + + for (const created of createdTerminals) { + await page.evaluate( + (id) => window.__store?.getState().setActiveWorktree(id), + created.worktreeId + ) + const tab = page.locator(`[data-testid="sortable-tab"][data-tab-id="${created.tabId}"]`) + await expect(tab).toBeVisible({ timeout: 30_000 }) + await tab.click() + const originalPtyId = await waitForActivePanePtyId(page, 30_000) + await callRuntime(page, 'terminal.send', { + terminal: created.terminal, + text: 'FILL', + enter: true, + client: { id: 'paired-retention-memory-e2e', type: 'desktop' } + }) + await expect + .poll(() => getTerminalContent(page), { timeout: 30_000 }) + .toContain(`FILLED:${created.marker}`) + remoteTabs.push({ ...created, originalPtyId }) + } + + await page.evaluate(() => window.__store?.getState().setActiveView('tasks')) + await expect + .poll( + () => + page.evaluate( + (ids) => ids.filter((id) => window.__paneManagers?.has(id)).length, + remoteTabs.map((tab) => tab.tabId) + ), + { timeout: 10_000 } + ) + .toBe(TARGET_WORKTREE_COUNT) + const baseline = await readPairedRetentionSample( + page, + remoteTabs.map((tab) => tab.tabId) + ) + expect(baseline.bufferCells).toBeGreaterThan(MIN_STAGED_BUFFER_CELLS) + + const lagProbe = await startRendererLagProbe(page) + let maxLagMs = Number.POSITIVE_INFINITY + let lagProbeStopped = false + try { + await page.evaluate(async () => { + await window.__store?.getState().updateSettings({ terminalHiddenViewParking: true }) + }) + await expect + .poll( + () => + page.evaluate( + ({ tabIds, worktreeIds }) => { + const verdicts = window.__terminalParkingDebug?.worktreeVerdicts() ?? [] + return { + forceParked: worktreeIds.map( + (id) => verdicts.find((verdict) => verdict.worktreeId === id)?.forceParked + ), + mounted: tabIds.filter((id) => window.__paneManagers?.has(id)).length, + ordinaryParkingCovers: worktreeIds.map( + (id) => + verdicts.find((verdict) => verdict.worktreeId === id)?.ordinaryParkingCovers + ), + parked: window.__terminalParkingDebug?.parkedTabIds().length, + retentionBudgetEnabled: + window.__store?.getState().settings?.terminalHiddenWorktreeRetentionBudget + } + }, + { + tabIds: remoteTabs.map((tab) => tab.tabId), + worktreeIds: remoteTabs.map((tab) => tab.worktreeId) + } + ), + { timeout: 10_000 } + ) + .toEqual({ + forceParked: Array(TARGET_WORKTREE_COUNT).fill(false), + mounted: 1, + ordinaryParkingCovers: Array(TARGET_WORKTREE_COUNT).fill(true), + parked: TARGET_WORKTREE_COUNT - 1, + retentionBudgetEnabled: false + }) + maxLagMs = await lagProbe.evaluate((probe) => probe.stop()) + lagProbeStopped = true + } finally { + if (!lagProbeStopped) { + await lagProbe.evaluate((probe) => probe.stop()).catch(() => undefined) + } + await lagProbe.dispose() + } + const after = await readPairedRetentionSample( + page, + remoteTabs.map((tab) => tab.tabId) + ) + expect(after.bufferCells).toBeLessThanOrEqual(baseline.bufferCells * MAX_RETAINED_CELL_FRACTION) + expect(after.mountedTargetManagers).toBe(1) + expect(maxLagMs).toBeLessThan(MAX_EVICTION_LAG_MS) + if (baseline.heapBytes !== null && after.heapBytes !== null) { + expect(after.heapBytes).toBeLessThanOrEqual(baseline.heapBytes + MAX_HEAP_GROWTH_BYTES) + } + + const evicted = await page.evaluate( + (tabs) => tabs.find((tab) => !window.__paneManagers?.has(tab.tabId)) ?? null, + remoteTabs + ) + if (!evicted) { + throw new Error('Ordinary parking did not unmount a paired terminal') + } + const parkedMarker = `WHILE_PARKED_${Date.now()}` + await callRuntime(page, 'terminal.send', { + terminal: evicted.terminal, + text: parkedMarker, + enter: true, + client: { id: 'paired-retention-memory-e2e', type: 'desktop' } + }) + await expect + .poll( + async () => { + const result = await callRuntime<{ terminal: RuntimeTerminalRead }>( + page, + 'terminal.read', + { terminal: evicted.terminal, limit: 1_000 } + ) + return result.terminal.tail.join('\n') + }, + { timeout: 30_000 } + ) + .toContain(`LIVE:${parkedMarker}`) + + await page.evaluate((worktreeId) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(worktreeId) + }, evicted.worktreeId) + const restored = page.locator(`[data-testid="sortable-tab"][data-tab-id="${evicted.tabId}"]`) + await expect(restored).toBeVisible({ timeout: 30_000 }) + await restored.click() + expect(await waitForActivePanePtyId(page, 30_000)).toBe(evicted.originalPtyId) + await expect + .poll(() => getTerminalContent(page, 1_000_000), { timeout: 30_000 }) + .toContain(`fill-${evicted.marker}-4000-`) + await expect + .poll(() => getTerminalContent(page, 1_000_000), { timeout: 30_000 }) + .toContain(`LIVE:${parkedMarker}`) + const liveMarker = `AFTER_RETENTION_${Date.now()}` + await callRuntime(page, 'terminal.send', { + terminal: evicted.terminal, + text: liveMarker, + enter: true, + client: { id: 'paired-retention-memory-e2e', type: 'desktop' } + }) + await expect + .poll(() => getTerminalContent(page), { timeout: 30_000 }) + .toContain(`LIVE:${liveMarker}`) + } finally { + for (const tab of remoteTabs) { + await callRuntime(page, 'terminal.closeTab', { terminal: tab.terminal }).catch( + () => undefined + ) + } + await page + .evaluate((id) => window.__store?.getState().setActiveWorktree(id), seed.fallbackWorktreeId) + .catch(() => undefined) + for (const worktreeId of createdWorktreeIds.toReversed()) { + await callRuntime(page, 'worktree.rm', { + worktree: `id:${worktreeId}`, + force: true, + runHooks: false + }).catch(() => undefined) + } + fixture.dispose() + } +} diff --git a/tests/e2e/helpers/paired-web-client-url.ts b/tests/e2e/helpers/paired-web-client-url.ts new file mode 100644 index 000000000..306f7bd24 --- /dev/null +++ b/tests/e2e/helpers/paired-web-client-url.ts @@ -0,0 +1,23 @@ +export type PairedWebClientOptions = { + disableRemoteTerminalStallRecovery?: boolean + terminalParkingDelayMs?: number + terminalRetentionLimit?: number + waitForWorkspace?: boolean +} + +export function createPairedWebClientUrl( + offerUrl: string, + options: PairedWebClientOptions +): string { + const clientUrl = new URL(offerUrl) + if (options.disableRemoteTerminalStallRecovery) { + clientUrl.searchParams.set('orcaE2EDisableRemoteTerminalStallRecovery', '1') + } + if (options.terminalParkingDelayMs !== undefined) { + clientUrl.searchParams.set('orcaE2ETerminalParkingDelayMs', `${options.terminalParkingDelayMs}`) + } + if (options.terminalRetentionLimit !== undefined) { + clientUrl.searchParams.set('orcaE2ETerminalRetentionLimit', `${options.terminalRetentionLimit}`) + } + return clientUrl.href +} diff --git a/tests/e2e/paired-remote-terminal-retention-memory.spec.ts b/tests/e2e/paired-remote-terminal-retention-memory.spec.ts new file mode 100644 index 000000000..621f4c248 --- /dev/null +++ b/tests/e2e/paired-remote-terminal-retention-memory.spec.ts @@ -0,0 +1,34 @@ +import { test } from './helpers/orca-app' +import { + createRuntimeDesktopPairingOffer, + launchPairedWebClient +} from './helpers/paired-electron-client' +import { runPairedTerminalParkingOracle } from './helpers/paired-terminal-parking-oracle' + +test('ordinary-parks paired terminals and restores authoritative host scrollback @headful', async ({ + electronApp, + orcaPage +}) => { + test.setTimeout(240_000) + const seed = await orcaPage.evaluate(() => { + const state = window.__store?.getState() + const worktrees = state?.allWorktrees() ?? [] + const active = worktrees.find((worktree) => worktree.id === state?.activeWorktreeId) + if (!active) { + throw new Error('Paired retention host has no active seeded worktree') + } + return { repoId: active.repoId, worktreeIds: worktrees.map((worktree) => worktree.id) } + }) + const offer = await createRuntimeDesktopPairingOffer(orcaPage) + const client = await launchPairedWebClient(electronApp, offer, { + terminalParkingDelayMs: 100 + }) + try { + await runPairedTerminalParkingOracle(client.page, { + fallbackWorktreeId: seed.worktreeIds[0]!, + repoId: seed.repoId + }) + } finally { + await client.dispose() + } +}) diff --git a/tests/e2e/paired-remote-terminal-stall-recovery.spec.ts b/tests/e2e/paired-remote-terminal-stall-recovery.spec.ts new file mode 100644 index 000000000..2f6026f0b --- /dev/null +++ b/tests/e2e/paired-remote-terminal-stall-recovery.spec.ts @@ -0,0 +1,251 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { Page } from '@stablyai/playwright-test' +import type { RuntimeTerminalRead } from '../../src/shared/runtime-types' +import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id' +import { expect, test } from './helpers/orca-app' +import { + createRuntimeDesktopPairingOffer, + launchPairedWebClient +} from './helpers/paired-electron-client' +import { getTerminalContent, waitForActivePanePtyId } from './helpers/terminal' + +const MIN_EXHAUSTED_ACK_BYTES = 400 * 1024 +const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-stalled-stream-')) +const fixturePath = path.join(scratch, 'stalled-stream-terminal.mjs') +writeFileSync( + fixturePath, + [ + "process.stdout.write('PAIRED_STALL_READY\\r\\n')", + "process.stdin.setEncoding('utf8')", + "let pending = ''", + "process.stdin.on('data', (data) => {", + ' pending += data', + ' const commands = pending.split(/\\r\\n|\\r|\\n/)', + ' pending = commands.pop() ?? ""', + ' for (const input of commands) {', + " if (input === 'GO') {", + " for (let row = 0; row < 16_000; row += 1) process.stdout.write(`flood-${row}-${'x'.repeat(80)}\\r\\n`)", + " process.stdout.write('HOST_FLOOD_COMPLETE\\r\\n')", + ' continue', + ' }', + ' process.stdout.write(`LIVE:${input}\\r\\n`)', + ' }', + '})', + 'process.stdin.resume()' + ].join('\n') +) + +test.afterAll(() => { + rmSync(scratch, { recursive: true, force: true }) +}) + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(): string { + const command = [process.execPath, fixturePath] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +async function callRuntime(page: Page, method: string, params: unknown): Promise { + return page.evaluate( + async ({ method, params }) => { + const response = await window.api.runtime.call({ method, params }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { method, params } + ) as Promise +} + +test('restarts one ACK-starved paired terminal stream without replacing its PTY @headful', async ({ + electronApp, + orcaPage +}) => { + test.setTimeout(120_000) + const liveMarker = `PAIRED_STALL_RECOVERED_${Date.now()}` + const worktree = await orcaPage.evaluate(() => { + const state = window.__store?.getState() + const id = state?.activeWorktreeId + const active = state?.allWorktrees().find((candidate) => candidate.id === id) + if (!active) { + throw new Error('Headed host did not select its seeded worktree') + } + return { id: active.id } + }) + const offer = await createRuntimeDesktopPairingOffer(orcaPage) + const client = await launchPairedWebClient(electronApp, offer, { + disableRemoteTerminalStallRecovery: + process.env.ORCA_E2E_DISABLE_REMOTE_TERMINAL_STALL_RECOVERY === '1' + }) + let terminal: string | null = null + try { + await expect + .poll( + () => + client.page.evaluate( + (worktreeId) => + window.__store + ?.getState() + .allWorktrees() + .some((candidate) => candidate.id === worktreeId), + worktree.id + ), + { timeout: 30_000 } + ) + .toBe(true) + const created = await callRuntime<{ + tab: { parentTabId: string; terminal: string | null } + }>(client.page, 'session.tabs.createTerminal', { + worktree: `id:${worktree.id}`, + command: fixtureCommand(), + activate: false, + select: false, + navigation: 'caller' + }) + terminal = created.tab.terminal + if (!terminal) { + throw new Error('Paired runtime did not publish the stalled-stream fixture') + } + const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId) + await expect + .poll( + () => + client.page.evaluate( + ({ tabId, worktreeId }) => + (window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some( + (tab) => tab.id === tabId + ), + { tabId: webTabId, worktreeId: worktree.id } + ), + { timeout: 30_000 } + ) + .toBe(true) + await client.page.evaluate( + (worktreeId) => window.__store?.getState().setActiveWorktree(worktreeId), + worktree.id + ) + const tab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`) + await expect(tab).toBeVisible({ timeout: 30_000 }) + await tab.click() + const originalPtyId = await waitForActivePanePtyId(client.page, 30_000) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain('PAIRED_STALL_READY') + + await client.page.evaluate((target) => { + const gate = ( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { hold: (terminals: string[]) => void } + } + ).__remoteTerminalMultiplexAckGate + if (!gate) { + throw new Error('Remote terminal multiplex ACK gate is unavailable') + } + gate.hold([target]) + }, terminal) + const textarea = client.page.locator('.xterm-helper-textarea:visible').first() + await textarea.focus() + await client.page.keyboard.type('GO') + await client.page.keyboard.press('Enter') + + await expect + .poll( + () => + client.page.evaluate(() => { + const gate = ( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { + snapshot: () => { heldAckChars: number } + } + } + ).__remoteTerminalMultiplexAckGate + return gate?.snapshot().heldAckChars ?? 0 + }), + { timeout: 30_000 } + ) + .toBeGreaterThan(MIN_EXHAUSTED_ACK_BYTES) + await expect + .poll( + async () => { + const result = await callRuntime<{ terminal: RuntimeTerminalRead }>( + client.page, + 'terminal.read', + { terminal } + ) + return result.terminal.tail.join('\n').includes('HOST_FLOOD_COMPLETE') + }, + { timeout: 30_000 } + ) + .toBe(true) + + const beforeInput = await callRuntime<{ terminal: RuntimeTerminalRead }>( + client.page, + 'terminal.read', + { terminal } + ) + const sent = await callRuntime<{ send: { accepted: boolean } }>(client.page, 'terminal.send', { + terminal, + text: liveMarker, + enter: true, + client: { id: 'paired-stalled-stream-e2e', type: 'desktop' } + }) + expect(sent.send.accepted).toBe(true) + await expect + .poll( + async () => { + const result = await callRuntime<{ terminal: RuntimeTerminalRead }>( + client.page, + 'terminal.read', + { terminal } + ) + return Number(result.terminal.latestCursor) + }, + { timeout: 30_000 } + ) + .toBeGreaterThan(Number(beforeInput.terminal.latestCursor)) + expect(await getTerminalContent(client.page)).not.toContain(liveMarker) + expect( + await client.page.evaluate( + ({ target, text }) => { + const gate = ( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { + sendInput: (terminal: string, text: string) => number + } + } + ).__remoteTerminalMultiplexAckGate + return gate?.sendInput(target, text) ?? 0 + }, + { target: terminal, text: '\r' } + ) + ).toBe(1) + + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain(`LIVE:${liveMarker}`) + expect(await waitForActivePanePtyId(client.page, 30_000)).toBe(originalPtyId) + await expect(tab).toHaveAttribute('data-active', 'true') + } finally { + await client.page + .evaluate(() => { + ;( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { release: () => void } + } + ).__remoteTerminalMultiplexAckGate?.release() + }) + .catch(() => undefined) + if (terminal) { + await callRuntime(client.page, 'terminal.closeTab', { terminal }).catch(() => undefined) + } + await client.dispose() + } +}) diff --git a/tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts b/tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts new file mode 100644 index 000000000..79c7145cd --- /dev/null +++ b/tests/e2e/paired-remote-terminal-truncated-tail-first-paint.spec.ts @@ -0,0 +1,483 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { Page } from '@stablyai/playwright-test' +import type { RuntimeTerminalRead } from '../../src/shared/runtime-types' +import { TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY } from '../../src/shared/protocol-version' +import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id' +import { expect, test } from './helpers/orca-app' +import { + createRuntimeDesktopPairingOffer, + launchPairedWebClient +} from './helpers/paired-electron-client' +import { getTerminalContent, waitForActivePanePtyId } from './helpers/terminal' + +const RETENTION_PARK_DELAY_MS = 100 +const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-truncated-tail-')) +const fixturePath = path.join(scratch, 'truncated-tail-terminal.mjs') +writeFileSync( + fixturePath, + [ + 'const marker = process.argv[2]', + 'let flooded = false', + "process.stdout.write('REMOTE_TRUNCATED_TAIL_READY\\r\\n')", + "process.stdin.setEncoding('utf8')", + "process.stdin.on('data', (data) => {", + " if (!flooded && data.includes('GO')) {", + ' flooded = true', + " for (let row = 0; row < 4_000; row += 1) process.stdout.write(`overflow-${row}-${'x'.repeat(80)}\\r\\n`)", + ' process.stdout.write(`${marker}\\r\\n`)', + ' return', + ' }', + ' process.stdout.write(`LIVE:${data.trim()}\\r\\n`)', + '})', + 'process.stdin.resume()' + ].join('\n') +) + +test.afterAll(() => { + rmSync(scratch, { recursive: true, force: true }) +}) + +test.use({ + orcaAppExtraEnv: { + ORCA_E2E_TERMINAL_PARKING_DELAY_MS: String(RETENTION_PARK_DELAY_MS), + ORCA_E2E_TERMINAL_RETENTION_LIMIT: '1' + } +}) + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(marker: string): string { + const command = [process.execPath, fixturePath, marker] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +async function callRuntime(page: Page, method: string, params: unknown): Promise { + return page.evaluate( + async ({ method, params }) => { + const response = await window.api.runtime.call({ method, params }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { method, params } + ) as Promise +} + +test('paints a paired remote terminal when only its retained text tail overflowed @headful', async ({ + electronApp, + orcaPage +}) => { + test.setTimeout(120_000) + const firstPaintMarker = `REMOTE_TRUNCATED_TAIL_FIRST_PAINT_${Date.now()}` + const liveMarker = `REMOTE_TRUNCATED_TAIL_LIVE_${Date.now()}` + const worktree = await orcaPage.evaluate(() => { + const state = window.__store?.getState() + const activeWorktreeId = state?.activeWorktreeId + if (!activeWorktreeId) { + throw new Error('Headed host did not select its seeded worktree') + } + const activeWorktree = state + .allWorktrees() + .find((candidate) => candidate.id === activeWorktreeId) + if (!activeWorktree) { + throw new Error('Headed host active worktree was absent from inventory') + } + return { id: activeWorktree.id, path: activeWorktree.path } + }) + const offer = await createRuntimeDesktopPairingOffer(orcaPage) + const client = await launchPairedWebClient(electronApp, offer) + let terminal: string | null = null + try { + await expect + .poll( + () => + client.page.evaluate( + (worktreeId) => + window.__store + ?.getState() + .allWorktrees() + .some((candidate) => candidate.id === worktreeId), + worktree.id + ), + { timeout: 30_000 } + ) + .toBe(true) + const created = await callRuntime<{ + tab: { + parentTabId: string + leafId: string + terminal: string | null + } + }>(client.page, 'session.tabs.createTerminal', { + worktree: `id:${worktree.id}`, + command: fixtureCommand(firstPaintMarker), + activate: false, + select: false, + navigation: 'caller' + }) + terminal = created.tab.terminal + if (!terminal) { + throw new Error('Paired runtime did not publish the overflow fixture terminal') + } + + const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId) + await expect + .poll( + () => + client.page.evaluate( + ({ tabId, worktreeId }) => + (window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some( + (tab) => tab.id === tabId + ), + { tabId: webTabId, worktreeId: worktree.id } + ), + { timeout: 30_000 } + ) + .toBe(true) + await client.page.evaluate( + (worktreeId) => window.__store?.getState().setActiveWorktree(worktreeId), + worktree.id + ) + const remoteTab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`) + await expect(remoteTab).toBeVisible({ timeout: 30_000 }) + await remoteTab.click() + await expect(remoteTab).toHaveAttribute('data-active', 'true') + await waitForActivePanePtyId(client.page, 30_000) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain('REMOTE_TRUNCATED_TAIL_READY') + await callRuntime(client.page, 'terminal.send', { + terminal, + text: 'WARMUP', + enter: true, + client: { id: 'paired-truncated-tail-e2e', type: 'desktop' } + }) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain('LIVE:WARMUP') + await callRuntime(client.page, 'terminal.send', { + terminal, + text: 'GO', + enter: true, + client: { id: 'paired-truncated-tail-e2e', type: 'desktop' } + }) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain(firstPaintMarker) + await expect + .poll( + async () => { + const result = await callRuntime<{ terminal: RuntimeTerminalRead }>( + client.page, + 'terminal.read', + { terminal } + ) + return { + marker: result.terminal.tail.join('\n').includes(firstPaintMarker), + truncated: result.terminal.truncated + } + }, + { timeout: 30_000 } + ) + .toEqual({ marker: true, truncated: true }) + + await client.page.reload() + await client.page.locator('[data-worktree-sidebar]').waitFor({ + state: 'visible', + timeout: 30_000 + }) + await client.page.evaluate( + (worktreeId) => window.__store?.getState().setActiveWorktree(worktreeId), + worktree.id + ) + const restoredRemoteTab = client.page.locator( + `[data-testid="sortable-tab"][data-tab-id="${webTabId}"]` + ) + await expect(restoredRemoteTab).toBeVisible({ timeout: 30_000 }) + await restoredRemoteTab.click() + await expect(restoredRemoteTab).toHaveAttribute('data-active', 'true') + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain(firstPaintMarker) + + await callRuntime(client.page, 'terminal.send', { + terminal, + text: liveMarker, + enter: true, + client: { id: 'paired-truncated-tail-e2e', type: 'desktop' } + }) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain(`LIVE:${liveMarker}`) + } finally { + if (terminal) { + await callRuntime(client.page, 'terminal.closeTab', { terminal }).catch(() => undefined) + } + await client.dispose() + } +}) + +test('legacy paired hosts retain the lossy hidden-manager budget fallback @headful', async ({ + electronApp, + orcaPage +}) => { + test.skip( + process.env.ORCA_E2E_DISABLE_PAIRED_TERMINAL_PARKING !== '1', + 'The legacy fallback requires a host without terminal.paired-parking.v1.' + ) + test.setTimeout(120_000) + const worktreeIds = await orcaPage.evaluate(() => + window.__store + ?.getState() + .allWorktrees() + .slice(0, 2) + .map((worktree) => worktree.id) + ) + if (!worktreeIds || worktreeIds.length < 2) { + throw new Error('Paired retention fixture requires two seeded worktrees') + } + const offer = await createRuntimeDesktopPairingOffer(orcaPage) + const client = await launchPairedWebClient(electronApp, offer, { + terminalParkingDelayMs: RETENTION_PARK_DELAY_MS, + terminalRetentionLimit: 1 + }) + const createdTerminals: string[] = [] + try { + expect(await client.page.evaluate(() => window.api.e2e.getConfig())).toMatchObject({ + exposeStore: true, + terminalParkingDelayMs: RETENTION_PARK_DELAY_MS, + terminalRetentionLimit: 1 + }) + expect( + await client.page.evaluate( + (capability) => + Array.from(window.__store?.getState().runtimeStatusByEnvironmentId.values() ?? []).some( + (entry) => entry.status?.capabilities?.includes(capability) + ), + TERMINAL_PAIRED_PARKING_RUNTIME_CAPABILITY + ) + ).toBe(false) + await expect + .poll( + () => + client.page.evaluate((ids) => { + const known = new Set( + window.__store + ?.getState() + .allWorktrees() + .map((worktree) => worktree.id) + ) + return ids.every((id) => known.has(id)) + }, worktreeIds), + { timeout: 30_000 } + ) + .toBe(true) + await client.page.evaluate(async () => { + await window.__store + ?.getState() + .updateSettings({ terminalHiddenWorktreeRetentionBudget: false }) + }) + + const remoteTabs: { tabId: string; terminal: string; worktreeId: string; marker: string }[] = [] + for (const [index, worktreeId] of worktreeIds.entries()) { + const marker = `PAIRED_RETENTION_${index}_${Date.now()}` + const created = await callRuntime<{ + tab: { parentTabId: string; terminal: string | null } + }>(client.page, 'session.tabs.createTerminal', { + worktree: `id:${worktreeId}`, + command: fixtureCommand(marker), + activate: false, + select: false, + navigation: 'caller' + }) + if (!created.tab.terminal) { + throw new Error(`Paired retention terminal ${index} was not published`) + } + createdTerminals.push(created.tab.terminal) + const tabId = toWebTerminalSurfaceTabId(created.tab.parentTabId) + await expect + .poll( + () => + client.page.evaluate( + ({ tabId, worktreeId }) => + (window.__store?.getState().tabsByWorktree[worktreeId] ?? []).some( + (tab) => tab.id === tabId + ), + { tabId, worktreeId } + ), + { timeout: 30_000 } + ) + .toBe(true) + await client.page.evaluate( + (id) => window.__store?.getState().setActiveWorktree(id), + worktreeId + ) + const tab = client.page.locator(`[data-testid="sortable-tab"][data-tab-id="${tabId}"]`) + await expect(tab).toBeVisible({ timeout: 30_000 }) + await tab.click() + await waitForActivePanePtyId(client.page, 30_000) + await callRuntime(client.page, 'terminal.send', { + terminal: created.tab.terminal, + text: marker, + enter: true, + client: { id: 'paired-retention-e2e', type: 'desktop' } + }) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain(`LIVE:${marker}`) + remoteTabs.push({ tabId, terminal: created.tab.terminal, worktreeId, marker }) + } + + await expect + .poll( + () => + client.page.evaluate(() => ({ + parkDelayMs: window.__terminalParkingDebug?.parkDelayMs, + retentionLimit: window.__terminalParkingDebug?.retentionLimit + })), + { timeout: 10_000 } + ) + .toEqual({ parkDelayMs: RETENTION_PARK_DELAY_MS, retentionLimit: 1 }) + + await client.page.evaluate(() => window.__store?.getState().setActiveView('tasks')) + const controlStartedAt = Date.now() + await expect + .poll( + () => + client.page.evaluate( + ({ tabIds, controlStartedAt, delayMs }) => ({ + heldLongEnough: Date.now() - controlStartedAt >= delayMs * 4, + mounted: tabIds.filter((tabId) => window.__paneManagers?.has(tabId)).length + }), + { + tabIds: remoteTabs.map((tab) => tab.tabId), + controlStartedAt, + delayMs: RETENTION_PARK_DELAY_MS + } + ), + { timeout: 10_000 } + ) + .toEqual({ heldLongEnough: true, mounted: 2 }) + + await client.page.evaluate(async () => { + await window.__store + ?.getState() + .updateSettings({ terminalHiddenWorktreeRetentionBudget: true }) + }) + await expect + .poll( + () => + client.page.evaluate( + ({ delayMs, tabIds: [olderTabId, newerTabId], worktreeIds }) => { + const state = window.__store?.getState() + const terminalTabs = Object.values(state?.tabsByWorktree ?? {}).flat() + const verdicts = window.__terminalParkingDebug?.worktreeVerdicts() ?? [] + return { + activeView: state?.activeView, + budgetEnabled: state?.settings?.terminalHiddenWorktreeRetentionBudget, + newerMounted: window.__paneManagers?.has(newerTabId), + olderMounted: window.__paneManagers?.has(olderTabId), + remotePtys: [olderTabId, newerTabId].map((tabId) => + terminalTabs.find((tab) => tab.id === tabId)?.ptyId?.startsWith('remote:') + ), + verdicts: worktreeIds.map((worktreeId) => { + const verdict = verdicts.find((candidate) => candidate.worktreeId === worktreeId) + return verdict + ? { + forceParked: verdict.forceParked, + hasActivityTerminalPortal: verdict.hasActivityTerminalPortal, + hasPendingSpawnWork: verdict.hasPendingSpawnWork, + hidden: verdict.hiddenSinceMs !== null, + hiddenPastDelay: + verdict.hiddenSinceMs !== null && + Date.now() - verdict.hiddenSinceMs >= delayMs, + isVisible: verdict.isVisible, + ordinaryParkingCovers: verdict.ordinaryParkingCovers, + parkCooldown: + verdict.parkCooldownUntilMs !== null && + Date.now() < verdict.parkCooldownUntilMs, + shouldMeasureHiddenWorktree: verdict.shouldMeasureHiddenWorktree + } + : null + }) + } + }, + { + delayMs: RETENTION_PARK_DELAY_MS, + tabIds: [remoteTabs[0]!.tabId, remoteTabs[1]!.tabId], + worktreeIds + } + ), + { timeout: 10_000 } + ) + .toEqual({ + activeView: 'tasks', + budgetEnabled: true, + newerMounted: true, + olderMounted: false, + remotePtys: [true, true], + verdicts: [ + { + forceParked: true, + hasActivityTerminalPortal: false, + hasPendingSpawnWork: false, + hidden: true, + hiddenPastDelay: true, + isVisible: false, + ordinaryParkingCovers: false, + parkCooldown: false, + shouldMeasureHiddenWorktree: false + }, + { + forceParked: false, + hasActivityTerminalPortal: false, + hasPendingSpawnWork: false, + hidden: true, + hiddenPastDelay: true, + isVisible: false, + ordinaryParkingCovers: false, + parkCooldown: false, + shouldMeasureHiddenWorktree: false + } + ] + }) + + const older = remoteTabs[0]! + await client.page.evaluate((worktreeId) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(worktreeId) + }, older.worktreeId) + const olderTab = client.page.locator( + `[data-testid="sortable-tab"][data-tab-id="${older.tabId}"]` + ) + await expect(olderTab).toBeVisible({ timeout: 30_000 }) + await olderTab.click() + await waitForActivePanePtyId(client.page, 30_000) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain(`LIVE:${older.marker}`) + await callRuntime(client.page, 'terminal.send', { + terminal: older.terminal, + text: 'AFTER_RESTORE', + enter: true, + client: { id: 'paired-retention-e2e', type: 'desktop' } + }) + await expect + .poll(() => getTerminalContent(client.page), { timeout: 30_000 }) + .toContain('LIVE:AFTER_RESTORE') + await expect(olderTab).toHaveAttribute('data-active', 'true') + } finally { + for (const terminal of createdTerminals) { + await callRuntime(client.page, 'terminal.closeTab', { terminal }).catch(() => undefined) + } + await client.dispose() + } +}) diff --git a/tests/e2e/paired-runtime-retention-metrics.ts b/tests/e2e/paired-runtime-retention-metrics.ts new file mode 100644 index 000000000..234b9fb46 --- /dev/null +++ b/tests/e2e/paired-runtime-retention-metrics.ts @@ -0,0 +1,67 @@ +import type { JSHandle, Page } from '@stablyai/playwright-test' + +export type PairedRetentionSample = { + bufferCells: number + heapBytes: number | null + mountedTargetManagers: number + targetPanes: number +} + +export async function readPairedRetentionSample( + page: Page, + tabIds: string[] +): Promise { + try { + const session = await page.context().newCDPSession(page) + await session.send('HeapProfiler.collectGarbage') + await session.detach() + } catch { + // GC only improves measurement fidelity. + } + return page.evaluate((targets) => { + let bufferCells = 0 + let mountedTargetManagers = 0 + let targetPanes = 0 + for (const tabId of targets) { + const manager = window.__paneManagers?.get(tabId) + if (!manager) { + continue + } + mountedTargetManagers += 1 + for (const pane of manager.getPanes?.() ?? []) { + const buffer = pane.terminal?.buffer?.active + if (!buffer) { + continue + } + targetPanes += 1 + bufferCells += buffer.length * pane.terminal.cols + } + } + const memory = (performance as Performance & { memory?: { usedJSHeapSize?: number } }).memory + return { + bufferCells, + heapBytes: memory?.usedJSHeapSize ?? null, + mountedTargetManagers, + targetPanes + } + }, tabIds) +} + +export async function startRendererLagProbe(page: Page): Promise number }>> { + return page.evaluateHandle(() => { + const sampleMs = 16 + let lastAt = performance.now() + let maxDriftMs = 0 + const timer = window.setInterval(() => { + const now = performance.now() + maxDriftMs = Math.max(maxDriftMs, now - lastAt - sampleMs) + lastAt = now + }, sampleMs) + return { + stop: () => { + window.clearInterval(timer) + return maxDriftMs + } + } + }) +}