fix(remote-runtime): recover terminals after network loss (#9774)
Co-authored-by: Brennan Benson <79079362+brennanb2025@users.noreply.github.com>
This commit is contained in:
OrcaWin
GitHub
parent
e986a7ba1a
commit
54c1ec5e7a
|
|
@ -1330,7 +1330,7 @@
|
|||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-07-20",
|
||||
"date": "2026-07-21",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/pty-size-reconcile.test.ts src/renderer/src/components/terminal-pane/pty-size-reassertion.test.ts src/renderer/src/components/terminal-pane/split-right-white-screen.test.ts src/renderer/src/components/terminal-pane/use-terminal-window-wake-recovery.test.ts src/main/ipc/pty.test.ts src/main/providers/ssh-pty-provider.test.ts src/main/runtime/orca-runtime.test.ts src/relay/pty-handler.test.ts",
|
||||
|
|
@ -2385,7 +2385,7 @@
|
|||
},
|
||||
{
|
||||
"id": "terminal-provider.ssh-remote-reattach-contract",
|
||||
"title": "SSH and remote terminal restore never turn unknown liveness into destructive cleanup",
|
||||
"title": "SSH restore and remote-runtime recovery preserve live terminal ownership",
|
||||
"maturity": "experimental",
|
||||
"protection": "partial",
|
||||
"owner": "terminal-provider",
|
||||
|
|
@ -2393,6 +2393,8 @@
|
|||
"surfaces": [
|
||||
"SSH deferred restore",
|
||||
"remote-runtime mirror polling",
|
||||
"remote-runtime network recovery",
|
||||
"terminal create idempotency",
|
||||
"provider listing",
|
||||
"reattach",
|
||||
"unknown liveness"
|
||||
|
|
@ -2414,7 +2416,7 @@
|
|||
"ssh",
|
||||
"remote-runtime"
|
||||
],
|
||||
"coverageNotes": "Deterministic renderer coverage proves startup publishes the state returned by ssh.connect, remote or unresolved hosts cannot enter local-daemon cold activation deferral, stale cleanup cannot unregister a replacement runtime terminal, and a mounted remote-runtime web mirror replaces a stale handle without retiring its pane or accumulating obsolete PTY identities. Bounded polling falls back to accepted host snapshots without a global store subscription, and concurrent panes share each in-flight inventory request. A macOS Electron journey connects to a real Linux SSH relay in Docker, persists six terminals, reloads the renderer, and verifies all six original relay PTYs remount eagerly and still execute remote input. Linux and Windows desktop clients, WSL, and live remote-runtime restore remain gaps.",
|
||||
"coverageNotes": "Deterministic renderer coverage proves startup publishes the state returned by ssh.connect, stale cleanup cannot unregister a replacement runtime terminal, and a mounted remote-runtime terminal survives repeated transport partitions without changing PTY identity. Client/server heartbeat tests cover timer suspension, socket generations fence stale callbacks, cold restored-terminal attachment retries, cached pixels remain unhealthy until authoritative replay, automatic retries stop after one minute, manual reconnect preserves the PTY, and pane closure releases recovery UI state. Capability-gated create retries adopt a provider-owned PTY by stable terminal identity after an unknown outcome or runtime-process restart, stop retrying after one minute without a fatal error, and remain manually retryable without accepting stale create completions. A macOS Electron journey covers live SSH restore; a Windows remote-runtime smoke covers reachability and PTY round-trip. A live partition journey using patched Mac and Windows builds remains a gap.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/pull/6951",
|
||||
"https://github.com/stablyai/orca/pull/6955",
|
||||
|
|
@ -2422,10 +2424,11 @@
|
|||
"https://github.com/stablyai/orca/pull/7009",
|
||||
"https://github.com/stablyai/orca/pull/8597"
|
||||
],
|
||||
"invariant": "SSH, WSL, and remote-runtime restore paths must treat provider listing failures and unknown liveness as unknown, not dead, while still avoiding duplicate spawn and clearing expired relay leases exactly once. Snapshot-backed cold activation deferral is local-daemon-only; every restored remote terminal must mount a manager eagerly and preserve its provider PTY identity. A stale remote-runtime mirror handle must keep its pane mounted until a replacement or explicit terminal-removal fact arrives, and each replacement must atomically supersede rather than accumulate PTY identity.",
|
||||
"oracle": "Deterministic tests assert startup publishes ssh.connect's authoritative state before terminal reconnect, only an explicit local execution host can defer cold activation, host ownership changes clear old restrictions, replacement runtime registrations survive stale cleanup, stale mirror polling is bounded, and replacement handles resubscribe without exit/disconnect callbacks or stale PTY index growth. Count tests prove concurrent panes share one in-flight inventory request per runtime/worktree and accepted-snapshot listeners are identity-scoped and released after rebind. The live Docker SSH journey commits six terminal records to the relay, reloads Electron, requires the exact six manager ids with no parked tabs, compares all restored PTY ids to their pre-reload identities, and sends a marker through the restored terminal to a proof file on the Linux host.",
|
||||
"invariant": "SSH, WSL, and remote-runtime restore paths must treat provider listing failures and unknown liveness as unknown, not dead, while still avoiding duplicate spawn and clearing expired relay leases exactly once. Every restored remote terminal must preserve its provider PTY identity. After a recoverable partition the same authenticated runtime must reattach the same PTY, reject detached input, apply the latest viewport, and report healthy only after authoritative replay. Automatic PTY recovery stops after one bounded minute without a fatal terminal error; a manual reconnect starts a newly fenced epoch against the same PTY, and closed panes retain no recovery UI state. One capability-gated terminal-create mutation must produce at most one host PTY across an unknown response outcome, remain manually retryable after cutoff, and never let a stale completion replace a newer pane lifecycle.",
|
||||
"oracle": "Deterministic tests cover bounded stale-handle replacement, suspended heartbeat clocks, cold and established subscription failure, ten partition/recovery cycles, automatic-recovery cutoff, and manual reconnect. They assert one unsubscribe per epoch, observable recovery phases, stable PTY identity, resumed snapshot/output/input, no healthy state before replay, no retry or input after cutoff, a new manual epoch against the same PTY, quiet recovery UI with an explicit Reconnect action, pane-close state cleanup, one stable create mutation id, one-minute create-retry cutoff, old-runtime no-retry behavior, authenticated client/worktree isolation, cross-process PTY adoption without rerunning startup, unavailable or legacy-incomplete inventory fail-closed behavior, and bounded in-flight coordination. Existing count tests prove concurrent panes share one in-flight inventory request per runtime/worktree and accepted-snapshot listeners are identity-scoped and released after rebind.",
|
||||
"commands": [
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/startup/ssh-startup-reconnect.test.ts src/renderer/src/lib/resolved-worktree-execution-host.test.ts src/renderer/src/components/terminal/background-terminal-worktree-mount.test.ts src/renderer/src/runtime/sync-runtime-graph-scheduling.test.ts src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/runtime/remote-runtime-session-tabs-inflight.test.ts src/renderer/src/runtime/web-session-terminal-handle-events.test.ts src/renderer/src/store/slices/terminal-pty-identity-replacement.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts",
|
||||
"pnpm exec electron-vite build --mode e2e",
|
||||
"SKIP_BUILD=1 pnpm exec playwright test tests/e2e/terminal-cold-activation-deferral.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-cold-activation-restore.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1"
|
||||
|
|
@ -2438,9 +2441,19 @@
|
|||
"src/renderer/src/components/terminal-pane/use-terminal-pane-lifecycle.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/pty-connection.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx",
|
||||
"src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts",
|
||||
"src/renderer/src/runtime/remote-runtime-session-tabs-inflight.test.ts",
|
||||
"src/renderer/src/runtime/web-session-terminal-handle-events.test.ts",
|
||||
"src/renderer/src/store/slices/terminal-pty-identity-replacement.test.ts",
|
||||
"src/shared/remote-runtime-socket-liveness.test.ts",
|
||||
"src/shared/remote-runtime-shared-control-connection.test.ts",
|
||||
"src/shared/remote-runtime-shared-control-socket-generation.test.ts",
|
||||
"src/shared/remote-runtime-client-error-classification.test.ts",
|
||||
"src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts",
|
||||
"src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts",
|
||||
"src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts",
|
||||
"tests/e2e/terminal-cold-activation-deferral.spec.ts",
|
||||
"tests/e2e/ssh-cold-activation-restore.spec.ts"
|
||||
],
|
||||
|
|
@ -2463,7 +2476,53 @@
|
|||
"assertions": [
|
||||
"a stale web-mirror handle polls until a different ready handle is published without resubscribing the stale handle",
|
||||
"replacement does not emit pane exit or disconnect callbacks and explicit terminal exit still retires the mirror",
|
||||
"replacement polling and each in-flight request share a 15-second deadline, then accepted snapshots own recovery without input re-arming polling"
|
||||
"replacement polling and each in-flight request share a 15-second deadline, then accepted snapshots own recovery without input re-arming polling",
|
||||
"cold restored-terminal subscription failure retries and resumes snapshot, output, and input without a fatal error",
|
||||
"a canonical close before subscription readiness opens exactly one replacement stream without surfacing a fatal error",
|
||||
"recovery exposes connecting, recovering, backoff, connected, and fatal-offline phases with fenced epochs",
|
||||
"ten partition cycles retain one PTY identity, reject detached input, and unsubscribe each epoch exactly once",
|
||||
"cached terminal pixels remain disconnected until authoritative replay completes",
|
||||
"automatic retries stop after one minute with no fatal error, no further requests, and no stale input",
|
||||
"manual reconnect starts a new epoch, resubscribes the same PTY exactly once, and becomes healthy only after its snapshot",
|
||||
"unknown terminal-create recovery stops all request activity after one minute without a fatal error, clips post-probe RPC timeouts to the remaining budget, then manual retry re-probes capability and reconciles the same mutation",
|
||||
"a stale create completion cannot replace or close a newer attachment, including when two runtimes use the same raw handle",
|
||||
"an authoritative capability-probe failure replaces the stale connectivity error"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts",
|
||||
"assertions": [
|
||||
"a healthy replacement invalidates the prior recovery epoch so a slower failure cannot re-arm retry",
|
||||
"the automatic-recovery deadline cancels pending backoff and a manual retry owns a new epoch",
|
||||
"a caller-owned recovery cutoff cancels scheduled work and remains disconnected"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx",
|
||||
"assertions": [
|
||||
"automatic recovery renders as a quiet bounded status without a button",
|
||||
"the disconnected state exposes one explicit Reconnect action"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts",
|
||||
"assertions": [
|
||||
"only actionable recovery phases retain UI state and pane closure removes disconnected state"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/shared/remote-runtime-socket-liveness.test.ts",
|
||||
"assertions": [
|
||||
"a suspended client receives a fresh probe deadline after resume and post-resume activity clears it"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts",
|
||||
"assertions": [
|
||||
"one authenticated client/worktree mutation derives a stable server-owned terminal handle",
|
||||
"an unknown-outcome retry adopts the same provider PTY after runtime-process restart without rerunning startup",
|
||||
"retry inventory failure, missing same-worktree identity metadata, or ownership mismatch fails closed without spawning",
|
||||
"in-flight mutation coordination is bounded and releases capacity after settlement"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -2513,6 +2572,15 @@
|
|||
"result": "passed",
|
||||
"durationSeconds": 15,
|
||||
"summary": "Ten provider-contract files and 566 tests passed, including a shared 15-second polling/RPC deadline, no input-triggered re-polling after the bound, post-timeout snapshot recovery, listener cleanup, in-flight inventory deduplication, and replacement-state migration."
|
||||
},
|
||||
{
|
||||
"date": "2026-07-21",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts src/renderer/src/components/terminal-pane/TerminalRemoteRuntimeReconnectBanner.test.tsx src/renderer/src/components/terminal-pane/terminal-remote-runtime-recovery-ui-state.test.ts src/shared/remote-runtime-socket-liveness.test.ts src/shared/remote-runtime-shared-control-connection.test.ts src/shared/remote-runtime-shared-control-socket-generation.test.ts src/shared/remote-runtime-client-error-classification.test.ts src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 5,
|
||||
"summary": "Eleven fault-injection and recovery-UI files and 127 tests passed, covering suspended heartbeat clocks, stale socket, PTY, and create generations, canonical pre-ready close recovery with one replacement subscription, cold and repeated PTY reattachment, authoritative health, bounded PTY and terminal-create recovery, post-probe timeout clipping, manually retryable create cutoff, accurate capability-probe failures, same-PTY manual reconnect, pane-state cleanup, fatal error deduplication, stable create identity, cross-process PTY adoption, and fail-closed legacy inventory."
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
|
|
@ -2525,11 +2593,11 @@
|
|||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "partial",
|
||||
"evidence": "Tests assert SSH attach uses relay pty.attach for saved sessions, expired relay attach does not fresh-spawn in the provider, a failed SSH listProcesses observation does not clear previously learned ownership, a rejected SSH hasPty probe resolves as unknown liveness, deferred SSH passphrase cancellation does not auto-reconnect, deferred attach uses saved leaf/tab session ids once connected, transient deferred reattach failure preserves the saved session id without clearing pane/tab bindings or fresh-spawning, deferred SSH no-result cleanup clears the pending serializer without clearing pane/tab bindings or consuming the saved restore id, disconnected SSH relay ids are retained as deferred reconnect metadata and sidebar wake hints rather than attached PTY proof, expired deferred relay state clears stale pane/tab bindings before one fresh replacement spawn, and remote-runtime stale handles rotate in place without stale resubscription, pane retirement, or PTY index growth. Needs WSL, live remote-runtime/SSH, and saved intentional-break artifacts before promotion."
|
||||
"evidence": "The remote-runtime fault tests failed before the recovery changes by leaving a cold restored subscription detached, reporting connected before authoritative replay, delivering a fatal setup error twice, and allowing an unknown create outcome to spawn again after process-local state was lost. The bounded-recovery tests additionally failed before the policy change because PTY and terminal-create recovery remained active after one minute and a retry reused the stale epoch. Final review tests failed before lifecycle fencing because create cutoff emitted a fatal error with no manual path and a delayed create completion replaced a newer cross-runtime attachment. The fixed tests pass with resumed snapshot/output/input, one-minute cutoffs, new manual epochs, capability re-probing, stale-create rejection, and cross-process provider PTY adoption without rerunning startup. Existing SSH and stale-handle reattach coverage remains green. Needs WSL, a patched live remote-runtime partition journey, and saved intentional-break artifacts before promotion."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "Host eligibility is resolved from existing store metadata during activation. Remote-runtime resubscribe polling runs only after a transport close, reserves replacement-only matching for typed stale-handle evidence, backs off from 150 ms to 1 s, caps each request to the remaining 15-second reconnect deadline, and shares each in-flight inventory request across panes in the same runtime/worktree. Accepted host snapshots use an identity-scoped listener only while recovery is pending; after the bound there is no timer or global store subscriber, and stale input cannot re-arm polling. The listener is released on rebind, removal, detach, or destroy, and PTY state is atomically replaced instead of growing. The local journey preserves cold deferral, while SSH eagerly mounts six managers as required; no work is added to typing or terminal-output hot paths."
|
||||
"evidence": "Remote-runtime recovery allocates at most one backoff timer and one one-minute deadline per detached pane, then stops all PTY retry work until explicit user action; regular PTYs and initial creates do not poll inventory. Each unknown-outcome create attempt performs one bounded provider inventory scan, coordinated per authenticated client/worktree mutation, and the renderer stops issuing attempts after one minute. Timers, accepted-snapshot listeners, stale streams, and pane UI entries are released on health, cutoff, rebind, removal, detach, or destroy; ten-cycle tests prove one unsubscribe per epoch and cutoff tests prove request counts stay fixed for five additional minutes. Client and server liveness each use one interval per socket/transport. At most 4,096 create promises are retained only while in flight, and capacity rejection happens before spawning. Common terminal input/output paths add only constant-time state checks; recovery UI updates only on deduplicated phase transitions."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Use deterministic fake providers for failure and unknown-liveness cases.",
|
||||
|
|
@ -2540,7 +2608,8 @@
|
|||
"Current command covers store wake-hint metadata, main-process SSH provider failure semantics, provider attach/expired-attach behavior, and renderer deferred SSH reconnect/transient-failure/expired-relay fallback with mocked transports.",
|
||||
"The live SSH journey is environment-dependent and currently runs from a macOS Electron client against a Linux Docker host.",
|
||||
"WSL restore remains inferred rather than directly covered.",
|
||||
"Linux and Windows desktop-client journeys and a live remote-runtime reconnect soak are not yet collected."
|
||||
"Linux and Windows desktop-client partition journeys using patched builds are not yet collected; the Windows smoke proves current reachability and PTY round-trip only.",
|
||||
"Terminal-create recovery depends on providers authoritatively listing live terminal handles and worktree ownership; older runtimes do not advertise the capability and are never retried after an unknown outcome."
|
||||
],
|
||||
"demotionRule": "Cannot promote if provider failure can close panes or if the oracle is screenshot-only."
|
||||
},
|
||||
|
|
|
|||
|
|
@ -730,7 +730,12 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => {
|
|||
|
||||
describe('listProcesses', () => {
|
||||
it('returns active sessions', async () => {
|
||||
await adapter.spawn({ cols: 80, rows: 24, cwd: '/repo/owned-before-osc7' })
|
||||
await adapter.spawn({
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
cwd: '/repo/owned-before-osc7',
|
||||
worktreeId: 'repo::/repo/owned-before-osc7'
|
||||
})
|
||||
await adapter.spawn({ cols: 80, rows: 24 })
|
||||
|
||||
const procs = await adapter.listProcesses()
|
||||
|
|
@ -739,6 +744,7 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => {
|
|||
expect(procs[0]).toHaveProperty('cwd')
|
||||
expect(procs[0]).toHaveProperty('title')
|
||||
expect(procs[0].cwd).toBe('/repo/owned-before-osc7')
|
||||
expect(procs[0].worktreeId).toBe('repo::/repo/owned-before-osc7')
|
||||
})
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -773,13 +773,17 @@ export class DaemonPtyAdapter implements IPtyProvider {
|
|||
)
|
||||
return result.sessions
|
||||
.filter((s) => s.isAlive)
|
||||
.map((s) => ({
|
||||
id: s.sessionId,
|
||||
// Why: OSC 7 may not arrive before cleanup; spawn cwd is authoritative until the daemon reports a live cwd.
|
||||
cwd: s.cwd ?? this.initialCwds.get(s.sessionId) ?? '',
|
||||
title: 'shell',
|
||||
...(s.terminalHandle ? { terminalHandle: s.terminalHandle } : {})
|
||||
}))
|
||||
.map((s) => {
|
||||
const { worktreeId } = parsePtySessionId(s.sessionId)
|
||||
return {
|
||||
id: s.sessionId,
|
||||
// Why: OSC 7 may not arrive before cleanup; spawn cwd is authoritative until the daemon reports a live cwd.
|
||||
cwd: s.cwd ?? this.initialCwds.get(s.sessionId) ?? '',
|
||||
title: 'shell',
|
||||
...(worktreeId ? { worktreeId } : {}),
|
||||
...(s.terminalHandle ? { terminalHandle: s.terminalHandle } : {})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Why keep both: the Manage Sessions panel needs full SessionInfo (pid/state/createdAt) that listProcesses drops for the IPtyProvider contract.
|
||||
|
|
|
|||
|
|
@ -1758,7 +1758,12 @@ describe('LocalPtyProvider', () => {
|
|||
describe('listProcesses', () => {
|
||||
it('returns spawned PTYs', async () => {
|
||||
const before = await provider.listProcesses()
|
||||
await provider.spawn({ cols: 80, rows: 24, cwd: '/tmp/owned-cwd' })
|
||||
await provider.spawn({
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
cwd: '/tmp/owned-cwd',
|
||||
worktreeId: 'repo::/tmp/owned-cwd'
|
||||
})
|
||||
await provider.spawn({ cols: 80, rows: 24 })
|
||||
const after = await provider.listProcesses()
|
||||
expect(after.length - before.length).toBe(2)
|
||||
|
|
@ -1766,6 +1771,7 @@ describe('LocalPtyProvider', () => {
|
|||
expect(newEntries[0]).toHaveProperty('id')
|
||||
expect(newEntries[0]).toHaveProperty('title', 'zsh')
|
||||
expect(newEntries[0]).toHaveProperty('cwd', '/tmp/owned-cwd')
|
||||
expect(newEntries[0]).toHaveProperty('worktreeId', 'repo::/tmp/owned-cwd')
|
||||
})
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ const ptyAgentForegroundContextPaths = new Map<string, string[]>()
|
|||
// Why: remember the last recognized agent foreground so a degraded scan doesn't report the shell and look like an exit.
|
||||
const ptyLastRecognizedForeground = new Map<string, string>()
|
||||
const ptyTerminalHandle = new Map<string, string>()
|
||||
const ptyWorktreeId = new Map<string, string>()
|
||||
const ptyInitialCwd = new Map<string, string>()
|
||||
// Why: reattach carries current settings, not the live process's launch context; keep the first creator's WSL/native identity.
|
||||
const ptyWslDistroById = new Map<string, string | null>()
|
||||
|
|
@ -240,6 +241,7 @@ function clearPtyState(id: string): void {
|
|||
ptyAgentForegroundContextPaths.delete(id)
|
||||
ptyLastRecognizedForeground.delete(id)
|
||||
ptyTerminalHandle.delete(id)
|
||||
ptyWorktreeId.delete(id)
|
||||
ptyInitialCwd.delete(id)
|
||||
ptyWslDistroById.delete(id)
|
||||
ptyLoadGeneration.delete(id)
|
||||
|
|
@ -852,6 +854,9 @@ export class LocalPtyProvider implements IPtyProvider {
|
|||
if (finalEnv.ORCA_TERMINAL_HANDLE) {
|
||||
ptyTerminalHandle.set(id, finalEnv.ORCA_TERMINAL_HANDLE)
|
||||
}
|
||||
if (args.worktreeId) {
|
||||
ptyWorktreeId.set(id, args.worktreeId)
|
||||
}
|
||||
ptyAgentForegroundContextPaths.set(
|
||||
id,
|
||||
getAgentForegroundContextPaths({ cwd: args.cwd, worktreeId: args.worktreeId })
|
||||
|
|
@ -1306,6 +1311,7 @@ export class LocalPtyProvider implements IPtyProvider {
|
|||
id,
|
||||
cwd: ptyInitialCwd.get(id) ?? '',
|
||||
title: proc.process || ptyShellName.get(id) || 'shell',
|
||||
...(ptyWorktreeId.get(id) ? { worktreeId: ptyWorktreeId.get(id) } : {}),
|
||||
...(ptyTerminalHandle.get(id) ? { terminalHandle: ptyTerminalHandle.get(id) } : {})
|
||||
}))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,255 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { PtyProcessInfo } from '../providers/types'
|
||||
import type { RuntimeTerminalCreate } from '../../shared/runtime-types'
|
||||
import { OrcaRuntimeService } from './orca-runtime'
|
||||
import { deriveRemoteRuntimeTerminalCreateHandle } from './remote-runtime-terminal-create-identity'
|
||||
import { RemoteRuntimeTerminalCreateIdempotency } from './remote-runtime-terminal-create-idempotency'
|
||||
|
||||
type CreateRun = (
|
||||
canonicalWorktreeSelector: string | undefined,
|
||||
preAllocatedHandle: string | undefined
|
||||
) => Promise<RuntimeTerminalCreate>
|
||||
|
||||
function createRuntimeForDedupe(listProcesses = vi.fn(async (): Promise<PtyProcessInfo[]> => [])) {
|
||||
const handleByPtyId = new Map<string, string>()
|
||||
const runtime = Object.create(OrcaRuntimeService.prototype) as OrcaRuntimeService
|
||||
Object.assign(runtime, {
|
||||
terminalCreateIdempotency: new RemoteRuntimeTerminalCreateIdempotency(),
|
||||
ptyController: { listProcesses },
|
||||
resolveTerminalWorkspaceLaunchScope: vi.fn(async (selector: string) => ({
|
||||
id: selector.startsWith('id:') ? selector.slice(3) : selector
|
||||
})),
|
||||
adoptControllerTerminalHandle: vi.fn((ptyId: string, handle: string) => {
|
||||
handleByPtyId.set(ptyId, handle)
|
||||
}),
|
||||
recordPtyWorktree: vi.fn((ptyId: string, worktreeId: string, state: { title?: string }) => ({
|
||||
ptyId,
|
||||
worktreeId,
|
||||
title: state.title ?? null
|
||||
})),
|
||||
issuePtyHandle: vi.fn((pty: { ptyId: string }) => handleByPtyId.get(pty.ptyId))
|
||||
})
|
||||
return { runtime, listProcesses }
|
||||
}
|
||||
|
||||
function createdTerminal(handle: string, worktreeId = 'worktree-1'): RuntimeTerminalCreate {
|
||||
return { handle, worktreeId, title: null }
|
||||
}
|
||||
|
||||
describe('terminal create idempotency', () => {
|
||||
it('derives a stable handle isolated by authenticated client and canonical worktree', () => {
|
||||
const first = deriveRemoteRuntimeTerminalCreateHandle('device-a', 'worktree-1', 'mutation-1')
|
||||
|
||||
expect(deriveRemoteRuntimeTerminalCreateHandle('device-a', 'worktree-1', 'mutation-1')).toBe(
|
||||
first
|
||||
)
|
||||
expect(
|
||||
deriveRemoteRuntimeTerminalCreateHandle('device-b', 'worktree-1', 'mutation-1')
|
||||
).not.toBe(first)
|
||||
expect(
|
||||
deriveRemoteRuntimeTerminalCreateHandle('device-a', 'worktree-2', 'mutation-1')
|
||||
).not.toBe(first)
|
||||
expect(first).toMatch(/^term_[0-9a-f]{32}$/)
|
||||
})
|
||||
|
||||
it('shares an in-flight create without scanning inventory on the initial request', async () => {
|
||||
const { runtime, listProcesses } = createRuntimeForDedupe()
|
||||
let resolveCreate: (value: RuntimeTerminalCreate) => void = () => {}
|
||||
const pending = new Promise<RuntimeTerminalCreate>((resolve) => {
|
||||
resolveCreate = resolve
|
||||
})
|
||||
const create = vi.fn<CreateRun>(() => pending)
|
||||
|
||||
const first = runtime.dedupeTerminalCreate(
|
||||
'device-a',
|
||||
'id:worktree-1',
|
||||
'mutation-1',
|
||||
false,
|
||||
create
|
||||
)
|
||||
const retry = runtime.dedupeTerminalCreate(
|
||||
'device-a',
|
||||
'worktree-1',
|
||||
'mutation-1',
|
||||
false,
|
||||
create
|
||||
)
|
||||
await vi.waitFor(() => expect(create).toHaveBeenCalledTimes(1))
|
||||
const stableHandle = create.mock.calls[0][1]
|
||||
resolveCreate(createdTerminal(stableHandle ?? 'missing'))
|
||||
|
||||
await expect(Promise.all([first, retry])).resolves.toEqual([
|
||||
createdTerminal(stableHandle ?? 'missing'),
|
||||
createdTerminal(stableHandle ?? 'missing')
|
||||
])
|
||||
expect(listProcesses).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('adopts the original PTY after a runtime-process restart without rerunning startup', async () => {
|
||||
const liveSessions: PtyProcessInfo[] = []
|
||||
const firstRuntime = createRuntimeForDedupe(vi.fn(async () => liveSessions)).runtime
|
||||
const secondInventory = vi.fn(async () => liveSessions)
|
||||
const secondRuntime = createRuntimeForDedupe(secondInventory).runtime
|
||||
const startup = vi.fn<CreateRun>(async (_selector, handle) => {
|
||||
liveSessions.push({
|
||||
id: 'worktree-1@@session-a',
|
||||
cwd: '/workspace',
|
||||
title: 'pwsh',
|
||||
worktreeId: 'worktree-1',
|
||||
terminalHandle: handle
|
||||
})
|
||||
return createdTerminal(handle ?? 'missing')
|
||||
})
|
||||
|
||||
const first = await firstRuntime.dedupeTerminalCreate(
|
||||
'device-a',
|
||||
'id:worktree-1',
|
||||
'mutation-1',
|
||||
false,
|
||||
startup
|
||||
)
|
||||
const retrySpawn = vi.fn<CreateRun>()
|
||||
const recovered = await secondRuntime.dedupeTerminalCreate(
|
||||
'device-a',
|
||||
'worktree-1',
|
||||
'mutation-1',
|
||||
true,
|
||||
retrySpawn
|
||||
)
|
||||
|
||||
expect(recovered.handle).toBe(first.handle)
|
||||
expect(recovered.ptyId).toBe('worktree-1@@session-a')
|
||||
expect(startup).toHaveBeenCalledTimes(1)
|
||||
expect(retrySpawn).not.toHaveBeenCalled()
|
||||
expect(secondInventory).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('creates with the same stable handle after authoritative inventory proves absence', async () => {
|
||||
const { runtime, listProcesses } = createRuntimeForDedupe()
|
||||
const create = vi.fn<CreateRun>(async (_selector, handle) =>
|
||||
createdTerminal(handle ?? 'missing')
|
||||
)
|
||||
|
||||
const result = await runtime.dedupeTerminalCreate(
|
||||
'device-a',
|
||||
'id:worktree-1',
|
||||
'mutation-1',
|
||||
true,
|
||||
create
|
||||
)
|
||||
|
||||
expect(listProcesses).toHaveBeenCalledTimes(1)
|
||||
expect(create).toHaveBeenCalledWith('id:worktree-1', result.handle)
|
||||
})
|
||||
|
||||
it('fails safely without spawning when retry inventory is unavailable', async () => {
|
||||
const listProcesses = vi.fn(async (): Promise<PtyProcessInfo[]> => {
|
||||
throw new Error('controller offline')
|
||||
})
|
||||
const { runtime } = createRuntimeForDedupe(listProcesses)
|
||||
const create = vi.fn<CreateRun>()
|
||||
|
||||
await expect(
|
||||
runtime.dedupeTerminalCreate('device-a', 'id:worktree-1', 'mutation-1', true, create)
|
||||
).rejects.toThrow('runtime_unavailable')
|
||||
expect(create).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('fails safely when an older provider omits identity for a live same-worktree PTY', async () => {
|
||||
const { runtime } = createRuntimeForDedupe(
|
||||
vi.fn(async () => [
|
||||
{
|
||||
id: 'worktree-1@@legacy-session',
|
||||
cwd: '/workspace',
|
||||
title: 'shell',
|
||||
worktreeId: 'worktree-1'
|
||||
}
|
||||
])
|
||||
)
|
||||
const create = vi.fn<CreateRun>()
|
||||
|
||||
await expect(
|
||||
runtime.dedupeTerminalCreate('device-a', 'id:worktree-1', 'mutation-1', true, create)
|
||||
).rejects.toThrow('runtime_unavailable')
|
||||
expect(create).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('adopts an exact identity even when another legacy PTY lacks metadata', async () => {
|
||||
const handle = deriveRemoteRuntimeTerminalCreateHandle('device-a', 'worktree-1', 'mutation-1')
|
||||
const { runtime } = createRuntimeForDedupe(
|
||||
vi.fn(async () => [
|
||||
{
|
||||
id: 'worktree-1@@legacy-session',
|
||||
cwd: '/workspace',
|
||||
title: 'shell',
|
||||
worktreeId: 'worktree-1'
|
||||
},
|
||||
{
|
||||
id: 'worktree-1@@created-session',
|
||||
cwd: '/workspace',
|
||||
title: 'pwsh',
|
||||
worktreeId: 'worktree-1',
|
||||
terminalHandle: handle
|
||||
}
|
||||
])
|
||||
)
|
||||
|
||||
await expect(
|
||||
runtime.dedupeTerminalCreate(
|
||||
'device-a',
|
||||
'id:worktree-1',
|
||||
'mutation-1',
|
||||
true,
|
||||
vi.fn<CreateRun>()
|
||||
)
|
||||
).resolves.toMatchObject({ handle, ptyId: 'worktree-1@@created-session' })
|
||||
})
|
||||
|
||||
it('fails closed when a matching handle belongs to another worktree', async () => {
|
||||
const handle = deriveRemoteRuntimeTerminalCreateHandle('device-a', 'worktree-1', 'mutation-1')
|
||||
const { runtime } = createRuntimeForDedupe(
|
||||
vi.fn(async () => [
|
||||
{
|
||||
id: 'worktree-2@@session-a',
|
||||
cwd: '/other',
|
||||
title: 'shell',
|
||||
worktreeId: 'worktree-2',
|
||||
terminalHandle: handle
|
||||
}
|
||||
])
|
||||
)
|
||||
const create = vi.fn<CreateRun>()
|
||||
|
||||
await expect(
|
||||
runtime.dedupeTerminalCreate('device-a', 'id:worktree-1', 'mutation-1', true, create)
|
||||
).rejects.toThrow('terminal_create_identity_conflict')
|
||||
expect(create).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('bounds concurrent creates and releases capacity after settlement', async () => {
|
||||
const idempotency = new RemoteRuntimeTerminalCreateIdempotency(1)
|
||||
let resolveFirst: (value: RuntimeTerminalCreate) => void = () => {}
|
||||
const first = idempotency.run(
|
||||
'device-a',
|
||||
'worktree-1',
|
||||
'mutation-1',
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveFirst = resolve
|
||||
})
|
||||
)
|
||||
|
||||
await expect(
|
||||
idempotency.run('device-a', 'worktree-1', 'mutation-2', async () =>
|
||||
createdTerminal('terminal-2')
|
||||
)
|
||||
).rejects.toThrow('Too many terminal creations')
|
||||
resolveFirst(createdTerminal('terminal-1'))
|
||||
await first
|
||||
await expect(
|
||||
idempotency.run('device-a', 'worktree-1', 'mutation-2', async () =>
|
||||
createdTerminal('terminal-2')
|
||||
)
|
||||
).resolves.toEqual(createdTerminal('terminal-2'))
|
||||
})
|
||||
})
|
||||
|
|
@ -375,6 +375,8 @@ import type {
|
|||
} from '../../shared/runtime-types'
|
||||
import type { AutomationService } from '../automations/service'
|
||||
import { RuntimeBrowserCommands } from './orca-runtime-browser'
|
||||
import { RemoteRuntimeTerminalCreateIdempotency } from './remote-runtime-terminal-create-idempotency'
|
||||
import { deriveRemoteRuntimeTerminalCreateHandle } from './remote-runtime-terminal-create-identity'
|
||||
import { buildHeadlessTerminalSplitLayout } from './headless-terminal-split-layout'
|
||||
import { RECENT_PTY_OUTPUT_LIMIT, RecentPtyOutputBuffer } from './recent-pty-output-buffer'
|
||||
import {
|
||||
|
|
@ -1116,6 +1118,7 @@ type TerminalCreateOptions = {
|
|||
leafId?: string
|
||||
sessionId?: string
|
||||
persistHostSessionBinding?: boolean
|
||||
preAllocatedHandle?: string
|
||||
// Why: the headless mobile-session create publishes its own authoritative
|
||||
// snapshot (with the correct target group) right after spawn. Skip the
|
||||
// intermediate pty-backed publish so the new tab doesn't briefly flash in
|
||||
|
|
@ -2295,6 +2298,7 @@ export class OrcaRuntimeService {
|
|||
string,
|
||||
Promise<RuntimeMobileSessionCreateTerminalResult>
|
||||
>()
|
||||
private readonly terminalCreateIdempotency = new RemoteRuntimeTerminalCreateIdempotency()
|
||||
// Why: idempotency map for worktree.create — a create interrupted by a mobile
|
||||
// connection migration is retried with the same clientMutationId and returns
|
||||
// the in-flight (or just-finished) operation instead of a duplicate worktree.
|
||||
|
|
@ -19530,7 +19534,8 @@ export class OrcaRuntimeService {
|
|||
const launchOpts = await this.resolveAgentTerminalCreateOptions(workspace, opts)
|
||||
const cwd =
|
||||
this.resolveWorkspaceTerminalStartupCwd(workspace, launchOpts.cwd) ?? workspace.path
|
||||
const preAllocatedHandle = this.createPreAllocatedTerminalHandle()
|
||||
const preAllocatedHandle =
|
||||
launchOpts.preAllocatedHandle ?? this.createPreAllocatedTerminalHandle()
|
||||
// Why: mint tabId in main before spawn so paneKey is known at PTY env
|
||||
// build time. Hook-based agent status (Claude/Codex/Cursor/Gemini) keys
|
||||
// off `${tabId}:${leafId}` — without these vars set on the PTY, the
|
||||
|
|
@ -19804,6 +19809,100 @@ export class OrcaRuntimeService {
|
|||
}
|
||||
}
|
||||
|
||||
async dedupeTerminalCreate(
|
||||
clientIdentity: string,
|
||||
worktreeSelector: string | undefined,
|
||||
clientMutationId: string | undefined,
|
||||
reconcileExisting: boolean,
|
||||
run: (
|
||||
canonicalWorktreeSelector: string | undefined,
|
||||
preAllocatedHandle: string | undefined
|
||||
) => Promise<RuntimeTerminalCreate>
|
||||
): Promise<RuntimeTerminalCreate> {
|
||||
if (!clientMutationId || !worktreeSelector) {
|
||||
if (reconcileExisting) {
|
||||
throw new Error('runtime_unavailable')
|
||||
}
|
||||
return await run(worktreeSelector, undefined)
|
||||
}
|
||||
const workspace = await this.resolveTerminalWorkspaceLaunchScope(worktreeSelector)
|
||||
const canonicalWorktreeSelector = `id:${workspace.id}`
|
||||
const preAllocatedHandle = deriveRemoteRuntimeTerminalCreateHandle(
|
||||
clientIdentity,
|
||||
workspace.id,
|
||||
clientMutationId
|
||||
)
|
||||
return this.terminalCreateIdempotency.run(
|
||||
clientIdentity,
|
||||
workspace.id,
|
||||
clientMutationId,
|
||||
async () => {
|
||||
if (reconcileExisting) {
|
||||
const adopted = await this.reconcileRemoteTerminalCreate(workspace.id, preAllocatedHandle)
|
||||
if (adopted) {
|
||||
return adopted
|
||||
}
|
||||
}
|
||||
return await run(canonicalWorktreeSelector, preAllocatedHandle)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private async reconcileRemoteTerminalCreate(
|
||||
worktreeId: string,
|
||||
terminalHandle: string
|
||||
): Promise<RuntimeTerminalCreate | null> {
|
||||
if (!this.ptyController?.listProcesses) {
|
||||
throw new Error('runtime_unavailable')
|
||||
}
|
||||
const listed = await withTimeoutResult(
|
||||
this.ptyController.listProcesses(),
|
||||
PTY_CONTROLLER_LIST_TIMEOUT_MS
|
||||
)
|
||||
if (!listed.ok) {
|
||||
// Why: unknown inventory cannot prove the first create failed, so spawning could duplicate a live shell.
|
||||
throw new Error('runtime_unavailable')
|
||||
}
|
||||
const matches = listed.value.filter((session) => session.terminalHandle === terminalHandle)
|
||||
if (matches.length > 1) {
|
||||
throw new Error('terminal_create_identity_conflict')
|
||||
}
|
||||
if (matches.length === 0) {
|
||||
const sameWorktreeHasUnknownIdentity = listed.value.some(
|
||||
(session) =>
|
||||
(session.worktreeId ?? inferWorktreeIdFromPtyId(session.id)) === worktreeId &&
|
||||
!session.terminalHandle
|
||||
)
|
||||
if (sameWorktreeHasUnknownIdentity) {
|
||||
// Why: older retained providers may list the first shell without its handle; absence is not authoritative in that shape.
|
||||
throw new Error('runtime_unavailable')
|
||||
}
|
||||
return null
|
||||
}
|
||||
const session = matches[0]
|
||||
const authoritativeWorktreeId = session.worktreeId ?? inferWorktreeIdFromPtyId(session.id)
|
||||
if (authoritativeWorktreeId !== worktreeId) {
|
||||
// Why: a reused address or forged provider record must never adopt a PTY from another workspace.
|
||||
throw new Error('terminal_create_identity_conflict')
|
||||
}
|
||||
this.adoptControllerTerminalHandle(session.id, terminalHandle)
|
||||
const pty = this.recordPtyWorktree(session.id, worktreeId, {
|
||||
connected: true,
|
||||
title: session.title
|
||||
})
|
||||
const adoptedHandle = this.issuePtyHandle(pty)
|
||||
if (adoptedHandle !== terminalHandle) {
|
||||
throw new Error('terminal_create_identity_conflict')
|
||||
}
|
||||
return {
|
||||
handle: adoptedHandle,
|
||||
ptyId: session.id,
|
||||
worktreeId,
|
||||
title: session.title || null,
|
||||
surface: 'background'
|
||||
}
|
||||
}
|
||||
|
||||
async launchAgentTerminal(
|
||||
worktreeSelector: string,
|
||||
opts: { agent: TuiAgent; prompt: string; title?: string }
|
||||
|
|
|
|||
|
|
@ -0,0 +1,37 @@
|
|||
import type { RuntimeTerminalCreate } from '../../shared/runtime-types'
|
||||
|
||||
const DEFAULT_MAX_IN_FLIGHT_TERMINAL_CREATES = 4_096
|
||||
|
||||
export class RemoteRuntimeTerminalCreateIdempotency {
|
||||
private readonly inFlight = new Map<string, Promise<RuntimeTerminalCreate>>()
|
||||
|
||||
constructor(private readonly maxInFlight = DEFAULT_MAX_IN_FLIGHT_TERMINAL_CREATES) {}
|
||||
|
||||
run(
|
||||
clientIdentity: string,
|
||||
worktreeId: string,
|
||||
clientMutationId: string,
|
||||
create: () => Promise<RuntimeTerminalCreate>
|
||||
): Promise<RuntimeTerminalCreate> {
|
||||
const key = `${clientIdentity}\0${worktreeId}\0${clientMutationId}`
|
||||
const existing = this.inFlight.get(key)
|
||||
if (existing) {
|
||||
return existing
|
||||
}
|
||||
if (this.inFlight.size >= this.maxInFlight) {
|
||||
return Promise.reject(
|
||||
new Error('Too many terminal creations are still pending; retry after they settle.')
|
||||
)
|
||||
}
|
||||
|
||||
const promise = create()
|
||||
this.inFlight.set(key, promise)
|
||||
const drop = (): void => {
|
||||
if (this.inFlight.get(key) === promise) {
|
||||
this.inFlight.delete(key)
|
||||
}
|
||||
}
|
||||
void promise.then(drop, drop)
|
||||
return promise
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
import { createHash } from 'node:crypto'
|
||||
|
||||
export function deriveRemoteRuntimeTerminalCreateHandle(
|
||||
clientIdentity: string,
|
||||
worktreeId: string,
|
||||
clientMutationId: string
|
||||
): string {
|
||||
const digest = createHash('sha256')
|
||||
.update('orca.remote-terminal-create.v2\0')
|
||||
.update(clientIdentity)
|
||||
.update('\0')
|
||||
.update(worktreeId)
|
||||
.update('\0')
|
||||
.update(clientMutationId)
|
||||
.digest('hex')
|
||||
return `term_${digest.slice(0, 32)}`
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RpcContext } from '../core'
|
||||
import { TERMINAL_METHODS } from './terminal'
|
||||
|
||||
describe('terminal.create RPC idempotency', () => {
|
||||
it('scopes the mutation key to the authenticated paired device and worktree', async () => {
|
||||
const terminal = { handle: 'terminal-1', worktreeId: 'worktree-1', title: null }
|
||||
const createTerminal = vi.fn(async () => terminal)
|
||||
const dedupeTerminalCreate = vi.fn(
|
||||
async (
|
||||
_clientIdentity: string,
|
||||
_worktree: string | undefined,
|
||||
_mutationId: string | undefined,
|
||||
_reconcileExisting: boolean,
|
||||
run: (worktree: string | undefined, handle: string | undefined) => Promise<typeof terminal>
|
||||
) => run('id:worktree-1', 'term_stable')
|
||||
)
|
||||
const method = TERMINAL_METHODS.find((candidate) => candidate.name === 'terminal.create')
|
||||
if (!method) {
|
||||
throw new Error('terminal.create method missing')
|
||||
}
|
||||
|
||||
const result = await method.handler(
|
||||
{
|
||||
worktree: 'id:worktree-1',
|
||||
clientMutationId: 'mutation-1',
|
||||
command: 'pwsh',
|
||||
resumeProviderSession: {
|
||||
key: 'session_id',
|
||||
id: 'session-1',
|
||||
transcriptPath: 'C:\\Users\\example\\.codex\\sessions\\rollout.jsonl'
|
||||
}
|
||||
},
|
||||
{
|
||||
runtime: { createTerminal, dedupeTerminalCreate },
|
||||
pairedDeviceId: 'device-a',
|
||||
clientId: 'bearer-token'
|
||||
} as unknown as RpcContext,
|
||||
vi.fn()
|
||||
)
|
||||
|
||||
expect(dedupeTerminalCreate).toHaveBeenCalledWith(
|
||||
'device-a',
|
||||
'id:worktree-1',
|
||||
'mutation-1',
|
||||
false,
|
||||
expect.any(Function)
|
||||
)
|
||||
expect(createTerminal).toHaveBeenCalledWith(
|
||||
'id:worktree-1',
|
||||
expect.objectContaining({
|
||||
command: 'pwsh',
|
||||
preAllocatedHandle: 'term_stable',
|
||||
resumeProviderSession: {
|
||||
key: 'session_id',
|
||||
id: 'session-1',
|
||||
transcriptPath: 'C:\\Users\\example\\.codex\\sessions\\rollout.jsonl'
|
||||
}
|
||||
})
|
||||
)
|
||||
expect(result).toEqual({ terminal })
|
||||
})
|
||||
})
|
||||
|
|
@ -910,6 +910,8 @@ const TerminalWait = TerminalHandle.extend({
|
|||
|
||||
const TerminalCreateParams = z.object({
|
||||
worktree: OptionalString,
|
||||
clientMutationId: z.string().min(1).max(128).optional(),
|
||||
reconcileExisting: z.boolean().optional(),
|
||||
command: OptionalString,
|
||||
startupCommandDelivery: z.enum(['fast', 'shell-ready']).optional(),
|
||||
env: z.record(z.string(), z.string()).optional(),
|
||||
|
|
@ -1362,29 +1364,37 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
|||
defineMethod({
|
||||
name: 'terminal.create',
|
||||
params: TerminalCreateParams,
|
||||
handler: async (params, { runtime }) => ({
|
||||
terminal: await runtime.createTerminal(params.worktree, {
|
||||
command: params.command,
|
||||
startupCommandDelivery: params.startupCommandDelivery,
|
||||
env: params.env,
|
||||
envToDelete: params.envToDelete,
|
||||
...(params.launchConfig ? { launchConfig: params.launchConfig } : {}),
|
||||
...(params.resumeProviderSession
|
||||
? { resumeProviderSession: params.resumeProviderSession }
|
||||
: {}),
|
||||
...(params.launchToken ? { launchToken: params.launchToken } : {}),
|
||||
...(params.launchAgent ? { launchAgent: params.launchAgent } : {}),
|
||||
...(params.terminalColorQueryReplies
|
||||
? { terminalColorQueryReplies: params.terminalColorQueryReplies }
|
||||
: {}),
|
||||
title: params.title,
|
||||
focus: params.focus === true,
|
||||
rendererBacked: params.rendererBacked === true,
|
||||
activate: params.activate === true,
|
||||
presentation: params.presentation,
|
||||
tabId: params.tabId,
|
||||
leafId: params.leafId
|
||||
})
|
||||
handler: async (params, { runtime, pairedDeviceId, clientId }) => ({
|
||||
terminal: await runtime.dedupeTerminalCreate(
|
||||
pairedDeviceId ?? clientId ?? 'local',
|
||||
params.worktree,
|
||||
params.clientMutationId,
|
||||
params.reconcileExisting === true,
|
||||
(canonicalWorktreeSelector, preAllocatedHandle) =>
|
||||
runtime.createTerminal(canonicalWorktreeSelector, {
|
||||
command: params.command,
|
||||
startupCommandDelivery: params.startupCommandDelivery,
|
||||
env: params.env,
|
||||
envToDelete: params.envToDelete,
|
||||
...(params.launchConfig ? { launchConfig: params.launchConfig } : {}),
|
||||
...(params.resumeProviderSession
|
||||
? { resumeProviderSession: params.resumeProviderSession }
|
||||
: {}),
|
||||
...(params.launchToken ? { launchToken: params.launchToken } : {}),
|
||||
...(params.launchAgent ? { launchAgent: params.launchAgent } : {}),
|
||||
...(params.terminalColorQueryReplies
|
||||
? { terminalColorQueryReplies: params.terminalColorQueryReplies }
|
||||
: {}),
|
||||
title: params.title,
|
||||
focus: params.focus === true,
|
||||
rendererBacked: params.rendererBacked === true,
|
||||
activate: params.activate === true,
|
||||
presentation: params.presentation,
|
||||
tabId: params.tabId,
|
||||
leafId: params.leafId,
|
||||
...(preAllocatedHandle ? { preAllocatedHandle } : {})
|
||||
})
|
||||
)
|
||||
})
|
||||
}),
|
||||
defineMethod({
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { WebSocket } from 'ws'
|
||||
import { RemoteRuntimeServerHeartbeat } from './remote-runtime-server-heartbeat'
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
describe('RemoteRuntimeServerHeartbeat', () => {
|
||||
it('grants clients a fresh probe after the server event loop resumes', async () => {
|
||||
vi.useFakeTimers()
|
||||
let now = 1_000
|
||||
const socket = { ping: vi.fn(), terminate: vi.fn() } as unknown as WebSocket
|
||||
const heartbeat = new RemoteRuntimeServerHeartbeat(100, () => now)
|
||||
heartbeat.noteAlive(socket)
|
||||
heartbeat.start(() => [socket])
|
||||
|
||||
now += 100
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
now += 3_600_000
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
|
||||
expect(socket.ping).toHaveBeenCalledTimes(2)
|
||||
expect(socket.terminate).not.toHaveBeenCalled()
|
||||
|
||||
now += 100
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
expect(socket.terminate).toHaveBeenCalledTimes(1)
|
||||
heartbeat.stop()
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import type { WebSocket } from 'ws'
|
||||
|
||||
export class RemoteRuntimeServerHeartbeat {
|
||||
private timer: ReturnType<typeof setInterval> | null = null
|
||||
private lastTickAt: number | null = null
|
||||
private readonly alive = new WeakSet<WebSocket>()
|
||||
|
||||
constructor(
|
||||
private readonly intervalMs: number,
|
||||
private readonly now: () => number = Date.now,
|
||||
private readonly warningClientCount = 128
|
||||
) {}
|
||||
|
||||
noteAlive(socket: WebSocket): void {
|
||||
this.alive.add(socket)
|
||||
}
|
||||
|
||||
start(getClients: () => Iterable<WebSocket>): void {
|
||||
if (this.timer) {
|
||||
return
|
||||
}
|
||||
this.lastTickAt = this.now()
|
||||
this.timer = setInterval(() => this.sweep(getClients()), this.intervalMs)
|
||||
this.timer.unref?.()
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (this.timer) {
|
||||
clearInterval(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
this.lastTickAt = null
|
||||
}
|
||||
|
||||
private sweep(clients: Iterable<WebSocket>): void {
|
||||
const tickAt = this.now()
|
||||
const elapsedMs = tickAt - (this.lastTickAt ?? tickAt)
|
||||
this.lastTickAt = tickAt
|
||||
const resumedFromPause = elapsedMs < 0 || elapsedMs > this.intervalMs * 1.5
|
||||
let reaped = 0
|
||||
let clientCount = 0
|
||||
for (const socket of clients) {
|
||||
clientCount += 1
|
||||
if (resumedFromPause) {
|
||||
// Why: a delayed server tick cannot infer that clients missed a probe they had no chance to answer.
|
||||
this.alive.add(socket)
|
||||
}
|
||||
if (!this.alive.has(socket)) {
|
||||
socket.terminate()
|
||||
reaped += 1
|
||||
continue
|
||||
}
|
||||
this.alive.delete(socket)
|
||||
try {
|
||||
socket.ping()
|
||||
} catch {
|
||||
// Why: a mid-teardown socket is finalized by its close/error listener.
|
||||
}
|
||||
}
|
||||
if (reaped > 0 || clientCount >= this.warningClientCount) {
|
||||
console.warn(`[ws-transport] heartbeat reaped ${reaped}; ${clientCount} tracked sockets`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@ import { createServer as createHttpServer, type Server as HttpServer } from 'nod
|
|||
import { WebSocketServer, type WebSocket } from 'ws'
|
||||
import type { RpcTransport } from './transport'
|
||||
import { createStaticWebClientHandler } from './static-web-client-handler'
|
||||
import { RemoteRuntimeServerHeartbeat } from './remote-runtime-server-heartbeat'
|
||||
|
||||
const MAX_WS_MESSAGE_BYTES = 1024 * 1024
|
||||
// Why: one desktop remote-host client can hold many concurrent streams, so keep the cap high enough that stale streams don't starve control RPCs.
|
||||
|
|
@ -30,6 +31,8 @@ export type WebSocketTransportOptions = {
|
|||
tlsKey?: string
|
||||
// Why: test-only override. Production uses HEARTBEAT_INTERVAL_MS.
|
||||
heartbeatIntervalMs?: number
|
||||
// Why: deterministic suspension tests must advance wall time independently from timer callbacks.
|
||||
heartbeatNow?: () => number
|
||||
// Why: test-only override. Production uses PRE_AUTH_TIMEOUT_MS.
|
||||
preAuthTimeoutMs?: number
|
||||
// Why: the pairing server can also serve the browser client, avoiding a second static server.
|
||||
|
|
@ -45,16 +48,13 @@ export class WebSocketTransport implements RpcTransport {
|
|||
private readonly port: number
|
||||
private readonly tlsCert: string | undefined
|
||||
private readonly tlsKey: string | undefined
|
||||
private readonly heartbeatIntervalMs: number
|
||||
private readonly heartbeat: RemoteRuntimeServerHeartbeat
|
||||
private readonly preAuthTimeoutMs: number
|
||||
private readonly staticRoot: string | undefined
|
||||
private readonly fallbackPort: number | undefined
|
||||
private readonly preferPinnedPort: boolean
|
||||
private httpServer: HttpsServer | HttpServer | null = null
|
||||
private wss: WebSocketServer | null = null
|
||||
private heartbeatTimer: ReturnType<typeof setInterval> | null = null
|
||||
// Why: a socket absent from this set at the next heartbeat sweep is presumed dead and terminated.
|
||||
private wsAlive = new WeakSet<WebSocket>()
|
||||
private messageHandler: WebSocketMessageHandler | null = null
|
||||
private connectionCloseHandler:
|
||||
| ((clientId: string | null, ws: WebSocket, hasOtherConnections: boolean) => void)
|
||||
|
|
@ -69,6 +69,7 @@ export class WebSocketTransport implements RpcTransport {
|
|||
tlsCert,
|
||||
tlsKey,
|
||||
heartbeatIntervalMs,
|
||||
heartbeatNow,
|
||||
preAuthTimeoutMs,
|
||||
staticRoot,
|
||||
fallbackPort,
|
||||
|
|
@ -78,7 +79,11 @@ export class WebSocketTransport implements RpcTransport {
|
|||
this.port = port
|
||||
this.tlsCert = tlsCert
|
||||
this.tlsKey = tlsKey
|
||||
this.heartbeatIntervalMs = heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS
|
||||
this.heartbeat = new RemoteRuntimeServerHeartbeat(
|
||||
heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS,
|
||||
heartbeatNow,
|
||||
MAX_WS_CONNECTIONS
|
||||
)
|
||||
this.preAuthTimeoutMs = preAuthTimeoutMs ?? PRE_AUTH_TIMEOUT_MS
|
||||
this.staticRoot = staticRoot
|
||||
this.fallbackPort = fallbackPort
|
||||
|
|
@ -194,7 +199,7 @@ export class WebSocketTransport implements RpcTransport {
|
|||
|
||||
this.httpServer = httpServer
|
||||
this.wss = wss
|
||||
this.startHeartbeat()
|
||||
this.heartbeat.start(() => this.wss?.clients ?? [])
|
||||
}
|
||||
|
||||
// Why: force-terminate soon after the 1013 close since a half-open phone may never ack and would hold the descriptor past the WS cap; the 'error' listener absorbs a reset while closing.
|
||||
|
|
@ -206,56 +211,12 @@ export class WebSocketTransport implements RpcTransport {
|
|||
ws.once('close', () => clearTimeout(terminateTimer))
|
||||
}
|
||||
|
||||
// Why: the only reliable reaper of half-open mobile sockets stranded by background suspension without a TCP FIN.
|
||||
private startHeartbeat(): void {
|
||||
if (this.heartbeatTimer) {
|
||||
return
|
||||
}
|
||||
this.heartbeatTimer = setInterval(() => {
|
||||
const wss = this.wss
|
||||
if (!wss) {
|
||||
return
|
||||
}
|
||||
let reaped = 0
|
||||
for (const ws of wss.clients) {
|
||||
if (!this.wsAlive.has(ws)) {
|
||||
// Why: terminate() frees the slot immediately; close() on a dead socket can hang for the OS-level TCP timeout.
|
||||
ws.terminate()
|
||||
reaped++
|
||||
continue
|
||||
}
|
||||
this.wsAlive.delete(ws)
|
||||
try {
|
||||
ws.ping()
|
||||
} catch {
|
||||
// Why: ping() can throw on a mid-teardown socket; the close handler runs regardless, so swallow it.
|
||||
}
|
||||
}
|
||||
// Why: steady reaping or riding the cap are early overload signals; stay quiet on healthy ticks.
|
||||
if (reaped > 0 || wss.clients.size >= MAX_WS_CONNECTIONS) {
|
||||
console.warn(
|
||||
`[ws-transport] heartbeat reaped ${reaped}; ${wss.clients.size} tracked sockets`
|
||||
)
|
||||
}
|
||||
}, this.heartbeatIntervalMs)
|
||||
if (typeof this.heartbeatTimer.unref === 'function') {
|
||||
this.heartbeatTimer.unref()
|
||||
}
|
||||
}
|
||||
|
||||
private stopHeartbeat(): void {
|
||||
if (this.heartbeatTimer) {
|
||||
clearInterval(this.heartbeatTimer)
|
||||
this.heartbeatTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
const wss = this.wss
|
||||
const httpServer = this.httpServer
|
||||
this.wss = null
|
||||
this.httpServer = null
|
||||
this.stopHeartbeat()
|
||||
this.heartbeat.stop()
|
||||
|
||||
if (wss) {
|
||||
for (const client of wss.clients) {
|
||||
|
|
@ -282,11 +243,11 @@ export class WebSocketTransport implements RpcTransport {
|
|||
private handleConnection(ws: WebSocket): void {
|
||||
let finalized = false
|
||||
const onPong = (): void => {
|
||||
this.wsAlive.add(ws)
|
||||
this.heartbeat.noteAlive(ws)
|
||||
}
|
||||
const onMessage = (data: WebSocket.RawData, isBinary: boolean): void => {
|
||||
// Why: any inbound frame counts as proof of life, so an actively-talking client isn't reaped mid-request.
|
||||
this.wsAlive.add(ws)
|
||||
this.heartbeat.noteAlive(ws)
|
||||
const msg =
|
||||
typeof data === 'string'
|
||||
? data
|
||||
|
|
@ -338,7 +299,7 @@ export class WebSocketTransport implements RpcTransport {
|
|||
this.preAuthTimers.set(ws, preAuthTimer)
|
||||
|
||||
// Why: seed alive so the first heartbeat tick doesn't reap a fresh socket before its first pong.
|
||||
this.wsAlive.add(ws)
|
||||
this.heartbeat.noteAlive(ws)
|
||||
|
||||
ws.on('pong', onPong)
|
||||
ws.on('message', onMessage)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import type {
|
|||
} from '@/lib/pane-manager/pane-manager'
|
||||
import TerminalSearch from '@/components/TerminalSearch'
|
||||
import type { PtyTransport } from './pty-transport'
|
||||
import type { PtyTransportRecoveryState } from './pty-transport-types'
|
||||
import { fitPanes, isWindowsUserAgent } from './pane-helpers'
|
||||
import { getConnectionId, getConnectionIdFromState } from '@/lib/connection-context'
|
||||
import {
|
||||
|
|
@ -155,7 +156,12 @@ import { scheduleImagePasteWebglAtlasRecovery } from './terminal-webgl-atlas-rec
|
|||
import { restoreTerminalFitToDesktop, restoreTerminalFitsToDesktop } from './terminal-fit-restore'
|
||||
import { useVisibleTerminalTabClaim } from './use-visible-terminal-tab-claim'
|
||||
import { TerminalSshReconnectOverlay } from './TerminalSshReconnectOverlay'
|
||||
import { TerminalRemoteRuntimeReconnectBanner } from './TerminalRemoteRuntimeReconnectBanner'
|
||||
import { selectTerminalTabAgentTypesByLeaf } from './terminal-tab-agent-type-index'
|
||||
import {
|
||||
updateTerminalRemoteRuntimeRecoveryUiState,
|
||||
type VisiblePtyRecoveryState
|
||||
} from './terminal-remote-runtime-recovery-ui-state'
|
||||
|
||||
const NATIVE_CHAT_ROOT_SELECTOR = '[data-native-chat-root="true"]'
|
||||
|
||||
|
|
@ -376,6 +382,9 @@ export default function TerminalPane({
|
|||
const [quickCommandDraft, setQuickCommandDraft] = useState(createTerminalQuickCommandDraft)
|
||||
const [agentSessionFork, setAgentSessionFork] = useState<PreparedAgentSessionFork | null>(null)
|
||||
const [terminalError, setTerminalError] = useState<string | null>(null)
|
||||
const [ptyRecoveryStatesByPaneId, setPtyRecoveryStatesByPaneId] = useState<
|
||||
Record<number, VisiblePtyRecoveryState>
|
||||
>({})
|
||||
const [sessionStateSaveFailureOpen, setSessionStateSaveFailureOpen] = useState(false)
|
||||
const daemonActions = useDaemonActions()
|
||||
// Why: override state lives in a Map for perf; this counter forces a re-render on override change so the mobile-fit banner toggles.
|
||||
|
|
@ -558,6 +567,13 @@ export default function TerminalPane({
|
|||
}
|
||||
setTerminalError((prev) => (prev ? `${prev}\n${message}` : message))
|
||||
})
|
||||
const onPtyRecoveryStateRef = useRef(
|
||||
(paneId: number, state: PtyTransportRecoveryState | null) => {
|
||||
setPtyRecoveryStatesByPaneId((previous) =>
|
||||
updateTerminalRemoteRuntimeRecoveryUiState(previous, paneId, state)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
const setTabPaneExpanded = useAppStore((store) => store.setTabPaneExpanded)
|
||||
const setTabCanExpandPane = useAppStore((store) => store.setTabCanExpandPane)
|
||||
|
|
@ -1406,6 +1422,7 @@ export default function TerminalPane({
|
|||
onPtyExitRef,
|
||||
onAgentExitedRef,
|
||||
onPtyErrorRef,
|
||||
onPtyRecoveryStateRef,
|
||||
clearTabPtyId,
|
||||
consumeSuppressedPtyExit: useAppStore((store) => store.consumeSuppressedPtyExit),
|
||||
updateTabTitle,
|
||||
|
|
@ -1608,6 +1625,7 @@ export default function TerminalPane({
|
|||
onPtyExitRef,
|
||||
onAgentExitedRef,
|
||||
onPtyErrorRef,
|
||||
onPtyRecoveryStateRef,
|
||||
clearTabPtyId,
|
||||
consumeSuppressedPtyExit: useAppStore.getState().consumeSuppressedPtyExit,
|
||||
updateTabTitle,
|
||||
|
|
@ -3014,6 +3032,25 @@ export default function TerminalPane({
|
|||
onRenameCancel={handleRenameCancel}
|
||||
onRenameBlur={handleRenameBlur}
|
||||
/>
|
||||
{!showSshReconnectOverlay
|
||||
? managedPanes.map((pane) => {
|
||||
const recoveryState = ptyRecoveryStatesByPaneId[pane.id]
|
||||
if (!recoveryState) {
|
||||
return null
|
||||
}
|
||||
return createPortal(
|
||||
<TerminalRemoteRuntimeReconnectBanner
|
||||
key={`remote-runtime-reconnect-${pane.id}-${recoveryState.epoch}`}
|
||||
phase={recoveryState.phase}
|
||||
onReconnect={() => {
|
||||
paneTransportsRef.current.get(pane.id)?.retryRecovery?.()
|
||||
}}
|
||||
/>,
|
||||
pane.container,
|
||||
`remote-runtime-reconnect-${pane.id}`
|
||||
)
|
||||
})
|
||||
: null}
|
||||
{managedPanes.map((pane) => {
|
||||
// Why: pane IDs collide across tabs, so key overlays by the transport's actual ptyId to avoid wrong-pane banners.
|
||||
const ptyId = paneTransportsRef.current.get(pane.id)?.getPtyId()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
// @vitest-environment happy-dom
|
||||
|
||||
import '@testing-library/jest-dom/vitest'
|
||||
import { cleanup, render, screen } from '@testing-library/react'
|
||||
import userEvent from '@testing-library/user-event'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { TerminalRemoteRuntimeReconnectBanner } from './TerminalRemoteRuntimeReconnectBanner'
|
||||
|
||||
vi.mock('@/i18n/i18n', () => ({
|
||||
translate: (_key: string, fallback: string) => fallback
|
||||
}))
|
||||
|
||||
afterEach(cleanup)
|
||||
|
||||
describe('TerminalRemoteRuntimeReconnectBanner', () => {
|
||||
it('shows quiet bounded automatic recovery without a manual action', () => {
|
||||
render(<TerminalRemoteRuntimeReconnectBanner phase="backoff" onReconnect={vi.fn()} />)
|
||||
|
||||
expect(screen.getByText('Reconnecting to remote runtime')).toBeInTheDocument()
|
||||
expect(screen.getByText(/retry for up to one minute/)).toBeInTheDocument()
|
||||
expect(screen.queryByRole('button')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('offers one explicit reconnect action after automatic recovery stops', async () => {
|
||||
const onReconnect = vi.fn()
|
||||
const user = userEvent.setup()
|
||||
render(<TerminalRemoteRuntimeReconnectBanner phase="disconnected" onReconnect={onReconnect} />)
|
||||
|
||||
expect(screen.getByText('Remote runtime disconnected')).toBeInTheDocument()
|
||||
await user.click(screen.getByRole('button', { name: 'Reconnect' }))
|
||||
expect(onReconnect).toHaveBeenCalledOnce()
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
import { Loader2, ServerOff } from 'lucide-react'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { translate } from '@/i18n/i18n'
|
||||
import type { PtyTransportRecoveryState } from './pty-transport-types'
|
||||
|
||||
type VisibleRecoveryPhase = Extract<
|
||||
PtyTransportRecoveryState['phase'],
|
||||
'recovering' | 'backoff' | 'disconnected'
|
||||
>
|
||||
|
||||
export function TerminalRemoteRuntimeReconnectBanner({
|
||||
phase,
|
||||
onReconnect
|
||||
}: {
|
||||
phase: VisibleRecoveryPhase
|
||||
onReconnect: () => void
|
||||
}): React.JSX.Element {
|
||||
const retrying = phase !== 'disconnected'
|
||||
|
||||
return (
|
||||
<div
|
||||
className="pointer-events-none absolute inset-x-3 bottom-3 z-30 flex justify-center"
|
||||
data-terminal-remote-runtime-reconnect-banner={phase}
|
||||
>
|
||||
<div
|
||||
className="pointer-events-auto flex w-full max-w-xl items-center gap-3 rounded-md border border-border bg-card/95 px-3 py-3 text-card-foreground shadow-xs backdrop-blur-[1px]"
|
||||
role="status"
|
||||
aria-live="polite"
|
||||
>
|
||||
<div className="flex size-8 shrink-0 items-center justify-center rounded-md border border-border bg-muted text-muted-foreground">
|
||||
{retrying ? (
|
||||
<Loader2 className="size-4 animate-spin" />
|
||||
) : (
|
||||
<ServerOff className="size-4" />
|
||||
)}
|
||||
</div>
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="text-sm font-semibold">
|
||||
{retrying
|
||||
? translate(
|
||||
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.retryingTitle',
|
||||
'Reconnecting to remote runtime'
|
||||
)
|
||||
: translate(
|
||||
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.disconnectedTitle',
|
||||
'Remote runtime disconnected'
|
||||
)}
|
||||
</div>
|
||||
<div className="mt-0.5 text-xs leading-5 text-muted-foreground">
|
||||
{retrying
|
||||
? translate(
|
||||
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.retryingBody',
|
||||
'Orca will retry for up to one minute. This terminal will resume if the connection returns.'
|
||||
)
|
||||
: translate(
|
||||
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.disconnectedBody',
|
||||
'Automatic retries stopped. Reconnect to resume this terminal session.'
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
{!retrying ? (
|
||||
<Button size="sm" onClick={onReconnect}>
|
||||
{translate(
|
||||
'auto.components.terminal.pane.TerminalRemoteRuntimeReconnectBanner.reconnectButton',
|
||||
'Reconnect'
|
||||
)}
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ import type {
|
|||
SleepingAgentLaunchConfig
|
||||
} from '../../../../shared/agent-session-resume'
|
||||
import type { TerminalKittyKeyboardModeTracker } from '../../../../shared/terminal-kitty-keyboard-mode-tracker'
|
||||
import type { PtyTransportRecoveryState } from './pty-transport-types'
|
||||
|
||||
export type PtyConnectionDeps = {
|
||||
tabId: string
|
||||
|
|
@ -55,6 +56,9 @@ export type PtyConnectionDeps = {
|
|||
onPtyExitRef: React.RefObject<(ptyId: string) => void>
|
||||
onAgentExitedRef: React.RefObject<(leafId: string) => void>
|
||||
onPtyErrorRef?: React.RefObject<(paneId: number, message: string) => void>
|
||||
onPtyRecoveryStateRef?: React.RefObject<
|
||||
(paneId: number, state: PtyTransportRecoveryState | null) => void
|
||||
>
|
||||
clearTabPtyId: (tabId: string, ptyId: string) => void
|
||||
consumeSuppressedPtyExit: (ptyId: string) => boolean
|
||||
updateTabTitle: (tabId: string, title: string) => void
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import { takeCurrentPtyDeliveryAckCredit } from './terminal-pty-ack-gate'
|
|||
import { serializeWithAbsoluteCursor } from '../../../../shared/terminal-serialize-absolute-cursor'
|
||||
import { isTerminalQueryReply } from '../../../../shared/terminal-query-reply'
|
||||
import type { PtyBufferSnapshot, PtyConnectResult } from './pty-transport'
|
||||
import type { PtyTransportRecoveryState } from './pty-transport-types'
|
||||
import { createIpcPtyTransport } from './pty-transport'
|
||||
import { createRemoteRuntimePtyTransport } from './remote-runtime-pty-transport'
|
||||
import { getConnectionId } from '@/lib/connection-context'
|
||||
|
|
@ -2233,6 +2234,7 @@ export function connectPanePty(
|
|||
activePanePtyBinding = null
|
||||
activePanePtyBindingBoundAt = null
|
||||
delete pane.container.dataset.ptyId
|
||||
delete pane.container.dataset.ptyRecoveryState
|
||||
}
|
||||
|
||||
const agentCompletionCoordinator = createAgentCompletionCoordinator({
|
||||
|
|
@ -5228,6 +5230,13 @@ export function connectPanePty(
|
|||
if (isCurrent()) {
|
||||
onError(message)
|
||||
}
|
||||
},
|
||||
onRecoveryStateChange: (state: PtyTransportRecoveryState): void => {
|
||||
if (isCurrent()) {
|
||||
// Why: cached pixels remain visible while detached; expose transport truth for diagnostics and recovery UI.
|
||||
pane.container.dataset.ptyRecoveryState = state.phase
|
||||
deps.onPtyRecoveryStateRef?.current?.(pane.id, state)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -72,6 +72,21 @@ type PtyCallbacks = {
|
|||
onStatus?: (shell: string) => void
|
||||
onError?: (message: string, errors?: string[]) => void
|
||||
onExit?: (code: number) => void
|
||||
onRecoveryStateChange?: (state: PtyTransportRecoveryState) => void
|
||||
}
|
||||
|
||||
export type PtyTransportRecoveryState = {
|
||||
phase:
|
||||
| 'connecting'
|
||||
| 'connected'
|
||||
| 'recovering'
|
||||
| 'backoff'
|
||||
| 'disconnected'
|
||||
| 'offline'
|
||||
| 'ended'
|
||||
| 'disposed'
|
||||
epoch: number
|
||||
attempt: number
|
||||
}
|
||||
|
||||
export type PtyTransport = {
|
||||
|
|
@ -125,6 +140,9 @@ export type PtyTransport = {
|
|||
}
|
||||
) => boolean
|
||||
isConnected: () => boolean
|
||||
getRecoveryState?: () => PtyTransportRecoveryState
|
||||
/** Starts a fresh connection epoch while preserving the authoritative remote PTY identity. */
|
||||
retryRecovery?: () => boolean
|
||||
getPtyId: () => string | null
|
||||
getConnectionId?: () => string | null | undefined
|
||||
/** The runtime captured by this transport; legacy remote PTY ids do not
|
||||
|
|
|
|||
|
|
@ -2005,6 +2005,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
method: 'terminal.create',
|
||||
params: {
|
||||
worktree: 'id:repo1::/remote/wt',
|
||||
clientMutationId: expect.any(String),
|
||||
command: 'claude',
|
||||
env: { ORCA_TAB_ID: 'tab-1' },
|
||||
tabId: 'tab-1',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { RemoteRuntimePtyRecoveryState } from './remote-runtime-pty-recovery-state'
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
describe('RemoteRuntimePtyRecoveryState', () => {
|
||||
it('cancels stale retry timers when a pane detaches', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const retry = vi.fn()
|
||||
const epoch = state.begin()
|
||||
state.schedule(epoch, retry)
|
||||
|
||||
state.cancel()
|
||||
await vi.runAllTimersAsync()
|
||||
|
||||
expect(retry).not.toHaveBeenCalled()
|
||||
expect(state.isCurrent(epoch)).toBe(false)
|
||||
expect(state.isActive).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps one epoch across retries and rejects it after disposal', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const retry = vi.fn()
|
||||
const epoch = state.begin()
|
||||
state.schedule(epoch, retry)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
expect(retry).toHaveBeenCalledWith(epoch)
|
||||
expect(state.begin()).toBe(epoch)
|
||||
|
||||
state.dispose()
|
||||
expect(state.isCurrent(epoch)).toBe(false)
|
||||
expect(state.isActive).toBe(false)
|
||||
})
|
||||
|
||||
it('rejects a stale retry after a newer attachment becomes healthy', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const retry = vi.fn()
|
||||
const epoch = state.begin()
|
||||
|
||||
state.markHealthy()
|
||||
|
||||
expect(state.isCurrent(epoch)).toBe(false)
|
||||
expect(state.schedule(epoch, retry)).toBe(false)
|
||||
await vi.runAllTimersAsync()
|
||||
expect(retry).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stops automatic recovery after the bounded recovery window', async () => {
|
||||
vi.useFakeTimers()
|
||||
const onChange = vi.fn()
|
||||
const state = new RemoteRuntimePtyRecoveryState(onChange)
|
||||
const retry = vi.fn()
|
||||
const epoch = state.begin()
|
||||
state.schedule(epoch, retry)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
|
||||
expect(state.currentPhase).toBe('disconnected')
|
||||
expect(state.isActive).toBe(false)
|
||||
expect(state.isCurrent(epoch)).toBe(false)
|
||||
expect(onChange).toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('starts a newly fenced recovery epoch after a manual retry', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const firstEpoch = state.begin()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
const manualEpoch = state.begin()
|
||||
|
||||
expect(manualEpoch).toBe(firstEpoch + 1)
|
||||
expect(state.currentPhase).toBe('recovering')
|
||||
expect(state.isCurrent(firstEpoch)).toBe(false)
|
||||
expect(state.isCurrent(manualEpoch)).toBe(true)
|
||||
})
|
||||
|
||||
it('cancels scheduled work when a caller reaches its own recovery cutoff', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const retry = vi.fn()
|
||||
const epoch = state.begin()
|
||||
state.schedule(epoch, retry)
|
||||
|
||||
state.markDisconnected()
|
||||
await vi.runAllTimersAsync()
|
||||
|
||||
expect(state.currentPhase).toBe('disconnected')
|
||||
expect(state.isCurrent(epoch)).toBe(false)
|
||||
expect(retry).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
const RECOVERY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const
|
||||
export const REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS = 60_000
|
||||
|
||||
export type RemoteRuntimePtyRecoveryPhase =
|
||||
| 'idle'
|
||||
| 'recovering'
|
||||
| 'backoff'
|
||||
| 'disconnected'
|
||||
| 'disposed'
|
||||
|
||||
export class RemoteRuntimePtyRecoveryState {
|
||||
private phase: RemoteRuntimePtyRecoveryPhase = 'idle'
|
||||
private epoch = 0
|
||||
private attempt = 0
|
||||
private retryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private deadlineTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
constructor(private readonly onChange?: () => void) {}
|
||||
|
||||
get isActive(): boolean {
|
||||
return this.phase === 'recovering' || this.phase === 'backoff'
|
||||
}
|
||||
|
||||
get currentPhase(): RemoteRuntimePtyRecoveryPhase {
|
||||
return this.phase
|
||||
}
|
||||
|
||||
get currentEpoch(): number {
|
||||
return this.epoch
|
||||
}
|
||||
|
||||
get attemptCount(): number {
|
||||
return this.attempt
|
||||
}
|
||||
|
||||
begin(): number {
|
||||
if (this.phase === 'disposed') {
|
||||
return this.epoch
|
||||
}
|
||||
if (!this.isActive) {
|
||||
this.epoch += 1
|
||||
this.attempt = 0
|
||||
this.armDeadline(this.epoch)
|
||||
}
|
||||
this.clearRetryTimer()
|
||||
this.phase = 'recovering'
|
||||
this.onChange?.()
|
||||
return this.epoch
|
||||
}
|
||||
|
||||
isCurrent(epoch: number): boolean {
|
||||
return this.isActive && epoch === this.epoch
|
||||
}
|
||||
|
||||
ownsEpoch(epoch: number): boolean {
|
||||
return this.phase !== 'disposed' && epoch === this.epoch
|
||||
}
|
||||
|
||||
schedule(epoch: number, retry: (epoch: number) => void): boolean {
|
||||
if (!this.isCurrent(epoch)) {
|
||||
return false
|
||||
}
|
||||
this.clearRetryTimer()
|
||||
this.phase = 'backoff'
|
||||
const delayMs = RECOVERY_DELAYS_MS[Math.min(this.attempt, RECOVERY_DELAYS_MS.length - 1)]
|
||||
this.attempt += 1
|
||||
this.onChange?.()
|
||||
const timer = setTimeout(() => {
|
||||
if (this.retryTimer !== timer || !this.isCurrent(epoch)) {
|
||||
return
|
||||
}
|
||||
this.retryTimer = null
|
||||
this.phase = 'recovering'
|
||||
this.onChange?.()
|
||||
retry(epoch)
|
||||
}, delayMs)
|
||||
timer.unref?.()
|
||||
this.retryTimer = timer
|
||||
return true
|
||||
}
|
||||
|
||||
markHealthy(): void {
|
||||
if (this.phase === 'disposed') {
|
||||
return
|
||||
}
|
||||
this.clearTimers()
|
||||
this.phase = 'idle'
|
||||
this.attempt = 0
|
||||
this.onChange?.()
|
||||
}
|
||||
|
||||
markDisconnected(): void {
|
||||
if (this.phase === 'disposed') {
|
||||
return
|
||||
}
|
||||
this.clearTimers()
|
||||
this.phase = 'disconnected'
|
||||
this.onChange?.()
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
if (this.phase === 'disposed') {
|
||||
return
|
||||
}
|
||||
this.epoch += 1
|
||||
this.clearTimers()
|
||||
this.phase = 'idle'
|
||||
this.attempt = 0
|
||||
this.onChange?.()
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.epoch += 1
|
||||
this.clearTimers()
|
||||
this.phase = 'disposed'
|
||||
this.onChange?.()
|
||||
}
|
||||
|
||||
private armDeadline(epoch: number): void {
|
||||
this.clearDeadlineTimer()
|
||||
const timer = setTimeout(() => {
|
||||
if (this.deadlineTimer !== timer || !this.isCurrent(epoch)) {
|
||||
return
|
||||
}
|
||||
this.deadlineTimer = null
|
||||
this.clearRetryTimer()
|
||||
this.phase = 'disconnected'
|
||||
this.onChange?.()
|
||||
}, REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
timer.unref?.()
|
||||
this.deadlineTimer = timer
|
||||
}
|
||||
|
||||
private clearTimers(): void {
|
||||
this.clearRetryTimer()
|
||||
this.clearDeadlineTimer()
|
||||
}
|
||||
|
||||
private clearRetryTimer(): void {
|
||||
if (this.retryTimer) {
|
||||
clearTimeout(this.retryTimer)
|
||||
this.retryTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
private clearDeadlineTimer(): void {
|
||||
if (this.deadlineTimer) {
|
||||
clearTimeout(this.deadlineTimer)
|
||||
this.deadlineTimer = null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@ import {
|
|||
TERMINAL_INPUT_MAX_BYTES
|
||||
} from '../../../../shared/terminal-input'
|
||||
import { CLIPBOARD_TEXT_MEASURE_YIELD_CODE_UNITS } from '../../../../shared/clipboard-text'
|
||||
import { TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version'
|
||||
|
||||
describe('createRemoteRuntimePtyTransport', () => {
|
||||
const runtimeCall = vi.fn()
|
||||
|
|
@ -191,6 +192,363 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
})
|
||||
})
|
||||
|
||||
it('does not report attachment health until the authoritative PTY snapshot arrives', async () => {
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
|
||||
transport.attach({ existingPtyId: 'remote:terminal-1', callbacks: {} })
|
||||
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
|
||||
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'authoritative state')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
transport.destroy?.()
|
||||
})
|
||||
|
||||
it('recovers when the first restored-terminal subscription attempt is offline', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let attempt = 0
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
|
||||
attempt += 1
|
||||
if (attempt === 1) {
|
||||
throw Object.assign(new Error('Could not connect to the remote Orca runtime.'), {
|
||||
code: 'remote_runtime_unavailable'
|
||||
})
|
||||
}
|
||||
subscriptionCallbacks = callbacks
|
||||
queueMicrotask(emitMultiplexReady)
|
||||
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const onData = vi.fn()
|
||||
const recoveryStates: { phase: string; epoch: number; attempt: number }[] = []
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
|
||||
transport.attach({
|
||||
existingPtyId: 'remote:terminal-1',
|
||||
callbacks: {
|
||||
onData,
|
||||
onError,
|
||||
onRecoveryStateChange: (state) => recoveryStates.push(state)
|
||||
}
|
||||
})
|
||||
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(1))
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2))
|
||||
await vi.waitFor(() => expect(latestSubscribePayload().terminal).toBe('terminal-1'))
|
||||
const { streamId } = latestSubscribePayload()
|
||||
emitSnapshot(streamId, 'restored')
|
||||
emitOutput(streamId, 'resumed-output')
|
||||
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
expect(onData).toHaveBeenCalledWith('resumed-output', expect.any(Object))
|
||||
expect(transport.sendInputImmediate('resumed-input')).toBe(true)
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('connected')
|
||||
expect(recoveryStates.map((state) => state.phase)).toEqual(
|
||||
expect.arrayContaining(['connecting', 'recovering', 'connected'])
|
||||
)
|
||||
const recoveryEpochs = new Set(
|
||||
recoveryStates
|
||||
.filter((state) => state.phase === 'recovering' || state.phase === 'backoff')
|
||||
.map((state) => state.epoch)
|
||||
)
|
||||
expect(recoveryEpochs.size).toBe(1)
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-1')
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('recovers when the runtime closes before a restored subscription becomes ready', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let attempt = 0
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
|
||||
attempt += 1
|
||||
subscriptionCallbacks = callbacks
|
||||
if (attempt === 1) {
|
||||
queueMicrotask(() => callbacks.onClose?.())
|
||||
} else {
|
||||
queueMicrotask(emitMultiplexReady)
|
||||
}
|
||||
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
|
||||
transport.attach({
|
||||
existingPtyId: 'remote:terminal-1',
|
||||
callbacks: { onError }
|
||||
})
|
||||
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(1))
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2))
|
||||
const { streamId } = latestSubscribePayload()
|
||||
emitSnapshot(streamId, 'restored')
|
||||
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-1')
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('surfaces a fatal error during subscription setup exactly once', async () => {
|
||||
const unsubscribe = vi.fn()
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
|
||||
subscriptionCallbacks = callbacks
|
||||
queueMicrotask(() =>
|
||||
callbacks.onError?.({
|
||||
code: 'unauthorized',
|
||||
message: 'Remote Orca runtime rejected the pairing token.'
|
||||
})
|
||||
)
|
||||
return { unsubscribe, sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
|
||||
transport.attach({ existingPtyId: 'remote:terminal-1', callbacks: { onError } })
|
||||
await vi.waitFor(() => expect(onError).toHaveBeenCalled())
|
||||
await Promise.resolve()
|
||||
|
||||
expect(onError).toHaveBeenCalledTimes(1)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('offline')
|
||||
expect(unsubscribe).toHaveBeenCalledTimes(1)
|
||||
expect(runtimeSubscribe).toHaveBeenCalledTimes(1)
|
||||
transport.destroy?.()
|
||||
})
|
||||
|
||||
it('retries an unknown terminal-create outcome exactly once with the same mutation id', async () => {
|
||||
let createCalls = 0
|
||||
runtimeCall.mockImplementation(async (args: { method: string; params?: unknown }) => {
|
||||
if (args.method === 'status.get') {
|
||||
return {
|
||||
ok: true,
|
||||
result: { capabilities: [TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY] }
|
||||
}
|
||||
}
|
||||
if (args.method === 'terminal.create') {
|
||||
createCalls += 1
|
||||
if (createCalls === 1) {
|
||||
throw Object.assign(new Error('Timed out waiting for the remote Orca runtime.'), {
|
||||
code: 'runtime_timeout'
|
||||
})
|
||||
}
|
||||
return { ok: true, result: { terminal: { handle: 'terminal-once' } } }
|
||||
}
|
||||
return { ok: true, result: {} }
|
||||
})
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onPtySpawn = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1',
|
||||
onPtySpawn
|
||||
})
|
||||
|
||||
await transport.connect({ url: '', callbacks: {} })
|
||||
|
||||
const creates = runtimeCall.mock.calls
|
||||
.map(
|
||||
([args]) =>
|
||||
args as {
|
||||
method: string
|
||||
params?: { clientMutationId?: string; reconcileExisting?: boolean }
|
||||
}
|
||||
)
|
||||
.filter((args) => args.method === 'terminal.create')
|
||||
expect(creates).toHaveLength(2)
|
||||
expect(creates[0].params?.clientMutationId).toMatch(/\S+/)
|
||||
expect(creates[1].params?.clientMutationId).toBe(creates[0].params?.clientMutationId)
|
||||
expect(creates[0].params?.reconcileExisting).toBeUndefined()
|
||||
expect(creates[1].params?.reconcileExisting).toBe(true)
|
||||
expect(onPtySpawn).toHaveBeenCalledTimes(1)
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-once')
|
||||
transport.destroy?.()
|
||||
})
|
||||
|
||||
it('clips a reconciled create timeout to the budget left after a slow capability probe', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const startedAt = Date.now()
|
||||
let createCalls = 0
|
||||
runtimeCall.mockImplementation(async (args: { method: string }) => {
|
||||
if (args.method === 'status.get') {
|
||||
vi.setSystemTime(startedAt + 59_000)
|
||||
return {
|
||||
ok: true,
|
||||
result: { capabilities: [TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY] }
|
||||
}
|
||||
}
|
||||
if (args.method === 'terminal.create') {
|
||||
createCalls += 1
|
||||
if (createCalls === 1) {
|
||||
throw Object.assign(new Error('Timed out waiting for the remote Orca runtime.'), {
|
||||
code: 'runtime_timeout'
|
||||
})
|
||||
}
|
||||
return { ok: true, result: { terminal: { handle: 'terminal-reconciled' } } }
|
||||
}
|
||||
return { ok: true, result: {} }
|
||||
})
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
|
||||
const connect = transport.connect({ url: '', callbacks: {} })
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
await connect
|
||||
|
||||
const createRequests = runtimeCall.mock.calls
|
||||
.map(([args]) => args as { method: string; timeoutMs: number })
|
||||
.filter((args) => args.method === 'terminal.create')
|
||||
expect(createRequests).toHaveLength(2)
|
||||
expect(createRequests[1].timeoutMs).toBe(1_000)
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not retry an unknown create outcome against an older runtime', async () => {
|
||||
runtimeCall.mockImplementation(async (args: { method: string }) => {
|
||||
if (args.method === 'status.get') {
|
||||
return { ok: true, result: { capabilities: [] } }
|
||||
}
|
||||
throw Object.assign(new Error('Timed out waiting for the remote Orca runtime.'), {
|
||||
code: 'runtime_timeout'
|
||||
})
|
||||
})
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
|
||||
await transport.connect({ url: '', callbacks: { onError } })
|
||||
|
||||
expect(
|
||||
runtimeCall.mock.calls.filter(([args]) => args.method === 'terminal.create')
|
||||
).toHaveLength(1)
|
||||
expect(onError).toHaveBeenCalledTimes(1)
|
||||
transport.destroy?.()
|
||||
})
|
||||
|
||||
it('surfaces an authoritative capability-probe failure after an unknown create outcome', async () => {
|
||||
runtimeCall.mockImplementation(async (args: { method: string }) => {
|
||||
if (args.method === 'status.get') {
|
||||
throw Object.assign(new Error('Remote runtime pairing credentials expired.'), {
|
||||
code: 'unauthorized'
|
||||
})
|
||||
}
|
||||
throw Object.assign(new Error('Timed out waiting for the remote Orca runtime.'), {
|
||||
code: 'runtime_timeout'
|
||||
})
|
||||
})
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
|
||||
await transport.connect({ url: '', callbacks: { onError } })
|
||||
|
||||
expect(onError).toHaveBeenCalledWith('Remote runtime pairing credentials expired.')
|
||||
expect(
|
||||
runtimeCall.mock.calls.filter(([args]) => args.method === 'terminal.create')
|
||||
).toHaveLength(1)
|
||||
transport.destroy?.()
|
||||
})
|
||||
|
||||
it('stops unknown terminal-create recovery after one minute and remains manually retryable', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let reachable = false
|
||||
let statusTimesOut = false
|
||||
runtimeCall.mockImplementation(async (args: { method: string; timeoutMs: number }) => {
|
||||
if (args.method === 'status.get') {
|
||||
if (statusTimesOut) {
|
||||
return new Promise((_, reject) => {
|
||||
setTimeout(() => {
|
||||
reject(
|
||||
Object.assign(new Error('Timed out waiting for the remote Orca runtime.'), {
|
||||
code: 'runtime_timeout'
|
||||
})
|
||||
)
|
||||
}, args.timeoutMs)
|
||||
})
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
result: { capabilities: [TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY] }
|
||||
}
|
||||
}
|
||||
if (args.method === 'terminal.create' && reachable) {
|
||||
return { ok: true, result: { terminal: { handle: 'terminal-recovered' } } }
|
||||
}
|
||||
throw Object.assign(new Error('Timed out waiting for the remote Orca runtime.'), {
|
||||
code: 'runtime_timeout'
|
||||
})
|
||||
})
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const recoveryStates: string[] = []
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
|
||||
const connect = transport.connect({
|
||||
url: '',
|
||||
callbacks: {
|
||||
onError,
|
||||
onRecoveryStateChange: (state) => recoveryStates.push(state.phase)
|
||||
}
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
await connect
|
||||
const callsAtCutoff = runtimeCall.mock.calls.length
|
||||
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
expect(recoveryStates).toContain('recovering')
|
||||
expect(runtimeCall.mock.calls.some(([args]) => args.method === 'terminal.create')).toBe(true)
|
||||
await vi.advanceTimersByTimeAsync(5 * 60_000)
|
||||
expect(runtimeCall).toHaveBeenCalledTimes(callsAtCutoff)
|
||||
|
||||
statusTimesOut = true
|
||||
expect(transport.retryRecovery?.()).toBe(true)
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
const callsAtManualCutoff = runtimeCall.mock.calls.length
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
await vi.advanceTimersByTimeAsync(5 * 60_000)
|
||||
expect(runtimeCall).toHaveBeenCalledTimes(callsAtManualCutoff)
|
||||
|
||||
statusTimesOut = false
|
||||
reachable = true
|
||||
expect(transport.retryRecovery?.()).toBe(true)
|
||||
await vi.waitFor(() => expect(transport.getPtyId()).toBe('remote:env-1@@terminal-recovered'))
|
||||
const createRequests = runtimeCall.mock.calls
|
||||
.map(([args]) => args as { method: string; params?: { reconcileExisting?: boolean } })
|
||||
.filter((args) => args.method === 'terminal.create')
|
||||
expect(createRequests[0].params?.reconcileExisting).toBeUndefined()
|
||||
expect(createRequests.slice(1).every((args) => args.params?.reconcileExisting === true)).toBe(
|
||||
true
|
||||
)
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('scopes the same legacy handle independently for each runtime environment', async () => {
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const first = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
|
|
@ -445,6 +803,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
|
||||
transport.destroy?.()
|
||||
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disposed')
|
||||
expect(runtimeCall).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
method: 'terminal.close'
|
||||
|
|
@ -671,7 +1030,8 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
expect(onError).not.toHaveBeenCalled()
|
||||
expect(onPtyExit).not.toHaveBeenCalled()
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-stale')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
// Cached pixels and a known PTY id do not imply that input/output is attached.
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
|
||||
|
||||
|
|
@ -718,6 +1078,9 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
)
|
||||
expect(onPtyExit).not.toHaveBeenCalled()
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-after-timeout')
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'reattached')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(0)
|
||||
const subscribedTerminals = subscriptionSendBinary.mock.calls
|
||||
.map((call) => decodeTerminalStreamFrame(call[0]))
|
||||
|
|
@ -813,6 +1176,8 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
expect(hostListCalls).toBe(1)
|
||||
expect(onPtyExit).not.toHaveBeenCalled()
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-reconnected')
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'reattached')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
})
|
||||
|
||||
|
|
@ -846,6 +1211,7 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
expect(onPtyExit).toHaveBeenCalledWith('remote:env-1@@terminal-exited')
|
||||
expect(transport.getPtyId()).toBeNull()
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('ended')
|
||||
})
|
||||
|
||||
it('ignores stale stream end after reattaching a newer remote terminal', async () => {
|
||||
|
|
@ -880,12 +1246,14 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
|
||||
expect(onPtyExit).not.toHaveBeenCalled()
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-new')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(latestSubscribePayload()).toMatchObject({ terminal: 'terminal-new' })
|
||||
})
|
||||
const newStreamId = latestSubscribePayload().streamId
|
||||
emitSnapshot(newStreamId, 'reattached')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
|
||||
subscriptionCallbacks?.onResponse({
|
||||
ok: true,
|
||||
|
|
@ -960,7 +1328,10 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
oldSubscription.reject = reject
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce(newStream)
|
||||
.mockImplementationOnce(async (args: { callbacks: { onSubscribed?: () => void } }) => {
|
||||
args.callbacks.onSubscribed?.()
|
||||
return newStream
|
||||
})
|
||||
vi.doMock('../../runtime/remote-runtime-terminal-multiplexer', () => ({
|
||||
getRemoteRuntimeTerminalMultiplexer: vi.fn(() => ({ subscribeTerminal }))
|
||||
}))
|
||||
|
|
@ -1081,6 +1452,39 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
})
|
||||
})
|
||||
|
||||
it('cannot let a stale create completion replace a newer attached terminal', async () => {
|
||||
let resolveCreate: (value: unknown) => void = () => {}
|
||||
runtimeCall.mockImplementation((args) => {
|
||||
if (args.method === 'terminal.create') {
|
||||
return new Promise((resolve) => {
|
||||
resolveCreate = resolve
|
||||
})
|
||||
}
|
||||
return Promise.resolve({ ok: true, result: {} })
|
||||
})
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onPtySpawn = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
onPtySpawn
|
||||
})
|
||||
|
||||
const connect = transport.connect({ url: '', callbacks: {} })
|
||||
transport.attach({ existingPtyId: 'remote:env-2@@terminal-late', callbacks: {} })
|
||||
resolveCreate({ ok: true, result: { terminal: { handle: 'terminal-late' } } })
|
||||
await connect
|
||||
|
||||
expect(transport.getPtyId()).toBe('remote:env-2@@terminal-late')
|
||||
expect(onPtySpawn).not.toHaveBeenCalled()
|
||||
expect(runtimeCall).toHaveBeenCalledWith({
|
||||
selector: 'env-1',
|
||||
method: 'terminal.close',
|
||||
params: { terminal: 'terminal-late' },
|
||||
timeoutMs: 15_000
|
||||
})
|
||||
transport.destroy?.()
|
||||
})
|
||||
|
||||
it('passes activation intent when creating the remote runtime terminal', async () => {
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
|
|
@ -1838,6 +2242,212 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(2))
|
||||
})
|
||||
|
||||
it('keeps retrying when the first post-partition terminal reattach fails', async () => {
|
||||
let subscribeAttempt = 0
|
||||
const recoveryPhases: string[] = []
|
||||
const transportCallbacks: NonNullable<typeof subscriptionCallbacks>[] = []
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
|
||||
subscribeAttempt += 1
|
||||
transportCallbacks.push(callbacks)
|
||||
subscriptionCallbacks = callbacks
|
||||
if (subscribeAttempt === 2) {
|
||||
throw new Error('Could not connect to the remote Orca runtime.')
|
||||
}
|
||||
queueMicrotask(emitMultiplexReady)
|
||||
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1'
|
||||
})
|
||||
|
||||
await transport.connect({
|
||||
url: '',
|
||||
callbacks: {
|
||||
onError,
|
||||
onRecoveryStateChange: (state) => recoveryPhases.push(state.phase)
|
||||
}
|
||||
})
|
||||
transportCallbacks[0].onError?.({
|
||||
code: 'remote_runtime_unavailable',
|
||||
message: 'Remote Orca runtime stopped responding; the stream connection was reset.'
|
||||
})
|
||||
|
||||
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(3))
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
expect(recoveryPhases).toContain('backoff')
|
||||
transport.destroy?.()
|
||||
})
|
||||
|
||||
it('surfaces fatal transport errors once without retrying or double-unsubscribing', async () => {
|
||||
const unsubscribe = vi.fn()
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
|
||||
subscriptionCallbacks = callbacks
|
||||
queueMicrotask(emitMultiplexReady)
|
||||
return { unsubscribe, sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' })
|
||||
await transport.connect({ url: '', callbacks: { onError } })
|
||||
|
||||
subscriptionCallbacks?.onError?.({
|
||||
code: 'unauthorized',
|
||||
message: 'Remote Orca runtime rejected the pairing token.'
|
||||
})
|
||||
|
||||
expect(onError).toHaveBeenCalledTimes(1)
|
||||
expect(unsubscribe).toHaveBeenCalledTimes(1)
|
||||
expect(runtimeSubscribe).toHaveBeenCalledTimes(1)
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
transport.destroy?.()
|
||||
expect(unsubscribe).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('recovers repeated partitions without changing PTY identity or accepting detached input', async () => {
|
||||
const callbacksByEpoch: NonNullable<typeof subscriptionCallbacks>[] = []
|
||||
const unsubscribeByEpoch: ReturnType<typeof vi.fn>[] = []
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
|
||||
callbacksByEpoch.push(callbacks)
|
||||
subscriptionCallbacks = callbacks
|
||||
const unsubscribe = vi.fn()
|
||||
unsubscribeByEpoch.push(unsubscribe)
|
||||
queueMicrotask(() => callbacks.onResponse({ ok: true, result: { type: 'ready' } }))
|
||||
return { unsubscribe, sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onData = vi.fn()
|
||||
const onError = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1'
|
||||
})
|
||||
|
||||
await transport.connect({ url: '', callbacks: { onData, onError } })
|
||||
const ptyId = transport.getPtyId()
|
||||
|
||||
for (let cycle = 0; cycle < 10; cycle += 1) {
|
||||
callbacksByEpoch.at(-1)?.onError?.({
|
||||
code: 'remote_runtime_unavailable',
|
||||
message: 'Remote runtime connection closed.'
|
||||
})
|
||||
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
expect(transport.sendInput(`detached-${cycle}`)).toBe(false)
|
||||
expect(unsubscribeByEpoch[cycle]).toHaveBeenCalledTimes(1)
|
||||
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(cycle + 2))
|
||||
await vi.waitFor(() => expect(latestSubscribePayload().terminal).toBe('terminal-1'))
|
||||
const { streamId } = latestSubscribePayload()
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
emitSnapshot(streamId, `snapshot-${cycle}`)
|
||||
await vi.waitFor(() => expect(transport.isConnected()).toBe(true))
|
||||
emitOutput(streamId, `output-${cycle}`)
|
||||
expect(transport.sendInputImmediate(`input-${cycle}`)).toBe(true)
|
||||
|
||||
expect(transport.getPtyId()).toBe(ptyId)
|
||||
expect(onData).toHaveBeenCalledWith(`output-${cycle}`, expect.any(Object))
|
||||
expect(
|
||||
decodeTerminalStreamText(
|
||||
latestFrameForOpcode(TerminalStreamOpcode.Input)?.payload ?? new Uint8Array()
|
||||
)
|
||||
).toBe(`input-${cycle}`)
|
||||
expect(callbacksByEpoch).toHaveLength(cycle + 2)
|
||||
}
|
||||
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
expect(runtimeSubscribe).toHaveBeenCalledTimes(11)
|
||||
transport.destroy?.()
|
||||
expect(unsubscribeByEpoch.every((unsubscribe) => unsubscribe.mock.calls.length === 1)).toBe(
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
it('stops automatic retries and manually reattaches the same PTY in a new epoch', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
let partitioned = false
|
||||
const callbacksByConnection: NonNullable<typeof subscriptionCallbacks>[] = []
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: NonNullable<typeof subscriptionCallbacks>) => {
|
||||
if (partitioned) {
|
||||
throw Object.assign(new Error('Could not connect to the remote Orca runtime.'), {
|
||||
code: 'remote_runtime_unavailable'
|
||||
})
|
||||
}
|
||||
callbacksByConnection.push(callbacks)
|
||||
subscriptionCallbacks = callbacks
|
||||
queueMicrotask(() => callbacks.onResponse({ ok: true, result: { type: 'ready' } }))
|
||||
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const recoveryStates: { phase: string; epoch: number; attempt: number }[] = []
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'tab-1',
|
||||
leafId: 'pane:1'
|
||||
})
|
||||
|
||||
transport.attach({
|
||||
existingPtyId: 'remote:env-1@@terminal-1',
|
||||
callbacks: {
|
||||
onError,
|
||||
onRecoveryStateChange: (state) => recoveryStates.push(state)
|
||||
}
|
||||
})
|
||||
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'before partition')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
|
||||
partitioned = true
|
||||
callbacksByConnection[0].onClose?.()
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
|
||||
const disconnectedState = transport.getRecoveryState?.()
|
||||
const callsAtCutoff = runtimeSubscribe.mock.calls.length
|
||||
expect(disconnectedState?.phase).toBe('disconnected')
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-1')
|
||||
expect(transport.isConnected()).toBe(false)
|
||||
expect(transport.sendInput('must not reach a stale socket')).toBe(false)
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
await vi.advanceTimersByTimeAsync(5 * 60_000)
|
||||
expect(runtimeSubscribe).toHaveBeenCalledTimes(callsAtCutoff)
|
||||
|
||||
partitioned = false
|
||||
expect(transport.retryRecovery?.()).toBe(true)
|
||||
expect(transport.retryRecovery?.()).toBe(false)
|
||||
await vi.waitFor(() => expect(runtimeSubscribe).toHaveBeenCalledTimes(callsAtCutoff + 1))
|
||||
await vi.waitFor(() => {
|
||||
const subscribeFrames = subscriptionSendBinary.mock.calls
|
||||
.map((call) => decodeTerminalStreamFrame(call[0]))
|
||||
.filter((frame) => frame?.opcode === TerminalStreamOpcode.Subscribe)
|
||||
expect(subscribeFrames).toHaveLength(2)
|
||||
})
|
||||
const manualStream = latestSubscribePayload()
|
||||
expect(manualStream.terminal).toBe('terminal-1')
|
||||
emitSnapshot(manualStream.streamId, 'after manual reconnect')
|
||||
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('connected')
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-1')
|
||||
expect(recoveryStates.at(-1)?.epoch).toBeGreaterThan(disconnectedState?.epoch ?? 0)
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('releases pending claimed input when reconnect subscription fails', async () => {
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
|
|
|
|||
|
|
@ -1,16 +1,27 @@
|
|||
/* eslint-disable max-lines -- Why: remote PTY transport keeps lifecycle, JSON fallback, and binary stream wiring together so reconnect/destroy ordering stays testable as one behavior surface. */
|
||||
import type { RuntimeRpcResponse } from '../../../../shared/runtime-rpc-envelope'
|
||||
import {
|
||||
isRecoverableRemoteRuntimeConnectionError,
|
||||
toRemoteRuntimeClientErrorLike
|
||||
} from '../../../../shared/remote-runtime-client-error-classification'
|
||||
import type {
|
||||
RuntimeMobileSessionTerminalClientTab,
|
||||
RuntimeMobileSessionTabsResult,
|
||||
RuntimeStatus,
|
||||
RuntimeTerminalCreate,
|
||||
RuntimeTerminalSend
|
||||
} from '../../../../shared/runtime-types'
|
||||
import { TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY } from '../../../../shared/protocol-version'
|
||||
import {
|
||||
isTerminalInputTooLargeWithDeferredMeasurement,
|
||||
iterateTerminalInputChunks
|
||||
} from '../../../../shared/terminal-input'
|
||||
import type { IpcPtyTransportOptions, PtyConnectResult, PtyTransport } from './pty-transport-types'
|
||||
import type {
|
||||
IpcPtyTransportOptions,
|
||||
PtyConnectResult,
|
||||
PtyTransport,
|
||||
PtyTransportRecoveryState
|
||||
} from './pty-transport-types'
|
||||
import { createPtyOutputProcessor } from './pty-transport'
|
||||
import { unwrapRuntimeRpcResult } from '../../runtime/runtime-rpc-client'
|
||||
import {
|
||||
|
|
@ -32,6 +43,10 @@ import {
|
|||
createRemoteRuntimePtyTextBatcher,
|
||||
createRemoteRuntimeViewportBatcher
|
||||
} from './remote-runtime-pty-batching'
|
||||
import {
|
||||
REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS,
|
||||
RemoteRuntimePtyRecoveryState
|
||||
} from './remote-runtime-pty-recovery-state'
|
||||
import { createBrowserUuid } from '@/lib/browser-uuid'
|
||||
import { replaceFitOverridePtyId, setFitOverride } from '@/lib/pane-manager/mobile-fit-overrides'
|
||||
import { replaceDriverPtyId, setDriverForPty } from '@/lib/pane-manager/mobile-driver-state'
|
||||
|
|
@ -44,6 +59,7 @@ const REMOTE_TERMINAL_VIEWPORT_FLUSH_MS = 33
|
|||
const HOST_SESSION_ATTACH_POLL_MS = 150
|
||||
const HOST_SESSION_REPLACEMENT_POLL_MAX_MS = 1_000
|
||||
const HOST_SESSION_ATTACH_TIMEOUT_MS = 15_000
|
||||
const TERMINAL_CREATE_RETRY_DELAYS_MS = [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000] as const
|
||||
|
||||
function isRemoteTerminalStaleMessage(message: string): boolean {
|
||||
return message.includes('terminal_handle_stale')
|
||||
|
|
@ -87,7 +103,12 @@ export function createRemoteRuntimePtyTransport(
|
|||
onAgentStatus
|
||||
} = opts
|
||||
let connected = false
|
||||
let attachmentReady = false
|
||||
let destroyed = false
|
||||
let terminalEnded = false
|
||||
let connecting = false
|
||||
// Why: transport methods overlap during remounts; only the latest pane lifecycle may install a returned PTY.
|
||||
let lifecycleEpoch = 0
|
||||
let handle: string | null = null
|
||||
let remotePtyId: string | null = null
|
||||
let currentRuntimeEnvironmentId = runtimeEnvironmentId
|
||||
|
|
@ -95,13 +116,32 @@ export function createRemoteRuntimePtyTransport(
|
|||
let multiplexedStreamHandle: string | null = null
|
||||
let desiredViewport: { cols: number; rows: number } | null = null
|
||||
let storedCallbacks: Parameters<PtyTransport['connect']>[0]['callbacks'] = {}
|
||||
let resubscribing = false
|
||||
let resubscribeEpoch: number | null = null
|
||||
let resubscribeRequestedHandle: string | null = null
|
||||
let resubscribeRequestedRequiresReplacement = false
|
||||
let recoveryRequiresReplacement = false
|
||||
let stopWaitingForPublishedHandle: (() => void) | null = null
|
||||
let subscriptionGeneration = 0
|
||||
const recovery = new RemoteRuntimePtyRecoveryState(() => {
|
||||
if (recovery.currentPhase === 'disconnected') {
|
||||
clearPublishedHandleWait()
|
||||
// Why: cached pixels may remain, but no stream from the exhausted epoch may keep delivering or accepting terminal traffic.
|
||||
subscriptionGeneration += 1
|
||||
closeMultiplexedStream()
|
||||
}
|
||||
emitRecoveryState()
|
||||
})
|
||||
let lastRecoveryStateKey = ''
|
||||
let pendingViewportClaim = false
|
||||
let pendingClaimInput = ''
|
||||
let terminalCreateRetryWait: {
|
||||
timer: ReturnType<typeof setTimeout>
|
||||
resolve: (continueRetrying: boolean) => void
|
||||
} | null = null
|
||||
// Why: after an unknown result, every later attempt must reconcile first so older runtimes cannot duplicate the PTY.
|
||||
let terminalCreateNeedsReconciliation = false
|
||||
let terminalCreateUnknownOutcomeError: unknown = null
|
||||
let lastConnectOptions: Parameters<PtyTransport['connect']>[0] | null = null
|
||||
const viewportClaimReadyWaiters = new Set<(ready: boolean) => void>()
|
||||
const clearPendingViewportClaim = (): void => {
|
||||
pendingViewportClaim = false
|
||||
|
|
@ -113,6 +153,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
// Why: tab/leaf ids are shared by paired viewers; the instance suffix keeps one viewer's refresh off peer records.
|
||||
const clientId = `desktop:${tabId ?? 'tab'}:${leafId ?? 'leaf'}:${createBrowserUuid()}`
|
||||
const terminalCreateMutationId = createBrowserUuid()
|
||||
const outputProcessor = createPtyOutputProcessor({
|
||||
onTitleChange,
|
||||
onBell,
|
||||
|
|
@ -122,6 +163,39 @@ export function createRemoteRuntimePtyTransport(
|
|||
onAgentStatus
|
||||
})
|
||||
|
||||
function getRecoveryState(): PtyTransportRecoveryState {
|
||||
const phase = destroyed
|
||||
? 'disposed'
|
||||
: terminalEnded
|
||||
? 'ended'
|
||||
: recovery.currentPhase === 'recovering'
|
||||
? 'recovering'
|
||||
: recovery.currentPhase === 'backoff'
|
||||
? 'backoff'
|
||||
: recovery.currentPhase === 'disconnected'
|
||||
? 'disconnected'
|
||||
: connecting
|
||||
? 'connecting'
|
||||
: connected && attachmentReady
|
||||
? 'connected'
|
||||
: 'offline'
|
||||
return {
|
||||
phase,
|
||||
epoch: recovery.currentEpoch,
|
||||
attempt: recovery.attemptCount
|
||||
}
|
||||
}
|
||||
|
||||
function emitRecoveryState(force = false): void {
|
||||
const state = getRecoveryState()
|
||||
const key = `${state.phase}:${state.epoch}:${state.attempt}`
|
||||
if (!force && key === lastRecoveryStateKey) {
|
||||
return
|
||||
}
|
||||
lastRecoveryStateKey = key
|
||||
storedCallbacks.onRecoveryStateChange?.(state)
|
||||
}
|
||||
|
||||
function findReadyHostSessionHandle(
|
||||
snapshot: RuntimeMobileSessionTabsResult,
|
||||
hostTabId: string
|
||||
|
|
@ -299,7 +373,13 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
onPtySpawn?.(remotePtyId)
|
||||
|
||||
await subscribeToHandle()
|
||||
try {
|
||||
await subscribeToHandle()
|
||||
} catch (error) {
|
||||
if (!recoverAfterSubscribeFailure(error, hostHandle, remotePtyId)) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
if (destroyed || !connected || !remotePtyId) {
|
||||
return undefined
|
||||
}
|
||||
|
|
@ -310,13 +390,14 @@ export function createRemoteRuntimePtyTransport(
|
|||
} satisfies PtyConnectResult
|
||||
}
|
||||
|
||||
async function callRuntime<TResult>(
|
||||
async function callRuntimeForEnvironment<TResult>(
|
||||
environmentId: string,
|
||||
method: string,
|
||||
params?: unknown,
|
||||
timeoutMs = 15_000
|
||||
): Promise<TResult> {
|
||||
const response = await window.api.runtimeEnvironments.call({
|
||||
selector: currentRuntimeEnvironmentId,
|
||||
selector: environmentId,
|
||||
method,
|
||||
params,
|
||||
timeoutMs
|
||||
|
|
@ -324,28 +405,198 @@ export function createRemoteRuntimePtyTransport(
|
|||
return unwrapRuntimeRpcResult(response as RuntimeRpcResponse<TResult>)
|
||||
}
|
||||
|
||||
async function closeRemoteTerminal(handleOverride?: string): Promise<void> {
|
||||
async function callRuntime<TResult>(
|
||||
method: string,
|
||||
params?: unknown,
|
||||
timeoutMs = 15_000
|
||||
): Promise<TResult> {
|
||||
return callRuntimeForEnvironment(currentRuntimeEnvironmentId, method, params, timeoutMs)
|
||||
}
|
||||
|
||||
function cancelTerminalCreateRetryWait(): void {
|
||||
const waiting = terminalCreateRetryWait
|
||||
terminalCreateRetryWait = null
|
||||
if (waiting) {
|
||||
clearTimeout(waiting.timer)
|
||||
waiting.resolve(false)
|
||||
}
|
||||
}
|
||||
|
||||
function waitForTerminalCreateRetry(delayMs: number): Promise<boolean> {
|
||||
if (destroyed) {
|
||||
return Promise.resolve(false)
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const timer = setTimeout(() => {
|
||||
if (terminalCreateRetryWait?.timer === timer) {
|
||||
terminalCreateRetryWait = null
|
||||
}
|
||||
resolve(!destroyed)
|
||||
}, delayMs)
|
||||
timer.unref?.()
|
||||
terminalCreateRetryWait = { timer, resolve }
|
||||
})
|
||||
}
|
||||
|
||||
function terminalCreateRecoveryCutoffReached(): boolean {
|
||||
return recovery.currentPhase === 'disconnected'
|
||||
}
|
||||
|
||||
async function createTerminalWithUnknownOutcomeRecovery(
|
||||
params: Record<string, unknown>,
|
||||
environmentId: string,
|
||||
expectedLifecycleEpoch: number
|
||||
): Promise<{ terminal: RuntimeTerminalCreate } | null> {
|
||||
let retryAttempt = 0
|
||||
let idempotencySupported = false
|
||||
let reconcileExisting = terminalCreateNeedsReconciliation
|
||||
let recoveryDeadlineAt: number | null = recovery.isActive
|
||||
? Date.now() + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS
|
||||
: null
|
||||
let lastError: unknown =
|
||||
terminalCreateUnknownOutcomeError ?? new Error('Remote terminal creation was cancelled.')
|
||||
while (
|
||||
!destroyed &&
|
||||
lifecycleEpoch === expectedLifecycleEpoch &&
|
||||
!terminalCreateRecoveryCutoffReached()
|
||||
) {
|
||||
if (recoveryDeadlineAt !== null && recoveryDeadlineAt - Date.now() <= 0) {
|
||||
break
|
||||
}
|
||||
while (
|
||||
reconcileExisting &&
|
||||
!idempotencySupported &&
|
||||
!destroyed &&
|
||||
lifecycleEpoch === expectedLifecycleEpoch &&
|
||||
!terminalCreateRecoveryCutoffReached()
|
||||
) {
|
||||
let status: RuntimeStatus
|
||||
try {
|
||||
const statusRemainingMs =
|
||||
recoveryDeadlineAt === null ? 5_000 : recoveryDeadlineAt - Date.now()
|
||||
if (statusRemainingMs <= 0) {
|
||||
break
|
||||
}
|
||||
status = await callRuntimeForEnvironment<RuntimeStatus>(
|
||||
environmentId,
|
||||
'status.get',
|
||||
undefined,
|
||||
Math.min(5_000, statusRemainingMs)
|
||||
)
|
||||
} catch (statusError) {
|
||||
const statusClientError = toRemoteRuntimeClientErrorLike(statusError)
|
||||
if (!isRecoverableRemoteRuntimeConnectionError(statusClientError)) {
|
||||
throw statusError
|
||||
}
|
||||
const startsRecovery = recoveryDeadlineAt === null
|
||||
recoveryDeadlineAt ??= Date.now() + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS
|
||||
if (startsRecovery && !recovery.isActive) {
|
||||
recovery.begin()
|
||||
}
|
||||
const statusDelayMs =
|
||||
TERMINAL_CREATE_RETRY_DELAYS_MS[
|
||||
Math.min(retryAttempt, TERMINAL_CREATE_RETRY_DELAYS_MS.length - 1)
|
||||
]
|
||||
retryAttempt += 1
|
||||
const remainingMs = recoveryDeadlineAt - Date.now()
|
||||
if (
|
||||
remainingMs <= 0 ||
|
||||
terminalCreateRecoveryCutoffReached() ||
|
||||
!(await waitForTerminalCreateRetry(Math.min(statusDelayMs, remainingMs)))
|
||||
) {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (!status.capabilities?.includes(TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY)) {
|
||||
throw lastError
|
||||
}
|
||||
idempotencySupported = true
|
||||
}
|
||||
if (
|
||||
destroyed ||
|
||||
lifecycleEpoch !== expectedLifecycleEpoch ||
|
||||
(recoveryDeadlineAt !== null && recoveryDeadlineAt - Date.now() <= 0)
|
||||
) {
|
||||
break
|
||||
}
|
||||
const createRemainingMs = recoveryDeadlineAt === null ? null : recoveryDeadlineAt - Date.now()
|
||||
if (createRemainingMs !== null && createRemainingMs <= 0) {
|
||||
break
|
||||
}
|
||||
try {
|
||||
return await callRuntimeForEnvironment<{ terminal: RuntimeTerminalCreate }>(
|
||||
environmentId,
|
||||
'terminal.create',
|
||||
{
|
||||
...params,
|
||||
...(reconcileExisting ? { reconcileExisting: true } : {})
|
||||
},
|
||||
Math.min(15_000, createRemainingMs ?? 15_000)
|
||||
)
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
const clientError = toRemoteRuntimeClientErrorLike(error)
|
||||
if (!isRecoverableRemoteRuntimeConnectionError(clientError)) {
|
||||
throw error
|
||||
}
|
||||
terminalCreateNeedsReconciliation = true
|
||||
terminalCreateUnknownOutcomeError ??= error
|
||||
reconcileExisting = true
|
||||
const startsRecovery = recoveryDeadlineAt === null
|
||||
recoveryDeadlineAt ??= Date.now() + REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS
|
||||
if (startsRecovery && !recovery.isActive) {
|
||||
recovery.begin()
|
||||
}
|
||||
if (destroyed || lifecycleEpoch !== expectedLifecycleEpoch) {
|
||||
break
|
||||
}
|
||||
const remainingMs = recoveryDeadlineAt - Date.now()
|
||||
if (remainingMs <= 0 || terminalCreateRecoveryCutoffReached()) {
|
||||
break
|
||||
}
|
||||
const delayMs =
|
||||
TERMINAL_CREATE_RETRY_DELAYS_MS[
|
||||
Math.min(retryAttempt, TERMINAL_CREATE_RETRY_DELAYS_MS.length - 1)
|
||||
]
|
||||
retryAttempt += 1
|
||||
if (!(await waitForTerminalCreateRetry(Math.min(delayMs, remainingMs)))) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
async function closeRemoteTerminal(
|
||||
handleOverride?: string,
|
||||
environmentId = currentRuntimeEnvironmentId
|
||||
): Promise<void> {
|
||||
const targetHandle = handleOverride ?? handle
|
||||
if (!targetHandle) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
await callRuntime('terminal.close', { terminal: targetHandle })
|
||||
await callRuntimeForEnvironment(environmentId, 'terminal.close', { terminal: targetHandle })
|
||||
} catch {
|
||||
// Best-effort parity with local disconnect/kill.
|
||||
}
|
||||
}
|
||||
|
||||
function recoveryBlocksIo(): boolean {
|
||||
return recovery.isActive || recovery.currentPhase === 'disconnected'
|
||||
}
|
||||
|
||||
async function sendInputAcceptedToRuntime(data: string): Promise<boolean> {
|
||||
const targetHandle = handle
|
||||
if (!connected || !targetHandle) {
|
||||
if (!connected || !targetHandle || recoveryBlocksIo()) {
|
||||
return false
|
||||
}
|
||||
if (!data) {
|
||||
return true
|
||||
}
|
||||
await inputBatcher.drain()
|
||||
if (!connected || handle !== targetHandle) {
|
||||
if (!connected || handle !== targetHandle || recoveryBlocksIo()) {
|
||||
return false
|
||||
}
|
||||
if (pendingViewportClaim && !getCurrentMultiplexedStream(targetHandle)) {
|
||||
|
|
@ -368,7 +619,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
try {
|
||||
for (const chunk of iterateTerminalInputChunks(text)) {
|
||||
if (!connected || handle !== targetHandle) {
|
||||
if (!connected || handle !== targetHandle || recoveryBlocksIo()) {
|
||||
return false
|
||||
}
|
||||
// Why: acknowledged sends order behind pending debounce text but must not collapse large paste back into one remote RPC.
|
||||
|
|
@ -392,7 +643,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
|
||||
const inputBatcher = createRemoteRuntimePtyTextBatcher(REMOTE_TERMINAL_INPUT_FLUSH_MS, (text) => {
|
||||
const targetHandle = handle
|
||||
if (!connected || !targetHandle) {
|
||||
if (!connected || !targetHandle || recoveryBlocksIo()) {
|
||||
return
|
||||
}
|
||||
const stream = getCurrentMultiplexedStream(targetHandle)
|
||||
|
|
@ -416,7 +667,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
|
||||
function sendViewportUpdate(cols: number, rows: number, claim = false): void {
|
||||
const targetHandle = handle
|
||||
if (!connected || !targetHandle) {
|
||||
if (!connected || !targetHandle || recoveryBlocksIo()) {
|
||||
return
|
||||
}
|
||||
const stream = getCurrentMultiplexedStream(targetHandle)
|
||||
|
|
@ -456,6 +707,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
multiplexedStream?.close()
|
||||
multiplexedStream = null
|
||||
multiplexedStreamHandle = null
|
||||
attachmentReady = false
|
||||
}
|
||||
|
||||
function clearPublishedHandleWait(): void {
|
||||
|
|
@ -474,13 +726,18 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
|
||||
function retireRemoteTerminalId(): void {
|
||||
recovery.cancel()
|
||||
recoveryRequiresReplacement = false
|
||||
connected = false
|
||||
connecting = false
|
||||
terminalEnded = true
|
||||
clearPublishedHandleWait()
|
||||
clearPendingViewportClaim()
|
||||
const stalePtyId = remotePtyId
|
||||
handle = null
|
||||
remotePtyId = null
|
||||
closeMultiplexedStream()
|
||||
emitRecoveryState()
|
||||
if (stalePtyId) {
|
||||
onPtyExit?.(stalePtyId)
|
||||
}
|
||||
|
|
@ -491,6 +748,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
const replacedPtyId = remotePtyId
|
||||
handle = nextHandle
|
||||
remotePtyId = toRemoteRuntimePtyId(nextHandle, currentRuntimeEnvironmentId)
|
||||
attachmentReady = false
|
||||
// Why: host handle rotation preserves the pane generation; only the store identity changes, not spawn/exit semantics.
|
||||
if (replacedPtyId) {
|
||||
replaceFitOverridePtyId(replacedPtyId, remotePtyId)
|
||||
|
|
@ -527,7 +785,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
const reboundHandle = handle
|
||||
const reboundPtyId = remotePtyId
|
||||
void subscribeToHandle().catch((error) => {
|
||||
if (reboundHandle && isCurrentRemoteTerminal(reboundHandle, reboundPtyId)) {
|
||||
if (reboundHandle && !recoverAfterSubscribeFailure(error, reboundHandle, reboundPtyId)) {
|
||||
handleRemoteTerminalError(error)
|
||||
}
|
||||
})
|
||||
|
|
@ -556,13 +814,40 @@ export function createRemoteRuntimePtyTransport(
|
|||
retireRemoteTerminalId()
|
||||
return
|
||||
}
|
||||
if (isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error))) {
|
||||
// Why: a partition is attachment state, not a terminal failure; keep the red error surface for actionable fatal errors.
|
||||
scheduleResubscribeAfterTransportClose()
|
||||
return
|
||||
}
|
||||
connecting = false
|
||||
emitRecoveryState()
|
||||
storedCallbacks.onError?.(message)
|
||||
}
|
||||
|
||||
function recoverAfterSubscribeFailure(
|
||||
error: unknown,
|
||||
targetHandle: string,
|
||||
targetPtyId: string | null
|
||||
): boolean {
|
||||
if (!isCurrentRemoteTerminal(targetHandle, targetPtyId)) {
|
||||
return true
|
||||
}
|
||||
if (multiplexedStreamHandle !== targetHandle) {
|
||||
closeMultiplexedStream()
|
||||
}
|
||||
clearPendingViewportClaim()
|
||||
if (!isRecoverableRemoteRuntimeConnectionError(toRemoteRuntimeClientErrorLike(error))) {
|
||||
return false
|
||||
}
|
||||
scheduleResubscribeAfterTransportClose()
|
||||
return true
|
||||
}
|
||||
|
||||
// Why: after a transport drop the host may have re-minted this handle; re-derive from the snapshot so we don't mirror/type into whatever PTY now sits behind the stale one (#7718).
|
||||
async function resubscribeAfterTransportClose(
|
||||
previousHandle: string,
|
||||
requireReplacement: boolean
|
||||
requireReplacement: boolean,
|
||||
recoveryEpoch: number
|
||||
): Promise<void> {
|
||||
if (tabId && isWebTerminalSurfaceTabId(tabId)) {
|
||||
const hostTabId = toHostSessionTabId(tabId)
|
||||
|
|
@ -571,7 +856,12 @@ export function createRemoteRuntimePtyTransport(
|
|||
previousHandle,
|
||||
requireReplacement
|
||||
)
|
||||
if (destroyed || !connected || handle !== previousHandle) {
|
||||
if (
|
||||
destroyed ||
|
||||
!connected ||
|
||||
handle !== previousHandle ||
|
||||
!recovery.isCurrent(recoveryEpoch)
|
||||
) {
|
||||
return
|
||||
}
|
||||
if (nextHandle === undefined) {
|
||||
|
|
@ -587,18 +877,33 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
}
|
||||
clearPublishedHandleWait()
|
||||
await subscribeToHandle()
|
||||
await subscribeToHandle(recoveryEpoch)
|
||||
}
|
||||
|
||||
function scheduleResubscribeAfterTransportClose(requireReplacement = false): void {
|
||||
function scheduleResubscribeAfterTransportClose(
|
||||
requireReplacement = false,
|
||||
requestedRecoveryEpoch?: number
|
||||
): void {
|
||||
if (destroyed || !connected || !handle) {
|
||||
return
|
||||
}
|
||||
const recoveryWasActive = recovery.isActive
|
||||
const recoveryEpoch = requestedRecoveryEpoch ?? recovery.begin()
|
||||
if (!recovery.isCurrent(recoveryEpoch)) {
|
||||
return
|
||||
}
|
||||
if (!recoveryWasActive) {
|
||||
// Why: bytes queued before a partition have unknown delivery; never replay them on a replacement stream.
|
||||
inputBatcher.clear()
|
||||
viewportBatcher.clear()
|
||||
clearPendingViewportClaim()
|
||||
}
|
||||
recoveryRequiresReplacement ||= requireReplacement
|
||||
if (requireReplacement && stopWaitingForPublishedHandle) {
|
||||
// Why: once recovery is handed to accepted snapshots, repeated sends to the stale handle must not re-arm inventory RPCs.
|
||||
return
|
||||
}
|
||||
if (resubscribing) {
|
||||
if (resubscribeEpoch === recoveryEpoch) {
|
||||
// Why: concurrent stale errors belong to their own handle; don't carry an old handle's replacement requirement onto its successor.
|
||||
if (resubscribeRequestedHandle !== handle) {
|
||||
resubscribeRequestedHandle = handle
|
||||
|
|
@ -614,39 +919,63 @@ export function createRemoteRuntimePtyTransport(
|
|||
// Why: subscribe before polling so a fresh host snapshot can't land in the gap between the inventory loop and its event-driven fallback.
|
||||
waitForPublishedHostSessionHandle(toHostSessionTabId(tabId), resubscribeHandle)
|
||||
}
|
||||
resubscribing = true
|
||||
void resubscribeAfterTransportClose(resubscribeHandle, requireReplacement)
|
||||
resubscribeEpoch = recoveryEpoch
|
||||
resubscribeRequestedHandle = null
|
||||
resubscribeRequestedRequiresReplacement = false
|
||||
let retryScheduled = false
|
||||
void resubscribeAfterTransportClose(resubscribeHandle, requireReplacement, recoveryEpoch)
|
||||
.catch((error) => {
|
||||
if (!destroyed && connected && handle) {
|
||||
if (!destroyed && connected && handle && recovery.isCurrent(recoveryEpoch)) {
|
||||
clearPendingViewportClaim()
|
||||
handleRemoteTerminalError(error)
|
||||
const clientError = toRemoteRuntimeClientErrorLike(error)
|
||||
if (isRecoverableRemoteRuntimeConnectionError(clientError)) {
|
||||
retryScheduled = recovery.schedule(recoveryEpoch, (nextEpoch) => {
|
||||
scheduleResubscribeAfterTransportClose(requireReplacement, nextEpoch)
|
||||
})
|
||||
} else {
|
||||
recovery.cancel()
|
||||
handleRemoteTerminalError(error)
|
||||
}
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
resubscribing = false
|
||||
if (resubscribeEpoch !== recoveryEpoch) {
|
||||
return
|
||||
}
|
||||
resubscribeEpoch = null
|
||||
const pendingHandle = resubscribeRequestedHandle
|
||||
const pendingRequiresReplacement = resubscribeRequestedRequiresReplacement
|
||||
resubscribeRequestedHandle = null
|
||||
resubscribeRequestedRequiresReplacement = false
|
||||
if (!stopWaitingForPublishedHandle && pendingHandle && pendingHandle === handle) {
|
||||
if (
|
||||
!retryScheduled &&
|
||||
recovery.isCurrent(recoveryEpoch) &&
|
||||
!stopWaitingForPublishedHandle &&
|
||||
pendingHandle &&
|
||||
pendingHandle === handle &&
|
||||
!getCurrentMultiplexedStream(pendingHandle)
|
||||
) {
|
||||
scheduleResubscribeAfterTransportClose(pendingRequiresReplacement)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function subscribeToHandle(): Promise<void> {
|
||||
async function subscribeToHandle(expectedRecoveryEpoch?: number): Promise<void> {
|
||||
if (!handle) {
|
||||
return
|
||||
}
|
||||
const subscribedHandle = handle
|
||||
const subscribedPtyId = remotePtyId
|
||||
const generation = ++subscriptionGeneration
|
||||
attachmentReady = false
|
||||
let transportClosed = false
|
||||
let subscriptionAttached = false
|
||||
// Why: viewport handed to subscribe; a resize during the round-trip falls back to the refresh-only one-shot RPC, replayed through the stream below once current.
|
||||
const subscribedViewport = desiredViewport
|
||||
const isCurrentSubscription = (): boolean =>
|
||||
!transportClosed &&
|
||||
generation === subscriptionGeneration &&
|
||||
(expectedRecoveryEpoch === undefined || recovery.ownsEpoch(expectedRecoveryEpoch)) &&
|
||||
isCurrentRemoteTerminal(subscribedHandle, subscribedPtyId)
|
||||
const nextStream = await getRemoteRuntimeTerminalMultiplexer(
|
||||
currentRuntimeEnvironmentId
|
||||
|
|
@ -676,6 +1005,12 @@ export function createRemoteRuntimePtyTransport(
|
|||
if (!isCurrentSubscription()) {
|
||||
return
|
||||
}
|
||||
subscriptionAttached = true
|
||||
attachmentReady = true
|
||||
connecting = false
|
||||
recoveryRequiresReplacement = false
|
||||
recovery.markHealthy()
|
||||
emitRecoveryState()
|
||||
storedCallbacks.onConnect?.()
|
||||
storedCallbacks.onStatus?.('shell')
|
||||
},
|
||||
|
|
@ -685,11 +1020,15 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
outputProcessor.clearAccumulatedState()
|
||||
connected = false
|
||||
connecting = false
|
||||
handle = null
|
||||
remotePtyId = null
|
||||
multiplexedStream = null
|
||||
multiplexedStreamHandle = null
|
||||
attachmentReady = false
|
||||
terminalEnded = true
|
||||
clearPendingViewportClaim()
|
||||
emitRecoveryState()
|
||||
storedCallbacks.onExit?.(0)
|
||||
storedCallbacks.onDisconnect?.()
|
||||
if (subscribedPtyId) {
|
||||
|
|
@ -711,7 +1050,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
setDriverForPty(subscribedPtyId, driver)
|
||||
}
|
||||
},
|
||||
onTransportClose: () => {
|
||||
onTransportClose: ({ recoverable }) => {
|
||||
transportClosed = true
|
||||
if (generation !== subscriptionGeneration) {
|
||||
return
|
||||
|
|
@ -724,13 +1063,21 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
multiplexedStream = null
|
||||
multiplexedStreamHandle = null
|
||||
scheduleResubscribeAfterTransportClose()
|
||||
attachmentReady = false
|
||||
if (recoverable) {
|
||||
scheduleResubscribeAfterTransportClose()
|
||||
} else {
|
||||
connecting = false
|
||||
recovery.cancel()
|
||||
emitRecoveryState()
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
if (
|
||||
transportClosed ||
|
||||
generation !== subscriptionGeneration ||
|
||||
(expectedRecoveryEpoch !== undefined && !recovery.ownsEpoch(expectedRecoveryEpoch)) ||
|
||||
destroyed ||
|
||||
!connected ||
|
||||
handle !== subscribedHandle ||
|
||||
|
|
@ -742,6 +1089,11 @@ export function createRemoteRuntimePtyTransport(
|
|||
closeMultiplexedStream()
|
||||
multiplexedStream = nextStream
|
||||
multiplexedStreamHandle = subscribedHandle
|
||||
attachmentReady = subscriptionAttached
|
||||
if (subscriptionAttached) {
|
||||
recoveryRequiresReplacement = false
|
||||
recovery.markHealthy()
|
||||
}
|
||||
// Why: a viewport change during the subscribe round-trip hit the no-op one-shot fallback; replay the latest viewport so the PTY isn't stuck at subscribe-time size.
|
||||
if (pendingViewportClaim && desiredViewport) {
|
||||
nextStream.claimViewport(desiredViewport.cols, desiredViewport.rows)
|
||||
|
|
@ -764,9 +1116,17 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
}
|
||||
|
||||
return {
|
||||
const transport: PtyTransport = {
|
||||
async connect(options) {
|
||||
cancelTerminalCreateRetryWait()
|
||||
const connectLifecycleEpoch = ++lifecycleEpoch
|
||||
const createEnvironmentId = currentRuntimeEnvironmentId
|
||||
lastConnectOptions = options
|
||||
storedCallbacks = options.callbacks
|
||||
recoveryRequiresReplacement = false
|
||||
terminalEnded = false
|
||||
connecting = true
|
||||
emitRecoveryState(true)
|
||||
if (destroyed || !worktreeId) {
|
||||
return
|
||||
}
|
||||
|
|
@ -785,34 +1145,50 @@ export function createRemoteRuntimePtyTransport(
|
|||
const resumeProviderSessionToSend = options.resumeProviderSession ?? resumeProviderSession
|
||||
const launchTokenToSend = options.launchToken ?? launchToken
|
||||
const launchAgentToSend = options.launchAgent ?? launchAgent
|
||||
const created = await callRuntime<{ terminal: RuntimeTerminalCreate }>('terminal.create', {
|
||||
worktree: toRuntimeTerminalWorktreeSelector(worktreeId),
|
||||
...(commandToSend !== undefined ? { command: commandToSend } : {}),
|
||||
...(startupCommandDeliveryToSend !== undefined
|
||||
? { startupCommandDelivery: startupCommandDeliveryToSend }
|
||||
: {}),
|
||||
...(envToSend !== undefined ? { env: envToSend } : {}),
|
||||
...(envToDeleteToSend !== undefined ? { envToDelete: envToDeleteToSend } : {}),
|
||||
...(launchConfigToSend !== undefined ? { launchConfig: launchConfigToSend } : {}),
|
||||
...(resumeProviderSessionToSend !== undefined
|
||||
? { resumeProviderSession: resumeProviderSessionToSend }
|
||||
: {}),
|
||||
...(launchTokenToSend !== undefined ? { launchToken: launchTokenToSend } : {}),
|
||||
...(launchAgentToSend !== undefined ? { launchAgent: launchAgentToSend } : {}),
|
||||
...(terminalColorQueryReplies ? { terminalColorQueryReplies } : {}),
|
||||
tabId,
|
||||
leafId,
|
||||
focus: false,
|
||||
// Why: transport backs an already-mounted pane; activation is local state, not permission for remote UI reveal.
|
||||
presentation: 'background',
|
||||
...(activate === true ? { activate: true } : {})
|
||||
})
|
||||
handle = created.terminal.handle
|
||||
if (destroyed) {
|
||||
// Why: cancelled launch, not a shared session; close the server PTY so rapid tab-open/close does not leak.
|
||||
await closeRemoteTerminal(created.terminal.handle)
|
||||
const created = await createTerminalWithUnknownOutcomeRecovery(
|
||||
{
|
||||
worktree: toRuntimeTerminalWorktreeSelector(worktreeId),
|
||||
clientMutationId: terminalCreateMutationId,
|
||||
...(commandToSend !== undefined ? { command: commandToSend } : {}),
|
||||
...(startupCommandDeliveryToSend !== undefined
|
||||
? { startupCommandDelivery: startupCommandDeliveryToSend }
|
||||
: {}),
|
||||
...(envToSend !== undefined ? { env: envToSend } : {}),
|
||||
...(envToDeleteToSend !== undefined ? { envToDelete: envToDeleteToSend } : {}),
|
||||
...(launchConfigToSend !== undefined ? { launchConfig: launchConfigToSend } : {}),
|
||||
...(resumeProviderSessionToSend !== undefined
|
||||
? { resumeProviderSession: resumeProviderSessionToSend }
|
||||
: {}),
|
||||
...(launchTokenToSend !== undefined ? { launchToken: launchTokenToSend } : {}),
|
||||
...(launchAgentToSend !== undefined ? { launchAgent: launchAgentToSend } : {}),
|
||||
...(terminalColorQueryReplies ? { terminalColorQueryReplies } : {}),
|
||||
tabId,
|
||||
leafId,
|
||||
focus: false,
|
||||
// Why: transport backs an already-mounted pane; activation is local state, not permission for remote UI reveal.
|
||||
presentation: 'background',
|
||||
...(activate === true ? { activate: true } : {})
|
||||
},
|
||||
createEnvironmentId,
|
||||
connectLifecycleEpoch
|
||||
)
|
||||
if (!created) {
|
||||
if (!destroyed && lifecycleEpoch === connectLifecycleEpoch) {
|
||||
connecting = false
|
||||
recovery.markDisconnected()
|
||||
}
|
||||
return
|
||||
}
|
||||
if (destroyed || lifecycleEpoch !== connectLifecycleEpoch) {
|
||||
if (
|
||||
created.terminal.handle !== handle ||
|
||||
createEnvironmentId !== currentRuntimeEnvironmentId
|
||||
) {
|
||||
await closeRemoteTerminal(created.terminal.handle, createEnvironmentId)
|
||||
}
|
||||
return
|
||||
}
|
||||
handle = created.terminal.handle
|
||||
|
||||
remotePtyId = toRemoteRuntimePtyId(handle, currentRuntimeEnvironmentId)
|
||||
connected = true
|
||||
|
|
@ -821,8 +1197,15 @@ export function createRemoteRuntimePtyTransport(
|
|||
rows: options.rows ?? 24
|
||||
}
|
||||
onPtySpawn?.(remotePtyId)
|
||||
emitRecoveryState()
|
||||
|
||||
await subscribeToHandle()
|
||||
try {
|
||||
await subscribeToHandle()
|
||||
} catch (error) {
|
||||
if (!recoverAfterSubscribeFailure(error, handle, remotePtyId)) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
if (destroyed || !connected || !remotePtyId) {
|
||||
return
|
||||
}
|
||||
|
|
@ -832,14 +1215,26 @@ export function createRemoteRuntimePtyTransport(
|
|||
replay: ''
|
||||
} satisfies PtyConnectResult
|
||||
} catch (error) {
|
||||
storedCallbacks.onError?.(runtimeTerminalErrorMessage(error))
|
||||
if (!destroyed && lifecycleEpoch === connectLifecycleEpoch) {
|
||||
connecting = false
|
||||
recovery.cancel()
|
||||
storedCallbacks.onError?.(runtimeTerminalErrorMessage(error))
|
||||
emitRecoveryState()
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
},
|
||||
|
||||
attach(options) {
|
||||
lifecycleEpoch += 1
|
||||
cancelTerminalCreateRetryWait()
|
||||
recovery.cancel()
|
||||
recoveryRequiresReplacement = false
|
||||
clearPublishedHandleWait()
|
||||
storedCallbacks = options.callbacks
|
||||
terminalEnded = false
|
||||
connecting = true
|
||||
emitRecoveryState(true)
|
||||
currentRuntimeEnvironmentId =
|
||||
getRemoteRuntimePtyEnvironmentId(options.existingPtyId) ?? runtimeEnvironmentId
|
||||
const previousHandle = handle
|
||||
|
|
@ -851,8 +1246,10 @@ export function createRemoteRuntimePtyTransport(
|
|||
handle = nextHandle
|
||||
if (!handle) {
|
||||
connected = false
|
||||
connecting = false
|
||||
remotePtyId = null
|
||||
closeMultiplexedStream()
|
||||
emitRecoveryState()
|
||||
storedCallbacks.onError?.('Remote runtime terminal id is invalid.')
|
||||
return
|
||||
}
|
||||
|
|
@ -865,19 +1262,19 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
const targetHandle = handle
|
||||
const targetPtyId = remotePtyId
|
||||
emitRecoveryState()
|
||||
void subscribeToHandle().catch((error) => {
|
||||
if (!isCurrentRemoteTerminal(targetHandle, targetPtyId)) {
|
||||
return
|
||||
if (!recoverAfterSubscribeFailure(error, targetHandle, targetPtyId)) {
|
||||
handleRemoteTerminalError(error)
|
||||
}
|
||||
if (handle === targetHandle && multiplexedStreamHandle !== targetHandle) {
|
||||
closeMultiplexedStream()
|
||||
}
|
||||
clearPendingViewportClaim()
|
||||
handleRemoteTerminalError(error)
|
||||
})
|
||||
},
|
||||
|
||||
disconnect() {
|
||||
lifecycleEpoch += 1
|
||||
cancelTerminalCreateRetryWait()
|
||||
recovery.cancel()
|
||||
recoveryRequiresReplacement = false
|
||||
clearPublishedHandleWait()
|
||||
inputBatcher.flush()
|
||||
inputBatcher.clear()
|
||||
|
|
@ -887,11 +1284,14 @@ export function createRemoteRuntimePtyTransport(
|
|||
return
|
||||
}
|
||||
connected = false
|
||||
connecting = false
|
||||
terminalEnded = true
|
||||
clearPendingViewportClaim()
|
||||
const id = remotePtyId
|
||||
closeMultiplexedStream()
|
||||
handle = null
|
||||
remotePtyId = null
|
||||
emitRecoveryState()
|
||||
storedCallbacks.onDisconnect?.()
|
||||
if (id) {
|
||||
onPtyExit?.(id)
|
||||
|
|
@ -899,19 +1299,25 @@ export function createRemoteRuntimePtyTransport(
|
|||
},
|
||||
|
||||
detach() {
|
||||
lifecycleEpoch += 1
|
||||
cancelTerminalCreateRetryWait()
|
||||
recovery.cancel()
|
||||
recoveryRequiresReplacement = false
|
||||
clearPublishedHandleWait()
|
||||
inputBatcher.flush()
|
||||
inputBatcher.clear()
|
||||
viewportBatcher.flush()
|
||||
outputProcessor.clearAccumulatedState()
|
||||
connected = false
|
||||
connecting = false
|
||||
clearPendingViewportClaim()
|
||||
closeMultiplexedStream()
|
||||
emitRecoveryState()
|
||||
storedCallbacks = {}
|
||||
},
|
||||
|
||||
sendInput(data: string): boolean {
|
||||
if (!connected || !handle) {
|
||||
if (!connected || !handle || recoveryBlocksIo()) {
|
||||
return false
|
||||
}
|
||||
if (!data) {
|
||||
|
|
@ -924,7 +1330,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
// Why: query replies (CPR/DSR/DA/OSC) are read in raw mode with a short timeout; the 8ms debounce would miss it and echo the reply onto the prompt (#7329).
|
||||
sendInputImmediate(data: string): boolean {
|
||||
const targetHandle = handle
|
||||
if (!connected || !targetHandle) {
|
||||
if (!connected || !targetHandle || recoveryBlocksIo()) {
|
||||
return false
|
||||
}
|
||||
if (!data) {
|
||||
|
|
@ -964,6 +1370,9 @@ export function createRemoteRuntimePtyTransport(
|
|||
return false
|
||||
}
|
||||
rememberViewport(cols, rows)
|
||||
if (recoveryBlocksIo()) {
|
||||
return true
|
||||
}
|
||||
viewportBatcher.clear()
|
||||
sendViewportUpdate(cols, rows, true)
|
||||
return true
|
||||
|
|
@ -974,6 +1383,9 @@ export function createRemoteRuntimePtyTransport(
|
|||
return false
|
||||
}
|
||||
rememberViewport(cols, rows)
|
||||
if (recoveryBlocksIo()) {
|
||||
return true
|
||||
}
|
||||
if (meta?.claim) {
|
||||
viewportBatcher.clear()
|
||||
sendViewportUpdate(cols, rows, true)
|
||||
|
|
@ -985,7 +1397,43 @@ export function createRemoteRuntimePtyTransport(
|
|||
},
|
||||
|
||||
isConnected() {
|
||||
return connected
|
||||
return (
|
||||
connected &&
|
||||
!recoveryBlocksIo() &&
|
||||
attachmentReady &&
|
||||
multiplexedStream !== null &&
|
||||
multiplexedStreamHandle === handle
|
||||
)
|
||||
},
|
||||
|
||||
getRecoveryState,
|
||||
|
||||
retryRecovery() {
|
||||
if (
|
||||
!destroyed &&
|
||||
!terminalEnded &&
|
||||
!connected &&
|
||||
!handle &&
|
||||
terminalCreateNeedsReconciliation &&
|
||||
lastConnectOptions &&
|
||||
recovery.currentPhase === 'disconnected'
|
||||
) {
|
||||
recovery.begin()
|
||||
void transport.connect(lastConnectOptions)
|
||||
return true
|
||||
}
|
||||
if (
|
||||
destroyed ||
|
||||
terminalEnded ||
|
||||
!connected ||
|
||||
!handle ||
|
||||
recovery.currentPhase !== 'disconnected'
|
||||
) {
|
||||
return false
|
||||
}
|
||||
const recoveryEpoch = recovery.begin()
|
||||
scheduleResubscribeAfterTransportClose(recoveryRequiresReplacement, recoveryEpoch)
|
||||
return true
|
||||
},
|
||||
|
||||
getPtyId() {
|
||||
|
|
@ -1010,8 +1458,10 @@ export function createRemoteRuntimePtyTransport(
|
|||
destroy() {
|
||||
destroyed = true
|
||||
this.disconnect()
|
||||
recovery.dispose()
|
||||
inputBatcher.clear()
|
||||
viewportBatcher.clear()
|
||||
}
|
||||
}
|
||||
return transport
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import { updateTerminalRemoteRuntimeRecoveryUiState } from './terminal-remote-runtime-recovery-ui-state'
|
||||
|
||||
describe('updateTerminalRemoteRuntimeRecoveryUiState', () => {
|
||||
it('tracks only recovery phases that need pane UI', () => {
|
||||
const recovering = updateTerminalRemoteRuntimeRecoveryUiState({}, 7, {
|
||||
phase: 'recovering',
|
||||
epoch: 2,
|
||||
attempt: 1
|
||||
})
|
||||
|
||||
expect(recovering[7]?.phase).toBe('recovering')
|
||||
expect(
|
||||
updateTerminalRemoteRuntimeRecoveryUiState(recovering, 7, {
|
||||
phase: 'connected',
|
||||
epoch: 2,
|
||||
attempt: 0
|
||||
})
|
||||
).toEqual({})
|
||||
})
|
||||
|
||||
it('drops disconnected state when the owning pane closes', () => {
|
||||
const disconnected = updateTerminalRemoteRuntimeRecoveryUiState({}, 7, {
|
||||
phase: 'disconnected',
|
||||
epoch: 2,
|
||||
attempt: 4
|
||||
})
|
||||
|
||||
expect(updateTerminalRemoteRuntimeRecoveryUiState(disconnected, 7, null)).toEqual({})
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
import type { PtyTransportRecoveryState } from './pty-transport-types'
|
||||
|
||||
export type VisiblePtyRecoveryState = Omit<PtyTransportRecoveryState, 'phase'> & {
|
||||
phase: Extract<PtyTransportRecoveryState['phase'], 'recovering' | 'backoff' | 'disconnected'>
|
||||
}
|
||||
|
||||
function isVisiblePtyRecoveryState(
|
||||
state: PtyTransportRecoveryState | null
|
||||
): state is VisiblePtyRecoveryState {
|
||||
return (
|
||||
state?.phase === 'recovering' || state?.phase === 'backoff' || state?.phase === 'disconnected'
|
||||
)
|
||||
}
|
||||
|
||||
export function updateTerminalRemoteRuntimeRecoveryUiState(
|
||||
previous: Record<number, VisiblePtyRecoveryState>,
|
||||
paneId: number,
|
||||
state: PtyTransportRecoveryState | null
|
||||
): Record<number, VisiblePtyRecoveryState> {
|
||||
if (isVisiblePtyRecoveryState(state)) {
|
||||
return previous[paneId] === state ? previous : { ...previous, [paneId]: state }
|
||||
}
|
||||
if (!(paneId in previous)) {
|
||||
return previous
|
||||
}
|
||||
const next = { ...previous }
|
||||
delete next[paneId]
|
||||
return next
|
||||
}
|
||||
|
|
@ -100,6 +100,7 @@ import type { EffectiveMacOptionAsAlt } from '@/lib/keyboard-layout/detect-optio
|
|||
import { resolveEffectiveTerminalAppearance } from '@/lib/terminal-theme'
|
||||
import { connectPanePty } from './pty-connection'
|
||||
import type { PtyTransport } from './pty-transport'
|
||||
import type { PtyTransportRecoveryState } from './pty-transport-types'
|
||||
import {
|
||||
reconcileMissingSessions,
|
||||
type ReconcilableBinding
|
||||
|
|
@ -264,6 +265,9 @@ type UseTerminalPaneLifecycleDeps = {
|
|||
onPtyExitRef: React.RefObject<(ptyId: string) => void>
|
||||
onAgentExitedRef: React.RefObject<(leafId: string) => void>
|
||||
onPtyErrorRef?: React.RefObject<(paneId: number, message: string) => void>
|
||||
onPtyRecoveryStateRef?: React.RefObject<
|
||||
(paneId: number, state: PtyTransportRecoveryState | null) => void
|
||||
>
|
||||
clearTabPtyId: (tabId: string, ptyId: string) => void
|
||||
consumeSuppressedPtyExit: (ptyId: string) => boolean
|
||||
updateTabTitle: (tabId: string, title: string) => void
|
||||
|
|
@ -519,6 +523,7 @@ export function useTerminalPaneLifecycle({
|
|||
onPtyExitRef,
|
||||
onAgentExitedRef,
|
||||
onPtyErrorRef,
|
||||
onPtyRecoveryStateRef,
|
||||
clearTabPtyId,
|
||||
consumeSuppressedPtyExit,
|
||||
updateTabTitle,
|
||||
|
|
@ -731,6 +736,7 @@ export function useTerminalPaneLifecycle({
|
|||
onPtyExitRef,
|
||||
onAgentExitedRef,
|
||||
onPtyErrorRef,
|
||||
onPtyRecoveryStateRef,
|
||||
clearTabPtyId,
|
||||
consumeSuppressedPtyExit,
|
||||
updateTabTitle,
|
||||
|
|
@ -1126,6 +1132,7 @@ export function useTerminalPaneLifecycle({
|
|||
queueResizeAll(true)
|
||||
},
|
||||
onPaneClosed: (paneId, closedPane) => {
|
||||
onPtyRecoveryStateRef?.current?.(paneId, null)
|
||||
const isDetachedToTab = closedPane?.reason === 'detach'
|
||||
const linkProviderDisposable = linkProviderDisposablesRef.current.get(paneId)
|
||||
if (linkProviderDisposable) {
|
||||
|
|
|
|||
|
|
@ -2737,6 +2737,13 @@
|
|||
"removedTitle": "SSH host removed",
|
||||
"removedBody": "The SSH host for this workspace was removed, so it can no longer connect. Remove the workspace to clear it — remote files are left untouched.",
|
||||
"removeWorkspaceButton": "Remove workspace"
|
||||
},
|
||||
"TerminalRemoteRuntimeReconnectBanner": {
|
||||
"retryingTitle": "Reconnecting to remote runtime",
|
||||
"disconnectedTitle": "Remote runtime disconnected",
|
||||
"retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.",
|
||||
"disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.",
|
||||
"reconnectButton": "Reconnect"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2714,6 +2714,13 @@
|
|||
"removedTitle": "Host SSH eliminado",
|
||||
"removedBody": "El host SSH de este espacio de trabajo fue eliminado, por lo que ya no puede conectarse. Elimina el espacio de trabajo para limpiarlo — los archivos remotos permanecen intactos.",
|
||||
"removeWorkspaceButton": "Eliminar espacio de trabajo"
|
||||
},
|
||||
"TerminalRemoteRuntimeReconnectBanner": {
|
||||
"retryingTitle": "Reconnecting to remote runtime",
|
||||
"disconnectedTitle": "Remote runtime disconnected",
|
||||
"retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.",
|
||||
"disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.",
|
||||
"reconnectButton": "Reconnect"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2714,6 +2714,13 @@
|
|||
"removedTitle": "SSHホストが削除されました",
|
||||
"removedBody": "このワークスペースのSSHホストが削除されたため、接続できません。ワークスペースを削除してクリアしてください — リモートファイルはそのまま残ります。",
|
||||
"removeWorkspaceButton": "ワークスペースを削除"
|
||||
},
|
||||
"TerminalRemoteRuntimeReconnectBanner": {
|
||||
"retryingTitle": "Reconnecting to remote runtime",
|
||||
"disconnectedTitle": "Remote runtime disconnected",
|
||||
"retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.",
|
||||
"disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.",
|
||||
"reconnectButton": "Reconnect"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2714,6 +2714,13 @@
|
|||
"removedTitle": "SSH 호스트가 제거됨",
|
||||
"removedBody": "이 워크트리의 SSH 호스트가 제거되어 더 이상 연결할 수 없습니다. 워크트리를 제거하여 정리하세요 — 원격 파일은 그대로 남아 있습니다.",
|
||||
"removeWorkspaceButton": "워크트리 제거"
|
||||
},
|
||||
"TerminalRemoteRuntimeReconnectBanner": {
|
||||
"retryingTitle": "Reconnecting to remote runtime",
|
||||
"disconnectedTitle": "Remote runtime disconnected",
|
||||
"retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.",
|
||||
"disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.",
|
||||
"reconnectButton": "Reconnect"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2714,6 +2714,13 @@
|
|||
"removedTitle": "SSH 主机已移除",
|
||||
"removedBody": "此工作区的 SSH 主机已被移除，无法再连接。请移除工作区以清理 — 远程文件不会受影响。",
|
||||
"removeWorkspaceButton": "移除工作区"
|
||||
},
|
||||
"TerminalRemoteRuntimeReconnectBanner": {
|
||||
"retryingTitle": "Reconnecting to remote runtime",
|
||||
"disconnectedTitle": "Remote runtime disconnected",
|
||||
"retryingBody": "Orca will retry for up to one minute. This terminal will resume if the connection returns.",
|
||||
"disconnectedBody": "Automatic retries stopped. Reconnect to resume this terminal session.",
|
||||
"reconnectButton": "Reconnect"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
/* eslint-disable max-lines -- Why: the remote terminal multiplexer owns one bridged subscription, stream lifecycle, binary frame parsing, and remote lock events as a single transport contract. */
|
||||
import type { RuntimeRpcResponse } from '../../../shared/runtime-rpc-envelope'
|
||||
import { isRecoverableRemoteRuntimeConnectionError } from '../../../shared/remote-runtime-client-error-classification'
|
||||
import {
|
||||
TerminalStreamOpcode,
|
||||
decodeTerminalStreamFrame,
|
||||
|
|
@ -50,7 +51,7 @@ export type RemoteRuntimeMultiplexedTerminalCallbacks = {
|
|||
onDriverChanged?: (
|
||||
driver: { kind: 'idle' } | { kind: 'desktop' } | { kind: 'mobile'; clientId: string }
|
||||
) => void
|
||||
onTransportClose?: () => void
|
||||
onTransportClose?: (event: { recoverable: boolean }) => void
|
||||
}
|
||||
|
||||
export type RemoteRuntimeMultiplexedTerminal = {
|
||||
|
|
@ -72,6 +73,7 @@ type RemoteRuntimeMultiplexedTerminalState = {
|
|||
streamId: number
|
||||
terminal: string
|
||||
callbacks: RemoteRuntimeMultiplexedTerminalCallbacks
|
||||
subscriptionRequested: boolean
|
||||
acknowledgeOutput: boolean
|
||||
heldAckBytes: number
|
||||
snapshotChunks: Uint8Array<ArrayBufferLike>[]
|
||||
|
|
@ -223,6 +225,7 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
streamId,
|
||||
terminal: args.terminal,
|
||||
callbacks: args.callbacks,
|
||||
subscriptionRequested: false,
|
||||
acknowledgeOutput: args.client.type === 'desktop',
|
||||
heldAckBytes: 0,
|
||||
snapshotChunks: [],
|
||||
|
|
@ -295,6 +298,7 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
if (!sent) {
|
||||
throw new Error('Remote terminal stream is not connected.')
|
||||
}
|
||||
state.subscriptionRequested = true
|
||||
} catch (error) {
|
||||
const terminalError = error instanceof Error ? error : new Error(String(error))
|
||||
if (this.streams.get(streamId) === state) {
|
||||
|
|
@ -340,7 +344,13 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
{
|
||||
onResponse: (response) => this.handleResponse(response),
|
||||
onBinary: (bytes) => this.handleBinary(bytes),
|
||||
onError: (error) => this.failConnection(new Error(error.message)),
|
||||
onError: (error) => {
|
||||
if (isRecoverableRemoteRuntimeConnectionError(error)) {
|
||||
this.handleClose(error.message)
|
||||
} else {
|
||||
this.failConnection(Object.assign(new Error(error.message), { code: error.code }))
|
||||
}
|
||||
},
|
||||
onClose: () => this.handleClose('Remote Orca runtime closed the connection.')
|
||||
}
|
||||
)
|
||||
|
|
@ -764,34 +774,36 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
this.readyResolver = null
|
||||
this.readyRejecter = null
|
||||
for (const stream of this.streams.values()) {
|
||||
stream.callbacks.onError?.(error.message)
|
||||
// Why: a stream still awaiting ensureConnected receives this failure through its rejected promise.
|
||||
if (stream.subscriptionRequested) {
|
||||
stream.callbacks.onError?.(error.message)
|
||||
}
|
||||
}
|
||||
this.subscription?.unsubscribe()
|
||||
this.handleClose()
|
||||
this.handleClose(undefined, false)
|
||||
}
|
||||
|
||||
private handleClose(message?: string): void {
|
||||
private handleClose(message?: string, recoverable = true): void {
|
||||
const streams = Array.from(this.streams.values())
|
||||
const closingSubscription = this.subscription
|
||||
this.ready = false
|
||||
this.connectPromise = null
|
||||
this.readyRejecter?.(new Error(message ?? 'Remote runtime connection closed.'))
|
||||
this.readyResolver = null
|
||||
this.readyRejecter = null
|
||||
this.subscription = null
|
||||
closingSubscription?.unsubscribe()
|
||||
this.streams.clear()
|
||||
// Why: close callbacks may resubscribe synchronously; release first so every replacement shares the new environment multiplexer.
|
||||
this.releaseIfCurrent(this.environmentId, this)
|
||||
for (const stream of streams) {
|
||||
clearSnapshot(stream)
|
||||
rejectPendingSnapshotRequest(stream, message ?? 'Remote runtime connection closed.')
|
||||
const canHandleClose = Boolean(stream.callbacks.onTransportClose)
|
||||
stream.callbacks.onTransportClose?.()
|
||||
stream.callbacks.onTransportClose?.({ recoverable })
|
||||
if (message && !canHandleClose) {
|
||||
stream.callbacks.onError?.(message)
|
||||
}
|
||||
}
|
||||
// Why: a closed transport has no live streams or subscription; keeping it
|
||||
// in the module map only retains callbacks for an environment that must
|
||||
// reconnect through a fresh subscription anyway.
|
||||
this.releaseIfCurrent(this.environmentId, this)
|
||||
}
|
||||
|
||||
private closeIfIdle(): void {
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ export const TERMINAL_QUICK_COMMANDS_RUNTIME_CAPABILITY = 'terminal.quick-comman
|
|||
// replay ambiguous cutovers when the host advertises idempotent create support.
|
||||
export const WORKTREE_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY =
|
||||
'worktree.create-idempotency.v1' as const
|
||||
// Why: older hosts cannot reconcile terminal.create's mutation after losing the reply, so clients may only retry unknown outcomes when advertised.
|
||||
export const TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY =
|
||||
'terminal.create-idempotency.v2' as const
|
||||
|
||||
export const RUNTIME_CAPABILITIES = [
|
||||
'runtime.status.compat.v1',
|
||||
|
|
@ -69,7 +72,8 @@ export const RUNTIME_CAPABILITIES = [
|
|||
AI_VAULT_RUNTIME_CAPABILITY,
|
||||
TERMINAL_QUERY_REPLY_INPUT_RUNTIME_CAPABILITY,
|
||||
TERMINAL_QUICK_COMMANDS_RUNTIME_CAPABILITY,
|
||||
WORKTREE_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY
|
||||
WORKTREE_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY,
|
||||
TERMINAL_CREATE_IDEMPOTENCY_RUNTIME_CAPABILITY
|
||||
] as const
|
||||
|
||||
export type RuntimeCapability = (typeof RUNTIME_CAPABILITIES)[number] | (string & {})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
isRecoverableRemoteRuntimeConnectionError,
|
||||
toRemoteRuntimeClientErrorLike
|
||||
} from './remote-runtime-client-error-classification'
|
||||
|
||||
describe('remote runtime client error classification', () => {
|
||||
it.each(['remote_runtime_unavailable', 'runtime_timeout', 'runtime_unavailable', 'reconnecting'])(
|
||||
'treats %s as recoverable',
|
||||
(code) => {
|
||||
expect(isRecoverableRemoteRuntimeConnectionError({ code, message: 'transport failed' })).toBe(
|
||||
true
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
it('does not retry authentication or protocol failures', () => {
|
||||
expect(
|
||||
isRecoverableRemoteRuntimeConnectionError({ code: 'unauthorized', message: 'bad token' })
|
||||
).toBe(false)
|
||||
expect(
|
||||
isRecoverableRemoteRuntimeConnectionError({
|
||||
code: 'invalid_runtime_response',
|
||||
message: 'bad frame'
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it.each([
|
||||
'Could not connect to the remote Orca runtime.',
|
||||
'Remote Orca runtime closed the connection.',
|
||||
'Remote Orca runtime connection closed.',
|
||||
'Remote Orca runtime is not connected.',
|
||||
'Remote runtime subscription closed before it started.'
|
||||
])('normalizes unstructured connection failure: %s', (message) => {
|
||||
const error = toRemoteRuntimeClientErrorLike(new Error(message))
|
||||
expect(isRecoverableRemoteRuntimeConnectionError(error)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
export type RemoteRuntimeClientErrorLike = { code?: string; message: string }
|
||||
|
||||
const RECOVERABLE_CODES = new Set([
|
||||
'remote_runtime_unavailable',
|
||||
'runtime_timeout',
|
||||
'runtime_unavailable',
|
||||
'reconnecting',
|
||||
'timeout'
|
||||
])
|
||||
|
||||
const RECOVERABLE_MESSAGE_FRAGMENTS = [
|
||||
'could not connect to the remote orca runtime',
|
||||
'remote orca runtime closed the connection',
|
||||
'remote orca runtime connection closed',
|
||||
'remote orca runtime is not connected',
|
||||
'remote runtime connection closed',
|
||||
'remote runtime subscription closed before it started',
|
||||
'remote terminal stream is not connected',
|
||||
'timed out waiting for the remote orca runtime'
|
||||
]
|
||||
|
||||
export function isRecoverableRemoteRuntimeConnectionError(
|
||||
error: RemoteRuntimeClientErrorLike
|
||||
): boolean {
|
||||
if (error.code && RECOVERABLE_CODES.has(error.code)) {
|
||||
return true
|
||||
}
|
||||
const message = error.message.toLowerCase()
|
||||
return RECOVERABLE_MESSAGE_FRAGMENTS.some((fragment) => message.includes(fragment))
|
||||
}
|
||||
|
||||
export function toRemoteRuntimeClientErrorLike(error: unknown): RemoteRuntimeClientErrorLike {
|
||||
if (error && typeof error === 'object') {
|
||||
const candidate = error as { code?: unknown; message?: unknown }
|
||||
if (typeof candidate.message === 'string') {
|
||||
return {
|
||||
...(typeof candidate.code === 'string' ? { code: candidate.code } : {}),
|
||||
message: candidate.message
|
||||
}
|
||||
}
|
||||
}
|
||||
return { message: String(error) }
|
||||
}
|
||||
|
|
@ -172,10 +172,11 @@ describe('RemoteRuntimeSharedControlConnection', () => {
|
|||
const server = await createServer({ closeAfterFirstStreamingResponse: true })
|
||||
const connection = new RemoteRuntimeSharedControlConnection(server.pairing)
|
||||
const onClose = vi.fn()
|
||||
const onError = vi.fn()
|
||||
|
||||
await connection.subscribe('runtime.clientEvents.subscribe', null, 1000, {
|
||||
onResponse: vi.fn(),
|
||||
onError: vi.fn(),
|
||||
onError,
|
||||
onClose
|
||||
})
|
||||
|
||||
|
|
@ -186,22 +187,23 @@ describe('RemoteRuntimeSharedControlConnection', () => {
|
|||
'runtime.clientEvents.subscribe'
|
||||
])
|
||||
)
|
||||
expect(onError).toHaveBeenCalledTimes(1)
|
||||
expect(onClose).not.toHaveBeenCalled()
|
||||
|
||||
connection.close()
|
||||
})
|
||||
|
||||
it('emits one final close when reconnect attempts are exhausted', async () => {
|
||||
it('keeps passive subscriptions alive after reaching the capped reconnect delay', async () => {
|
||||
const server = await createServer()
|
||||
const connection = new RemoteRuntimeSharedControlConnection(server.pairing)
|
||||
const onClose = vi.fn()
|
||||
|
||||
const unsafe = connection as unknown as {
|
||||
reconnectAttempt: number
|
||||
reconnect: { attempt: number }
|
||||
subscriptions: Map<string, unknown>
|
||||
scheduleReconnect: () => void
|
||||
}
|
||||
unsafe.reconnectAttempt = 7
|
||||
unsafe.reconnect.attempt = 7
|
||||
unsafe.subscriptions.set('sub-1', {
|
||||
requestId: 'sub-1',
|
||||
method: 'runtime.clientEvents.subscribe',
|
||||
|
|
@ -215,11 +217,11 @@ describe('RemoteRuntimeSharedControlConnection', () => {
|
|||
|
||||
unsafe.scheduleReconnect()
|
||||
|
||||
expect(onClose).toHaveBeenCalledTimes(1)
|
||||
expect(onClose).not.toHaveBeenCalled()
|
||||
expect(connection.getDiagnostics()).toMatchObject({
|
||||
state: 'closed',
|
||||
reconnectAttempt: 7,
|
||||
subscriptionCount: 0
|
||||
state: 'reconnecting',
|
||||
reconnectAttempt: 8,
|
||||
subscriptionCount: 1
|
||||
})
|
||||
|
||||
connection.close()
|
||||
|
|
@ -234,7 +236,7 @@ describe('RemoteRuntimeSharedControlConnection', () => {
|
|||
await expect(connection.request('worktree.ps', undefined, 1000)).resolves.toMatchObject({
|
||||
ok: true
|
||||
})
|
||||
;(connection as unknown as { reconnectAttempt: number }).reconnectAttempt = 3
|
||||
;(connection as unknown as { reconnect: { attempt: number } }).reconnect.attempt = 3
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(connection.getDiagnostics()).toMatchObject({ reconnectAttempt: 0 })
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import {
|
|||
isSharedControlReady,
|
||||
waitForSharedControlReadyWithTimeout
|
||||
} from './remote-runtime-shared-control-ready'
|
||||
import { scheduleSharedControlReconnectOrFinish } from './remote-runtime-shared-control-reconnect'
|
||||
import { SharedControlReconnectScheduler } from './remote-runtime-shared-control-reconnect'
|
||||
import { requestSharedControl } from './remote-runtime-shared-control-requests'
|
||||
import { SharedControlReadyStableResetTimer } from './remote-runtime-shared-control-stability'
|
||||
import * as sharedControlState from './remote-runtime-shared-control-state'
|
||||
|
|
@ -22,6 +22,7 @@ import { closeSharedControlSocket } from './remote-runtime-shared-control-socket
|
|||
import type { RemoteRuntimeSocketLivenessOptions } from './remote-runtime-socket-liveness'
|
||||
import * as sharedControlSubscriptions from './remote-runtime-shared-control-subscriptions'
|
||||
import { startSharedControlSubscription } from './remote-runtime-shared-control-subscription-start'
|
||||
import { SharedControlSocketGeneration } from './remote-runtime-shared-control-socket-generation'
|
||||
import type {
|
||||
RemoteRuntimeSharedConnectionDiagnostics,
|
||||
RemoteRuntimeSharedSubscription,
|
||||
|
|
@ -37,9 +38,8 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
private ws: WebSocket | null = null
|
||||
private sharedKey: Uint8Array | null = null
|
||||
private socketCleanup: (() => void) | null = null
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private readonly reconnect = new SharedControlReconnectScheduler()
|
||||
private readonly readyStableReset: SharedControlReadyStableResetTimer
|
||||
private reconnectAttempt = 0
|
||||
private intentionallyClosed = false
|
||||
private lastConnectedAt: number | null = null
|
||||
private lastClose: { code: number; reason: string } | null = null
|
||||
|
|
@ -48,6 +48,7 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
private readonly subscriptions = new Map<string, SharedControlLogicalSubscription<unknown>>()
|
||||
private readonly readyWaiters: SharedControlReadyWaiter[] = []
|
||||
private everReady = false
|
||||
private readonly socketGeneration = new SharedControlSocketGeneration()
|
||||
|
||||
constructor(
|
||||
private readonly pairing: PairingOffer,
|
||||
|
|
@ -97,7 +98,8 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
|
||||
close(error?: Error): void {
|
||||
this.intentionallyClosed = true
|
||||
this.clearReconnectTimer()
|
||||
this.socketGeneration.invalidate()
|
||||
this.reconnect.clear()
|
||||
for (const subscription of Array.from(this.subscriptions.values())) {
|
||||
this.closeSubscription(subscription.requestId)
|
||||
}
|
||||
|
|
@ -107,10 +109,10 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
getDiagnostics(): RemoteRuntimeSharedConnectionDiagnostics {
|
||||
return sharedControlState.buildSharedControlDiagnostics({
|
||||
state: this.state,
|
||||
reconnecting: this.reconnectTimer !== null,
|
||||
reconnecting: this.reconnect.isScheduled,
|
||||
pendingRequestCount: this.pendingRequests.size,
|
||||
subscriptionCount: this.subscriptions.size,
|
||||
reconnectAttempt: this.reconnectAttempt,
|
||||
reconnectAttempt: this.reconnect.attemptCount,
|
||||
lastConnectedAt: this.lastConnectedAt,
|
||||
lastClose: this.lastClose,
|
||||
lastError: this.lastError
|
||||
|
|
@ -144,25 +146,25 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
)
|
||||
return
|
||||
}
|
||||
this.clearReconnectTimer()
|
||||
this.reconnect.clear()
|
||||
const socketGeneration = this.socketGeneration.begin()
|
||||
const opened = openSharedControlSocket(this.pairing, {
|
||||
getCurrentSocket: () => this.ws,
|
||||
onClose: (close, error) => {
|
||||
this.lastClose = close
|
||||
this.handleSocketClosed(error)
|
||||
if (this.socketGeneration.isCurrent(socketGeneration)) {
|
||||
this.lastClose = close
|
||||
}
|
||||
this.handleSocketClosed(error, socketGeneration)
|
||||
},
|
||||
onError: (error) => {
|
||||
this.lastError = error.message
|
||||
this.handleSocketClosed(error)
|
||||
},
|
||||
onTextFrame: (frame) => this.handleTextFrame(frame),
|
||||
onError: (error) => this.handleSocketClosed(error, socketGeneration),
|
||||
onTextFrame: (frame) => this.handleTextFrame(frame, socketGeneration),
|
||||
liveness: {
|
||||
options: this.options.liveness,
|
||||
onDead: (error) => this.handleSocketClosed(error)
|
||||
onDead: (error) => this.handleSocketClosed(error, socketGeneration)
|
||||
}
|
||||
})
|
||||
if (!opened.ok) {
|
||||
this.handleSocketClosed(opened.error)
|
||||
this.handleSocketClosed(opened.error, socketGeneration)
|
||||
return
|
||||
}
|
||||
this.ws = opened.socket.ws
|
||||
|
|
@ -171,7 +173,10 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
this.state = 'awaiting_ready'
|
||||
}
|
||||
|
||||
private handleTextFrame(frame: string): void {
|
||||
private handleTextFrame(frame: string, socketGeneration: number): void {
|
||||
if (!this.socketGeneration.isCurrent(socketGeneration)) {
|
||||
return
|
||||
}
|
||||
handleSharedControlTextFrame({
|
||||
frame,
|
||||
state: this.state,
|
||||
|
|
@ -184,7 +189,7 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
setState: (state) => {
|
||||
this.state = state
|
||||
},
|
||||
handleSocketClosed: (error) => this.handleSocketClosed(error),
|
||||
handleSocketClosed: (error) => this.handleSocketClosed(error, socketGeneration),
|
||||
sendEncrypted: (payload) => this.sendEncrypted(payload),
|
||||
markReady: () => {
|
||||
this.lastConnectedAt = Date.now()
|
||||
|
|
@ -236,6 +241,7 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
subscription,
|
||||
request: (method, params) => this.sendSubscriptionCleanupRequest(method, params)
|
||||
})
|
||||
this.reconnect.clearWhenIdle(this.subscriptions.size === 0 && this.state === 'closed')
|
||||
}
|
||||
|
||||
private sendEncrypted(payload: unknown): boolean {
|
||||
|
|
@ -256,9 +262,19 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
})
|
||||
}
|
||||
|
||||
private handleSocketClosed(error: RemoteRuntimeClientError): void {
|
||||
private handleSocketClosed(error: RemoteRuntimeClientError, socketGeneration: number): void {
|
||||
if (
|
||||
!this.socketGeneration.acceptClose({
|
||||
generation: socketGeneration,
|
||||
error,
|
||||
everReady: this.everReady,
|
||||
subscriptions: this.subscriptions,
|
||||
closeSocket: () => this.closeSocket(error)
|
||||
})
|
||||
) {
|
||||
return
|
||||
}
|
||||
this.lastError = error.message
|
||||
this.closeSocket(error)
|
||||
if (this.subscriptions.size > 0 && !this.intentionallyClosed) {
|
||||
this.scheduleReconnect()
|
||||
}
|
||||
|
|
@ -286,35 +302,18 @@ export class RemoteRuntimeSharedControlConnection {
|
|||
}
|
||||
|
||||
private scheduleReconnect(): void {
|
||||
const scheduled = scheduleSharedControlReconnectOrFinish({
|
||||
current: this.reconnectTimer,
|
||||
this.reconnect.schedule({
|
||||
intentionallyClosed: this.intentionallyClosed,
|
||||
reconnectAttempt: this.reconnectAttempt,
|
||||
delaysMs: [250, 500, 1000, 2000, 4000, 8000, 15_000],
|
||||
subscriptions: this.subscriptions,
|
||||
open: () => {
|
||||
this.reconnectTimer = null
|
||||
this.open()
|
||||
}
|
||||
delaysMs: [250, 500, 1000, 2000, 4000, 8000, 15_000, 30_000],
|
||||
open: () => this.open()
|
||||
})
|
||||
this.reconnectTimer = scheduled.timer
|
||||
this.reconnectAttempt = scheduled.reconnectAttempt
|
||||
}
|
||||
|
||||
private clearReconnectTimer(): void {
|
||||
if (this.reconnectTimer) {
|
||||
clearTimeout(this.reconnectTimer)
|
||||
this.reconnectTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleReconnectAttemptReset(): void {
|
||||
this.readyStableReset.schedule({
|
||||
getState: () => this.state,
|
||||
getSocket: () => this.ws,
|
||||
reset: () => {
|
||||
this.reconnectAttempt = 0
|
||||
}
|
||||
reset: () => this.reconnect.resetAttempt()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,26 +1,50 @@
|
|||
import { remoteRuntimeUnavailableError } from './remote-runtime-request-frames'
|
||||
import {
|
||||
finishSharedControlSubscription,
|
||||
scheduleSharedControlReconnect
|
||||
} from './remote-runtime-shared-control-state'
|
||||
import type { SharedControlLogicalSubscription } from './remote-runtime-shared-control-types'
|
||||
import { scheduleSharedControlReconnect } from './remote-runtime-shared-control-state'
|
||||
|
||||
export function scheduleSharedControlReconnectOrFinish(args: {
|
||||
current: ReturnType<typeof setTimeout> | null
|
||||
intentionallyClosed: boolean
|
||||
reconnectAttempt: number
|
||||
delaysMs: readonly number[]
|
||||
subscriptions: Map<string, SharedControlLogicalSubscription<unknown>>
|
||||
open: () => void
|
||||
}): { timer: ReturnType<typeof setTimeout> | null; reconnectAttempt: number } {
|
||||
if (args.reconnectAttempt >= args.delaysMs.length) {
|
||||
const error = remoteRuntimeUnavailableError(
|
||||
'Remote Orca runtime connection could not be restored.'
|
||||
)
|
||||
for (const subscription of Array.from(args.subscriptions.values())) {
|
||||
finishSharedControlSubscription(args.subscriptions, subscription, true, error)
|
||||
}
|
||||
return { timer: null, reconnectAttempt: args.reconnectAttempt }
|
||||
export class SharedControlReconnectScheduler {
|
||||
private timer: ReturnType<typeof setTimeout> | null = null
|
||||
private attempt = 0
|
||||
|
||||
get isScheduled(): boolean {
|
||||
return this.timer !== null
|
||||
}
|
||||
|
||||
get attemptCount(): number {
|
||||
return this.attempt
|
||||
}
|
||||
|
||||
schedule(args: {
|
||||
intentionallyClosed: boolean
|
||||
delaysMs: readonly number[]
|
||||
open: () => void
|
||||
}): void {
|
||||
// Why: a passive subscription owns recovery until its caller closes it; roaming outages are unbounded.
|
||||
const scheduled = scheduleSharedControlReconnect({
|
||||
...args,
|
||||
current: this.timer,
|
||||
reconnectAttempt: this.attempt,
|
||||
open: () => {
|
||||
this.timer = null
|
||||
args.open()
|
||||
}
|
||||
})
|
||||
this.timer = scheduled.timer
|
||||
this.attempt = scheduled.reconnectAttempt
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
if (this.timer) {
|
||||
clearTimeout(this.timer)
|
||||
this.timer = null
|
||||
}
|
||||
}
|
||||
|
||||
clearWhenIdle(isIdle: boolean): void {
|
||||
if (isIdle) {
|
||||
this.clear()
|
||||
}
|
||||
}
|
||||
|
||||
resetAttempt(): void {
|
||||
this.attempt = 0
|
||||
}
|
||||
return scheduleSharedControlReconnect(args)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { remoteRuntimeUnavailableError } from './remote-runtime-request-frames'
|
||||
import { SharedControlSocketGeneration } from './remote-runtime-shared-control-socket-generation'
|
||||
|
||||
describe('SharedControlSocketGeneration', () => {
|
||||
it('ignores stale close callbacks and accepts one close for the current socket', () => {
|
||||
const generations = new SharedControlSocketGeneration()
|
||||
const stale = generations.begin()
|
||||
const current = generations.begin()
|
||||
const closeSocket = vi.fn()
|
||||
const onError = vi.fn()
|
||||
const throwingOnError = vi.fn(() => {
|
||||
throw new Error('consumer failed')
|
||||
})
|
||||
const subscriptions = new Map([
|
||||
[
|
||||
'subscription-1',
|
||||
{
|
||||
requestId: 'subscription-1',
|
||||
method: 'session.tabs.subscribeAll',
|
||||
params: null,
|
||||
callbacks: { onResponse: vi.fn(), onError },
|
||||
sent: true,
|
||||
closed: false,
|
||||
closeAfterReady: false,
|
||||
remoteSubscriptionId: 'remote-1'
|
||||
}
|
||||
],
|
||||
[
|
||||
'subscription-2',
|
||||
{
|
||||
requestId: 'subscription-2',
|
||||
method: 'runtime.clientEvents.subscribe',
|
||||
params: null,
|
||||
callbacks: { onResponse: vi.fn(), onError: throwingOnError },
|
||||
sent: true,
|
||||
closed: false,
|
||||
closeAfterReady: false,
|
||||
remoteSubscriptionId: 'remote-2'
|
||||
}
|
||||
]
|
||||
])
|
||||
const args = {
|
||||
error: remoteRuntimeUnavailableError(),
|
||||
everReady: true,
|
||||
subscriptions,
|
||||
closeSocket
|
||||
}
|
||||
|
||||
expect(generations.acceptClose({ ...args, generation: stale })).toBe(false)
|
||||
expect(generations.acceptClose({ ...args, generation: current })).toBe(true)
|
||||
expect(generations.acceptClose({ ...args, generation: current })).toBe(false)
|
||||
expect(closeSocket).toHaveBeenCalledTimes(1)
|
||||
expect(onError).toHaveBeenCalledTimes(1)
|
||||
expect(throwingOnError).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
import type { RemoteRuntimeClientError } from './remote-runtime-client-error'
|
||||
import type { SharedControlLogicalSubscription } from './remote-runtime-shared-control-types'
|
||||
|
||||
export class SharedControlSocketGeneration {
|
||||
private current = 0
|
||||
|
||||
isCurrent(generation: number): boolean {
|
||||
return generation === this.current
|
||||
}
|
||||
|
||||
begin(): number {
|
||||
this.current += 1
|
||||
return this.current
|
||||
}
|
||||
|
||||
invalidate(): void {
|
||||
this.current += 1
|
||||
}
|
||||
|
||||
acceptClose(args: {
|
||||
generation: number
|
||||
error: RemoteRuntimeClientError
|
||||
everReady: boolean
|
||||
subscriptions: Map<string, SharedControlLogicalSubscription<unknown>>
|
||||
closeSocket: () => void
|
||||
}): boolean {
|
||||
if (!this.isCurrent(args.generation)) {
|
||||
return false
|
||||
}
|
||||
// Why: error, close, and liveness callbacks can race for one socket; only its first callback owns recovery.
|
||||
this.current += 1
|
||||
args.closeSocket()
|
||||
if (args.everReady) {
|
||||
for (const subscription of Array.from(args.subscriptions.values())) {
|
||||
try {
|
||||
subscription.callbacks.onError(args.error)
|
||||
} catch {
|
||||
// Why: one consumer callback cannot block sibling notification or reconnect scheduling.
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { startRemoteRuntimeSocketLiveness } from './remote-runtime-socket-liveness'
|
||||
|
||||
describe('remote runtime socket liveness', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('grants a fresh probe window after a suspended client resumes', async () => {
|
||||
let now = 1_000
|
||||
const ping = vi.fn()
|
||||
const onDead = vi.fn()
|
||||
const monitor = startRemoteRuntimeSocketLiveness({
|
||||
ping,
|
||||
onDead,
|
||||
options: { pingIntervalMs: 100, livenessTimeoutMs: 250 },
|
||||
now: () => now
|
||||
})
|
||||
|
||||
now += 3_600_000
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
|
||||
expect(ping).toHaveBeenCalledTimes(1)
|
||||
expect(onDead).not.toHaveBeenCalled()
|
||||
|
||||
for (const delta of [100, 100]) {
|
||||
now += delta
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
}
|
||||
expect(onDead).not.toHaveBeenCalled()
|
||||
|
||||
now += 100
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
expect(onDead).toHaveBeenCalledTimes(1)
|
||||
monitor.stop()
|
||||
})
|
||||
|
||||
it('clears the resumed probe when the socket answers', async () => {
|
||||
let now = 1_000
|
||||
const ping = vi.fn()
|
||||
const onDead = vi.fn()
|
||||
const monitor = startRemoteRuntimeSocketLiveness({
|
||||
ping,
|
||||
onDead,
|
||||
options: { pingIntervalMs: 100, livenessTimeoutMs: 250 },
|
||||
now: () => now
|
||||
})
|
||||
|
||||
now += 3_600_000
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
monitor.noteActivity()
|
||||
for (const delta of [100, 100, 100]) {
|
||||
now += delta
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
}
|
||||
|
||||
expect(onDead).not.toHaveBeenCalled()
|
||||
monitor.stop()
|
||||
})
|
||||
})
|
||||
|
|
@ -33,23 +33,31 @@ export function startRemoteRuntimeSocketLiveness(args: {
|
|||
const pingIntervalMs = args.options?.pingIntervalMs ?? REMOTE_RUNTIME_SOCKET_PING_INTERVAL_MS
|
||||
const livenessTimeoutMs =
|
||||
args.options?.livenessTimeoutMs ?? REMOTE_RUNTIME_SOCKET_LIVENESS_TIMEOUT_MS
|
||||
let lastActivityAt = now()
|
||||
let lastTickAt = now()
|
||||
let probeSentAt: number | null = null
|
||||
let stopped = false
|
||||
|
||||
const timer = setInterval(() => {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
if (now() - lastActivityAt > livenessTimeoutMs) {
|
||||
const tickAt = now()
|
||||
const tickElapsedMs = tickAt - lastTickAt
|
||||
lastTickAt = tickAt
|
||||
// Why: sleep and background throttling age sockets without giving them a chance to answer.
|
||||
if (tickElapsedMs < 0 || tickElapsedMs > pingIntervalMs * 1.5) {
|
||||
probeSentAt = tickAt
|
||||
tryPing()
|
||||
return
|
||||
}
|
||||
if (probeSentAt !== null && tickAt - probeSentAt > livenessTimeoutMs) {
|
||||
stop()
|
||||
args.onDead()
|
||||
return
|
||||
}
|
||||
try {
|
||||
args.ping()
|
||||
} catch {
|
||||
// Why: ping() can throw while a socket is mid-teardown; the liveness
|
||||
// window (or the close handler) settles the socket's fate either way.
|
||||
if (probeSentAt === null) {
|
||||
probeSentAt = tickAt
|
||||
tryPing()
|
||||
}
|
||||
}, pingIntervalMs)
|
||||
// Why: mobile typechecks shared code with DOM timer types where unref is absent.
|
||||
|
|
@ -66,9 +74,17 @@ export function startRemoteRuntimeSocketLiveness(args: {
|
|||
clearInterval(timer)
|
||||
}
|
||||
|
||||
function tryPing(): void {
|
||||
try {
|
||||
args.ping()
|
||||
} catch {
|
||||
// Why: ping() can throw while a socket is mid-teardown; the probe deadline still settles it.
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
noteActivity: () => {
|
||||
lastActivityAt = now()
|
||||
probeSentAt = null
|
||||
},
|
||||
stop
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue