Let a timed-out remote terminal pane reconnect when the host session is still alive (#12213)
* fix(terminal): let a timed-out remote terminal pane reconnect again A remote-runtime PTY pane that missed the 60s auto-recovery window latched to "disconnected" permanently, even while the host kept the session alive and streaming. The cutoff destroyed every path back at once: it dropped the pending retry and evicted the pane from the shared retry registry (so window `online` and system-resume became no-ops), and the transport tore down the accepted host-snapshot listener. The listener was also inert for the common case, since a host that keeps publishing the same live handle never rotates it. The cutoff now only stops the retry timer; the pane keeps its pending retry and stays revivable. The accepted-snapshot listener survives the latch, and a post-cutoff snapshot is accepted as reattach evidence whether the host rotated the handle or republished the same one. The require-replacement published-wait guard is scoped to its own recovery epoch so Reconnect and online/resume are no longer swallowed. No new polling: a latched pane still issues zero self-initiated RPCs, and the same-handle reattach consumes a snapshot the client already receives. Closes #12097 * fix(terminal): drop the settled attach retry when the recovery cutoff lands The recovery cutoff now keeps a pending retry so online/resume can revive a latched pane, but the host-session attach wait schedules a single-shot closure that the cutoff itself resolves. Retaining it left the pane registered as revivable work: 'online' would bump the epoch, arm a fresh 60s deadline and flip the phase to 'recovering' while invoking a no-op, hiding the working Reconnect button and the same-handle snapshot reattach for a full minute. The attach wait now discards its own scheduled retry as it settles, so a pane latched on that path stays 'disconnected' and Reconnect keeps working. Retries scheduled by the resubscribe paths re-enter real work and are unaffected. Co-authored-by: Orca <help@stably.ai> * fix(test): type the reattach host snapshot factory so status narrows to 'ready' Co-authored-by: Orca <help@stably.ai> * fix(terminal): keep an exhausted remote pane reattachable through Reconnect and resume Reconnect and the online/system-resume trigger both opened a fresh recovery epoch, which switched off the accepted-snapshot reattach path (gated on the 'disconnected' phase) for 60s while the require-replacement inventory wait dead-ended without scheduling anything. Gate the same-handle reattach on a spent auto-recovery window instead of the live phase, consumed once per window, and park an unarmed retry at the require-replacement dead end so online/resume/Reconnect have work to revive. Co-authored-by: Orca <help@stably.ai> * test(terminal): pin latched-pane retention to one listener and one registry entry The recovery cutoff now keeps the retry-registry entry and the accepted-snapshot listener alive, so cover the two module-global collections that could accumulate: destroy/detach cycles, concurrent latched panes, revive storms and snapshot churn all return to baseline. Co-authored-by: Orca <help@stably.ai> * refactor(terminal): use the returned reattach epoch and pin the no-RPC claim Addresses both CodeRabbit nitpicks: subscribeToHandle now takes the epoch begin() returned rather than re-reading currentEpoch, and the latched attach-wait test asserts the runtime call count is unchanged instead of only the advanced count. Co-authored-by: Orca <help@stably.ai> --------- Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Neil
GitHub
parent
dc0cb1806a
commit
dbbeca0916
|
|
@ -0,0 +1,381 @@
|
|||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
TerminalStreamOpcode,
|
||||
decodeTerminalStreamFrame,
|
||||
decodeTerminalStreamJson,
|
||||
encodeTerminalStreamFrame,
|
||||
encodeTerminalStreamJson,
|
||||
encodeTerminalStreamText
|
||||
} from '../../../../shared/terminal-stream-protocol'
|
||||
import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types'
|
||||
|
||||
describe('remote runtime pty reattach after the bounded recovery window', () => {
|
||||
const runtimeCall = vi.fn()
|
||||
const runtimeSubscribe = vi.fn()
|
||||
const refreshSessionTabsSnapshot = vi.fn(async () => {})
|
||||
const subscriptionSendBinary = vi.fn()
|
||||
let subscriptionCallbacks: {
|
||||
onResponse: (response: unknown) => void
|
||||
onBinary?: (bytes: Uint8Array<ArrayBufferLike>) => void
|
||||
onError?: (error: { code: string; message: string }) => void
|
||||
onClose?: () => void
|
||||
} | null = null
|
||||
let hostListCalls = 0
|
||||
|
||||
function emitMultiplexReady(): void {
|
||||
subscriptionCallbacks?.onResponse({ ok: true, result: { type: 'ready' } })
|
||||
}
|
||||
|
||||
function latestSubscribePayload(): { streamId: number; terminal: string } {
|
||||
const frame = subscriptionSendBinary.mock.calls
|
||||
.map((call) => decodeTerminalStreamFrame(call[0]))
|
||||
.findLast((candidate) => candidate?.opcode === TerminalStreamOpcode.Subscribe)
|
||||
if (!frame) {
|
||||
throw new Error('missing terminal subscribe frame')
|
||||
}
|
||||
const payload = decodeTerminalStreamJson<{ streamId: number; terminal: string }>(frame.payload)
|
||||
if (!payload) {
|
||||
throw new Error('invalid terminal subscribe payload')
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
function subscribedTerminalHandles(): string[] {
|
||||
return subscriptionSendBinary.mock.calls
|
||||
.map((call) => decodeTerminalStreamFrame(call[0]))
|
||||
.flatMap((frame) => {
|
||||
if (frame?.opcode !== TerminalStreamOpcode.Subscribe) {
|
||||
return []
|
||||
}
|
||||
const payload = decodeTerminalStreamJson<{ terminal: string }>(frame.payload)
|
||||
return payload ? [payload.terminal] : []
|
||||
})
|
||||
}
|
||||
|
||||
function emitSnapshot(streamId: number, data: string): void {
|
||||
subscriptionCallbacks?.onBinary?.(
|
||||
encodeTerminalStreamFrame({
|
||||
opcode: TerminalStreamOpcode.SnapshotStart,
|
||||
streamId,
|
||||
seq: 1,
|
||||
payload: encodeTerminalStreamJson({ kind: 'scrollback' })
|
||||
})
|
||||
)
|
||||
subscriptionCallbacks?.onBinary?.(
|
||||
encodeTerminalStreamFrame({
|
||||
opcode: TerminalStreamOpcode.SnapshotChunk,
|
||||
streamId,
|
||||
seq: 2,
|
||||
payload: encodeTerminalStreamText(data)
|
||||
})
|
||||
)
|
||||
subscriptionCallbacks?.onBinary?.(
|
||||
encodeTerminalStreamFrame({
|
||||
opcode: TerminalStreamOpcode.SnapshotEnd,
|
||||
streamId,
|
||||
seq: 3,
|
||||
payload: new Uint8Array()
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
function hostSnapshot(
|
||||
terminal: string,
|
||||
snapshotVersion: number,
|
||||
publicationEpoch: string
|
||||
): RuntimeMobileSessionTabsResult {
|
||||
return {
|
||||
worktree: 'wt-1',
|
||||
publicationEpoch,
|
||||
snapshotVersion,
|
||||
activeGroupId: null,
|
||||
activeTabId: 'tab-1::pane:1',
|
||||
activeTabType: 'terminal' as const,
|
||||
tabs: [
|
||||
{
|
||||
type: 'terminal' as const,
|
||||
id: 'tab-1::pane:1',
|
||||
parentTabId: 'tab-1',
|
||||
leafId: 'pane:1',
|
||||
title: 'Claude Code',
|
||||
isActive: true,
|
||||
status: 'ready',
|
||||
terminal
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
async function attachStalePane() {
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const onError = vi.fn()
|
||||
const onPtyExit = vi.fn()
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'web-terminal-tab-1',
|
||||
leafId: 'pane:1',
|
||||
onPtyExit,
|
||||
onPtyRebind: vi.fn()
|
||||
})
|
||||
transport.attach({
|
||||
existingPtyId: 'remote:env-1@@terminal-stale',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
callbacks: { onError }
|
||||
})
|
||||
await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled())
|
||||
|
||||
// The host keeps publishing the same live handle, so bounded replacement polling finds
|
||||
// no replacement and stops quietly without retiring the pane.
|
||||
runtimeCall.mockImplementation(async (args: { method: string }) => {
|
||||
if (args.method !== 'session.tabs.list') {
|
||||
return { ok: true, result: {} }
|
||||
}
|
||||
hostListCalls += 1
|
||||
return { ok: true, result: hostSnapshot('terminal-stale', hostListCalls + 1, 'epoch-1') }
|
||||
})
|
||||
subscriptionCallbacks?.onResponse({
|
||||
ok: true,
|
||||
result: {
|
||||
type: 'error',
|
||||
streamId: latestSubscribePayload().streamId,
|
||||
message: 'terminal_handle_stale'
|
||||
}
|
||||
})
|
||||
return { transport, onError, onPtyExit }
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules()
|
||||
vi.doUnmock('../../runtime/remote-runtime-terminal-multiplexer')
|
||||
vi.doMock('@/runtime/web-runtime-session', () => ({
|
||||
refreshWebRuntimeSessionTabsSnapshot: refreshSessionTabsSnapshot
|
||||
}))
|
||||
vi.clearAllMocks()
|
||||
subscriptionCallbacks = null
|
||||
hostListCalls = 0
|
||||
subscriptionSendBinary.mockReset()
|
||||
runtimeCall.mockImplementation(async (request: { method: string; params?: unknown }) => {
|
||||
if (request.method === 'session.tabs.activate') {
|
||||
return { ok: true, result: hostSnapshot('terminal-stale', 1, 'epoch-1') }
|
||||
}
|
||||
if (request.method === 'terminal.resolvePane') {
|
||||
const params = request.params as { paneKey: string; worktreeId: string }
|
||||
const separator = params.paneKey.indexOf(':')
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
terminal: {
|
||||
handle: 'terminal-stale',
|
||||
tabId: params.paneKey.slice(0, separator),
|
||||
leafId: params.paneKey.slice(separator + 1),
|
||||
worktreeId: params.worktreeId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return { ok: true, result: { terminal: { handle: 'terminal-stale' } } }
|
||||
})
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: typeof subscriptionCallbacks) => {
|
||||
subscriptionCallbacks = callbacks
|
||||
queueMicrotask(emitMultiplexReady)
|
||||
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
vi.stubGlobal('window', {
|
||||
api: { runtimeEnvironments: { call: runtimeCall, subscribe: runtimeSubscribe } }
|
||||
})
|
||||
})
|
||||
|
||||
it('reattaches from a rotated host handle published after the recovery cutoff', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { transport, onPtyExit } = await attachStalePane()
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(16_000)
|
||||
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
|
||||
expect(transport.getRecoveryState?.().phase).not.toBe('disconnected')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(50_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
// The cutoff must not tear down the accepted-snapshot listener; it is the only path back.
|
||||
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
|
||||
|
||||
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
|
||||
hostSnapshot('terminal-after-timeout', 3, 'epoch-2'),
|
||||
'env-1'
|
||||
)
|
||||
await vi.waitFor(() =>
|
||||
expect(latestSubscribePayload()).toMatchObject({ terminal: 'terminal-after-timeout' })
|
||||
)
|
||||
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'reattached')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(transport.getPtyId()).toBe('remote:env-1@@terminal-after-timeout')
|
||||
expect(onPtyExit).not.toHaveBeenCalled()
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('reattaches when the post-cutoff snapshot republishes the same live handle', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { transport, onError } = await attachStalePane()
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
const listCallsAtCutoff = hostListCalls
|
||||
|
||||
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
|
||||
hostSnapshot('terminal-stale', 3, 'epoch-2'),
|
||||
'env-1'
|
||||
)
|
||||
await vi.waitFor(() => expect(subscribedTerminalHandles()).toHaveLength(2))
|
||||
|
||||
expect(subscribedTerminalHandles()).toEqual(['terminal-stale', 'terminal-stale'])
|
||||
// The snapshot is already-received host evidence; reattaching must cost no inventory RPC.
|
||||
expect(hostListCalls).toBe(listCallsAtCutoff)
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'reattached')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('ignores a same-handle snapshot while automatic recovery is still running', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { transport } = await attachStalePane()
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(16_000)
|
||||
expect(transport.getRecoveryState?.().phase).not.toBe('disconnected')
|
||||
|
||||
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
|
||||
hostSnapshot('terminal-stale', 3, 'epoch-2'),
|
||||
'env-1'
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
|
||||
expect(subscribedTerminalHandles()).toEqual(['terminal-stale'])
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('reattaches from a same-handle snapshot delivered inside the Reconnect window', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { transport, onError } = await attachStalePane()
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1)
|
||||
const listCallsAtCutoff = hostListCalls
|
||||
|
||||
expect(transport.retryRecovery?.()).toBe(true)
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(hostListCalls).toBeGreaterThan(listCallsAtCutoff)
|
||||
|
||||
// Reconnect must not disarm the only path back: the click opens a window, it does not spend one.
|
||||
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
|
||||
hostSnapshot('terminal-stale', 4, 'epoch-3'),
|
||||
'env-1'
|
||||
)
|
||||
await vi.waitFor(() => expect(subscribedTerminalHandles()).toHaveLength(2))
|
||||
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'reattached')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('revives a latched require-replacement pane when online or system resume fires', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { transport, onError } = await attachStalePane()
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNow } =
|
||||
await import('./remote-runtime-pty-recovery-state')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
const listCallsAtCutoff = hostListCalls
|
||||
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(hostListCalls).toBeGreaterThan(listCallsAtCutoff)
|
||||
|
||||
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
|
||||
hostSnapshot('terminal-stale', 4, 'epoch-3'),
|
||||
'env-1'
|
||||
)
|
||||
await vi.waitFor(() => expect(subscribedTerminalHandles()).toHaveLength(2))
|
||||
|
||||
emitSnapshot(latestSubscribePayload().streamId, 'reattached')
|
||||
expect(transport.isConnected()).toBe(true)
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('leaves Reconnect available when online fires on a pane latched during the attach wait', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNow } =
|
||||
await import('./remote-runtime-pty-recovery-state')
|
||||
runtimeCall.mockImplementation(async (request: { method: string }) => {
|
||||
if (request.method === 'session.tabs.activate') {
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
code: 'remote_runtime_unavailable',
|
||||
message: 'Remote Orca runtime connection closed'
|
||||
}
|
||||
}
|
||||
}
|
||||
return { ok: true, result: {} }
|
||||
})
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'web-terminal-tab-1',
|
||||
leafId: 'pane:1',
|
||||
onPtyExit: vi.fn(),
|
||||
onPtyRebind: vi.fn()
|
||||
})
|
||||
transport.attach({
|
||||
existingPtyId: 'remote:env-1@@terminal-stale',
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
callbacks: { onError: vi.fn() }
|
||||
})
|
||||
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
const callsBeforeRetry = runtimeCall.mock.calls.length
|
||||
// The attach wait is single-shot and the cutoff already settled it; replaying it would spin the banner with no RPC in flight.
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
expect(runtimeCall.mock.calls.length).toBe(callsBeforeRetry)
|
||||
transport.destroy?.()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,358 @@
|
|||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
TerminalStreamOpcode,
|
||||
decodeTerminalStreamFrame,
|
||||
decodeTerminalStreamJson
|
||||
} from '../../../../shared/terminal-stream-protocol'
|
||||
import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types'
|
||||
|
||||
// Why: the recovery cutoff no longer tears down the retry registry entry or the accepted-snapshot
|
||||
// listener, so those two module-global collections are the only places a latched pane can accumulate.
|
||||
describe('remote runtime pty latched-pane retention', () => {
|
||||
const runtimeCall = vi.fn()
|
||||
const runtimeSubscribe = vi.fn()
|
||||
const refreshSessionTabsSnapshot = vi.fn(async () => {})
|
||||
const subscriptionSendBinary = vi.fn()
|
||||
let subscriptionCallbacks: {
|
||||
onResponse: (response: unknown) => void
|
||||
onBinary?: (bytes: Uint8Array<ArrayBufferLike>) => void
|
||||
onError?: (error: { code: string; message: string }) => void
|
||||
onClose?: () => void
|
||||
} | null = null
|
||||
let hostListCalls = 0
|
||||
|
||||
function emitMultiplexReady(): void {
|
||||
subscriptionCallbacks?.onResponse({ ok: true, result: { type: 'ready' } })
|
||||
}
|
||||
|
||||
function latestSubscribePayload(): { streamId: number; terminal: string } {
|
||||
const frame = subscriptionSendBinary.mock.calls
|
||||
.map((call) => decodeTerminalStreamFrame(call[0]))
|
||||
.findLast((candidate) => candidate?.opcode === TerminalStreamOpcode.Subscribe)
|
||||
if (!frame) {
|
||||
throw new Error('missing terminal subscribe frame')
|
||||
}
|
||||
const payload = decodeTerminalStreamJson<{ streamId: number; terminal: string }>(frame.payload)
|
||||
if (!payload) {
|
||||
throw new Error('invalid terminal subscribe payload')
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
function subscribeFrameCount(): number {
|
||||
return subscriptionSendBinary.mock.calls
|
||||
.map((call) => decodeTerminalStreamFrame(call[0]))
|
||||
.filter((frame) => frame?.opcode === TerminalStreamOpcode.Subscribe).length
|
||||
}
|
||||
|
||||
// Why: every pane needs its own host surface, otherwise a later pane reuses the earlier pane's
|
||||
// multiplexed stream and never enters recovery at all.
|
||||
const paneIdentity = (pane: number) => ({
|
||||
hostTabId: `tab-${pane}`,
|
||||
tabId: `web-terminal-tab-${pane}`,
|
||||
leafId: `pane:${pane}`,
|
||||
handle: `terminal-stale-${pane}`
|
||||
})
|
||||
|
||||
const PANE_COUNT = 24
|
||||
|
||||
// Why: one payload carries every pane's surface, matching the real per-worktree session.tabs
|
||||
// response that any polling pane receives.
|
||||
function hostSnapshot(
|
||||
snapshotVersion: number,
|
||||
publicationEpoch: string
|
||||
): RuntimeMobileSessionTabsResult {
|
||||
return {
|
||||
worktree: 'wt-1',
|
||||
publicationEpoch,
|
||||
snapshotVersion,
|
||||
activeGroupId: null,
|
||||
activeTabId: `${paneIdentity(0).hostTabId}::${paneIdentity(0).leafId}`,
|
||||
activeTabType: 'terminal' as const,
|
||||
tabs: Array.from({ length: PANE_COUNT }, (_unused, pane) => {
|
||||
const identity = paneIdentity(pane)
|
||||
return {
|
||||
type: 'terminal' as const,
|
||||
id: `${identity.hostTabId}::${identity.leafId}`,
|
||||
parentTabId: identity.hostTabId,
|
||||
leafId: identity.leafId,
|
||||
title: 'Claude Code',
|
||||
isActive: pane === 0,
|
||||
status: 'ready' as const,
|
||||
terminal: identity.handle
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function attachStalePane(pane: number) {
|
||||
const identity = paneIdentity(pane)
|
||||
const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport')
|
||||
const transport = createRemoteRuntimePtyTransport('env-1', {
|
||||
worktreeId: 'wt-1',
|
||||
tabId: identity.tabId,
|
||||
leafId: identity.leafId,
|
||||
onPtyExit: vi.fn(),
|
||||
onPtyRebind: vi.fn()
|
||||
})
|
||||
const subscribesBefore = subscribeFrameCount()
|
||||
transport.attach({
|
||||
existingPtyId: `remote:env-1@@${identity.handle}`,
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
callbacks: { onError: vi.fn() }
|
||||
})
|
||||
await vi.waitFor(() => expect(subscribeFrameCount()).toBeGreaterThan(subscribesBefore))
|
||||
|
||||
// The host keeps publishing the same live handle, so bounded replacement polling finds no
|
||||
// replacement and stops quietly: the exact shape that leaves the pane latched but revivable.
|
||||
subscriptionCallbacks?.onResponse({
|
||||
ok: true,
|
||||
result: {
|
||||
type: 'error',
|
||||
streamId: latestSubscribePayload().streamId,
|
||||
message: 'terminal_handle_stale'
|
||||
}
|
||||
})
|
||||
return transport
|
||||
}
|
||||
|
||||
async function registries() {
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
const recoveryState = await import('./remote-runtime-pty-recovery-state')
|
||||
return {
|
||||
subscribers: handleEvents.getWebSessionTerminalHandleSubscriberCountForTests(),
|
||||
scheduled: recoveryState.getScheduledRemoteRuntimePtyRecoveryCountForTests()
|
||||
}
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetModules()
|
||||
vi.doUnmock('../../runtime/remote-runtime-terminal-multiplexer')
|
||||
vi.doMock('@/runtime/web-runtime-session', () => ({
|
||||
refreshWebRuntimeSessionTabsSnapshot: refreshSessionTabsSnapshot
|
||||
}))
|
||||
vi.clearAllMocks()
|
||||
subscriptionCallbacks = null
|
||||
hostListCalls = 0
|
||||
subscriptionSendBinary.mockReset()
|
||||
runtimeCall.mockImplementation(async (request: { method: string; params?: unknown }) => {
|
||||
if (request.method === 'session.tabs.list') {
|
||||
hostListCalls += 1
|
||||
return { ok: true, result: hostSnapshot(hostListCalls + 1, 'epoch-1') }
|
||||
}
|
||||
if (request.method === 'session.tabs.activate') {
|
||||
return { ok: true, result: hostSnapshot(1, 'epoch-1') }
|
||||
}
|
||||
if (request.method === 'terminal.resolvePane') {
|
||||
const params = request.params as { paneKey: string; worktreeId: string }
|
||||
const separator = params.paneKey.indexOf(':')
|
||||
const paneTabId = params.paneKey.slice(0, separator)
|
||||
const pane = Number(paneTabId.slice(paneTabId.lastIndexOf('-') + 1))
|
||||
return {
|
||||
ok: true,
|
||||
result: {
|
||||
terminal: {
|
||||
handle: paneIdentity(pane).handle,
|
||||
tabId: paneTabId,
|
||||
leafId: params.paneKey.slice(separator + 1),
|
||||
worktreeId: params.worktreeId
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return { ok: true, result: {} }
|
||||
})
|
||||
runtimeSubscribe.mockImplementation(
|
||||
async (_args: unknown, callbacks: typeof subscriptionCallbacks) => {
|
||||
subscriptionCallbacks = callbacks
|
||||
queueMicrotask(emitMultiplexReady)
|
||||
return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary }
|
||||
}
|
||||
)
|
||||
vi.stubGlobal('window', {
|
||||
api: { runtimeEnvironments: { call: runtimeCall, subscribe: runtimeSubscribe } }
|
||||
})
|
||||
})
|
||||
|
||||
it('returns listener and retry-registry counts to baseline across destroy cycles', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 })
|
||||
const latched: { subscribers: number; scheduled: number }[] = []
|
||||
const settled: { subscribers: number; scheduled: number }[] = []
|
||||
|
||||
for (let cycle = 0; cycle < 20; cycle += 1) {
|
||||
const transport = await attachStalePane(cycle)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
latched.push(await registries())
|
||||
transport.destroy?.()
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
settled.push(await registries())
|
||||
}
|
||||
|
||||
// A latched pane deliberately holds exactly one listener and one registry entry...
|
||||
expect(latched).toEqual(Array.from({ length: 20 }, () => ({ subscribers: 1, scheduled: 1 })))
|
||||
// ...and destroy releases both, so twenty cycles do not accumulate anything.
|
||||
expect(settled).toEqual(Array.from({ length: 20 }, () => ({ subscribers: 0, scheduled: 0 })))
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('returns to baseline when a latched pane is detached rather than destroyed', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const settled: { subscribers: number; scheduled: number }[] = []
|
||||
for (let cycle = 0; cycle < 20; cycle += 1) {
|
||||
const transport = await attachStalePane(cycle)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
transport.detach?.()
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
settled.push(await registries())
|
||||
}
|
||||
expect(settled).toEqual(Array.from({ length: 20 }, () => ({ subscribers: 0, scheduled: 0 })))
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('holds one listener and one registry entry per concurrently latched pane', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const transports: Awaited<ReturnType<typeof attachStalePane>>[] = []
|
||||
for (let pane = 0; pane < 8; pane += 1) {
|
||||
transports.push(await attachStalePane(pane))
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
}
|
||||
// Retention is per live pane, not per timeout: eight latched panes hold eight of each.
|
||||
expect(await registries()).toEqual({ subscribers: 8, scheduled: 8 })
|
||||
|
||||
for (const transport of transports) {
|
||||
transport.destroy?.()
|
||||
}
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 })
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('leaves a latched pane fully quiescent — no timers, no RPCs, no growth', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const transport = await attachStalePane(0)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
const baseline = await registries()
|
||||
const callsAtLatch = runtimeCall.mock.calls.length
|
||||
const timersAtLatch = vi.getTimerCount()
|
||||
// Nothing is armed once the window is spent: the pane costs one listener and one registry
|
||||
// entry, and nothing else, no matter how long it stays latched.
|
||||
expect(timersAtLatch).toBe(0)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10 * 60_000)
|
||||
|
||||
expect(runtimeCall.mock.calls.length).toBe(callsAtLatch)
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
expect(await registries()).toEqual(baseline)
|
||||
|
||||
transport.destroy?.()
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 })
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not stack listeners, registry entries or timers across repeated revive cycles', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const { retryAllRemoteRuntimePtyRecoveriesNow } = await import(
|
||||
'./remote-runtime-pty-recovery-state'
|
||||
)
|
||||
const transport = await attachStalePane(0)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
|
||||
const baseline = await registries()
|
||||
const timersAtFirstLatch = vi.getTimerCount()
|
||||
const subscribesAtFirstLatch = subscribeFrameCount()
|
||||
const observed: {
|
||||
subscribers: number
|
||||
scheduled: number
|
||||
timers: number
|
||||
revived: number
|
||||
}[] = []
|
||||
|
||||
for (let cycle = 0; cycle < 25; cycle += 1) {
|
||||
const revived = retryAllRemoteRuntimePtyRecoveriesNow()
|
||||
// A second trigger in the same window must find nothing to advance, so an online/resume
|
||||
// storm cannot stack fresh recovery epochs on one pane.
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
observed.push({ ...(await registries()), timers: vi.getTimerCount(), revived })
|
||||
}
|
||||
|
||||
expect(observed).toEqual(
|
||||
Array.from({ length: 25 }, () => ({
|
||||
subscribers: baseline.subscribers,
|
||||
scheduled: baseline.scheduled,
|
||||
timers: timersAtFirstLatch,
|
||||
revived: 1
|
||||
}))
|
||||
)
|
||||
// Each revive re-derives the handle; it must not leave extra live stream subscriptions behind.
|
||||
expect(subscribeFrameCount()).toBe(subscribesAtFirstLatch)
|
||||
|
||||
transport.destroy?.()
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 })
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not stack anything when host snapshots arrive repeatedly at a latched pane', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const handleEvents = await import('../../runtime/web-session-terminal-handle-events')
|
||||
const transport = await attachStalePane(0)
|
||||
await vi.advanceTimersByTimeAsync(66_000)
|
||||
expect(transport.getRecoveryState?.().phase).toBe('disconnected')
|
||||
const baseline = await registries()
|
||||
|
||||
const subscribesAtLatch = subscribeFrameCount()
|
||||
|
||||
for (let cycle = 0; cycle < 50; cycle += 1) {
|
||||
handleEvents.queueAcceptedWebSessionTerminalSnapshot(
|
||||
hostSnapshot(10 + cycle, `epoch-${cycle}`),
|
||||
'env-1'
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
}
|
||||
|
||||
const after = await registries()
|
||||
expect(after.subscribers).toBeLessThanOrEqual(baseline.subscribers)
|
||||
expect(after.scheduled).toBeLessThanOrEqual(baseline.scheduled)
|
||||
// At most one same-handle resubscribe per spent window: 50 snapshots inside one window
|
||||
// must not become 50 subscribes.
|
||||
expect(subscribeFrameCount() - subscribesAtLatch).toBeLessThanOrEqual(1)
|
||||
|
||||
transport.destroy?.()
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 })
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
@ -69,6 +69,7 @@ describe('RemoteRuntimePtyRecoveryState', () => {
|
|||
expect(state.isActive).toBe(false)
|
||||
expect(state.isCurrent(epoch)).toBe(false)
|
||||
expect(onChange).toHaveBeenCalled()
|
||||
state.dispose()
|
||||
})
|
||||
|
||||
it('advances a pending backoff immediately via retryNow and the active registry', async () => {
|
||||
|
|
@ -154,16 +155,56 @@ describe('RemoteRuntimePtyRecoveryState', () => {
|
|||
expect(retryNow).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('removes timed-out panes from the scheduled recovery registry', async () => {
|
||||
it('keeps a timed-out pane revivable through the scheduled recovery registry', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const retry = vi.fn()
|
||||
const firstEpoch = state.begin()
|
||||
// Why: the backoff ladder outlasts the recovery window, so a retry is still armed when the cutoff lands.
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS - 100)
|
||||
state.schedule(firstEpoch, retry)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
expect(state.currentPhase).toBe('disconnected')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(300_000)
|
||||
expect(retry).not.toHaveBeenCalled()
|
||||
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
|
||||
expect(retry).toHaveBeenCalledWith(firstEpoch + 1)
|
||||
expect(state.currentPhase).toBe('recovering')
|
||||
state.dispose()
|
||||
})
|
||||
|
||||
it('revives a parked retry that never armed a backoff timer', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const retry = vi.fn()
|
||||
const firstEpoch = state.begin()
|
||||
expect(state.parkRetryForExternalTrigger(firstEpoch, retry)).toBe(true)
|
||||
|
||||
// Why: parking must not fire on its own, so the pane still latches at the cutoff.
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 300_000)
|
||||
expect(state.currentPhase).toBe('disconnected')
|
||||
expect(retry).not.toHaveBeenCalled()
|
||||
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1)
|
||||
expect(retry).toHaveBeenCalledWith(firstEpoch + 1)
|
||||
expect(state.currentPhase).toBe('recovering')
|
||||
state.dispose()
|
||||
})
|
||||
|
||||
it('refuses to park over an armed backoff or a stale epoch', () => {
|
||||
vi.useFakeTimers()
|
||||
const state = new RemoteRuntimePtyRecoveryState()
|
||||
const epoch = state.begin()
|
||||
state.schedule(epoch, vi.fn())
|
||||
const retryNow = vi.spyOn(state, 'retryNow')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
retryAllRemoteRuntimePtyRecoveriesNow()
|
||||
expect(state.parkRetryForExternalTrigger(epoch, vi.fn())).toBe(false)
|
||||
|
||||
expect(retryNow).not.toHaveBeenCalled()
|
||||
state.cancel()
|
||||
expect(state.parkRetryForExternalTrigger(epoch, vi.fn())).toBe(false)
|
||||
expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0)
|
||||
state.dispose()
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ export type RemoteRuntimePtyRecoveryPhase =
|
|||
// Why: system resume / network online need to advance pending pane backoffs without a second coordinator.
|
||||
const scheduledRecoveries = new Set<RemoteRuntimePtyRecoveryState>()
|
||||
|
||||
export function getScheduledRemoteRuntimePtyRecoveryCountForTests(): number {
|
||||
return scheduledRecoveries.size
|
||||
}
|
||||
|
||||
export function retryAllRemoteRuntimePtyRecoveriesNow(): number {
|
||||
let advanced = 0
|
||||
// Why: a synchronous retry failure can schedule the same state again.
|
||||
|
|
@ -101,14 +105,43 @@ export class RemoteRuntimePtyRecoveryState {
|
|||
return true
|
||||
}
|
||||
|
||||
// Why: a wait that ends with no liveness evidence arms no timer, so park a retry or online/resume/reconnect find nothing to revive.
|
||||
parkRetryForExternalTrigger(epoch: number, retry: (epoch: number) => void): boolean {
|
||||
if (!this.isCurrent(epoch) || this.pendingRetry !== null) {
|
||||
return false
|
||||
}
|
||||
this.pendingRetry = retry
|
||||
this.pendingEpoch = epoch
|
||||
scheduledRecoveries.add(this)
|
||||
return true
|
||||
}
|
||||
|
||||
// Why: a one-shot retry whose owner already resolved elsewhere would otherwise survive the cutoff as fake revivable work.
|
||||
discardPendingRetry(retry: (epoch: number) => void): void {
|
||||
if (this.pendingRetry !== retry) {
|
||||
return
|
||||
}
|
||||
this.clearRetryTimer()
|
||||
}
|
||||
|
||||
// Why: resume/online should fire an already-scheduled backoff immediately, not start a new epoch.
|
||||
retryNow(): boolean {
|
||||
if (this.phase !== 'backoff' || this.pendingRetry === null || this.pendingEpoch === null) {
|
||||
if (this.pendingRetry === null || this.pendingEpoch === null) {
|
||||
return false
|
||||
}
|
||||
if (this.phase !== 'backoff' && this.phase !== 'disconnected') {
|
||||
return false
|
||||
}
|
||||
const retry = this.pendingRetry
|
||||
const epoch = this.pendingEpoch
|
||||
const latched = this.phase === 'disconnected'
|
||||
this.clearRetryTimer()
|
||||
if (latched) {
|
||||
// Why: the deadline only stops auto-retry; an explicit trigger opens a fresh recovery window.
|
||||
this.epoch += 1
|
||||
this.attempt = 0
|
||||
this.armDeadline(this.epoch)
|
||||
}
|
||||
const epoch = this.epoch
|
||||
this.phase = 'recovering'
|
||||
this.onChange?.()
|
||||
retry(epoch)
|
||||
|
|
@ -159,7 +192,8 @@ export class RemoteRuntimePtyRecoveryState {
|
|||
return
|
||||
}
|
||||
this.deadlineTimer = null
|
||||
this.clearRetryTimer()
|
||||
// Why: the cutoff stops self-initiated retries but must keep the pane revivable by online/resume/reconnect.
|
||||
this.stopRetryTimer()
|
||||
this.phase = 'disconnected'
|
||||
this.onChange?.()
|
||||
}, REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS)
|
||||
|
|
@ -172,11 +206,15 @@ export class RemoteRuntimePtyRecoveryState {
|
|||
this.clearDeadlineTimer()
|
||||
}
|
||||
|
||||
private clearRetryTimer(): void {
|
||||
private stopRetryTimer(): void {
|
||||
if (this.retryTimer) {
|
||||
clearTimeout(this.retryTimer)
|
||||
this.retryTimer = null
|
||||
}
|
||||
}
|
||||
|
||||
private clearRetryTimer(): void {
|
||||
this.stopRetryTimer()
|
||||
this.pendingRetry = null
|
||||
this.pendingEpoch = null
|
||||
scheduledRecoveries.delete(this)
|
||||
|
|
|
|||
|
|
@ -187,6 +187,9 @@ export function createRemoteRuntimePtyTransport(
|
|||
let recoveryReplacementPolicy: HostHandleReplacementPolicy = 'reuse'
|
||||
let recoveryReplacementPolicyHandle: string | null = null
|
||||
let stopWaitingForPublishedHandle: (() => void) | null = null
|
||||
let publishedHandleWaitEpoch: number | null = null
|
||||
// Why: a spent auto-recovery window is the evidence that licenses reattaching the fenced handle; explicit retries must not erase it.
|
||||
let autoRecoveryWindowSpent = false
|
||||
let settleHostSessionAttachRetry: ((retry: boolean) => void) | null = null
|
||||
let resubscribeInventoryEpoch: number | null = null
|
||||
let resubscribeInventoryWindows = 0
|
||||
|
|
@ -236,12 +239,18 @@ export function createRemoteRuntimePtyTransport(
|
|||
}
|
||||
|
||||
const recovery = new RemoteRuntimePtyRecoveryState(() => {
|
||||
if (recovery.currentPhase === 'disconnected') {
|
||||
if (recovery.currentPhase === 'disposed') {
|
||||
clearPublishedHandleWait()
|
||||
}
|
||||
if (recovery.currentPhase === 'disconnected') {
|
||||
autoRecoveryWindowSpent = true
|
||||
// Why: cached pixels may remain, but no stream from the exhausted epoch may keep delivering or accepting terminal traffic.
|
||||
subscriptionGeneration += 1
|
||||
closeMultiplexedStream()
|
||||
}
|
||||
if (recovery.currentPhase === 'idle') {
|
||||
autoRecoveryWindowSpent = false
|
||||
}
|
||||
if (
|
||||
recovery.currentPhase === 'disconnected' ||
|
||||
recovery.currentPhase === 'disposed' ||
|
||||
|
|
@ -562,11 +571,16 @@ export function createRemoteRuntimePtyTransport(
|
|||
if (settleHostSessionAttachRetry === settle) {
|
||||
settleHostSessionAttachRetry = null
|
||||
}
|
||||
// Why: this wait is single-shot, so replaying it after the cutoff would strand the pane in 'recovering' with no RPC in flight.
|
||||
recovery.discardPendingRetry(scheduledRetry)
|
||||
resolve(retry)
|
||||
}
|
||||
const scheduledRetry = (): void => {
|
||||
settle(true)
|
||||
}
|
||||
settleHostSessionAttachRetry?.(false)
|
||||
settleHostSessionAttachRetry = settle
|
||||
if (!recovery.schedule(recoveryEpoch, () => settle(true))) {
|
||||
if (!recovery.schedule(recoveryEpoch, scheduledRetry)) {
|
||||
settle(false)
|
||||
}
|
||||
})
|
||||
|
|
@ -1257,6 +1271,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
function clearPublishedHandleWait(): void {
|
||||
stopWaitingForPublishedHandle?.()
|
||||
stopWaitingForPublishedHandle = null
|
||||
publishedHandleWaitEpoch = null
|
||||
}
|
||||
|
||||
function isCurrentRemoteTerminal(targetHandle: string, targetPtyId: string | null): boolean {
|
||||
|
|
@ -1329,9 +1344,30 @@ export function createRemoteRuntimePtyTransport(
|
|||
retireRemoteTerminalId()
|
||||
return
|
||||
}
|
||||
if (!update.terminalHandle || update.terminalHandle === previousHandle) {
|
||||
if (!update.terminalHandle) {
|
||||
return
|
||||
}
|
||||
if (update.terminalHandle === previousHandle) {
|
||||
// Why: once the auto-recovery window is spent, a host still publishing this surface is evidence the fenced handle outlived the stale error.
|
||||
if (!autoRecoveryWindowSpent || getCurrentMultiplexedStream(previousHandle)) {
|
||||
return
|
||||
}
|
||||
// Why: one reattach per spent window, so a handle that really is dead is not retried on every host snapshot.
|
||||
autoRecoveryWindowSpent = false
|
||||
const reattachEpoch = recovery.begin()
|
||||
clearPublishedHandleWait()
|
||||
const reusedPtyId = remotePtyId
|
||||
void subscribeToHandle(reattachEpoch, true).catch((error) => {
|
||||
if (!recoverAfterSubscribeFailure(error, previousHandle, reusedPtyId)) {
|
||||
handleRemoteTerminalError(error)
|
||||
}
|
||||
})
|
||||
return
|
||||
}
|
||||
if (recovery.currentPhase === 'disconnected') {
|
||||
// Why: without a live epoch a failed resubscribe is swallowed as already-latched, leaving a pane with no handle and no way back.
|
||||
recovery.begin()
|
||||
}
|
||||
rebindRemoteTerminalHandle(update.terminalHandle)
|
||||
const reboundHandle = handle
|
||||
const reboundPtyId = remotePtyId
|
||||
|
|
@ -1440,6 +1476,13 @@ export function createRemoteRuntimePtyTransport(
|
|||
code: 'remote_runtime_unavailable'
|
||||
})
|
||||
}
|
||||
// Why: liveness is unknown, so auto-retry stops here; keep an unarmed retry parked for online/resume/reconnect to fire.
|
||||
recovery.parkRetryForExternalTrigger(recoveryEpoch, (nextEpoch) => {
|
||||
scheduleResubscribeAfterTransportClose(
|
||||
handle ? getRecoveryReplacementPolicy(handle) : 'reuse',
|
||||
nextEpoch
|
||||
)
|
||||
})
|
||||
return
|
||||
}
|
||||
if (!nextHandle) {
|
||||
|
|
@ -1507,8 +1550,12 @@ export function createRemoteRuntimePtyTransport(
|
|||
clearPendingViewportClaim()
|
||||
}
|
||||
strengthenRecoveryReplacementPolicy(handle, replacementPolicy)
|
||||
if (replacementPolicy === 'require-replacement' && stopWaitingForPublishedHandle) {
|
||||
// Why: once recovery is handed to accepted snapshots, repeated sends to the stale handle must not re-arm inventory RPCs.
|
||||
if (
|
||||
replacementPolicy === 'require-replacement' &&
|
||||
stopWaitingForPublishedHandle &&
|
||||
// Why: only the epoch that handed recovery to accepted snapshots is blocked; a newer epoch is a fresh attempt, not a repeated stale send.
|
||||
publishedHandleWaitEpoch === recoveryEpoch
|
||||
) {
|
||||
return
|
||||
}
|
||||
if (resubscribeEpoch === recoveryEpoch) {
|
||||
|
|
@ -1529,6 +1576,7 @@ export function createRemoteRuntimePtyTransport(
|
|||
if (tabId && isWebTerminalSurfaceTabId(tabId)) {
|
||||
// Why: subscribe before polling so a fresh host snapshot can't land in the gap between the inventory loop and its event-driven fallback.
|
||||
waitForPublishedHostSessionHandle(toHostSessionTabId(tabId), resubscribeHandle)
|
||||
publishedHandleWaitEpoch = recoveryEpoch
|
||||
}
|
||||
resubscribeEpoch = recoveryEpoch
|
||||
resubscribeRequestedHandle = null
|
||||
|
|
|
|||
Loading…
Reference in New Issue