From dbbeca091644b0e577eb19e4e8c3363604d5bb4e Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:05:22 -0700 Subject: [PATCH] Let a timed-out remote terminal pane reconnect when the host session is still alive (#12213) * fix(terminal): let a timed-out remote terminal pane reconnect again A remote-runtime PTY pane that missed the 60s auto-recovery window latched to "disconnected" permanently, even while the host kept the session alive and streaming. The cutoff destroyed every path back at once: it dropped the pending retry and evicted the pane from the shared retry registry (so window `online` and system-resume became no-ops), and the transport tore down the accepted host-snapshot listener. The listener was also inert for the common case, since a host that keeps publishing the same live handle never rotates it. The cutoff now only stops the retry timer; the pane keeps its pending retry and stays revivable. The accepted-snapshot listener survives the latch, and a post-cutoff snapshot is accepted as reattach evidence whether the host rotated the handle or republished the same one. The require-replacement published-wait guard is scoped to its own recovery epoch so Reconnect and online/resume are no longer swallowed. No new polling: a latched pane still issues zero self-initiated RPCs, and the same-handle reattach consumes a snapshot the client already receives. Closes #12097 * fix(terminal): drop the settled attach retry when the recovery cutoff lands The recovery cutoff now keeps a pending retry so online/resume can revive a latched pane, but the host-session attach wait schedules a single-shot closure that the cutoff itself resolves. Retaining it left the pane registered as revivable work: 'online' would bump the epoch, arm a fresh 60s deadline and flip the phase to 'recovering' while invoking a no-op, hiding the working Reconnect button and the same-handle snapshot reattach for a full minute. The attach wait now discards its own scheduled retry as it settles, so a pane latched on that path stays 'disconnected' and Reconnect keeps working. Retries scheduled by the resubscribe paths re-enter real work and are unaffected. Co-authored-by: Orca * fix(test): type the reattach host snapshot factory so status narrows to 'ready' Co-authored-by: Orca * fix(terminal): keep an exhausted remote pane reattachable through Reconnect and resume Reconnect and the online/system-resume trigger both opened a fresh recovery epoch, which switched off the accepted-snapshot reattach path (gated on the 'disconnected' phase) for 60s while the require-replacement inventory wait dead-ended without scheduling anything. Gate the same-handle reattach on a spent auto-recovery window instead of the live phase, consumed once per window, and park an unarmed retry at the require-replacement dead end so online/resume/Reconnect have work to revive. Co-authored-by: Orca * test(terminal): pin latched-pane retention to one listener and one registry entry The recovery cutoff now keeps the retry-registry entry and the accepted-snapshot listener alive, so cover the two module-global collections that could accumulate: destroy/detach cycles, concurrent latched panes, revive storms and snapshot churn all return to baseline. Co-authored-by: Orca * refactor(terminal): use the returned reattach epoch and pin the no-RPC claim Addresses both CodeRabbit nitpicks: subscribeToHandle now takes the epoch begin() returned rather than re-reading currentEpoch, and the latched attach-wait test asserts the runtime call count is unchanged instead of only the advanced count. Co-authored-by: Orca --------- Co-authored-by: Orca --- ...mote-runtime-pty-deadline-reattach.test.ts | 381 ++++++++++++++++++ ...runtime-pty-latched-pane-retention.test.ts | 358 ++++++++++++++++ .../remote-runtime-pty-recovery-state.test.ts | 51 ++- .../remote-runtime-pty-recovery-state.ts | 46 ++- .../remote-runtime-pty-transport.ts | 58 ++- 5 files changed, 880 insertions(+), 14 deletions(-) create mode 100644 src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts create mode 100644 src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts new file mode 100644 index 000000000..886516414 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-deadline-reattach.test.ts @@ -0,0 +1,381 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson, + encodeTerminalStreamFrame, + encodeTerminalStreamJson, + encodeTerminalStreamText +} from '../../../../shared/terminal-stream-protocol' +import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types' + +describe('remote runtime pty reattach after the bounded recovery window', () => { + const runtimeCall = vi.fn() + const runtimeSubscribe = vi.fn() + const refreshSessionTabsSnapshot = vi.fn(async () => {}) + const subscriptionSendBinary = vi.fn() + let subscriptionCallbacks: { + onResponse: (response: unknown) => void + onBinary?: (bytes: Uint8Array) => void + onError?: (error: { code: string; message: string }) => void + onClose?: () => void + } | null = null + let hostListCalls = 0 + + function emitMultiplexReady(): void { + subscriptionCallbacks?.onResponse({ ok: true, result: { type: 'ready' } }) + } + + function latestSubscribePayload(): { streamId: number; terminal: string } { + const frame = subscriptionSendBinary.mock.calls + .map((call) => decodeTerminalStreamFrame(call[0])) + .findLast((candidate) => candidate?.opcode === TerminalStreamOpcode.Subscribe) + if (!frame) { + throw new Error('missing terminal subscribe frame') + } + const payload = decodeTerminalStreamJson<{ streamId: number; terminal: string }>(frame.payload) + if (!payload) { + throw new Error('invalid terminal subscribe payload') + } + return payload + } + + function subscribedTerminalHandles(): string[] { + return subscriptionSendBinary.mock.calls + .map((call) => decodeTerminalStreamFrame(call[0])) + .flatMap((frame) => { + if (frame?.opcode !== TerminalStreamOpcode.Subscribe) { + return [] + } + const payload = decodeTerminalStreamJson<{ terminal: string }>(frame.payload) + return payload ? [payload.terminal] : [] + }) + } + + function emitSnapshot(streamId: number, data: string): void { + subscriptionCallbacks?.onBinary?.( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.SnapshotStart, + streamId, + seq: 1, + payload: encodeTerminalStreamJson({ kind: 'scrollback' }) + }) + ) + subscriptionCallbacks?.onBinary?.( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.SnapshotChunk, + streamId, + seq: 2, + payload: encodeTerminalStreamText(data) + }) + ) + subscriptionCallbacks?.onBinary?.( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.SnapshotEnd, + streamId, + seq: 3, + payload: new Uint8Array() + }) + ) + } + + function hostSnapshot( + terminal: string, + snapshotVersion: number, + publicationEpoch: string + ): RuntimeMobileSessionTabsResult { + return { + worktree: 'wt-1', + publicationEpoch, + snapshotVersion, + activeGroupId: null, + activeTabId: 'tab-1::pane:1', + activeTabType: 'terminal' as const, + tabs: [ + { + type: 'terminal' as const, + id: 'tab-1::pane:1', + parentTabId: 'tab-1', + leafId: 'pane:1', + title: 'Claude Code', + isActive: true, + status: 'ready', + terminal + } + ] + } + } + + async function attachStalePane() { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onError = vi.fn() + const onPtyExit = vi.fn() + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'web-terminal-tab-1', + leafId: 'pane:1', + onPtyExit, + onPtyRebind: vi.fn() + }) + transport.attach({ + existingPtyId: 'remote:env-1@@terminal-stale', + cols: 80, + rows: 24, + callbacks: { onError } + }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + + // The host keeps publishing the same live handle, so bounded replacement polling finds + // no replacement and stops quietly without retiring the pane. + runtimeCall.mockImplementation(async (args: { method: string }) => { + if (args.method !== 'session.tabs.list') { + return { ok: true, result: {} } + } + hostListCalls += 1 + return { ok: true, result: hostSnapshot('terminal-stale', hostListCalls + 1, 'epoch-1') } + }) + subscriptionCallbacks?.onResponse({ + ok: true, + result: { + type: 'error', + streamId: latestSubscribePayload().streamId, + message: 'terminal_handle_stale' + } + }) + return { transport, onError, onPtyExit } + } + + beforeEach(() => { + vi.resetModules() + vi.doUnmock('../../runtime/remote-runtime-terminal-multiplexer') + vi.doMock('@/runtime/web-runtime-session', () => ({ + refreshWebRuntimeSessionTabsSnapshot: refreshSessionTabsSnapshot + })) + vi.clearAllMocks() + subscriptionCallbacks = null + hostListCalls = 0 + subscriptionSendBinary.mockReset() + runtimeCall.mockImplementation(async (request: { method: string; params?: unknown }) => { + if (request.method === 'session.tabs.activate') { + return { ok: true, result: hostSnapshot('terminal-stale', 1, 'epoch-1') } + } + if (request.method === 'terminal.resolvePane') { + const params = request.params as { paneKey: string; worktreeId: string } + const separator = params.paneKey.indexOf(':') + return { + ok: true, + result: { + terminal: { + handle: 'terminal-stale', + tabId: params.paneKey.slice(0, separator), + leafId: params.paneKey.slice(separator + 1), + worktreeId: params.worktreeId + } + } + } + } + return { ok: true, result: { terminal: { handle: 'terminal-stale' } } } + }) + runtimeSubscribe.mockImplementation( + async (_args: unknown, callbacks: typeof subscriptionCallbacks) => { + subscriptionCallbacks = callbacks + queueMicrotask(emitMultiplexReady) + return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary } + } + ) + vi.stubGlobal('window', { + api: { runtimeEnvironments: { call: runtimeCall, subscribe: runtimeSubscribe } } + }) + }) + + it('reattaches from a rotated host handle published after the recovery cutoff', async () => { + vi.useFakeTimers() + try { + const { transport, onPtyExit } = await attachStalePane() + const handleEvents = await import('../../runtime/web-session-terminal-handle-events') + + await vi.advanceTimersByTimeAsync(16_000) + expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1) + expect(transport.getRecoveryState?.().phase).not.toBe('disconnected') + + await vi.advanceTimersByTimeAsync(50_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + // The cutoff must not tear down the accepted-snapshot listener; it is the only path back. + expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1) + + handleEvents.queueAcceptedWebSessionTerminalSnapshot( + hostSnapshot('terminal-after-timeout', 3, 'epoch-2'), + 'env-1' + ) + await vi.waitFor(() => + expect(latestSubscribePayload()).toMatchObject({ terminal: 'terminal-after-timeout' }) + ) + + emitSnapshot(latestSubscribePayload().streamId, 'reattached') + expect(transport.isConnected()).toBe(true) + expect(transport.getPtyId()).toBe('remote:env-1@@terminal-after-timeout') + expect(onPtyExit).not.toHaveBeenCalled() + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + + it('reattaches when the post-cutoff snapshot republishes the same live handle', async () => { + vi.useFakeTimers() + try { + const { transport, onError } = await attachStalePane() + const handleEvents = await import('../../runtime/web-session-terminal-handle-events') + + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + const listCallsAtCutoff = hostListCalls + + handleEvents.queueAcceptedWebSessionTerminalSnapshot( + hostSnapshot('terminal-stale', 3, 'epoch-2'), + 'env-1' + ) + await vi.waitFor(() => expect(subscribedTerminalHandles()).toHaveLength(2)) + + expect(subscribedTerminalHandles()).toEqual(['terminal-stale', 'terminal-stale']) + // The snapshot is already-received host evidence; reattaching must cost no inventory RPC. + expect(hostListCalls).toBe(listCallsAtCutoff) + emitSnapshot(latestSubscribePayload().streamId, 'reattached') + expect(transport.isConnected()).toBe(true) + expect(onError).not.toHaveBeenCalled() + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + + it('ignores a same-handle snapshot while automatic recovery is still running', async () => { + vi.useFakeTimers() + try { + const { transport } = await attachStalePane() + const handleEvents = await import('../../runtime/web-session-terminal-handle-events') + + await vi.advanceTimersByTimeAsync(16_000) + expect(transport.getRecoveryState?.().phase).not.toBe('disconnected') + + handleEvents.queueAcceptedWebSessionTerminalSnapshot( + hostSnapshot('terminal-stale', 3, 'epoch-2'), + 'env-1' + ) + await vi.advanceTimersByTimeAsync(1_000) + + expect(subscribedTerminalHandles()).toEqual(['terminal-stale']) + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + + it('reattaches from a same-handle snapshot delivered inside the Reconnect window', async () => { + vi.useFakeTimers() + try { + const { transport, onError } = await attachStalePane() + const handleEvents = await import('../../runtime/web-session-terminal-handle-events') + + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + expect(handleEvents.getWebSessionTerminalHandleSubscriberCountForTests()).toBe(1) + const listCallsAtCutoff = hostListCalls + + expect(transport.retryRecovery?.()).toBe(true) + await vi.advanceTimersByTimeAsync(1_000) + expect(hostListCalls).toBeGreaterThan(listCallsAtCutoff) + + // Reconnect must not disarm the only path back: the click opens a window, it does not spend one. + handleEvents.queueAcceptedWebSessionTerminalSnapshot( + hostSnapshot('terminal-stale', 4, 'epoch-3'), + 'env-1' + ) + await vi.waitFor(() => expect(subscribedTerminalHandles()).toHaveLength(2)) + + emitSnapshot(latestSubscribePayload().streamId, 'reattached') + expect(transport.isConnected()).toBe(true) + expect(onError).not.toHaveBeenCalled() + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + + it('revives a latched require-replacement pane when online or system resume fires', async () => { + vi.useFakeTimers() + try { + const { transport, onError } = await attachStalePane() + const handleEvents = await import('../../runtime/web-session-terminal-handle-events') + const { retryAllRemoteRuntimePtyRecoveriesNow } = + await import('./remote-runtime-pty-recovery-state') + + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + const listCallsAtCutoff = hostListCalls + + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1) + await vi.advanceTimersByTimeAsync(1_000) + expect(hostListCalls).toBeGreaterThan(listCallsAtCutoff) + + handleEvents.queueAcceptedWebSessionTerminalSnapshot( + hostSnapshot('terminal-stale', 4, 'epoch-3'), + 'env-1' + ) + await vi.waitFor(() => expect(subscribedTerminalHandles()).toHaveLength(2)) + + emitSnapshot(latestSubscribePayload().streamId, 'reattached') + expect(transport.isConnected()).toBe(true) + expect(onError).not.toHaveBeenCalled() + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + + it('leaves Reconnect available when online fires on a pane latched during the attach wait', async () => { + vi.useFakeTimers() + try { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const { retryAllRemoteRuntimePtyRecoveriesNow } = + await import('./remote-runtime-pty-recovery-state') + runtimeCall.mockImplementation(async (request: { method: string }) => { + if (request.method === 'session.tabs.activate') { + return { + ok: false, + error: { + code: 'remote_runtime_unavailable', + message: 'Remote Orca runtime connection closed' + } + } + } + return { ok: true, result: {} } + }) + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'web-terminal-tab-1', + leafId: 'pane:1', + onPtyExit: vi.fn(), + onPtyRebind: vi.fn() + }) + transport.attach({ + existingPtyId: 'remote:env-1@@terminal-stale', + cols: 80, + rows: 24, + callbacks: { onError: vi.fn() } + }) + + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + + const callsBeforeRetry = runtimeCall.mock.calls.length + // The attach wait is single-shot and the cutoff already settled it; replaying it would spin the banner with no RPC in flight. + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + expect(runtimeCall.mock.calls.length).toBe(callsBeforeRetry) + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts new file mode 100644 index 000000000..e9c83bf67 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-latched-pane-retention.test.ts @@ -0,0 +1,358 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson +} from '../../../../shared/terminal-stream-protocol' +import type { RuntimeMobileSessionTabsResult } from '../../../../shared/runtime-types' + +// Why: the recovery cutoff no longer tears down the retry registry entry or the accepted-snapshot +// listener, so those two module-global collections are the only places a latched pane can accumulate. +describe('remote runtime pty latched-pane retention', () => { + const runtimeCall = vi.fn() + const runtimeSubscribe = vi.fn() + const refreshSessionTabsSnapshot = vi.fn(async () => {}) + const subscriptionSendBinary = vi.fn() + let subscriptionCallbacks: { + onResponse: (response: unknown) => void + onBinary?: (bytes: Uint8Array) => void + onError?: (error: { code: string; message: string }) => void + onClose?: () => void + } | null = null + let hostListCalls = 0 + + function emitMultiplexReady(): void { + subscriptionCallbacks?.onResponse({ ok: true, result: { type: 'ready' } }) + } + + function latestSubscribePayload(): { streamId: number; terminal: string } { + const frame = subscriptionSendBinary.mock.calls + .map((call) => decodeTerminalStreamFrame(call[0])) + .findLast((candidate) => candidate?.opcode === TerminalStreamOpcode.Subscribe) + if (!frame) { + throw new Error('missing terminal subscribe frame') + } + const payload = decodeTerminalStreamJson<{ streamId: number; terminal: string }>(frame.payload) + if (!payload) { + throw new Error('invalid terminal subscribe payload') + } + return payload + } + + function subscribeFrameCount(): number { + return subscriptionSendBinary.mock.calls + .map((call) => decodeTerminalStreamFrame(call[0])) + .filter((frame) => frame?.opcode === TerminalStreamOpcode.Subscribe).length + } + + // Why: every pane needs its own host surface, otherwise a later pane reuses the earlier pane's + // multiplexed stream and never enters recovery at all. + const paneIdentity = (pane: number) => ({ + hostTabId: `tab-${pane}`, + tabId: `web-terminal-tab-${pane}`, + leafId: `pane:${pane}`, + handle: `terminal-stale-${pane}` + }) + + const PANE_COUNT = 24 + + // Why: one payload carries every pane's surface, matching the real per-worktree session.tabs + // response that any polling pane receives. + function hostSnapshot( + snapshotVersion: number, + publicationEpoch: string + ): RuntimeMobileSessionTabsResult { + return { + worktree: 'wt-1', + publicationEpoch, + snapshotVersion, + activeGroupId: null, + activeTabId: `${paneIdentity(0).hostTabId}::${paneIdentity(0).leafId}`, + activeTabType: 'terminal' as const, + tabs: Array.from({ length: PANE_COUNT }, (_unused, pane) => { + const identity = paneIdentity(pane) + return { + type: 'terminal' as const, + id: `${identity.hostTabId}::${identity.leafId}`, + parentTabId: identity.hostTabId, + leafId: identity.leafId, + title: 'Claude Code', + isActive: pane === 0, + status: 'ready' as const, + terminal: identity.handle + } + }) + } + } + + async function attachStalePane(pane: number) { + const identity = paneIdentity(pane) + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: identity.tabId, + leafId: identity.leafId, + onPtyExit: vi.fn(), + onPtyRebind: vi.fn() + }) + const subscribesBefore = subscribeFrameCount() + transport.attach({ + existingPtyId: `remote:env-1@@${identity.handle}`, + cols: 80, + rows: 24, + callbacks: { onError: vi.fn() } + }) + await vi.waitFor(() => expect(subscribeFrameCount()).toBeGreaterThan(subscribesBefore)) + + // The host keeps publishing the same live handle, so bounded replacement polling finds no + // replacement and stops quietly: the exact shape that leaves the pane latched but revivable. + subscriptionCallbacks?.onResponse({ + ok: true, + result: { + type: 'error', + streamId: latestSubscribePayload().streamId, + message: 'terminal_handle_stale' + } + }) + return transport + } + + async function registries() { + const handleEvents = await import('../../runtime/web-session-terminal-handle-events') + const recoveryState = await import('./remote-runtime-pty-recovery-state') + return { + subscribers: handleEvents.getWebSessionTerminalHandleSubscriberCountForTests(), + scheduled: recoveryState.getScheduledRemoteRuntimePtyRecoveryCountForTests() + } + } + + beforeEach(() => { + vi.resetModules() + vi.doUnmock('../../runtime/remote-runtime-terminal-multiplexer') + vi.doMock('@/runtime/web-runtime-session', () => ({ + refreshWebRuntimeSessionTabsSnapshot: refreshSessionTabsSnapshot + })) + vi.clearAllMocks() + subscriptionCallbacks = null + hostListCalls = 0 + subscriptionSendBinary.mockReset() + runtimeCall.mockImplementation(async (request: { method: string; params?: unknown }) => { + if (request.method === 'session.tabs.list') { + hostListCalls += 1 + return { ok: true, result: hostSnapshot(hostListCalls + 1, 'epoch-1') } + } + if (request.method === 'session.tabs.activate') { + return { ok: true, result: hostSnapshot(1, 'epoch-1') } + } + if (request.method === 'terminal.resolvePane') { + const params = request.params as { paneKey: string; worktreeId: string } + const separator = params.paneKey.indexOf(':') + const paneTabId = params.paneKey.slice(0, separator) + const pane = Number(paneTabId.slice(paneTabId.lastIndexOf('-') + 1)) + return { + ok: true, + result: { + terminal: { + handle: paneIdentity(pane).handle, + tabId: paneTabId, + leafId: params.paneKey.slice(separator + 1), + worktreeId: params.worktreeId + } + } + } + } + return { ok: true, result: {} } + }) + runtimeSubscribe.mockImplementation( + async (_args: unknown, callbacks: typeof subscriptionCallbacks) => { + subscriptionCallbacks = callbacks + queueMicrotask(emitMultiplexReady) + return { unsubscribe: vi.fn(), sendBinary: subscriptionSendBinary } + } + ) + vi.stubGlobal('window', { + api: { runtimeEnvironments: { call: runtimeCall, subscribe: runtimeSubscribe } } + }) + }) + + it('returns listener and retry-registry counts to baseline across destroy cycles', async () => { + vi.useFakeTimers() + try { + expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 }) + const latched: { subscribers: number; scheduled: number }[] = [] + const settled: { subscribers: number; scheduled: number }[] = [] + + for (let cycle = 0; cycle < 20; cycle += 1) { + const transport = await attachStalePane(cycle) + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + latched.push(await registries()) + transport.destroy?.() + await vi.advanceTimersByTimeAsync(1_000) + settled.push(await registries()) + } + + // A latched pane deliberately holds exactly one listener and one registry entry... + expect(latched).toEqual(Array.from({ length: 20 }, () => ({ subscribers: 1, scheduled: 1 }))) + // ...and destroy releases both, so twenty cycles do not accumulate anything. + expect(settled).toEqual(Array.from({ length: 20 }, () => ({ subscribers: 0, scheduled: 0 }))) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('returns to baseline when a latched pane is detached rather than destroyed', async () => { + vi.useFakeTimers() + try { + const settled: { subscribers: number; scheduled: number }[] = [] + for (let cycle = 0; cycle < 20; cycle += 1) { + const transport = await attachStalePane(cycle) + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + transport.detach?.() + await vi.advanceTimersByTimeAsync(1_000) + settled.push(await registries()) + } + expect(settled).toEqual(Array.from({ length: 20 }, () => ({ subscribers: 0, scheduled: 0 }))) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('holds one listener and one registry entry per concurrently latched pane', async () => { + vi.useFakeTimers() + try { + const transports: Awaited>[] = [] + for (let pane = 0; pane < 8; pane += 1) { + transports.push(await attachStalePane(pane)) + await vi.advanceTimersByTimeAsync(66_000) + } + // Retention is per live pane, not per timeout: eight latched panes hold eight of each. + expect(await registries()).toEqual({ subscribers: 8, scheduled: 8 }) + + for (const transport of transports) { + transport.destroy?.() + } + await vi.advanceTimersByTimeAsync(1_000) + expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 }) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('leaves a latched pane fully quiescent — no timers, no RPCs, no growth', async () => { + vi.useFakeTimers() + try { + const transport = await attachStalePane(0) + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + + const baseline = await registries() + const callsAtLatch = runtimeCall.mock.calls.length + const timersAtLatch = vi.getTimerCount() + // Nothing is armed once the window is spent: the pane costs one listener and one registry + // entry, and nothing else, no matter how long it stays latched. + expect(timersAtLatch).toBe(0) + + await vi.advanceTimersByTimeAsync(10 * 60_000) + + expect(runtimeCall.mock.calls.length).toBe(callsAtLatch) + expect(vi.getTimerCount()).toBe(0) + expect(await registries()).toEqual(baseline) + + transport.destroy?.() + await vi.advanceTimersByTimeAsync(1_000) + expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 }) + } finally { + vi.useRealTimers() + } + }) + + it('does not stack listeners, registry entries or timers across repeated revive cycles', async () => { + vi.useFakeTimers() + try { + const { retryAllRemoteRuntimePtyRecoveriesNow } = await import( + './remote-runtime-pty-recovery-state' + ) + const transport = await attachStalePane(0) + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + + const baseline = await registries() + const timersAtFirstLatch = vi.getTimerCount() + const subscribesAtFirstLatch = subscribeFrameCount() + const observed: { + subscribers: number + scheduled: number + timers: number + revived: number + }[] = [] + + for (let cycle = 0; cycle < 25; cycle += 1) { + const revived = retryAllRemoteRuntimePtyRecoveriesNow() + // A second trigger in the same window must find nothing to advance, so an online/resume + // storm cannot stack fresh recovery epochs on one pane. + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0) + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + observed.push({ ...(await registries()), timers: vi.getTimerCount(), revived }) + } + + expect(observed).toEqual( + Array.from({ length: 25 }, () => ({ + subscribers: baseline.subscribers, + scheduled: baseline.scheduled, + timers: timersAtFirstLatch, + revived: 1 + })) + ) + // Each revive re-derives the handle; it must not leave extra live stream subscriptions behind. + expect(subscribeFrameCount()).toBe(subscribesAtFirstLatch) + + transport.destroy?.() + await vi.advanceTimersByTimeAsync(1_000) + expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 }) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it('does not stack anything when host snapshots arrive repeatedly at a latched pane', async () => { + vi.useFakeTimers() + try { + const handleEvents = await import('../../runtime/web-session-terminal-handle-events') + const transport = await attachStalePane(0) + await vi.advanceTimersByTimeAsync(66_000) + expect(transport.getRecoveryState?.().phase).toBe('disconnected') + const baseline = await registries() + + const subscribesAtLatch = subscribeFrameCount() + + for (let cycle = 0; cycle < 50; cycle += 1) { + handleEvents.queueAcceptedWebSessionTerminalSnapshot( + hostSnapshot(10 + cycle, `epoch-${cycle}`), + 'env-1' + ) + await vi.advanceTimersByTimeAsync(100) + } + + const after = await registries() + expect(after.subscribers).toBeLessThanOrEqual(baseline.subscribers) + expect(after.scheduled).toBeLessThanOrEqual(baseline.scheduled) + // At most one same-handle resubscribe per spent window: 50 snapshots inside one window + // must not become 50 subscribes. + expect(subscribeFrameCount() - subscribesAtLatch).toBeLessThanOrEqual(1) + + transport.destroy?.() + await vi.advanceTimersByTimeAsync(1_000) + expect(await registries()).toEqual({ subscribers: 0, scheduled: 0 }) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) +}) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts index f2556b737..07a9199d3 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.test.ts @@ -69,6 +69,7 @@ describe('RemoteRuntimePtyRecoveryState', () => { expect(state.isActive).toBe(false) expect(state.isCurrent(epoch)).toBe(false) expect(onChange).toHaveBeenCalled() + state.dispose() }) it('advances a pending backoff immediately via retryNow and the active registry', async () => { @@ -154,16 +155,56 @@ describe('RemoteRuntimePtyRecoveryState', () => { expect(retryNow).not.toHaveBeenCalled() }) - it('removes timed-out panes from the scheduled recovery registry', async () => { + it('keeps a timed-out pane revivable through the scheduled recovery registry', async () => { + vi.useFakeTimers() + const state = new RemoteRuntimePtyRecoveryState() + const retry = vi.fn() + const firstEpoch = state.begin() + // Why: the backoff ladder outlasts the recovery window, so a retry is still armed when the cutoff lands. + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS - 100) + state.schedule(firstEpoch, retry) + + await vi.advanceTimersByTimeAsync(100) + expect(state.currentPhase).toBe('disconnected') + + await vi.advanceTimersByTimeAsync(300_000) + expect(retry).not.toHaveBeenCalled() + + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1) + expect(retry).toHaveBeenCalledWith(firstEpoch + 1) + expect(state.currentPhase).toBe('recovering') + state.dispose() + }) + + it('revives a parked retry that never armed a backoff timer', async () => { + vi.useFakeTimers() + const state = new RemoteRuntimePtyRecoveryState() + const retry = vi.fn() + const firstEpoch = state.begin() + expect(state.parkRetryForExternalTrigger(firstEpoch, retry)).toBe(true) + + // Why: parking must not fire on its own, so the pane still latches at the cutoff. + await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS + 300_000) + expect(state.currentPhase).toBe('disconnected') + expect(retry).not.toHaveBeenCalled() + + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(1) + expect(retry).toHaveBeenCalledWith(firstEpoch + 1) + expect(state.currentPhase).toBe('recovering') + state.dispose() + }) + + it('refuses to park over an armed backoff or a stale epoch', () => { vi.useFakeTimers() const state = new RemoteRuntimePtyRecoveryState() const epoch = state.begin() state.schedule(epoch, vi.fn()) - const retryNow = vi.spyOn(state, 'retryNow') - await vi.advanceTimersByTimeAsync(REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) - retryAllRemoteRuntimePtyRecoveriesNow() + expect(state.parkRetryForExternalTrigger(epoch, vi.fn())).toBe(false) - expect(retryNow).not.toHaveBeenCalled() + state.cancel() + expect(state.parkRetryForExternalTrigger(epoch, vi.fn())).toBe(false) + expect(retryAllRemoteRuntimePtyRecoveriesNow()).toBe(0) + state.dispose() }) }) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts index df6814a40..0b609ff87 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-recovery-state.ts @@ -11,6 +11,10 @@ export type RemoteRuntimePtyRecoveryPhase = // Why: system resume / network online need to advance pending pane backoffs without a second coordinator. const scheduledRecoveries = new Set() +export function getScheduledRemoteRuntimePtyRecoveryCountForTests(): number { + return scheduledRecoveries.size +} + export function retryAllRemoteRuntimePtyRecoveriesNow(): number { let advanced = 0 // Why: a synchronous retry failure can schedule the same state again. @@ -101,14 +105,43 @@ export class RemoteRuntimePtyRecoveryState { return true } + // Why: a wait that ends with no liveness evidence arms no timer, so park a retry or online/resume/reconnect find nothing to revive. + parkRetryForExternalTrigger(epoch: number, retry: (epoch: number) => void): boolean { + if (!this.isCurrent(epoch) || this.pendingRetry !== null) { + return false + } + this.pendingRetry = retry + this.pendingEpoch = epoch + scheduledRecoveries.add(this) + return true + } + + // Why: a one-shot retry whose owner already resolved elsewhere would otherwise survive the cutoff as fake revivable work. + discardPendingRetry(retry: (epoch: number) => void): void { + if (this.pendingRetry !== retry) { + return + } + this.clearRetryTimer() + } + // Why: resume/online should fire an already-scheduled backoff immediately, not start a new epoch. retryNow(): boolean { - if (this.phase !== 'backoff' || this.pendingRetry === null || this.pendingEpoch === null) { + if (this.pendingRetry === null || this.pendingEpoch === null) { + return false + } + if (this.phase !== 'backoff' && this.phase !== 'disconnected') { return false } const retry = this.pendingRetry - const epoch = this.pendingEpoch + const latched = this.phase === 'disconnected' this.clearRetryTimer() + if (latched) { + // Why: the deadline only stops auto-retry; an explicit trigger opens a fresh recovery window. + this.epoch += 1 + this.attempt = 0 + this.armDeadline(this.epoch) + } + const epoch = this.epoch this.phase = 'recovering' this.onChange?.() retry(epoch) @@ -159,7 +192,8 @@ export class RemoteRuntimePtyRecoveryState { return } this.deadlineTimer = null - this.clearRetryTimer() + // Why: the cutoff stops self-initiated retries but must keep the pane revivable by online/resume/reconnect. + this.stopRetryTimer() this.phase = 'disconnected' this.onChange?.() }, REMOTE_RUNTIME_AUTO_RECOVERY_TIMEOUT_MS) @@ -172,11 +206,15 @@ export class RemoteRuntimePtyRecoveryState { this.clearDeadlineTimer() } - private clearRetryTimer(): void { + private stopRetryTimer(): void { if (this.retryTimer) { clearTimeout(this.retryTimer) this.retryTimer = null } + } + + private clearRetryTimer(): void { + this.stopRetryTimer() this.pendingRetry = null this.pendingEpoch = null scheduledRecoveries.delete(this) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index eedd195c7..c9e0ea435 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -187,6 +187,9 @@ export function createRemoteRuntimePtyTransport( let recoveryReplacementPolicy: HostHandleReplacementPolicy = 'reuse' let recoveryReplacementPolicyHandle: string | null = null let stopWaitingForPublishedHandle: (() => void) | null = null + let publishedHandleWaitEpoch: number | null = null + // Why: a spent auto-recovery window is the evidence that licenses reattaching the fenced handle; explicit retries must not erase it. + let autoRecoveryWindowSpent = false let settleHostSessionAttachRetry: ((retry: boolean) => void) | null = null let resubscribeInventoryEpoch: number | null = null let resubscribeInventoryWindows = 0 @@ -236,12 +239,18 @@ export function createRemoteRuntimePtyTransport( } const recovery = new RemoteRuntimePtyRecoveryState(() => { - if (recovery.currentPhase === 'disconnected') { + if (recovery.currentPhase === 'disposed') { clearPublishedHandleWait() + } + if (recovery.currentPhase === 'disconnected') { + autoRecoveryWindowSpent = true // Why: cached pixels may remain, but no stream from the exhausted epoch may keep delivering or accepting terminal traffic. subscriptionGeneration += 1 closeMultiplexedStream() } + if (recovery.currentPhase === 'idle') { + autoRecoveryWindowSpent = false + } if ( recovery.currentPhase === 'disconnected' || recovery.currentPhase === 'disposed' || @@ -562,11 +571,16 @@ export function createRemoteRuntimePtyTransport( if (settleHostSessionAttachRetry === settle) { settleHostSessionAttachRetry = null } + // Why: this wait is single-shot, so replaying it after the cutoff would strand the pane in 'recovering' with no RPC in flight. + recovery.discardPendingRetry(scheduledRetry) resolve(retry) } + const scheduledRetry = (): void => { + settle(true) + } settleHostSessionAttachRetry?.(false) settleHostSessionAttachRetry = settle - if (!recovery.schedule(recoveryEpoch, () => settle(true))) { + if (!recovery.schedule(recoveryEpoch, scheduledRetry)) { settle(false) } }) @@ -1257,6 +1271,7 @@ export function createRemoteRuntimePtyTransport( function clearPublishedHandleWait(): void { stopWaitingForPublishedHandle?.() stopWaitingForPublishedHandle = null + publishedHandleWaitEpoch = null } function isCurrentRemoteTerminal(targetHandle: string, targetPtyId: string | null): boolean { @@ -1329,9 +1344,30 @@ export function createRemoteRuntimePtyTransport( retireRemoteTerminalId() return } - if (!update.terminalHandle || update.terminalHandle === previousHandle) { + if (!update.terminalHandle) { return } + if (update.terminalHandle === previousHandle) { + // Why: once the auto-recovery window is spent, a host still publishing this surface is evidence the fenced handle outlived the stale error. + if (!autoRecoveryWindowSpent || getCurrentMultiplexedStream(previousHandle)) { + return + } + // Why: one reattach per spent window, so a handle that really is dead is not retried on every host snapshot. + autoRecoveryWindowSpent = false + const reattachEpoch = recovery.begin() + clearPublishedHandleWait() + const reusedPtyId = remotePtyId + void subscribeToHandle(reattachEpoch, true).catch((error) => { + if (!recoverAfterSubscribeFailure(error, previousHandle, reusedPtyId)) { + handleRemoteTerminalError(error) + } + }) + return + } + if (recovery.currentPhase === 'disconnected') { + // Why: without a live epoch a failed resubscribe is swallowed as already-latched, leaving a pane with no handle and no way back. + recovery.begin() + } rebindRemoteTerminalHandle(update.terminalHandle) const reboundHandle = handle const reboundPtyId = remotePtyId @@ -1440,6 +1476,13 @@ export function createRemoteRuntimePtyTransport( code: 'remote_runtime_unavailable' }) } + // Why: liveness is unknown, so auto-retry stops here; keep an unarmed retry parked for online/resume/reconnect to fire. + recovery.parkRetryForExternalTrigger(recoveryEpoch, (nextEpoch) => { + scheduleResubscribeAfterTransportClose( + handle ? getRecoveryReplacementPolicy(handle) : 'reuse', + nextEpoch + ) + }) return } if (!nextHandle) { @@ -1507,8 +1550,12 @@ export function createRemoteRuntimePtyTransport( clearPendingViewportClaim() } strengthenRecoveryReplacementPolicy(handle, replacementPolicy) - if (replacementPolicy === 'require-replacement' && stopWaitingForPublishedHandle) { - // Why: once recovery is handed to accepted snapshots, repeated sends to the stale handle must not re-arm inventory RPCs. + if ( + replacementPolicy === 'require-replacement' && + stopWaitingForPublishedHandle && + // Why: only the epoch that handed recovery to accepted snapshots is blocked; a newer epoch is a fresh attempt, not a repeated stale send. + publishedHandleWaitEpoch === recoveryEpoch + ) { return } if (resubscribeEpoch === recoveryEpoch) { @@ -1529,6 +1576,7 @@ export function createRemoteRuntimePtyTransport( if (tabId && isWebTerminalSurfaceTabId(tabId)) { // Why: subscribe before polling so a fresh host snapshot can't land in the gap between the inventory loop and its event-driven fallback. waitForPublishedHostSessionHandle(toHostSessionTabId(tabId), resubscribeHandle) + publishedHandleWaitEpoch = recoveryEpoch } resubscribeEpoch = recoveryEpoch resubscribeRequestedHandle = null