diff --git a/src/main/runtime/rpc/methods/terminal.ts b/src/main/runtime/rpc/methods/terminal.ts index bd6cad49a..8040ce4e0 100644 --- a/src/main/runtime/rpc/methods/terminal.ts +++ b/src/main/runtime/rpc/methods/terminal.ts @@ -71,6 +71,7 @@ import { sameTerminalOutputSourceIdentity, type TerminalOutputSourceRange } from '../../../../shared/terminal-output-source-range' +import type { TerminalSnapshotUnavailableReason } from '../../../../shared/terminal-snapshot-unavailability' import type { RemoteTerminalSourceRangeReplacementReservation } from '../../remote-terminal-source-range-consumer' import { withTerminalCloseAttribution } from '../terminal-close-attribution' @@ -93,6 +94,8 @@ type SnapshotFrameOptions = { cwd?: string | null truncated?: boolean truncatedByByteBudget?: boolean + // Why: distinguishes "I could not answer right now" from a genuinely empty buffer; omitted on success. + unavailable?: TerminalSnapshotUnavailableReason source?: 'headless' | 'renderer' oscLinks?: TerminalOscLinkRange[] pendingEscapeTailAnsi?: string @@ -628,6 +631,7 @@ function sendSnapshotFrames( requestId: options.requestId, displayMode: options.displayMode, reason: options.reason, + unavailable: options.unavailable, seq: options.seq, cwd: options.cwd, source: options.source, @@ -2279,6 +2283,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ displayMode, truncated: true, truncatedByByteBudget: false, + unavailable: 'pending-output-overflowed', data: '' }) return @@ -2298,6 +2303,8 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi, truncated: false, truncatedByByteBudget: serialized?.truncatedByByteBudget, + // Why: no serializer answered, which is not proof the pane is empty — say so instead of passing off '' as the buffer. + unavailable: serialized ? undefined : 'no-serializable-buffer', data: serialized?.data ?? '' }) } catch (error) { diff --git a/src/main/runtime/rpc/terminal-requested-snapshot-unavailability.test.ts b/src/main/runtime/rpc/terminal-requested-snapshot-unavailability.test.ts new file mode 100644 index 000000000..c5f05cb04 --- /dev/null +++ b/src/main/runtime/rpc/terminal-requested-snapshot-unavailability.test.ts @@ -0,0 +1,233 @@ +import { describe, expect, it, vi } from 'vitest' +import { RpcDispatcher } from './dispatcher' +import type { RpcRequest } from './core' +import type { OrcaRuntimeService } from '../orca-runtime' +import { TERMINAL_METHODS } from './methods/terminal' +import type { RuntimeTerminalWait } from '../../../shared/runtime-types' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson, + decodeTerminalStreamText, + encodeTerminalStreamFrame, + encodeTerminalStreamJson +} from '../../../shared/terminal-stream-protocol' + +type SerializedBuffer = { data: string; cols: number; rows: number } | null +type SnapshotStartPayload = Record + +// Why: 256 KiB is the pending-output budget, so this many 1 KiB chunks always trips the overflow guard. +const OVERFLOW_CHUNKS = 400 + +function makeRequest(method: string, params?: unknown): RpcRequest { + return { id: 'req-1', authToken: 'tok', method, params } +} + +/** Drives one desktop multiplex stream up to a requested-snapshot reply and returns its SnapshotStart payload. */ +async function requestSnapshotReply(options: { + connectionId: string + /** Called for each requested-snapshot serialization attempt (attempt 1 is the initial subscribe snapshot). */ + serializeRequested: (attempt: number) => Promise + /** Runs while a requested-snapshot serialization is in flight, e.g. to flood pending output. */ + duringSerialize?: (attempt: number, pushOutput: (data: string) => void) => void +}): Promise<{ start: SnapshotStartPayload; chunks: string }> { + const messages: string[] = [] + const binaryFrames: Uint8Array[] = [] + const handlers = new Map< + number, + (frame: NonNullable>) => void + >() + const cleanups = new Map void>() + const dataListenerRef: { current?: (data: string) => void } = {} + let attempt = 0 + const serializeTerminalBuffer = vi.fn(async () => { + attempt += 1 + if (attempt === 1) { + return { data: 'initial', cols: 120, rows: 40 } + } + const requestedAttempt = attempt - 1 + options.duringSerialize?.(requestedAttempt, (data) => dataListenerRef.current?.(data)) + return options.serializeRequested(requestedAttempt) + }) + const runtime = { + getRuntimeId: () => 'test-runtime', + registerRemoteTerminalViewSubscriber: () => () => {}, + resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }), + requestRendererTerminalTabMount: vi.fn().mockReturnValue(true), + updateRemoteDesktopViewer: vi.fn().mockResolvedValue(true), + unregisterRemoteDesktopViewer: vi.fn().mockResolvedValue(true), + unregisterRemoteDesktopViewers: vi.fn().mockResolvedValue(true), + isPtyResizeDrivenRemotely: vi.fn().mockReturnValue(false), + getRemoteDesktopFitHold: vi.fn().mockReturnValue({ mode: 'desktop-fit', cols: 120, rows: 40 }), + isRemoteDesktopViewerOwner: vi.fn().mockReturnValue(false), + getPtyOutputSequence: vi.fn().mockReturnValue(0), + serializeTerminalBuffer, + serializeAuthoritativeTerminalBuffer: serializeTerminalBuffer, + readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false }), + getTerminalSize: vi.fn().mockReturnValue({ cols: 120, rows: 40 }), + getMobileDisplayMode: vi.fn().mockReturnValue('auto'), + getLayout: vi.fn().mockReturnValue({ seq: 1 }), + subscribeToTerminalData: vi.fn((_: string, listener: (data: string) => void) => { + dataListenerRef.current = listener + return vi.fn() + }), + subscribeToTerminalResize: vi.fn().mockReturnValue(vi.fn()), + subscribeToFitOverrideChanges: vi.fn().mockReturnValue(vi.fn()), + subscribeToDriverChanges: vi.fn().mockReturnValue(vi.fn()), + getTerminalFitOverride: vi.fn().mockReturnValue(null), + getDriver: vi.fn().mockReturnValue({ kind: 'idle' }), + registerSubscriptionCleanup: vi.fn((id: string, cleanup: () => void) => { + cleanups.set(id, cleanup) + }), + cleanupSubscription: vi.fn((id: string) => cleanups.get(id)?.()), + waitForTerminal: vi.fn(() => new Promise(() => {})), + updateDesktopViewport: vi.fn().mockResolvedValue(true) + } as unknown as OrcaRuntimeService + + const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS }) + const dispatchPromise = dispatcher.dispatchStreaming( + makeRequest('terminal.multiplex', {}), + (msg) => messages.push(msg), + { + connectionId: options.connectionId, + sendBinary: (bytes) => { + binaryFrames.push(bytes) + }, + registerBinaryStreamHandler: (streamId, handler) => { + handlers.set(streamId, handler) + return () => handlers.delete(streamId) + } + } + ) + + await vi.waitFor(() => + expect(messages.some((msg) => JSON.parse(msg).result?.type === 'ready')).toBe(true) + ) + handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 12, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + viewport: { cols: 120, rows: 40 } + }) + }) + )! + ) + await vi.waitFor(() => + expect(messages.some((msg) => JSON.parse(msg).result?.type === 'subscribed')).toBe(true) + ) + const framesBeforeRequest = binaryFrames.length + + handlers.get(12)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.SnapshotRequest, + streamId: 12, + seq: 2, + payload: encodeTerminalStreamJson({ requestId: 77, scrollbackRows: 5000 }) + }) + )! + ) + + await vi.waitFor(() => + expect( + binaryFrames + .slice(framesBeforeRequest) + .map((frame) => decodeTerminalStreamFrame(frame)) + .some((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotEnd) + ).toBe(true) + ) + const replyFrames = binaryFrames + .slice(framesBeforeRequest) + .map((frame) => decodeTerminalStreamFrame(frame)) + const start = replyFrames.find((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart)! + cleanups.get(`terminal-multiplex:${options.connectionId}`)?.() + await dispatchPromise + return { + start: decodeTerminalStreamJson(start.payload)!, + chunks: replyFrames + .filter((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotChunk) + .map((frame) => (frame ? decodeTerminalStreamText(frame.payload) : '')) + .join('') + } +} + +describe('requested terminal snapshot unavailability reasons', () => { + it('omits a reason when the host serialized a real buffer', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-success', + serializeRequested: async () => ({ data: 'restored output', cols: 120, rows: 40 }) + }) + expect(chunks).toBe('restored output') + expect(start).toMatchObject({ requestId: 77, truncated: false }) + expect(start.unavailable).toBeUndefined() + }) + + it('omits a reason for a proven-empty buffer so absence stays distinguishable from failure', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-empty', + serializeRequested: async () => ({ data: '', cols: 120, rows: 40 }) + }) + expect(chunks).toBe('') + expect(start).toMatchObject({ requestId: 77, truncated: false }) + expect(start.unavailable).toBeUndefined() + }) + + it('reports no-serializable-buffer when no serializer answered', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-null', + serializeRequested: async () => null + }) + expect(chunks).toBe('') + // Legacy fields stay exactly as an old client expects them. + expect(start).toMatchObject({ + requestId: 77, + truncated: false, + cols: 120, + rows: 40, + unavailable: 'no-serializable-buffer' + }) + }) + + it('reports pending-output-overflowed when the retry also overflowed', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-overflow', + serializeRequested: async () => ({ data: 'never delivered', cols: 120, rows: 40 }), + duringSerialize: (_attempt, pushOutput) => { + for (let index = 0; index < OVERFLOW_CHUNKS; index += 1) { + pushOutput(String(index).padStart(3, '0') + 'x'.repeat(1021)) + } + } + }) + expect(chunks).toBe('') + expect(start).toMatchObject({ + requestId: 77, + truncated: true, + truncatedByByteBudget: false, + unavailable: 'pending-output-overflowed' + }) + }) + + it('recovers without a reason when only the first attempt overflowed', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-overflow-once', + serializeRequested: async () => ({ data: 'retry snapshot', cols: 120, rows: 40 }), + duringSerialize: (attempt, pushOutput) => { + if (attempt > 1) { + return + } + for (let index = 0; index < OVERFLOW_CHUNKS; index += 1) { + pushOutput(String(index).padStart(3, '0') + 'x'.repeat(1021)) + } + } + }) + expect(chunks).toBe('retry snapshot') + expect(start).toMatchObject({ requestId: 77, truncated: false }) + expect(start.unavailable).toBeUndefined() + }) +}) diff --git a/src/renderer/src/components/terminal-pane/pty-connection.test.ts b/src/renderer/src/components/terminal-pane/pty-connection.test.ts index 1d3936155..7ad97e6b6 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.test.ts @@ -12077,7 +12077,7 @@ describe('connectPanePty', () => { it('abandons the restore on queue overflow, writes the stream through, and repaints once', async () => { const { pane, dataCallback, getMainBufferSnapshot, resolveFirstSnapshot } = await startInFlightRestore() - const { markTerminalPinnedViewport } = + const { markTerminalFollowOutput, markTerminalPinnedViewport } = await import('@/lib/pane-manager/terminal-scroll-intent') const parseCallbacks: (() => void)[] = [] pane.terminal.write.mockImplementation((_data: string, callback?: () => void) => { @@ -12116,6 +12116,9 @@ describe('connectPanePty', () => { expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1) expect(writtenFloodData(pane)).toContain('AFTER-FLOOD') + // The repaint only runs while the viewport follows output; return there first. + markTerminalFollowOutput(pane.terminal) + // After the flood goes quiet: exactly ONE deferred repaint. vi.advanceTimersByTime(2_100) await flushAsyncTicks(20) @@ -12128,6 +12131,42 @@ describe('connectPanePty', () => { } }) + it('holds the post-flood repaint while the user reads scrollback and runs it on return to the bottom', async () => { + const { pane, dataCallback, getMainBufferSnapshot, resolveFirstSnapshot } = + await startInFlightRestore() + const { markTerminalFollowOutput, markTerminalPinnedViewport } = + await import('@/lib/pane-manager/terminal-scroll-intent') + pane.terminal.buffer.active.viewportY = 42 + pane.terminal.buffer.active.baseY = 100 + markTerminalPinnedViewport(pane.terminal) + + dataCallback('f'.repeat(300 * 1024), { seq: 300 * 1024 + 64, rawLength: 300 * 1024 }) + dataCallback('g'.repeat(300 * 1024), { seq: 600 * 1024 + 64, rawLength: 300 * 1024 }) + + try { + vi.useFakeTimers() + resolveFirstSnapshot({ data: 'flood snapshot\r\n', cols: 100, rows: 30, seq: 64 }) + await flushAsyncTicks(20) + expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1) + + // Quiet flood, but the user is still scrolled back: the clear-and-replay repaint must not move their viewport. + vi.advanceTimersByTime(2_100) + await flushAsyncTicks(20) + expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1) + vi.advanceTimersByTime(60_000) + await flushAsyncTicks(20) + expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1) + + // Returning to the bottom is the event that releases it — the heal is deferred, never dropped. + pane.terminal.buffer.active.viewportY = pane.terminal.buffer.active.baseY + markTerminalFollowOutput(pane.terminal) + await flushAsyncTicks(20) + expect(getMainBufferSnapshot).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + } + }) + it('sends salvaged queries immediately from an overflowing restore queue', async () => { const { pane, transport, dataCallback } = await startInFlightRestore() diff --git a/src/renderer/src/components/terminal-pane/pty-connection.ts b/src/renderer/src/components/terminal-pane/pty-connection.ts index fa7ce5737..3075a7217 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.ts @@ -148,6 +148,7 @@ import { inspectRuntimeTerminalProcess } from '@/runtime/runtime-terminal-inspec // actually attached — nothing is inspectable while the session hydrates. import { notifyCodexPaneBoundForStaleSweep } from '@/lib/codex-stale-pane-sweep' import { getRemoteRuntimePtyEnvironmentId } from '@/runtime/runtime-terminal-stream' +import { isHostAnsweredSnapshotRetryCause } from '@/runtime/remote-runtime-terminal-multiplexer' import { discardTerminalOutput, flushTerminalOutput, @@ -166,7 +167,8 @@ import { clearTerminalScrollbackAndFollowOutput } from '@/lib/pane-manager/termi import { enforceTerminalCurrentScrollIntent, getTerminalScrollIntentKind, - markTerminalFollowOutput + markTerminalFollowOutput, + onTerminalScrollIntentFollowOutput } from '@/lib/pane-manager/terminal-scroll-intent' import { cancelTerminalScrollIntentBufferRebuildCompletions, @@ -358,6 +360,23 @@ const HIDDEN_OUTPUT_RESTORE_FLOOD_SUPPRESS_MS = 2000 // (fresh-snapshot marks, unmappable slices) only this many times before it // abandons and lets live bytes flow. const HIDDEN_OUTPUT_RESTORE_MAX_LOOP_ITERATIONS = 3 +// Why: remote-runtime PTYs have no local main fallback — the host transport is +// the only recovery and legitimately answers null while it resyncs, trims a +// flooded snapshot, or waits out link RTT. Those nulls are not proof of loss, +// so an abandoned restore re-arms one quiet post-suppression repaint instead of +// claiming the bytes are gone. Five cycles (~2.15s each) outlast both the +// multiplexer resync window and the remote snapshot request timeout (10s); +// past that the host really is unreachable and the loss banner is honest. +const HIDDEN_OUTPUT_RESTORE_REMOTE_REARM_MAX = 5 +// Why: the host declined seven separate times; each one cost it a real serialize attempt, so stop asking. +const HIDDEN_OUTPUT_RESTORE_REMOTE_OUTCOME_MAX_ATTEMPTS = 7 +// Why separate and larger: these causes are decided before any frame leaves the +// client (resync gate, occupied request lane, detached stream), so the host +// declined nothing and charging them to its budget would banner a healthy pane — +// the exact elapsed-time guess this change removes. They cost zero host traffic, +// so the only job of this cap is termination. At the ~2s post-abandon re-arm +// cadence, 30 outlasts several full 10s resync watchdog cycles. +const HIDDEN_OUTPUT_RESTORE_LOCAL_GATE_MAX_ATTEMPTS = 30 const TERMINAL_RENDERER_RISK_SCAN_TAIL_CHARS = 256 const SYNCHRONIZED_OUTPUT_START_SEQUENCE = '\x1b[?2026h' const SYNCHRONIZED_OUTPUT_END_SEQUENCE = '\x1b[?2026l' @@ -5848,6 +5867,11 @@ export function connectPanePty( let hiddenOutputRestoreDeferredRetryTimer: ReturnType | null = null let hiddenOutputRestoreForegroundDeadlineTimer: ReturnType | null = null let hiddenOutputRestoreDeferredRetryAttempts = 0 + let hiddenOutputRestoreRemoteOutcomeAttempts = 0 + let hiddenOutputRestoreLocalGateAttempts = 0 + let hiddenOutputRestoreLegacyPtyId: string | null = null + // Bounded remote re-arms spent instead of the loss banner (per PTY stream). + let hiddenOutputRestoreRemoteAbandonCycles = 0 let hiddenOutputSnapshotScrollRestore: { ptyId: string | null generation: number @@ -5870,6 +5894,7 @@ export function connectPanePty( pendingDeliveryStartSeq?: number } | null = null let hiddenOutputRestoreFloodRepaintTimer: ReturnType | null = null + let cancelHiddenOutputRestoreFloodRepaintPark: (() => void) | null = null // Why: after a snapshot restore, main can still drain ACK-backlog chunks // whose bytes the snapshot already covers — writing them unguarded // duplicates visible output. Track the restored baseline seq (per PTY) @@ -5950,21 +5975,59 @@ export function connectPanePty( return transport.getPtyId() === ptyId && typeof transport.serializeBuffer === 'function' } + type HiddenOutputSnapshotResult = + | { kind: 'snapshot'; snapshot: PtyBufferSnapshot } + // `source` picks the budget: only 'host' answers cost the host a serialize attempt. + | { kind: 'retry-worthy'; source: 'host' | 'local' } + | { kind: 'permanently-unavailable' } + | { kind: 'unknown-legacy-host' } + | { kind: 'unavailable' } + async function serializeHiddenOutputSnapshot( ptyId: string, opts: { scrollbackRows?: number } - ): Promise { + ): Promise { const e2eSnapshot = readE2eHiddenSnapshotOverride(ptyId) if (e2eSnapshot) { - return e2eSnapshot + const snapshot = await e2eSnapshot + return snapshot ? { kind: 'snapshot', snapshot } : { kind: 'unavailable' } } if (canUseMainBufferSnapshot(ptyId)) { - return window.api.pty.getMainBufferSnapshot(ptyId, opts) + const snapshot = await window.api.pty.getMainBufferSnapshot(ptyId, opts) + return snapshot ? { kind: 'snapshot', snapshot } : { kind: 'unavailable' } } if (transport.getPtyId() !== ptyId || typeof transport.serializeBuffer !== 'function') { - return null + return { kind: 'unavailable' } + } + if ( + hiddenOutputRestoreLegacyPtyId === ptyId || + typeof transport.serializeBufferOutcome !== 'function' + ) { + const snapshot = await transport.serializeBuffer(opts) + return snapshot ? { kind: 'snapshot', snapshot } : { kind: 'unknown-legacy-host' } + } + try { + const outcome = await transport.serializeBufferOutcome(opts) + if (outcome.availability.kind === 'snapshot') { + // A success frame with no image is still the host's own answer to a request it received. + return outcome.snapshot + ? { kind: 'snapshot', snapshot: outcome.snapshot } + : { kind: 'retry-worthy', source: 'host' } + } + if (outcome.availability.kind === 'retry-worthy') { + return { + kind: 'retry-worthy', + source: isHostAnsweredSnapshotRetryCause(outcome.availability.cause) ? 'host' : 'local' + } + } + if (outcome.availability.kind === 'permanently-unavailable') { + return { kind: 'permanently-unavailable' } + } + return { kind: 'unknown-legacy-host' } + } catch { + // Why 'host': the reject path is the request timeout — the frame went out and the host stayed silent. + return { kind: 'retry-worthy', source: 'host' } } - return transport.serializeBuffer(opts) } // Why: hidden/parked panes used to mark hidden only at the first @@ -6009,6 +6072,8 @@ export function connectPanePty( } function clearHiddenOutputRestoreFloodRepaintTimer(): void { + cancelHiddenOutputRestoreFloodRepaintPark?.() + cancelHiddenOutputRestoreFloodRepaintPark = null if (hiddenOutputRestoreFloodRepaintTimer === null) { return } @@ -6017,6 +6082,24 @@ export function connectPanePty( } cleanupHiddenOutputRestoreFloodRepaint = clearHiddenOutputRestoreFloodRepaintTimer + // Why: the repaint discards the buffer and replays a full snapshot, which repositions the viewport; a user reading scrollback must not be yanked to the bottom, so hold it until their own scroll intent returns to follow-output. + function repaintAfterFloodWhenFollowingOutput(ptyId: string): void { + cancelHiddenOutputRestoreFloodRepaintPark?.() + cancelHiddenOutputRestoreFloodRepaintPark = null + let repainted = false + const cancelPark = onTerminalScrollIntentFollowOutput(pane.terminal, () => { + repainted = true + cancelHiddenOutputRestoreFloodRepaintPark = null + if (disposed || transport.getPtyId() !== ptyId) { + return + } + markHiddenOutputRestoreNeeded() + }) + if (!repainted) { + cancelHiddenOutputRestoreFloodRepaintPark = cancelPark + } + } + function resetHiddenOutputRestoreFloodSuppression(): void { hiddenOutputRestoreFloodSuppressedUntil = 0 clearHiddenOutputRestoreFloodRepaintTimer() @@ -6036,7 +6119,7 @@ export function connectPanePty( return } // Why one repaint: flood-dropped bytes leave a gap the live stream can't heal; once quiet, one snapshot restore repaints from main's authoritative buffer. - markHiddenOutputRestoreNeeded() + repaintAfterFloodWhenFollowingOutput(ptyId) }, HIDDEN_OUTPUT_RESTORE_FLOOD_SUPPRESS_MS) } @@ -6901,6 +6984,9 @@ export function connectPanePty( disposed || hiddenOutputRestoreForegroundDeadlineTimer !== null || !shouldWritePtyOutputForeground(deps.isVisibleRef.current) || + (isRemoteRuntimePtyId(hiddenOutputRestorePtyId) && + hiddenOutputRestoreLegacyPtyId !== hiddenOutputRestorePtyId && + typeof transport.serializeBufferOutcome === 'function') || (hiddenOutputRestorePendingChunks.length === 0 && !hiddenOutputRestorePendingOverflow) ) { return @@ -6925,14 +7011,42 @@ export function connectPanePty( }, HIDDEN_OUTPUT_RESTORE_FOREGROUND_TIMEOUT_MS) } + // Trades the loss banner for one bounded post-suppression repaint from the + // host's authoritative buffer. Ordered before the state reset so the repaint + // timer arms against the live ptyId (mirrors the flood-abandon call sites). + function rearmRemoteHiddenOutputRestoreInsteadOfWarning( + ptyId: string, + reason: string + ): boolean { + if ( + !isRemoteRuntimePtyId(ptyId) || + hiddenOutputRestoreRemoteAbandonCycles >= HIDDEN_OUTPUT_RESTORE_REMOTE_REARM_MAX + ) { + return false + } + hiddenOutputRestoreRemoteAbandonCycles += 1 + recordTerminalFreezeBreadcrumb('restore-abandon-rearm', { + id: redactPtyIdForDiagnostics(ptyId), + reason, + cycle: hiddenOutputRestoreRemoteAbandonCycles + }) + noteHiddenOutputRestoreFloodBackpressure() + return true + } + function abandonHiddenOutputRestoreAndDrainPendingForeground( expectedPtyId: string, - opts: { quiet?: boolean } = {} + opts: { quiet?: boolean; rearmRemote?: boolean } = {} ): void { if (transport.getPtyId() !== expectedPtyId || hiddenOutputRestorePtyId !== expectedPtyId) { resetHiddenOutputRestoreIfPtyChanged() return } + const rearmedRemoteRestore = + opts.rearmRemote !== false && + !opts.quiet && + canUseHiddenOutputSnapshot(expectedPtyId) && + rearmRemoteHiddenOutputRestoreInsteadOfWarning(expectedPtyId, 'abandon-deadline') const pendingChunks = hiddenOutputRestorePendingOverflow ? [] : hiddenOutputRestorePendingChunks.slice() @@ -6965,7 +7079,10 @@ export function connectPanePty( hiddenOutputRestoreDeferredRetryAttempts = 0 // Why quiet: flood cuts abandon deliberately and repaint post-flood, so the "restore unavailable" warning would be noise the repaint wipes. - if (!opts.quiet) { + if (!opts.quiet && !rearmedRemoteRestore) { + // Why: this abandon declares the bytes unrecoverable, so a repaint armed by earlier live + // backpressure must not outlive it — it would re-open recovery and banner a second time. + clearHiddenOutputRestoreFloodRepaintTimer() writeRestoreUnavailableWarning() } if (hadPendingOverflow) { @@ -7024,6 +7141,10 @@ export function connectPanePty( function clearHiddenOutputRestoreState(): void { cancelSnapshotScrollRestore() clearPendingLiveChunksDuringRestore() + // Re-arm budget is per PTY stream, like the rest of this state. + hiddenOutputRestoreRemoteAbandonCycles = 0 + hiddenOutputRestoreRemoteOutcomeAttempts = 0 + hiddenOutputRestoreLocalGateAttempts = 0 hiddenStartupRendererQueryPending = '' hiddenRendererStateDirty = false resetHiddenRendererRiskState() @@ -7335,7 +7456,13 @@ export function connectPanePty( if (hiddenOutputRestorePtyId === currentPtyId) { clearHiddenOutputRestoreState() } - writeRestoreUnavailableWarning() + // Remote-only path: the transport swapped PTYs mid-restore, which is a + // stream change, not proof the hidden bytes are unrecoverable. + if ( + !rearmRemoteHiddenOutputRestoreInsteadOfWarning(currentPtyId, 'restore-pty-swapped') + ) { + writeRestoreUnavailableWarning() + } return } if (transport.getPtyId() !== currentPtyId) { @@ -7346,13 +7473,19 @@ export function connectPanePty( } const restoreGeneration = hiddenOutputRestoreGeneration hiddenOutputRestoreNeeded = false - let snapshot: PtyBufferSnapshot | null = null + let snapshotResult: HiddenOutputSnapshotResult try { - snapshot = await serializeHiddenOutputSnapshot(currentPtyId, { + snapshotResult = await serializeHiddenOutputSnapshot(currentPtyId, { scrollbackRows: resolveHiddenRestoreScrollbackRows(pane.terminal.options.scrollback) }) } catch { - snapshot = null + snapshotResult = + !isRemoteRuntimePtyId(currentPtyId) || + hiddenOutputRestoreLegacyPtyId === currentPtyId || + typeof transport.serializeBufferOutcome !== 'function' + ? { kind: 'unavailable' } + : // Why 'host': the only reject here is the request timeout — the frame went out and the host stayed silent. + { kind: 'retry-worthy', source: 'host' } } if (disposed) { return @@ -7367,14 +7500,53 @@ export function connectPanePty( } return } - if (!snapshot) { + if (snapshotResult.kind === 'retry-worthy') { + let budgetExhausted: boolean + if (snapshotResult.source === 'host') { + hiddenOutputRestoreRemoteOutcomeAttempts += 1 + budgetExhausted = + hiddenOutputRestoreRemoteOutcomeAttempts >= + HIDDEN_OUTPUT_RESTORE_REMOTE_OUTCOME_MAX_ATTEMPTS + } else { + hiddenOutputRestoreLocalGateAttempts += 1 + budgetExhausted = + hiddenOutputRestoreLocalGateAttempts >= + HIDDEN_OUTPUT_RESTORE_LOCAL_GATE_MAX_ATTEMPTS + } + if (budgetExhausted) { + abandonHiddenOutputRestoreAndDrainPendingForeground(currentPtyId, { + rearmRemote: false + }) + return + } + hiddenOutputRestoreNeeded = true + hiddenOutputRestoreFreshSnapshotNeeded = false + noteHiddenOutputRestoreFloodBackpressure() + abandonHiddenOutputRestoreAndDrainPendingForeground(currentPtyId, { quiet: true }) + return + } + if (snapshotResult.kind === 'permanently-unavailable') { + abandonHiddenOutputRestoreAndDrainPendingForeground(currentPtyId, { + rearmRemote: false + }) + return + } + if (snapshotResult.kind === 'unknown-legacy-host') { + hiddenOutputRestoreLegacyPtyId = currentPtyId + armHiddenOutputRestoreForegroundDeadline() + } + if (snapshotResult.kind !== 'snapshot') { hiddenOutputRestoreNeeded = true hiddenOutputRestoreFreshSnapshotNeeded = false hiddenOutputRestoreRetryDeferred = true scheduleHiddenOutputRestoreDeferredRetry() return } + const snapshot = snapshotResult.snapshot hiddenOutputRestoreDeferredRetryAttempts = 0 + hiddenOutputRestoreRemoteAbandonCycles = 0 + hiddenOutputRestoreRemoteOutcomeAttempts = 0 + hiddenOutputRestoreLocalGateAttempts = 0 restoreIterations += 1 await applyMainBufferSnapshot(snapshot) if ( @@ -8025,9 +8197,10 @@ export function connectPanePty( prefetchedParkModelSnapshot = await fetchSshMainModelReattachSnapshot() } else { try { - prefetchedParkModelSnapshot = await serializeHiddenOutputSnapshot(ptyId, { + const result = await serializeHiddenOutputSnapshot(ptyId, { scrollbackRows: resolveHiddenRestoreScrollbackRows(pane.terminal.options.scrollback) }) + prefetchedParkModelSnapshot = result.kind === 'snapshot' ? result.snapshot : null } catch { prefetchedParkModelSnapshot = null } diff --git a/src/renderer/src/components/terminal-pane/pty-transport-types.ts b/src/renderer/src/components/terminal-pane/pty-transport-types.ts index c64b93934..3833468ed 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport-types.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport-types.ts @@ -14,6 +14,7 @@ import type { TerminalOscColorQueryReplyColors } from '../../../../shared/termin import type { TuiAgent } from '../../../../shared/types' import type { ExecutionHostId } from '../../../../shared/execution-host' import type { PtyDataMeta } from './pty-dispatcher' +import type { RemoteRuntimeSnapshotOutcome } from '../../runtime/remote-runtime-terminal-multiplexer' export type PtyBufferSnapshot = { data: string @@ -178,6 +179,9 @@ export type PtyTransport = { * would corrupt the next live chunk. IPC transports only. */ resetCrossChunkParserState?: () => void serializeBuffer?: (opts?: { scrollbackRows?: number }) => Promise + serializeBufferOutcome?: (opts?: { + scrollbackRows?: number + }) => Promise preserve?: () => void detach?: (options?: { preserveExitObserver?: boolean }) => void destroy?: () => void | Promise diff --git a/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-outcomes.test.ts b/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-outcomes.test.ts new file mode 100644 index 000000000..47bc5af02 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-outcomes.test.ts @@ -0,0 +1,858 @@ +import type * as React from 'react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { resetAgentStartupDelayedDeliveryForTests } from '@/lib/agent-startup-delayed-delivery' + +async function flushAsyncTicks(count = 6): Promise { + for (let i = 0; i < count; i++) { + await Promise.resolve() + } +} + +async function drainFakeTimerWork(limit = 20): Promise { + await flushAsyncTicks(20) + if (!vi.isFakeTimers()) { + return + } + for (let iteration = 0; iteration < limit && vi.getTimerCount() > 0; iteration += 1) { + await vi.runOnlyPendingTimersAsync() + await flushAsyncTicks(20) + } + vi.clearAllTimers() + await flushAsyncTicks(20) + vi.clearAllTimers() +} + +const LEAF_1 = '11111111-1111-4111-8111-111111111111' as const +const LEAF_2 = '22222222-2222-4222-8222-222222222222' as const + +function leafIdForPane(paneId: number): string { + return paneId === 2 ? LEAF_2 : LEAF_1 +} + +type ConnectCallbacks = { + onReattachDetermined?: () => void + onConnect?: () => void + onData?: ( + data: string, + meta?: { seq?: number; rawLength?: number; background?: boolean; droppedOutput?: boolean } + ) => void + onReplayData?: (data: string, meta?: { clearBeforeReplay?: boolean }) => void + onError?: (msg: string) => void + onWriteUnavailable?: () => void + onOutputPauseChanged?: (paused: boolean, supported: boolean) => void +} + +type MockTransport = { + attach: ReturnType + connect: ReturnType & { + mockImplementation: ( + impl: (opts: { callbacks?: ConnectCallbacks } & Record) => Promise + ) => unknown + } + disconnect: ReturnType + sendInput: ReturnType + sendInputImmediate?: ReturnType + sendInputAccepted?: ReturnType + claimViewport: ReturnType + resize: ReturnType + getPtyId: ReturnType + getConnectionId: ReturnType + serializeBuffer?: ReturnType + serializeBufferOutcome?: ReturnType +} + +const scheduleRuntimeGraphSync = vi.fn() +const shouldSeedCacheTimerOnInitialTitle = vi.fn(() => false) +const scheduleTerminalWebglAtlasRecovery = vi.fn() +const toastInfo = vi.fn() +const notifyCodexPaneBoundForStaleSweep = vi.fn() + +let mockStoreState: Record +let transportFactoryQueue: MockTransport[] = [] +let createdTransportOptions: Record[] = [] +let storeSubscribers: ((state: Record) => void)[] = [] + +vi.mock('@/runtime/sync-runtime-graph', () => ({ + scheduleRuntimeGraphSync +})) + +vi.mock('@/store', () => ({ + useAppStore: { + getState: () => mockStoreState, + subscribe: (listener: (state: Record) => void) => { + storeSubscribers.push(listener) + return () => { + storeSubscribers = storeSubscribers.filter((candidate) => candidate !== listener) + } + } + } +})) + +vi.mock('./terminal-webgl-atlas-recovery', () => ({ + scheduleTerminalWebglAtlasRecovery +})) + +vi.mock('@/lib/agent-status', async (importOriginal) => { + const actual = await importOriginal>() + const isGeminiTerminalTitle = actual.isGeminiTerminalTitle as (title: string) => boolean + return { + ...actual, + isGeminiTerminalTitle: vi.fn((title: string) => isGeminiTerminalTitle(title)), + isClaudeAgent: vi.fn(() => false), + detectAgentStatusFromTitle: vi.fn(() => null) + } +}) + +vi.mock('./cache-timer-seeding', () => ({ + shouldSeedCacheTimerOnInitialTitle +})) + +vi.mock('sonner', () => ({ + toast: { + info: toastInfo + } +})) + +vi.mock('@/lib/codex-stale-pane-sweep', () => ({ + notifyCodexPaneBoundForStaleSweep +})) + +vi.mock('react', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + useCallback: unknown>(fn: T): T => fn + } +}) + +vi.mock('./pty-transport', () => ({ + createIpcPtyTransport: vi.fn((options: Record) => { + createdTransportOptions.push(options) + const nextTransport = transportFactoryQueue.shift() + if (!nextTransport) { + throw new Error('No mock transport queued') + } + return nextTransport + }) +})) + +vi.mock('./remote-runtime-pty-transport', () => ({ + createRemoteRuntimePtyTransport: vi.fn( + (_environmentId: string, options: Record) => { + createdTransportOptions.push(options) + const nextTransport = transportFactoryQueue.shift() + if (!nextTransport) { + throw new Error('No mock transport queued') + } + return nextTransport + } + ) +})) + +vi.mock('./pty-dispatcher', async (importOriginal) => { + const actual = await importOriginal>() + return { + ...actual, + getEagerPtyBufferHandle: vi.fn(() => undefined) + } +}) + +function createMockTransport(initialPtyId: string | null = null): MockTransport { + let ptyId = initialPtyId + const transport = { + attach: vi.fn(({ existingPtyId }: { existingPtyId: string }) => { + ptyId = existingPtyId + }), + connect: vi.fn().mockImplementation(async (opts: { sessionId?: string }) => { + if (opts.sessionId) { + ptyId = opts.sessionId + return { id: opts.sessionId } + } + return ptyId + }), + disconnect: vi.fn(() => { + ptyId = null + }), + sendInput: vi.fn(() => true), + claimViewport: vi.fn(() => true), + resize: vi.fn(() => true), + getPtyId: vi.fn(() => ptyId), + getConnectionId: vi.fn(() => null), + serializeBuffer: undefined + } as MockTransport + const sendInput = transport.sendInput as unknown as (data: string) => boolean + transport.sendInputImmediate = vi.fn((data: string) => sendInput(data)) + transport.sendInputAccepted = vi.fn(async (data: string) => sendInput(data)) + return transport +} + +function createPaneContainer(): HTMLElement { + const container = new EventTarget() as HTMLElement + Object.defineProperty(container, 'dataset', { + configurable: true, + value: {} + }) + return container +} + +function createPane(paneId: number) { + const leafId = leafIdForPane(paneId) + const activeBuffer = { + type: 'normal' as const, + viewportY: 0, + baseY: 0, + cursorY: 0, + cursorX: 0 + } + const terminal = { + cols: 120, + rows: 40, + element: {}, + buffer: { + active: activeBuffer + }, + modes: { + bracketedPasteMode: false, + sendFocusMode: false + }, + options: { + scrollback: 5_000, + ignoreBracketedPasteMode: false, + theme: { + foreground: '#eeeeee', + background: '#111111' + } + }, + write: vi.fn<(data: string, callback?: () => void) => void>(function write(...args): void { + const [data, callback] = args + if (data === '' || callback?.name === 'runParsedSteps') { + callback?.() + } + }), + resize: vi.fn(), + clear: vi.fn(), + scrollToBottom: vi.fn(() => { + activeBuffer.viewportY = activeBuffer.baseY + }), + scrollToLine: vi.fn((line: number) => { + activeBuffer.viewportY = line + }), + scrollLines: vi.fn((amount: number) => { + activeBuffer.viewportY = Math.max( + 0, + Math.min(activeBuffer.baseY, activeBuffer.viewportY + amount) + ) + }), + paste: vi.fn(), + onData: vi.fn(() => ({ dispose: vi.fn() })), + onResize: vi.fn(() => ({ dispose: vi.fn() })), + onRender: vi.fn((_listener: () => void) => ({ dispose: vi.fn() })), + onTitleChange: vi.fn(() => ({ dispose: vi.fn() })), + hasSelection: vi.fn(() => false), + parser: { + registerCsiHandler: vi.fn(() => ({ dispose: vi.fn() })), + registerOscHandler: vi.fn(() => ({ dispose: vi.fn() })) + } + } + return { + id: paneId, + leafId, + stablePaneId: leafId, + terminal, + container: createPaneContainer(), + fitAddon: { + fit: vi.fn(), + proposeDimensions: vi.fn(() => ({ cols: terminal.cols, rows: terminal.rows })) + } + } +} + +function createManager(paneCount = 1, initialActivePaneId: number | null = null) { + let activePaneId = initialActivePaneId + const panes = Array.from({ length: paneCount }, (_, index) => ({ + id: index + 1, + leafId: leafIdForPane(index + 1) + })) + return { + setPaneGpuRendering: vi.fn(), + markPaneHasComplexScriptOutput: vi.fn(), + rebuildPaneWebgl: vi.fn(), + hasWebglRenderer: vi.fn(() => false), + getPanes: vi.fn(() => panes), + closePane: vi.fn(), + getActivePane: vi.fn<() => { id: number; leafId?: string } | null>(() => + activePaneId === null + ? null + : (panes.find((candidate) => candidate.id === activePaneId) ?? null) + ), + getNumericIdForLeaf: vi.fn((leafId: string) => { + return panes.find((candidate) => candidate.leafId === leafId)?.id ?? null + }), + setActivePane: vi.fn((paneId: number) => { + activePaneId = paneId + }) + } +} + +function createDeps(overrides: Record = {}) { + return { + tabId: 'tab-1', + worktreeId: 'wt-1', + cwd: '/tmp/wt-1', + startup: null, + restoredLeafId: null, + restoredPtyIdByLeafId: {}, + paneTransportsRef: { current: new Map() }, + paneMode2031Ref: { current: new Map() }, + paneKittyKeyboardModesRef: { current: new Map() }, + paneLastThemeModeRef: { current: new Map() }, + replayingPanesRef: { current: new Map() }, + isActiveRef: { current: true }, + isVisibleRef: { current: true }, + onPtyExitRef: { current: vi.fn() }, + onAgentExitedRef: { current: vi.fn() }, + onPtyErrorRef: { current: vi.fn() }, + clearTabPtyId: vi.fn(), + consumeSuppressedPtyExit: vi.fn(() => false), + isPtyShutdownPending: vi.fn(() => false), + updateTabTitle: vi.fn(), + setRuntimePaneTitle: vi.fn(), + clearRuntimePaneTitle: vi.fn(), + updateTabPtyId: vi.fn((tabId: string, ptyId: string, replacedPtyId?: string) => { + const byTab = (mockStoreState.ptyIdsByTabId ?? {}) as Record + const current = byTab[tabId] ?? [] + const next = + replacedPtyId && current.includes(replacedPtyId) + ? current.map((candidate) => (candidate === replacedPtyId ? ptyId : candidate)) + : current.includes(ptyId) + ? current + : [...current, ptyId] + mockStoreState.ptyIdsByTabId = { ...byTab, [tabId]: next } + }), + markWorktreeUnread: vi.fn(), + markTerminalTabUnread: vi.fn(), + markTerminalPaneUnread: vi.fn(), + clearWorktreeUnread: vi.fn(), + clearTerminalTabUnread: vi.fn(), + clearTerminalPaneUnread: vi.fn(), + dispatchNotification: vi.fn(), + onShowSessionRestoredBanner: vi.fn(), + setCacheTimerStartedAt: vi.fn(), + syncPanePtyLayoutBinding: vi.fn(), + clearExitedPanePtyLayoutBinding: vi.fn(), + ...overrides + } +} + +function createDeferred(): { + promise: Promise + resolve: (value: T) => void + reject: (reason?: unknown) => void +} { + let resolveDeferred!: (value: T) => void + let rejectDeferred!: (reason?: unknown) => void + const promise = new Promise((resolve, reject) => { + resolveDeferred = resolve + rejectDeferred = reject + }) + return { promise, resolve: resolveDeferred, reject: rejectDeferred } +} + +// ── Scenario identities ───────────────────────────────────────────────────── +const REMOTE_PTY_ID = 'remote:env-1@@terminal-agent-1' +// Overflows the renderer's hidden background queue (2MB lossy cap), dropping the +// backlog and latching model restore — mirrors the real remote pause semantics +// where hidden-time bytes exist ONLY in the host's authoritative buffer. +const HIDDEN_BYTES = 'x'.repeat(2 * 1024 * 1024 + 1) +const LIVE_AGENT_CHUNK = 'LIVE_AGENT_CHUNK\r\n' +const HOST_SNAPSHOT_MARKER = 'HOST_SNAPSHOT_HIDDEN_AGENT_CONTENT' +const HOST_SNAPSHOT = { + data: `${HOST_SNAPSHOT_MARKER}\r\n`, + cols: 120, + rows: 40, + seq: HIDDEN_BYTES.length + LIVE_AGENT_CHUNK.length, + source: 'headless' +} +const BANNER_FRAGMENT = 'main recovery was unavailable' + +type HostSnapshot = typeof HOST_SNAPSHOT + +type RemotePaneDrive = { + transport: MockTransport + pane: ReturnType + deps: ReturnType + disposable: { dispose: () => void } + deliver: (data: string, seq: number) => void + setOutputPaused: (paused: boolean) => void + writtenChunks: () => string[] +} + +async function connectHiddenRemoteAgentPane( + serializeBuffer: ReturnType, + serializeBufferOutcome?: ReturnType +): Promise { + const { connectPanePty } = await import('./pty-connection') + const transport = createMockTransport(REMOTE_PTY_ID) + transport.serializeBuffer = serializeBuffer + transport.serializeBufferOutcome = serializeBufferOutcome + const capturedDataCallback: { + current: ((data: string, meta?: { seq?: number; rawLength?: number }) => void) | null + } = { current: null } + const capturedOutputPauseCallback: { + current: ((paused: boolean, supported: boolean) => void) | null + } = { current: null } + transport.connect.mockImplementation(async ({ callbacks }: { callbacks?: ConnectCallbacks }) => { + capturedDataCallback.current = callbacks?.onData ?? null + capturedOutputPauseCallback.current = callbacks?.onOutputPauseChanged ?? null + return REMOTE_PTY_ID + }) + transportFactoryQueue.push(transport) + const pane = createPane(1) + const manager = createManager(1) + const deps = createDeps({ isVisibleRef: { current: false } }) + const disposable = connectPanePty(pane as never, manager as never, deps as never) + await flushAsyncTicks(6) + expect(capturedDataCallback.current).not.toBeNull() + return { + transport, + pane, + deps, + disposable, + deliver: (data, seq) => capturedDataCallback.current?.(data, { seq, rawLength: data.length }), + setOutputPaused: (paused) => capturedOutputPauseCallback.current?.(paused, true), + writtenChunks: () => pane.terminal.write.mock.calls.map(([data]) => String(data)) + } +} + +// Delivers the hidden backlog, reveals the pane, and delivers one live agent +// chunk — the exact user flow in the screenshot (agent streaming on reveal). +function driveHiddenBacklogThenReveal(drive: RemotePaneDrive): void { + drive.deliver(HIDDEN_BYTES, HIDDEN_BYTES.length) + ;(drive.deps.isVisibleRef as { current: boolean }).current = true + drive.deliver(LIVE_AGENT_CHUNK, HIDDEN_BYTES.length + LIVE_AGENT_CHUNK.length) +} + +async function advanceThroughNullRetryBudget(drive: RemotePaneDrive): Promise { + for (let retry = 0; retry < 3; retry++) { + // Live output stays gated behind the in-flight restore during the window. + expect(drive.pane.terminal.write).not.toHaveBeenCalledWith( + LIVE_AGENT_CHUNK, + expect.any(Function) + ) + vi.advanceTimersByTime(50) + vi.advanceTimersByTime(0) + await flushAsyncTicks(10) + } +} + +// Generous self-heal window: lets any corrected pipeline (extended retries, +// restore re-arm, post-abandon repaint) run to completion under fake timers. +async function advanceModernRetryProbe(): Promise { + await vi.advanceTimersByTimeAsync(2_000) + await flushAsyncTicks(20) +} + +describe('remote hidden-output restore outcomes', () => { + const originalRequestAnimationFrame = globalThis.requestAnimationFrame + const originalCancelAnimationFrame = globalThis.cancelAnimationFrame + const originalDocument = globalThis.document + + beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + transportFactoryQueue = [] + createdTransportOptions = [] + storeSubscribers = [] + mockStoreState = { + activeWorktreeId: 'wt-1', + tabsByWorktree: { + 'wt-1': [{ id: 'tab-1', ptyId: 'tab-pty' }] + }, + ptyIdsByTabId: { + 'tab-1': ['tab-pty'] + }, + terminalLayoutsByTabId: { + 'tab-1': { + root: { type: 'leaf', leafId: LEAF_1 }, + activeLeafId: LEAF_1, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF_1]: 'tab-pty' } + } + }, + unreadTerminalTabs: {}, + deleteStateByWorktreeId: {}, + worktreesByRepo: { + repo1: [{ id: 'wt-1', repoId: 'repo1', path: '/tmp/wt-1', displayName: 'feat/notis' }] + }, + runtimeStatusByEnvironmentId: new Map(), + repos: [{ id: 'repo1', connectionId: null, displayName: 'orca' }], + projects: [], + sshConnectionStates: new Map(), + transientClearedAgentStatusConnectionIds: {}, + cacheTimerByKey: {}, + settings: { + promptCacheTimerEnabled: true, + experimentalTerminalAttention: true, + terminalMainSideEffectAuthority: false + }, + codexRestartNoticeByPtyId: {}, + deferredSshReconnectTargets: [], + deferredSshSessionIdsByTabId: {}, + removeDeferredSshReconnectTarget: vi.fn(), + removeDeferredSshSessionId: vi.fn(), + consumePendingColdRestore: vi.fn(() => null), + consumePendingSnapshot: vi.fn(() => null), + runtimePaneTitlesByTabId: {}, + agentStatusByPaneKey: {} as Record, + retainedAgentsByPaneKey: {}, + paneForegroundAgentByPaneKey: {} as Record, + sleepingAgentSessionsByPaneKey: {} as Record, + suppressedPtyExitIds: {}, + agentLaunchConfigByPaneKey: {} as Record, + getAgentLaunchConfigForStatusEntry: vi.fn((entry: { paneKey: string }) => { + const byPaneKey = mockStoreState.agentLaunchConfigByPaneKey as Record< + string, + { launchConfig: unknown } | undefined + > + return byPaneKey[entry.paneKey]?.launchConfig + }), + getAgentLaunchConfigForStatusMetadata: vi.fn(() => undefined), + clearSleepingAgentSession: vi.fn((paneKey: string) => { + delete (mockStoreState.sleepingAgentSessionsByPaneKey as Record)[paneKey] + }), + registerAgentLaunchConfig: vi.fn(), + clearAgentLaunchConfig: vi.fn(), + markWorktreeUnread: vi.fn(), + observeTerminalGitHubPullRequestLink: vi.fn(), + recordTerminalInput: vi.fn(), + setAgentStatus: vi.fn(), + removeAgentStatus: vi.fn(), + dropAgentStatus: vi.fn(), + retireAgentPaneAuthority: vi.fn(), + setPaneForegroundAgent: vi.fn((paneKey: string, entry: unknown) => { + ;(mockStoreState.paneForegroundAgentByPaneKey as Record)[paneKey] = entry + }), + clearPaneForegroundAgent: vi.fn((paneKey: string) => { + delete (mockStoreState.paneForegroundAgentByPaneKey as Record)[paneKey] + }), + markTerminalTabUnread: vi.fn(), + markTerminalPaneUnread: vi.fn(), + markAgentCompletionPaneUnread: vi.fn() + } + ;(globalThis as unknown as { window: unknown }).window = { + api: { + ssh: { + connect: vi.fn().mockResolvedValue({ status: 'connected' }), + needsPassphrasePrompt: vi.fn().mockResolvedValue(false) + }, + pty: { + kill: vi.fn(), + signal: vi.fn(), + listSessions: vi.fn().mockResolvedValue([]), + hasPty: vi.fn().mockResolvedValue(true), + getSize: vi.fn().mockResolvedValue(null), + reportGeometry: vi.fn(), + getMainBufferSnapshot: vi.fn().mockResolvedValue(null), + getForegroundProcess: vi.fn().mockResolvedValue(null), + inspectProcess: vi.fn().mockResolvedValue({ + foregroundProcess: null, + hasChildProcesses: false + }), + confirmForegroundProcess: vi.fn().mockResolvedValue(null), + hasChildProcesses: vi.fn().mockResolvedValue(false), + write: vi.fn(), + writeAccepted: vi.fn().mockResolvedValue(true), + setHiddenRendererPty: vi.fn(), + setPtyDeliveryInterest: vi.fn(), + ackColdRestore: vi.fn(), + onClearBufferRequest: vi.fn(() => vi.fn()), + onSerializeBufferRequest: vi.fn(() => vi.fn()), + sendSerializedBuffer: vi.fn(), + declarePendingPaneSerializer: vi.fn().mockResolvedValue(1), + settlePaneSerializer: vi.fn().mockResolvedValue(undefined), + clearPendingPaneSerializer: vi.fn().mockResolvedValue(undefined), + reportRendererSerializerReady: vi.fn().mockResolvedValue(undefined) + }, + platform: { + get: vi.fn(() => ({ platform: 'darwin', osRelease: '25.0.0' })) + }, + notifications: { + dispatch: vi.fn().mockResolvedValue({ delivered: true }), + playSound: vi.fn().mockResolvedValue({ played: true }) + }, + runtime: { + restoreTerminalFit: vi.fn().mockResolvedValue({ restored: true }) + }, + agentStatus: { + inferInterrupt: vi.fn().mockResolvedValue(false) + } + }, + dispatchEvent: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn() + } + globalThis.requestAnimationFrame = vi.fn((callback: FrameRequestCallback) => { + callback(0) + return 1 + }) + globalThis.cancelAnimationFrame = vi.fn() + }) + + afterEach(async () => { + await drainFakeTimerWork() + vi.useRealTimers() + vi.restoreAllMocks() + if (originalRequestAnimationFrame) { + globalThis.requestAnimationFrame = originalRequestAnimationFrame + } else { + delete (globalThis as { requestAnimationFrame?: typeof requestAnimationFrame }) + .requestAnimationFrame + } + if (originalCancelAnimationFrame) { + globalThis.cancelAnimationFrame = originalCancelAnimationFrame + } else { + delete (globalThis as { cancelAnimationFrame?: typeof cancelAnimationFrame }) + .cancelAnimationFrame + } + if (originalDocument) { + globalThis.document = originalDocument + } else { + delete (globalThis as { document?: Document }).document + } + delete (globalThis as unknown as { window?: unknown }).window + resetAgentStartupDelayedDeliveryForTests() + }) + + it('[modern] accepts an empty snapshot as successful recovery without a loss banner', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'snapshot' }, + snapshot: { ...HOST_SNAPSHOT, data: '' } + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(serializeBuffer).not.toHaveBeenCalled() + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + drive.disposable.dispose() + }) + + it('[modern] waits for a reported outcome instead of applying the old elapsed-time deadline', async () => { + const pendingOutcome = createDeferred<{ + availability: { kind: 'snapshot' } + snapshot: HostSnapshot + }>() + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockReturnValue(pendingOutcome.promise) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + + pendingOutcome.resolve({ availability: { kind: 'snapshot' }, snapshot: HOST_SNAPSHOT }) + await flushAsyncTicks(20) + expect(drive.writtenChunks().join('')).toContain(HOST_SNAPSHOT_MARKER) + expect(serializeBuffer).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + it('[modern] banners on the seventh retry-worthy host answer and never sends an eighth request', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'retry-worthy', cause: 'host-pending-output-overflowed' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + for (let expectedRequests = 2; expectedRequests <= 7; expectedRequests += 1) { + await advanceModernRetryProbe() + expect(serializeBufferOutcome).toHaveBeenCalledTimes(expectedRequests) + expect(drive.writtenChunks().join('').includes(BANNER_FRAGMENT)).toBe(expectedRequests === 7) + } + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(7) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + expect(serializeBuffer).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + // Regression: these causes are returned before any frame leaves the client, so the host + // declined nothing. Charging them to its budget banners a healthy pane at ~12s — the very + // elapsed-time guess this change removes. + it('[modern] does not spend the host answer budget on locally-gated retries', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'retry-worthy', cause: 'resync-in-flight' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + // Well past the 7-answer host budget: a stuck local gate must not be mistaken for host refusals. + for (let expectedRequests = 2; expectedRequests <= 12; expectedRequests += 1) { + await advanceModernRetryProbe() + expect(serializeBufferOutcome).toHaveBeenCalledTimes(expectedRequests) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + } + + // The gate clears: the pane recovers the hidden bytes it would otherwise have declared lost. + serializeBufferOutcome.mockResolvedValue({ + availability: { kind: 'snapshot' }, + snapshot: HOST_SNAPSHOT + }) + await advanceModernRetryProbe() + expect(drive.writtenChunks().join('')).toContain(HOST_SNAPSHOT_MARKER) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + expect(serializeBuffer).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + it('[modern] still bounds locally-gated retries at their own cap', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'retry-worthy', cause: 'connection-not-ready' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + for (let expectedRequests = 2; expectedRequests <= 30; expectedRequests += 1) { + await advanceModernRetryProbe() + expect(serializeBufferOutcome).toHaveBeenCalledTimes(expectedRequests) + expect(drive.writtenChunks().join('').includes(BANNER_FRAGMENT)).toBe(expectedRequests === 30) + } + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(30) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + drive.disposable.dispose() + }) + + it('[modern] banners immediately on permanent unavailability without retrying', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { + kind: 'permanently-unavailable', + reason: 'exceeds-client-replay-limit' + }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + await vi.advanceTimersByTimeAsync(0) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(serializeBuffer).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + it('[modern] cancels a live flood repaint when it declares the output unrecoverable', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi + .fn() + .mockResolvedValueOnce({ + availability: { kind: 'retry-worthy', cause: 'host-no-serializable-buffer' }, + snapshot: null + }) + .mockResolvedValue({ + availability: { kind: 'permanently-unavailable', reason: 'exceeds-client-replay-limit' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + + // The retry answer arms a post-flood repaint 2s out; interrupt it partway with an + // ungated re-arm (remote output unpause) whose answer is permanently unavailable. + await vi.advanceTimersByTimeAsync(1_000) + drive.setOutputPaused(true) + drive.setOutputPaused(false) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(2) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + + // The still-armed repaint must not resurrect recovery the pane already declared dead. + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(2) + drive.disposable.dispose() + }) + + it('[legacy] switches an explicit unknown-host answer to the existing four-request heuristic', async () => { + const serializeBuffer = vi.fn().mockResolvedValue(null) + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'unknown-legacy-host' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + await advanceThroughNullRetryBudget(drive) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(serializeBuffer).toHaveBeenCalledTimes(3) + expect(drive.writtenChunks().join('')).toContain(LIVE_AGENT_CHUNK) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + drive.disposable.dispose() + }) + + it('[modern] parks retry repaint while scrolled back and resumes when following output', async () => { + const { markTerminalFollowOutput, markTerminalPinnedViewport } = + await import('@/lib/pane-manager/terminal-scroll-intent') + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'retry-worthy', cause: 'request-already-in-flight' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + drive.pane.terminal.buffer.active.baseY = 100 + drive.pane.terminal.buffer.active.viewportY = 42 + markTerminalPinnedViewport(drive.pane.terminal) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(drive.pane.terminal.buffer.active.viewportY).toBe(42) + + drive.pane.terminal.buffer.active.viewportY = drive.pane.terminal.buffer.active.baseY + markTerminalFollowOutput(drive.pane.terminal) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(2) + drive.disposable.dispose() + }) +}) diff --git a/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-unavailable-banner.repro.test.ts b/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-unavailable-banner.repro.test.ts new file mode 100644 index 000000000..3ce3f3ad1 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-unavailable-banner.repro.test.ts @@ -0,0 +1,827 @@ +import type * as React from 'react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { resetAgentStartupDelayedDeliveryForTests } from '@/lib/agent-startup-delayed-delivery' + +// Deterministic reproduction for issue2-hidden-output-skip: +// "[Orca skipped hidden terminal output because main recovery was unavailable.]" +// printed into a live remote-runtime agent pane. +// +// Topology modeled (renderer contract level, per reproduce-orca-remote-server-issues): +// remote-runtime-owned PTY ("remote:" id) -> transport.serializeBuffer is the ONLY +// recovery for hidden-time output (canUseMainBufferSnapshot is structurally false; +// the host DROPS paused/hidden stream output, so the reveal snapshot is the sole +// carrier of hidden bytes). The fault is injected directly below the restore +// pipeline at transport.serializeBuffer — the first seam that distinguishes +// "host recovery is available but transiently null/slow" (flood-truncated +// SnapshotResponse, resync window, Tailscale RTT) from true unavailability. +// +// Two mechanism tests pin the exact transition (they asserted the defect on main +// and now assert its fixed form); the invariant tests (RED on main) assert the +// contract: a host snapshot that is one retry tick away must not be declared +// "unavailable", hidden output must be recovered, and the added tolerance stays +// bounded — a host that never answers still banners once and stops requesting. +// +// Repro command: +// pnpm exec vitest run --config config/vitest.config.ts \ +// src/renderer/src/components/terminal-pane/remote-hidden-output-restore-unavailable-banner.repro.test.ts + +async function flushAsyncTicks(count = 6): Promise { + for (let i = 0; i < count; i++) { + await Promise.resolve() + } +} + +async function drainFakeTimerWork(limit = 20): Promise { + await flushAsyncTicks(20) + if (!vi.isFakeTimers()) { + return + } + for (let iteration = 0; iteration < limit && vi.getTimerCount() > 0; iteration += 1) { + await vi.runOnlyPendingTimersAsync() + await flushAsyncTicks(20) + } + vi.clearAllTimers() + await flushAsyncTicks(20) + vi.clearAllTimers() +} + +const LEAF_1 = '11111111-1111-4111-8111-111111111111' as const +const LEAF_2 = '22222222-2222-4222-8222-222222222222' as const + +function leafIdForPane(paneId: number): string { + return paneId === 2 ? LEAF_2 : LEAF_1 +} + +type ConnectCallbacks = { + onReattachDetermined?: () => void + onConnect?: () => void + onData?: ( + data: string, + meta?: { seq?: number; rawLength?: number; background?: boolean; droppedOutput?: boolean } + ) => void + onReplayData?: (data: string, meta?: { clearBeforeReplay?: boolean }) => void + onError?: (msg: string) => void + onWriteUnavailable?: () => void + onOutputPauseChanged?: (paused: boolean, supported: boolean) => void +} + +type MockTransport = { + attach: ReturnType + connect: ReturnType & { + mockImplementation: ( + impl: (opts: { callbacks?: ConnectCallbacks } & Record) => Promise + ) => unknown + } + disconnect: ReturnType + sendInput: ReturnType + sendInputImmediate?: ReturnType + sendInputAccepted?: ReturnType + claimViewport: ReturnType + resize: ReturnType + getPtyId: ReturnType + getConnectionId: ReturnType + serializeBuffer?: ReturnType +} + +const scheduleRuntimeGraphSync = vi.fn() +const shouldSeedCacheTimerOnInitialTitle = vi.fn(() => false) +const scheduleTerminalWebglAtlasRecovery = vi.fn() +const toastInfo = vi.fn() +const notifyCodexPaneBoundForStaleSweep = vi.fn() + +let mockStoreState: Record +let transportFactoryQueue: MockTransport[] = [] +let createdTransportOptions: Record[] = [] +let storeSubscribers: ((state: Record) => void)[] = [] + +vi.mock('@/runtime/sync-runtime-graph', () => ({ + scheduleRuntimeGraphSync +})) + +vi.mock('@/store', () => ({ + useAppStore: { + getState: () => mockStoreState, + subscribe: (listener: (state: Record) => void) => { + storeSubscribers.push(listener) + return () => { + storeSubscribers = storeSubscribers.filter((candidate) => candidate !== listener) + } + } + } +})) + +vi.mock('./terminal-webgl-atlas-recovery', () => ({ + scheduleTerminalWebglAtlasRecovery +})) + +vi.mock('@/lib/agent-status', async (importOriginal) => { + const actual = await importOriginal>() + const isGeminiTerminalTitle = actual.isGeminiTerminalTitle as (title: string) => boolean + return { + ...actual, + isGeminiTerminalTitle: vi.fn((title: string) => isGeminiTerminalTitle(title)), + isClaudeAgent: vi.fn(() => false), + detectAgentStatusFromTitle: vi.fn(() => null) + } +}) + +vi.mock('./cache-timer-seeding', () => ({ + shouldSeedCacheTimerOnInitialTitle +})) + +vi.mock('sonner', () => ({ + toast: { + info: toastInfo + } +})) + +vi.mock('@/lib/codex-stale-pane-sweep', () => ({ + notifyCodexPaneBoundForStaleSweep +})) + +vi.mock('react', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + useCallback: unknown>(fn: T): T => fn + } +}) + +vi.mock('./pty-transport', () => ({ + createIpcPtyTransport: vi.fn((options: Record) => { + createdTransportOptions.push(options) + const nextTransport = transportFactoryQueue.shift() + if (!nextTransport) { + throw new Error('No mock transport queued') + } + return nextTransport + }) +})) + +vi.mock('./remote-runtime-pty-transport', () => ({ + createRemoteRuntimePtyTransport: vi.fn( + (_environmentId: string, options: Record) => { + createdTransportOptions.push(options) + const nextTransport = transportFactoryQueue.shift() + if (!nextTransport) { + throw new Error('No mock transport queued') + } + return nextTransport + } + ) +})) + +vi.mock('./pty-dispatcher', async (importOriginal) => { + const actual = await importOriginal>() + return { + ...actual, + getEagerPtyBufferHandle: vi.fn(() => undefined) + } +}) + +function createMockTransport(initialPtyId: string | null = null): MockTransport { + let ptyId = initialPtyId + const transport = { + attach: vi.fn(({ existingPtyId }: { existingPtyId: string }) => { + ptyId = existingPtyId + }), + connect: vi.fn().mockImplementation(async (opts: { sessionId?: string }) => { + if (opts.sessionId) { + ptyId = opts.sessionId + return { id: opts.sessionId } + } + return ptyId + }), + disconnect: vi.fn(() => { + ptyId = null + }), + sendInput: vi.fn(() => true), + claimViewport: vi.fn(() => true), + resize: vi.fn(() => true), + getPtyId: vi.fn(() => ptyId), + getConnectionId: vi.fn(() => null), + serializeBuffer: undefined + } as MockTransport + const sendInput = transport.sendInput as unknown as (data: string) => boolean + transport.sendInputImmediate = vi.fn((data: string) => sendInput(data)) + transport.sendInputAccepted = vi.fn(async (data: string) => sendInput(data)) + return transport +} + +function createPaneContainer(): HTMLElement { + const container = new EventTarget() as HTMLElement + Object.defineProperty(container, 'dataset', { + configurable: true, + value: {} + }) + return container +} + +function createPane(paneId: number) { + const leafId = leafIdForPane(paneId) + const activeBuffer = { + type: 'normal' as const, + viewportY: 0, + baseY: 0, + cursorY: 0, + cursorX: 0 + } + const terminal = { + cols: 120, + rows: 40, + element: {}, + buffer: { + active: activeBuffer + }, + modes: { + bracketedPasteMode: false, + sendFocusMode: false + }, + options: { + scrollback: 5_000, + ignoreBracketedPasteMode: false, + theme: { + foreground: '#eeeeee', + background: '#111111' + } + }, + write: vi.fn<(data: string, callback?: () => void) => void>(function write(...args): void { + const [data, callback] = args + if (data === '' || callback?.name === 'runParsedSteps') { + callback?.() + } + }), + resize: vi.fn(), + clear: vi.fn(), + scrollToBottom: vi.fn(() => { + activeBuffer.viewportY = activeBuffer.baseY + }), + scrollToLine: vi.fn((line: number) => { + activeBuffer.viewportY = line + }), + scrollLines: vi.fn((amount: number) => { + activeBuffer.viewportY = Math.max( + 0, + Math.min(activeBuffer.baseY, activeBuffer.viewportY + amount) + ) + }), + paste: vi.fn(), + onData: vi.fn(() => ({ dispose: vi.fn() })), + onResize: vi.fn(() => ({ dispose: vi.fn() })), + onRender: vi.fn((_listener: () => void) => ({ dispose: vi.fn() })), + onTitleChange: vi.fn(() => ({ dispose: vi.fn() })), + hasSelection: vi.fn(() => false), + parser: { + registerCsiHandler: vi.fn(() => ({ dispose: vi.fn() })), + registerOscHandler: vi.fn(() => ({ dispose: vi.fn() })) + } + } + return { + id: paneId, + leafId, + stablePaneId: leafId, + terminal, + container: createPaneContainer(), + fitAddon: { + fit: vi.fn(), + proposeDimensions: vi.fn(() => ({ cols: terminal.cols, rows: terminal.rows })) + } + } +} + +function createManager(paneCount = 1, initialActivePaneId: number | null = null) { + let activePaneId = initialActivePaneId + const panes = Array.from({ length: paneCount }, (_, index) => ({ + id: index + 1, + leafId: leafIdForPane(index + 1) + })) + return { + setPaneGpuRendering: vi.fn(), + markPaneHasComplexScriptOutput: vi.fn(), + rebuildPaneWebgl: vi.fn(), + hasWebglRenderer: vi.fn(() => false), + getPanes: vi.fn(() => panes), + closePane: vi.fn(), + getActivePane: vi.fn<() => { id: number; leafId?: string } | null>(() => + activePaneId === null + ? null + : (panes.find((candidate) => candidate.id === activePaneId) ?? null) + ), + getNumericIdForLeaf: vi.fn((leafId: string) => { + return panes.find((candidate) => candidate.leafId === leafId)?.id ?? null + }), + setActivePane: vi.fn((paneId: number) => { + activePaneId = paneId + }) + } +} + +function createDeps(overrides: Record = {}) { + return { + tabId: 'tab-1', + worktreeId: 'wt-1', + cwd: '/tmp/wt-1', + startup: null, + restoredLeafId: null, + restoredPtyIdByLeafId: {}, + paneTransportsRef: { current: new Map() }, + paneMode2031Ref: { current: new Map() }, + paneKittyKeyboardModesRef: { current: new Map() }, + paneLastThemeModeRef: { current: new Map() }, + replayingPanesRef: { current: new Map() }, + isActiveRef: { current: true }, + isVisibleRef: { current: true }, + onPtyExitRef: { current: vi.fn() }, + onAgentExitedRef: { current: vi.fn() }, + onPtyErrorRef: { current: vi.fn() }, + clearTabPtyId: vi.fn(), + consumeSuppressedPtyExit: vi.fn(() => false), + isPtyShutdownPending: vi.fn(() => false), + updateTabTitle: vi.fn(), + setRuntimePaneTitle: vi.fn(), + clearRuntimePaneTitle: vi.fn(), + updateTabPtyId: vi.fn((tabId: string, ptyId: string, replacedPtyId?: string) => { + const byTab = (mockStoreState.ptyIdsByTabId ?? {}) as Record + const current = byTab[tabId] ?? [] + const next = + replacedPtyId && current.includes(replacedPtyId) + ? current.map((candidate) => (candidate === replacedPtyId ? ptyId : candidate)) + : current.includes(ptyId) + ? current + : [...current, ptyId] + mockStoreState.ptyIdsByTabId = { ...byTab, [tabId]: next } + }), + markWorktreeUnread: vi.fn(), + markTerminalTabUnread: vi.fn(), + markTerminalPaneUnread: vi.fn(), + clearWorktreeUnread: vi.fn(), + clearTerminalTabUnread: vi.fn(), + clearTerminalPaneUnread: vi.fn(), + dispatchNotification: vi.fn(), + onShowSessionRestoredBanner: vi.fn(), + setCacheTimerStartedAt: vi.fn(), + syncPanePtyLayoutBinding: vi.fn(), + clearExitedPanePtyLayoutBinding: vi.fn(), + ...overrides + } +} + +function createDeferred(): { + promise: Promise + resolve: (value: T) => void + reject: (reason?: unknown) => void +} { + let resolveDeferred!: (value: T) => void + let rejectDeferred!: (reason?: unknown) => void + const promise = new Promise((resolve, reject) => { + resolveDeferred = resolve + rejectDeferred = reject + }) + return { promise, resolve: resolveDeferred, reject: rejectDeferred } +} + +// ── Scenario identities ───────────────────────────────────────────────────── +const REMOTE_PTY_ID = 'remote:env-1@@terminal-agent-1' +// Overflows the renderer's hidden background queue (2MB lossy cap), dropping the +// backlog and latching model restore — mirrors the real remote pause semantics +// where hidden-time bytes exist ONLY in the host's authoritative buffer. +const HIDDEN_BYTES = 'x'.repeat(2 * 1024 * 1024 + 1) +const LIVE_AGENT_CHUNK = 'LIVE_AGENT_CHUNK\r\n' +const HOST_SNAPSHOT_MARKER = 'HOST_SNAPSHOT_HIDDEN_AGENT_CONTENT' +const HOST_SNAPSHOT = { + data: `${HOST_SNAPSHOT_MARKER}\r\n`, + cols: 120, + rows: 40, + seq: HIDDEN_BYTES.length + LIVE_AGENT_CHUNK.length, + source: 'headless' +} +const BANNER_FRAGMENT = 'main recovery was unavailable' + +type HostSnapshot = typeof HOST_SNAPSHOT + +type RemotePaneDrive = { + transport: MockTransport + pane: ReturnType + deps: ReturnType + disposable: { dispose: () => void } + deliver: (data: string, seq: number) => void + writtenChunks: () => string[] +} + +async function connectHiddenRemoteAgentPane( + serializeBuffer: ReturnType +): Promise { + const { connectPanePty } = await import('./pty-connection') + const transport = createMockTransport(REMOTE_PTY_ID) + transport.serializeBuffer = serializeBuffer + const capturedDataCallback: { + current: ((data: string, meta?: { seq?: number; rawLength?: number }) => void) | null + } = { current: null } + transport.connect.mockImplementation(async ({ callbacks }: { callbacks?: ConnectCallbacks }) => { + capturedDataCallback.current = callbacks?.onData ?? null + return REMOTE_PTY_ID + }) + transportFactoryQueue.push(transport) + const pane = createPane(1) + const manager = createManager(1) + const deps = createDeps({ isVisibleRef: { current: false } }) + const disposable = connectPanePty(pane as never, manager as never, deps as never) + await flushAsyncTicks(6) + expect(capturedDataCallback.current).not.toBeNull() + return { + transport, + pane, + deps, + disposable, + deliver: (data, seq) => capturedDataCallback.current?.(data, { seq, rawLength: data.length }), + writtenChunks: () => pane.terminal.write.mock.calls.map(([data]) => String(data)) + } +} + +// Delivers the hidden backlog, reveals the pane, and delivers one live agent +// chunk — the exact user flow in the screenshot (agent streaming on reveal). +function driveHiddenBacklogThenReveal(drive: RemotePaneDrive): void { + drive.deliver(HIDDEN_BYTES, HIDDEN_BYTES.length) + ;(drive.deps.isVisibleRef as { current: boolean }).current = true + drive.deliver(LIVE_AGENT_CHUNK, HIDDEN_BYTES.length + LIVE_AGENT_CHUNK.length) +} + +async function advanceThroughNullRetryBudget(drive: RemotePaneDrive): Promise { + for (let retry = 0; retry < 3; retry++) { + // Live output stays gated behind the in-flight restore during the window. + expect(drive.pane.terminal.write).not.toHaveBeenCalledWith( + LIVE_AGENT_CHUNK, + expect.any(Function) + ) + vi.advanceTimersByTime(50) + vi.advanceTimersByTime(0) + await flushAsyncTicks(10) + } +} + +// Generous self-heal window: lets any corrected pipeline (extended retries, +// restore re-arm, post-abandon repaint) run to completion under fake timers. +async function allowSelfHealWindow(): Promise { + for (let step = 0; step < 10; step += 1) { + await vi.advanceTimersByTimeAsync(500) + await flushAsyncTicks(20) + } +} + +function observeFinalPaneState(drive: RemotePaneDrive): { + localMainSnapshotCalls: number + rawHiddenBacklogWritten: boolean + liveAgentChunkWritten: boolean + unavailableBannerWritten: boolean + hiddenOutputRecoveredFromHostSnapshot: boolean +} { + const written = drive.writtenChunks() + const joined = written.join('') + return { + localMainSnapshotCalls: vi.mocked(window.api.pty.getMainBufferSnapshot).mock.calls.length, + rawHiddenBacklogWritten: joined.includes('x'.repeat(1024)), + liveAgentChunkWritten: joined.includes('LIVE_AGENT_CHUNK'), + unavailableBannerWritten: joined.includes(BANNER_FRAGMENT), + hiddenOutputRecoveredFromHostSnapshot: joined.includes(HOST_SNAPSHOT_MARKER) + } +} + +describe('remote hidden-output restore abandonment (issue2-hidden-output-skip)', () => { + const originalRequestAnimationFrame = globalThis.requestAnimationFrame + const originalCancelAnimationFrame = globalThis.cancelAnimationFrame + const originalDocument = globalThis.document + + beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + transportFactoryQueue = [] + createdTransportOptions = [] + storeSubscribers = [] + mockStoreState = { + activeWorktreeId: 'wt-1', + tabsByWorktree: { + 'wt-1': [{ id: 'tab-1', ptyId: 'tab-pty' }] + }, + ptyIdsByTabId: { + 'tab-1': ['tab-pty'] + }, + terminalLayoutsByTabId: { + 'tab-1': { + root: { type: 'leaf', leafId: LEAF_1 }, + activeLeafId: LEAF_1, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF_1]: 'tab-pty' } + } + }, + unreadTerminalTabs: {}, + deleteStateByWorktreeId: {}, + worktreesByRepo: { + repo1: [{ id: 'wt-1', repoId: 'repo1', path: '/tmp/wt-1', displayName: 'feat/notis' }] + }, + runtimeStatusByEnvironmentId: new Map(), + repos: [{ id: 'repo1', connectionId: null, displayName: 'orca' }], + projects: [], + sshConnectionStates: new Map(), + transientClearedAgentStatusConnectionIds: {}, + cacheTimerByKey: {}, + settings: { + promptCacheTimerEnabled: true, + experimentalTerminalAttention: true, + terminalMainSideEffectAuthority: false + }, + codexRestartNoticeByPtyId: {}, + deferredSshReconnectTargets: [], + deferredSshSessionIdsByTabId: {}, + removeDeferredSshReconnectTarget: vi.fn(), + removeDeferredSshSessionId: vi.fn(), + consumePendingColdRestore: vi.fn(() => null), + consumePendingSnapshot: vi.fn(() => null), + runtimePaneTitlesByTabId: {}, + agentStatusByPaneKey: {} as Record, + retainedAgentsByPaneKey: {}, + paneForegroundAgentByPaneKey: {} as Record, + sleepingAgentSessionsByPaneKey: {} as Record, + suppressedPtyExitIds: {}, + agentLaunchConfigByPaneKey: {} as Record, + getAgentLaunchConfigForStatusEntry: vi.fn((entry: { paneKey: string }) => { + const byPaneKey = mockStoreState.agentLaunchConfigByPaneKey as Record< + string, + { launchConfig: unknown } | undefined + > + return byPaneKey[entry.paneKey]?.launchConfig + }), + getAgentLaunchConfigForStatusMetadata: vi.fn(() => undefined), + clearSleepingAgentSession: vi.fn((paneKey: string) => { + delete (mockStoreState.sleepingAgentSessionsByPaneKey as Record)[paneKey] + }), + registerAgentLaunchConfig: vi.fn(), + clearAgentLaunchConfig: vi.fn(), + markWorktreeUnread: vi.fn(), + observeTerminalGitHubPullRequestLink: vi.fn(), + recordTerminalInput: vi.fn(), + setAgentStatus: vi.fn(), + removeAgentStatus: vi.fn(), + dropAgentStatus: vi.fn(), + retireAgentPaneAuthority: vi.fn(), + setPaneForegroundAgent: vi.fn((paneKey: string, entry: unknown) => { + ;(mockStoreState.paneForegroundAgentByPaneKey as Record)[paneKey] = entry + }), + clearPaneForegroundAgent: vi.fn((paneKey: string) => { + delete (mockStoreState.paneForegroundAgentByPaneKey as Record)[paneKey] + }), + markTerminalTabUnread: vi.fn(), + markTerminalPaneUnread: vi.fn(), + markAgentCompletionPaneUnread: vi.fn() + } + ;(globalThis as unknown as { window: unknown }).window = { + api: { + ssh: { + connect: vi.fn().mockResolvedValue({ status: 'connected' }), + needsPassphrasePrompt: vi.fn().mockResolvedValue(false) + }, + pty: { + kill: vi.fn(), + signal: vi.fn(), + listSessions: vi.fn().mockResolvedValue([]), + hasPty: vi.fn().mockResolvedValue(true), + getSize: vi.fn().mockResolvedValue(null), + reportGeometry: vi.fn(), + getMainBufferSnapshot: vi.fn().mockResolvedValue(null), + getForegroundProcess: vi.fn().mockResolvedValue(null), + inspectProcess: vi.fn().mockResolvedValue({ + foregroundProcess: null, + hasChildProcesses: false + }), + confirmForegroundProcess: vi.fn().mockResolvedValue(null), + hasChildProcesses: vi.fn().mockResolvedValue(false), + write: vi.fn(), + writeAccepted: vi.fn().mockResolvedValue(true), + setHiddenRendererPty: vi.fn(), + setPtyDeliveryInterest: vi.fn(), + ackColdRestore: vi.fn(), + onClearBufferRequest: vi.fn(() => vi.fn()), + onSerializeBufferRequest: vi.fn(() => vi.fn()), + sendSerializedBuffer: vi.fn(), + declarePendingPaneSerializer: vi.fn().mockResolvedValue(1), + settlePaneSerializer: vi.fn().mockResolvedValue(undefined), + clearPendingPaneSerializer: vi.fn().mockResolvedValue(undefined), + reportRendererSerializerReady: vi.fn().mockResolvedValue(undefined) + }, + platform: { + get: vi.fn(() => ({ platform: 'darwin', osRelease: '25.0.0' })) + }, + notifications: { + dispatch: vi.fn().mockResolvedValue({ delivered: true }), + playSound: vi.fn().mockResolvedValue({ played: true }) + }, + runtime: { + restoreTerminalFit: vi.fn().mockResolvedValue({ restored: true }) + }, + agentStatus: { + inferInterrupt: vi.fn().mockResolvedValue(false) + } + }, + dispatchEvent: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn() + } + globalThis.requestAnimationFrame = vi.fn((callback: FrameRequestCallback) => { + callback(0) + return 1 + }) + globalThis.cancelAnimationFrame = vi.fn() + }) + + afterEach(async () => { + await drainFakeTimerWork() + vi.useRealTimers() + vi.restoreAllMocks() + if (originalRequestAnimationFrame) { + globalThis.requestAnimationFrame = originalRequestAnimationFrame + } else { + delete (globalThis as { requestAnimationFrame?: typeof requestAnimationFrame }) + .requestAnimationFrame + } + if (originalCancelAnimationFrame) { + globalThis.cancelAnimationFrame = originalCancelAnimationFrame + } else { + delete (globalThis as { cancelAnimationFrame?: typeof cancelAnimationFrame }) + .cancelAnimationFrame + } + if (originalDocument) { + globalThis.document = originalDocument + } else { + delete (globalThis as { document?: Document }).document + } + delete (globalThis as unknown as { window?: unknown }).window + resetAgentStartupDelayedDeliveryForTests() + }) + + // ── Mechanism (GREEN on main): pins the defective transition exactly ────── + + it('[mechanism] 4 transient-null host snapshots end the retry budget: live drains immediately, no banner, and the restore re-arms until the host answers', async () => { + const serializeBuffer = vi.fn(async () => + serializeBuffer.mock.calls.length <= 4 ? null : HOST_SNAPSHOT + ) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + // Reveal triggered exactly one remote snapshot request against the owning runtime. + expect(serializeBuffer).toHaveBeenCalledTimes(1) + expect(serializeBuffer).toHaveBeenNthCalledWith(1, { scrollbackRows: 5000 }) + + await advanceThroughNullRetryBudget(drive) + + const written = drive.writtenChunks() + // 1 initial + 3 deferred retries (50ms each), then abandonment unblocks live + // output — but for a remote pane it must stay quiet, not claim loss. + expect(serializeBuffer).toHaveBeenCalledTimes(4) + expect(serializeBuffer).toHaveBeenNthCalledWith(4, { scrollbackRows: 5000 }) + expect(written.some((data) => data.includes('LIVE_AGENT_CHUNK'))).toBe(true) + expect(written.some((data) => data.includes(BANNER_FRAGMENT))).toBe(false) + // Remote pane never consulted local main recovery — the banner's "main + // recovery" claim is structurally impossible for this PTY. + expect(window.api.pty.getMainBufferSnapshot).not.toHaveBeenCalled() + expect((drive.transport.getPtyId as unknown as () => string | null)()).toBe(REMOTE_PTY_ID) + + // Abandonment re-arms: the host answers the next request and the hidden + // bytes are repainted from its authoritative buffer. + await allowSelfHealWindow() + expect(serializeBuffer.mock.calls.length).toBeGreaterThan(4) + const joined = drive.writtenChunks().join('') + expect(joined).toContain(HOST_SNAPSHOT_MARKER) + expect(joined).not.toContain('x'.repeat(1024)) + drive.disposable.dispose() + }) + + it('[mechanism] slow-but-valid host snapshot: the 750ms foreground deadline unblocks live output quietly and refetches the discarded snapshot', async () => { + const slowSnapshot = createDeferred() + const serializeBuffer = vi.fn().mockReturnValue(slowSnapshot.promise) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + expect(serializeBuffer).toHaveBeenCalledTimes(1) + expect(drive.pane.terminal.write).not.toHaveBeenCalledWith( + LIVE_AGENT_CHUNK, + expect.any(Function) + ) + + // Negative control: one tick before the deadline no banner exists. + vi.advanceTimersByTime(749) + await flushAsyncTicks(6) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + + vi.advanceTimersByTime(1) + vi.advanceTimersByTime(0) + await flushAsyncTicks(10) + + const written = drive.writtenChunks() + expect(written.some((data) => data.includes('LIVE_AGENT_CHUNK'))).toBe(true) + expect(written.some((data) => data.includes(BANNER_FRAGMENT))).toBe(false) + + // The host's serialization completes just after the deadline; the stale + // generation guard discards that snapshot, so the re-arm must refetch it. + slowSnapshot.resolve(HOST_SNAPSHOT) + await flushAsyncTicks(20) + expect(serializeBuffer).toHaveBeenCalledTimes(1) + expect(drive.writtenChunks().join('')).not.toContain(HOST_SNAPSHOT_MARKER) + + await allowSelfHealWindow() + expect(serializeBuffer.mock.calls.length).toBeGreaterThan(1) + expect(drive.writtenChunks().join('')).toContain(HOST_SNAPSHOT_MARKER) + expect(window.api.pty.getMainBufferSnapshot).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + // ── Invariants (RED on main): the contract a fix must satisfy ───────────── + + it('[invariant] must not declare recovery unavailable when the host snapshot is one retry tick away', async () => { + const serializeBuffer = vi.fn(async () => + serializeBuffer.mock.calls.length <= 4 ? null : HOST_SNAPSHOT + ) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + await advanceThroughNullRetryBudget(drive) + await allowSelfHealWindow() + + expect(observeFinalPaneState(drive)).toEqual({ + localMainSnapshotCalls: 0, + rawHiddenBacklogWritten: false, + liveAgentChunkWritten: true, + unavailableBannerWritten: false, + hiddenOutputRecoveredFromHostSnapshot: true + }) + drive.disposable.dispose() + }) + + it('[invariant] must not turn a merely-slow host snapshot into a loss banner and a permanent scrollback gap', async () => { + const slowSnapshot = createDeferred() + const serializeBuffer = vi.fn(async () => HOST_SNAPSHOT) + serializeBuffer.mockReturnValueOnce(slowSnapshot.promise) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + // Deadline elapses while the host is still serializing its buffer. + vi.advanceTimersByTime(750) + vi.advanceTimersByTime(0) + await flushAsyncTicks(10) + // Host serialization finishes with the authoritative hidden bytes; every + // later snapshot request would succeed instantly. + slowSnapshot.resolve(HOST_SNAPSHOT) + await flushAsyncTicks(20) + await allowSelfHealWindow() + + expect(observeFinalPaneState(drive)).toEqual({ + localMainSnapshotCalls: 0, + rawHiddenBacklogWritten: false, + liveAgentChunkWritten: true, + unavailableBannerWritten: false, + hiddenOutputRecoveredFromHostSnapshot: true + }) + drive.disposable.dispose() + }) + + // Bounds the tolerance: patience must not become an unbounded snapshot poll, + // and a host that truly never answers still owes the user a loss signal. + it('[invariant] a permanently silent host banners once after a bounded number of re-arms and then stops requesting', async () => { + const serializeBuffer = vi.fn(async () => null) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + await advanceThroughNullRetryBudget(drive) + + // Live output is never held hostage by the re-arm budget, and the first + // abandonment stays quiet. + expect(drive.writtenChunks().join('')).toContain('LIVE_AGENT_CHUNK') + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + + // ~30s: far past the 5 re-arm cycles (~2.15s each). + for (let step = 0; step < 6; step += 1) { + await allowSelfHealWindow() + } + const bannerCount = drive + .writtenChunks() + .filter((data) => data.includes(BANNER_FRAGMENT)).length + expect(bannerCount).toBe(1) + const settledRequests = serializeBuffer.mock.calls.length + // Re-armed past the initial budget, but bounded — no permanent polling. + expect(settledRequests).toBeGreaterThan(4) + expect(settledRequests).toBeLessThan(40) + + for (let step = 0; step < 4; step += 1) { + await allowSelfHealWindow() + } + expect(serializeBuffer.mock.calls.length).toBe(settledRequests) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT)).length).toBe(1) + drive.disposable.dispose() + }) +}) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts index 198c24818..16b83b005 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts @@ -5801,6 +5801,41 @@ describe('createRemoteRuntimePtyTransport', () => { expect(onData).not.toHaveBeenCalledWith('requested snapshot', expect.anything()) }) + it('forwards requested snapshot availability through the remote transport', async () => { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' }) + + await transport.connect({ url: '', callbacks: {} }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + const { streamId } = latestSubscribePayload() + emitSnapshot(streamId, 'initial') + + const outcomePromise = transport.serializeBufferOutcome?.({ scrollbackRows: 5000 }) + await vi.waitFor(() => + expect(latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest)).toBeDefined() + ) + const requestFrame = latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest) + const request = requestFrame + ? decodeTerminalStreamJson<{ requestId?: number }>(requestFrame.payload) + : null + emitSnapshotFrame( + streamId, + TerminalStreamOpcode.SnapshotStart, + encodeTerminalStreamJson({ + requestId: request?.requestId, + cols: 120, + rows: 40, + unavailable: 'no-serializable-buffer' + }) + ) + emitSnapshotFrame(streamId, TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + + await expect(outcomePromise).resolves.toMatchObject({ + availability: { kind: 'retry-worthy', cause: 'host-no-serializable-buffer' }, + snapshot: { data: '' } + }) + }) + it('keeps initial replay separate from in-flight explicit binary snapshot requests', async () => { const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') const onReplayData = vi.fn() diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index a74fed287..fb6caea57 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -43,7 +43,8 @@ import { import { getRemoteRuntimeTerminalMultiplexer, REMOTE_TERMINAL_SNAPSHOT_TOO_LARGE, - type RemoteRuntimeMultiplexedTerminal + type RemoteRuntimeMultiplexedTerminal, + type RemoteRuntimeSnapshotOutcome } from '../../runtime/remote-runtime-terminal-multiplexer' import { toRuntimeTerminalWorktreeSelector, @@ -2461,6 +2462,23 @@ export function createRemoteRuntimePtyTransport( return getCurrentMultiplexedStream(handle)?.serializeBuffer(opts) ?? null }, + async serializeBufferOutcome(opts): Promise { + if (!connected || !handle) { + return { + availability: { kind: 'retry-worthy', cause: 'connection-not-ready' }, + snapshot: null + } + } + const stream = getCurrentMultiplexedStream(handle) + if (!stream) { + return { + availability: { kind: 'retry-worthy', cause: 'stream-detached' }, + snapshot: null + } + } + return stream.serializeBufferOutcome(opts) + }, + destroy() { destroyed = true setAttachmentUnavailable() diff --git a/src/renderer/src/lib/pane-manager/terminal-follow-output-waiters.ts b/src/renderer/src/lib/pane-manager/terminal-follow-output-waiters.ts new file mode 100644 index 000000000..a5a187f71 --- /dev/null +++ b/src/renderer/src/lib/pane-manager/terminal-follow-output-waiters.ts @@ -0,0 +1,30 @@ +import type { TerminalScrollIntentTarget } from './terminal-scroll-intent' + +// Why: work that repositions the viewport (clear-and-replay repaints) must wait +// out a pinned reading position instead of racing it on a timer. Waiters are +// one-shot: the intent write that turns follow-output back on releases them all. +const followOutputWaitersByTerminal = new WeakMap void>>() + +/** Registers a one-shot listener for the terminal's next follow-output intent; returns a canceller. */ +export function addTerminalFollowOutputWaiter( + terminal: TerminalScrollIntentTarget, + listener: () => void +): () => void { + const waiters = followOutputWaitersByTerminal.get(terminal) ?? new Set<() => void>() + followOutputWaitersByTerminal.set(terminal, waiters) + waiters.add(listener) + return () => { + waiters.delete(listener) + } +} + +export function notifyTerminalFollowOutputWaiters(terminal: TerminalScrollIntentTarget): void { + const waiters = followOutputWaitersByTerminal.get(terminal) + if (!waiters?.size) { + return + } + followOutputWaitersByTerminal.delete(terminal) + for (const waiter of waiters) { + waiter() + } +} diff --git a/src/renderer/src/lib/pane-manager/terminal-scroll-intent.ts b/src/renderer/src/lib/pane-manager/terminal-scroll-intent.ts index 8fa96d62c..bd0b63611 100644 --- a/src/renderer/src/lib/pane-manager/terminal-scroll-intent.ts +++ b/src/renderer/src/lib/pane-manager/terminal-scroll-intent.ts @@ -1,3 +1,7 @@ +import { + addTerminalFollowOutputWaiter, + notifyTerminalFollowOutputWaiters +} from './terminal-follow-output-waiters' import { isTerminalScrollIntentRebuildInFlight } from './terminal-scroll-intent-rebuild' import { clampTerminalViewportY, @@ -55,6 +59,18 @@ const terminalScrollIntentBindingByKey = new Map void +): () => void { + if (getTerminalScrollIntentKind(terminal) === 'followOutput') { + listener() + return () => {} + } + return addTerminalFollowOutputWaiter(terminal, listener) +} + function writeIntent( terminal: TerminalScrollIntentTarget, kind: TerminalScrollIntentKind @@ -78,6 +94,9 @@ function writeIntentSnapshot( if (key) { terminalScrollIntentByKey.set(key, intent) } + if (kind === 'followOutput') { + notifyTerminalFollowOutputWaiters(terminal) + } return intent } diff --git a/src/renderer/src/runtime/remote-runtime-snapshot-outcome.test.ts b/src/renderer/src/runtime/remote-runtime-snapshot-outcome.test.ts new file mode 100644 index 000000000..1d903d5bd --- /dev/null +++ b/src/renderer/src/runtime/remote-runtime-snapshot-outcome.test.ts @@ -0,0 +1,301 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson, + encodeTerminalStreamFrame, + encodeTerminalStreamJson, + encodeTerminalStreamText +} from '../../../shared/terminal-stream-protocol' +import { + getRemoteRuntimeTerminalMultiplexer, + resetRemoteRuntimeTerminalMultiplexersForTests, + type RemoteRuntimeMultiplexedTerminal +} from './remote-runtime-terminal-multiplexer' +import { replaceRuntimeEnvironmentRevisions } from './runtime-environment-revision' + +// Why: the client used to collapse every unusable snapshot reply to `null`, so a slow +// host and a genuinely empty pane were indistinguishable. These tests pin the reason +// each reply now carries, and pin that the legacy `serializeBuffer` result is unchanged. + +type SubscribeCallbacks = { + onResponse: (response: unknown) => void + onBinary?: (bytes: Uint8Array) => void + onError?: (error: { message: string }) => void + onClose?: () => void +} + +type RequestedReply = + | { kind: 'buffer'; data: string } + | { kind: 'buffer'; data: string; unavailable: string } + | { kind: 'truncated'; unavailable?: string } + | { kind: 'oversized'; chunks: number } + | { kind: 'hold' } + +/** Answers SnapshotRequests with a scripted reply so each unavailability shape can be replayed exactly. */ +class ScriptedSnapshotServer { + private streamId = 0 + private cursorUnits = 0 + requestIds: (number | undefined)[] = [] + nextRequestedReply: RequestedReply = { kind: 'buffer', data: 'MANUAL' } + dropNextOutput = false + holdResyncReplies = false + private heldRequestId: number | null = null + + constructor(private readonly toClient: (bytes: Uint8Array) => void) {} + + receive(bytes: Uint8Array): void { + const frame = decodeTerminalStreamFrame(bytes) + if (!frame) { + return + } + if (frame.opcode === TerminalStreamOpcode.Subscribe) { + this.streamId = decodeTerminalStreamJson<{ streamId: number }>(frame.payload)?.streamId ?? 0 + this.sendStart({}) + this.send(TerminalStreamOpcode.SnapshotChunk, encodeTerminalStreamText('INITIAL')) + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + return + } + if (frame.opcode !== TerminalStreamOpcode.SnapshotRequest) { + return + } + const requestId = decodeTerminalStreamJson<{ requestId?: number }>(frame.payload)?.requestId + this.requestIds.push(requestId) + if (typeof requestId !== 'number') { + if (this.holdResyncReplies) { + return + } + // Untagged resync request: answer it so the resync gate does not stay latched. + this.sendStart({}) + this.send(TerminalStreamOpcode.SnapshotChunk, encodeTerminalStreamText('RECOVERED')) + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + return + } + this.replyToRequest(requestId, this.nextRequestedReply) + } + + /** Answers a request parked by `{ kind: 'hold' }` so no promise is left dangling on a real timer. */ + releaseHeldRequest(reply: RequestedReply): void { + const requestId = this.heldRequestId + this.heldRequestId = null + if (requestId !== null) { + this.replyToRequest(requestId, reply) + } + } + + private replyToRequest(requestId: number, reply: RequestedReply): void { + if (reply.kind === 'hold') { + this.heldRequestId = requestId + return + } + if (reply.kind === 'truncated') { + this.sendStart({ requestId, truncated: true, unavailable: reply.unavailable }) + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + return + } + if (reply.kind === 'oversized') { + this.sendStart({ requestId }) + for (let index = 0; index < reply.chunks; index += 1) { + this.send(TerminalStreamOpcode.SnapshotChunk, encodeTerminalStreamText('x'.repeat(512_000))) + } + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + return + } + this.sendStart({ + requestId, + unavailable: 'unavailable' in reply ? reply.unavailable : undefined + }) + if (reply.data.length > 0) { + this.send(TerminalStreamOpcode.SnapshotChunk, encodeTerminalStreamText(reply.data)) + } + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + } + + private sendStart(meta: { requestId?: number; truncated?: boolean; unavailable?: string }): void { + this.send( + TerminalStreamOpcode.SnapshotStart, + encodeTerminalStreamJson({ + cols: 80, + rows: 24, + seq: meta.truncated ? undefined : this.cursorUnits, + ...meta + }) + ) + } + + private send(opcode: TerminalStreamOpcode, payload: Uint8Array): void { + this.toClient( + encodeTerminalStreamFrame({ opcode, streamId: this.streamId, seq: this.cursorUnits, payload }) + ) + } + + output(text: string): void { + this.cursorUnits += text.length + if (this.dropNextOutput) { + this.dropNextOutput = false + return + } + this.send(TerminalStreamOpcode.Output, encodeTerminalStreamText(text)) + } +} + +describe('remote terminal snapshot outcome reasons', () => { + let server: ScriptedSnapshotServer + + beforeEach(() => { + vi.clearAllMocks() + resetRemoteRuntimeTerminalMultiplexersForTests() + replaceRuntimeEnvironmentRevisions([]) + vi.stubGlobal('window', { + api: { + runtimeEnvironments: { + subscribe: vi.fn(async (_args: unknown, callbacks: SubscribeCallbacks) => { + server = new ScriptedSnapshotServer((bytes) => callbacks.onBinary?.(bytes)) + queueMicrotask(() => callbacks.onResponse({ ok: true, result: { type: 'ready' } })) + return { + unsubscribe: vi.fn(), + sendBinary: (bytes: Uint8Array) => server.receive(bytes) + } + }) + } + } + }) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + async function subscribeClient(): Promise { + const stream = await getRemoteRuntimeTerminalMultiplexer('env-1').subscribeTerminal({ + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + callbacks: { onData: () => {}, onSnapshot: () => {} } + }) + await Promise.resolve() + await Promise.resolve() + return stream + } + + it('reports a real buffer as a snapshot and hands legacy callers the same image', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'buffer', data: 'RESTORED' } + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'snapshot' }, + snapshot: { + data: 'RESTORED', + cols: 80, + rows: 24, + seq: 0, + source: undefined, + pendingEscapeTailAnsi: undefined + } + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toMatchObject({ + data: 'RESTORED' + }) + }) + + it('reports a proven-empty buffer as a snapshot, not as an unavailable reply', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'buffer', data: '' } + + const outcome = await stream.serializeBufferOutcome({ scrollbackRows: 100 }) + expect(outcome.availability).toEqual({ kind: 'snapshot' }) + expect(outcome.snapshot?.data).toBe('') + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toMatchObject({ + data: '' + }) + }) + + it('reports a host that could not serialize as retry-worthy without changing the legacy result', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { + kind: 'buffer', + data: '', + unavailable: 'no-serializable-buffer' + } + + const outcome = await stream.serializeBufferOutcome({ scrollbackRows: 100 }) + expect(outcome.availability).toEqual({ + kind: 'retry-worthy', + cause: 'host-no-serializable-buffer' + }) + // Legacy callers still see the host's (empty) image, exactly as before the reason existed. + expect(outcome.snapshot?.data).toBe('') + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toMatchObject({ + data: '' + }) + }) + + it('reports a pending-output overflow as retry-worthy', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'truncated', unavailable: 'pending-output-overflowed' } + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'retry-worthy', cause: 'host-pending-output-overflowed' }, + snapshot: null + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toBeNull() + }) + + it('maps an old host that truncated without a reason to the explicit legacy case', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'truncated' } + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'unknown-legacy-host' }, + snapshot: null + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toBeNull() + }) + + it('reports a reply past the client replay limit as permanently unavailable', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'oversized', chunks: 5 } + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { + kind: 'permanently-unavailable', + reason: 'exceeds-client-replay-limit' + }, + snapshot: null + }) + }) + + it('reports a concurrent request as retry-worthy while the legacy path still rejects', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'hold' } + const held = stream.serializeBufferOutcome({ scrollbackRows: 100 }) + await Promise.resolve() + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'retry-worthy', cause: 'request-already-in-flight' }, + snapshot: null + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).rejects.toThrow( + 'Remote terminal snapshot already in flight.' + ) + + server.releaseHeldRequest({ kind: 'buffer', data: 'LATE' }) + await expect(held).resolves.toMatchObject({ availability: { kind: 'snapshot' } }) + }) + + it('reports an in-flight resync as retry-worthy instead of an empty answer', async () => { + const stream = await subscribeClient() + // Force a seq gap so the client latches its resync gate before the manual request. + server.holdResyncReplies = true + server.output('aaa') + server.dropNextOutput = true + server.output('bbb') + server.output('ccc') + expect(server.requestIds).toEqual([undefined]) + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'retry-worthy', cause: 'resync-in-flight' }, + snapshot: null + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toBeNull() + }) +}) diff --git a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts index 0485eb729..88d5633c3 100644 --- a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts +++ b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts @@ -10,6 +10,10 @@ import { encodeTerminalStreamJson, encodeTerminalStreamText } from '../../../shared/terminal-stream-protocol' +import { + parseTerminalSnapshotUnavailableReason, + type TerminalSnapshotUnavailableReason +} from '../../../shared/terminal-snapshot-unavailability' import { e2eConfig, e2eDisableRemoteTerminalStallRecovery } from '@/lib/e2e-config' import { recordRendererCrashBreadcrumb } from '@/lib/crash-breadcrumb-recorder' import { deliverTerminalDataWithDeferredCredit } from '@/lib/pane-manager/terminal-delivery-credit' @@ -72,6 +76,64 @@ export type RemoteRuntimeMultiplexedTerminalCallbacks = { onTransportClose?: (event: { recoverable: boolean; retryWithBackoff?: boolean }) => void } +export type RemoteRuntimeSnapshotImage = { + data: string + cols: number + rows: number + seq?: number + source?: 'headless' | 'renderer' + pendingEscapeTailAnsi?: string +} + +/** Transient causes the host itself reported: a request reached it and it declined to serialize now. */ +export type RemoteRuntimeSnapshotHostRetryCause = + | 'host-pending-output-overflowed' + | 'host-no-serializable-buffer' + +/** Transient causes decided entirely client-side: no request frame ever reached the host, so it answered nothing. */ +export type RemoteRuntimeSnapshotLocalRetryCause = + | 'resync-in-flight' + | 'stream-detached' + | 'connection-not-ready' + | 'request-already-in-flight' + | 'request-frame-not-sent' + +/** Transient causes: the same request may succeed later, so an absent buffer proves nothing about the pane. */ +export type RemoteRuntimeSnapshotRetryCause = + | RemoteRuntimeSnapshotHostRetryCause + | RemoteRuntimeSnapshotLocalRetryCause + +const HOST_ANSWERED_SNAPSHOT_RETRY_CAUSES = new Set([ + 'host-pending-output-overflowed', + 'host-no-serializable-buffer' +]) + +/** Callers budget host answers separately from local gates; only the former cost the host a request. */ +export function isHostAnsweredSnapshotRetryCause( + cause: RemoteRuntimeSnapshotRetryCause +): cause is RemoteRuntimeSnapshotHostRetryCause { + return HOST_ANSWERED_SNAPSHOT_RETRY_CAUSES.has(cause) +} + +/** Final causes: the host answered and repeating this exact request cannot produce the buffer. */ +export type RemoteRuntimeSnapshotPermanentReason = 'exceeds-client-replay-limit' + +export type RemoteRuntimeSnapshotAvailability = + | { kind: 'snapshot' } + | { kind: 'permanently-unavailable'; reason: RemoteRuntimeSnapshotPermanentReason } + | { kind: 'retry-worthy'; cause: RemoteRuntimeSnapshotRetryCause } + // Why: a pre-`unavailable` host sent an empty reply with no reason; the caller must fall back to its own heuristic. + | { kind: 'unknown-legacy-host' } + +/** + * `availability` is what the reply proves; `snapshot` is the buffer image the host actually sent. + * They are orthogonal so legacy callers can keep reading `snapshot` alone while new callers read the reason. + */ +export type RemoteRuntimeSnapshotOutcome = { + availability: RemoteRuntimeSnapshotAvailability + snapshot: RemoteRuntimeSnapshotImage | null +} + export type RemoteRuntimeMultiplexedTerminal = { streamId: number sendInput: (text: string) => boolean @@ -85,6 +147,10 @@ export type RemoteRuntimeMultiplexedTerminal = { seq?: number source?: 'headless' | 'renderer' } | null> + // Why: same request as serializeBuffer, but keeps the host's reason for an absent buffer instead of collapsing it to null. + serializeBufferOutcome: (opts?: { + scrollbackRows?: number + }) => Promise close: () => void } @@ -132,6 +198,7 @@ type RemoteRuntimeSnapshotInfo = { source?: 'headless' | 'renderer' requestId?: number truncated?: boolean + unavailable?: TerminalSnapshotUnavailableReason // Why: a mid-escape tail the emulator could not serialize; the transport // must write it AFTER the replay reset so the next live chunk completes it // instead of rendering literally (#7329). @@ -140,16 +207,7 @@ type RemoteRuntimeSnapshotInfo = { type RemoteRuntimeSnapshotRequest = { requestId: number - resolve: ( - snapshot: { - data: string - cols: number - rows: number - seq?: number - source?: 'headless' | 'renderer' - pendingEscapeTailAnsi?: string - } | null - ) => void + resolve: (outcome: RemoteRuntimeSnapshotOutcome) => void reject: (error: Error) => void timer: ReturnType } @@ -417,6 +475,7 @@ class RemoteRuntimeTerminalMultiplexer { }, setOutputPaused: (paused) => this.setOutputPaused(state, paused), serializeBuffer: (opts) => this.requestSnapshot(state, opts), + serializeBufferOutcome: (opts) => this.requestSnapshotOutcome(state, opts), close: () => { if (this.streams.get(streamId) === state) { discardOutputAcknowledgements(state) @@ -807,12 +866,15 @@ class RemoteRuntimeTerminalMultiplexer { if (snapshotApplied) { if (matchesPendingRequest) { pendingRequest.resolve({ - data: data ?? '', - cols: info?.cols ?? 80, - rows: info?.rows ?? 24, - seq: info?.seq, - source: info?.source, - pendingEscapeTailAnsi: info?.pendingEscapeTailAnsi + availability: classifySnapshotAvailability(stream.snapshotOverflowed, info), + snapshot: { + data: data ?? '', + cols: info?.cols ?? 80, + rows: info?.rows ?? 24, + seq: info?.seq, + source: info?.source, + pendingEscapeTailAnsi: info?.pendingEscapeTailAnsi + } }) clearPendingSnapshotRequest(stream) } else if (target === 'initial') { @@ -829,7 +891,10 @@ class RemoteRuntimeTerminalMultiplexer { }) } } else if (matchesPendingRequest) { - pendingRequest.resolve(null) + pendingRequest.resolve({ + availability: classifySnapshotAvailability(stream.snapshotOverflowed, info), + snapshot: null + }) clearPendingSnapshotRequest(stream) } clearSnapshot(stream) @@ -997,7 +1062,7 @@ class RemoteRuntimeTerminalMultiplexer { stream.resyncTimer = timer } - private requestSnapshot( + private async requestSnapshot( stream: RemoteRuntimeMultiplexedTerminalState, opts?: { scrollbackRows?: number } ): Promise<{ @@ -1007,16 +1072,34 @@ class RemoteRuntimeTerminalMultiplexer { seq?: number source?: 'headless' | 'renderer' } | null> { - if (this.streams.get(stream.streamId) !== stream || !this.ready || !this.subscription) { - return Promise.resolve(null) + const outcome = await this.requestSnapshotOutcome(stream, opts) + // Why: the concurrent-request guard used to reject before the outcome existed; keep that contract for legacy callers. + if ( + outcome.availability.kind === 'retry-worthy' && + outcome.availability.cause === 'request-already-in-flight' + ) { + throw new Error('Remote terminal snapshot already in flight.') + } + return outcome.snapshot + } + + private requestSnapshotOutcome( + stream: RemoteRuntimeMultiplexedTerminalState, + opts?: { scrollbackRows?: number } + ): Promise { + if (this.streams.get(stream.streamId) !== stream) { + return Promise.resolve(retryWorthySnapshotOutcome('stream-detached')) + } + if (!this.ready || !this.subscription) { + return Promise.resolve(retryWorthySnapshotOutcome('connection-not-ready')) } // Recovery uses an untagged snapshot frame group; callers can retry after // it completes instead of racing another request onto the same frame lane. if (stream.resyncInFlight) { - return Promise.resolve(null) + return Promise.resolve(retryWorthySnapshotOutcome('resync-in-flight')) } if (stream.pendingSnapshotRequest) { - return Promise.reject(new Error('Remote terminal snapshot already in flight.')) + return Promise.resolve(retryWorthySnapshotOutcome('request-already-in-flight')) } const requestId = this.allocateSnapshotRequestId() return new Promise((resolve, reject) => { @@ -1039,7 +1122,7 @@ class RemoteRuntimeTerminalMultiplexer { ) ) { clearPendingSnapshotRequest(stream) - resolve(null) + resolve(retryWorthySnapshotOutcome('request-frame-not-sent')) } }) } @@ -1438,6 +1521,7 @@ function decodeSnapshotInfo( source?: unknown requestId?: unknown truncated?: unknown + unavailable?: unknown pendingEscapeTailAnsi?: unknown }>(payload) if (!raw) { @@ -1450,11 +1534,38 @@ function decodeSnapshotInfo( source: raw.source === 'headless' || raw.source === 'renderer' ? raw.source : undefined, requestId: typeof raw.requestId === 'number' ? raw.requestId : undefined, truncated: raw.truncated === true, + unavailable: parseTerminalSnapshotUnavailableReason(raw.unavailable), pendingEscapeTailAnsi: typeof raw.pendingEscapeTailAnsi === 'string' ? raw.pendingEscapeTailAnsi : undefined } } +function retryWorthySnapshotOutcome( + cause: RemoteRuntimeSnapshotRetryCause +): RemoteRuntimeSnapshotOutcome { + return { availability: { kind: 'retry-worthy', cause }, snapshot: null } +} + +function classifySnapshotAvailability( + clientOverflowed: boolean, + info: RemoteRuntimeSnapshotInfo | null +): RemoteRuntimeSnapshotAvailability { + if (clientOverflowed) { + return { kind: 'permanently-unavailable', reason: 'exceeds-client-replay-limit' } + } + if (info?.unavailable === 'pending-output-overflowed') { + return { kind: 'retry-worthy', cause: 'host-pending-output-overflowed' } + } + if (info?.unavailable === 'no-serializable-buffer') { + return { kind: 'retry-worthy', cause: 'host-no-serializable-buffer' } + } + // Why: a truncated reply with no stated reason can only come from a host that predates `unavailable`. + if (info?.truncated === true) { + return { kind: 'unknown-legacy-host' } + } + return { kind: 'snapshot' } +} + function isTerminalDriverState( value: unknown ): value is { kind: 'idle' } | { kind: 'desktop' } | { kind: 'mobile'; clientId: string } { diff --git a/src/shared/terminal-snapshot-unavailability.ts b/src/shared/terminal-snapshot-unavailability.ts new file mode 100644 index 000000000..6d3ac2677 --- /dev/null +++ b/src/shared/terminal-snapshot-unavailability.ts @@ -0,0 +1,24 @@ +/** + * Why a host answered a requested terminal-buffer snapshot without a usable buffer image. + * + * Sent as the additive `unavailable` field on the SnapshotStart frame. Hosts that predate + * this field omit it, so an absent value means "the host did not say" — never "nothing exists". + * Both current reasons are transient: the host could not answer *now*, not that the pane has + * no retained output. A pane that genuinely has nothing still comes back as a real snapshot + * whose `data` is empty, because the host successfully serialized and found it empty. + */ +export const TERMINAL_SNAPSHOT_UNAVAILABLE_REASONS = [ + // The pending-output buffer overflowed twice while serializing, so the reply was truncated to nothing. + 'pending-output-overflowed', + // No serializer (provider, renderer, headless) produced a buffer for this pty at request time. + 'no-serializable-buffer' +] as const + +export type TerminalSnapshotUnavailableReason = + (typeof TERMINAL_SNAPSHOT_UNAVAILABLE_REASONS)[number] + +export function parseTerminalSnapshotUnavailableReason( + value: unknown +): TerminalSnapshotUnavailableReason | undefined { + return TERMINAL_SNAPSHOT_UNAVAILABLE_REASONS.find((reason) => reason === value) +}