From 4e370062a8991a15e275becc87c209ca4a8892ea Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:51:39 -0700 Subject: [PATCH] fix(remote-runtime): make hidden-output recovery reason-driven instead of timer-guessed (#12655) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a remote terminal tab is hidden, the host stops sending its output and discards what it queued, so on reveal the only way to recover the missed output is to ask the host to serialize its buffer. That reply was ambiguous — one empty answer covered several unrelated situations — so the client inferred "output is lost" from elapsed time, using budgets sized for local IPC. Over a network that guess was routinely wrong: users saw "[Orca skipped hidden terminal output because main recovery was unavailable.]" on a healthy pane and got a permanent scrollback gap, worst exactly when an agent was streaming heavily and there was the most to lose. The key insight is that there is no provable-absence case at all. A pane with genuinely no retained output returns a SUCCESSFUL snapshot with empty data, because the host serialized fine and found nothing. The real defect was the host sending an untagged empty reply when no serializer answered — reporting an unprovable failure as proven emptiness. The host now states why a snapshot is unavailable and the client acts on that reason: an empty snapshot is success; retry-worthy retries and then gives up honestly; permanently-unavailable banners immediately with no waiting; and a host too old to say latches that pane to the pre-existing timer heuristic. Local panes are unchanged. The self-heal repaint no longer yanks the viewport of a user scrolled back reading — it waits for the terminal to return to following output. Retries are bounded by COUNTING REPORTED OUTCOMES, never elapsed time. Independent review found that the single budget also charged attempts for causes returned locally, where the host was never asked — meaning a re-arming resync could exhaust it and banner on a perfectly healthy host, a residual instance of this very bug. Host answers and local gates now have separate budgets; local gates send zero frames, so retrying them cannot pressure the host. Review also found a duplicate-banner path where a repaint timer armed before a permanent answer survived the abandon; the clear is scoped to the branch that banners, since the retry loop deliberately arms that timer. Wire change is additive: an optional field on an existing frame, dropped on the success path, so old clients see an unchanged frame. STA-3476 tracks replacing the legacy-host detection (currently inferred from an absent field) with a positive capability signal. Closes STA-3457. --- src/main/runtime/rpc/methods/terminal.ts | 7 + ...-requested-snapshot-unavailability.test.ts | 233 +++++ .../terminal-pane/pty-connection.test.ts | 41 +- .../terminal-pane/pty-connection.ts | 203 ++++- .../terminal-pane/pty-transport-types.ts | 4 + ...ote-hidden-output-restore-outcomes.test.ts | 858 ++++++++++++++++++ ...t-restore-unavailable-banner.repro.test.ts | 827 +++++++++++++++++ .../remote-runtime-pty-transport.test.ts | 35 + .../remote-runtime-pty-transport.ts | 20 +- .../terminal-follow-output-waiters.ts | 30 + .../pane-manager/terminal-scroll-intent.ts | 19 + .../remote-runtime-snapshot-outcome.test.ts | 301 ++++++ .../remote-runtime-terminal-multiplexer.ts | 157 +++- .../terminal-snapshot-unavailability.ts | 24 + 14 files changed, 2719 insertions(+), 40 deletions(-) create mode 100644 src/main/runtime/rpc/terminal-requested-snapshot-unavailability.test.ts create mode 100644 src/renderer/src/components/terminal-pane/remote-hidden-output-restore-outcomes.test.ts create mode 100644 src/renderer/src/components/terminal-pane/remote-hidden-output-restore-unavailable-banner.repro.test.ts create mode 100644 src/renderer/src/lib/pane-manager/terminal-follow-output-waiters.ts create mode 100644 src/renderer/src/runtime/remote-runtime-snapshot-outcome.test.ts create mode 100644 src/shared/terminal-snapshot-unavailability.ts diff --git a/src/main/runtime/rpc/methods/terminal.ts b/src/main/runtime/rpc/methods/terminal.ts index bd6cad49a..8040ce4e0 100644 --- a/src/main/runtime/rpc/methods/terminal.ts +++ b/src/main/runtime/rpc/methods/terminal.ts @@ -71,6 +71,7 @@ import { sameTerminalOutputSourceIdentity, type TerminalOutputSourceRange } from '../../../../shared/terminal-output-source-range' +import type { TerminalSnapshotUnavailableReason } from '../../../../shared/terminal-snapshot-unavailability' import type { RemoteTerminalSourceRangeReplacementReservation } from '../../remote-terminal-source-range-consumer' import { withTerminalCloseAttribution } from '../terminal-close-attribution' @@ -93,6 +94,8 @@ type SnapshotFrameOptions = { cwd?: string | null truncated?: boolean truncatedByByteBudget?: boolean + // Why: distinguishes "I could not answer right now" from a genuinely empty buffer; omitted on success. + unavailable?: TerminalSnapshotUnavailableReason source?: 'headless' | 'renderer' oscLinks?: TerminalOscLinkRange[] pendingEscapeTailAnsi?: string @@ -628,6 +631,7 @@ function sendSnapshotFrames( requestId: options.requestId, displayMode: options.displayMode, reason: options.reason, + unavailable: options.unavailable, seq: options.seq, cwd: options.cwd, source: options.source, @@ -2279,6 +2283,7 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ displayMode, truncated: true, truncatedByByteBudget: false, + unavailable: 'pending-output-overflowed', data: '' }) return @@ -2298,6 +2303,8 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [ pendingEscapeTailAnsi: serialized?.pendingEscapeTailAnsi, truncated: false, truncatedByByteBudget: serialized?.truncatedByByteBudget, + // Why: no serializer answered, which is not proof the pane is empty — say so instead of passing off '' as the buffer. + unavailable: serialized ? undefined : 'no-serializable-buffer', data: serialized?.data ?? '' }) } catch (error) { diff --git a/src/main/runtime/rpc/terminal-requested-snapshot-unavailability.test.ts b/src/main/runtime/rpc/terminal-requested-snapshot-unavailability.test.ts new file mode 100644 index 000000000..c5f05cb04 --- /dev/null +++ b/src/main/runtime/rpc/terminal-requested-snapshot-unavailability.test.ts @@ -0,0 +1,233 @@ +import { describe, expect, it, vi } from 'vitest' +import { RpcDispatcher } from './dispatcher' +import type { RpcRequest } from './core' +import type { OrcaRuntimeService } from '../orca-runtime' +import { TERMINAL_METHODS } from './methods/terminal' +import type { RuntimeTerminalWait } from '../../../shared/runtime-types' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson, + decodeTerminalStreamText, + encodeTerminalStreamFrame, + encodeTerminalStreamJson +} from '../../../shared/terminal-stream-protocol' + +type SerializedBuffer = { data: string; cols: number; rows: number } | null +type SnapshotStartPayload = Record + +// Why: 256 KiB is the pending-output budget, so this many 1 KiB chunks always trips the overflow guard. +const OVERFLOW_CHUNKS = 400 + +function makeRequest(method: string, params?: unknown): RpcRequest { + return { id: 'req-1', authToken: 'tok', method, params } +} + +/** Drives one desktop multiplex stream up to a requested-snapshot reply and returns its SnapshotStart payload. */ +async function requestSnapshotReply(options: { + connectionId: string + /** Called for each requested-snapshot serialization attempt (attempt 1 is the initial subscribe snapshot). */ + serializeRequested: (attempt: number) => Promise + /** Runs while a requested-snapshot serialization is in flight, e.g. to flood pending output. */ + duringSerialize?: (attempt: number, pushOutput: (data: string) => void) => void +}): Promise<{ start: SnapshotStartPayload; chunks: string }> { + const messages: string[] = [] + const binaryFrames: Uint8Array[] = [] + const handlers = new Map< + number, + (frame: NonNullable>) => void + >() + const cleanups = new Map void>() + const dataListenerRef: { current?: (data: string) => void } = {} + let attempt = 0 + const serializeTerminalBuffer = vi.fn(async () => { + attempt += 1 + if (attempt === 1) { + return { data: 'initial', cols: 120, rows: 40 } + } + const requestedAttempt = attempt - 1 + options.duringSerialize?.(requestedAttempt, (data) => dataListenerRef.current?.(data)) + return options.serializeRequested(requestedAttempt) + }) + const runtime = { + getRuntimeId: () => 'test-runtime', + registerRemoteTerminalViewSubscriber: () => () => {}, + resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }), + requestRendererTerminalTabMount: vi.fn().mockReturnValue(true), + updateRemoteDesktopViewer: vi.fn().mockResolvedValue(true), + unregisterRemoteDesktopViewer: vi.fn().mockResolvedValue(true), + unregisterRemoteDesktopViewers: vi.fn().mockResolvedValue(true), + isPtyResizeDrivenRemotely: vi.fn().mockReturnValue(false), + getRemoteDesktopFitHold: vi.fn().mockReturnValue({ mode: 'desktop-fit', cols: 120, rows: 40 }), + isRemoteDesktopViewerOwner: vi.fn().mockReturnValue(false), + getPtyOutputSequence: vi.fn().mockReturnValue(0), + serializeTerminalBuffer, + serializeAuthoritativeTerminalBuffer: serializeTerminalBuffer, + readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false }), + getTerminalSize: vi.fn().mockReturnValue({ cols: 120, rows: 40 }), + getMobileDisplayMode: vi.fn().mockReturnValue('auto'), + getLayout: vi.fn().mockReturnValue({ seq: 1 }), + subscribeToTerminalData: vi.fn((_: string, listener: (data: string) => void) => { + dataListenerRef.current = listener + return vi.fn() + }), + subscribeToTerminalResize: vi.fn().mockReturnValue(vi.fn()), + subscribeToFitOverrideChanges: vi.fn().mockReturnValue(vi.fn()), + subscribeToDriverChanges: vi.fn().mockReturnValue(vi.fn()), + getTerminalFitOverride: vi.fn().mockReturnValue(null), + getDriver: vi.fn().mockReturnValue({ kind: 'idle' }), + registerSubscriptionCleanup: vi.fn((id: string, cleanup: () => void) => { + cleanups.set(id, cleanup) + }), + cleanupSubscription: vi.fn((id: string) => cleanups.get(id)?.()), + waitForTerminal: vi.fn(() => new Promise(() => {})), + updateDesktopViewport: vi.fn().mockResolvedValue(true) + } as unknown as OrcaRuntimeService + + const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS }) + const dispatchPromise = dispatcher.dispatchStreaming( + makeRequest('terminal.multiplex', {}), + (msg) => messages.push(msg), + { + connectionId: options.connectionId, + sendBinary: (bytes) => { + binaryFrames.push(bytes) + }, + registerBinaryStreamHandler: (streamId, handler) => { + handlers.set(streamId, handler) + return () => handlers.delete(streamId) + } + } + ) + + await vi.waitFor(() => + expect(messages.some((msg) => JSON.parse(msg).result?.type === 'ready')).toBe(true) + ) + handlers.get(0)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Subscribe, + streamId: 0, + seq: 1, + payload: encodeTerminalStreamJson({ + streamId: 12, + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + viewport: { cols: 120, rows: 40 } + }) + }) + )! + ) + await vi.waitFor(() => + expect(messages.some((msg) => JSON.parse(msg).result?.type === 'subscribed')).toBe(true) + ) + const framesBeforeRequest = binaryFrames.length + + handlers.get(12)?.( + decodeTerminalStreamFrame( + encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.SnapshotRequest, + streamId: 12, + seq: 2, + payload: encodeTerminalStreamJson({ requestId: 77, scrollbackRows: 5000 }) + }) + )! + ) + + await vi.waitFor(() => + expect( + binaryFrames + .slice(framesBeforeRequest) + .map((frame) => decodeTerminalStreamFrame(frame)) + .some((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotEnd) + ).toBe(true) + ) + const replyFrames = binaryFrames + .slice(framesBeforeRequest) + .map((frame) => decodeTerminalStreamFrame(frame)) + const start = replyFrames.find((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotStart)! + cleanups.get(`terminal-multiplex:${options.connectionId}`)?.() + await dispatchPromise + return { + start: decodeTerminalStreamJson(start.payload)!, + chunks: replyFrames + .filter((frame) => frame?.opcode === TerminalStreamOpcode.SnapshotChunk) + .map((frame) => (frame ? decodeTerminalStreamText(frame.payload) : '')) + .join('') + } +} + +describe('requested terminal snapshot unavailability reasons', () => { + it('omits a reason when the host serialized a real buffer', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-success', + serializeRequested: async () => ({ data: 'restored output', cols: 120, rows: 40 }) + }) + expect(chunks).toBe('restored output') + expect(start).toMatchObject({ requestId: 77, truncated: false }) + expect(start.unavailable).toBeUndefined() + }) + + it('omits a reason for a proven-empty buffer so absence stays distinguishable from failure', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-empty', + serializeRequested: async () => ({ data: '', cols: 120, rows: 40 }) + }) + expect(chunks).toBe('') + expect(start).toMatchObject({ requestId: 77, truncated: false }) + expect(start.unavailable).toBeUndefined() + }) + + it('reports no-serializable-buffer when no serializer answered', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-null', + serializeRequested: async () => null + }) + expect(chunks).toBe('') + // Legacy fields stay exactly as an old client expects them. + expect(start).toMatchObject({ + requestId: 77, + truncated: false, + cols: 120, + rows: 40, + unavailable: 'no-serializable-buffer' + }) + }) + + it('reports pending-output-overflowed when the retry also overflowed', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-overflow', + serializeRequested: async () => ({ data: 'never delivered', cols: 120, rows: 40 }), + duringSerialize: (_attempt, pushOutput) => { + for (let index = 0; index < OVERFLOW_CHUNKS; index += 1) { + pushOutput(String(index).padStart(3, '0') + 'x'.repeat(1021)) + } + } + }) + expect(chunks).toBe('') + expect(start).toMatchObject({ + requestId: 77, + truncated: true, + truncatedByByteBudget: false, + unavailable: 'pending-output-overflowed' + }) + }) + + it('recovers without a reason when only the first attempt overflowed', async () => { + const { start, chunks } = await requestSnapshotReply({ + connectionId: 'conn-reason-overflow-once', + serializeRequested: async () => ({ data: 'retry snapshot', cols: 120, rows: 40 }), + duringSerialize: (attempt, pushOutput) => { + if (attempt > 1) { + return + } + for (let index = 0; index < OVERFLOW_CHUNKS; index += 1) { + pushOutput(String(index).padStart(3, '0') + 'x'.repeat(1021)) + } + } + }) + expect(chunks).toBe('retry snapshot') + expect(start).toMatchObject({ requestId: 77, truncated: false }) + expect(start.unavailable).toBeUndefined() + }) +}) diff --git a/src/renderer/src/components/terminal-pane/pty-connection.test.ts b/src/renderer/src/components/terminal-pane/pty-connection.test.ts index 1d3936155..7ad97e6b6 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.test.ts @@ -12077,7 +12077,7 @@ describe('connectPanePty', () => { it('abandons the restore on queue overflow, writes the stream through, and repaints once', async () => { const { pane, dataCallback, getMainBufferSnapshot, resolveFirstSnapshot } = await startInFlightRestore() - const { markTerminalPinnedViewport } = + const { markTerminalFollowOutput, markTerminalPinnedViewport } = await import('@/lib/pane-manager/terminal-scroll-intent') const parseCallbacks: (() => void)[] = [] pane.terminal.write.mockImplementation((_data: string, callback?: () => void) => { @@ -12116,6 +12116,9 @@ describe('connectPanePty', () => { expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1) expect(writtenFloodData(pane)).toContain('AFTER-FLOOD') + // The repaint only runs while the viewport follows output; return there first. + markTerminalFollowOutput(pane.terminal) + // After the flood goes quiet: exactly ONE deferred repaint. vi.advanceTimersByTime(2_100) await flushAsyncTicks(20) @@ -12128,6 +12131,42 @@ describe('connectPanePty', () => { } }) + it('holds the post-flood repaint while the user reads scrollback and runs it on return to the bottom', async () => { + const { pane, dataCallback, getMainBufferSnapshot, resolveFirstSnapshot } = + await startInFlightRestore() + const { markTerminalFollowOutput, markTerminalPinnedViewport } = + await import('@/lib/pane-manager/terminal-scroll-intent') + pane.terminal.buffer.active.viewportY = 42 + pane.terminal.buffer.active.baseY = 100 + markTerminalPinnedViewport(pane.terminal) + + dataCallback('f'.repeat(300 * 1024), { seq: 300 * 1024 + 64, rawLength: 300 * 1024 }) + dataCallback('g'.repeat(300 * 1024), { seq: 600 * 1024 + 64, rawLength: 300 * 1024 }) + + try { + vi.useFakeTimers() + resolveFirstSnapshot({ data: 'flood snapshot\r\n', cols: 100, rows: 30, seq: 64 }) + await flushAsyncTicks(20) + expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1) + + // Quiet flood, but the user is still scrolled back: the clear-and-replay repaint must not move their viewport. + vi.advanceTimersByTime(2_100) + await flushAsyncTicks(20) + expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1) + vi.advanceTimersByTime(60_000) + await flushAsyncTicks(20) + expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1) + + // Returning to the bottom is the event that releases it — the heal is deferred, never dropped. + pane.terminal.buffer.active.viewportY = pane.terminal.buffer.active.baseY + markTerminalFollowOutput(pane.terminal) + await flushAsyncTicks(20) + expect(getMainBufferSnapshot).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + } + }) + it('sends salvaged queries immediately from an overflowing restore queue', async () => { const { pane, transport, dataCallback } = await startInFlightRestore() diff --git a/src/renderer/src/components/terminal-pane/pty-connection.ts b/src/renderer/src/components/terminal-pane/pty-connection.ts index fa7ce5737..3075a7217 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.ts @@ -148,6 +148,7 @@ import { inspectRuntimeTerminalProcess } from '@/runtime/runtime-terminal-inspec // actually attached — nothing is inspectable while the session hydrates. import { notifyCodexPaneBoundForStaleSweep } from '@/lib/codex-stale-pane-sweep' import { getRemoteRuntimePtyEnvironmentId } from '@/runtime/runtime-terminal-stream' +import { isHostAnsweredSnapshotRetryCause } from '@/runtime/remote-runtime-terminal-multiplexer' import { discardTerminalOutput, flushTerminalOutput, @@ -166,7 +167,8 @@ import { clearTerminalScrollbackAndFollowOutput } from '@/lib/pane-manager/termi import { enforceTerminalCurrentScrollIntent, getTerminalScrollIntentKind, - markTerminalFollowOutput + markTerminalFollowOutput, + onTerminalScrollIntentFollowOutput } from '@/lib/pane-manager/terminal-scroll-intent' import { cancelTerminalScrollIntentBufferRebuildCompletions, @@ -358,6 +360,23 @@ const HIDDEN_OUTPUT_RESTORE_FLOOD_SUPPRESS_MS = 2000 // (fresh-snapshot marks, unmappable slices) only this many times before it // abandons and lets live bytes flow. const HIDDEN_OUTPUT_RESTORE_MAX_LOOP_ITERATIONS = 3 +// Why: remote-runtime PTYs have no local main fallback — the host transport is +// the only recovery and legitimately answers null while it resyncs, trims a +// flooded snapshot, or waits out link RTT. Those nulls are not proof of loss, +// so an abandoned restore re-arms one quiet post-suppression repaint instead of +// claiming the bytes are gone. Five cycles (~2.15s each) outlast both the +// multiplexer resync window and the remote snapshot request timeout (10s); +// past that the host really is unreachable and the loss banner is honest. +const HIDDEN_OUTPUT_RESTORE_REMOTE_REARM_MAX = 5 +// Why: the host declined seven separate times; each one cost it a real serialize attempt, so stop asking. +const HIDDEN_OUTPUT_RESTORE_REMOTE_OUTCOME_MAX_ATTEMPTS = 7 +// Why separate and larger: these causes are decided before any frame leaves the +// client (resync gate, occupied request lane, detached stream), so the host +// declined nothing and charging them to its budget would banner a healthy pane — +// the exact elapsed-time guess this change removes. They cost zero host traffic, +// so the only job of this cap is termination. At the ~2s post-abandon re-arm +// cadence, 30 outlasts several full 10s resync watchdog cycles. +const HIDDEN_OUTPUT_RESTORE_LOCAL_GATE_MAX_ATTEMPTS = 30 const TERMINAL_RENDERER_RISK_SCAN_TAIL_CHARS = 256 const SYNCHRONIZED_OUTPUT_START_SEQUENCE = '\x1b[?2026h' const SYNCHRONIZED_OUTPUT_END_SEQUENCE = '\x1b[?2026l' @@ -5848,6 +5867,11 @@ export function connectPanePty( let hiddenOutputRestoreDeferredRetryTimer: ReturnType | null = null let hiddenOutputRestoreForegroundDeadlineTimer: ReturnType | null = null let hiddenOutputRestoreDeferredRetryAttempts = 0 + let hiddenOutputRestoreRemoteOutcomeAttempts = 0 + let hiddenOutputRestoreLocalGateAttempts = 0 + let hiddenOutputRestoreLegacyPtyId: string | null = null + // Bounded remote re-arms spent instead of the loss banner (per PTY stream). + let hiddenOutputRestoreRemoteAbandonCycles = 0 let hiddenOutputSnapshotScrollRestore: { ptyId: string | null generation: number @@ -5870,6 +5894,7 @@ export function connectPanePty( pendingDeliveryStartSeq?: number } | null = null let hiddenOutputRestoreFloodRepaintTimer: ReturnType | null = null + let cancelHiddenOutputRestoreFloodRepaintPark: (() => void) | null = null // Why: after a snapshot restore, main can still drain ACK-backlog chunks // whose bytes the snapshot already covers — writing them unguarded // duplicates visible output. Track the restored baseline seq (per PTY) @@ -5950,21 +5975,59 @@ export function connectPanePty( return transport.getPtyId() === ptyId && typeof transport.serializeBuffer === 'function' } + type HiddenOutputSnapshotResult = + | { kind: 'snapshot'; snapshot: PtyBufferSnapshot } + // `source` picks the budget: only 'host' answers cost the host a serialize attempt. + | { kind: 'retry-worthy'; source: 'host' | 'local' } + | { kind: 'permanently-unavailable' } + | { kind: 'unknown-legacy-host' } + | { kind: 'unavailable' } + async function serializeHiddenOutputSnapshot( ptyId: string, opts: { scrollbackRows?: number } - ): Promise { + ): Promise { const e2eSnapshot = readE2eHiddenSnapshotOverride(ptyId) if (e2eSnapshot) { - return e2eSnapshot + const snapshot = await e2eSnapshot + return snapshot ? { kind: 'snapshot', snapshot } : { kind: 'unavailable' } } if (canUseMainBufferSnapshot(ptyId)) { - return window.api.pty.getMainBufferSnapshot(ptyId, opts) + const snapshot = await window.api.pty.getMainBufferSnapshot(ptyId, opts) + return snapshot ? { kind: 'snapshot', snapshot } : { kind: 'unavailable' } } if (transport.getPtyId() !== ptyId || typeof transport.serializeBuffer !== 'function') { - return null + return { kind: 'unavailable' } + } + if ( + hiddenOutputRestoreLegacyPtyId === ptyId || + typeof transport.serializeBufferOutcome !== 'function' + ) { + const snapshot = await transport.serializeBuffer(opts) + return snapshot ? { kind: 'snapshot', snapshot } : { kind: 'unknown-legacy-host' } + } + try { + const outcome = await transport.serializeBufferOutcome(opts) + if (outcome.availability.kind === 'snapshot') { + // A success frame with no image is still the host's own answer to a request it received. + return outcome.snapshot + ? { kind: 'snapshot', snapshot: outcome.snapshot } + : { kind: 'retry-worthy', source: 'host' } + } + if (outcome.availability.kind === 'retry-worthy') { + return { + kind: 'retry-worthy', + source: isHostAnsweredSnapshotRetryCause(outcome.availability.cause) ? 'host' : 'local' + } + } + if (outcome.availability.kind === 'permanently-unavailable') { + return { kind: 'permanently-unavailable' } + } + return { kind: 'unknown-legacy-host' } + } catch { + // Why 'host': the reject path is the request timeout — the frame went out and the host stayed silent. + return { kind: 'retry-worthy', source: 'host' } } - return transport.serializeBuffer(opts) } // Why: hidden/parked panes used to mark hidden only at the first @@ -6009,6 +6072,8 @@ export function connectPanePty( } function clearHiddenOutputRestoreFloodRepaintTimer(): void { + cancelHiddenOutputRestoreFloodRepaintPark?.() + cancelHiddenOutputRestoreFloodRepaintPark = null if (hiddenOutputRestoreFloodRepaintTimer === null) { return } @@ -6017,6 +6082,24 @@ export function connectPanePty( } cleanupHiddenOutputRestoreFloodRepaint = clearHiddenOutputRestoreFloodRepaintTimer + // Why: the repaint discards the buffer and replays a full snapshot, which repositions the viewport; a user reading scrollback must not be yanked to the bottom, so hold it until their own scroll intent returns to follow-output. + function repaintAfterFloodWhenFollowingOutput(ptyId: string): void { + cancelHiddenOutputRestoreFloodRepaintPark?.() + cancelHiddenOutputRestoreFloodRepaintPark = null + let repainted = false + const cancelPark = onTerminalScrollIntentFollowOutput(pane.terminal, () => { + repainted = true + cancelHiddenOutputRestoreFloodRepaintPark = null + if (disposed || transport.getPtyId() !== ptyId) { + return + } + markHiddenOutputRestoreNeeded() + }) + if (!repainted) { + cancelHiddenOutputRestoreFloodRepaintPark = cancelPark + } + } + function resetHiddenOutputRestoreFloodSuppression(): void { hiddenOutputRestoreFloodSuppressedUntil = 0 clearHiddenOutputRestoreFloodRepaintTimer() @@ -6036,7 +6119,7 @@ export function connectPanePty( return } // Why one repaint: flood-dropped bytes leave a gap the live stream can't heal; once quiet, one snapshot restore repaints from main's authoritative buffer. - markHiddenOutputRestoreNeeded() + repaintAfterFloodWhenFollowingOutput(ptyId) }, HIDDEN_OUTPUT_RESTORE_FLOOD_SUPPRESS_MS) } @@ -6901,6 +6984,9 @@ export function connectPanePty( disposed || hiddenOutputRestoreForegroundDeadlineTimer !== null || !shouldWritePtyOutputForeground(deps.isVisibleRef.current) || + (isRemoteRuntimePtyId(hiddenOutputRestorePtyId) && + hiddenOutputRestoreLegacyPtyId !== hiddenOutputRestorePtyId && + typeof transport.serializeBufferOutcome === 'function') || (hiddenOutputRestorePendingChunks.length === 0 && !hiddenOutputRestorePendingOverflow) ) { return @@ -6925,14 +7011,42 @@ export function connectPanePty( }, HIDDEN_OUTPUT_RESTORE_FOREGROUND_TIMEOUT_MS) } + // Trades the loss banner for one bounded post-suppression repaint from the + // host's authoritative buffer. Ordered before the state reset so the repaint + // timer arms against the live ptyId (mirrors the flood-abandon call sites). + function rearmRemoteHiddenOutputRestoreInsteadOfWarning( + ptyId: string, + reason: string + ): boolean { + if ( + !isRemoteRuntimePtyId(ptyId) || + hiddenOutputRestoreRemoteAbandonCycles >= HIDDEN_OUTPUT_RESTORE_REMOTE_REARM_MAX + ) { + return false + } + hiddenOutputRestoreRemoteAbandonCycles += 1 + recordTerminalFreezeBreadcrumb('restore-abandon-rearm', { + id: redactPtyIdForDiagnostics(ptyId), + reason, + cycle: hiddenOutputRestoreRemoteAbandonCycles + }) + noteHiddenOutputRestoreFloodBackpressure() + return true + } + function abandonHiddenOutputRestoreAndDrainPendingForeground( expectedPtyId: string, - opts: { quiet?: boolean } = {} + opts: { quiet?: boolean; rearmRemote?: boolean } = {} ): void { if (transport.getPtyId() !== expectedPtyId || hiddenOutputRestorePtyId !== expectedPtyId) { resetHiddenOutputRestoreIfPtyChanged() return } + const rearmedRemoteRestore = + opts.rearmRemote !== false && + !opts.quiet && + canUseHiddenOutputSnapshot(expectedPtyId) && + rearmRemoteHiddenOutputRestoreInsteadOfWarning(expectedPtyId, 'abandon-deadline') const pendingChunks = hiddenOutputRestorePendingOverflow ? [] : hiddenOutputRestorePendingChunks.slice() @@ -6965,7 +7079,10 @@ export function connectPanePty( hiddenOutputRestoreDeferredRetryAttempts = 0 // Why quiet: flood cuts abandon deliberately and repaint post-flood, so the "restore unavailable" warning would be noise the repaint wipes. - if (!opts.quiet) { + if (!opts.quiet && !rearmedRemoteRestore) { + // Why: this abandon declares the bytes unrecoverable, so a repaint armed by earlier live + // backpressure must not outlive it — it would re-open recovery and banner a second time. + clearHiddenOutputRestoreFloodRepaintTimer() writeRestoreUnavailableWarning() } if (hadPendingOverflow) { @@ -7024,6 +7141,10 @@ export function connectPanePty( function clearHiddenOutputRestoreState(): void { cancelSnapshotScrollRestore() clearPendingLiveChunksDuringRestore() + // Re-arm budget is per PTY stream, like the rest of this state. + hiddenOutputRestoreRemoteAbandonCycles = 0 + hiddenOutputRestoreRemoteOutcomeAttempts = 0 + hiddenOutputRestoreLocalGateAttempts = 0 hiddenStartupRendererQueryPending = '' hiddenRendererStateDirty = false resetHiddenRendererRiskState() @@ -7335,7 +7456,13 @@ export function connectPanePty( if (hiddenOutputRestorePtyId === currentPtyId) { clearHiddenOutputRestoreState() } - writeRestoreUnavailableWarning() + // Remote-only path: the transport swapped PTYs mid-restore, which is a + // stream change, not proof the hidden bytes are unrecoverable. + if ( + !rearmRemoteHiddenOutputRestoreInsteadOfWarning(currentPtyId, 'restore-pty-swapped') + ) { + writeRestoreUnavailableWarning() + } return } if (transport.getPtyId() !== currentPtyId) { @@ -7346,13 +7473,19 @@ export function connectPanePty( } const restoreGeneration = hiddenOutputRestoreGeneration hiddenOutputRestoreNeeded = false - let snapshot: PtyBufferSnapshot | null = null + let snapshotResult: HiddenOutputSnapshotResult try { - snapshot = await serializeHiddenOutputSnapshot(currentPtyId, { + snapshotResult = await serializeHiddenOutputSnapshot(currentPtyId, { scrollbackRows: resolveHiddenRestoreScrollbackRows(pane.terminal.options.scrollback) }) } catch { - snapshot = null + snapshotResult = + !isRemoteRuntimePtyId(currentPtyId) || + hiddenOutputRestoreLegacyPtyId === currentPtyId || + typeof transport.serializeBufferOutcome !== 'function' + ? { kind: 'unavailable' } + : // Why 'host': the only reject here is the request timeout — the frame went out and the host stayed silent. + { kind: 'retry-worthy', source: 'host' } } if (disposed) { return @@ -7367,14 +7500,53 @@ export function connectPanePty( } return } - if (!snapshot) { + if (snapshotResult.kind === 'retry-worthy') { + let budgetExhausted: boolean + if (snapshotResult.source === 'host') { + hiddenOutputRestoreRemoteOutcomeAttempts += 1 + budgetExhausted = + hiddenOutputRestoreRemoteOutcomeAttempts >= + HIDDEN_OUTPUT_RESTORE_REMOTE_OUTCOME_MAX_ATTEMPTS + } else { + hiddenOutputRestoreLocalGateAttempts += 1 + budgetExhausted = + hiddenOutputRestoreLocalGateAttempts >= + HIDDEN_OUTPUT_RESTORE_LOCAL_GATE_MAX_ATTEMPTS + } + if (budgetExhausted) { + abandonHiddenOutputRestoreAndDrainPendingForeground(currentPtyId, { + rearmRemote: false + }) + return + } + hiddenOutputRestoreNeeded = true + hiddenOutputRestoreFreshSnapshotNeeded = false + noteHiddenOutputRestoreFloodBackpressure() + abandonHiddenOutputRestoreAndDrainPendingForeground(currentPtyId, { quiet: true }) + return + } + if (snapshotResult.kind === 'permanently-unavailable') { + abandonHiddenOutputRestoreAndDrainPendingForeground(currentPtyId, { + rearmRemote: false + }) + return + } + if (snapshotResult.kind === 'unknown-legacy-host') { + hiddenOutputRestoreLegacyPtyId = currentPtyId + armHiddenOutputRestoreForegroundDeadline() + } + if (snapshotResult.kind !== 'snapshot') { hiddenOutputRestoreNeeded = true hiddenOutputRestoreFreshSnapshotNeeded = false hiddenOutputRestoreRetryDeferred = true scheduleHiddenOutputRestoreDeferredRetry() return } + const snapshot = snapshotResult.snapshot hiddenOutputRestoreDeferredRetryAttempts = 0 + hiddenOutputRestoreRemoteAbandonCycles = 0 + hiddenOutputRestoreRemoteOutcomeAttempts = 0 + hiddenOutputRestoreLocalGateAttempts = 0 restoreIterations += 1 await applyMainBufferSnapshot(snapshot) if ( @@ -8025,9 +8197,10 @@ export function connectPanePty( prefetchedParkModelSnapshot = await fetchSshMainModelReattachSnapshot() } else { try { - prefetchedParkModelSnapshot = await serializeHiddenOutputSnapshot(ptyId, { + const result = await serializeHiddenOutputSnapshot(ptyId, { scrollbackRows: resolveHiddenRestoreScrollbackRows(pane.terminal.options.scrollback) }) + prefetchedParkModelSnapshot = result.kind === 'snapshot' ? result.snapshot : null } catch { prefetchedParkModelSnapshot = null } diff --git a/src/renderer/src/components/terminal-pane/pty-transport-types.ts b/src/renderer/src/components/terminal-pane/pty-transport-types.ts index c64b93934..3833468ed 100644 --- a/src/renderer/src/components/terminal-pane/pty-transport-types.ts +++ b/src/renderer/src/components/terminal-pane/pty-transport-types.ts @@ -14,6 +14,7 @@ import type { TerminalOscColorQueryReplyColors } from '../../../../shared/termin import type { TuiAgent } from '../../../../shared/types' import type { ExecutionHostId } from '../../../../shared/execution-host' import type { PtyDataMeta } from './pty-dispatcher' +import type { RemoteRuntimeSnapshotOutcome } from '../../runtime/remote-runtime-terminal-multiplexer' export type PtyBufferSnapshot = { data: string @@ -178,6 +179,9 @@ export type PtyTransport = { * would corrupt the next live chunk. IPC transports only. */ resetCrossChunkParserState?: () => void serializeBuffer?: (opts?: { scrollbackRows?: number }) => Promise + serializeBufferOutcome?: (opts?: { + scrollbackRows?: number + }) => Promise preserve?: () => void detach?: (options?: { preserveExitObserver?: boolean }) => void destroy?: () => void | Promise diff --git a/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-outcomes.test.ts b/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-outcomes.test.ts new file mode 100644 index 000000000..47bc5af02 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-outcomes.test.ts @@ -0,0 +1,858 @@ +import type * as React from 'react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { resetAgentStartupDelayedDeliveryForTests } from '@/lib/agent-startup-delayed-delivery' + +async function flushAsyncTicks(count = 6): Promise { + for (let i = 0; i < count; i++) { + await Promise.resolve() + } +} + +async function drainFakeTimerWork(limit = 20): Promise { + await flushAsyncTicks(20) + if (!vi.isFakeTimers()) { + return + } + for (let iteration = 0; iteration < limit && vi.getTimerCount() > 0; iteration += 1) { + await vi.runOnlyPendingTimersAsync() + await flushAsyncTicks(20) + } + vi.clearAllTimers() + await flushAsyncTicks(20) + vi.clearAllTimers() +} + +const LEAF_1 = '11111111-1111-4111-8111-111111111111' as const +const LEAF_2 = '22222222-2222-4222-8222-222222222222' as const + +function leafIdForPane(paneId: number): string { + return paneId === 2 ? LEAF_2 : LEAF_1 +} + +type ConnectCallbacks = { + onReattachDetermined?: () => void + onConnect?: () => void + onData?: ( + data: string, + meta?: { seq?: number; rawLength?: number; background?: boolean; droppedOutput?: boolean } + ) => void + onReplayData?: (data: string, meta?: { clearBeforeReplay?: boolean }) => void + onError?: (msg: string) => void + onWriteUnavailable?: () => void + onOutputPauseChanged?: (paused: boolean, supported: boolean) => void +} + +type MockTransport = { + attach: ReturnType + connect: ReturnType & { + mockImplementation: ( + impl: (opts: { callbacks?: ConnectCallbacks } & Record) => Promise + ) => unknown + } + disconnect: ReturnType + sendInput: ReturnType + sendInputImmediate?: ReturnType + sendInputAccepted?: ReturnType + claimViewport: ReturnType + resize: ReturnType + getPtyId: ReturnType + getConnectionId: ReturnType + serializeBuffer?: ReturnType + serializeBufferOutcome?: ReturnType +} + +const scheduleRuntimeGraphSync = vi.fn() +const shouldSeedCacheTimerOnInitialTitle = vi.fn(() => false) +const scheduleTerminalWebglAtlasRecovery = vi.fn() +const toastInfo = vi.fn() +const notifyCodexPaneBoundForStaleSweep = vi.fn() + +let mockStoreState: Record +let transportFactoryQueue: MockTransport[] = [] +let createdTransportOptions: Record[] = [] +let storeSubscribers: ((state: Record) => void)[] = [] + +vi.mock('@/runtime/sync-runtime-graph', () => ({ + scheduleRuntimeGraphSync +})) + +vi.mock('@/store', () => ({ + useAppStore: { + getState: () => mockStoreState, + subscribe: (listener: (state: Record) => void) => { + storeSubscribers.push(listener) + return () => { + storeSubscribers = storeSubscribers.filter((candidate) => candidate !== listener) + } + } + } +})) + +vi.mock('./terminal-webgl-atlas-recovery', () => ({ + scheduleTerminalWebglAtlasRecovery +})) + +vi.mock('@/lib/agent-status', async (importOriginal) => { + const actual = await importOriginal>() + const isGeminiTerminalTitle = actual.isGeminiTerminalTitle as (title: string) => boolean + return { + ...actual, + isGeminiTerminalTitle: vi.fn((title: string) => isGeminiTerminalTitle(title)), + isClaudeAgent: vi.fn(() => false), + detectAgentStatusFromTitle: vi.fn(() => null) + } +}) + +vi.mock('./cache-timer-seeding', () => ({ + shouldSeedCacheTimerOnInitialTitle +})) + +vi.mock('sonner', () => ({ + toast: { + info: toastInfo + } +})) + +vi.mock('@/lib/codex-stale-pane-sweep', () => ({ + notifyCodexPaneBoundForStaleSweep +})) + +vi.mock('react', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + useCallback: unknown>(fn: T): T => fn + } +}) + +vi.mock('./pty-transport', () => ({ + createIpcPtyTransport: vi.fn((options: Record) => { + createdTransportOptions.push(options) + const nextTransport = transportFactoryQueue.shift() + if (!nextTransport) { + throw new Error('No mock transport queued') + } + return nextTransport + }) +})) + +vi.mock('./remote-runtime-pty-transport', () => ({ + createRemoteRuntimePtyTransport: vi.fn( + (_environmentId: string, options: Record) => { + createdTransportOptions.push(options) + const nextTransport = transportFactoryQueue.shift() + if (!nextTransport) { + throw new Error('No mock transport queued') + } + return nextTransport + } + ) +})) + +vi.mock('./pty-dispatcher', async (importOriginal) => { + const actual = await importOriginal>() + return { + ...actual, + getEagerPtyBufferHandle: vi.fn(() => undefined) + } +}) + +function createMockTransport(initialPtyId: string | null = null): MockTransport { + let ptyId = initialPtyId + const transport = { + attach: vi.fn(({ existingPtyId }: { existingPtyId: string }) => { + ptyId = existingPtyId + }), + connect: vi.fn().mockImplementation(async (opts: { sessionId?: string }) => { + if (opts.sessionId) { + ptyId = opts.sessionId + return { id: opts.sessionId } + } + return ptyId + }), + disconnect: vi.fn(() => { + ptyId = null + }), + sendInput: vi.fn(() => true), + claimViewport: vi.fn(() => true), + resize: vi.fn(() => true), + getPtyId: vi.fn(() => ptyId), + getConnectionId: vi.fn(() => null), + serializeBuffer: undefined + } as MockTransport + const sendInput = transport.sendInput as unknown as (data: string) => boolean + transport.sendInputImmediate = vi.fn((data: string) => sendInput(data)) + transport.sendInputAccepted = vi.fn(async (data: string) => sendInput(data)) + return transport +} + +function createPaneContainer(): HTMLElement { + const container = new EventTarget() as HTMLElement + Object.defineProperty(container, 'dataset', { + configurable: true, + value: {} + }) + return container +} + +function createPane(paneId: number) { + const leafId = leafIdForPane(paneId) + const activeBuffer = { + type: 'normal' as const, + viewportY: 0, + baseY: 0, + cursorY: 0, + cursorX: 0 + } + const terminal = { + cols: 120, + rows: 40, + element: {}, + buffer: { + active: activeBuffer + }, + modes: { + bracketedPasteMode: false, + sendFocusMode: false + }, + options: { + scrollback: 5_000, + ignoreBracketedPasteMode: false, + theme: { + foreground: '#eeeeee', + background: '#111111' + } + }, + write: vi.fn<(data: string, callback?: () => void) => void>(function write(...args): void { + const [data, callback] = args + if (data === '' || callback?.name === 'runParsedSteps') { + callback?.() + } + }), + resize: vi.fn(), + clear: vi.fn(), + scrollToBottom: vi.fn(() => { + activeBuffer.viewportY = activeBuffer.baseY + }), + scrollToLine: vi.fn((line: number) => { + activeBuffer.viewportY = line + }), + scrollLines: vi.fn((amount: number) => { + activeBuffer.viewportY = Math.max( + 0, + Math.min(activeBuffer.baseY, activeBuffer.viewportY + amount) + ) + }), + paste: vi.fn(), + onData: vi.fn(() => ({ dispose: vi.fn() })), + onResize: vi.fn(() => ({ dispose: vi.fn() })), + onRender: vi.fn((_listener: () => void) => ({ dispose: vi.fn() })), + onTitleChange: vi.fn(() => ({ dispose: vi.fn() })), + hasSelection: vi.fn(() => false), + parser: { + registerCsiHandler: vi.fn(() => ({ dispose: vi.fn() })), + registerOscHandler: vi.fn(() => ({ dispose: vi.fn() })) + } + } + return { + id: paneId, + leafId, + stablePaneId: leafId, + terminal, + container: createPaneContainer(), + fitAddon: { + fit: vi.fn(), + proposeDimensions: vi.fn(() => ({ cols: terminal.cols, rows: terminal.rows })) + } + } +} + +function createManager(paneCount = 1, initialActivePaneId: number | null = null) { + let activePaneId = initialActivePaneId + const panes = Array.from({ length: paneCount }, (_, index) => ({ + id: index + 1, + leafId: leafIdForPane(index + 1) + })) + return { + setPaneGpuRendering: vi.fn(), + markPaneHasComplexScriptOutput: vi.fn(), + rebuildPaneWebgl: vi.fn(), + hasWebglRenderer: vi.fn(() => false), + getPanes: vi.fn(() => panes), + closePane: vi.fn(), + getActivePane: vi.fn<() => { id: number; leafId?: string } | null>(() => + activePaneId === null + ? null + : (panes.find((candidate) => candidate.id === activePaneId) ?? null) + ), + getNumericIdForLeaf: vi.fn((leafId: string) => { + return panes.find((candidate) => candidate.leafId === leafId)?.id ?? null + }), + setActivePane: vi.fn((paneId: number) => { + activePaneId = paneId + }) + } +} + +function createDeps(overrides: Record = {}) { + return { + tabId: 'tab-1', + worktreeId: 'wt-1', + cwd: '/tmp/wt-1', + startup: null, + restoredLeafId: null, + restoredPtyIdByLeafId: {}, + paneTransportsRef: { current: new Map() }, + paneMode2031Ref: { current: new Map() }, + paneKittyKeyboardModesRef: { current: new Map() }, + paneLastThemeModeRef: { current: new Map() }, + replayingPanesRef: { current: new Map() }, + isActiveRef: { current: true }, + isVisibleRef: { current: true }, + onPtyExitRef: { current: vi.fn() }, + onAgentExitedRef: { current: vi.fn() }, + onPtyErrorRef: { current: vi.fn() }, + clearTabPtyId: vi.fn(), + consumeSuppressedPtyExit: vi.fn(() => false), + isPtyShutdownPending: vi.fn(() => false), + updateTabTitle: vi.fn(), + setRuntimePaneTitle: vi.fn(), + clearRuntimePaneTitle: vi.fn(), + updateTabPtyId: vi.fn((tabId: string, ptyId: string, replacedPtyId?: string) => { + const byTab = (mockStoreState.ptyIdsByTabId ?? {}) as Record + const current = byTab[tabId] ?? [] + const next = + replacedPtyId && current.includes(replacedPtyId) + ? current.map((candidate) => (candidate === replacedPtyId ? ptyId : candidate)) + : current.includes(ptyId) + ? current + : [...current, ptyId] + mockStoreState.ptyIdsByTabId = { ...byTab, [tabId]: next } + }), + markWorktreeUnread: vi.fn(), + markTerminalTabUnread: vi.fn(), + markTerminalPaneUnread: vi.fn(), + clearWorktreeUnread: vi.fn(), + clearTerminalTabUnread: vi.fn(), + clearTerminalPaneUnread: vi.fn(), + dispatchNotification: vi.fn(), + onShowSessionRestoredBanner: vi.fn(), + setCacheTimerStartedAt: vi.fn(), + syncPanePtyLayoutBinding: vi.fn(), + clearExitedPanePtyLayoutBinding: vi.fn(), + ...overrides + } +} + +function createDeferred(): { + promise: Promise + resolve: (value: T) => void + reject: (reason?: unknown) => void +} { + let resolveDeferred!: (value: T) => void + let rejectDeferred!: (reason?: unknown) => void + const promise = new Promise((resolve, reject) => { + resolveDeferred = resolve + rejectDeferred = reject + }) + return { promise, resolve: resolveDeferred, reject: rejectDeferred } +} + +// ── Scenario identities ───────────────────────────────────────────────────── +const REMOTE_PTY_ID = 'remote:env-1@@terminal-agent-1' +// Overflows the renderer's hidden background queue (2MB lossy cap), dropping the +// backlog and latching model restore — mirrors the real remote pause semantics +// where hidden-time bytes exist ONLY in the host's authoritative buffer. +const HIDDEN_BYTES = 'x'.repeat(2 * 1024 * 1024 + 1) +const LIVE_AGENT_CHUNK = 'LIVE_AGENT_CHUNK\r\n' +const HOST_SNAPSHOT_MARKER = 'HOST_SNAPSHOT_HIDDEN_AGENT_CONTENT' +const HOST_SNAPSHOT = { + data: `${HOST_SNAPSHOT_MARKER}\r\n`, + cols: 120, + rows: 40, + seq: HIDDEN_BYTES.length + LIVE_AGENT_CHUNK.length, + source: 'headless' +} +const BANNER_FRAGMENT = 'main recovery was unavailable' + +type HostSnapshot = typeof HOST_SNAPSHOT + +type RemotePaneDrive = { + transport: MockTransport + pane: ReturnType + deps: ReturnType + disposable: { dispose: () => void } + deliver: (data: string, seq: number) => void + setOutputPaused: (paused: boolean) => void + writtenChunks: () => string[] +} + +async function connectHiddenRemoteAgentPane( + serializeBuffer: ReturnType, + serializeBufferOutcome?: ReturnType +): Promise { + const { connectPanePty } = await import('./pty-connection') + const transport = createMockTransport(REMOTE_PTY_ID) + transport.serializeBuffer = serializeBuffer + transport.serializeBufferOutcome = serializeBufferOutcome + const capturedDataCallback: { + current: ((data: string, meta?: { seq?: number; rawLength?: number }) => void) | null + } = { current: null } + const capturedOutputPauseCallback: { + current: ((paused: boolean, supported: boolean) => void) | null + } = { current: null } + transport.connect.mockImplementation(async ({ callbacks }: { callbacks?: ConnectCallbacks }) => { + capturedDataCallback.current = callbacks?.onData ?? null + capturedOutputPauseCallback.current = callbacks?.onOutputPauseChanged ?? null + return REMOTE_PTY_ID + }) + transportFactoryQueue.push(transport) + const pane = createPane(1) + const manager = createManager(1) + const deps = createDeps({ isVisibleRef: { current: false } }) + const disposable = connectPanePty(pane as never, manager as never, deps as never) + await flushAsyncTicks(6) + expect(capturedDataCallback.current).not.toBeNull() + return { + transport, + pane, + deps, + disposable, + deliver: (data, seq) => capturedDataCallback.current?.(data, { seq, rawLength: data.length }), + setOutputPaused: (paused) => capturedOutputPauseCallback.current?.(paused, true), + writtenChunks: () => pane.terminal.write.mock.calls.map(([data]) => String(data)) + } +} + +// Delivers the hidden backlog, reveals the pane, and delivers one live agent +// chunk — the exact user flow in the screenshot (agent streaming on reveal). +function driveHiddenBacklogThenReveal(drive: RemotePaneDrive): void { + drive.deliver(HIDDEN_BYTES, HIDDEN_BYTES.length) + ;(drive.deps.isVisibleRef as { current: boolean }).current = true + drive.deliver(LIVE_AGENT_CHUNK, HIDDEN_BYTES.length + LIVE_AGENT_CHUNK.length) +} + +async function advanceThroughNullRetryBudget(drive: RemotePaneDrive): Promise { + for (let retry = 0; retry < 3; retry++) { + // Live output stays gated behind the in-flight restore during the window. + expect(drive.pane.terminal.write).not.toHaveBeenCalledWith( + LIVE_AGENT_CHUNK, + expect.any(Function) + ) + vi.advanceTimersByTime(50) + vi.advanceTimersByTime(0) + await flushAsyncTicks(10) + } +} + +// Generous self-heal window: lets any corrected pipeline (extended retries, +// restore re-arm, post-abandon repaint) run to completion under fake timers. +async function advanceModernRetryProbe(): Promise { + await vi.advanceTimersByTimeAsync(2_000) + await flushAsyncTicks(20) +} + +describe('remote hidden-output restore outcomes', () => { + const originalRequestAnimationFrame = globalThis.requestAnimationFrame + const originalCancelAnimationFrame = globalThis.cancelAnimationFrame + const originalDocument = globalThis.document + + beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + transportFactoryQueue = [] + createdTransportOptions = [] + storeSubscribers = [] + mockStoreState = { + activeWorktreeId: 'wt-1', + tabsByWorktree: { + 'wt-1': [{ id: 'tab-1', ptyId: 'tab-pty' }] + }, + ptyIdsByTabId: { + 'tab-1': ['tab-pty'] + }, + terminalLayoutsByTabId: { + 'tab-1': { + root: { type: 'leaf', leafId: LEAF_1 }, + activeLeafId: LEAF_1, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF_1]: 'tab-pty' } + } + }, + unreadTerminalTabs: {}, + deleteStateByWorktreeId: {}, + worktreesByRepo: { + repo1: [{ id: 'wt-1', repoId: 'repo1', path: '/tmp/wt-1', displayName: 'feat/notis' }] + }, + runtimeStatusByEnvironmentId: new Map(), + repos: [{ id: 'repo1', connectionId: null, displayName: 'orca' }], + projects: [], + sshConnectionStates: new Map(), + transientClearedAgentStatusConnectionIds: {}, + cacheTimerByKey: {}, + settings: { + promptCacheTimerEnabled: true, + experimentalTerminalAttention: true, + terminalMainSideEffectAuthority: false + }, + codexRestartNoticeByPtyId: {}, + deferredSshReconnectTargets: [], + deferredSshSessionIdsByTabId: {}, + removeDeferredSshReconnectTarget: vi.fn(), + removeDeferredSshSessionId: vi.fn(), + consumePendingColdRestore: vi.fn(() => null), + consumePendingSnapshot: vi.fn(() => null), + runtimePaneTitlesByTabId: {}, + agentStatusByPaneKey: {} as Record, + retainedAgentsByPaneKey: {}, + paneForegroundAgentByPaneKey: {} as Record, + sleepingAgentSessionsByPaneKey: {} as Record, + suppressedPtyExitIds: {}, + agentLaunchConfigByPaneKey: {} as Record, + getAgentLaunchConfigForStatusEntry: vi.fn((entry: { paneKey: string }) => { + const byPaneKey = mockStoreState.agentLaunchConfigByPaneKey as Record< + string, + { launchConfig: unknown } | undefined + > + return byPaneKey[entry.paneKey]?.launchConfig + }), + getAgentLaunchConfigForStatusMetadata: vi.fn(() => undefined), + clearSleepingAgentSession: vi.fn((paneKey: string) => { + delete (mockStoreState.sleepingAgentSessionsByPaneKey as Record)[paneKey] + }), + registerAgentLaunchConfig: vi.fn(), + clearAgentLaunchConfig: vi.fn(), + markWorktreeUnread: vi.fn(), + observeTerminalGitHubPullRequestLink: vi.fn(), + recordTerminalInput: vi.fn(), + setAgentStatus: vi.fn(), + removeAgentStatus: vi.fn(), + dropAgentStatus: vi.fn(), + retireAgentPaneAuthority: vi.fn(), + setPaneForegroundAgent: vi.fn((paneKey: string, entry: unknown) => { + ;(mockStoreState.paneForegroundAgentByPaneKey as Record)[paneKey] = entry + }), + clearPaneForegroundAgent: vi.fn((paneKey: string) => { + delete (mockStoreState.paneForegroundAgentByPaneKey as Record)[paneKey] + }), + markTerminalTabUnread: vi.fn(), + markTerminalPaneUnread: vi.fn(), + markAgentCompletionPaneUnread: vi.fn() + } + ;(globalThis as unknown as { window: unknown }).window = { + api: { + ssh: { + connect: vi.fn().mockResolvedValue({ status: 'connected' }), + needsPassphrasePrompt: vi.fn().mockResolvedValue(false) + }, + pty: { + kill: vi.fn(), + signal: vi.fn(), + listSessions: vi.fn().mockResolvedValue([]), + hasPty: vi.fn().mockResolvedValue(true), + getSize: vi.fn().mockResolvedValue(null), + reportGeometry: vi.fn(), + getMainBufferSnapshot: vi.fn().mockResolvedValue(null), + getForegroundProcess: vi.fn().mockResolvedValue(null), + inspectProcess: vi.fn().mockResolvedValue({ + foregroundProcess: null, + hasChildProcesses: false + }), + confirmForegroundProcess: vi.fn().mockResolvedValue(null), + hasChildProcesses: vi.fn().mockResolvedValue(false), + write: vi.fn(), + writeAccepted: vi.fn().mockResolvedValue(true), + setHiddenRendererPty: vi.fn(), + setPtyDeliveryInterest: vi.fn(), + ackColdRestore: vi.fn(), + onClearBufferRequest: vi.fn(() => vi.fn()), + onSerializeBufferRequest: vi.fn(() => vi.fn()), + sendSerializedBuffer: vi.fn(), + declarePendingPaneSerializer: vi.fn().mockResolvedValue(1), + settlePaneSerializer: vi.fn().mockResolvedValue(undefined), + clearPendingPaneSerializer: vi.fn().mockResolvedValue(undefined), + reportRendererSerializerReady: vi.fn().mockResolvedValue(undefined) + }, + platform: { + get: vi.fn(() => ({ platform: 'darwin', osRelease: '25.0.0' })) + }, + notifications: { + dispatch: vi.fn().mockResolvedValue({ delivered: true }), + playSound: vi.fn().mockResolvedValue({ played: true }) + }, + runtime: { + restoreTerminalFit: vi.fn().mockResolvedValue({ restored: true }) + }, + agentStatus: { + inferInterrupt: vi.fn().mockResolvedValue(false) + } + }, + dispatchEvent: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn() + } + globalThis.requestAnimationFrame = vi.fn((callback: FrameRequestCallback) => { + callback(0) + return 1 + }) + globalThis.cancelAnimationFrame = vi.fn() + }) + + afterEach(async () => { + await drainFakeTimerWork() + vi.useRealTimers() + vi.restoreAllMocks() + if (originalRequestAnimationFrame) { + globalThis.requestAnimationFrame = originalRequestAnimationFrame + } else { + delete (globalThis as { requestAnimationFrame?: typeof requestAnimationFrame }) + .requestAnimationFrame + } + if (originalCancelAnimationFrame) { + globalThis.cancelAnimationFrame = originalCancelAnimationFrame + } else { + delete (globalThis as { cancelAnimationFrame?: typeof cancelAnimationFrame }) + .cancelAnimationFrame + } + if (originalDocument) { + globalThis.document = originalDocument + } else { + delete (globalThis as { document?: Document }).document + } + delete (globalThis as unknown as { window?: unknown }).window + resetAgentStartupDelayedDeliveryForTests() + }) + + it('[modern] accepts an empty snapshot as successful recovery without a loss banner', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'snapshot' }, + snapshot: { ...HOST_SNAPSHOT, data: '' } + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(serializeBuffer).not.toHaveBeenCalled() + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + drive.disposable.dispose() + }) + + it('[modern] waits for a reported outcome instead of applying the old elapsed-time deadline', async () => { + const pendingOutcome = createDeferred<{ + availability: { kind: 'snapshot' } + snapshot: HostSnapshot + }>() + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockReturnValue(pendingOutcome.promise) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + + pendingOutcome.resolve({ availability: { kind: 'snapshot' }, snapshot: HOST_SNAPSHOT }) + await flushAsyncTicks(20) + expect(drive.writtenChunks().join('')).toContain(HOST_SNAPSHOT_MARKER) + expect(serializeBuffer).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + it('[modern] banners on the seventh retry-worthy host answer and never sends an eighth request', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'retry-worthy', cause: 'host-pending-output-overflowed' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + for (let expectedRequests = 2; expectedRequests <= 7; expectedRequests += 1) { + await advanceModernRetryProbe() + expect(serializeBufferOutcome).toHaveBeenCalledTimes(expectedRequests) + expect(drive.writtenChunks().join('').includes(BANNER_FRAGMENT)).toBe(expectedRequests === 7) + } + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(7) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + expect(serializeBuffer).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + // Regression: these causes are returned before any frame leaves the client, so the host + // declined nothing. Charging them to its budget banners a healthy pane at ~12s — the very + // elapsed-time guess this change removes. + it('[modern] does not spend the host answer budget on locally-gated retries', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'retry-worthy', cause: 'resync-in-flight' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + // Well past the 7-answer host budget: a stuck local gate must not be mistaken for host refusals. + for (let expectedRequests = 2; expectedRequests <= 12; expectedRequests += 1) { + await advanceModernRetryProbe() + expect(serializeBufferOutcome).toHaveBeenCalledTimes(expectedRequests) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + } + + // The gate clears: the pane recovers the hidden bytes it would otherwise have declared lost. + serializeBufferOutcome.mockResolvedValue({ + availability: { kind: 'snapshot' }, + snapshot: HOST_SNAPSHOT + }) + await advanceModernRetryProbe() + expect(drive.writtenChunks().join('')).toContain(HOST_SNAPSHOT_MARKER) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + expect(serializeBuffer).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + it('[modern] still bounds locally-gated retries at their own cap', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'retry-worthy', cause: 'connection-not-ready' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + for (let expectedRequests = 2; expectedRequests <= 30; expectedRequests += 1) { + await advanceModernRetryProbe() + expect(serializeBufferOutcome).toHaveBeenCalledTimes(expectedRequests) + expect(drive.writtenChunks().join('').includes(BANNER_FRAGMENT)).toBe(expectedRequests === 30) + } + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(30) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + drive.disposable.dispose() + }) + + it('[modern] banners immediately on permanent unavailability without retrying', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { + kind: 'permanently-unavailable', + reason: 'exceeds-client-replay-limit' + }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + await vi.advanceTimersByTimeAsync(0) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(serializeBuffer).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + it('[modern] cancels a live flood repaint when it declares the output unrecoverable', async () => { + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi + .fn() + .mockResolvedValueOnce({ + availability: { kind: 'retry-worthy', cause: 'host-no-serializable-buffer' }, + snapshot: null + }) + .mockResolvedValue({ + availability: { kind: 'permanently-unavailable', reason: 'exceeds-client-replay-limit' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + + // The retry answer arms a post-flood repaint 2s out; interrupt it partway with an + // ungated re-arm (remote output unpause) whose answer is permanently unavailable. + await vi.advanceTimersByTimeAsync(1_000) + drive.setOutputPaused(true) + drive.setOutputPaused(false) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(2) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + + // The still-armed repaint must not resurrect recovery the pane already declared dead. + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT))).toHaveLength(1) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(2) + drive.disposable.dispose() + }) + + it('[legacy] switches an explicit unknown-host answer to the existing four-request heuristic', async () => { + const serializeBuffer = vi.fn().mockResolvedValue(null) + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'unknown-legacy-host' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + await advanceThroughNullRetryBudget(drive) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(serializeBuffer).toHaveBeenCalledTimes(3) + expect(drive.writtenChunks().join('')).toContain(LIVE_AGENT_CHUNK) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + drive.disposable.dispose() + }) + + it('[modern] parks retry repaint while scrolled back and resumes when following output', async () => { + const { markTerminalFollowOutput, markTerminalPinnedViewport } = + await import('@/lib/pane-manager/terminal-scroll-intent') + const serializeBuffer = vi.fn() + const serializeBufferOutcome = vi.fn().mockResolvedValue({ + availability: { kind: 'retry-worthy', cause: 'request-already-in-flight' }, + snapshot: null + }) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer, serializeBufferOutcome) + drive.pane.terminal.buffer.active.baseY = 100 + drive.pane.terminal.buffer.active.viewportY = 42 + markTerminalPinnedViewport(drive.pane.terminal) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(60_000) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(1) + expect(drive.pane.terminal.buffer.active.viewportY).toBe(42) + + drive.pane.terminal.buffer.active.viewportY = drive.pane.terminal.buffer.active.baseY + markTerminalFollowOutput(drive.pane.terminal) + await flushAsyncTicks(20) + expect(serializeBufferOutcome).toHaveBeenCalledTimes(2) + drive.disposable.dispose() + }) +}) diff --git a/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-unavailable-banner.repro.test.ts b/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-unavailable-banner.repro.test.ts new file mode 100644 index 000000000..3ce3f3ad1 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/remote-hidden-output-restore-unavailable-banner.repro.test.ts @@ -0,0 +1,827 @@ +import type * as React from 'react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { resetAgentStartupDelayedDeliveryForTests } from '@/lib/agent-startup-delayed-delivery' + +// Deterministic reproduction for issue2-hidden-output-skip: +// "[Orca skipped hidden terminal output because main recovery was unavailable.]" +// printed into a live remote-runtime agent pane. +// +// Topology modeled (renderer contract level, per reproduce-orca-remote-server-issues): +// remote-runtime-owned PTY ("remote:" id) -> transport.serializeBuffer is the ONLY +// recovery for hidden-time output (canUseMainBufferSnapshot is structurally false; +// the host DROPS paused/hidden stream output, so the reveal snapshot is the sole +// carrier of hidden bytes). The fault is injected directly below the restore +// pipeline at transport.serializeBuffer — the first seam that distinguishes +// "host recovery is available but transiently null/slow" (flood-truncated +// SnapshotResponse, resync window, Tailscale RTT) from true unavailability. +// +// Two mechanism tests pin the exact transition (they asserted the defect on main +// and now assert its fixed form); the invariant tests (RED on main) assert the +// contract: a host snapshot that is one retry tick away must not be declared +// "unavailable", hidden output must be recovered, and the added tolerance stays +// bounded — a host that never answers still banners once and stops requesting. +// +// Repro command: +// pnpm exec vitest run --config config/vitest.config.ts \ +// src/renderer/src/components/terminal-pane/remote-hidden-output-restore-unavailable-banner.repro.test.ts + +async function flushAsyncTicks(count = 6): Promise { + for (let i = 0; i < count; i++) { + await Promise.resolve() + } +} + +async function drainFakeTimerWork(limit = 20): Promise { + await flushAsyncTicks(20) + if (!vi.isFakeTimers()) { + return + } + for (let iteration = 0; iteration < limit && vi.getTimerCount() > 0; iteration += 1) { + await vi.runOnlyPendingTimersAsync() + await flushAsyncTicks(20) + } + vi.clearAllTimers() + await flushAsyncTicks(20) + vi.clearAllTimers() +} + +const LEAF_1 = '11111111-1111-4111-8111-111111111111' as const +const LEAF_2 = '22222222-2222-4222-8222-222222222222' as const + +function leafIdForPane(paneId: number): string { + return paneId === 2 ? LEAF_2 : LEAF_1 +} + +type ConnectCallbacks = { + onReattachDetermined?: () => void + onConnect?: () => void + onData?: ( + data: string, + meta?: { seq?: number; rawLength?: number; background?: boolean; droppedOutput?: boolean } + ) => void + onReplayData?: (data: string, meta?: { clearBeforeReplay?: boolean }) => void + onError?: (msg: string) => void + onWriteUnavailable?: () => void + onOutputPauseChanged?: (paused: boolean, supported: boolean) => void +} + +type MockTransport = { + attach: ReturnType + connect: ReturnType & { + mockImplementation: ( + impl: (opts: { callbacks?: ConnectCallbacks } & Record) => Promise + ) => unknown + } + disconnect: ReturnType + sendInput: ReturnType + sendInputImmediate?: ReturnType + sendInputAccepted?: ReturnType + claimViewport: ReturnType + resize: ReturnType + getPtyId: ReturnType + getConnectionId: ReturnType + serializeBuffer?: ReturnType +} + +const scheduleRuntimeGraphSync = vi.fn() +const shouldSeedCacheTimerOnInitialTitle = vi.fn(() => false) +const scheduleTerminalWebglAtlasRecovery = vi.fn() +const toastInfo = vi.fn() +const notifyCodexPaneBoundForStaleSweep = vi.fn() + +let mockStoreState: Record +let transportFactoryQueue: MockTransport[] = [] +let createdTransportOptions: Record[] = [] +let storeSubscribers: ((state: Record) => void)[] = [] + +vi.mock('@/runtime/sync-runtime-graph', () => ({ + scheduleRuntimeGraphSync +})) + +vi.mock('@/store', () => ({ + useAppStore: { + getState: () => mockStoreState, + subscribe: (listener: (state: Record) => void) => { + storeSubscribers.push(listener) + return () => { + storeSubscribers = storeSubscribers.filter((candidate) => candidate !== listener) + } + } + } +})) + +vi.mock('./terminal-webgl-atlas-recovery', () => ({ + scheduleTerminalWebglAtlasRecovery +})) + +vi.mock('@/lib/agent-status', async (importOriginal) => { + const actual = await importOriginal>() + const isGeminiTerminalTitle = actual.isGeminiTerminalTitle as (title: string) => boolean + return { + ...actual, + isGeminiTerminalTitle: vi.fn((title: string) => isGeminiTerminalTitle(title)), + isClaudeAgent: vi.fn(() => false), + detectAgentStatusFromTitle: vi.fn(() => null) + } +}) + +vi.mock('./cache-timer-seeding', () => ({ + shouldSeedCacheTimerOnInitialTitle +})) + +vi.mock('sonner', () => ({ + toast: { + info: toastInfo + } +})) + +vi.mock('@/lib/codex-stale-pane-sweep', () => ({ + notifyCodexPaneBoundForStaleSweep +})) + +vi.mock('react', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + useCallback: unknown>(fn: T): T => fn + } +}) + +vi.mock('./pty-transport', () => ({ + createIpcPtyTransport: vi.fn((options: Record) => { + createdTransportOptions.push(options) + const nextTransport = transportFactoryQueue.shift() + if (!nextTransport) { + throw new Error('No mock transport queued') + } + return nextTransport + }) +})) + +vi.mock('./remote-runtime-pty-transport', () => ({ + createRemoteRuntimePtyTransport: vi.fn( + (_environmentId: string, options: Record) => { + createdTransportOptions.push(options) + const nextTransport = transportFactoryQueue.shift() + if (!nextTransport) { + throw new Error('No mock transport queued') + } + return nextTransport + } + ) +})) + +vi.mock('./pty-dispatcher', async (importOriginal) => { + const actual = await importOriginal>() + return { + ...actual, + getEagerPtyBufferHandle: vi.fn(() => undefined) + } +}) + +function createMockTransport(initialPtyId: string | null = null): MockTransport { + let ptyId = initialPtyId + const transport = { + attach: vi.fn(({ existingPtyId }: { existingPtyId: string }) => { + ptyId = existingPtyId + }), + connect: vi.fn().mockImplementation(async (opts: { sessionId?: string }) => { + if (opts.sessionId) { + ptyId = opts.sessionId + return { id: opts.sessionId } + } + return ptyId + }), + disconnect: vi.fn(() => { + ptyId = null + }), + sendInput: vi.fn(() => true), + claimViewport: vi.fn(() => true), + resize: vi.fn(() => true), + getPtyId: vi.fn(() => ptyId), + getConnectionId: vi.fn(() => null), + serializeBuffer: undefined + } as MockTransport + const sendInput = transport.sendInput as unknown as (data: string) => boolean + transport.sendInputImmediate = vi.fn((data: string) => sendInput(data)) + transport.sendInputAccepted = vi.fn(async (data: string) => sendInput(data)) + return transport +} + +function createPaneContainer(): HTMLElement { + const container = new EventTarget() as HTMLElement + Object.defineProperty(container, 'dataset', { + configurable: true, + value: {} + }) + return container +} + +function createPane(paneId: number) { + const leafId = leafIdForPane(paneId) + const activeBuffer = { + type: 'normal' as const, + viewportY: 0, + baseY: 0, + cursorY: 0, + cursorX: 0 + } + const terminal = { + cols: 120, + rows: 40, + element: {}, + buffer: { + active: activeBuffer + }, + modes: { + bracketedPasteMode: false, + sendFocusMode: false + }, + options: { + scrollback: 5_000, + ignoreBracketedPasteMode: false, + theme: { + foreground: '#eeeeee', + background: '#111111' + } + }, + write: vi.fn<(data: string, callback?: () => void) => void>(function write(...args): void { + const [data, callback] = args + if (data === '' || callback?.name === 'runParsedSteps') { + callback?.() + } + }), + resize: vi.fn(), + clear: vi.fn(), + scrollToBottom: vi.fn(() => { + activeBuffer.viewportY = activeBuffer.baseY + }), + scrollToLine: vi.fn((line: number) => { + activeBuffer.viewportY = line + }), + scrollLines: vi.fn((amount: number) => { + activeBuffer.viewportY = Math.max( + 0, + Math.min(activeBuffer.baseY, activeBuffer.viewportY + amount) + ) + }), + paste: vi.fn(), + onData: vi.fn(() => ({ dispose: vi.fn() })), + onResize: vi.fn(() => ({ dispose: vi.fn() })), + onRender: vi.fn((_listener: () => void) => ({ dispose: vi.fn() })), + onTitleChange: vi.fn(() => ({ dispose: vi.fn() })), + hasSelection: vi.fn(() => false), + parser: { + registerCsiHandler: vi.fn(() => ({ dispose: vi.fn() })), + registerOscHandler: vi.fn(() => ({ dispose: vi.fn() })) + } + } + return { + id: paneId, + leafId, + stablePaneId: leafId, + terminal, + container: createPaneContainer(), + fitAddon: { + fit: vi.fn(), + proposeDimensions: vi.fn(() => ({ cols: terminal.cols, rows: terminal.rows })) + } + } +} + +function createManager(paneCount = 1, initialActivePaneId: number | null = null) { + let activePaneId = initialActivePaneId + const panes = Array.from({ length: paneCount }, (_, index) => ({ + id: index + 1, + leafId: leafIdForPane(index + 1) + })) + return { + setPaneGpuRendering: vi.fn(), + markPaneHasComplexScriptOutput: vi.fn(), + rebuildPaneWebgl: vi.fn(), + hasWebglRenderer: vi.fn(() => false), + getPanes: vi.fn(() => panes), + closePane: vi.fn(), + getActivePane: vi.fn<() => { id: number; leafId?: string } | null>(() => + activePaneId === null + ? null + : (panes.find((candidate) => candidate.id === activePaneId) ?? null) + ), + getNumericIdForLeaf: vi.fn((leafId: string) => { + return panes.find((candidate) => candidate.leafId === leafId)?.id ?? null + }), + setActivePane: vi.fn((paneId: number) => { + activePaneId = paneId + }) + } +} + +function createDeps(overrides: Record = {}) { + return { + tabId: 'tab-1', + worktreeId: 'wt-1', + cwd: '/tmp/wt-1', + startup: null, + restoredLeafId: null, + restoredPtyIdByLeafId: {}, + paneTransportsRef: { current: new Map() }, + paneMode2031Ref: { current: new Map() }, + paneKittyKeyboardModesRef: { current: new Map() }, + paneLastThemeModeRef: { current: new Map() }, + replayingPanesRef: { current: new Map() }, + isActiveRef: { current: true }, + isVisibleRef: { current: true }, + onPtyExitRef: { current: vi.fn() }, + onAgentExitedRef: { current: vi.fn() }, + onPtyErrorRef: { current: vi.fn() }, + clearTabPtyId: vi.fn(), + consumeSuppressedPtyExit: vi.fn(() => false), + isPtyShutdownPending: vi.fn(() => false), + updateTabTitle: vi.fn(), + setRuntimePaneTitle: vi.fn(), + clearRuntimePaneTitle: vi.fn(), + updateTabPtyId: vi.fn((tabId: string, ptyId: string, replacedPtyId?: string) => { + const byTab = (mockStoreState.ptyIdsByTabId ?? {}) as Record + const current = byTab[tabId] ?? [] + const next = + replacedPtyId && current.includes(replacedPtyId) + ? current.map((candidate) => (candidate === replacedPtyId ? ptyId : candidate)) + : current.includes(ptyId) + ? current + : [...current, ptyId] + mockStoreState.ptyIdsByTabId = { ...byTab, [tabId]: next } + }), + markWorktreeUnread: vi.fn(), + markTerminalTabUnread: vi.fn(), + markTerminalPaneUnread: vi.fn(), + clearWorktreeUnread: vi.fn(), + clearTerminalTabUnread: vi.fn(), + clearTerminalPaneUnread: vi.fn(), + dispatchNotification: vi.fn(), + onShowSessionRestoredBanner: vi.fn(), + setCacheTimerStartedAt: vi.fn(), + syncPanePtyLayoutBinding: vi.fn(), + clearExitedPanePtyLayoutBinding: vi.fn(), + ...overrides + } +} + +function createDeferred(): { + promise: Promise + resolve: (value: T) => void + reject: (reason?: unknown) => void +} { + let resolveDeferred!: (value: T) => void + let rejectDeferred!: (reason?: unknown) => void + const promise = new Promise((resolve, reject) => { + resolveDeferred = resolve + rejectDeferred = reject + }) + return { promise, resolve: resolveDeferred, reject: rejectDeferred } +} + +// ── Scenario identities ───────────────────────────────────────────────────── +const REMOTE_PTY_ID = 'remote:env-1@@terminal-agent-1' +// Overflows the renderer's hidden background queue (2MB lossy cap), dropping the +// backlog and latching model restore — mirrors the real remote pause semantics +// where hidden-time bytes exist ONLY in the host's authoritative buffer. +const HIDDEN_BYTES = 'x'.repeat(2 * 1024 * 1024 + 1) +const LIVE_AGENT_CHUNK = 'LIVE_AGENT_CHUNK\r\n' +const HOST_SNAPSHOT_MARKER = 'HOST_SNAPSHOT_HIDDEN_AGENT_CONTENT' +const HOST_SNAPSHOT = { + data: `${HOST_SNAPSHOT_MARKER}\r\n`, + cols: 120, + rows: 40, + seq: HIDDEN_BYTES.length + LIVE_AGENT_CHUNK.length, + source: 'headless' +} +const BANNER_FRAGMENT = 'main recovery was unavailable' + +type HostSnapshot = typeof HOST_SNAPSHOT + +type RemotePaneDrive = { + transport: MockTransport + pane: ReturnType + deps: ReturnType + disposable: { dispose: () => void } + deliver: (data: string, seq: number) => void + writtenChunks: () => string[] +} + +async function connectHiddenRemoteAgentPane( + serializeBuffer: ReturnType +): Promise { + const { connectPanePty } = await import('./pty-connection') + const transport = createMockTransport(REMOTE_PTY_ID) + transport.serializeBuffer = serializeBuffer + const capturedDataCallback: { + current: ((data: string, meta?: { seq?: number; rawLength?: number }) => void) | null + } = { current: null } + transport.connect.mockImplementation(async ({ callbacks }: { callbacks?: ConnectCallbacks }) => { + capturedDataCallback.current = callbacks?.onData ?? null + return REMOTE_PTY_ID + }) + transportFactoryQueue.push(transport) + const pane = createPane(1) + const manager = createManager(1) + const deps = createDeps({ isVisibleRef: { current: false } }) + const disposable = connectPanePty(pane as never, manager as never, deps as never) + await flushAsyncTicks(6) + expect(capturedDataCallback.current).not.toBeNull() + return { + transport, + pane, + deps, + disposable, + deliver: (data, seq) => capturedDataCallback.current?.(data, { seq, rawLength: data.length }), + writtenChunks: () => pane.terminal.write.mock.calls.map(([data]) => String(data)) + } +} + +// Delivers the hidden backlog, reveals the pane, and delivers one live agent +// chunk — the exact user flow in the screenshot (agent streaming on reveal). +function driveHiddenBacklogThenReveal(drive: RemotePaneDrive): void { + drive.deliver(HIDDEN_BYTES, HIDDEN_BYTES.length) + ;(drive.deps.isVisibleRef as { current: boolean }).current = true + drive.deliver(LIVE_AGENT_CHUNK, HIDDEN_BYTES.length + LIVE_AGENT_CHUNK.length) +} + +async function advanceThroughNullRetryBudget(drive: RemotePaneDrive): Promise { + for (let retry = 0; retry < 3; retry++) { + // Live output stays gated behind the in-flight restore during the window. + expect(drive.pane.terminal.write).not.toHaveBeenCalledWith( + LIVE_AGENT_CHUNK, + expect.any(Function) + ) + vi.advanceTimersByTime(50) + vi.advanceTimersByTime(0) + await flushAsyncTicks(10) + } +} + +// Generous self-heal window: lets any corrected pipeline (extended retries, +// restore re-arm, post-abandon repaint) run to completion under fake timers. +async function allowSelfHealWindow(): Promise { + for (let step = 0; step < 10; step += 1) { + await vi.advanceTimersByTimeAsync(500) + await flushAsyncTicks(20) + } +} + +function observeFinalPaneState(drive: RemotePaneDrive): { + localMainSnapshotCalls: number + rawHiddenBacklogWritten: boolean + liveAgentChunkWritten: boolean + unavailableBannerWritten: boolean + hiddenOutputRecoveredFromHostSnapshot: boolean +} { + const written = drive.writtenChunks() + const joined = written.join('') + return { + localMainSnapshotCalls: vi.mocked(window.api.pty.getMainBufferSnapshot).mock.calls.length, + rawHiddenBacklogWritten: joined.includes('x'.repeat(1024)), + liveAgentChunkWritten: joined.includes('LIVE_AGENT_CHUNK'), + unavailableBannerWritten: joined.includes(BANNER_FRAGMENT), + hiddenOutputRecoveredFromHostSnapshot: joined.includes(HOST_SNAPSHOT_MARKER) + } +} + +describe('remote hidden-output restore abandonment (issue2-hidden-output-skip)', () => { + const originalRequestAnimationFrame = globalThis.requestAnimationFrame + const originalCancelAnimationFrame = globalThis.cancelAnimationFrame + const originalDocument = globalThis.document + + beforeEach(() => { + vi.resetModules() + vi.clearAllMocks() + transportFactoryQueue = [] + createdTransportOptions = [] + storeSubscribers = [] + mockStoreState = { + activeWorktreeId: 'wt-1', + tabsByWorktree: { + 'wt-1': [{ id: 'tab-1', ptyId: 'tab-pty' }] + }, + ptyIdsByTabId: { + 'tab-1': ['tab-pty'] + }, + terminalLayoutsByTabId: { + 'tab-1': { + root: { type: 'leaf', leafId: LEAF_1 }, + activeLeafId: LEAF_1, + expandedLeafId: null, + ptyIdsByLeafId: { [LEAF_1]: 'tab-pty' } + } + }, + unreadTerminalTabs: {}, + deleteStateByWorktreeId: {}, + worktreesByRepo: { + repo1: [{ id: 'wt-1', repoId: 'repo1', path: '/tmp/wt-1', displayName: 'feat/notis' }] + }, + runtimeStatusByEnvironmentId: new Map(), + repos: [{ id: 'repo1', connectionId: null, displayName: 'orca' }], + projects: [], + sshConnectionStates: new Map(), + transientClearedAgentStatusConnectionIds: {}, + cacheTimerByKey: {}, + settings: { + promptCacheTimerEnabled: true, + experimentalTerminalAttention: true, + terminalMainSideEffectAuthority: false + }, + codexRestartNoticeByPtyId: {}, + deferredSshReconnectTargets: [], + deferredSshSessionIdsByTabId: {}, + removeDeferredSshReconnectTarget: vi.fn(), + removeDeferredSshSessionId: vi.fn(), + consumePendingColdRestore: vi.fn(() => null), + consumePendingSnapshot: vi.fn(() => null), + runtimePaneTitlesByTabId: {}, + agentStatusByPaneKey: {} as Record, + retainedAgentsByPaneKey: {}, + paneForegroundAgentByPaneKey: {} as Record, + sleepingAgentSessionsByPaneKey: {} as Record, + suppressedPtyExitIds: {}, + agentLaunchConfigByPaneKey: {} as Record, + getAgentLaunchConfigForStatusEntry: vi.fn((entry: { paneKey: string }) => { + const byPaneKey = mockStoreState.agentLaunchConfigByPaneKey as Record< + string, + { launchConfig: unknown } | undefined + > + return byPaneKey[entry.paneKey]?.launchConfig + }), + getAgentLaunchConfigForStatusMetadata: vi.fn(() => undefined), + clearSleepingAgentSession: vi.fn((paneKey: string) => { + delete (mockStoreState.sleepingAgentSessionsByPaneKey as Record)[paneKey] + }), + registerAgentLaunchConfig: vi.fn(), + clearAgentLaunchConfig: vi.fn(), + markWorktreeUnread: vi.fn(), + observeTerminalGitHubPullRequestLink: vi.fn(), + recordTerminalInput: vi.fn(), + setAgentStatus: vi.fn(), + removeAgentStatus: vi.fn(), + dropAgentStatus: vi.fn(), + retireAgentPaneAuthority: vi.fn(), + setPaneForegroundAgent: vi.fn((paneKey: string, entry: unknown) => { + ;(mockStoreState.paneForegroundAgentByPaneKey as Record)[paneKey] = entry + }), + clearPaneForegroundAgent: vi.fn((paneKey: string) => { + delete (mockStoreState.paneForegroundAgentByPaneKey as Record)[paneKey] + }), + markTerminalTabUnread: vi.fn(), + markTerminalPaneUnread: vi.fn(), + markAgentCompletionPaneUnread: vi.fn() + } + ;(globalThis as unknown as { window: unknown }).window = { + api: { + ssh: { + connect: vi.fn().mockResolvedValue({ status: 'connected' }), + needsPassphrasePrompt: vi.fn().mockResolvedValue(false) + }, + pty: { + kill: vi.fn(), + signal: vi.fn(), + listSessions: vi.fn().mockResolvedValue([]), + hasPty: vi.fn().mockResolvedValue(true), + getSize: vi.fn().mockResolvedValue(null), + reportGeometry: vi.fn(), + getMainBufferSnapshot: vi.fn().mockResolvedValue(null), + getForegroundProcess: vi.fn().mockResolvedValue(null), + inspectProcess: vi.fn().mockResolvedValue({ + foregroundProcess: null, + hasChildProcesses: false + }), + confirmForegroundProcess: vi.fn().mockResolvedValue(null), + hasChildProcesses: vi.fn().mockResolvedValue(false), + write: vi.fn(), + writeAccepted: vi.fn().mockResolvedValue(true), + setHiddenRendererPty: vi.fn(), + setPtyDeliveryInterest: vi.fn(), + ackColdRestore: vi.fn(), + onClearBufferRequest: vi.fn(() => vi.fn()), + onSerializeBufferRequest: vi.fn(() => vi.fn()), + sendSerializedBuffer: vi.fn(), + declarePendingPaneSerializer: vi.fn().mockResolvedValue(1), + settlePaneSerializer: vi.fn().mockResolvedValue(undefined), + clearPendingPaneSerializer: vi.fn().mockResolvedValue(undefined), + reportRendererSerializerReady: vi.fn().mockResolvedValue(undefined) + }, + platform: { + get: vi.fn(() => ({ platform: 'darwin', osRelease: '25.0.0' })) + }, + notifications: { + dispatch: vi.fn().mockResolvedValue({ delivered: true }), + playSound: vi.fn().mockResolvedValue({ played: true }) + }, + runtime: { + restoreTerminalFit: vi.fn().mockResolvedValue({ restored: true }) + }, + agentStatus: { + inferInterrupt: vi.fn().mockResolvedValue(false) + } + }, + dispatchEvent: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn() + } + globalThis.requestAnimationFrame = vi.fn((callback: FrameRequestCallback) => { + callback(0) + return 1 + }) + globalThis.cancelAnimationFrame = vi.fn() + }) + + afterEach(async () => { + await drainFakeTimerWork() + vi.useRealTimers() + vi.restoreAllMocks() + if (originalRequestAnimationFrame) { + globalThis.requestAnimationFrame = originalRequestAnimationFrame + } else { + delete (globalThis as { requestAnimationFrame?: typeof requestAnimationFrame }) + .requestAnimationFrame + } + if (originalCancelAnimationFrame) { + globalThis.cancelAnimationFrame = originalCancelAnimationFrame + } else { + delete (globalThis as { cancelAnimationFrame?: typeof cancelAnimationFrame }) + .cancelAnimationFrame + } + if (originalDocument) { + globalThis.document = originalDocument + } else { + delete (globalThis as { document?: Document }).document + } + delete (globalThis as unknown as { window?: unknown }).window + resetAgentStartupDelayedDeliveryForTests() + }) + + // ── Mechanism (GREEN on main): pins the defective transition exactly ────── + + it('[mechanism] 4 transient-null host snapshots end the retry budget: live drains immediately, no banner, and the restore re-arms until the host answers', async () => { + const serializeBuffer = vi.fn(async () => + serializeBuffer.mock.calls.length <= 4 ? null : HOST_SNAPSHOT + ) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + // Reveal triggered exactly one remote snapshot request against the owning runtime. + expect(serializeBuffer).toHaveBeenCalledTimes(1) + expect(serializeBuffer).toHaveBeenNthCalledWith(1, { scrollbackRows: 5000 }) + + await advanceThroughNullRetryBudget(drive) + + const written = drive.writtenChunks() + // 1 initial + 3 deferred retries (50ms each), then abandonment unblocks live + // output — but for a remote pane it must stay quiet, not claim loss. + expect(serializeBuffer).toHaveBeenCalledTimes(4) + expect(serializeBuffer).toHaveBeenNthCalledWith(4, { scrollbackRows: 5000 }) + expect(written.some((data) => data.includes('LIVE_AGENT_CHUNK'))).toBe(true) + expect(written.some((data) => data.includes(BANNER_FRAGMENT))).toBe(false) + // Remote pane never consulted local main recovery — the banner's "main + // recovery" claim is structurally impossible for this PTY. + expect(window.api.pty.getMainBufferSnapshot).not.toHaveBeenCalled() + expect((drive.transport.getPtyId as unknown as () => string | null)()).toBe(REMOTE_PTY_ID) + + // Abandonment re-arms: the host answers the next request and the hidden + // bytes are repainted from its authoritative buffer. + await allowSelfHealWindow() + expect(serializeBuffer.mock.calls.length).toBeGreaterThan(4) + const joined = drive.writtenChunks().join('') + expect(joined).toContain(HOST_SNAPSHOT_MARKER) + expect(joined).not.toContain('x'.repeat(1024)) + drive.disposable.dispose() + }) + + it('[mechanism] slow-but-valid host snapshot: the 750ms foreground deadline unblocks live output quietly and refetches the discarded snapshot', async () => { + const slowSnapshot = createDeferred() + const serializeBuffer = vi.fn().mockReturnValue(slowSnapshot.promise) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + expect(serializeBuffer).toHaveBeenCalledTimes(1) + expect(drive.pane.terminal.write).not.toHaveBeenCalledWith( + LIVE_AGENT_CHUNK, + expect.any(Function) + ) + + // Negative control: one tick before the deadline no banner exists. + vi.advanceTimersByTime(749) + await flushAsyncTicks(6) + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + + vi.advanceTimersByTime(1) + vi.advanceTimersByTime(0) + await flushAsyncTicks(10) + + const written = drive.writtenChunks() + expect(written.some((data) => data.includes('LIVE_AGENT_CHUNK'))).toBe(true) + expect(written.some((data) => data.includes(BANNER_FRAGMENT))).toBe(false) + + // The host's serialization completes just after the deadline; the stale + // generation guard discards that snapshot, so the re-arm must refetch it. + slowSnapshot.resolve(HOST_SNAPSHOT) + await flushAsyncTicks(20) + expect(serializeBuffer).toHaveBeenCalledTimes(1) + expect(drive.writtenChunks().join('')).not.toContain(HOST_SNAPSHOT_MARKER) + + await allowSelfHealWindow() + expect(serializeBuffer.mock.calls.length).toBeGreaterThan(1) + expect(drive.writtenChunks().join('')).toContain(HOST_SNAPSHOT_MARKER) + expect(window.api.pty.getMainBufferSnapshot).not.toHaveBeenCalled() + drive.disposable.dispose() + }) + + // ── Invariants (RED on main): the contract a fix must satisfy ───────────── + + it('[invariant] must not declare recovery unavailable when the host snapshot is one retry tick away', async () => { + const serializeBuffer = vi.fn(async () => + serializeBuffer.mock.calls.length <= 4 ? null : HOST_SNAPSHOT + ) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + await advanceThroughNullRetryBudget(drive) + await allowSelfHealWindow() + + expect(observeFinalPaneState(drive)).toEqual({ + localMainSnapshotCalls: 0, + rawHiddenBacklogWritten: false, + liveAgentChunkWritten: true, + unavailableBannerWritten: false, + hiddenOutputRecoveredFromHostSnapshot: true + }) + drive.disposable.dispose() + }) + + it('[invariant] must not turn a merely-slow host snapshot into a loss banner and a permanent scrollback gap', async () => { + const slowSnapshot = createDeferred() + const serializeBuffer = vi.fn(async () => HOST_SNAPSHOT) + serializeBuffer.mockReturnValueOnce(slowSnapshot.promise) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + + // Deadline elapses while the host is still serializing its buffer. + vi.advanceTimersByTime(750) + vi.advanceTimersByTime(0) + await flushAsyncTicks(10) + // Host serialization finishes with the authoritative hidden bytes; every + // later snapshot request would succeed instantly. + slowSnapshot.resolve(HOST_SNAPSHOT) + await flushAsyncTicks(20) + await allowSelfHealWindow() + + expect(observeFinalPaneState(drive)).toEqual({ + localMainSnapshotCalls: 0, + rawHiddenBacklogWritten: false, + liveAgentChunkWritten: true, + unavailableBannerWritten: false, + hiddenOutputRecoveredFromHostSnapshot: true + }) + drive.disposable.dispose() + }) + + // Bounds the tolerance: patience must not become an unbounded snapshot poll, + // and a host that truly never answers still owes the user a loss signal. + it('[invariant] a permanently silent host banners once after a bounded number of re-arms and then stops requesting', async () => { + const serializeBuffer = vi.fn(async () => null) + const drive = await connectHiddenRemoteAgentPane(serializeBuffer) + vi.useFakeTimers() + driveHiddenBacklogThenReveal(drive) + await flushAsyncTicks(20) + await advanceThroughNullRetryBudget(drive) + + // Live output is never held hostage by the re-arm budget, and the first + // abandonment stays quiet. + expect(drive.writtenChunks().join('')).toContain('LIVE_AGENT_CHUNK') + expect(drive.writtenChunks().join('')).not.toContain(BANNER_FRAGMENT) + + // ~30s: far past the 5 re-arm cycles (~2.15s each). + for (let step = 0; step < 6; step += 1) { + await allowSelfHealWindow() + } + const bannerCount = drive + .writtenChunks() + .filter((data) => data.includes(BANNER_FRAGMENT)).length + expect(bannerCount).toBe(1) + const settledRequests = serializeBuffer.mock.calls.length + // Re-armed past the initial budget, but bounded — no permanent polling. + expect(settledRequests).toBeGreaterThan(4) + expect(settledRequests).toBeLessThan(40) + + for (let step = 0; step < 4; step += 1) { + await allowSelfHealWindow() + } + expect(serializeBuffer.mock.calls.length).toBe(settledRequests) + expect(drive.writtenChunks().filter((data) => data.includes(BANNER_FRAGMENT)).length).toBe(1) + drive.disposable.dispose() + }) +}) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts index 198c24818..16b83b005 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts @@ -5801,6 +5801,41 @@ describe('createRemoteRuntimePtyTransport', () => { expect(onData).not.toHaveBeenCalledWith('requested snapshot', expect.anything()) }) + it('forwards requested snapshot availability through the remote transport', async () => { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const transport = createRemoteRuntimePtyTransport('env-1', { worktreeId: 'wt-1' }) + + await transport.connect({ url: '', callbacks: {} }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + const { streamId } = latestSubscribePayload() + emitSnapshot(streamId, 'initial') + + const outcomePromise = transport.serializeBufferOutcome?.({ scrollbackRows: 5000 }) + await vi.waitFor(() => + expect(latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest)).toBeDefined() + ) + const requestFrame = latestFrameForOpcode(TerminalStreamOpcode.SnapshotRequest) + const request = requestFrame + ? decodeTerminalStreamJson<{ requestId?: number }>(requestFrame.payload) + : null + emitSnapshotFrame( + streamId, + TerminalStreamOpcode.SnapshotStart, + encodeTerminalStreamJson({ + requestId: request?.requestId, + cols: 120, + rows: 40, + unavailable: 'no-serializable-buffer' + }) + ) + emitSnapshotFrame(streamId, TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + + await expect(outcomePromise).resolves.toMatchObject({ + availability: { kind: 'retry-worthy', cause: 'host-no-serializable-buffer' }, + snapshot: { data: '' } + }) + }) + it('keeps initial replay separate from in-flight explicit binary snapshot requests', async () => { const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') const onReplayData = vi.fn() diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index a74fed287..fb6caea57 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -43,7 +43,8 @@ import { import { getRemoteRuntimeTerminalMultiplexer, REMOTE_TERMINAL_SNAPSHOT_TOO_LARGE, - type RemoteRuntimeMultiplexedTerminal + type RemoteRuntimeMultiplexedTerminal, + type RemoteRuntimeSnapshotOutcome } from '../../runtime/remote-runtime-terminal-multiplexer' import { toRuntimeTerminalWorktreeSelector, @@ -2461,6 +2462,23 @@ export function createRemoteRuntimePtyTransport( return getCurrentMultiplexedStream(handle)?.serializeBuffer(opts) ?? null }, + async serializeBufferOutcome(opts): Promise { + if (!connected || !handle) { + return { + availability: { kind: 'retry-worthy', cause: 'connection-not-ready' }, + snapshot: null + } + } + const stream = getCurrentMultiplexedStream(handle) + if (!stream) { + return { + availability: { kind: 'retry-worthy', cause: 'stream-detached' }, + snapshot: null + } + } + return stream.serializeBufferOutcome(opts) + }, + destroy() { destroyed = true setAttachmentUnavailable() diff --git a/src/renderer/src/lib/pane-manager/terminal-follow-output-waiters.ts b/src/renderer/src/lib/pane-manager/terminal-follow-output-waiters.ts new file mode 100644 index 000000000..a5a187f71 --- /dev/null +++ b/src/renderer/src/lib/pane-manager/terminal-follow-output-waiters.ts @@ -0,0 +1,30 @@ +import type { TerminalScrollIntentTarget } from './terminal-scroll-intent' + +// Why: work that repositions the viewport (clear-and-replay repaints) must wait +// out a pinned reading position instead of racing it on a timer. Waiters are +// one-shot: the intent write that turns follow-output back on releases them all. +const followOutputWaitersByTerminal = new WeakMap void>>() + +/** Registers a one-shot listener for the terminal's next follow-output intent; returns a canceller. */ +export function addTerminalFollowOutputWaiter( + terminal: TerminalScrollIntentTarget, + listener: () => void +): () => void { + const waiters = followOutputWaitersByTerminal.get(terminal) ?? new Set<() => void>() + followOutputWaitersByTerminal.set(terminal, waiters) + waiters.add(listener) + return () => { + waiters.delete(listener) + } +} + +export function notifyTerminalFollowOutputWaiters(terminal: TerminalScrollIntentTarget): void { + const waiters = followOutputWaitersByTerminal.get(terminal) + if (!waiters?.size) { + return + } + followOutputWaitersByTerminal.delete(terminal) + for (const waiter of waiters) { + waiter() + } +} diff --git a/src/renderer/src/lib/pane-manager/terminal-scroll-intent.ts b/src/renderer/src/lib/pane-manager/terminal-scroll-intent.ts index 8fa96d62c..bd0b63611 100644 --- a/src/renderer/src/lib/pane-manager/terminal-scroll-intent.ts +++ b/src/renderer/src/lib/pane-manager/terminal-scroll-intent.ts @@ -1,3 +1,7 @@ +import { + addTerminalFollowOutputWaiter, + notifyTerminalFollowOutputWaiters +} from './terminal-follow-output-waiters' import { isTerminalScrollIntentRebuildInFlight } from './terminal-scroll-intent-rebuild' import { clampTerminalViewportY, @@ -55,6 +59,18 @@ const terminalScrollIntentBindingByKey = new Map void +): () => void { + if (getTerminalScrollIntentKind(terminal) === 'followOutput') { + listener() + return () => {} + } + return addTerminalFollowOutputWaiter(terminal, listener) +} + function writeIntent( terminal: TerminalScrollIntentTarget, kind: TerminalScrollIntentKind @@ -78,6 +94,9 @@ function writeIntentSnapshot( if (key) { terminalScrollIntentByKey.set(key, intent) } + if (kind === 'followOutput') { + notifyTerminalFollowOutputWaiters(terminal) + } return intent } diff --git a/src/renderer/src/runtime/remote-runtime-snapshot-outcome.test.ts b/src/renderer/src/runtime/remote-runtime-snapshot-outcome.test.ts new file mode 100644 index 000000000..1d903d5bd --- /dev/null +++ b/src/renderer/src/runtime/remote-runtime-snapshot-outcome.test.ts @@ -0,0 +1,301 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + TerminalStreamOpcode, + decodeTerminalStreamFrame, + decodeTerminalStreamJson, + encodeTerminalStreamFrame, + encodeTerminalStreamJson, + encodeTerminalStreamText +} from '../../../shared/terminal-stream-protocol' +import { + getRemoteRuntimeTerminalMultiplexer, + resetRemoteRuntimeTerminalMultiplexersForTests, + type RemoteRuntimeMultiplexedTerminal +} from './remote-runtime-terminal-multiplexer' +import { replaceRuntimeEnvironmentRevisions } from './runtime-environment-revision' + +// Why: the client used to collapse every unusable snapshot reply to `null`, so a slow +// host and a genuinely empty pane were indistinguishable. These tests pin the reason +// each reply now carries, and pin that the legacy `serializeBuffer` result is unchanged. + +type SubscribeCallbacks = { + onResponse: (response: unknown) => void + onBinary?: (bytes: Uint8Array) => void + onError?: (error: { message: string }) => void + onClose?: () => void +} + +type RequestedReply = + | { kind: 'buffer'; data: string } + | { kind: 'buffer'; data: string; unavailable: string } + | { kind: 'truncated'; unavailable?: string } + | { kind: 'oversized'; chunks: number } + | { kind: 'hold' } + +/** Answers SnapshotRequests with a scripted reply so each unavailability shape can be replayed exactly. */ +class ScriptedSnapshotServer { + private streamId = 0 + private cursorUnits = 0 + requestIds: (number | undefined)[] = [] + nextRequestedReply: RequestedReply = { kind: 'buffer', data: 'MANUAL' } + dropNextOutput = false + holdResyncReplies = false + private heldRequestId: number | null = null + + constructor(private readonly toClient: (bytes: Uint8Array) => void) {} + + receive(bytes: Uint8Array): void { + const frame = decodeTerminalStreamFrame(bytes) + if (!frame) { + return + } + if (frame.opcode === TerminalStreamOpcode.Subscribe) { + this.streamId = decodeTerminalStreamJson<{ streamId: number }>(frame.payload)?.streamId ?? 0 + this.sendStart({}) + this.send(TerminalStreamOpcode.SnapshotChunk, encodeTerminalStreamText('INITIAL')) + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + return + } + if (frame.opcode !== TerminalStreamOpcode.SnapshotRequest) { + return + } + const requestId = decodeTerminalStreamJson<{ requestId?: number }>(frame.payload)?.requestId + this.requestIds.push(requestId) + if (typeof requestId !== 'number') { + if (this.holdResyncReplies) { + return + } + // Untagged resync request: answer it so the resync gate does not stay latched. + this.sendStart({}) + this.send(TerminalStreamOpcode.SnapshotChunk, encodeTerminalStreamText('RECOVERED')) + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + return + } + this.replyToRequest(requestId, this.nextRequestedReply) + } + + /** Answers a request parked by `{ kind: 'hold' }` so no promise is left dangling on a real timer. */ + releaseHeldRequest(reply: RequestedReply): void { + const requestId = this.heldRequestId + this.heldRequestId = null + if (requestId !== null) { + this.replyToRequest(requestId, reply) + } + } + + private replyToRequest(requestId: number, reply: RequestedReply): void { + if (reply.kind === 'hold') { + this.heldRequestId = requestId + return + } + if (reply.kind === 'truncated') { + this.sendStart({ requestId, truncated: true, unavailable: reply.unavailable }) + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + return + } + if (reply.kind === 'oversized') { + this.sendStart({ requestId }) + for (let index = 0; index < reply.chunks; index += 1) { + this.send(TerminalStreamOpcode.SnapshotChunk, encodeTerminalStreamText('x'.repeat(512_000))) + } + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + return + } + this.sendStart({ + requestId, + unavailable: 'unavailable' in reply ? reply.unavailable : undefined + }) + if (reply.data.length > 0) { + this.send(TerminalStreamOpcode.SnapshotChunk, encodeTerminalStreamText(reply.data)) + } + this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array()) + } + + private sendStart(meta: { requestId?: number; truncated?: boolean; unavailable?: string }): void { + this.send( + TerminalStreamOpcode.SnapshotStart, + encodeTerminalStreamJson({ + cols: 80, + rows: 24, + seq: meta.truncated ? undefined : this.cursorUnits, + ...meta + }) + ) + } + + private send(opcode: TerminalStreamOpcode, payload: Uint8Array): void { + this.toClient( + encodeTerminalStreamFrame({ opcode, streamId: this.streamId, seq: this.cursorUnits, payload }) + ) + } + + output(text: string): void { + this.cursorUnits += text.length + if (this.dropNextOutput) { + this.dropNextOutput = false + return + } + this.send(TerminalStreamOpcode.Output, encodeTerminalStreamText(text)) + } +} + +describe('remote terminal snapshot outcome reasons', () => { + let server: ScriptedSnapshotServer + + beforeEach(() => { + vi.clearAllMocks() + resetRemoteRuntimeTerminalMultiplexersForTests() + replaceRuntimeEnvironmentRevisions([]) + vi.stubGlobal('window', { + api: { + runtimeEnvironments: { + subscribe: vi.fn(async (_args: unknown, callbacks: SubscribeCallbacks) => { + server = new ScriptedSnapshotServer((bytes) => callbacks.onBinary?.(bytes)) + queueMicrotask(() => callbacks.onResponse({ ok: true, result: { type: 'ready' } })) + return { + unsubscribe: vi.fn(), + sendBinary: (bytes: Uint8Array) => server.receive(bytes) + } + }) + } + } + }) + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + async function subscribeClient(): Promise { + const stream = await getRemoteRuntimeTerminalMultiplexer('env-1').subscribeTerminal({ + terminal: 'terminal-1', + client: { id: 'desktop-1', type: 'desktop' }, + callbacks: { onData: () => {}, onSnapshot: () => {} } + }) + await Promise.resolve() + await Promise.resolve() + return stream + } + + it('reports a real buffer as a snapshot and hands legacy callers the same image', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'buffer', data: 'RESTORED' } + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'snapshot' }, + snapshot: { + data: 'RESTORED', + cols: 80, + rows: 24, + seq: 0, + source: undefined, + pendingEscapeTailAnsi: undefined + } + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toMatchObject({ + data: 'RESTORED' + }) + }) + + it('reports a proven-empty buffer as a snapshot, not as an unavailable reply', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'buffer', data: '' } + + const outcome = await stream.serializeBufferOutcome({ scrollbackRows: 100 }) + expect(outcome.availability).toEqual({ kind: 'snapshot' }) + expect(outcome.snapshot?.data).toBe('') + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toMatchObject({ + data: '' + }) + }) + + it('reports a host that could not serialize as retry-worthy without changing the legacy result', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { + kind: 'buffer', + data: '', + unavailable: 'no-serializable-buffer' + } + + const outcome = await stream.serializeBufferOutcome({ scrollbackRows: 100 }) + expect(outcome.availability).toEqual({ + kind: 'retry-worthy', + cause: 'host-no-serializable-buffer' + }) + // Legacy callers still see the host's (empty) image, exactly as before the reason existed. + expect(outcome.snapshot?.data).toBe('') + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toMatchObject({ + data: '' + }) + }) + + it('reports a pending-output overflow as retry-worthy', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'truncated', unavailable: 'pending-output-overflowed' } + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'retry-worthy', cause: 'host-pending-output-overflowed' }, + snapshot: null + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toBeNull() + }) + + it('maps an old host that truncated without a reason to the explicit legacy case', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'truncated' } + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'unknown-legacy-host' }, + snapshot: null + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toBeNull() + }) + + it('reports a reply past the client replay limit as permanently unavailable', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'oversized', chunks: 5 } + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { + kind: 'permanently-unavailable', + reason: 'exceeds-client-replay-limit' + }, + snapshot: null + }) + }) + + it('reports a concurrent request as retry-worthy while the legacy path still rejects', async () => { + const stream = await subscribeClient() + server.nextRequestedReply = { kind: 'hold' } + const held = stream.serializeBufferOutcome({ scrollbackRows: 100 }) + await Promise.resolve() + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'retry-worthy', cause: 'request-already-in-flight' }, + snapshot: null + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).rejects.toThrow( + 'Remote terminal snapshot already in flight.' + ) + + server.releaseHeldRequest({ kind: 'buffer', data: 'LATE' }) + await expect(held).resolves.toMatchObject({ availability: { kind: 'snapshot' } }) + }) + + it('reports an in-flight resync as retry-worthy instead of an empty answer', async () => { + const stream = await subscribeClient() + // Force a seq gap so the client latches its resync gate before the manual request. + server.holdResyncReplies = true + server.output('aaa') + server.dropNextOutput = true + server.output('bbb') + server.output('ccc') + expect(server.requestIds).toEqual([undefined]) + + await expect(stream.serializeBufferOutcome({ scrollbackRows: 100 })).resolves.toEqual({ + availability: { kind: 'retry-worthy', cause: 'resync-in-flight' }, + snapshot: null + }) + await expect(stream.serializeBuffer({ scrollbackRows: 100 })).resolves.toBeNull() + }) +}) diff --git a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts index 0485eb729..88d5633c3 100644 --- a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts +++ b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts @@ -10,6 +10,10 @@ import { encodeTerminalStreamJson, encodeTerminalStreamText } from '../../../shared/terminal-stream-protocol' +import { + parseTerminalSnapshotUnavailableReason, + type TerminalSnapshotUnavailableReason +} from '../../../shared/terminal-snapshot-unavailability' import { e2eConfig, e2eDisableRemoteTerminalStallRecovery } from '@/lib/e2e-config' import { recordRendererCrashBreadcrumb } from '@/lib/crash-breadcrumb-recorder' import { deliverTerminalDataWithDeferredCredit } from '@/lib/pane-manager/terminal-delivery-credit' @@ -72,6 +76,64 @@ export type RemoteRuntimeMultiplexedTerminalCallbacks = { onTransportClose?: (event: { recoverable: boolean; retryWithBackoff?: boolean }) => void } +export type RemoteRuntimeSnapshotImage = { + data: string + cols: number + rows: number + seq?: number + source?: 'headless' | 'renderer' + pendingEscapeTailAnsi?: string +} + +/** Transient causes the host itself reported: a request reached it and it declined to serialize now. */ +export type RemoteRuntimeSnapshotHostRetryCause = + | 'host-pending-output-overflowed' + | 'host-no-serializable-buffer' + +/** Transient causes decided entirely client-side: no request frame ever reached the host, so it answered nothing. */ +export type RemoteRuntimeSnapshotLocalRetryCause = + | 'resync-in-flight' + | 'stream-detached' + | 'connection-not-ready' + | 'request-already-in-flight' + | 'request-frame-not-sent' + +/** Transient causes: the same request may succeed later, so an absent buffer proves nothing about the pane. */ +export type RemoteRuntimeSnapshotRetryCause = + | RemoteRuntimeSnapshotHostRetryCause + | RemoteRuntimeSnapshotLocalRetryCause + +const HOST_ANSWERED_SNAPSHOT_RETRY_CAUSES = new Set([ + 'host-pending-output-overflowed', + 'host-no-serializable-buffer' +]) + +/** Callers budget host answers separately from local gates; only the former cost the host a request. */ +export function isHostAnsweredSnapshotRetryCause( + cause: RemoteRuntimeSnapshotRetryCause +): cause is RemoteRuntimeSnapshotHostRetryCause { + return HOST_ANSWERED_SNAPSHOT_RETRY_CAUSES.has(cause) +} + +/** Final causes: the host answered and repeating this exact request cannot produce the buffer. */ +export type RemoteRuntimeSnapshotPermanentReason = 'exceeds-client-replay-limit' + +export type RemoteRuntimeSnapshotAvailability = + | { kind: 'snapshot' } + | { kind: 'permanently-unavailable'; reason: RemoteRuntimeSnapshotPermanentReason } + | { kind: 'retry-worthy'; cause: RemoteRuntimeSnapshotRetryCause } + // Why: a pre-`unavailable` host sent an empty reply with no reason; the caller must fall back to its own heuristic. + | { kind: 'unknown-legacy-host' } + +/** + * `availability` is what the reply proves; `snapshot` is the buffer image the host actually sent. + * They are orthogonal so legacy callers can keep reading `snapshot` alone while new callers read the reason. + */ +export type RemoteRuntimeSnapshotOutcome = { + availability: RemoteRuntimeSnapshotAvailability + snapshot: RemoteRuntimeSnapshotImage | null +} + export type RemoteRuntimeMultiplexedTerminal = { streamId: number sendInput: (text: string) => boolean @@ -85,6 +147,10 @@ export type RemoteRuntimeMultiplexedTerminal = { seq?: number source?: 'headless' | 'renderer' } | null> + // Why: same request as serializeBuffer, but keeps the host's reason for an absent buffer instead of collapsing it to null. + serializeBufferOutcome: (opts?: { + scrollbackRows?: number + }) => Promise close: () => void } @@ -132,6 +198,7 @@ type RemoteRuntimeSnapshotInfo = { source?: 'headless' | 'renderer' requestId?: number truncated?: boolean + unavailable?: TerminalSnapshotUnavailableReason // Why: a mid-escape tail the emulator could not serialize; the transport // must write it AFTER the replay reset so the next live chunk completes it // instead of rendering literally (#7329). @@ -140,16 +207,7 @@ type RemoteRuntimeSnapshotInfo = { type RemoteRuntimeSnapshotRequest = { requestId: number - resolve: ( - snapshot: { - data: string - cols: number - rows: number - seq?: number - source?: 'headless' | 'renderer' - pendingEscapeTailAnsi?: string - } | null - ) => void + resolve: (outcome: RemoteRuntimeSnapshotOutcome) => void reject: (error: Error) => void timer: ReturnType } @@ -417,6 +475,7 @@ class RemoteRuntimeTerminalMultiplexer { }, setOutputPaused: (paused) => this.setOutputPaused(state, paused), serializeBuffer: (opts) => this.requestSnapshot(state, opts), + serializeBufferOutcome: (opts) => this.requestSnapshotOutcome(state, opts), close: () => { if (this.streams.get(streamId) === state) { discardOutputAcknowledgements(state) @@ -807,12 +866,15 @@ class RemoteRuntimeTerminalMultiplexer { if (snapshotApplied) { if (matchesPendingRequest) { pendingRequest.resolve({ - data: data ?? '', - cols: info?.cols ?? 80, - rows: info?.rows ?? 24, - seq: info?.seq, - source: info?.source, - pendingEscapeTailAnsi: info?.pendingEscapeTailAnsi + availability: classifySnapshotAvailability(stream.snapshotOverflowed, info), + snapshot: { + data: data ?? '', + cols: info?.cols ?? 80, + rows: info?.rows ?? 24, + seq: info?.seq, + source: info?.source, + pendingEscapeTailAnsi: info?.pendingEscapeTailAnsi + } }) clearPendingSnapshotRequest(stream) } else if (target === 'initial') { @@ -829,7 +891,10 @@ class RemoteRuntimeTerminalMultiplexer { }) } } else if (matchesPendingRequest) { - pendingRequest.resolve(null) + pendingRequest.resolve({ + availability: classifySnapshotAvailability(stream.snapshotOverflowed, info), + snapshot: null + }) clearPendingSnapshotRequest(stream) } clearSnapshot(stream) @@ -997,7 +1062,7 @@ class RemoteRuntimeTerminalMultiplexer { stream.resyncTimer = timer } - private requestSnapshot( + private async requestSnapshot( stream: RemoteRuntimeMultiplexedTerminalState, opts?: { scrollbackRows?: number } ): Promise<{ @@ -1007,16 +1072,34 @@ class RemoteRuntimeTerminalMultiplexer { seq?: number source?: 'headless' | 'renderer' } | null> { - if (this.streams.get(stream.streamId) !== stream || !this.ready || !this.subscription) { - return Promise.resolve(null) + const outcome = await this.requestSnapshotOutcome(stream, opts) + // Why: the concurrent-request guard used to reject before the outcome existed; keep that contract for legacy callers. + if ( + outcome.availability.kind === 'retry-worthy' && + outcome.availability.cause === 'request-already-in-flight' + ) { + throw new Error('Remote terminal snapshot already in flight.') + } + return outcome.snapshot + } + + private requestSnapshotOutcome( + stream: RemoteRuntimeMultiplexedTerminalState, + opts?: { scrollbackRows?: number } + ): Promise { + if (this.streams.get(stream.streamId) !== stream) { + return Promise.resolve(retryWorthySnapshotOutcome('stream-detached')) + } + if (!this.ready || !this.subscription) { + return Promise.resolve(retryWorthySnapshotOutcome('connection-not-ready')) } // Recovery uses an untagged snapshot frame group; callers can retry after // it completes instead of racing another request onto the same frame lane. if (stream.resyncInFlight) { - return Promise.resolve(null) + return Promise.resolve(retryWorthySnapshotOutcome('resync-in-flight')) } if (stream.pendingSnapshotRequest) { - return Promise.reject(new Error('Remote terminal snapshot already in flight.')) + return Promise.resolve(retryWorthySnapshotOutcome('request-already-in-flight')) } const requestId = this.allocateSnapshotRequestId() return new Promise((resolve, reject) => { @@ -1039,7 +1122,7 @@ class RemoteRuntimeTerminalMultiplexer { ) ) { clearPendingSnapshotRequest(stream) - resolve(null) + resolve(retryWorthySnapshotOutcome('request-frame-not-sent')) } }) } @@ -1438,6 +1521,7 @@ function decodeSnapshotInfo( source?: unknown requestId?: unknown truncated?: unknown + unavailable?: unknown pendingEscapeTailAnsi?: unknown }>(payload) if (!raw) { @@ -1450,11 +1534,38 @@ function decodeSnapshotInfo( source: raw.source === 'headless' || raw.source === 'renderer' ? raw.source : undefined, requestId: typeof raw.requestId === 'number' ? raw.requestId : undefined, truncated: raw.truncated === true, + unavailable: parseTerminalSnapshotUnavailableReason(raw.unavailable), pendingEscapeTailAnsi: typeof raw.pendingEscapeTailAnsi === 'string' ? raw.pendingEscapeTailAnsi : undefined } } +function retryWorthySnapshotOutcome( + cause: RemoteRuntimeSnapshotRetryCause +): RemoteRuntimeSnapshotOutcome { + return { availability: { kind: 'retry-worthy', cause }, snapshot: null } +} + +function classifySnapshotAvailability( + clientOverflowed: boolean, + info: RemoteRuntimeSnapshotInfo | null +): RemoteRuntimeSnapshotAvailability { + if (clientOverflowed) { + return { kind: 'permanently-unavailable', reason: 'exceeds-client-replay-limit' } + } + if (info?.unavailable === 'pending-output-overflowed') { + return { kind: 'retry-worthy', cause: 'host-pending-output-overflowed' } + } + if (info?.unavailable === 'no-serializable-buffer') { + return { kind: 'retry-worthy', cause: 'host-no-serializable-buffer' } + } + // Why: a truncated reply with no stated reason can only come from a host that predates `unavailable`. + if (info?.truncated === true) { + return { kind: 'unknown-legacy-host' } + } + return { kind: 'snapshot' } +} + function isTerminalDriverState( value: unknown ): value is { kind: 'idle' } | { kind: 'desktop' } | { kind: 'mobile'; clientId: string } { diff --git a/src/shared/terminal-snapshot-unavailability.ts b/src/shared/terminal-snapshot-unavailability.ts new file mode 100644 index 000000000..6d3ac2677 --- /dev/null +++ b/src/shared/terminal-snapshot-unavailability.ts @@ -0,0 +1,24 @@ +/** + * Why a host answered a requested terminal-buffer snapshot without a usable buffer image. + * + * Sent as the additive `unavailable` field on the SnapshotStart frame. Hosts that predate + * this field omit it, so an absent value means "the host did not say" — never "nothing exists". + * Both current reasons are transient: the host could not answer *now*, not that the pane has + * no retained output. A pane that genuinely has nothing still comes back as a real snapshot + * whose `data` is empty, because the host successfully serialized and found it empty. + */ +export const TERMINAL_SNAPSHOT_UNAVAILABLE_REASONS = [ + // The pending-output buffer overflowed twice while serializing, so the reply was truncated to nothing. + 'pending-output-overflowed', + // No serializer (provider, renderer, headless) produced a buffer for this pty at request time. + 'no-serializable-buffer' +] as const + +export type TerminalSnapshotUnavailableReason = + (typeof TERMINAL_SNAPSHOT_UNAVAILABLE_REASONS)[number] + +export function parseTerminalSnapshotUnavailableReason( + value: unknown +): TerminalSnapshotUnavailableReason | undefined { + return TERMINAL_SNAPSHOT_UNAVAILABLE_REASONS.find((reason) => reason === value) +}