From a0d9505ba515bf85c1c764d39e5eedbeb9d6be24 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Mon, 22 Jun 2026 16:42:20 -0700 Subject: [PATCH] Fix SSH terminal replay artifacts Fix SSH reconnect replay ownership and add a Docker-backed regression harness for terminal replay artifacts. --- .../run-ssh-codex-artifacts-repro-e2e.mjs | 45 +++ package.json | 1 + src/main/ipc/ssh.test.ts | 2 + src/main/providers/ssh-pty-provider.test.ts | 12 + src/main/providers/ssh-pty-provider.ts | 10 + src/main/ssh/ssh-relay-session.test.ts | 66 +++- src/main/ssh/ssh-relay-session.ts | 42 ++- .../terminal-pane/pty-connection.test.ts | 48 +++ .../terminal-pane/pty-connection.ts | 28 +- .../ssh-codex-display-artifacts-repro.spec.ts | 199 ++++++++++++ tests/e2e/ssh-codex-real-remote.ts | 59 ++++ .../e2e/ssh-codex-reconnect-replay-driver.ts | 234 +++++++++++++++ tests/e2e/ssh-codex-repro-remote-fixtures.ts | 233 ++++++++++++++ tests/e2e/ssh-codex-terminal-observers.ts | 147 +++++++++ .../e2e/terminal-raster-artifact-analysis.ts | 283 ++++++++++++++++++ tests/e2e/terminal-repro-evidence.ts | 19 ++ tests/e2e/terminal-webgl-reset-capture.ts | 30 ++ 17 files changed, 1446 insertions(+), 12 deletions(-) create mode 100644 config/scripts/run-ssh-codex-artifacts-repro-e2e.mjs create mode 100644 tests/e2e/ssh-codex-display-artifacts-repro.spec.ts create mode 100644 tests/e2e/ssh-codex-real-remote.ts create mode 100644 tests/e2e/ssh-codex-reconnect-replay-driver.ts create mode 100644 tests/e2e/ssh-codex-repro-remote-fixtures.ts create mode 100644 tests/e2e/ssh-codex-terminal-observers.ts create mode 100644 tests/e2e/terminal-raster-artifact-analysis.ts create mode 100644 tests/e2e/terminal-repro-evidence.ts create mode 100644 tests/e2e/terminal-webgl-reset-capture.ts diff --git a/config/scripts/run-ssh-codex-artifacts-repro-e2e.mjs b/config/scripts/run-ssh-codex-artifacts-repro-e2e.mjs new file mode 100644 index 000000000..a4e3fec42 --- /dev/null +++ b/config/scripts/run-ssh-codex-artifacts-repro-e2e.mjs @@ -0,0 +1,45 @@ +import { spawnSync } from 'node:child_process' + +const extraArgs = process.argv.slice(2) +const pnpm = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm' +const env = { + ...process.env, + ORCA_E2E_SSH_DOCKER: '1' +} + +const runtime = spawnSync(pnpm, ['run', 'ensure:electron-runtime'], { + stdio: 'inherit', + env +}) + +if (runtime.error) { + throw runtime.error +} +if (runtime.status !== 0) { + process.exit(runtime.status ?? 1) +} + +const result = spawnSync( + pnpm, + [ + 'exec', + 'playwright', + 'test', + 'tests/e2e/ssh-codex-display-artifacts-repro.spec.ts', + '--config', + 'tests/playwright.config.ts', + '--project', + 'electron-headless', + '--workers=1', + ...extraArgs + ], + { + stdio: 'inherit', + env + } +) + +if (result.error) { + throw result.error +} +process.exit(result.status ?? 1) diff --git a/package.json b/package.json index 14869d56e..9ab4f4fef 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "test:e2e:terminal-perf:check-report": "node config/scripts/check-terminal-perf-report-budgets.mjs", "test:e2e:terminal-perf:summarize": "node config/scripts/summarize-terminal-perf-report.mjs", "test:e2e:ssh-docker-perf": "node config/scripts/run-ssh-docker-perf-e2e.mjs", + "test:e2e:ssh-codex-artifacts-repro": "node config/scripts/run-ssh-codex-artifacts-repro-e2e.mjs", "test:e2e:headful": "pnpm run ensure:electron-runtime && npx playwright test --config tests/playwright.config.ts --project electron-headful", "test:e2e:computer": "vitest run --config tests/e2e/vitest.config.ts", "bench:idle-cpu": "pnpm run ensure:electron-runtime && node config/scripts/run-idle-cpu-benchmark.mjs" diff --git a/src/main/ipc/ssh.test.ts b/src/main/ipc/ssh.test.ts index 148d53271..00a7bf20d 100644 --- a/src/main/ipc/ssh.test.ts +++ b/src/main/ipc/ssh.test.ts @@ -57,6 +57,7 @@ const { onExit: vi.fn(), onReplay: vi.fn(), attach: vi.fn(), + attachForReconnect: vi.fn().mockResolvedValue({}), shutdown: vi.fn() }, mockFsProvider: {}, @@ -285,6 +286,7 @@ describe('SSH IPC handlers', () => { mockPtyProvider.onData.mockReset() mockPtyProvider.onExit.mockReset() mockPtyProvider.onReplay.mockReset() + mockPtyProvider.attachForReconnect.mockReset().mockResolvedValue({}) mockPtyProvider.shutdown.mockReset() mockPortForwardManager.addForward.mockReset() mockPortForwardManager.updateForward.mockReset() diff --git a/src/main/providers/ssh-pty-provider.test.ts b/src/main/providers/ssh-pty-provider.test.ts index ba4a47be8..9de9aa20b 100644 --- a/src/main/providers/ssh-pty-provider.test.ts +++ b/src/main/providers/ssh-pty-provider.test.ts @@ -227,6 +227,18 @@ describe('SshPtyProvider', () => { expect(mux.request).toHaveBeenCalledWith('pty.attach', { id: 'pty-1' }) }) + it('attachForReconnect returns replay without relay notification', async () => { + mux.request.mockResolvedValue({ replay: 'restored output' }) + + const result = await provider.attachForReconnect(scopedPty1) + + expect(result).toEqual({ replay: 'restored output' }) + expect(mux.request).toHaveBeenCalledWith('pty.attach', { + id: 'pty-1', + suppressReplayNotification: true + }) + }) + it('write sends pty.data notification', () => { provider.write(scopedPty1, 'hello') expect(mux.notify).toHaveBeenCalledWith('pty.data', { id: 'pty-1', data: 'hello' }) diff --git a/src/main/providers/ssh-pty-provider.ts b/src/main/providers/ssh-pty-provider.ts index 327bbb9d7..56c4b94c7 100644 --- a/src/main/providers/ssh-pty-provider.ts +++ b/src/main/providers/ssh-pty-provider.ts @@ -177,6 +177,16 @@ export class SshPtyProvider implements IPtyProvider { await this.mux.request('pty.attach', { id: this.toRelayPtyId(id) }) } + async attachForReconnect(id: string): Promise<{ replay?: string }> { + // Why: reconnect owns replay delivery so stale/duplicate attach results can + // be filtered before they reach the renderer. + const result = (await this.mux.request('pty.attach', { + id: this.toRelayPtyId(id), + suppressReplayNotification: true + })) as { replay?: string } | undefined + return result ?? {} + } + write(id: string, data: string): void { this.mux.notify('pty.data', { id: this.toRelayPtyId(id), data }) } diff --git a/src/main/ssh/ssh-relay-session.test.ts b/src/main/ssh/ssh-relay-session.test.ts index 66c865022..9131375d5 100644 --- a/src/main/ssh/ssh-relay-session.test.ts +++ b/src/main/ssh/ssh-relay-session.test.ts @@ -48,6 +48,7 @@ vi.mock('../providers/ssh-pty-provider', () => ({ onReplay = vi.fn().mockReturnValue(() => {}) onExit = vi.fn().mockReturnValue(() => {}) attach = vi.fn().mockResolvedValue(undefined) + attachForReconnect = vi.fn().mockResolvedValue({}) dispose = vi.fn() } })) @@ -67,7 +68,8 @@ vi.mock('../ipc/pty', () => ({ unregisterSshPtyProvider: vi.fn(), getSshPtyProvider: vi.fn().mockReturnValue({ dispose: vi.fn(), - attach: vi.fn().mockResolvedValue(undefined) + attach: vi.fn().mockResolvedValue(undefined), + attachForReconnect: vi.fn().mockResolvedValue({}) }), getPtyIdsForConnection: vi.fn().mockReturnValue([]), clearPtyOwnershipForConnection: vi.fn(), @@ -355,7 +357,7 @@ describe('SshRelaySession', () => { const { getSshPtyProvider } = await import('../ipc/pty') const mockAttach = vi.fn().mockResolvedValue(undefined) vi.mocked(getSshPtyProvider).mockReturnValue({ - attach: mockAttach, + attachForReconnect: mockAttach, dispose: vi.fn() } as unknown as ReturnType) vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-1', 'pty-2']) @@ -366,12 +368,60 @@ describe('SshRelaySession', () => { expect(mockAttach).toHaveBeenCalledWith('pty-2') }) + it('forwards reconnect replay after the attach attempt is still current', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + vi.clearAllMocks() + mockDeploySuccess() + + const { getSshPtyProvider } = await import('../ipc/pty') + const mockAttach = vi.fn().mockResolvedValue({ replay: 'restored-output' }) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect: mockAttach, + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-1']) + + await session.reconnect(mockConn) + + expect(mockWindow.webContents.send).toHaveBeenCalledWith('pty:replay', { + id: 'ssh:target-1@@pty-1', + data: 'restored-output' + }) + }) + + it('drops identical reconnect replay payloads inside one reconnect burst', async () => { + const { mockConn, mockStore, mockPortForward, getMainWindow, mockWindow } = createMockDeps() + const session = new SshRelaySession('target-1', getMainWindow, mockStore, mockPortForward) + await session.establish(mockConn) + vi.clearAllMocks() + mockDeploySuccess() + + const { getSshPtyProvider } = await import('../ipc/pty') + const mockAttach = vi.fn().mockResolvedValue({ replay: 'same-output' }) + vi.mocked(getSshPtyProvider).mockReturnValue({ + attachForReconnect: mockAttach, + dispose: vi.fn() + } as unknown as ReturnType) + vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-1']) + + await session.reconnect(mockConn) + await session.reconnect(mockConn) + + const replaySends = vi + .mocked(mockWindow.webContents.send) + .mock.calls.filter(([channel]) => channel === 'pty:replay') + expect(mockAttach).toHaveBeenCalledTimes(2) + expect(replaySends).toHaveLength(1) + }) + it('establish re-attaches owned PTYs after explicit disconnect', async () => { const { mockConn, mockStore, mockPortForward, getMainWindow } = createMockDeps() const { getSshPtyProvider } = await import('../ipc/pty') const mockAttach = vi.fn().mockResolvedValue(undefined) vi.mocked(getSshPtyProvider).mockReturnValue({ - attach: mockAttach, + attachForReconnect: mockAttach, dispose: vi.fn() } as unknown as ReturnType) vi.mocked(getPtyIdsForConnection).mockReturnValue(['ssh:target-1@@pty-1']) @@ -390,7 +440,7 @@ describe('SshRelaySession', () => { const { getSshPtyProvider } = await import('../ipc/pty') const mockAttach = vi.fn().mockResolvedValue(undefined) vi.mocked(getSshPtyProvider).mockReturnValue({ - attach: mockAttach, + attachForReconnect: mockAttach, dispose: vi.fn() } as unknown as ReturnType) vi.mocked(getPtyIdsForConnection).mockReturnValue([]) @@ -419,7 +469,7 @@ describe('SshRelaySession', () => { }) ) vi.mocked(getSshPtyProvider).mockReturnValue({ - attach: mockAttach, + attachForReconnect: mockAttach, dispose: vi.fn() } as unknown as ReturnType) vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-1']) @@ -454,7 +504,7 @@ describe('SshRelaySession', () => { }) ) vi.mocked(getSshPtyProvider).mockReturnValue({ - attach: mockAttach, + attachForReconnect: mockAttach, dispose: vi.fn() } as unknown as ReturnType) vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-1']) @@ -486,7 +536,7 @@ describe('SshRelaySession', () => { .mockRejectedValueOnce(new Error('PTY "pty-stale" not found')) .mockResolvedValueOnce(undefined) vi.mocked(getSshPtyProvider).mockReturnValue({ - attach: mockAttach, + attachForReconnect: mockAttach, dispose: vi.fn() } as unknown as ReturnType) vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-stale', 'pty-live']) @@ -515,7 +565,7 @@ describe('SshRelaySession', () => { const { getSshPtyProvider } = await import('../ipc/pty') const mockAttach = vi.fn().mockRejectedValue(new Error('Multiplexer disposed')) vi.mocked(getSshPtyProvider).mockReturnValue({ - attach: mockAttach, + attachForReconnect: mockAttach, dispose: vi.fn() } as unknown as ReturnType) vi.mocked(getPtyIdsForConnection).mockReturnValue(['pty-live']) diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index ecbf8b280..d124aca82 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -73,6 +73,14 @@ type RemoteCliBridgeEnv = { pathDelimiter?: ':' | ';' } +type ForwardedReplayFingerprint = { + fingerprint: string + deliveredAt: number +} + +const RECONNECT_REPLAY_DUPLICATE_WINDOW_MS = 1000 +const REPLAY_FINGERPRINT_EDGE_CHARS = 128 + function normalizeRelayGracePeriodSeconds(graceTimeSeconds: number | undefined): number { const raw = graceTimeSeconds ?? DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS const requested = Number.isFinite(raw) ? Math.floor(raw) : DEFAULT_SSH_RELAY_GRACE_PERIOD_SECONDS @@ -111,6 +119,7 @@ export class SshRelaySession { private currentConnection: SshConnection | null = null private hostPlatform: RemoteHostPlatform | null = null private remoteCliBridgeEnv: RemoteCliBridgeEnv | null = null + private forwardedReattachReplayByPty = new Map() constructor( readonly targetId: string, @@ -950,6 +959,7 @@ export class SshRelaySession { const relayPtyId = toRelaySshPtyId(this.targetId, payload.id) clearProviderPtyState(payload.id) deletePtyOwnership(payload.id) + this.forwardedReattachReplayByPty.delete(payload.id) this.store.markSshRemotePtyLease(this.targetId, relayPtyId, 'terminated') this.runtime?.onPtyExit(payload.id, payload.code) const win = this.getMainWindow() @@ -959,6 +969,34 @@ export class SshRelaySession { }) } + private replayFingerprint(data: string): string { + const head = data.slice(0, REPLAY_FINGERPRINT_EDGE_CHARS) + const tail = data.slice(-REPLAY_FINGERPRINT_EDGE_CHARS) + return `${data.length}:${head}:${tail}` + } + + private shouldForwardReattachReplay(appPtyId: string, data: string): boolean { + const now = Date.now() + const fingerprint = this.replayFingerprint(data) + const previous = this.forwardedReattachReplayByPty.get(appPtyId) + this.forwardedReattachReplayByPty.set(appPtyId, { fingerprint, deliveredAt: now }) + return ( + !previous || + previous.fingerprint !== fingerprint || + now - previous.deliveredAt > RECONNECT_REPLAY_DUPLICATE_WINDOW_MS + ) + } + + private forwardReattachReplay(appPtyId: string, data: string): void { + if (!data || !this.shouldForwardReattachReplay(appPtyId, data)) { + return + } + const win = this.getMainWindow() + if (win && !win.isDestroyed()) { + win.webContents.send('pty:replay', { id: appPtyId, data }) + } + } + private async reattachKnownPtys(shouldContinue: () => boolean): Promise { const leasedPtyIds = this.store .getSshRemotePtyLeases(this.targetId) @@ -983,13 +1021,14 @@ export class SshRelaySession { return } try { - await ptyProvider.attach(ptyId) + const attachResult = (await ptyProvider.attachForReconnect(ptyId)) ?? {} if (!shouldContinue()) { return } const appPtyId = toAppSshPtyId(this.targetId, ptyId) setPtyOwnership(appPtyId, this.targetId) this.store.markSshRemotePtyLease(this.targetId, ptyId, 'attached') + this.forwardReattachReplay(appPtyId, attachResult.replay ?? '') } catch (err) { if (!isSshPtyNotFoundError(err)) { throw err @@ -1002,6 +1041,7 @@ export class SshRelaySession { const appPtyId = toAppSshPtyId(this.targetId, ptyId) clearProviderPtyState(appPtyId) deletePtyOwnership(appPtyId) + this.forwardedReattachReplayByPty.delete(appPtyId) this.store.markSshRemotePtyLease(this.targetId, ptyId, 'expired') // Why: if the new relay cannot reattach this id, the remote backing // process is gone. Tell the renderer so it clears stale pane bindings diff --git a/src/renderer/src/components/terminal-pane/pty-connection.test.ts b/src/renderer/src/components/terminal-pane/pty-connection.test.ts index af5821fdb..ab99e7b28 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.test.ts @@ -6383,6 +6383,54 @@ describe('connectPanePty', () => { disposable.dispose() }) + it('coalesces remote replay payloads that overlap before parsing starts', async () => { + const { connectPanePty } = await import('./pty-connection') + enableActiveRuntimeEnvironment() + const transport = createMockTransport('remote:env-1@@terminal-1') + const capturedReplayCallback: { + current: ((data: string) => void) | null + } = { current: null } + transport.connect.mockImplementation(async ({ callbacks }: { callbacks: ConnectCallbacks }) => { + capturedReplayCallback.current = callbacks.onReplayData ?? null + return { id: 'remote:env-1@@terminal-1', replay: '' } + }) + transportFactoryQueue.push(transport) + + const pane = createPane(1) + const pendingParses: (() => void)[] = [] + pane.terminal.write = vi.fn((_data: string, callback?: () => void) => { + if (callback) { + pendingParses.push(callback) + } + }) + const manager = createManager(1) + const deps = createDeps() + const disposable = connectPanePty(pane as never, manager as never, deps as never) + await flushAsyncTicks(6) + + capturedReplayCallback.current?.('first replay') + capturedReplayCallback.current?.('second replay') + await flushAsyncTicks(2) + + expect(pane.terminal.write).toHaveBeenCalledTimes(1) + expect(pane.terminal.write).toHaveBeenNthCalledWith( + 1, + '\x1b[2J\x1b[3J\x1b[H', + expect.any(Function) + ) + + for (let index = 0; index < 8; index += 1) { + await flushAsyncTicks(2) + pendingParses.shift()?.() + } + await flushAsyncTicks(4) + + expect(pane.terminal.write).not.toHaveBeenCalledWith('first replay', expect.any(Function)) + expect(pane.terminal.write).toHaveBeenCalledWith('second replay', expect.any(Function)) + expect(manager.rebuildPaneWebgl).toHaveBeenCalledTimes(1) + disposable.dispose() + }) + it('does not switch renderers for Arabic output', async () => { const { connectPanePty } = await import('./pty-connection') const transport = createMockTransport() diff --git a/src/renderer/src/components/terminal-pane/pty-connection.ts b/src/renderer/src/components/terminal-pane/pty-connection.ts index 70d9edcc3..5807981a5 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.ts @@ -2491,8 +2491,13 @@ export function connectPanePty( return replayIntoTerminalAsync(pane, deps.replayingPanesRef, data) } - const replayDataCallback = (data: string): void => { - void (async () => { + let replayWriteQueue = Promise.resolve() + let pendingReplayData: string | null = null + let replayDrainQueued = false + const drainReplayDataQueue = async (): Promise => { + while (pendingReplayData !== null) { + const data = pendingReplayData + pendingReplayData = null // Relay replay buffer holds the last 100 KB of output, which may // overlap with content already rendered in xterm before the // disconnect. Clear first to prevent duplication on SSH reconnect. @@ -2500,13 +2505,30 @@ export function connectPanePty( await writeReplayDataAsync(data) await writeReplayDataAsync(POST_REPLAY_REATTACH_RESET) if (disposed) { + pendingReplayData = null return } // Why: remote-runtime snapshots can arrive after WebGL attached to an // empty buffer; rebuilding after replay parses seeds the glyph atlas // from the now-populated xterm state. manager.rebuildPaneWebgl(pane.id) - })() + } + } + const replayDataCallback = (data: string): void => { + pendingReplayData = data + if (replayDrainQueued) { + return + } + replayDrainQueued = true + replayWriteQueue = replayWriteQueue + .catch(() => undefined) + .then(drainReplayDataQueue) + .finally(() => { + replayDrainQueued = false + if (pendingReplayData !== null) { + replayDataCallback(pendingReplayData) + } + }) } type PendingHiddenOutputRestoreChunk = { diff --git a/tests/e2e/ssh-codex-display-artifacts-repro.spec.ts b/tests/e2e/ssh-codex-display-artifacts-repro.spec.ts new file mode 100644 index 000000000..e19a090df --- /dev/null +++ b/tests/e2e/ssh-codex-display-artifacts-repro.spec.ts @@ -0,0 +1,199 @@ +import type { TestInfo } from '@stablyai/playwright-test' +import { test, expect } from './helpers/orca-app' +import { + ensureTerminalVisible, + switchToWorktree, + waitForActiveWorktree, + waitForSessionReady +} from './helpers/store' +import { + execInTerminal, + waitForActivePanePtyId, + waitForActiveTerminalManager, + waitForTerminalOutput +} from './helpers/terminal' +import { + cleanupDockerSshRelayTarget, + startDockerSshRelayTarget, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' +import { + REMOTE_CODEX_FIXTURE_CLEAN_FINAL_TEXT, + REMOTE_TUI_DONE, + installRemoteCodexArtifactTui, + installRemoteCodexFixture, + shellQuote +} from './ssh-codex-repro-remote-fixtures' +import { + connectDockerRemote, + dropDockerSshClientSessions, + enableRiskyTerminalRendererPath, + installPtyReplayProbe, + readDuplicateStatusRows, + readReplayProbeSnapshot, + switchToNonRemoteWorktree, + waitForDockerRemoteReconnected +} from './ssh-codex-reconnect-replay-driver' +import { installRemoteRealCodex, realRemoteCodexCommand } from './ssh-codex-real-remote' +import { + clearRemoteTerminalAfterCodex, + scrollActiveTerminalToArtifactHistory, + stressRestoreRemoteTerminalDuringCodex, + waitForRealRemoteCodexCompletion, + waitForRemoteFixtureCleanFinalInHiddenPane +} from './ssh-codex-terminal-observers' +import { MAX_FINAL_GRAY_SLABS, captureGraySlabAnalysis } from './terminal-raster-artifact-analysis' +import { persistReproEvidence } from './terminal-repro-evidence' +import { resetWebglAndCaptureGraySlabAnalysis } from './terminal-webgl-reset-capture' + +const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1' +const RUN_REAL_REMOTE_CODEX = process.env.ORCA_E2E_REAL_REMOTE_CODEX === '1' +const EXPECT_NO_ARTIFACTS = process.env.ORCA_E2E_EXPECT_NO_CODEX_ARTIFACTS === '1' +const CAPTURE_WHILE_REMOTE_TUI_RUNNING = + process.env.ORCA_E2E_CAPTURE_WHILE_REMOTE_TUI_RUNNING === '1' +const HIDE_UNTIL_REMOTE_TUI_DONE = process.env.ORCA_E2E_HIDE_UNTIL_REMOTE_TUI_DONE === '1' +const CAPTURE_SCROLLBACK_ARTIFACT_REGION = + process.env.ORCA_E2E_CAPTURE_SCROLLBACK_ARTIFACT_REGION === '1' +const FORCE_SSH_RECONNECT_DURING_TUI = process.env.ORCA_E2E_FORCE_SSH_RECONNECT_DURING_TUI === '1' +const KEEP_SSH_REPRO_TARGET = process.env.ORCA_E2E_KEEP_SSH_REPRO_TARGET === '1' + +test.describe('Remote SSH Codex display artifacts repro', () => { + test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker-backed SSH repro.') + test.skip(process.platform === 'win32', 'Docker SSH repro uses POSIX ssh tooling.') + + test('does not leave duplicated Codex status output after SSH replay', async ({ + orcaPage + }, testInfo: TestInfo) => { + test.slow() + let target: DockerSshRelayTarget | null = null + try { + target = startDockerSshRelayTarget(testInfo) + installRemoteCodexArtifactTui(target) + if (RUN_REAL_REMOTE_CODEX) { + installRemoteRealCodex(target) + } else { + installRemoteCodexFixture(target) + } + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + const remote = await connectDockerRemote(orcaPage, target) + expect(remote.targetId).toBeTruthy() + expect(remote.worktreeId).toBeTruthy() + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 60_000) + await enableRiskyTerminalRendererPath(orcaPage) + await installPtyReplayProbe(orcaPage) + + const ptyId = await waitForActivePanePtyId(orcaPage, 60_000) + const doneMarker = RUN_REAL_REMOTE_CODEX + ? `ORCA_REAL_REMOTE_CODEX_DONE_${Date.now()}` + : REMOTE_TUI_DONE + const cleanMarker = RUN_REAL_REMOTE_CODEX + ? `ORCA_REAL_REMOTE_CODEX_CLEAN_${Date.now()}` + : doneMarker + await execInTerminal( + orcaPage, + ptyId, + RUN_REAL_REMOTE_CODEX + ? realRemoteCodexCommand(doneMarker) + : `codex --no-alt-screen --dangerously-bypass-approvals-and-sandbox ${shellQuote( + doneMarker + )}` + ) + await orcaPage.waitForTimeout(1_200) + if (FORCE_SSH_RECONNECT_DURING_TUI) { + dropDockerSshClientSessions(target) + await waitForDockerRemoteReconnected(orcaPage, remote.targetId) + await orcaPage.waitForTimeout(2_000) + } + await (RUN_REAL_REMOTE_CODEX + ? (async () => { + await stressRestoreRemoteTerminalDuringCodex(orcaPage, remote.worktreeId) + await waitForRealRemoteCodexCompletion(orcaPage, doneMarker) + })() + : (async () => { + if (CAPTURE_WHILE_REMOTE_TUI_RUNNING) { + await orcaPage.waitForTimeout(10_000) + } else { + await switchToNonRemoteWorktree(orcaPage, remote.worktreeId) + await (HIDE_UNTIL_REMOTE_TUI_DONE + ? waitForRemoteFixtureCleanFinalInHiddenPane(orcaPage, remote.worktreeId) + : orcaPage.waitForTimeout(10_000)) + } + if (CAPTURE_WHILE_REMOTE_TUI_RUNNING) { + await orcaPage.waitForTimeout(900) + return + } + await switchToWorktree(orcaPage, remote.worktreeId) + await ensureTerminalVisible(orcaPage, 45_000) + await waitForActiveTerminalManager(orcaPage, 60_000) + await waitForTerminalOutput( + orcaPage, + REMOTE_CODEX_FIXTURE_CLEAN_FINAL_TEXT, + 60_000, + 120_000 + ) + })()) + await orcaPage.waitForTimeout(600) + if (CAPTURE_SCROLLBACK_ARTIFACT_REGION) { + await scrollActiveTerminalToArtifactHistory(orcaPage) + } + + const { analysis, screenshot } = await captureGraySlabAnalysis(orcaPage) + analysis.replayDebug = await readReplayProbeSnapshot(orcaPage) + analysis.duplicateStatusRows = await readDuplicateStatusRows(orcaPage) + const evidenceLabel = RUN_REAL_REMOTE_CODEX + ? 'real-remote-codex-reconnect-replay' + : 'fixture-codex-reconnect-replay' + persistReproEvidence(evidenceLabel, analysis, screenshot) + const resetEvidence = await resetWebglAndCaptureGraySlabAnalysis(orcaPage) + resetEvidence.analysis.replayDebug = await readReplayProbeSnapshot(orcaPage) + resetEvidence.analysis.duplicateStatusRows = await readDuplicateStatusRows(orcaPage) + persistReproEvidence( + `${evidenceLabel}-after-webgl-reset`, + resetEvidence.analysis, + resetEvidence.screenshot + ) + await testInfo.attach('remote-codex-artifact-final-screen', { + body: screenshot, + contentType: 'image/png' + }) + await testInfo.attach('remote-codex-artifact-after-webgl-reset', { + body: resetEvidence.screenshot, + contentType: 'image/png' + }) + testInfo.annotations.push({ + type: 'remote-codex-artifact-analysis', + description: JSON.stringify(analysis) + }) + testInfo.annotations.push({ + type: 'remote-codex-artifact-after-webgl-reset-analysis', + description: JSON.stringify(resetEvidence.analysis) + }) + + // Why: this spec supports both repro mode and strict regression mode so + // the same harness can prove a failure and lock the fixed behavior. + if (EXPECT_NO_ARTIFACTS) { + expect(analysis.slabCount).toBeLessThanOrEqual(MAX_FINAL_GRAY_SLABS) + expect(analysis.staleStatusGlyphRowCount).toBe(0) + expect(analysis.duplicateStatusRows ?? []).toEqual([]) + } else { + expect(analysis.rawSlabCount + analysis.staleStatusGlyphRowCount).toBeGreaterThan(0) + } + if (FORCE_SSH_RECONNECT_DURING_TUI) { + expect(Number(analysis.replayDebug?.replayCount ?? 0)).toBeGreaterThan(0) + } + if (RUN_REAL_REMOTE_CODEX) { + await clearRemoteTerminalAfterCodex(orcaPage, ptyId, cleanMarker) + } + } finally { + if (KEEP_SSH_REPRO_TARGET && target) { + console.log( + `[ssh-codex-repro] keeping Docker SSH target ${target.containerName} on port ${target.port}` + ) + } else { + cleanupDockerSshRelayTarget(target) + } + } + }) +}) diff --git a/tests/e2e/ssh-codex-real-remote.ts b/tests/e2e/ssh-codex-real-remote.ts new file mode 100644 index 000000000..60c96eb0d --- /dev/null +++ b/tests/e2e/ssh-codex-real-remote.ts @@ -0,0 +1,59 @@ +import { existsSync, readFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { + DOCKER_SSH_RELAY_REMOTE_REPO_PATH, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' +import { dockerExec, dockerWriteFile, shellQuote } from './ssh-codex-repro-remote-fixtures' + +const REMOTE_CODEX_VERSION = '0.141.0' + +function tomlString(value: string): string { + return value.replaceAll('\\', '\\\\').replaceAll('"', '\\"') +} + +export function installRemoteRealCodex(target: DockerSshRelayTarget): void { + const codexAuthPath = path.join(os.homedir(), '.codex', 'auth.json') + if (!existsSync(codexAuthPath)) { + throw new Error(`Real remote Codex repro needs local auth at ${codexAuthPath}`) + } + dockerExec(target, 'mkdir -p /root/.codex') + dockerWriteFile(target, '/root/.codex/auth.json', readFileSync(codexAuthPath), '600') + const trustedRemotePath = tomlString(DOCKER_SSH_RELAY_REMOTE_REPO_PATH) + dockerWriteFile( + target, + '/root/.codex/config.toml', + [ + 'approval_policy = "never"', + '', + `[projects."${trustedRemotePath}"]`, + 'trust_level = "trusted"', + '' + ].join('\n'), + '600' + ) + dockerExec(target, `npm install -g @openai/codex@${REMOTE_CODEX_VERSION}`, 180_000) +} + +export function realRemoteCodexCommand(doneMarker: string): string { + const prompt = [ + 'This is an automated terminal rendering reproduction.', + 'Run these three shell commands one at a time, waiting for each one to finish before starting the next:', + 'node -e "let i = 0; const timer = setInterval(() => { console.log(\'REMOTE_CODEX_PHASE_0_\' + i); i += 1; if (i >= 100) clearInterval(timer) }, 250)"', + 'node -e "let i = 0; const timer = setInterval(() => { console.log(\'REMOTE_CODEX_PHASE_1_\' + i); i += 1; if (i >= 100) clearInterval(timer) }, 250)"', + 'node -e "let i = 0; const timer = setInterval(() => { console.log(\'REMOTE_CODEX_PHASE_2_\' + i); i += 1; if (i >= 100) clearInterval(timer) }, 250)"', + 'The commands are intentionally slow. Keep waiting until all three complete.', + 'Then briefly summarize that all three commands ran.', + `End your final response with this exact marker: ${doneMarker}` + ].join(' ') + return [ + 'codex', + '--no-alt-screen', + '--dangerously-bypass-approvals-and-sandbox', + '--dangerously-bypass-hook-trust', + '-C', + shellQuote(DOCKER_SSH_RELAY_REMOTE_REPO_PATH), + shellQuote(prompt) + ].join(' ') +} diff --git a/tests/e2e/ssh-codex-reconnect-replay-driver.ts b/tests/e2e/ssh-codex-reconnect-replay-driver.ts new file mode 100644 index 000000000..75fc21a47 --- /dev/null +++ b/tests/e2e/ssh-codex-reconnect-replay-driver.ts @@ -0,0 +1,234 @@ +import { execFileSync } from 'node:child_process' +import type { Page } from '@stablyai/playwright-test' +import { expect } from './helpers/orca-app' +import { + DOCKER_SSH_RELAY_REMOTE_REPO_PATH, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' + +export type ConnectedDockerRemote = { + targetId: string + worktreeId: string +} + +export function dropDockerSshClientSessions(target: DockerSshRelayTarget): void { + execFileSync( + 'docker', + [ + 'exec', + target.containerName, + 'bash', + '-lc', + `ps -eo pid=,comm=,args= | awk '$2 == "sshd" && index($0, "sshd: root") { print $1 }' | xargs -r kill -9` + ], + { stdio: ['ignore', 'pipe', 'pipe'], timeout: 60_000 } + ) +} + +export async function connectDockerRemote( + page: Page, + target: DockerSshRelayTarget +): Promise { + return await page.evaluate( + async ({ target, remotePath }) => { + const store = window.__store + if (!store) { + throw new Error('Store unavailable') + } + const credentialUnsub = window.api.ssh.onCredentialRequest((request) => { + void window.api.ssh.submitCredential({ requestId: request.requestId, value: null }) + }) + try { + const createdTarget = await window.api.ssh.addTarget({ + target: { + label: `Docker SSH Codex Artifact Repro ${Date.now()}`, + host: '127.0.0.1', + port: target.port, + username: 'root', + identityFile: target.identityFile, + identitiesOnly: true, + relayGracePeriodSeconds: 1 + } + }) + const state = await window.api.ssh.connect({ targetId: createdTarget.id }) + if (!state || state.status !== 'connected') { + throw new Error(`SSH target did not connect: ${JSON.stringify(state)}`) + } + store.getState().setSshConnectionState(createdTarget.id, state) + const labels = new Map(store.getState().sshTargetLabels) + labels.set(createdTarget.id, createdTarget.label) + store.getState().setSshTargetLabels(labels) + + const result = await window.api.repos.addRemote({ + connectionId: createdTarget.id, + remotePath, + displayName: 'Docker SSH Codex Artifact Repro' + }) + if ('error' in result) { + throw new Error(result.error) + } + await store.getState().fetchRepos() + await store.getState().fetchWorktrees(result.repo.id) + const worktree = (store.getState().worktreesByRepo[result.repo.id] ?? [])[0] + if (!worktree) { + throw new Error(`No remote worktree found for ${result.repo.path}`) + } + store.getState().setActiveWorktree(worktree.id) + if ((store.getState().tabsByWorktree[worktree.id] ?? []).length === 0) { + store.getState().createTab(worktree.id) + } + store.getState().setActiveTabType('terminal') + return { targetId: createdTarget.id, worktreeId: worktree.id } + } finally { + credentialUnsub() + } + }, + { target, remotePath: DOCKER_SSH_RELAY_REMOTE_REPO_PATH } + ) +} + +export async function switchToNonRemoteWorktree( + page: Page, + remoteWorktreeId: string +): Promise { + const otherWorktreeId = await page.evaluate((remoteWorktreeId) => { + const store = window.__store + if (!store) { + return null + } + const state = store.getState() + const other = Object.values(state.worktreesByRepo) + .flat() + .find((worktree) => worktree.id !== remoteWorktreeId) + if (!other) { + return null + } + state.setActiveWorktree(other.id) + return other.id + }, remoteWorktreeId) + if (!otherWorktreeId) { + throw new Error('No non-remote worktree available to hide the SSH terminal') + } + return otherWorktreeId +} + +export async function installPtyReplayProbe(page: Page): Promise { + await page.evaluate(() => { + const api = window.api?.pty + if (!api || typeof api.onReplay !== 'function') { + throw new Error('PTY replay API unavailable') + } + const holder = window as unknown as { + __orcaSshCodexReplayProbe?: { + payloads: { id: string; length: number; preview: string }[] + dispose: () => void + } + } + holder.__orcaSshCodexReplayProbe?.dispose() + const payloads: { id: string; length: number; preview: string }[] = [] + const dispose = api.onReplay(({ id, data }) => { + payloads.push({ + id, + length: data.length, + preview: data.slice(-400) + }) + }) + holder.__orcaSshCodexReplayProbe = { payloads, dispose } + }) +} + +export async function waitForDockerRemoteReconnected(page: Page, targetId: string): Promise { + let observedNonConnected = false + await expect + .poll( + async () => { + const status = await page.evaluate((targetId) => { + const state = window.__store?.getState().sshConnectionStates.get(targetId) + return state?.status ?? null + }, targetId) + if (status !== 'connected') { + observedNonConnected = true + } + return observedNonConnected && status === 'connected' + }, + { + timeout: 90_000, + message: 'Docker SSH target did not auto-reconnect after transport drop' + } + ) + .toBe(true) +} + +export async function readReplayProbeSnapshot(page: Page): Promise> { + return page.evaluate(() => { + const probe = ( + window as unknown as { + __orcaSshCodexReplayProbe?: { + payloads: { id: string; length: number; preview: string }[] + } + } + ).__orcaSshCodexReplayProbe + return { + replayCount: probe?.payloads.length ?? 0, + replayPayloads: probe?.payloads.slice(-8) ?? [] + } + }) +} + +export async function readDuplicateStatusRows(page: Page): Promise { + return page.evaluate(() => { + const state = window.__store?.getState() + const worktreeId = state?.activeWorktreeId + const tabId = + state?.activeTabType === 'terminal' + ? state.activeTabId + : worktreeId + ? (state?.activeTabIdByWorktree?.[worktreeId] ?? null) + : null + const manager = tabId ? window.__paneManagers?.get(tabId) : null + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + const text = pane?.serializeAddon?.serialize?.() ?? '' + const counts = new Map() + const escapeSequencePattern = new RegExp( + `${String.fromCharCode(27)}\\[[0-9;?]*[ -/]*[@-~]`, + 'g' + ) + for (const line of text.split(/\r?\n/)) { + const normalized = line.replace(escapeSequencePattern, '').trim() + if (!/gpt-5\.5|background terminal|\/ps to view|\/stop to close/i.test(normalized)) { + continue + } + counts.set(normalized, (counts.get(normalized) ?? 0) + 1) + } + return Array.from(counts) + .filter(([, count]) => count > 1) + .map(([line, count]) => `${count}x ${line}`) + .slice(0, 12) + }) +} + +export async function enableRiskyTerminalRendererPath(page: Page): Promise { + await page.evaluate(() => { + const store = window.__store + if (!store) { + throw new Error('window.__store unavailable') + } + const state = store.getState() + store.setState({ + settings: { + ...state.settings!, + terminalGpuAcceleration: 'on', + theme: 'dark' + } + }) + const worktreeId = state.activeWorktreeId + const tabId = + state.activeTabType === 'terminal' + ? state.activeTabId + : worktreeId + ? (state.activeTabIdByWorktree?.[worktreeId] ?? null) + : null + const manager = tabId ? window.__paneManagers?.get(tabId) : null + manager?.setTerminalGpuAcceleration('on') + }) +} diff --git a/tests/e2e/ssh-codex-repro-remote-fixtures.ts b/tests/e2e/ssh-codex-repro-remote-fixtures.ts new file mode 100644 index 000000000..3ee48187e --- /dev/null +++ b/tests/e2e/ssh-codex-repro-remote-fixtures.ts @@ -0,0 +1,233 @@ +import { execFileSync } from 'node:child_process' +import type { DockerSshRelayTarget } from './helpers/docker-ssh-relay-target' + +const REMOTE_TUI_PATH = '/tmp/orca-codex-display-artifacts-repro.mjs' +export const REMOTE_TUI_DONE = 'ORCA_REMOTE_CODEX_ARTIFACT_TUI_DONE' +export const REMOTE_CODEX_FIXTURE_CLEAN_FINAL_TEXT = + 'Any gray slab visible now is stale renderer state.' +const REMOTE_TUI_FRAMES = 900 +const REMOTE_CODEX_FIXTURE_FRAMES = parseEnvNumber(process.env.ORCA_E2E_CODEX_FIXTURE_FRAMES, 34) +const REMOTE_CODEX_FIXTURE_FRAME_DELAY_MS = parseEnvNumber( + process.env.ORCA_E2E_CODEX_FIXTURE_FRAME_DELAY_MS, + 45 +) + +function parseEnvNumber(value: string | undefined, fallback: number): number { + const parsed = Number(value) + return Number.isFinite(parsed) ? parsed : fallback +} + +export function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'` +} + +function remoteCodexArtifactTuiScript(): string { + return ` +const cols = Number(process.env.COLUMNS || 120) +const statusWidth = Math.max(48, Math.min(cols - 4, 104)) +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + +function write(chunk) { + return new Promise((resolve) => process.stdout.write(chunk, resolve)) +} + +function pad(text, width) { + const raw = text.length > width ? text.slice(0, width) : text + return raw + ' '.repeat(Math.max(0, width - raw.length)) +} + +await write('\\x1b]0;codex\\x07') +await write('\\x1b[2J\\x1b[H\\x1b[?25l') + +for (let frame = 0; frame < ${REMOTE_TUI_FRAMES}; frame += 1) { + const statusRow = 8 + (frame % 13) + const priorRow = 8 + ((frame + 12) % 13) + const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧'][frame % 8] + const bands = [ + { row: statusRow, width: statusWidth, text: \`\${spinner} Working for background terminal · frame \${String(frame).padStart(3, '0')}\` }, + { row: 27 + (frame % 9), width: Math.max(32, statusWidth - 18), text: \`gpt-5.5 high · ~/remote/repro/pr-5969 · /ps to view · /stop to close \${frame}\` }, + { row: 40 + (frame % 11), width: Math.max(28, statusWidth - 8), text: \`• Working for background terminal · rtk bun run e2e:ui --filter=@dalp/app \${frame}\` } + ] + await write('\\x1b[?2026h') + await write(\`\\x1b[\${priorRow};1H\\x1b[2K\`) + await write(\`\\x1b[\${27 + ((frame + 8) % 9)};1H\\x1b[2K\`) + await write(\`\\x1b[\${40 + ((frame + 10) % 11)};1H\\x1b[2K\`) + await write('\\x1b[1;1H\\x1b[38;2;142;196;255mgpt-5.5 high\\x1b[0m ') + await write('\\x1b[38;2;106;176;76m~/remote/repro/pr-5969\\x1b[0m ') + await write('/ps to view · /stop to close') + await write('\\x1b[3;1H• Reproducing remote Codex SSH display artifacts with fast status movement') + await write('\\x1b[4;1H• The moving status band intentionally uses gray background during frames') + await write('\\x1b[5;1H• Final screen is clean; any remaining gray slab is stale renderer state') + for (const band of bands) { + await write(\`\\x1b[\${band.row};1H\\x1b[48;2;72;72;72m\${pad('', band.width)}\\x1b[0m\`) + await write(\`\\x1b[\${band.row};3H\\x1b[38;2;220;220;220;48;2;72;72;72m\${pad(band.text, band.width - 4)}\\x1b[0m\`) + } + await write(\`\\x1b[22;1H\\x1b[38;2;106;176;76m+\${' added code '.repeat(8)}\${frame}\\x1b[0m\`) + await write(\`\\x1b[23;1H\\x1b[38;2;230;90;75m-\${' removed code '.repeat(8)}\${frame}\\x1b[0m\`) + await write(\`\\x1b[25;1H\\x1b[38;2;153;199;255m› \${'Run focused and full validation gates '.repeat(3)}\${frame}\\x1b[0m\`) + if (frame % 18 === 0) { + await write(\`\\x1b[52;1H\\x1b[0m• Waited for background terminal · rtk bun run e2e:ui --filter=@dalp/app \${frame}\\r\\n\`) + } + await write('\\x1b[?2026l') + await sleep(8) +} + +await write('\\x1b[?2026h\\x1b[2J\\x1b[H') +await write('\\x1b[38;2;142;196;255mgpt-5.5 high\\x1b[0m ') +await write('\\x1b[38;2;106;176;76m~/remote/repro/pr-5969\\x1b[0m clean final frame\\r\\n\\r\\n') +await write('Final screen intentionally has no gray background bands.\\r\\n') +await write('If Orca leaves wide gray rectangles here, they are stale remote-PTY render artifacts.\\r\\n') +await write('${REMOTE_TUI_DONE}\\r\\n') +await write('\\x1b[?25h\\x1b[?2026l') +setTimeout(() => process.exit(0), 50) +` +} + +export function installRemoteCodexArtifactTui(target: DockerSshRelayTarget): void { + const body = remoteCodexArtifactTuiScript() + dockerWriteFile(target, REMOTE_TUI_PATH, body, '755') +} + +function remoteCodexFixtureScript(): string { + return `#!/usr/bin/env node +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) +const write = (chunk) => new Promise((resolve) => process.stdout.write(chunk, resolve)) +const cols = Number(process.env.COLUMNS || 120) +const rows = Number(process.env.LINES || 48) +const width = Math.max(42, cols - 4) +const marker = process.argv.join(' ').match(/ORCA_REMOTE_CODEX_ARTIFACT_TUI_DONE|ORCA_REAL_REMOTE_CODEX_DONE_[0-9]+/)?.[0] || '${REMOTE_TUI_DONE}' +const scrollTop = 1 +const viewportTop = Math.max(14, rows - 17) +const viewportBottom = Math.max(viewportTop + 8, rows - 3) + +function pad(text, size) { + const raw = text.length > size ? text.slice(0, size) : text + return raw + ' '.repeat(Math.max(0, size - raw.length)) +} + +async function grayLine(row, text) { + await write(\`\\x1b[\${row};1H\\x1b[48;2;72;72;72m\${pad('', width)}\\x1b[0m\`) + await write(\`\\x1b[\${row};3H\\x1b[38;2;220;220;220;48;2;72;72;72m\${pad(text, width - 4)}\\x1b[0m\`) +} + +async function grayScrollLine(text) { + await write(\`\\x1b[48;2;72;72;72m\${pad(text, width)}\\x1b[0m\\r\\n\`) +} + +async function codexViewportFrame(frame) { + await write('\\x1b[?2026h') + await write('\\x1b[?25l') + await write(\`\\x1b[\${viewportTop};1H\\x1b[2K\\x1b[38;2;142;196;255m>_ OpenAI Codex\\x1b[0m \`) + await write('\\x1b[38;2;106;176;76mgpt-5.5 high\\x1b[0m ') + await write('\\x1b[38;2;180;180;180m/model to change · /ps to view · /stop to close\\x1b[0m') + await grayLine(viewportTop + 2 + (frame % 5), \`• Working for background terminal · rtk bun run e2e:ui --filter=@dalp/app \${frame}\`) + await write(\`\\x1b[\${viewportTop + 8};1H\\x1b[2K\\x1b[38;2;153;199;255m› \${pad('Reviewing terminal renderer state after remote SSH output burst ' + frame, width - 4)}\\x1b[0m\`) + await write('\\x1b[?2026l') +} + +async function insertCodexHistory(frame) { + const historyTop = scrollTop + const historyBottom = Math.max(historyTop + 3, viewportTop - 1) + const cursorTop = Math.max(historyTop, historyBottom - 1) + await write(\`\\x1b[\${historyTop};\${historyBottom}r\`) + await write(\`\\x1b[\${cursorTop};1H\`) + for (let index = 0; index < 3; index += 1) { + const phase = String(frame).padStart(4, '0') + '.' + index + await write('\\r\\n') + await write(\`\\x1b[48;2;72;72;72m\\x1b[K\`) + await write(\`\\x1b[38;2;220;220;220;48;2;72;72;72m\${pad('gpt-5.5 high · ~/code/pr-12250-migration-compare-move-baseprice-claim · /ps to view · /stop to close ' + phase, width)}\\x1b[0m\`) + } + await write('\\x1b[r') + await write(\`\\x1b[\${viewportBottom};1H\`) +} + +async function reverseIndexCodexHistory(frame) { + const historyTop = scrollTop + const historyBottom = Math.max(historyTop + 4, viewportTop - 1) + const scrollAmount = 2 + (frame % 3) + await write(\`\\x1b[\${historyTop};\${rows}r\`) + await write(\`\\x1b[\${viewportTop};1H\`) + for (let index = 0; index < scrollAmount; index += 1) { + await write('\\x1bM') + } + await write('\\x1b[r') + await write(\`\\x1b[\${historyTop};\${historyBottom}r\`) + await write(\`\\x1b[\${historyBottom};1H\`) + for (let index = 0; index < scrollAmount; index += 1) { + await write('\\r\\n') + await write(\`\\x1b[48;2;72;72;72m\\x1b[K\\x1b[38;2;220;220;220;48;2;72;72;72m\${pad('• Waited for background terminal · rtk bun run e2e:ui --filter=@dalp/app ' + frame + ':' + index, width)}\\x1b[0m\`) + } + await write('\\x1b[r') + await write(\`\\x1b[\${viewportBottom};1H\`) +} + +await write('\\x1b]0;codex\\x07') +await write('\\x1b[?25l') +await write('>_ OpenAI Codex (fixture)\\r\\n') +await write('model: gpt-5.5 /model to change\\r\\n') +await write('directory: ' + process.cwd() + '\\r\\n') +await write('permissions: YOLO mode\\r\\n\\r\\n') +await write('Tip: deterministic fixture for Orca SSH Codex display artifacts.\\r\\n\\r\\n') + +for (let frame = 0; frame < ${REMOTE_CODEX_FIXTURE_FRAMES}; frame += 1) { + await codexViewportFrame(frame) + await insertCodexHistory(frame) + if (frame % 4 === 0) { + await reverseIndexCodexHistory(frame) + } + if (frame % 9 === 0) { + await grayScrollLine(\`gpt-5.5 high · ~/code/pr-12250-migration-compare-move-baseprice-claim · /ps to view · /stop to close \${frame}\`) + } + await sleep(${REMOTE_CODEX_FIXTURE_FRAME_DELAY_MS}) +} + +await write('Updated Plan\\r\\n') +await write(' ✓ Reproduce remote Codex SSH display artifact\\r\\n') +await write(' ✓ Capture repeated gray status bands in scrollback\\r\\n') +await write('\\x1b[r\\x1b[?2026h\\x1b[2J\\x1b[H') +await write('Clean final frame after Codex-style gray status redraws.\\r\\n') +await write('There should be no gray background bands on this screen.\\r\\n') +await write('${REMOTE_CODEX_FIXTURE_CLEAN_FINAL_TEXT}\\r\\n') +await write(marker + '\\r\\n') +await write('\\x1b[?25h\\x1b[?2026l') +` +} + +export function installRemoteCodexFixture(target: DockerSshRelayTarget): void { + dockerWriteFile(target, '/usr/local/bin/codex', remoteCodexFixtureScript(), '755') +} + +export function dockerExec( + target: DockerSshRelayTarget, + command: string, + timeoutMs = 60_000 +): void { + execFileSync('docker', ['exec', target.containerName, 'bash', '-lc', command], { + stdio: ['ignore', 'pipe', 'pipe'], + timeout: timeoutMs + }) +} + +export function dockerWriteFile( + target: DockerSshRelayTarget, + remotePath: string, + body: string | Uint8Array, + mode: string +): void { + execFileSync( + 'docker', + [ + 'exec', + '-i', + target.containerName, + 'bash', + '-lc', + `cat > ${shellQuote(remotePath)} && chmod ${shellQuote(mode)} ${shellQuote(remotePath)}` + ], + { + input: body, + stdio: ['pipe', 'ignore', 'pipe'], + timeout: 60_000 + } + ) +} diff --git a/tests/e2e/ssh-codex-terminal-observers.ts b/tests/e2e/ssh-codex-terminal-observers.ts new file mode 100644 index 000000000..276441779 --- /dev/null +++ b/tests/e2e/ssh-codex-terminal-observers.ts @@ -0,0 +1,147 @@ +import type { Page } from '@stablyai/playwright-test' +import { expect } from './helpers/orca-app' +import { ensureTerminalVisible, switchToWorktree } from './helpers/store' +import { + execInTerminal, + sendToTerminal, + waitForActiveTerminalManager, + waitForTerminalOutput +} from './helpers/terminal' +import { REMOTE_CODEX_FIXTURE_CLEAN_FINAL_TEXT } from './ssh-codex-repro-remote-fixtures' +import { switchToNonRemoteWorktree } from './ssh-codex-reconnect-replay-driver' + +export async function waitForRemoteFixtureCleanFinalInHiddenPane( + page: Page, + remoteWorktreeId: string +): Promise { + await expect + .poll( + async () => + page.evaluate( + ({ remoteWorktreeId, cleanFinalText }) => { + const state = window.__store?.getState() + const tabId = state?.activeTabIdByWorktree?.[remoteWorktreeId] ?? null + const manager = tabId ? window.__paneManagers?.get(tabId) : null + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return pane?.serializeAddon?.serialize?.().includes(cleanFinalText) === true + }, + { remoteWorktreeId, cleanFinalText: REMOTE_CODEX_FIXTURE_CLEAN_FINAL_TEXT } + ), + { + timeout: 180_000, + message: 'Remote fixture did not reach its clean final frame while hidden' + } + ) + .toBe(true) +} + +export async function waitForRealRemoteCodexCompletion( + page: Page, + doneMarker: string +): Promise { + await expect + .poll( + async () => { + const content = await page.evaluate(() => { + const state = window.__store?.getState() + const worktreeId = state?.activeWorktreeId + const tabId = + state?.activeTabType === 'terminal' + ? state.activeTabId + : worktreeId + ? (state?.activeTabIdByWorktree?.[worktreeId] ?? null) + : null + const manager = tabId ? window.__paneManagers?.get(tabId) : null + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return pane?.serializeAddon?.serialize?.() ?? '' + }) + return content.split(doneMarker).length - 1 + }, + { + timeout: 300_000, + message: 'Real remote Codex did not emit its final marker' + } + ) + .toBeGreaterThanOrEqual(2) +} + +export async function clearRemoteTerminalAfterCodex( + page: Page, + ptyId: string, + cleanMarker: string +): Promise { + await sendToTerminal(page, ptyId, '/quit\r') + await waitForTerminalOutput(page, 'root@', 20_000, 120_000) + await execInTerminal( + page, + ptyId, + `printf '\\033[2J\\033[H${cleanMarker}\\nREAL_CODEX_CLEAN_SCREEN\\n'` + ) + await waitForTerminalOutput(page, cleanMarker, 20_000, 120_000) +} + +export async function waitForRealRemoteCodexBackgroundStatus(page: Page): Promise { + await expect + .poll( + async () => { + const content = await page.evaluate(() => { + const state = window.__store?.getState() + const worktreeId = state?.activeWorktreeId + const tabId = + state?.activeTabType === 'terminal' + ? state.activeTabId + : worktreeId + ? (state?.activeTabIdByWorktree?.[worktreeId] ?? null) + : null + const manager = tabId ? window.__paneManagers?.get(tabId) : null + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + return pane?.serializeAddon?.serialize?.() ?? '' + }) + return /background terminal|Working for background terminal|REMOTE_CODEX_PHASE/i.test( + content + ) + }, + { + timeout: 180_000, + message: 'Real remote Codex did not enter the long-running background-command state' + } + ) + .toBe(true) +} + +export async function scrollActiveTerminalToArtifactHistory(page: Page): Promise { + await page.evaluate(() => { + const state = window.__store?.getState() + const worktreeId = state?.activeWorktreeId + const tabId = + state?.activeTabType === 'terminal' + ? state.activeTabId + : worktreeId + ? (state?.activeTabIdByWorktree?.[worktreeId] ?? null) + : null + const manager = tabId ? window.__paneManagers?.get(tabId) : null + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + if (!pane) { + throw new Error('No active terminal pane to scroll') + } + const historyDepth = pane.terminal.buffer.active.baseY + pane.terminal.scrollToLine(Math.max(0, historyDepth - pane.terminal.rows * 3)) + pane.terminal.refresh(0, pane.terminal.rows - 1) + }) + await page.waitForTimeout(800) +} + +export async function stressRestoreRemoteTerminalDuringCodex( + page: Page, + remoteWorktreeId: string +): Promise { + for (let cycle = 0; cycle < 5; cycle += 1) { + await switchToNonRemoteWorktree(page, remoteWorktreeId) + await page.waitForTimeout(8_000) + await switchToWorktree(page, remoteWorktreeId) + await ensureTerminalVisible(page, 45_000) + await waitForActiveTerminalManager(page, 60_000) + await waitForRealRemoteCodexBackgroundStatus(page) + await page.waitForTimeout(900) + } +} diff --git a/tests/e2e/terminal-raster-artifact-analysis.ts b/tests/e2e/terminal-raster-artifact-analysis.ts new file mode 100644 index 000000000..765805422 --- /dev/null +++ b/tests/e2e/terminal-raster-artifact-analysis.ts @@ -0,0 +1,283 @@ +import { Buffer } from 'node:buffer' +import { PNG } from 'pngjs' +import type { Page } from '@stablyai/playwright-test' + +export type TerminalRasterTarget = { + clip: { x: number; y: number; width: number; height: number } + cellWidth: number + cellHeight: number + rows: number + cols: number + renderer: 'webgl' | 'dom' + modelGrayRows: number[] + modelStatusRows: number[] +} + +export type GraySlab = { + x: number + y: number + width: number + height: number +} +export type GraySlabAnalysis = { + slabCount: number + slabs: GraySlab[] + rawSlabCount: number + rawSlabs: GraySlab[] + staleStatusGlyphRowCount: number + staleStatusGlyphRows: number[] + target: TerminalRasterTarget + schedulerDebug: Record | null + replayDebug?: Record + duplicateStatusRows?: string[] +} + +export const MAX_FINAL_GRAY_SLABS = 0 + +async function readActiveTerminalRasterTarget(page: Page): Promise { + return page.evaluate(() => { + const isGrayRgb = (red: number, green: number, blue: number): boolean => { + const max = Math.max(red, green, blue) + const min = Math.min(red, green, blue) + return max - min <= 9 && max >= 48 && max <= 112 + } + + const isBufferGrayBackground = (cell: unknown): boolean => { + const record = cell as { + isBgRGB?: () => boolean + isBgPalette?: () => boolean + getBgColor?: () => number + } + if (record?.isBgRGB?.()) { + const color = record.getBgColor?.() ?? 0 + return isGrayRgb((color >> 16) & 0xff, (color >> 8) & 0xff, color & 0xff) + } + if (record?.isBgPalette?.()) { + const index = record.getBgColor?.() ?? -1 + const rgba = + pane.terminal._core?._themeService?.colors?.ansi?.[index]?.rgba ?? + pane.terminal._core?._themeService?.colors?.background?.rgba + if (typeof rgba !== 'number') { + return false + } + return isGrayRgb((rgba >> 24) & 0xff, (rgba >> 16) & 0xff, (rgba >> 8) & 0xff) + } + return false + } + + const state = window.__store?.getState() + const worktreeId = state?.activeWorktreeId + const tabId = + state?.activeTabType === 'terminal' + ? state.activeTabId + : worktreeId + ? (state?.activeTabIdByWorktree?.[worktreeId] ?? null) + : null + const manager = tabId ? window.__paneManagers?.get(tabId) : null + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + if (!pane) { + throw new Error('No active terminal pane') + } + const screen = pane.container.querySelector('.xterm-screen') + const dimensions = pane.terminal._core?._renderService?.dimensions?.css?.cell + if (!screen || !dimensions) { + throw new Error('Active terminal has no measurable xterm screen') + } + const diagnostics = manager + ?.getRenderingDiagnostics() + .find((diagnostic) => diagnostic.paneId === pane.id) + const rect = screen.getBoundingClientRect() + if (rect.width <= 0 || rect.height <= 0) { + throw new Error('Active terminal screen is not visible for raster capture') + } + const activeBuffer = pane.terminal.buffer.active + const modelGrayRows: number[] = [] + const modelStatusRows: number[] = [] + for (let row = 0; row < pane.terminal.rows; row += 1) { + const line = activeBuffer.getLine(activeBuffer.viewportY + row) + const rowText = line?.translateToString(true) ?? '' + if (/gpt-5\.5|background terminal|\/ps to view|\/stop to close/i.test(rowText)) { + modelStatusRows.push(row) + } + let grayCellCount = 0 + for (let col = 0; col < pane.terminal.cols; col += 1) { + if (isBufferGrayBackground(line?.getCell(col))) { + grayCellCount += 1 + } + } + if (grayCellCount >= Math.min(8, Math.ceil(pane.terminal.cols * 0.08))) { + modelGrayRows.push(row) + } + } + return { + clip: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, + cellWidth: dimensions.width, + cellHeight: dimensions.height, + rows: pane.terminal.rows, + cols: pane.terminal.cols, + renderer: diagnostics?.hasWebgl ? 'webgl' : 'dom', + modelGrayRows, + modelStatusRows + } + }) +} + +function isGraySlabPixel(red: number, green: number, blue: number, alpha: number): boolean { + if (alpha < 245) { + return false + } + const max = Math.max(red, green, blue) + const min = Math.min(red, green, blue) + return max - min <= 9 && max >= 48 && max <= 112 +} + +function isCodexStatusCyanPixel(red: number, green: number, blue: number, alpha: number): boolean { + if (alpha < 160) { + return false + } + return blue >= 160 && green >= 130 && red >= 80 && red <= 180 +} + +function isCodexStatusGreenPixel(red: number, green: number, blue: number, alpha: number): boolean { + if (alpha < 160) { + return false + } + return green >= 120 && red >= 60 && red <= 145 && blue >= 45 && blue <= 130 +} + +function analyzeGraySlabs( + buffer: Buffer, + target: TerminalRasterTarget, + viewport: { width: number; height: number } +): GraySlabAnalysis { + const image = PNG.sync.read(buffer) + const scaleX = image.width / viewport.width + const scaleY = image.height / viewport.height + const originX = Math.round(target.clip.x * scaleX) + const originY = Math.round(target.clip.y * scaleY) + const maxX = Math.min(image.width, originX + Math.round(target.clip.width * scaleX)) + const maxY = Math.min(image.height, originY + Math.round(target.clip.height * scaleY)) + // Tuned to ignore short gray fragments while still catching replay slab bands. + const minRunWidth = Math.max(32, Math.round(target.cellWidth * scaleX * 14)) + // Require multi-pixel vertical continuity to avoid one-line anti-alias noise. + const minRunHeight = Math.max(4, Math.round(target.cellHeight * scaleY * 0.35)) + const runs: GraySlab[] = [] + + for (let y = originY; y < maxY; y += 1) { + let runStart: number | null = null + for (let x = originX; x <= maxX; x += 1) { + const inside = x < maxX + const offset = (y * image.width + x) * 4 + const gray = + inside && + isGraySlabPixel( + image.data[offset] ?? 0, + image.data[offset + 1] ?? 0, + image.data[offset + 2] ?? 0, + image.data[offset + 3] ?? 0 + ) + if (gray && runStart === null) { + runStart = x + } else if (!gray && runStart !== null) { + const width = x - runStart + if (width >= minRunWidth) { + runs.push({ x: runStart - originX, y: y - originY, width, height: 1 }) + } + runStart = null + } + } + } + + const slabs: GraySlab[] = [] + for (const run of runs) { + const previous = slabs.at(-1) + if ( + previous && + Math.abs(previous.x - run.x) <= 3 && + Math.abs(previous.width - run.width) <= 8 && + previous.y + previous.height === run.y + ) { + previous.height += 1 + continue + } + slabs.push({ ...run }) + } + + const meaningfulSlabs = slabs.filter((slab) => slab.height >= minRunHeight) + const artifactSlabs = meaningfulSlabs.filter((slab) => { + const slabCenterCssY = (slab.y + slab.height / 2) / scaleY + const row = Math.floor(slabCenterCssY / target.cellHeight) + return !target.modelGrayRows.some((grayRow) => Math.abs(grayRow - row) <= 1) + }) + const statusGlyphRows: number[] = [] + for (let row = 0; row < target.rows; row += 1) { + let cyanPixelCount = 0 + let greenPixelCount = 0 + const yStart = originY + Math.round(row * target.cellHeight * scaleY) + const yEnd = Math.min(maxY, yStart + Math.round(target.cellHeight * scaleY)) + for (let y = yStart; y < yEnd; y += 1) { + for (let x = originX; x < maxX; x += 1) { + const offset = (y * image.width + x) * 4 + if ( + isCodexStatusCyanPixel( + image.data[offset] ?? 0, + image.data[offset + 1] ?? 0, + image.data[offset + 2] ?? 0, + image.data[offset + 3] ?? 0 + ) + ) { + cyanPixelCount += 1 + } else if ( + isCodexStatusGreenPixel( + image.data[offset] ?? 0, + image.data[offset + 1] ?? 0, + image.data[offset + 2] ?? 0, + image.data[offset + 3] ?? 0 + ) + ) { + greenPixelCount += 1 + } + } + } + if ( + row >= 8 && + // Thresholds tuned to classify stale Codex status glyph rows in screenshots. + cyanPixelCount >= Math.max(12, Math.round(target.cellWidth * scaleX * 3)) && + greenPixelCount >= Math.max(24, Math.round(target.cellWidth * scaleX * 8)) && + !target.modelStatusRows.some((statusRow) => Math.abs(statusRow - row) <= 1) + ) { + statusGlyphRows.push(row) + } + } + return { + slabCount: artifactSlabs.length, + slabs: artifactSlabs.slice(0, 12), + rawSlabCount: meaningfulSlabs.length, + rawSlabs: meaningfulSlabs.slice(0, 12), + staleStatusGlyphRowCount: statusGlyphRows.length, + staleStatusGlyphRows: statusGlyphRows.slice(0, 12), + target, + schedulerDebug: null + } +} + +export async function captureGraySlabAnalysis(page: Page): Promise<{ + analysis: GraySlabAnalysis + screenshot: Buffer +}> { + const target = await readActiveTerminalRasterTarget(page) + const schedulerDebug = await page.evaluate( + () => window.__terminalOutputSchedulerDebug?.snapshot?.() ?? null + ) + const viewport = await page.evaluate(() => ({ + width: window.innerWidth, + height: window.innerHeight + })) + const screenshot = Buffer.from(await page.screenshot()) + const analysis = analyzeGraySlabs(screenshot, target, viewport) + analysis.schedulerDebug = schedulerDebug + return { + analysis, + screenshot + } +} diff --git a/tests/e2e/terminal-repro-evidence.ts b/tests/e2e/terminal-repro-evidence.ts new file mode 100644 index 000000000..1ae6bd0ef --- /dev/null +++ b/tests/e2e/terminal-repro-evidence.ts @@ -0,0 +1,19 @@ +import type { Buffer } from 'node:buffer' +import { mkdirSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import type { GraySlabAnalysis } from './terminal-raster-artifact-analysis' + +const REPRO_ARTIFACT_DIR = path.join(process.cwd(), '.tmp', 'issue-5969-repro') + +export function persistReproEvidence( + label: string, + analysis: GraySlabAnalysis, + screenshot: Buffer +): void { + mkdirSync(REPRO_ARTIFACT_DIR, { recursive: true }) + writeFileSync(path.join(REPRO_ARTIFACT_DIR, `${label}.png`), screenshot) + writeFileSync( + path.join(REPRO_ARTIFACT_DIR, `${label}.json`), + `${JSON.stringify(analysis, null, 2)}\n` + ) +} diff --git a/tests/e2e/terminal-webgl-reset-capture.ts b/tests/e2e/terminal-webgl-reset-capture.ts new file mode 100644 index 000000000..93da7610c --- /dev/null +++ b/tests/e2e/terminal-webgl-reset-capture.ts @@ -0,0 +1,30 @@ +import type { Buffer } from 'node:buffer' +import type { Page } from '@stablyai/playwright-test' +import { captureGraySlabAnalysis, type GraySlabAnalysis } from './terminal-raster-artifact-analysis' + +export async function resetWebglAndCaptureGraySlabAnalysis(page: Page): Promise<{ + analysis: GraySlabAnalysis + screenshot: Buffer +}> { + await page.evaluate(() => { + const state = window.__store?.getState() + const worktreeId = state?.activeWorktreeId + const tabId = + state?.activeTabType === 'terminal' + ? state.activeTabId + : worktreeId + ? (state?.activeTabIdByWorktree?.[worktreeId] ?? null) + : null + const manager = tabId ? window.__paneManagers?.get(tabId) : null + const pane = manager?.getActivePane?.() ?? manager?.getPanes?.()[0] ?? null + manager?.resetWebglTextureAtlases?.() + pane?.terminal?.refresh?.(0, Math.max(0, (pane.terminal.rows ?? 1) - 1)) + }) + await page.evaluate( + () => + new Promise((resolve) => { + requestAnimationFrame(() => requestAnimationFrame(() => resolve())) + }) + ) + return captureGraySlabAnalysis(page) +}