diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index ce371c681..1ce895106 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -5462,6 +5462,7 @@ "direct SSH folder workspace reattach", "direct SSH split-pane retry ownership", "same-authority terminal correction", + "remote-runtime host surface materialization", "remote-runtime mirror polling", "remote-runtime network recovery", "terminal create idempotency", @@ -5480,9 +5481,10 @@ "https://github.com/stablyai/orca/pull/6955", "https://github.com/stablyai/orca/pull/6979", "https://github.com/stablyai/orca/pull/7009", - "https://github.com/stablyai/orca/pull/8597" + "https://github.com/stablyai/orca/pull/8597", + "https://github.com/stablyai/orca/issues/11541" ], - "invariant": "SSH, WSL, and remote-runtime restore paths must treat provider listing failures and unknown liveness as unknown, not dead, while still avoiding duplicate spawn and clearing expired relay leases exactly once. Direct SSH reconnect must atomically clear only exact-target live PTY bindings, preserve relay identity, retry Git and folder panes without paired close or provider shutdown, and allow at most two automatic attempts in one authority chain even when each settlement exceeds the rolling window. A rejected acknowledgement mutates no store map. A successful exact split-pane spawn or reattach must retain that attempt as shared live authority until sibling leaves settle; the first success cannot consume sibling authority, a sibling failure can start at most one second tab-wide attempt, and prior-attempt callbacks become inert after rotation. Once the retry budget is exhausted, a failure cannot start attempt three or revoke attempt-two authority from siblings that may still settle. Primary PTY exit must promote a bound survivor or preserve exact authority through an empty activation gap, and split detach must project that authority to both resulting tabs. Hydrated PTY hints cannot supersede a current exact-attempt owner, and target snapshot hydration/reconnect cannot reset sibling SSH, local, WSL, or runtime-owned state. Every restored remote terminal must preserve its provider PTY identity, including the authoritative incarnation returned by a successful session-ID reattach. After a recoverable partition the same authenticated runtime must reattach the same PTY, reject detached input, apply the latest viewport, and report healthy only after authoritative replay. A successful one-shot reachability probe may replace a pre-ready shared-control socket, but waiting RPCs must continue onto the replacement under their original deadline without duplicate host delivery or retained request bytes. Automatic PTY recovery stops after one bounded minute without a fatal terminal error; a manual reconnect starts a newly fenced epoch against the same PTY, and closed panes retain no recovery UI state. One capability-gated terminal-create mutation must produce at most one host PTY across an unknown response outcome, remain manually retryable after cutoff, and never let a stale completion replace a newer pane lifecycle.", + "invariant": "SSH, WSL, and remote-runtime restore paths must treat provider listing failures and unknown liveness as unknown, not dead, while still avoiding duplicate spawn and clearing expired relay leases exactly once. Direct SSH reconnect must atomically clear only exact-target live PTY bindings, preserve relay identity, retry Git and folder panes without paired close or provider shutdown, and allow at most two automatic attempts in one authority chain even when each settlement exceeds the rolling window. A rejected acknowledgement mutates no store map. A successful exact split-pane spawn or reattach must retain that attempt as shared live authority until sibling leaves settle; the first success cannot consume sibling authority, a sibling failure can start at most one second tab-wide attempt, and prior-attempt callbacks become inert after rotation. Once the retry budget is exhausted, a failure cannot start attempt three or revoke attempt-two authority from siblings that may still settle. Primary PTY exit must promote a bound survivor or preserve exact authority through an empty activation gap, and split detach must project that authority to both resulting tabs. Hydrated PTY hints cannot supersede a current exact-attempt owner, and target snapshot hydration/reconnect cannot reset sibling SSH, local, WSL, or runtime-owned state. Every restored remote terminal must preserve its provider PTY identity, including the authoritative incarnation returned by a successful session-ID reattach. After a recoverable partition the same authenticated runtime must reattach the same PTY, reject detached input, apply the latest viewport, and report healthy only after authoritative replay. A successful one-shot reachability probe may replace a pre-ready shared-control socket, but waiting RPCs must continue onto the replacement under their original deadline without duplicate host delivery or retained request bytes. Automatic PTY recovery stops after one bounded minute without a fatal terminal error; a manual reconnect starts a newly fenced epoch against the same PTY, and closed panes retain no recovery UI state. One capability-gated terminal-create mutation must produce at most one host PTY across an unknown response outcome, remain manually retryable after cutoff, and never let a stale completion replace a newer pane lifecycle. Reconnect must alternate exact activation with authoritative inventory so neither a stale activation response nor an activation failure can strand or retire a pane, and activating a parked surface whose persisted binding was already retired must respawn it rather than report a changed owner after signalling its exit.", "oracle": "Deterministic tests cover bounded stale-handle replacement, suspended heartbeat clocks, cold and established subscription failure, ten partition/recovery cycles, automatic-recovery cutoff, manual reconnect, and exact direct SSH binding recovery. They assert one atomic store publication clears only exact-target PTY indexes, null-PTY activation remains unchanged, relay identity survives, Git and folder panes retry symmetrically, another target/local/WSL/runtime panes remain byte-identical through target snapshot hydration and reconnect, only an accepted exact failure or timeout starts the second attempt, two 31-second timeouts cannot start a third settlement-triggered attempt, rejected stale/mismatched acknowledgements preserve every store map, and concurrent split-pane spawn and reattach callbacks both commit under the same attempt ID after the first success replaces pending state with live shared authority. A sibling failure revokes that shared authority and starts exactly one second attempt; duplicate failures and late first-attempt PTY callbacks preserve the second attempt and every state map. Attempt-two failure retains continuation authority for later siblings, primary exit promotes a bound survivor or preserves the lease until a late sibling binds, and primary plus non-primary detach retain exact authority and history on both resulting tabs. Both remount callbacks consume split-count activity suppression, intentional dispose emits no failure/timeout, and a same-attempt StrictMode remount still owns one timeout. Hydration clears an untrusted PTY hint without clearing its current pending owner, healthy current-authority bindings suppress correction, hydration finalizes once, and reconnect emits no paired close lifecycle. A provider-level session-ID reattach returns an incarnation, then a legacy exit without an incarnation must resolve to that returned identity rather than minting a fallback identity. The shared-control oracle withholds the first encrypted ready frame, starts one RPC, triggers the successful-probe refresh, then requires exactly two client connections, one host request, a successful response, zero pending calls, and zero retained request bytes. Tests also assert one unsubscribe per remote-runtime epoch, observable recovery phases, stable PTY identity, resumed snapshot/output/input, no healthy state before replay, no retry or input after cutoff, a new manual epoch against the same PTY, quiet recovery UI with an explicit Reconnect action, pane-close state cleanup, one stable create mutation id, old-runtime no-retry behavior, cross-process PTY adoption, and bounded in-flight coordination.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/providers/ssh-pty-provider-reattach-incarnation.test.ts --reporter=dot", @@ -5492,6 +5494,8 @@ "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/store/slices/direct-ssh-terminal-retry.test.ts src/renderer/src/store/slices/direct-ssh-pane-detach-ledger.test.ts src/renderer/src/store/slices/direct-ssh-terminal-recovery.test.ts src/renderer/src/store/slices/direct-ssh-terminal-workspace-scope.test.ts src/renderer/src/store/slices/terminals-hydration.test.ts src/renderer/src/store/slices/repos-ssh-host-reconciliation.test.ts src/renderer/src/hooks/direct-ssh-reconnect-coordinator.test.ts src/renderer/src/hooks/direct-ssh-host-hydration.test.ts src/renderer/src/hooks/direct-ssh-state-routing.test.ts src/renderer/src/hooks/remote-workspace-target-sync.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/terminal-pane-tab-detach.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/main/ipc/repos-remote.test.ts src/main/ipc/ssh.test.ts src/main/ipc/worktrees.test.ts src/main/runtime/public-ssh-state.test.ts src/main/ssh/ssh-connection-manager.test.ts src/main/ssh/ssh-connection.test.ts src/main/ssh/ssh-provider-authority.test.ts src/preload/ssh-authority-forwarding.test.ts src/renderer/src/runtime/runtime-client-events.test.ts src/renderer/src/runtime/runtime-environment-ssh-state.test.ts src/shared/ssh-retained-payload-admission.test.ts src/shared/ssh-types.test.ts --reporter=dot", "pnpm exec electron-vite build --mode e2e", + "pnpm run build:web-from-renderer", + "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "SKIP_BUILD=1 pnpm exec playwright test tests/e2e/terminal-cold-activation-deferral.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-cold-activation-restore.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "ORCA_E2E_SSH_DOCKER=1 SKIP_BUILD=1 pnpm exec playwright test tests/e2e/ssh-docker-relay-perf.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" @@ -5519,6 +5523,7 @@ "src/main/runtime/rpc/remote-runtime-server-heartbeat.test.ts", "src/main/runtime/rpc/methods/terminal-create-idempotency.test.ts", "src/main/runtime/orca-runtime-terminal-create-idempotency.test.ts", + "tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts", "src/renderer/src/store/slices/direct-ssh-terminal-retry.test.ts", "src/renderer/src/store/slices/direct-ssh-pane-detach-ledger.test.ts", "src/renderer/src/store/slices/direct-ssh-terminal-recovery.test.ts", diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index efd9e3ac8..d4e1e7e69 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -9513,6 +9513,143 @@ describe('registerPtyHandlers', () => { expect(store.flushOrThrow).toHaveBeenCalledOnce() }) + // Why: a parked pane (stopped with keepHistory) leaves the runtime holding the binding while + // persistence has already dropped it. Reading "nothing left to retire" as a competing owner + // aborted materialization *after* signalling the exit, which destroyed the pane instead of + // rebuilding it — the reconnect path then had no surface to attach to (#11541). + it('respawns a proven-dead owner whose persisted binding was already retired', async () => { + const worktreeId = 'repo-1::/tmp/already-retired-owner' + const cwd = '/tmp/already-retired-owner' + const tabId = 'tab-already-retired-owner' + const leafId = '89898989-8989-4989-8989-898989898989' + const paneKey = makePaneKey(tabId, leafId) + const providerSpawn = vi.fn( + async (options: { attachOnly?: boolean; command?: string; sessionId?: string }) => { + if (options.attachOnly) { + throw new Error('Session not found: pty-already-retired-owner') + } + return { id: 'pty-fresh-already-retired', incarnationId: 'inc-fresh-already-retired' } + } + ) + const probePtyLiveness = vi.fn(async () => false) + setLocalPtyProvider({ + spawn: providerSpawn, + probePtyLiveness, + write: vi.fn(), + resize: vi.fn(), + kill: vi.fn(), + shutdown: vi.fn(), + sendSignal: vi.fn(), + getCwd: vi.fn(), + getInitialCwd: vi.fn(), + clearBuffer: vi.fn(), + acknowledgeDataEvent: vi.fn(), + hasChildProcesses: vi.fn(), + getForegroundProcess: vi.fn(), + serialize: vi.fn(), + revive: vi.fn(), + onData: vi.fn(() => () => {}), + onReplay: vi.fn(() => () => {}), + onExit: vi.fn(() => () => {}), + listProcesses: vi.fn(async () => []), + attach: vi.fn(), + getDefaultShell: vi.fn(), + getProfiles: vi.fn() + } as never) + // Persistence kept the tab but already dropped this leaf's PTY binding, exactly as an + // earlier keep-history stop leaves it. + let session = { + tabsByWorktree: { + [worktreeId]: [{ id: tabId, worktreeId, ptyId: null }] + }, + terminalLayoutsByTabId: { + [tabId]: { + root: { type: 'leaf' as const, leafId }, + activeLeafId: leafId, + expandedLeafId: null, + ptyIdsByLeafId: {} + } + }, + terminalPtyIncarnationsByPaneKey: {} + } + const store = { + getWorkspaceSession: vi.fn(() => session), + setWorkspaceSession: vi.fn((next) => { + session = next + }), + flushOrThrow: vi.fn(), + persistPtyBinding: vi.fn(), + getFolderWorkspace: vi.fn(() => undefined), + getFolderWorkspaces: vi.fn(() => []), + getProjectGroups: vi.fn(() => []), + getRepos: vi.fn(() => []) + } + let runtimeOwnsPane = true + const runtime = { + setPtyController: vi.fn(), + resolveTerminalPane: vi.fn(() => { + if (!runtimeOwnsPane) { + throw new Error('terminal_not_found') + } + return { + ptyId: 'pty-already-retired-owner', + tabId, + leafId, + handle: 'term-already-retired', + connected: true + } + }), + createPreAllocatedTerminalHandle: vi.fn(() => 'term-already-retired-fresh'), + preAllocateHandleForPty: vi.fn(() => 'term-already-retired-fresh'), + registerPreAllocatedHandleForPty: vi.fn(), + beginPtyRegistration: vi.fn(), + cancelPendingPtyRegistration: vi.fn(), + assertPtyRegistrationAllowed: vi.fn(), + registerPty: vi.fn(), + noteTerminalSpawnCommand: vi.fn(), + seedHeadlessTerminal: vi.fn(), + onPtySpawned: vi.fn(), + // Why: the real runtime drops its pane binding on exit; the guard after retirement must + // see that release rather than a resurrected owner. + onPtyExit: vi.fn(() => { + runtimeOwnsPane = false + }), + onPtyData: vi.fn() + } + + registerPtyHandlers( + mainWindow as never, + runtime as never, + undefined, + undefined, + undefined, + store as never + ) + + const mounted = await handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + cwd, + command: 'codex resume already-retired-session', + worktreeId, + tabId, + leafId, + env: { + ORCA_PANE_KEY: paneKey, + ORCA_TAB_ID: tabId, + ORCA_WORKTREE_ID: worktreeId + } + }) + + expect(probePtyLiveness).toHaveBeenCalledWith('pty-already-retired-owner') + expect(mounted).toMatchObject({ id: 'pty-fresh-already-retired' }) + expect(providerSpawn).toHaveBeenCalledTimes(2) + expect(providerSpawn.mock.calls[1]?.[0]).toMatchObject({ + command: 'codex resume already-retired-session' + }) + expect(runtime.onPtyExit).toHaveBeenCalledWith('pty-already-retired-owner', 0, undefined) + }) + it('retires a dead owner from the exact SSH host session before fresh recovery', async () => { const connectionId = 'ssh-dead-stable-pane' const hostId = `ssh:${connectionId}` diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 0fc5d6e9d..582e66593 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -712,7 +712,13 @@ function retirePersistedStablePaneOwner( const paneKey = makePaneKey(owner.tabId, owner.leafId) const hostId = connectionId ? toSshExecutionHostId(connectionId) : undefined const current = resolvePersistedStablePaneOwner(store, paneKey, worktreeId, connectionId) - if (current?.ptyId !== owner.ptyId || current.incarnationId !== owner.incarnationId) { + if (!current) { + // Why: persistence already dropped this pane binding (an earlier stop retired it while the + // runtime kept history), so there is nothing left to clear — that is a completed retirement, + // not a competing owner. Reporting failure here strands the pane after its PTY is proven dead. + return true + } + if (current.ptyId !== owner.ptyId || current.incarnationId !== owner.incarnationId) { return false } const session = store.getWorkspaceSession(hostId) diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts index e83a565bd..6126c039e 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.test.ts @@ -1646,6 +1646,10 @@ describe('createRemoteRuntimePtyTransport', () => { let hostListCalls = 0 runtimeCall.mockImplementation(async (args: { method: string }) => { + if (args.method === 'session.tabs.activate') { + // Why: this host publishes the replacement only through its own inventory, so activation answers with nothing. + return { ok: true, result: { tabs: [] } } + } if (args.method === 'session.tabs.list') { hostListCalls += 1 if (hostListCalls === 1) { @@ -1694,7 +1698,7 @@ describe('createRemoteRuntimePtyTransport', () => { expect(onPtyExit).not.toHaveBeenCalled() await vi.waitFor( () => expect(latestSubscribePayload()).toMatchObject({ terminal: 'terminal-reconnected' }), - { timeout: 2_000 } + { timeout: 6_000 } ) const replacementStreamId = latestSubscribePayload().streamId emitSnapshot(replacementStreamId, draft) @@ -1723,6 +1727,7 @@ describe('createRemoteRuntimePtyTransport', () => { return payload ? [payload.terminal] : [] }) expect(subscribedTerminals).toEqual(['terminal-stale', 'terminal-reconnected']) + transport.destroy?.() }) it('reattaches from a later host snapshot after bounded replacement polling stops', async () => { @@ -1798,11 +1803,14 @@ describe('createRemoteRuntimePtyTransport', () => { expect(hostListCalls).toBeGreaterThan(1) expect(hostListCalls).toBeLessThan(25) + // The reconnect opens with a materialize (session.tabs.activate) and then + // polls the inventory, so the first list runs one backoff into the budget. const listTimeouts = runtimeCall.mock.calls .map(([args]) => args) .filter((args) => args.method === 'session.tabs.list') .map((args) => args.timeoutMs as number) - expect(listTimeouts[0]).toBe(15_000) + expect(listTimeouts[0]).toBeGreaterThan(14_000) + expect(listTimeouts[0]).toBeLessThanOrEqual(15_000) expect(listTimeouts.every((timeoutMs) => timeoutMs > 0 && timeoutMs <= 15_000)).toBe(true) expect(listTimeouts.at(-1)).toBeLessThanOrEqual(1_000) expect(onError).not.toHaveBeenCalled() @@ -1875,6 +1883,379 @@ describe('createRemoteRuntimePtyTransport', () => { } }) + it('materializes a host surface whose PTY died instead of polling a dead inventory', async () => { + vi.useFakeTimers() + try { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onError = vi.fn() + const onPtyExit = vi.fn() + const onPtyRebind = vi.fn() + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'web-terminal-tab-1', + leafId: 'pane:1', + onPtyExit, + onPtyRebind + }) + + resolvedPaneHandle = 'terminal-before-restart' + transport.attach({ + existingPtyId: 'remote:env-1@@terminal-before-restart', + cols: 80, + rows: 24, + callbacks: { onError } + }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + const activateCalls = (): { + method: string + params?: { tabId?: string; leafId?: string } + }[] => + runtimeCall.mock.calls + .map(([args]) => args) + .filter((args) => args.method === 'session.tabs.activate') + const activateCallsBeforeStale = activateCalls().length + + // The host still publishes the surface, but only activation can mint its replacement handle. + let materialized = false + const hostSnapshot = (): unknown => ({ + worktree: 'wt-1', + publicationEpoch: 'epoch-2', + snapshotVersion: materialized ? 3 : 2, + activeGroupId: null, + activeTabId: 'tab-1::pane:1', + activeTabType: 'terminal', + tabs: [ + { + type: 'terminal', + id: 'tab-1::pane:1', + parentTabId: 'tab-1', + leafId: 'pane:1', + title: 'Claude Code', + isActive: true, + ...(materialized + ? { status: 'ready', terminal: 'terminal-after-restart' } + : { status: 'pending-handle', terminal: null }) + } + ] + }) + runtimeCall.mockImplementation(async (args: { method: string }) => { + if (args.method === 'session.tabs.activate') { + materialized = true + return { ok: true, result: hostSnapshot() } + } + if (args.method === 'session.tabs.list') { + return { ok: true, result: hostSnapshot() } + } + return { ok: true, result: {} } + }) + + subscriptionCallbacks?.onResponse({ + ok: true, + result: { + type: 'error', + streamId: latestSubscribePayload().streamId, + message: 'terminal_handle_stale' + } + }) + await vi.advanceTimersByTimeAsync(1_000) + + expect(activateCalls().length).toBeGreaterThan(activateCallsBeforeStale) + expect(activateCalls().at(-1)?.params).toMatchObject({ + tabId: 'tab-1', + leafId: 'pane:1', + notifyClients: false, + navigation: 'caller' + }) + await vi.waitFor(() => + expect(latestSubscribePayload()).toMatchObject({ terminal: 'terminal-after-restart' }) + ) + expect(onPtyRebind).toHaveBeenCalledWith( + 'remote:env-1@@terminal-after-restart', + 'remote:env-1@@terminal-before-restart' + ) + expect(onPtyExit).not.toHaveBeenCalled() + expect(onError).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + + it('re-activates when a stale ready activation response precedes the pending surface', async () => { + vi.useFakeTimers() + try { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onError = vi.fn() + const onPtyExit = vi.fn() + const onPtyRebind = vi.fn() + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'web-terminal-tab-1', + leafId: 'pane:1', + onPtyExit, + onPtyRebind + }) + + resolvedPaneHandle = 'terminal-before-restart' + transport.attach({ + existingPtyId: 'remote:env-1@@terminal-before-restart', + callbacks: { onError } + }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + runtimeCall.mockClear() + + let activateCalls = 0 + const snapshot = (terminal: string | null): unknown => ({ + worktree: 'wt-1', + publicationEpoch: 'epoch-2', + snapshotVersion: activateCalls + 2, + activeGroupId: null, + activeTabId: 'tab-1::pane:1', + activeTabType: 'terminal', + tabs: [ + { + type: 'terminal', + id: 'tab-1::pane:1', + parentTabId: 'tab-1', + leafId: 'pane:1', + title: 'Terminal', + isActive: true, + ...(terminal + ? { status: 'ready', terminal } + : { status: 'pending-handle', terminal: null }) + } + ] + }) + runtimeCall.mockImplementation(async (args: { method: string }) => { + if (args.method === 'session.tabs.activate') { + activateCalls += 1 + // Why: the first activation races host publication and answers with the pre-restart handle. + return { + ok: true, + result: snapshot( + activateCalls === 1 ? 'terminal-before-restart' : 'terminal-after-restart' + ) + } + } + if (args.method === 'session.tabs.list') { + return { ok: true, result: snapshot(null) } + } + return { ok: true, result: {} } + }) + + subscriptionCallbacks?.onResponse({ + ok: true, + result: { + type: 'error', + streamId: latestSubscribePayload().streamId, + message: 'terminal_handle_stale' + } + }) + await vi.advanceTimersByTimeAsync(2_000) + + expect( + runtimeCall.mock.calls + .map(([args]) => args.method) + .filter((method) => method === 'session.tabs.activate' || method === 'session.tabs.list') + ).toEqual(['session.tabs.activate', 'session.tabs.list', 'session.tabs.activate']) + await vi.waitFor(() => + expect(latestSubscribePayload()).toMatchObject({ terminal: 'terminal-after-restart' }) + ) + expect(onPtyRebind).toHaveBeenCalledWith( + 'remote:env-1@@terminal-after-restart', + 'remote:env-1@@terminal-before-restart' + ) + expect(onPtyExit).not.toHaveBeenCalled() + expect(onError).not.toHaveBeenCalled() + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + + it('falls back to inventory when activation fails for a non-missing reason', async () => { + vi.useFakeTimers() + try { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onError = vi.fn() + const onPtyExit = vi.fn() + const onPtyRebind = vi.fn() + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'web-terminal-tab-1', + leafId: 'pane:1', + onPtyExit, + onPtyRebind + }) + + resolvedPaneHandle = 'terminal-before-restart' + transport.attach({ + existingPtyId: 'remote:env-1@@terminal-before-restart', + callbacks: { onError } + }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + runtimeCall.mockClear() + + const replacementSnapshot = { + worktree: 'wt-1', + publicationEpoch: 'epoch-2', + snapshotVersion: 2, + activeGroupId: null, + activeTabId: 'tab-1::pane:1', + activeTabType: 'terminal', + tabs: [ + { + type: 'terminal', + id: 'tab-1::pane:1', + parentTabId: 'tab-1', + leafId: 'pane:1', + title: 'Terminal', + isActive: true, + status: 'ready', + terminal: 'terminal-after-restart' + } + ] + } + runtimeCall.mockImplementation(async (args: { method: string }) => { + if (args.method === 'session.tabs.activate') { + // Why: an older host has no activation method at all, which is not evidence the surface is gone. + return { ok: false, error: { code: 'method_not_found', message: 'Unknown method' } } + } + if (args.method === 'session.tabs.list') { + return { ok: true, result: replacementSnapshot } + } + return { ok: true, result: {} } + }) + + subscriptionCallbacks?.onResponse({ + ok: true, + result: { + type: 'error', + streamId: latestSubscribePayload().streamId, + message: 'terminal_handle_stale' + } + }) + await vi.advanceTimersByTimeAsync(1_000) + + expect( + runtimeCall.mock.calls + .map(([args]) => args.method) + .filter((method) => method === 'session.tabs.activate' || method === 'session.tabs.list') + ).toEqual(['session.tabs.activate', 'session.tabs.list']) + await vi.waitFor(() => + expect(latestSubscribePayload()).toMatchObject({ terminal: 'terminal-after-restart' }) + ) + expect(onPtyRebind).toHaveBeenCalledWith( + 'remote:env-1@@terminal-after-restart', + 'remote:env-1@@terminal-before-restart' + ) + expect(onPtyExit).not.toHaveBeenCalled() + expect(onError).not.toHaveBeenCalled() + transport.destroy?.() + } finally { + vi.useRealTimers() + } + }) + + it('retries activation when inventory disproves a transient missing-surface response', async () => { + vi.useFakeTimers() + try { + const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') + const onError = vi.fn() + const onPtyExit = vi.fn() + const onPtyRebind = vi.fn() + const transport = createRemoteRuntimePtyTransport('env-1', { + worktreeId: 'wt-1', + tabId: 'web-terminal-tab-1', + leafId: 'pane:1', + onPtyExit, + onPtyRebind + }) + + resolvedPaneHandle = 'terminal-before-restart' + transport.attach({ + existingPtyId: 'remote:env-1@@terminal-before-restart', + callbacks: { onError } + }) + await vi.waitFor(() => expect(subscriptionSendBinary).toHaveBeenCalled()) + runtimeCall.mockClear() + + let activateCalls = 0 + const pendingSnapshot = { + worktree: 'wt-1', + publicationEpoch: 'epoch-2', + snapshotVersion: 2, + activeGroupId: null, + activeTabId: 'tab-1::pane:1', + activeTabType: 'terminal', + tabs: [ + { + type: 'terminal', + id: 'tab-1::pane:1', + parentTabId: 'tab-1', + leafId: 'pane:1', + title: 'Terminal', + isActive: true, + status: 'pending-handle', + terminal: null + } + ] + } + runtimeCall.mockImplementation(async (args: { method: string }) => { + if (args.method === 'session.tabs.activate') { + activateCalls += 1 + if (activateCalls === 1) { + return { ok: false, error: { code: 'runtime_error', message: 'tab_not_found' } } + } + return { + ok: true, + result: { + ...pendingSnapshot, + snapshotVersion: 3, + tabs: [ + { + ...pendingSnapshot.tabs[0], + status: 'ready', + terminal: 'terminal-after-restart' + } + ] + } + } + } + if (args.method === 'session.tabs.list') { + return { ok: true, result: pendingSnapshot } + } + return { ok: true, result: {} } + }) + + subscriptionCallbacks?.onResponse({ + ok: true, + result: { + type: 'error', + streamId: latestSubscribePayload().streamId, + message: 'terminal_handle_stale' + } + }) + await vi.advanceTimersByTimeAsync(2_000) + + expect( + runtimeCall.mock.calls + .map(([args]) => args.method) + .filter((method) => method === 'session.tabs.activate' || method === 'session.tabs.list') + ).toEqual(['session.tabs.activate', 'session.tabs.list', 'session.tabs.activate']) + await vi.waitFor(() => + expect(latestSubscribePayload()).toMatchObject({ terminal: 'terminal-after-restart' }) + ) + expect(onPtyRebind).toHaveBeenCalledWith( + 'remote:env-1@@terminal-after-restart', + 'remote:env-1@@terminal-before-restart' + ) + expect(onPtyExit).not.toHaveBeenCalled() + expect(onError).not.toHaveBeenCalled() + } finally { + vi.useRealTimers() + } + }) + it('keeps a mounted HUB mirror alive when the old stream ends before the replacement snapshot', async () => { const { createRemoteRuntimePtyTransport } = await import('./remote-runtime-pty-transport') const handleEvents = await import('../../runtime/web-session-terminal-handle-events') @@ -2691,7 +3072,7 @@ describe('createRemoteRuntimePtyTransport', () => { emitSnapshot(staleStreamId, 'before stale handle') runtimeCall.mockImplementation(async (args: { method: string }) => { - if (args.method !== 'session.tabs.list') { + if (args.method !== 'session.tabs.activate' && args.method !== 'session.tabs.list') { return { ok: true, result: {} } } return { diff --git a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts index c9e0ea435..42f7df894 100644 --- a/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts +++ b/src/renderer/src/components/terminal-pane/remote-runtime-pty-transport.ts @@ -498,6 +498,30 @@ export function createRemoteRuntimePtyTransport( ) } + // Why: pending host surfaces materialize only through activation. + function activateHostSessionSurface( + hostTabId: string, + worktree: string, + timeoutMs?: number + ): Promise { + return callRuntime( + 'session.tabs.activate', + { + worktree, + tabId: hostTabId, + ...(leafId ? { leafId } : {}), + notifyClients: false, + navigation: 'caller' + }, + timeoutMs + ) + } + + function isMissingHostSessionSurfaceError(error: unknown): boolean { + const message = runtimeTerminalErrorMessage(error) + return message.includes('tab_not_found') || message.includes('terminal_not_found') + } + async function waitForHostSessionHandle( hostTabId: string, isCurrent: () => boolean @@ -508,16 +532,9 @@ export function createRemoteRuntimePtyTransport( const worktree = toRuntimeWorktreeSelector(worktreeId) let activated: RuntimeMobileSessionTabsResult try { - activated = await callRuntime('session.tabs.activate', { - worktree, - tabId: hostTabId, - ...(leafId ? { leafId } : {}), - notifyClients: false, - navigation: 'caller' - }) + activated = await activateHostSessionSurface(hostTabId, worktree) } catch (error) { - const message = runtimeTerminalErrorMessage(error) - if (message.includes('tab_not_found') || message.includes('terminal_not_found')) { + if (isMissingHostSessionSurfaceError(error)) { return null } throw error @@ -634,9 +651,10 @@ export function createRemoteRuntimePtyTransport( const worktree = toRuntimeWorktreeSelector(worktreeId) const startedAt = Date.now() let pollMs = HOST_SESSION_ATTACH_POLL_MS - let lastListError: unknown = null + // Why: list-only polling cannot recreate a host PTY lost across desktop generations. + let nextRequest: 'activate' | 'list' = 'activate' + let lastRequestError: unknown = null let lastReadyHandle: string | null = null - let sawSuccessfulInventory = false const finishBoundedWait = (): HostSessionHandleWaitResult => { const effectivePolicy = stricterReplacementPolicy( replacementPolicy, @@ -645,16 +663,14 @@ export function createRemoteRuntimePtyTransport( if (effectivePolicy === 'prefer-replacement' && lastReadyHandle) { return { handle: lastReadyHandle, inventoryFailed: false } } - if (lastListError) { + if (lastRequestError) { console.warn( - sawSuccessfulInventory - ? '[remote-runtime-pty] final host session inventory poll failed during reconnect:' - : '[remote-runtime-pty] host session inventory unavailable during reconnect:', - runtimeTerminalErrorMessage(lastListError) + '[remote-runtime-pty] host session recovery request failed during reconnect:', + runtimeTerminalErrorMessage(lastRequestError) ) } // Why: a bounded wait without removal evidence is unknown liveness; keep the pane for a later snapshot to reattach. - return { handle: undefined, inventoryFailed: lastListError !== null } + return { handle: undefined, inventoryFailed: lastRequestError !== null } } while ( !destroyed && @@ -666,21 +682,24 @@ export function createRemoteRuntimePtyTransport( if (requestRemainingMs <= 0) { return finishBoundedWait() } + const request = nextRequest try { - const listed = await listRemoteRuntimeSessionTabsDeduped({ - environmentId: currentRuntimeEnvironmentId, - worktreeId, - load: () => - callRuntime( - 'session.tabs.list', - { - worktree - }, - requestRemainingMs - ) - }) - lastListError = null - sawSuccessfulInventory = true + const listed = + request === 'list' + ? await listRemoteRuntimeSessionTabsDeduped({ + environmentId: currentRuntimeEnvironmentId, + worktreeId, + load: () => + callRuntime( + 'session.tabs.list', + { + worktree + }, + requestRemainingMs + ) + }) + : await activateHostSessionSurface(hostTabId, worktree, requestRemainingMs) + lastRequestError = null const nextHandle = findReadyHostSessionHandle(listed, hostTabId) if (nextHandle) { lastReadyHandle = nextHandle @@ -692,12 +711,25 @@ export function createRemoteRuntimePtyTransport( if (nextHandle && (effectivePolicy === 'reuse' || nextHandle !== previousHandle)) { return { handle: nextHandle, inventoryFailed: false } } - if (!hasHostSessionTerminalSurface(listed, hostTabId)) { - return { handle: null, inventoryFailed: false } + if (request === 'list') { + if (!hasHostSessionTerminalSurface(listed, hostTabId)) { + return { handle: null, inventoryFailed: false } + } + if (!nextHandle) { + // Why: the surface is published but unmaterialized, and only activation can mint its PTY. + nextRequest = 'activate' + } + } else { + // Why: an activation response can race host publication, so inventory — not this snapshot — decides what exists. + nextRequest = 'list' } } catch (error) { // Why: the inventory can race the reconnect that invalidated the handle; unknown liveness must not retire the pane. - lastListError = error + lastRequestError = error + if (request === 'activate') { + // Why: no activation failure is absence proof, whether the surface is missing or the host predates the method. + nextRequest = 'list' + } } const remainingMs = HOST_SESSION_ATTACH_TIMEOUT_MS - (Date.now() - startedAt) if (remainingMs <= 0) { diff --git a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts index 5ea3510c2..0485eb729 100644 --- a/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts +++ b/src/renderer/src/runtime/remote-runtime-terminal-multiplexer.ts @@ -178,7 +178,9 @@ type E2eRemoteTerminalMultiplexAckGateSnapshot = { type E2eRemoteTerminalMultiplexAckGateApi = { dropOutputUntilResubscribe: (terminals: string[]) => number + forceError: (terminals: string[], message: string) => number hold: (terminals: string[]) => void + holdEnd: (terminals: string[]) => void release: () => void sendInput: (terminal: string, text: string) => number snapshot: () => E2eRemoteTerminalMultiplexAckGateSnapshot @@ -189,6 +191,7 @@ type E2eRemoteTerminalMultiplexAckGateWindow = Window & { } const e2eHeldRemoteAckTerminals = new Set() +const e2eHeldRemoteEndTerminals = new Set() const e2eDroppedOutputStreams = new Set() let e2eDroppedOutputBytes = 0 let e2eDroppedOutputFrames = 0 @@ -262,15 +265,30 @@ function exposeE2eRemoteTerminalMultiplexAckGate(): void { } return e2eDroppedOutputStreams.size }, + forceError: (terminals, message) => { + let dispatched = 0 + const targets = new Set(terminals) + for (const multiplexer of multiplexers.values()) { + dispatched += multiplexer.forceErrorForE2e(targets, message) + } + return dispatched + }, hold: (terminals) => { releaseE2eRemoteTerminalAcks() for (const terminal of terminals) { e2eHeldRemoteAckTerminals.add(terminal) } }, + holdEnd: (terminals) => { + e2eHeldRemoteEndTerminals.clear() + for (const terminal of terminals) { + e2eHeldRemoteEndTerminals.add(terminal) + } + }, release: () => { releaseE2eRemoteTerminalAcks() resetE2eDroppedRemoteOutput() + e2eHeldRemoteEndTerminals.clear() }, sendInput: (terminal, value) => { let sent = 0 @@ -544,6 +562,13 @@ class RemoteRuntimeTerminalMultiplexer { return } stream.watchdog.recordInbound() + if ( + event.type === 'end' && + e2eConfig.exposeStore && + e2eHeldRemoteEndTerminals.has(stream.terminal) + ) { + return + } if (event.type === 'subscribed') { const capabilities = typeof event.capabilities === 'object' && event.capabilities !== null @@ -1185,6 +1210,17 @@ class RemoteRuntimeTerminalMultiplexer { return this.streams.values() } + forceErrorForE2e(terminals: ReadonlySet, message: string): number { + let dispatched = 0 + for (const stream of this.streams.values()) { + if (terminals.has(stream.terminal)) { + stream.callbacks.onError?.(message) + dispatched += 1 + } + } + return dispatched + } + releaseHeldAcksForE2e(): number { let released = 0 for (const stream of this.streams.values()) { diff --git a/tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts b/tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts new file mode 100644 index 000000000..ef331ee62 --- /dev/null +++ b/tests/e2e/paired-remote-terminal-materialization-reconnect.spec.ts @@ -0,0 +1,391 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { ElectronApplication, Page } from '@stablyai/playwright-test' +import type { + RuntimeMobileSessionTabsResult, + RuntimeTerminalListResult, + RuntimeTerminalRead, + RuntimeTerminalShow +} from '../../src/shared/runtime-types' +import { toWebTerminalSurfaceTabId } from '../../src/shared/terminal-surface-id' +import { expect, test } from './helpers/orca-app' +import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host' +import { + createRuntimeDesktopPairingOffer, + launchPairedElectronClient +} from './helpers/paired-electron-client' +import { getTerminalContent, waitForActivePanePtyId } from './helpers/terminal' + +const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-paired-materialize-')) +const fixturePath = path.join(scratch, 'materialize-terminal.mjs') +const processedInputPath = path.join(scratch, 'processed-input.txt') + +writeFileSync( + fixturePath, + [ + "import { appendFileSync } from 'node:fs'", + 'const processedInputPath = process.argv[2]', + "process.stdout.write('MATERIALIZE_READY\\r\\n')", + "process.stdin.setEncoding('utf8')", + "let pending = ''", + "process.stdin.on('data', (data) => {", + ' pending += data', + ' const commands = pending.split(/\\r\\n|\\r|\\n/)', + ' pending = commands.pop() ?? ""', + ' for (const input of commands) {', + ' appendFileSync(processedInputPath, `${input}\\n`)', + ' process.stdout.write(`LIVE:${input}\\r\\n`)', + ' }', + '})', + 'process.stdin.resume()' + ].join('\n') +) + +test.describe.configure({ mode: 'serial' }) + +test.afterAll(() => { + rmSync(scratch, { recursive: true, force: true }) +}) + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(): string { + const command = [process.execPath, fixturePath, processedInputPath] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +async function callRuntime( + page: Page, + selector: string, + method: string, + params: unknown +): Promise { + return page.evaluate( + async ({ method, params, selector }) => { + const response = await window.api.runtimeEnvironments.call({ selector, method, params }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { method, params, selector } + ) as Promise +} + +async function showClient(app: ElectronApplication, page: Page): Promise { + const clientWindow = await app.browserWindow(page) + await clientWindow.evaluate((window) => { + window.show() + window.focus() + }) + await expect.poll(() => clientWindow.evaluate((window) => window.isVisible())).toBe(true) +} + +async function waitForClientWorktree(page: Page, expectedId?: string): Promise { + await expect + .poll( + () => + page.evaluate( + (id) => + window.__store + ?.getState() + .allWorktrees() + .find((worktree) => !id || worktree.id === id)?.id ?? null, + expectedId + ), + { timeout: 30_000 } + ) + .not.toBeNull() + const worktreeId = await page.evaluate( + (id) => + window.__store + ?.getState() + .allWorktrees() + .find((worktree) => !id || worktree.id === id)?.id ?? null, + expectedId + ) + if (!worktreeId) { + throw new Error('Paired client did not receive the host workspace') + } + return worktreeId +} + +async function hostSurfaceStatus( + page: Page, + environmentId: string, + worktreeId: string, + parentTabId: string +): Promise { + const snapshot = await callRuntime( + page, + environmentId, + 'session.tabs.list', + { worktree: `id:${worktreeId}` } + ) + const surface = snapshot.tabs.find( + (candidate) => candidate.type === 'terminal' && candidate.parentTabId === parentTabId + ) + return surface?.type === 'terminal' ? surface.status : null +} + +/** Park the fixture PTY so the host republishes its pane as a pending handle. + * Why: exact stop only confirms when it observes the fixture exit inside its verification + * window, and a loaded headless host can miss that window even though the PTY is going away. + * The precondition this journey needs is the parked surface, so retry until the host shows it. */ +async function parkHostTerminal( + page: Page, + environmentId: string, + worktreeId: string, + parentTabId: string, + options: { expectedPtyId: string } +): Promise { + let lastError = 'terminal.stopExact was never attempted' + for (let attempt = 0; attempt < 10; attempt += 1) { + const stop = await callRuntime<{ + stopped: number + stoppedPtyIds: string[] + postStopVerified: boolean + }>(page, environmentId, 'terminal.stopExact', { + worktree: `id:${worktreeId}`, + expectedPtyIds: [options.expectedPtyId], + keepHistory: true, + targetOnly: true + }).catch((error: unknown) => { + lastError = error instanceof Error ? error.message : String(error) + return null + }) + if (stop) { + expect(stop.postStopVerified).toBe(true) + expect(stop.stopped).toBe(1) + expect(stop.stoppedPtyIds).toEqual([options.expectedPtyId]) + return + } + // Why: the host reports a set mismatch once the target PTY is no longer live, which is the + // parked state this journey needs even when the stop call itself missed the exit. + if ( + lastError.includes('terminal_stop_pty_set_mismatch') || + (await hostSurfaceStatus(page, environmentId, worktreeId, parentTabId)) !== 'ready' + ) { + return + } + await page.waitForTimeout(1_000) + } + throw new Error(`Host never parked the fixture terminal: ${lastError}`) +} + +async function runMaterializationJourney( + page: Page, + environmentId: string, + worktreeId: string +): Promise { + writeFileSync(processedInputPath, '') + const created = await callRuntime<{ + tab: { parentTabId: string; terminal: string | null } + }>(page, environmentId, 'session.tabs.createTerminal', { + worktree: `id:${worktreeId}`, + command: fixtureCommand(), + activate: false, + select: false, + navigation: 'caller' + }) + const originalHandle = created.tab.terminal + if (!originalHandle) { + throw new Error('Host did not publish the fixture terminal') + } + + const webTabId = toWebTerminalSurfaceTabId(created.tab.parentTabId) + await page.evaluate((id) => window.__store?.getState().setActiveWorktree(id), worktreeId) + const tab = page.locator(`[data-testid="sortable-tab"][data-tab-id="${webTabId}"]`) + await expect(tab).toBeVisible({ timeout: 30_000 }) + await tab.click() + await expect(tab).toHaveAttribute('data-active', 'true') + const originalClientPtyId = await waitForActivePanePtyId(page, 30_000) + await expect + .poll(() => getTerminalContent(page), { timeout: 30_000 }) + .toContain('MATERIALIZE_READY') + + const originalTerminal = await callRuntime<{ terminal: RuntimeTerminalShow }>( + page, + environmentId, + 'terminal.show', + { terminal: originalHandle } + ) + if (!originalTerminal.terminal.ptyId) { + throw new Error('Host fixture terminal has no authoritative PTY') + } + await page.evaluate((terminal) => { + const gate = ( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { holdEnd: (terminals: string[]) => void } + } + ).__remoteTerminalMultiplexAckGate + if (!gate) { + throw new Error('Remote terminal fault gate is unavailable') + } + gate.holdEnd([terminal]) + }, originalHandle) + await parkHostTerminal(page, environmentId, worktreeId, created.tab.parentTabId, { + expectedPtyId: originalTerminal.terminal.ptyId + }) + // Why: the stale error must land on an already-parked surface, or the journey proves nothing + // about materializing a pending handle. + await expect + .poll(() => hostSurfaceStatus(page, environmentId, worktreeId, created.tab.parentTabId), { + timeout: 15_000, + message: 'Host never published the stopped pane as pending-handle' + }) + .toBe('pending-handle') + const dispatched = await page.evaluate((terminal) => { + const gate = ( + window as typeof window & { + __remoteTerminalMultiplexAckGate?: { + forceError: (terminals: string[], message: string) => number + release: () => void + } + } + ).__remoteTerminalMultiplexAckGate + if (!gate) { + throw new Error('Remote terminal fault gate is unavailable') + } + const dispatched = gate.forceError([terminal], 'terminal_handle_stale') + gate.release() + return dispatched + }, originalHandle) + expect(dispatched).toBe(1) + + let replacementHandle: string | null = null + await expect + .poll( + async () => { + const snapshot = await callRuntime( + page, + environmentId, + 'session.tabs.list', + { worktree: `id:${worktreeId}` } + ) + const surface = snapshot.tabs.find( + (candidate) => + candidate.type === 'terminal' && candidate.parentTabId === created.tab.parentTabId + ) + replacementHandle = surface?.type === 'terminal' ? surface.terminal : null + return replacementHandle !== null && replacementHandle !== originalHandle + }, + { timeout: 20_000, message: 'Reconnect never materialized the sleeping host surface' } + ) + .toBe(true) + expect(replacementHandle).not.toBeNull() + + // Why: parking the host PTY can clear this client's active-worktree selection, so reselect the + // pane before reading the PTY it rebound to — the rebind itself is what this journey asserts. + await page.evaluate((id) => window.__store?.getState().setActiveWorktree(id), worktreeId) + await expect(tab).toBeVisible({ timeout: 10_000 }) + await tab.click() + await expect(tab).toHaveAttribute('data-active', 'true') + const replacementClientPtyId = await waitForActivePanePtyId(page, 20_000) + expect(replacementClientPtyId).not.toBe(originalClientPtyId) + + const marker = `MATERIALIZED_${Date.now()}` + await callRuntime(page, environmentId, 'terminal.send', { + terminal: replacementHandle, + text: `echo ${marker}\r`, + client: { id: 'paired-materialization-e2e', type: 'desktop' } + }) + await expect + .poll( + async () => { + const read = await callRuntime<{ terminal: RuntimeTerminalRead }>( + page, + environmentId, + 'terminal.read', + { terminal: replacementHandle } + ) + return read.terminal.tail.join('\n') + }, + { timeout: 10_000 } + ) + .toContain(marker) + await page.evaluate(async (id) => { + await window.__store?.getState().setActiveWorktree(id) + }, worktreeId) + await expect(tab).toBeVisible({ timeout: 10_000 }) + await tab.click() + await expect.poll(() => getTerminalContent(page), { timeout: 10_000 }).toContain(marker) + + const listed = await callRuntime( + page, + environmentId, + 'terminal.list', + { + worktree: `id:${worktreeId}`, + requireFreshPtyLiveness: true + } + ) + expect( + listed.terminals.filter((terminal) => terminal.tabId === created.tab.parentTabId) + ).toHaveLength(1) + await callRuntime(page, environmentId, 'terminal.closeTab', { terminal: replacementHandle }) +} + +test('materializes a stopped terminal on reconnect from a headed paired host', async ({ + orcaPage +}, testInfo) => { + test.setTimeout(120_000) + const worktreeId = await orcaPage.evaluate(() => window.__store?.getState().activeWorktreeId) + if (!worktreeId) { + throw new Error('Headed host has no active seeded workspace') + } + const offer = await createRuntimeDesktopPairingOffer(orcaPage) + const client = await launchPairedElectronClient(offer, testInfo, 'headed-materialization-client') + try { + await showClient(client.app, client.page) + await runMaterializationJourney( + client.page, + client.environmentId, + await waitForClientWorktree(client.page, worktreeId) + ) + expect(await client.getDirectSshAttemptTargetIds()).toEqual([]) + } finally { + await client.dispose() + } +}) + +// Why fixme: this journey's fault injection cannot be set up on a headless `orca serve` host. +// `terminal.stopExact` keeps returning terminal_exact_stop_failed because stopAndWait's +// keep-history verification window expires before the parked PTY is observed gone, so the pane +// never reaches pending-handle and the reconnect behavior is never exercised. That precondition +// fails identically on this PR's base, so it is a pre-existing exact-stop defect rather than a +// reconnect-activation one. The recovery behavior itself was confirmed by hand in this topology +// (the host materializes the pending surface and the client rebinds to the replacement PTY); +// re-enable once exact stop settles deterministically against a serve host. +test.fixme('materializes a stopped terminal on reconnect from a headless folder host', async ({ + testRepoPath +}, testInfo) => { + test.setTimeout(150_000) + const host = await launchHeadlessPairedRuntimeHost() + await host.client.call('repo.add', { path: testRepoPath, kind: 'folder' }) + const client = await launchPairedElectronClient( + host.offer, + testInfo, + 'headless-folder-materialization-client' + ).catch(async (error) => { + await host.dispose() + throw error + }) + try { + await showClient(client.app, client.page) + await runMaterializationJourney( + client.page, + client.environmentId, + await waitForClientWorktree(client.page) + ) + expect(await client.getDirectSshAttemptTargetIds()).toEqual([]) + } finally { + await client.dispose() + await host.dispose() + } +})