fix(remote): harden terminal, relay, and SSH reliability (#8141)
Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
parent
629b57549f
commit
c9919e57b3
|
|
@ -7,6 +7,7 @@ type MockMultiplexer = {
|
|||
notify: ReturnType<typeof vi.fn>
|
||||
onNotification: ReturnType<typeof vi.fn>
|
||||
onNotificationByMethod: ReturnType<typeof vi.fn>
|
||||
onDispose: ReturnType<typeof vi.fn>
|
||||
dispose: ReturnType<typeof vi.fn>
|
||||
isDisposed: ReturnType<typeof vi.fn>
|
||||
}
|
||||
|
|
@ -17,6 +18,9 @@ function createMockMux(): MockMultiplexer {
|
|||
notify: vi.fn(),
|
||||
onNotification: vi.fn(),
|
||||
onNotificationByMethod: vi.fn().mockReturnValue(vi.fn()),
|
||||
// Why: requestGitStreamable subscribes to onDispose before awaiting the
|
||||
// response so it can reject in-flight reassembly if the link drops.
|
||||
onDispose: vi.fn().mockReturnValue(vi.fn()),
|
||||
dispose: vi.fn(),
|
||||
isDisposed: vi.fn().mockReturnValue(false)
|
||||
}
|
||||
|
|
@ -320,7 +324,10 @@ describe('SshGitProvider', () => {
|
|||
'git.exec',
|
||||
{
|
||||
args: ['clone', '--progress', '--', 'git@example.com:repo.git', 'repo'],
|
||||
cwd: '/home/user'
|
||||
cwd: '/home/user',
|
||||
// Why: exec opts into response streaming so a large stdout is chunked
|
||||
// onto the bulk lane; old relays ignore the flag.
|
||||
__streamResponse: true
|
||||
},
|
||||
{
|
||||
signal: controller.signal,
|
||||
|
|
@ -353,7 +360,8 @@ describe('SshGitProvider', () => {
|
|||
})
|
||||
expect(mux.request).toHaveBeenCalledWith('git.exec', {
|
||||
args: ['diff', '--cached', '--patch', '--minimal', '--no-color', '--no-ext-diff'],
|
||||
cwd: '/home/user/repo'
|
||||
cwd: '/home/user/repo',
|
||||
__streamResponse: true
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -636,7 +644,10 @@ describe('SshGitProvider', () => {
|
|||
expect(mux.request).toHaveBeenCalledWith('git.diff', {
|
||||
worktreePath: '/home/user/repo',
|
||||
filePath: 'src/index.ts',
|
||||
staged: true
|
||||
staged: true,
|
||||
// Why: opts into response streaming; a small result still comes back as a
|
||||
// single frame (relay decides), and old relays ignore the flag.
|
||||
__streamResponse: true
|
||||
})
|
||||
expect(result).toEqual(diffResult)
|
||||
})
|
||||
|
|
@ -880,7 +891,8 @@ describe('SshGitProvider', () => {
|
|||
const result = await provider.getBranchDiff('/home/user/repo', 'main')
|
||||
expect(mux.request).toHaveBeenCalledWith('git.branchDiff', {
|
||||
worktreePath: '/home/user/repo',
|
||||
baseRef: 'main'
|
||||
baseRef: 'main',
|
||||
__streamResponse: true
|
||||
})
|
||||
expect(result).toEqual(diffs)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import type {
|
|||
import type { GitHistoryOptions, GitHistoryResult } from '../../shared/git-history'
|
||||
import { buildHostedRemoteCommitUrl, buildHostedRemoteFileUrl } from '../git/hosted-remote-url'
|
||||
import { JsonRpcErrorCode } from '../ssh/relay-protocol'
|
||||
import { requestGitStreamable } from '../ssh/ssh-git-response-stream-reader'
|
||||
import type { CommitMessageDraftContext } from '../../shared/commit-message-generation'
|
||||
import type { CommitMessagePlan } from '../../shared/commit-message-plan'
|
||||
import type { RemoteCommitMessageExecResult } from '../text-generation/commit-message-text-generation'
|
||||
|
|
@ -373,7 +374,7 @@ export class SshGitProvider implements IGitProvider {
|
|||
return this.gitDiffReadDedupe.run(
|
||||
stableInFlightKey(['diff', worktreePath, filePath, staged, compareAgainstHead]),
|
||||
async () =>
|
||||
(await this.mux.request('git.diff', {
|
||||
(await requestGitStreamable(this.mux, 'git.diff', {
|
||||
worktreePath,
|
||||
filePath,
|
||||
staged,
|
||||
|
|
@ -597,7 +598,7 @@ export class SshGitProvider implements IGitProvider {
|
|||
keyOptions.oldPath ?? null
|
||||
]),
|
||||
async () =>
|
||||
(await this.mux.request('git.branchDiff', {
|
||||
(await requestGitStreamable(this.mux, 'git.branchDiff', {
|
||||
worktreePath,
|
||||
baseRef,
|
||||
...options
|
||||
|
|
@ -619,7 +620,7 @@ export class SshGitProvider implements IGitProvider {
|
|||
args.oldPath ?? null
|
||||
]),
|
||||
async () =>
|
||||
(await this.mux.request('git.commitDiff', {
|
||||
(await requestGitStreamable(this.mux, 'git.commitDiff', {
|
||||
worktreePath,
|
||||
...args
|
||||
})) as GitDiffResult
|
||||
|
|
@ -761,8 +762,8 @@ export class SshGitProvider implements IGitProvider {
|
|||
options?: { signal?: AbortSignal; timeoutMs?: number }
|
||||
): Promise<{ stdout: string; stderr: string }> {
|
||||
const result = options
|
||||
? await this.mux.request('git.exec', { args, cwd }, options)
|
||||
: await this.mux.request('git.exec', { args, cwd })
|
||||
? await requestGitStreamable(this.mux, 'git.exec', { args, cwd }, options)
|
||||
: await requestGitStreamable(this.mux, 'git.exec', { args, cwd })
|
||||
return result as {
|
||||
stdout: string
|
||||
stderr: string
|
||||
|
|
|
|||
|
|
@ -0,0 +1,120 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { createMobileSessionTabsNotifyCoalescer } from './mobile-session-tabs-notify-coalescer'
|
||||
|
||||
describe('createMobileSessionTabsNotifyCoalescer', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('coalesces a rapid burst of title/status flips into one trailing emit (repro)', () => {
|
||||
// Repro of the churn bug: a spinner-in-title agent flips a PTY title many
|
||||
// times a second. Without coalescing every flip fans out an emit; with it,
|
||||
// the whole burst settles into a single trailing-edge emit per worktree.
|
||||
const emit = vi.fn()
|
||||
const coalescer = createMobileSessionTabsNotifyCoalescer(emit)
|
||||
|
||||
const FLIPS = 20
|
||||
for (let i = 0; i < FLIPS; i++) {
|
||||
coalescer.schedule('worktree-1')
|
||||
// Each flip lands well inside the trailing window, resetting the timer.
|
||||
vi.advanceTimersByTime(10)
|
||||
}
|
||||
expect(emit).not.toHaveBeenCalled()
|
||||
|
||||
// Let the trailing window elapse after the last flip.
|
||||
vi.advanceTimersByTime(50)
|
||||
|
||||
expect(emit).toHaveBeenCalledTimes(1)
|
||||
expect(emit).toHaveBeenCalledWith('worktree-1')
|
||||
})
|
||||
|
||||
it('force-flushes under sustained churn so the emit is never starved', () => {
|
||||
const emit = vi.fn()
|
||||
const coalescer = createMobileSessionTabsNotifyCoalescer(emit)
|
||||
|
||||
// Keep flipping faster than the trailing window forever; the max-wait cap
|
||||
// (250ms) must force at least one emit within that budget.
|
||||
for (let i = 0; i < 100; i++) {
|
||||
coalescer.schedule('worktree-1')
|
||||
vi.advanceTimersByTime(30)
|
||||
}
|
||||
|
||||
expect(emit).toHaveBeenCalled()
|
||||
expect(emit).toHaveBeenCalledWith('worktree-1')
|
||||
})
|
||||
|
||||
it('keeps per-worktree windows independent', () => {
|
||||
const emit = vi.fn()
|
||||
const coalescer = createMobileSessionTabsNotifyCoalescer(emit)
|
||||
|
||||
coalescer.schedule('worktree-1')
|
||||
coalescer.schedule('worktree-2')
|
||||
vi.advanceTimersByTime(50)
|
||||
|
||||
expect(emit).toHaveBeenCalledTimes(2)
|
||||
expect(emit).toHaveBeenCalledWith('worktree-1')
|
||||
expect(emit).toHaveBeenCalledWith('worktree-2')
|
||||
})
|
||||
|
||||
it('cancel() drops a pending notify so a structural emit can supersede it', () => {
|
||||
const emit = vi.fn()
|
||||
const coalescer = createMobileSessionTabsNotifyCoalescer(emit)
|
||||
|
||||
coalescer.schedule('worktree-1')
|
||||
// A structural change (tab add/remove) cancels the coalesced notify because
|
||||
// it emits immediately elsewhere.
|
||||
coalescer.cancel('worktree-1')
|
||||
vi.advanceTimersByTime(50)
|
||||
|
||||
expect(emit).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('flush() emits the pending notify immediately', () => {
|
||||
const emit = vi.fn()
|
||||
const coalescer = createMobileSessionTabsNotifyCoalescer(emit)
|
||||
|
||||
coalescer.schedule('worktree-1')
|
||||
coalescer.flush('worktree-1')
|
||||
|
||||
expect(emit).toHaveBeenCalledTimes(1)
|
||||
// The timer must not double-fire afterward.
|
||||
vi.advanceTimersByTime(50)
|
||||
expect(emit).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('flush() is a no-op when nothing is pending', () => {
|
||||
const emit = vi.fn()
|
||||
const coalescer = createMobileSessionTabsNotifyCoalescer(emit)
|
||||
|
||||
coalescer.flush('worktree-1')
|
||||
|
||||
expect(emit).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('flushAll() drains every pending worktree (subscription close)', () => {
|
||||
const emit = vi.fn()
|
||||
const coalescer = createMobileSessionTabsNotifyCoalescer(emit)
|
||||
|
||||
coalescer.schedule('worktree-1')
|
||||
coalescer.schedule('worktree-2')
|
||||
coalescer.flushAll()
|
||||
|
||||
expect(emit).toHaveBeenCalledTimes(2)
|
||||
vi.advanceTimersByTime(50)
|
||||
expect(emit).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('dispose() drops pending timers without emitting', () => {
|
||||
const emit = vi.fn()
|
||||
const coalescer = createMobileSessionTabsNotifyCoalescer(emit)
|
||||
|
||||
coalescer.schedule('worktree-1')
|
||||
coalescer.dispose()
|
||||
vi.advanceTimersByTime(50)
|
||||
|
||||
expect(emit).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
// Why: spinner-in-title agents (e.g. the Claude Code braille spinner) flip a
|
||||
// PTY title several times per second. Each flip touched every session.tabs
|
||||
// snapshot and fanned a fresh emit out to every subscriber, where the ws layer
|
||||
// JSON.stringifies it per client — O(clients × snapshot size) of churn work
|
||||
// with no debounce. Clients gate on snapshotVersion freshness, so only the
|
||||
// newest version per worktree matters; coalescing the intermediate emits is
|
||||
// safe. Structural changes (tab added/removed/activated) bypass this via an
|
||||
// immediate flush so they still propagate promptly.
|
||||
|
||||
// Trailing-edge window: title/status is latency-sensitive UI, so this is
|
||||
// tighter than files.watch's 150ms but looser than native-chat's 40ms.
|
||||
const SESSION_TABS_FLUSH_MS = 50
|
||||
// Force a flush after this long even under sustained churn, so a title that
|
||||
// keeps spinning never starves the emit indefinitely.
|
||||
const SESSION_TABS_MAX_WAIT_MS = 250
|
||||
|
||||
export type MobileSessionTabsNotifyCoalescer = {
|
||||
// Schedule a coalesced (trailing-edge) notify for a worktree.
|
||||
schedule: (worktreeId: string) => void
|
||||
// Cancel any pending notify for a worktree without emitting. Use when an
|
||||
// immediate emit has already superseded the pending state, or the worktree
|
||||
// was removed and a stale notify must not fire.
|
||||
cancel: (worktreeId: string) => void
|
||||
// Flush a worktree's pending notify now (emit if one is pending).
|
||||
flush: (worktreeId: string) => void
|
||||
// Flush every pending worktree now.
|
||||
flushAll: () => void
|
||||
// Drop all pending state without emitting (runtime teardown).
|
||||
dispose: () => void
|
||||
}
|
||||
|
||||
type PendingNotify = {
|
||||
timer: ReturnType<typeof setTimeout>
|
||||
firstScheduledAt: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Coalesces per-worktree session.tabs notifications on a short trailing-edge
|
||||
* window. `emit` is invoked once per settled worktree and is expected to read
|
||||
* the latest snapshot itself, so only the freshest snapshotVersion is ever
|
||||
* published — dropped intermediate versions are exactly what clients discard.
|
||||
*/
|
||||
export function createMobileSessionTabsNotifyCoalescer(
|
||||
emit: (worktreeId: string) => void
|
||||
): MobileSessionTabsNotifyCoalescer {
|
||||
const pending = new Map<string, PendingNotify>()
|
||||
|
||||
const clear = (worktreeId: string): void => {
|
||||
const entry = pending.get(worktreeId)
|
||||
if (!entry) {
|
||||
return
|
||||
}
|
||||
clearTimeout(entry.timer)
|
||||
pending.delete(worktreeId)
|
||||
}
|
||||
|
||||
const fire = (worktreeId: string): void => {
|
||||
clear(worktreeId)
|
||||
emit(worktreeId)
|
||||
}
|
||||
|
||||
const arm = (worktreeId: string): ReturnType<typeof setTimeout> => {
|
||||
const timer = setTimeout(() => fire(worktreeId), SESSION_TABS_FLUSH_MS)
|
||||
if (typeof timer.unref === 'function') {
|
||||
timer.unref()
|
||||
}
|
||||
return timer
|
||||
}
|
||||
|
||||
return {
|
||||
schedule(worktreeId: string): void {
|
||||
const now = Date.now()
|
||||
const existing = pending.get(worktreeId)
|
||||
if (existing) {
|
||||
// Cap total delay so sustained churn can't starve the emit forever.
|
||||
if (now - existing.firstScheduledAt >= SESSION_TABS_MAX_WAIT_MS) {
|
||||
fire(worktreeId)
|
||||
return
|
||||
}
|
||||
clearTimeout(existing.timer)
|
||||
existing.timer = arm(worktreeId)
|
||||
return
|
||||
}
|
||||
pending.set(worktreeId, { timer: arm(worktreeId), firstScheduledAt: now })
|
||||
},
|
||||
cancel(worktreeId: string): void {
|
||||
clear(worktreeId)
|
||||
},
|
||||
flush(worktreeId: string): void {
|
||||
if (pending.has(worktreeId)) {
|
||||
fire(worktreeId)
|
||||
}
|
||||
},
|
||||
flushAll(): void {
|
||||
// Snapshot keys first: fire() deletes from `pending`, and emit may
|
||||
// schedule new work, so mutating the live map mid-iteration is unsafe.
|
||||
const worktreeIds = Array.from(pending.keys())
|
||||
for (const worktreeId of worktreeIds) {
|
||||
fire(worktreeId)
|
||||
}
|
||||
},
|
||||
dispose(): void {
|
||||
for (const entry of pending.values()) {
|
||||
clearTimeout(entry.timer)
|
||||
}
|
||||
pending.clear()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'
|
||||
import type * as GitUsernameModule from '../git/git-username'
|
||||
import type { RuntimeMobileSessionTabsSnapshot } from '../../shared/runtime-types'
|
||||
import { OrcaRuntimeService } from './orca-runtime'
|
||||
|
||||
vi.mock('../git/worktree', () => ({
|
||||
|
|
@ -983,4 +984,109 @@ describe('mobile subscribe integration', () => {
|
|||
expect(await runtime.reclaimTerminalForDesktop('pty-1')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('session.tabs title/status churn coalescing', () => {
|
||||
type SessionTabsPrivate = {
|
||||
mobileSessionTabsByWorktree: Map<string, RuntimeMobileSessionTabsSnapshot>
|
||||
touchMobileSessionSnapshotsForPty: (ptyId: string) => void
|
||||
notifyMobileSessionTabsChanged: (worktreeId?: string) => void
|
||||
}
|
||||
|
||||
function seedPtyBackedSnapshot(runtime: OrcaRuntimeService, ptyId: string): void {
|
||||
const priv = runtime as unknown as SessionTabsPrivate
|
||||
priv.mobileSessionTabsByWorktree.set('worktree-a', {
|
||||
worktree: 'worktree-a',
|
||||
publicationEpoch: 'epoch-1',
|
||||
snapshotVersion: 1,
|
||||
activeGroupId: 'group-1',
|
||||
activeTabId: 'tab-1',
|
||||
activeTabType: 'terminal',
|
||||
tabs: [
|
||||
{
|
||||
type: 'terminal',
|
||||
id: 'tab-1',
|
||||
title: 'agent',
|
||||
parentTabId: 'group-1',
|
||||
leafId: 'leaf-1',
|
||||
ptyId,
|
||||
isActive: true
|
||||
}
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
it('collapses a rapid title-flip burst into a single emit (repro)', () => {
|
||||
const { runtime } = createRuntime()
|
||||
seedPtyBackedSnapshot(runtime, 'pty-1')
|
||||
const priv = runtime as unknown as SessionTabsPrivate
|
||||
|
||||
const emits: number[] = []
|
||||
const unsubscribe = runtime.onMobileSessionTabsChanged((snapshot) => {
|
||||
emits.push(snapshot.snapshotVersion)
|
||||
})
|
||||
|
||||
// A spinner-in-title agent flips the title ~20 times within a second.
|
||||
const FLIPS = 20
|
||||
for (let i = 0; i < FLIPS; i++) {
|
||||
priv.touchMobileSessionSnapshotsForPty('pty-1')
|
||||
vi.advanceTimersByTime(10)
|
||||
}
|
||||
// Nothing has fired yet — all flips landed inside the trailing window.
|
||||
expect(emits).toHaveLength(0)
|
||||
|
||||
vi.advanceTimersByTime(50)
|
||||
|
||||
// Pre-fix this fanned out one emit per flip; now it is a single emit.
|
||||
expect(emits).toHaveLength(1)
|
||||
// The emitted version is the freshest (monotonic, never a stale one).
|
||||
const stored = priv.mobileSessionTabsByWorktree.get('worktree-a')
|
||||
expect(emits[0]).toBe(stored?.snapshotVersion)
|
||||
expect(emits[0]).toBe(1 + FLIPS)
|
||||
|
||||
unsubscribe()
|
||||
})
|
||||
|
||||
it('lets a structural change emit immediately and supersede the coalesced notify', () => {
|
||||
const { runtime } = createRuntime()
|
||||
seedPtyBackedSnapshot(runtime, 'pty-1')
|
||||
const priv = runtime as unknown as SessionTabsPrivate
|
||||
|
||||
const emits: number[] = []
|
||||
const unsubscribe = runtime.onMobileSessionTabsChanged((snapshot) => {
|
||||
emits.push(snapshot.snapshotVersion)
|
||||
})
|
||||
|
||||
// Title churn schedules a coalesced notify...
|
||||
priv.touchMobileSessionSnapshotsForPty('pty-1')
|
||||
// ...then a structural change (e.g. tab activated) demands a prompt emit.
|
||||
priv.notifyMobileSessionTabsChanged('worktree-a')
|
||||
expect(emits).toHaveLength(1)
|
||||
|
||||
// The pending coalesced notify was cancelled by the immediate emit, so no
|
||||
// duplicate trailing emit fires.
|
||||
vi.advanceTimersByTime(50)
|
||||
expect(emits).toHaveLength(1)
|
||||
|
||||
unsubscribe()
|
||||
})
|
||||
|
||||
it('flushes a pending notify when the last subscriber closes', () => {
|
||||
const { runtime } = createRuntime()
|
||||
seedPtyBackedSnapshot(runtime, 'pty-1')
|
||||
const priv = runtime as unknown as SessionTabsPrivate
|
||||
|
||||
const emits: number[] = []
|
||||
const unsubscribe = runtime.onMobileSessionTabsChanged((snapshot) => {
|
||||
emits.push(snapshot.snapshotVersion)
|
||||
})
|
||||
|
||||
priv.touchMobileSessionSnapshotsForPty('pty-1')
|
||||
expect(emits).toHaveLength(0)
|
||||
|
||||
// Closing the subscription flushes the pending window so the final state
|
||||
// still reaches the listener before it is dropped.
|
||||
unsubscribe()
|
||||
expect(emits).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -89,6 +89,13 @@ import { TERMINAL_METHODS } from './rpc/methods/terminal'
|
|||
const ORIGINAL_PLATFORM = process.platform
|
||||
const ORIGINAL_PLATFORM_DESCRIPTOR = Object.getOwnPropertyDescriptor(process, 'platform')
|
||||
|
||||
async function waitForMobileSessionTabsEvents(
|
||||
events: RuntimeMobileSessionTabsResult[],
|
||||
count: number
|
||||
): Promise<void> {
|
||||
await vi.waitFor(() => expect(events).toHaveLength(count))
|
||||
}
|
||||
|
||||
function setPlatform(platform: NodeJS.Platform): void {
|
||||
Object.defineProperty(process, 'platform', {
|
||||
configurable: true,
|
||||
|
|
@ -11336,6 +11343,7 @@ describe('OrcaRuntimeService', () => {
|
|||
runtime.onPtyData('laptop-created-pty', '\x1b]0;⠴ - Thinking - grok\x07', 101)
|
||||
runtime.onPtyData('laptop-created-pty', '\x1b]0;⠙ - Responding - grok\x07', 102)
|
||||
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
expect(events).toHaveLength(1)
|
||||
expect(events[0]?.tabs[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
|
|
@ -11375,6 +11383,7 @@ describe('OrcaRuntimeService', () => {
|
|||
100
|
||||
)
|
||||
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
expect(events).toHaveLength(1)
|
||||
expect(events[0]?.tabs[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
|
|
@ -11393,6 +11402,7 @@ describe('OrcaRuntimeService', () => {
|
|||
101
|
||||
)
|
||||
|
||||
await waitForMobileSessionTabsEvents(events, 2)
|
||||
expect(events).toHaveLength(2)
|
||||
expect(events[1]?.tabs[0]?.type === 'terminal' && events[1].tabs[0].agentStatus).toEqual(
|
||||
expect.objectContaining({ state: 'waiting' })
|
||||
|
|
@ -11424,6 +11434,7 @@ describe('OrcaRuntimeService', () => {
|
|||
runtime.onPtyData('hook-ping-pty', payload, 101)
|
||||
runtime.onPtyData('hook-ping-pty', payload, 102)
|
||||
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
expect(events).toHaveLength(1)
|
||||
|
||||
unsubscribe()
|
||||
|
|
@ -11456,6 +11467,7 @@ describe('OrcaRuntimeService', () => {
|
|||
// The stuck-spinner guard (#1437) must win over the retained hook row.
|
||||
runtime.onPtyData('hook-exit-pty', '\x1b]0;zsh\x07', 101)
|
||||
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
const last = events.at(-1)?.tabs[0]
|
||||
expect(last?.type).toBe('terminal')
|
||||
expect(last?.type === 'terminal' ? last.agentStatus : null).toBeFalsy()
|
||||
|
|
@ -13681,6 +13693,7 @@ describe('OrcaRuntimeService', () => {
|
|||
|
||||
foregroundProcess.resolve('omp')
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({
|
||||
|
|
@ -13731,6 +13744,7 @@ describe('OrcaRuntimeService', () => {
|
|||
|
||||
foregroundProcess.resolve('omp')
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({
|
||||
|
|
@ -13792,6 +13806,7 @@ describe('OrcaRuntimeService', () => {
|
|||
|
||||
freshForegroundProcess.resolve('omp')
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
|
||||
expect(getForegroundProcess).toHaveBeenCalledTimes(2)
|
||||
expect(events).toEqual([
|
||||
|
|
@ -13856,6 +13871,7 @@ describe('OrcaRuntimeService', () => {
|
|||
|
||||
freshForegroundProcess.resolve('omp')
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({
|
||||
|
|
@ -15942,19 +15958,8 @@ describe('OrcaRuntimeService', () => {
|
|||
runtime.onPtyData('laptop-created-pty', '\x1b]0;Claude working\x07', 123)
|
||||
runtime.onPtyData('laptop-created-pty', '\x1b]0;Claude waiting for permission\x07', 124)
|
||||
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({
|
||||
tabs: [
|
||||
expect.objectContaining({
|
||||
type: 'terminal',
|
||||
title: 'Claude working',
|
||||
agentStatus: expect.objectContaining({
|
||||
state: 'working',
|
||||
terminalHandle: laptopTerminal.handle
|
||||
})
|
||||
})
|
||||
]
|
||||
}),
|
||||
expect.objectContaining({
|
||||
tabs: [
|
||||
expect.objectContaining({
|
||||
|
|
@ -15967,7 +15972,6 @@ describe('OrcaRuntimeService', () => {
|
|||
]
|
||||
})
|
||||
])
|
||||
expect(events[1]!.snapshotVersion).toBeGreaterThan(events[0]!.snapshotVersion)
|
||||
|
||||
unsubscribe()
|
||||
})
|
||||
|
|
@ -15993,19 +15997,14 @@ describe('OrcaRuntimeService', () => {
|
|||
runtime.onPtyData('laptop-created-pty', '\x1b]0;Claude working\x07', 123)
|
||||
runtime.onPtyData('laptop-created-pty', '\x1b]0;claude agents\x07', 124)
|
||||
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
expect(events[0]?.tabs[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
type: 'terminal',
|
||||
agentStatus: expect.objectContaining({ state: 'working' })
|
||||
})
|
||||
)
|
||||
expect(events[1]?.tabs[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
type: 'terminal',
|
||||
title: 'claude agents'
|
||||
})
|
||||
)
|
||||
expect(events[1]?.tabs[0]).not.toHaveProperty('agentStatus')
|
||||
expect(events[0]?.tabs[0]).not.toHaveProperty('agentStatus')
|
||||
|
||||
unsubscribe()
|
||||
})
|
||||
|
|
@ -17624,6 +17623,7 @@ describe('OrcaRuntimeService', () => {
|
|||
events.length = 0
|
||||
runtime.onPtyData('headless-pty-2', '\x1b]0;tab-other running\x07', 200)
|
||||
|
||||
await waitForMobileSessionTabsEvents(events, 1)
|
||||
const afterPtyData = events.at(-1)
|
||||
expect(afterPtyData?.activeTabId).toBe(`tab-other::${SECOND_LEAF}`)
|
||||
expect(afterPtyData?.activeTabType).toBe('terminal')
|
||||
|
|
|
|||
|
|
@ -685,6 +685,10 @@ import { closeLocalWatcherForWorktreePath } from '../ipc/filesystem-watcher'
|
|||
import { HeadlessEmulator, type HeadlessEmulatorOptions } from '../daemon/headless-emulator'
|
||||
import { killAllProcessesForWorktree } from './worktree-teardown'
|
||||
import { MOBILE_SUBSCRIBE_SCROLLBACK_ROWS } from './scrollback-limits'
|
||||
import {
|
||||
createMobileSessionTabsNotifyCoalescer,
|
||||
type MobileSessionTabsNotifyCoalescer
|
||||
} from './mobile-session-tabs-notify-coalescer'
|
||||
import type { IFilesystemProvider, IPtyProvider, PtyProcessInfo } from '../providers/types'
|
||||
import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch'
|
||||
import {
|
||||
|
|
@ -2060,6 +2064,13 @@ export class OrcaRuntimeService {
|
|||
{ activate: boolean; selectIfNoActiveTab: boolean }
|
||||
>()
|
||||
private mobileSessionTabListeners = new Set<(snapshot: RuntimeMobileSessionTabsResult) => void>()
|
||||
// Why: coalesces title/status-driven session.tabs emits so spinner churn
|
||||
// doesn't fan out (and per-client JSON.stringify) a snapshot several times a
|
||||
// second. Emit reads the latest snapshot, so only the freshest version ships.
|
||||
private readonly mobileSessionTabsNotifyCoalescer: MobileSessionTabsNotifyCoalescer =
|
||||
createMobileSessionTabsNotifyCoalescer((worktreeId) =>
|
||||
this.notifyMobileSessionTabsChangedNow(worktreeId)
|
||||
)
|
||||
private leaves = new Map<string, RuntimeLeafRecord>()
|
||||
// Why: PTY output is a per-keystroke hot path. Looking up affected leaves by
|
||||
// ptyId keeps active TUI redraws independent of the total open terminal count.
|
||||
|
|
@ -3532,7 +3543,10 @@ export class OrcaRuntimeService {
|
|||
this.notifyMobileSessionTabsChanged(worktreeId)
|
||||
}
|
||||
|
||||
private touchMobileSessionSnapshotsForPty(ptyId: string): void {
|
||||
private touchMobileSessionSnapshotsForPty(
|
||||
ptyId: string,
|
||||
options: { immediate?: boolean } = {}
|
||||
): void {
|
||||
for (const [worktreeId, snapshot] of this.mobileSessionTabsByWorktree) {
|
||||
const hasPtyBackedTab = snapshot.tabs.some(
|
||||
(tab) =>
|
||||
|
|
@ -3546,7 +3560,15 @@ export class OrcaRuntimeService {
|
|||
...snapshot,
|
||||
snapshotVersion: snapshot.snapshotVersion + 1
|
||||
})
|
||||
this.notifyMobileSessionTabsChanged(worktreeId)
|
||||
if (options.immediate) {
|
||||
// Why: readiness/lifecycle changes are structural and must not wait
|
||||
// behind the title/status coalescing window.
|
||||
this.notifyMobileSessionTabsChanged(worktreeId)
|
||||
} else {
|
||||
// Why: title/status flips several times a second under spinner-in-title
|
||||
// agents. Coalesce the emit instead of fanning out every version.
|
||||
this.mobileSessionTabsNotifyCoalescer.schedule(worktreeId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -5208,6 +5230,9 @@ export class OrcaRuntimeService {
|
|||
): () => void {
|
||||
this.mobileSessionTabListeners.add(listener)
|
||||
return () => {
|
||||
// Why: flush pending coalesced notifies before dropping this listener so a
|
||||
// subscriber closing mid-window still receives the latest settled state.
|
||||
this.mobileSessionTabsNotifyCoalescer.flushAll()
|
||||
this.mobileSessionTabListeners.delete(listener)
|
||||
}
|
||||
}
|
||||
|
|
@ -7240,7 +7265,7 @@ export class OrcaRuntimeService {
|
|||
pty.lastExitCode = exitCode
|
||||
this.resolvePtyExitWaiters(pty, ptyId)
|
||||
this.pruneDisconnectedPtyTranscript(pty)
|
||||
this.touchMobileSessionSnapshotsForPty(ptyId)
|
||||
this.touchMobileSessionSnapshotsForPty(ptyId, { immediate: true })
|
||||
}
|
||||
|
||||
for (const leaf of this.getLeavesForPty(ptyId)) {
|
||||
|
|
@ -18884,6 +18909,9 @@ export class OrcaRuntimeService {
|
|||
nextWorktrees.add(worktreeId)
|
||||
} else {
|
||||
this.mobileSessionTabsByWorktree.delete(worktreeId)
|
||||
// Why: drop any pending coalesced notify so a stale snapshot can't
|
||||
// land after the removed frame.
|
||||
this.mobileSessionTabsNotifyCoalescer.cancel(worktreeId)
|
||||
this.notifyMobileSessionTabsRemoved(worktreeId)
|
||||
}
|
||||
}
|
||||
|
|
@ -19058,6 +19086,14 @@ export class OrcaRuntimeService {
|
|||
this.notifyMobileSessionTabSnapshots()
|
||||
return
|
||||
}
|
||||
// Why: structural changes (tab add/remove/activate) must propagate promptly,
|
||||
// so cancel any pending coalesced title/status notify — this immediate emit
|
||||
// already reflects the latest snapshot and supersedes it.
|
||||
this.mobileSessionTabsNotifyCoalescer.cancel(worktreeId)
|
||||
this.notifyMobileSessionTabsChangedNow(worktreeId)
|
||||
}
|
||||
|
||||
private notifyMobileSessionTabsChangedNow(worktreeId: string): void {
|
||||
if (this.mobileSessionTabListeners.size === 0) {
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ export class RpcDispatcher {
|
|||
signal?: AbortSignal
|
||||
clientId?: string
|
||||
clientKind?: 'mobile' | 'runtime'
|
||||
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => void
|
||||
sendBinary?: (bytes: Uint8Array<ArrayBufferLike>) => boolean | void
|
||||
registerBinaryStreamHandler?: (
|
||||
streamId: number,
|
||||
handler: (frame: TerminalStreamFrame) => void
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { WebSocket } from 'ws'
|
||||
import { E2EEChannel, type E2EEChannelOptions } from './e2ee-channel'
|
||||
import { deriveSharedKey, decrypt, encrypt, generateKeyPair } from './e2ee-crypto'
|
||||
|
||||
// Repro for gap (a): the streaming JSON reply path (encryptedReply) had no
|
||||
// bufferedAmount gate, so a fast producer over a slow link (legacy
|
||||
// terminal.subscribe, which has NO seq/resync) ballooned ws.bufferedAmount
|
||||
// without bound. The fix holds replies in order and drains on recovery — never
|
||||
// dropping a frame (which would recreate the corruption bug on the legacy path).
|
||||
|
||||
function publicKeyToBase64(key: Uint8Array): string {
|
||||
return Buffer.from(key).toString('base64')
|
||||
}
|
||||
|
||||
function createMockWs() {
|
||||
const sent: string[] = []
|
||||
return {
|
||||
OPEN: 1 as const,
|
||||
readyState: 1,
|
||||
bufferedAmount: 0,
|
||||
send: vi.fn((data: string) => {
|
||||
sent.push(data)
|
||||
}),
|
||||
close: vi.fn(),
|
||||
sent
|
||||
}
|
||||
}
|
||||
|
||||
function setup(overrides?: Partial<E2EEChannelOptions>) {
|
||||
const serverKeys = generateKeyPair()
|
||||
const clientKeys = generateKeyPair()
|
||||
const ws = createMockWs()
|
||||
const onError = vi.fn()
|
||||
const channel = new E2EEChannel(ws as unknown as WebSocket, {
|
||||
serverSecretKey: serverKeys.secretKey,
|
||||
validateToken: (token) => token === 'valid-token',
|
||||
onReady: vi.fn(),
|
||||
onError,
|
||||
...overrides
|
||||
})
|
||||
const sharedKey = deriveSharedKey(clientKeys.secretKey, serverKeys.publicKey)
|
||||
channel.handleRawMessage(
|
||||
JSON.stringify({ type: 'e2ee_hello', publicKeyB64: publicKeyToBase64(clientKeys.publicKey) })
|
||||
)
|
||||
channel.handleRawMessage(
|
||||
encrypt(JSON.stringify({ type: 'e2ee_auth', deviceToken: 'valid-token' }), sharedKey)
|
||||
)
|
||||
return { channel, ws, sharedKey, onError }
|
||||
}
|
||||
|
||||
/** Fire a streaming reply through the real channel's encryptedReply closure. */
|
||||
function emitReply(ctx: ReturnType<typeof setup>, payload: string): void {
|
||||
ctx.channel.onMessage((_plaintext, encryptedReply) => {
|
||||
encryptedReply(payload)
|
||||
})
|
||||
ctx.channel.handleRawMessage(encrypt('{"id":"x","method":"status.get"}', ctx.sharedKey))
|
||||
}
|
||||
|
||||
describe('E2EE text reply backpressure', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('holds text replies while over the buffer cap and drains them in order', () => {
|
||||
const ctx = setup()
|
||||
const baseline = ctx.ws.sent.length // ready + authenticated control frames
|
||||
|
||||
// Simulate a congested socket: bufferedAmount pinned over the 8 MiB cap.
|
||||
ctx.ws.bufferedAmount = 9 * 1024 * 1024
|
||||
|
||||
emitReply(ctx, '{"seq":1}')
|
||||
emitReply(ctx, '{"seq":2}')
|
||||
emitReply(ctx, '{"seq":3}')
|
||||
|
||||
// Not dropped, not sent yet — parked in order on the queue.
|
||||
expect(ctx.ws.sent.length).toBe(baseline)
|
||||
|
||||
// Link drains; the queue flushes every reply, in order, none lost.
|
||||
ctx.ws.bufferedAmount = 0
|
||||
vi.runOnlyPendingTimers()
|
||||
|
||||
const replies = ctx.ws.sent.slice(baseline).map((frame) => decrypt(frame, ctx.sharedKey))
|
||||
expect(replies).toEqual(['{"seq":1}', '{"seq":2}', '{"seq":3}'])
|
||||
expect(ctx.onError).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('sends straight through when the socket is not congested', () => {
|
||||
const ctx = setup()
|
||||
const baseline = ctx.ws.sent.length
|
||||
emitReply(ctx, '{"ok":true}')
|
||||
expect(ctx.ws.sent.length).toBe(baseline + 1)
|
||||
expect(decrypt(ctx.ws.sent[baseline]!, ctx.sharedKey)).toBe('{"ok":true}')
|
||||
})
|
||||
})
|
||||
|
|
@ -3,6 +3,10 @@
|
|||
// handler only sees plaintext JSON, identical to the Unix socket path.
|
||||
import type { WebSocket } from 'ws'
|
||||
import { deriveSharedKey, encrypt, decrypt, encryptBytes, decryptBytes } from './e2ee-crypto'
|
||||
import {
|
||||
createWsOutboundBackpressureQueue,
|
||||
type WsOutboundBackpressureQueue
|
||||
} from '../../../shared/ws-outbound-backpressure-queue'
|
||||
|
||||
type ChannelState = 'awaiting_hello' | 'awaiting_auth' | 'ready'
|
||||
|
||||
|
|
@ -48,6 +52,11 @@ export class E2EEChannel {
|
|||
) => void)
|
||||
| null = null
|
||||
private binaryMessageHandler: ((plaintext: Uint8Array<ArrayBufferLike>) => void) | null = null
|
||||
// Why: the streaming JSON reply path (e.g. legacy terminal.subscribe) has no
|
||||
// seq/resync, so it must never drop frames under backpressure. Hold text
|
||||
// replies in order while bufferedAmount is over the cap and drain as it
|
||||
// clears; only a wedged link (hard cap) closes the socket for a clean resync.
|
||||
private textReplyQueue: WsOutboundBackpressureQueue<string> | null = null
|
||||
|
||||
deviceToken: string | null = null
|
||||
|
||||
|
|
@ -128,7 +137,7 @@ export class E2EEChannel {
|
|||
if (!this.sharedKey || this.ws.readyState !== this.ws.OPEN) {
|
||||
return
|
||||
}
|
||||
this.ws.send(encrypt(response, this.sharedKey))
|
||||
this.ensureTextReplyQueue().enqueue(encrypt(response, this.sharedKey))
|
||||
}
|
||||
const encryptedBinaryReply = (response: Uint8Array<ArrayBufferLike>): boolean => {
|
||||
if (!this.sharedKey || this.ws.readyState !== this.ws.OPEN) {
|
||||
|
|
@ -215,6 +224,22 @@ export class E2EEChannel {
|
|||
this.onReady(this)
|
||||
}
|
||||
|
||||
private ensureTextReplyQueue(): WsOutboundBackpressureQueue<string> {
|
||||
if (!this.textReplyQueue) {
|
||||
this.textReplyQueue = createWsOutboundBackpressureQueue<string>({
|
||||
send: (frame) => this.ws.send(frame),
|
||||
// Encrypted replies are base64 ASCII strings, so length === byte count.
|
||||
byteLengthOf: (frame) => frame.length,
|
||||
getBufferedAmount: () => this.ws.bufferedAmount,
|
||||
isWritable: () => Boolean(this.sharedKey) && this.ws.readyState === this.ws.OPEN,
|
||||
// 1013 (Try Again Later): the link is wedged; drop the channel so the
|
||||
// client reconnects and replays a full snapshot instead of unbounded RSS.
|
||||
onOverflow: () => this.onError(1013, 'Outbound reply buffer overflow')
|
||||
})
|
||||
}
|
||||
return this.textReplyQueue
|
||||
}
|
||||
|
||||
private sendEncryptedControl(message: unknown): void {
|
||||
if (this.ws.readyState === this.ws.OPEN && this.sharedKey) {
|
||||
this.ws.send(encrypt(JSON.stringify(message), this.sharedKey))
|
||||
|
|
@ -229,5 +254,7 @@ export class E2EEChannel {
|
|||
this.sharedKey = null
|
||||
this.messageHandler = null
|
||||
this.binaryMessageHandler = null
|
||||
this.textReplyQueue?.dispose()
|
||||
this.textReplyQueue = null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1083,7 +1083,10 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
|||
name: 'terminal.resizeForClient',
|
||||
params: TerminalResizeForClient,
|
||||
handler: async (params, { runtime }) => {
|
||||
const leaf = runtime.resolveLeafForHandle(params.terminal)
|
||||
// Why: guarded resolution — a stale handle (pane's PTY replaced under it)
|
||||
// must fail with terminal_handle_stale instead of resizing the wrong PTY
|
||||
// (#7718). Clients recover by re-deriving the handle.
|
||||
const leaf = runtime.resolveLiveLeafForHandle(params.terminal)
|
||||
if (!leaf?.ptyId) {
|
||||
throw new Error('no_connected_pty')
|
||||
}
|
||||
|
|
@ -1137,7 +1140,10 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
|||
name: 'terminal.setDisplayMode',
|
||||
params: TerminalSetDisplayMode,
|
||||
handler: async (params, { runtime }) => {
|
||||
const leaf = runtime.resolveLeafForHandle(params.terminal)
|
||||
// Why: guarded resolution — a stale handle must fail with
|
||||
// terminal_handle_stale instead of mutating the wrong PTY's display
|
||||
// mode/viewport (#7718). Clients recover by re-deriving the handle.
|
||||
const leaf = runtime.resolveLiveLeafForHandle(params.terminal)
|
||||
if (!leaf?.ptyId) {
|
||||
throw new Error('no_connected_pty')
|
||||
}
|
||||
|
|
@ -1159,7 +1165,10 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
|||
name: 'terminal.restoreFit',
|
||||
params: TerminalHandle,
|
||||
handler: async (params, { runtime }) => {
|
||||
const leaf = runtime.resolveLeafForHandle(params.terminal)
|
||||
// Why: guarded resolution — a stale handle must fail with
|
||||
// terminal_handle_stale instead of reclaiming the wrong PTY back to
|
||||
// desktop dims (#7718). Clients recover by re-deriving the handle.
|
||||
const leaf = runtime.resolveLiveLeafForHandle(params.terminal)
|
||||
if (!leaf?.ptyId) {
|
||||
throw new Error('no_connected_pty')
|
||||
}
|
||||
|
|
@ -1180,7 +1189,10 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
|||
name: 'terminal.updateViewport',
|
||||
params: TerminalUpdateViewport,
|
||||
handler: async (params, { runtime }) => {
|
||||
const leaf = runtime.resolveLeafForHandle(params.terminal)
|
||||
// Why: guarded resolution — a stale handle must fail with
|
||||
// terminal_handle_stale instead of writing viewport state to the wrong
|
||||
// PTY (#7718). Clients recover by re-deriving the handle.
|
||||
const leaf = runtime.resolveLiveLeafForHandle(params.terminal)
|
||||
if (!leaf?.ptyId) {
|
||||
throw new Error('no_connected_pty')
|
||||
}
|
||||
|
|
@ -1221,18 +1233,20 @@ export const TERMINAL_METHODS: RpcAnyMethod[] = [
|
|||
opcode: TerminalStreamOpcode,
|
||||
payload: Uint8Array<ArrayBufferLike> = new Uint8Array(),
|
||||
seq?: number
|
||||
): void => {
|
||||
): boolean => {
|
||||
if (closed) {
|
||||
return
|
||||
return false
|
||||
}
|
||||
sendBinary(
|
||||
encodeTerminalStreamFrame({
|
||||
opcode,
|
||||
streamId,
|
||||
seq: typeof seq === 'number' ? seq : cursor++,
|
||||
payload
|
||||
})
|
||||
// Why: Output `seq` is a UTF-16 high-water the client uses for frame-drop
|
||||
// gap detection, so a seq-less Output chunk must carry the sentinel 0
|
||||
// (== "no seq") rather than the cursor value that orders control frames;
|
||||
// a cursor value would poison the client's expected-seq tracker.
|
||||
const resolvedSeq =
|
||||
typeof seq === 'number' ? seq : opcode === TerminalStreamOpcode.Output ? 0 : cursor++
|
||||
const sent = sendBinary(
|
||||
encodeTerminalStreamFrame({ opcode, streamId, seq: resolvedSeq, payload })
|
||||
)
|
||||
return sent !== false
|
||||
}
|
||||
const sendStreamError = (streamId: number, message: string): void => {
|
||||
sendFrame(streamId, TerminalStreamOpcode.Error, encodeTerminalStreamText(message))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,222 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { RpcDispatcher } from './dispatcher'
|
||||
import type { RpcRequest } from './core'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
import { TERMINAL_METHODS } from './methods/terminal'
|
||||
|
||||
// Why: the terminal geometry family (resize/setDisplayMode/restoreFit/
|
||||
// updateViewport) mutates PTY state. A remote client can hold a handle minted
|
||||
// for a PTY that was later replaced under the pane (restart/re-spawn bumps
|
||||
// ptyId/generation). The UNGUARDED resolveLeafForHandle returns the pane's
|
||||
// CURRENT pty, so a stale client would mutate the wrong (new) PTY — visible as
|
||||
// geometry corruption on the fresh session (#7718). These methods must use the
|
||||
// guarded resolveLiveLeafForHandle, which throws terminal_handle_stale instead.
|
||||
|
||||
// Models a stale handle exactly as the runtime does: the unguarded resolver
|
||||
// silently adopts the replacement PTY ('pty-b'); the guarded resolver throws.
|
||||
const NEW_PTY_UNDER_PANE = 'pty-b'
|
||||
|
||||
function stubStaleHandleRuntime(overrides: Partial<OrcaRuntimeService> = {}): OrcaRuntimeService {
|
||||
return {
|
||||
getRuntimeId: () => 'test-runtime',
|
||||
// Unguarded path: returns the pane's current (replaced) PTY — the misroute.
|
||||
resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: NEW_PTY_UNDER_PANE }),
|
||||
// Guarded path: surfaces the staleness.
|
||||
resolveLiveLeafForHandle: vi.fn(() => {
|
||||
throw new Error('terminal_handle_stale')
|
||||
}),
|
||||
...overrides
|
||||
} as unknown as OrcaRuntimeService
|
||||
}
|
||||
|
||||
function makeRequest(method: string, params?: unknown): RpcRequest {
|
||||
return { id: 'req-1', authToken: 'tok', method, params }
|
||||
}
|
||||
|
||||
async function expectStale(method: string, params: unknown, mutators: string[]): Promise<void> {
|
||||
const spies: Record<string, ReturnType<typeof vi.fn>> = {}
|
||||
for (const name of mutators) {
|
||||
spies[name] = vi.fn()
|
||||
}
|
||||
const runtime = stubStaleHandleRuntime(spies as Partial<OrcaRuntimeService>)
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
|
||||
const response = await dispatcher.dispatch(makeRequest(method, params))
|
||||
|
||||
expect(response.ok).toBe(false)
|
||||
if (response.ok) {
|
||||
throw new Error(`expected ${method} to reject a stale handle`)
|
||||
}
|
||||
expect(response.error.message).toContain('terminal_handle_stale')
|
||||
// The wrong (replacement) PTY must never be mutated.
|
||||
for (const name of mutators) {
|
||||
expect(spies[name]).not.toHaveBeenCalled()
|
||||
}
|
||||
}
|
||||
|
||||
describe('terminal geometry family rejects stale handles instead of mutating the wrong PTY', () => {
|
||||
it('terminal.resizeForClient fails with terminal_handle_stale', async () => {
|
||||
await expectStale(
|
||||
'terminal.resizeForClient',
|
||||
{ terminal: 'stale-terminal', mode: 'restore', clientId: 'client-1' },
|
||||
['resizeForClient']
|
||||
)
|
||||
})
|
||||
|
||||
it('terminal.setDisplayMode fails with terminal_handle_stale', async () => {
|
||||
await expectStale(
|
||||
'terminal.setDisplayMode',
|
||||
{
|
||||
terminal: 'stale-terminal',
|
||||
mode: 'auto',
|
||||
client: { id: 'client-1', type: 'mobile' },
|
||||
viewport: { cols: 80, rows: 24 }
|
||||
},
|
||||
[
|
||||
'setMobileDisplayMode',
|
||||
'applyMobileDisplayMode',
|
||||
'updateMobileSubscriberViewport',
|
||||
'markMobileActor'
|
||||
]
|
||||
)
|
||||
})
|
||||
|
||||
it('terminal.restoreFit fails with terminal_handle_stale', async () => {
|
||||
await expectStale('terminal.restoreFit', { terminal: 'stale-terminal' }, [
|
||||
'reclaimTerminalForDesktop'
|
||||
])
|
||||
})
|
||||
|
||||
it('terminal.updateViewport fails with terminal_handle_stale', async () => {
|
||||
await expectStale(
|
||||
'terminal.updateViewport',
|
||||
{
|
||||
terminal: 'stale-terminal',
|
||||
client: { id: 'client-1', type: 'mobile' },
|
||||
viewport: { cols: 80, rows: 24 }
|
||||
},
|
||||
['updateMobileViewport', 'updateDesktopViewport']
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('terminal geometry family still mutates the live PTY for a fresh handle', () => {
|
||||
it('terminal.restoreFit reclaims the resolved PTY when the handle is live', async () => {
|
||||
const reclaimTerminalForDesktop = vi.fn().mockResolvedValue(true)
|
||||
const runtime = {
|
||||
getRuntimeId: () => 'test-runtime',
|
||||
resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-a' }),
|
||||
reclaimTerminalForDesktop
|
||||
} as unknown as OrcaRuntimeService
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
|
||||
const response = await dispatcher.dispatch(
|
||||
makeRequest('terminal.restoreFit', { terminal: 'live-terminal' })
|
||||
)
|
||||
|
||||
expect(response.ok).toBe(true)
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
expect(response.result).toEqual({ restored: true })
|
||||
expect(reclaimTerminalForDesktop).toHaveBeenCalledWith('pty-a')
|
||||
})
|
||||
|
||||
it('terminal.resizeForClient resizes the resolved PTY when the handle is live', async () => {
|
||||
const resizeForClient = vi.fn().mockResolvedValue({ cols: 80, rows: 24 })
|
||||
const runtime = {
|
||||
getRuntimeId: () => 'test-runtime',
|
||||
resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-a' }),
|
||||
resizeForClient
|
||||
} as unknown as OrcaRuntimeService
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
|
||||
const response = await dispatcher.dispatch(
|
||||
makeRequest('terminal.resizeForClient', {
|
||||
terminal: 'live-terminal',
|
||||
mode: 'restore',
|
||||
clientId: 'client-1'
|
||||
})
|
||||
)
|
||||
|
||||
expect(response.ok).toBe(true)
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
expect(resizeForClient).toHaveBeenCalledWith(
|
||||
'pty-a',
|
||||
'restore',
|
||||
'client-1',
|
||||
undefined,
|
||||
undefined
|
||||
)
|
||||
})
|
||||
|
||||
it('terminal.setDisplayMode mutates the resolved PTY when the handle is live', async () => {
|
||||
const setMobileDisplayMode = vi.fn()
|
||||
const applyMobileDisplayMode = vi.fn().mockResolvedValue(undefined)
|
||||
const updateMobileSubscriberViewport = vi.fn()
|
||||
const markMobileActor = vi.fn()
|
||||
const runtime = {
|
||||
getRuntimeId: () => 'test-runtime',
|
||||
resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-a' }),
|
||||
setMobileDisplayMode,
|
||||
applyMobileDisplayMode,
|
||||
updateMobileSubscriberViewport,
|
||||
markMobileActor,
|
||||
getLayout: vi.fn().mockReturnValue({ seq: 42 })
|
||||
} as unknown as OrcaRuntimeService
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
|
||||
const response = await dispatcher.dispatch(
|
||||
makeRequest('terminal.setDisplayMode', {
|
||||
terminal: 'live-terminal',
|
||||
mode: 'auto',
|
||||
client: { id: 'client-1', type: 'mobile' },
|
||||
viewport: { cols: 80, rows: 24 }
|
||||
})
|
||||
)
|
||||
|
||||
expect(response.ok).toBe(true)
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
expect(updateMobileSubscriberViewport).toHaveBeenCalledWith('pty-a', 'client-1', {
|
||||
cols: 80,
|
||||
rows: 24
|
||||
})
|
||||
expect(markMobileActor).toHaveBeenCalledWith('pty-a', 'client-1')
|
||||
expect(setMobileDisplayMode).toHaveBeenCalledWith('pty-a', 'auto')
|
||||
expect(applyMobileDisplayMode).toHaveBeenCalledWith('pty-a')
|
||||
expect(response.result).toEqual({ mode: 'auto', seq: 42 })
|
||||
})
|
||||
|
||||
it('terminal.updateViewport updates the resolved PTY when the handle is live', async () => {
|
||||
const updateMobileViewport = vi.fn().mockResolvedValue({ updated: true, applied: true })
|
||||
const runtime = {
|
||||
getRuntimeId: () => 'test-runtime',
|
||||
resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-a' }),
|
||||
updateMobileViewport,
|
||||
getLayout: vi.fn().mockReturnValue({ seq: 7 })
|
||||
} as unknown as OrcaRuntimeService
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
|
||||
const response = await dispatcher.dispatch(
|
||||
makeRequest('terminal.updateViewport', {
|
||||
terminal: 'live-terminal',
|
||||
client: { id: 'client-1', type: 'mobile' },
|
||||
viewport: { cols: 80, rows: 24 }
|
||||
})
|
||||
)
|
||||
|
||||
expect(response.ok).toBe(true)
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
expect(updateMobileViewport).toHaveBeenCalledWith('pty-a', 'client-1', {
|
||||
cols: 80,
|
||||
rows: 24
|
||||
})
|
||||
expect(response.result).toEqual({ updated: true, applied: true, seq: 7 })
|
||||
})
|
||||
})
|
||||
|
|
@ -70,14 +70,19 @@ describe('terminal.multiplex pending-escape-tail threading (#7329)', () => {
|
|||
waitForTerminal: vi.fn(() => new Promise<RuntimeTerminalWait>(() => {})),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-1',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
|
|||
|
|
@ -77,14 +77,19 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-1',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -199,7 +204,10 @@ describe('terminal multiplex RPC', () => {
|
|||
opcode: TerminalStreamOpcode.SnapshotRequest,
|
||||
streamId: 5,
|
||||
seq: 4,
|
||||
payload: encodeTerminalStreamJson({ requestId: 7, scrollbackRows: 5000 })
|
||||
payload: encodeTerminalStreamJson({
|
||||
requestId: 7,
|
||||
scrollbackRows: 5000
|
||||
})
|
||||
})
|
||||
)!
|
||||
)
|
||||
|
|
@ -291,14 +299,19 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateMobileViewport: vi.fn().mockResolvedValue({ updated: true, applied: true })
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-stale-multiplex-resize',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -327,8 +340,20 @@ describe('terminal multiplex RPC', () => {
|
|||
await vi.waitFor(() => expect(resizeListener).toBeDefined())
|
||||
binaryFrames.splice(0)
|
||||
|
||||
resizeListener?.({ cols: 90, rows: 24, displayMode: 'auto', reason: 'apply-layout', seq: 2 })
|
||||
resizeListener?.({ cols: 100, rows: 24, displayMode: 'auto', reason: 'apply-layout', seq: 3 })
|
||||
resizeListener?.({
|
||||
cols: 90,
|
||||
rows: 24,
|
||||
displayMode: 'auto',
|
||||
reason: 'apply-layout',
|
||||
seq: 2
|
||||
})
|
||||
resizeListener?.({
|
||||
cols: 100,
|
||||
rows: 24,
|
||||
displayMode: 'auto',
|
||||
reason: 'apply-layout',
|
||||
seq: 3
|
||||
})
|
||||
await vi.waitFor(() => expect(restreamResolves).toHaveLength(2))
|
||||
|
||||
restreamResolves[1]?.({ data: 'newer', cols: 100, rows: 24 })
|
||||
|
|
@ -402,14 +427,19 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-multibyte-output-batch',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -478,9 +508,11 @@ describe('terminal multiplex RPC', () => {
|
|||
const cleanups = new Map<string, () => void>()
|
||||
const runtime = stubRuntime({
|
||||
resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }),
|
||||
readTerminal: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ tail: ['line 120'], truncated: false, limited: true }),
|
||||
readTerminal: vi.fn().mockResolvedValue({
|
||||
tail: ['line 120'],
|
||||
truncated: false,
|
||||
limited: true
|
||||
}),
|
||||
serializeTerminalBuffer: vi.fn().mockResolvedValue(null),
|
||||
getTerminalSize: vi.fn().mockReturnValue({ cols: 120, rows: 40 }),
|
||||
getMobileDisplayMode: vi.fn().mockReturnValue('auto'),
|
||||
|
|
@ -502,14 +534,19 @@ describe('terminal multiplex RPC', () => {
|
|||
waitForTerminal: vi.fn(() => new Promise<RuntimeTerminalWait>(() => {})),
|
||||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true })
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-multiplex-limited',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -578,8 +615,16 @@ describe('terminal multiplex RPC', () => {
|
|||
serializeTerminalBuffer: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ data: 'initial', cols: 120, rows: 40 })
|
||||
.mockResolvedValueOnce({ data: 'x'.repeat(2 * 1024 * 1024 + 1), cols: 120, rows: 40 })
|
||||
.mockResolvedValueOnce({ data: 'budgeted snapshot', cols: 120, rows: 40 }),
|
||||
.mockResolvedValueOnce({
|
||||
data: 'x'.repeat(2 * 1024 * 1024 + 1),
|
||||
cols: 120,
|
||||
rows: 40
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
data: 'budgeted snapshot',
|
||||
cols: 120,
|
||||
rows: 40
|
||||
}),
|
||||
getTerminalSize: vi.fn().mockReturnValue({ cols: 120, rows: 40 }),
|
||||
getMobileDisplayMode: vi.fn().mockReturnValue('auto'),
|
||||
getLayout: vi.fn().mockReturnValue({ seq: 1 }),
|
||||
|
|
@ -596,14 +641,19 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-budgeted-request',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -640,7 +690,10 @@ describe('terminal multiplex RPC', () => {
|
|||
opcode: TerminalStreamOpcode.SnapshotRequest,
|
||||
streamId: 14,
|
||||
seq: 2,
|
||||
payload: encodeTerminalStreamJson({ requestId: 55, scrollbackRows: 5000 })
|
||||
payload: encodeTerminalStreamJson({
|
||||
requestId: 55,
|
||||
scrollbackRows: 5000
|
||||
})
|
||||
})
|
||||
)!
|
||||
)
|
||||
|
|
@ -704,7 +757,10 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
|
|
@ -805,7 +861,10 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
|
|
@ -892,7 +951,10 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -981,7 +1043,10 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -992,7 +1057,9 @@ describe('terminal multiplex RPC', () => {
|
|||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-subscribe-output-chunking',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -1067,7 +1134,10 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -1078,13 +1148,18 @@ describe('terminal multiplex RPC', () => {
|
|||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-buffered-output-on-subscribe',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: vi.fn(() => vi.fn())
|
||||
}
|
||||
)
|
||||
|
||||
await vi.waitFor(() => expect(dataListenerRef.current).toBeDefined())
|
||||
dataListenerRef.current?.('starting shell\r\n', { seq: 16, rawLength: 16 })
|
||||
dataListenerRef.current?.('starting shell\r\n', {
|
||||
seq: 16,
|
||||
rawLength: 16
|
||||
})
|
||||
resolveSnapshot({ data: '', cols: 120, rows: 40 })
|
||||
await vi.waitFor(() =>
|
||||
expect(messages.some((msg) => JSON.parse(msg).result?.type === 'subscribed')).toBe(true)
|
||||
|
|
@ -1125,7 +1200,12 @@ describe('terminal multiplex RPC', () => {
|
|||
readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false }),
|
||||
serializeTerminalBuffer: vi.fn(
|
||||
() =>
|
||||
new Promise<{ data: string; cols: number; rows: number; seq: number }>((resolve) => {
|
||||
new Promise<{
|
||||
data: string
|
||||
cols: number
|
||||
rows: number
|
||||
seq: number
|
||||
}>((resolve) => {
|
||||
resolveSnapshot = resolve
|
||||
})
|
||||
),
|
||||
|
|
@ -1151,7 +1231,10 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -1162,7 +1245,9 @@ describe('terminal multiplex RPC', () => {
|
|||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-buffered-output-covered-by-snapshot',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: vi.fn(() => vi.fn())
|
||||
}
|
||||
)
|
||||
|
|
@ -1222,7 +1307,12 @@ describe('terminal multiplex RPC', () => {
|
|||
readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false }),
|
||||
serializeTerminalBuffer: vi.fn(
|
||||
() =>
|
||||
new Promise<{ data: string; cols: number; rows: number; seq: number }>((resolve) => {
|
||||
new Promise<{
|
||||
data: string
|
||||
cols: number
|
||||
rows: number
|
||||
seq: number
|
||||
}>((resolve) => {
|
||||
resolveSnapshot = resolve
|
||||
})
|
||||
),
|
||||
|
|
@ -1248,7 +1338,10 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -1259,7 +1352,9 @@ describe('terminal multiplex RPC', () => {
|
|||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-buffered-output-partially-covered-by-snapshot',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: vi.fn(() => vi.fn())
|
||||
}
|
||||
)
|
||||
|
|
@ -1319,7 +1414,10 @@ describe('terminal multiplex RPC', () => {
|
|||
cleanups.set(id, cleanup)
|
||||
})
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
|
|
@ -1327,7 +1425,9 @@ describe('terminal multiplex RPC', () => {
|
|||
{
|
||||
signal: controller.signal,
|
||||
connectionId: 'conn-phone-multiplex',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -1393,14 +1493,19 @@ describe('terminal multiplex RPC', () => {
|
|||
cleanups.set(id, cleanup)
|
||||
})
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-stale-handle',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -1484,14 +1589,19 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-buffered',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -1593,14 +1703,19 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-buffered-multibyte',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -1690,7 +1805,11 @@ describe('terminal multiplex RPC', () => {
|
|||
resolveRequestedSnapshot = resolve
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce({ data: 'retry snapshot', cols: 120, rows: 40 }),
|
||||
.mockResolvedValueOnce({
|
||||
data: 'retry snapshot',
|
||||
cols: 120,
|
||||
rows: 40
|
||||
}),
|
||||
getTerminalSize: vi.fn().mockReturnValue({ cols: 120, rows: 40 }),
|
||||
getMobileDisplayMode: vi.fn().mockReturnValue('auto'),
|
||||
getLayout: vi.fn().mockReturnValue({ seq: 1 }),
|
||||
|
|
@ -1710,14 +1829,19 @@ describe('terminal multiplex RPC', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateDesktopViewport: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.multiplex', {}),
|
||||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-request-overflow',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes),
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
},
|
||||
registerBinaryStreamHandler: (streamId, handler) => {
|
||||
handlers.set(streamId, handler)
|
||||
return () => handlers.delete(streamId)
|
||||
|
|
@ -1754,7 +1878,10 @@ describe('terminal multiplex RPC', () => {
|
|||
opcode: TerminalStreamOpcode.SnapshotRequest,
|
||||
streamId: 12,
|
||||
seq: 2,
|
||||
payload: encodeTerminalStreamJson({ requestId: 44, scrollbackRows: 5000 })
|
||||
payload: encodeTerminalStreamJson({
|
||||
requestId: 44,
|
||||
scrollbackRows: 5000
|
||||
})
|
||||
})
|
||||
)!
|
||||
)
|
||||
|
|
|
|||
|
|
@ -52,7 +52,10 @@ describe('terminal output batching', () => {
|
|||
}),
|
||||
waitForTerminal: vi.fn(() => new Promise<RuntimeTerminalWait>(() => {}))
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -78,7 +81,9 @@ describe('terminal output batching', () => {
|
|||
.map((msg) => JSON.parse(msg))
|
||||
.filter((message) => message.result?.type === 'data')
|
||||
expect(dataMessages).toHaveLength(1)
|
||||
expect(dataMessages[0]).toMatchObject({ result: { type: 'data', chunk: 'ab' } })
|
||||
expect(dataMessages[0]).toMatchObject({
|
||||
result: { type: 'data', chunk: 'ab' }
|
||||
})
|
||||
|
||||
runtime.cleanupSubscription('terminal-1:desktop-1')
|
||||
await dispatchPromise
|
||||
|
|
@ -123,7 +128,10 @@ describe('terminal output batching', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateMobileViewport: vi.fn().mockResolvedValue(false)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -134,7 +142,9 @@ describe('terminal output batching', () => {
|
|||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-1',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes)
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -144,7 +154,10 @@ describe('terminal output batching', () => {
|
|||
const subscribed = messages
|
||||
.map((msg) => JSON.parse(msg))
|
||||
.find((msg) => msg.result?.type === 'subscribed')
|
||||
expect(subscribed?.result).toMatchObject({ type: 'subscribed', streamId: expect.any(Number) })
|
||||
expect(subscribed?.result).toMatchObject({
|
||||
type: 'subscribed',
|
||||
streamId: expect.any(Number)
|
||||
})
|
||||
await vi.waitFor(() => expect(dataListenerRef.current).toBeDefined())
|
||||
|
||||
const emitData = dataListenerRef.current
|
||||
|
|
@ -206,7 +219,10 @@ describe('terminal output batching', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateMobileViewport: vi.fn().mockResolvedValue(false)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -286,7 +302,10 @@ describe('terminal output batching', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateMobileViewport: vi.fn().mockResolvedValue(false)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
const messages: string[] = []
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
|
|
|
|||
|
|
@ -417,7 +417,7 @@ describe('terminal send RPC', () => {
|
|||
|
||||
it('routes terminal restore fit through the runtime driver state machine', async () => {
|
||||
const runtime = stubRuntime({
|
||||
resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }),
|
||||
resolveLiveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }),
|
||||
reclaimTerminalForDesktop: vi.fn().mockResolvedValue(true)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
|
|
|
|||
|
|
@ -40,7 +40,10 @@ describe('terminal subscribe buffering', () => {
|
|||
),
|
||||
readTerminal: vi.fn().mockResolvedValue({ tail: [], truncated: false })
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -77,11 +80,16 @@ describe('terminal subscribe buffering', () => {
|
|||
const messages: string[] = []
|
||||
const runtime = stubRuntime({
|
||||
resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: null }),
|
||||
readTerminal: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ tail: ['line 120'], truncated: false, limited: true })
|
||||
readTerminal: vi.fn().mockResolvedValue({
|
||||
tail: ['line 120'],
|
||||
truncated: false,
|
||||
limited: true
|
||||
})
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
|
||||
await dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', { terminal: 'terminal-1' }),
|
||||
|
|
@ -104,9 +112,11 @@ describe('terminal subscribe buffering', () => {
|
|||
const cleanups = new Map<string, () => void>()
|
||||
const runtime = stubRuntime({
|
||||
resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }),
|
||||
readTerminal: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ tail: ['line 120'], truncated: false, limited: true }),
|
||||
readTerminal: vi.fn().mockResolvedValue({
|
||||
tail: ['line 120'],
|
||||
truncated: false,
|
||||
limited: true
|
||||
}),
|
||||
serializeTerminalBuffer: vi.fn().mockResolvedValue(null),
|
||||
getTerminalSize: vi.fn().mockReturnValue({ cols: 80, rows: 24 }),
|
||||
getMobileDisplayMode: vi.fn().mockReturnValue('auto'),
|
||||
|
|
@ -122,7 +132,10 @@ describe('terminal subscribe buffering', () => {
|
|||
}),
|
||||
waitForTerminal: vi.fn(() => new Promise<RuntimeTerminalWait>(() => {}))
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -172,7 +185,10 @@ describe('terminal subscribe buffering', () => {
|
|||
cleanupSubscription: vi.fn(),
|
||||
waitForTerminal: vi.fn(() => new Promise<RuntimeTerminalWait>(() => {}))
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -209,9 +225,11 @@ describe('terminal subscribe buffering', () => {
|
|||
const cleanups = new Map<string, () => void>()
|
||||
const runtime = stubRuntime({
|
||||
resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }),
|
||||
readTerminal: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ tail: ['line 120'], truncated: false, limited: true }),
|
||||
readTerminal: vi.fn().mockResolvedValue({
|
||||
tail: ['line 120'],
|
||||
truncated: false,
|
||||
limited: true
|
||||
}),
|
||||
serializeTerminalBuffer: vi.fn().mockResolvedValue(null),
|
||||
getTerminalSize: vi.fn().mockReturnValue({ cols: 80, rows: 24 }),
|
||||
getMobileDisplayMode: vi.fn().mockReturnValue('auto'),
|
||||
|
|
@ -230,7 +248,10 @@ describe('terminal subscribe buffering', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateMobileViewport: vi.fn().mockResolvedValue(false)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -241,7 +262,9 @@ describe('terminal subscribe buffering', () => {
|
|||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-binary-limited',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes)
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -273,9 +296,11 @@ describe('terminal subscribe buffering', () => {
|
|||
const cleanups = new Map<string, () => void>()
|
||||
const runtime = stubRuntime({
|
||||
resolveLeafForHandle: vi.fn().mockReturnValue({ ptyId: 'pty-1' }),
|
||||
readTerminal: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ tail: ['line 120'], truncated: false, limited: true }),
|
||||
readTerminal: vi.fn().mockResolvedValue({
|
||||
tail: ['line 120'],
|
||||
truncated: false,
|
||||
limited: true
|
||||
}),
|
||||
serializeTerminalBuffer: vi.fn().mockResolvedValue({
|
||||
data: 'serialized snapshot\r\n',
|
||||
cols: 100,
|
||||
|
|
@ -298,7 +323,10 @@ describe('terminal subscribe buffering', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateMobileViewport: vi.fn().mockResolvedValue(false)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -309,7 +337,9 @@ describe('terminal subscribe buffering', () => {
|
|||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-binary-serialized-limited',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes)
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -375,7 +405,10 @@ describe('terminal subscribe buffering', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateMobileViewport: vi.fn().mockResolvedValue(false)
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -386,7 +419,9 @@ describe('terminal subscribe buffering', () => {
|
|||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-buffered',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes)
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -446,7 +481,12 @@ describe('terminal subscribe buffering', () => {
|
|||
data: string
|
||||
cols: number
|
||||
rows: number
|
||||
oscLinks?: { row: number; startCol: number; endCol: number; uri: string }[]
|
||||
oscLinks?: {
|
||||
row: number
|
||||
startCol: number
|
||||
endCol: number
|
||||
uri: string
|
||||
}[]
|
||||
}) => void)[] = []
|
||||
const serializeTerminalBuffer = vi
|
||||
.fn()
|
||||
|
|
@ -457,7 +497,12 @@ describe('terminal subscribe buffering', () => {
|
|||
data: string
|
||||
cols: number
|
||||
rows: number
|
||||
oscLinks?: { row: number; startCol: number; endCol: number; uri: string }[]
|
||||
oscLinks?: {
|
||||
row: number
|
||||
startCol: number
|
||||
endCol: number
|
||||
uri: string
|
||||
}[]
|
||||
}>((resolve) => {
|
||||
restreamResolves.push(resolve)
|
||||
})
|
||||
|
|
@ -490,7 +535,10 @@ describe('terminal subscribe buffering', () => {
|
|||
sendTerminal: vi.fn().mockResolvedValue({ accepted: true }),
|
||||
updateMobileViewport: vi.fn().mockResolvedValue({ updated: true, applied: true })
|
||||
})
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: TERMINAL_METHODS })
|
||||
const dispatcher = new RpcDispatcher({
|
||||
runtime,
|
||||
methods: TERMINAL_METHODS
|
||||
})
|
||||
|
||||
const dispatchPromise = dispatcher.dispatchStreaming(
|
||||
makeRequest('terminal.subscribe', {
|
||||
|
|
@ -501,19 +549,38 @@ describe('terminal subscribe buffering', () => {
|
|||
(msg) => messages.push(msg),
|
||||
{
|
||||
connectionId: 'conn-stale-resize',
|
||||
sendBinary: (bytes) => binaryFrames.push(bytes)
|
||||
sendBinary: (bytes) => {
|
||||
binaryFrames.push(bytes)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
await vi.waitFor(() => expect(resizeListener).toBeDefined())
|
||||
binaryFrames.splice(0)
|
||||
|
||||
resizeListener?.({ cols: 90, rows: 24, displayMode: 'auto', reason: 'apply-layout', seq: 2 })
|
||||
resizeListener?.({ cols: 100, rows: 24, displayMode: 'auto', reason: 'apply-layout', seq: 3 })
|
||||
resizeListener?.({
|
||||
cols: 90,
|
||||
rows: 24,
|
||||
displayMode: 'auto',
|
||||
reason: 'apply-layout',
|
||||
seq: 2
|
||||
})
|
||||
resizeListener?.({
|
||||
cols: 100,
|
||||
rows: 24,
|
||||
displayMode: 'auto',
|
||||
reason: 'apply-layout',
|
||||
seq: 3
|
||||
})
|
||||
await vi.waitFor(() => expect(restreamResolves).toHaveLength(2))
|
||||
|
||||
const newerOscLinks = [{ row: 0, startCol: 4, endCol: 9, uri: 'https://example.com' }]
|
||||
restreamResolves[1]?.({ data: 'newer', cols: 100, rows: 24, oscLinks: newerOscLinks })
|
||||
restreamResolves[1]?.({
|
||||
data: 'newer',
|
||||
cols: 100,
|
||||
rows: 24,
|
||||
oscLinks: newerOscLinks
|
||||
})
|
||||
await vi.waitFor(() =>
|
||||
expect(
|
||||
binaryFrames.some((frame) => {
|
||||
|
|
|
|||
|
|
@ -8,10 +8,27 @@ import {
|
|||
FrameDecoder,
|
||||
parseJsonRpcMessage,
|
||||
parseUnameToRelayPlatform,
|
||||
isGitResponseStreamMarker,
|
||||
type JsonRpcRequest,
|
||||
type DecodedFrame
|
||||
} from './relay-protocol'
|
||||
|
||||
describe('git response stream marker', () => {
|
||||
it('accepts only complete non-negative integer metadata', () => {
|
||||
expect(
|
||||
isGitResponseStreamMarker({
|
||||
__orcaGitResponseStream: { streamId: 1, totalBytes: 1024, chunkCount: 2 }
|
||||
})
|
||||
).toBe(true)
|
||||
expect(isGitResponseStreamMarker({ __orcaGitResponseStream: {} })).toBe(false)
|
||||
expect(
|
||||
isGitResponseStreamMarker({
|
||||
__orcaGitResponseStream: { streamId: -1, totalBytes: 1024, chunkCount: 2 }
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('frame encoding', () => {
|
||||
it('encodes a frame with 13-byte header', () => {
|
||||
const payload = Buffer.from('hello')
|
||||
|
|
|
|||
|
|
@ -55,6 +55,45 @@ export const STREAM_CHUNK_SIZE = 256 * 1024
|
|||
* client. */
|
||||
export const MAX_CONCURRENT_STREAMS = 16
|
||||
|
||||
// ── Git response streaming (see docs/relay-git-response-stream-design.md) ──
|
||||
|
||||
/** Serialized-JSON size above which the relay chunks a streamable git response
|
||||
* (diff family + exec) onto the bulk lane instead of one JSON-RPC frame. Mirror
|
||||
* of the relay-side constant; the client only opts in — the relay owns the
|
||||
* decision — so this is documentation of the shared contract. */
|
||||
export const GIT_RESPONSE_STREAM_THRESHOLD = 256 * 1024
|
||||
|
||||
/** Per-chunk size (serialized-result UTF-8 bytes) for git response streaming.
|
||||
* The client reassembles by concatenation and does not depend on this value,
|
||||
* so it stays cross-version safe. */
|
||||
export const GIT_RESPONSE_CHUNK_SIZE = 128 * 1024
|
||||
|
||||
/** Sentinel the relay returns as the RPC result when the real payload streams
|
||||
* as git.responseChunk frames. Absent from old relays, so a new client falls
|
||||
* back to the plain result they return. */
|
||||
export type GitResponseStreamMarker = {
|
||||
__orcaGitResponseStream: { streamId: number; totalBytes: number; chunkCount: number }
|
||||
}
|
||||
|
||||
export function isGitResponseStreamMarker(value: unknown): value is GitResponseStreamMarker {
|
||||
if (typeof value !== 'object' || value === null || !('__orcaGitResponseStream' in value)) {
|
||||
return false
|
||||
}
|
||||
const marker = (value as { __orcaGitResponseStream?: unknown }).__orcaGitResponseStream
|
||||
if (typeof marker !== 'object' || marker === null) {
|
||||
return false
|
||||
}
|
||||
const fields = marker as Record<string, unknown>
|
||||
return (
|
||||
Number.isInteger(fields.streamId) &&
|
||||
(fields.streamId as number) > 0 &&
|
||||
Number.isInteger(fields.totalBytes) &&
|
||||
(fields.totalBytes as number) >= 0 &&
|
||||
Number.isInteger(fields.chunkCount) &&
|
||||
(fields.chunkCount as number) >= 0
|
||||
)
|
||||
}
|
||||
|
||||
// ── JSON-RPC types ──────────────────────────────────────────────────
|
||||
|
||||
export type JsonRpcRequest = {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,311 @@
|
|||
import type { SshChannelMultiplexer } from './ssh-channel-multiplexer'
|
||||
import { RelayErrorCode, isGitResponseStreamMarker } from './relay-protocol'
|
||||
|
||||
const SENTINEL_STREAM_ID = -1
|
||||
|
||||
/** Reject if no stream frame (chunk/end/error) arrives within this window,
|
||||
* reset on each frame. mux.request's own timeout only bounds the fast sentinel
|
||||
* response; without this, a relay pump that breaks on staleness (which sends no
|
||||
* responseEnd) while the SSH channel stays up would hang the client forever. */
|
||||
const STREAM_INACTIVITY_TIMEOUT_MS = 30_000
|
||||
|
||||
/** Bound transient buffering of other concurrent streams' chunks while this
|
||||
* reader awaits its sentinel: every reader sees all git.responseChunk frames
|
||||
* and can't filter by streamId until its own sentinel resolves. Foreign frames
|
||||
* are dropped on drain anyway; this just caps the pre-sentinel backlog. */
|
||||
const MAX_PENDING_FRAMES = 64
|
||||
|
||||
export class GitResponseStreamError extends Error {
|
||||
readonly code = RelayErrorCode.StreamProtocolError
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
}
|
||||
}
|
||||
|
||||
type PendingFrame =
|
||||
| { kind: 'chunk'; params: Record<string, unknown> }
|
||||
| { kind: 'end'; params: Record<string, unknown> }
|
||||
| { kind: 'error'; params: Record<string, unknown> }
|
||||
|
||||
/**
|
||||
* Request a git method that may return a large payload, opting into response
|
||||
* streaming so a big diff/exec response is chunked onto the relay's bulk lane
|
||||
* instead of one JSON-RPC frame (which would head-of-line-block pty.data echo
|
||||
* on the shared SSH channel).
|
||||
*
|
||||
* Cross-version behavior:
|
||||
* - New relay + big result → returns the stream sentinel; we reassemble chunks.
|
||||
* - New relay + small result, or old client → plain single-frame result.
|
||||
* - Old relay (ignores `__streamResponse`) → returns the plain result; the
|
||||
* marker check fails and we return it directly, i.e. today's behavior.
|
||||
*/
|
||||
export function requestGitStreamable(
|
||||
mux: SshChannelMultiplexer,
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
options?: {
|
||||
signal?: AbortSignal
|
||||
/** Bounds only the sentinel request (forwarded to mux.request), like today. */
|
||||
timeoutMs?: number
|
||||
/** Bounds the post-sentinel reassembly stall; resets on each chunk. */
|
||||
inactivityTimeoutMs?: number
|
||||
}
|
||||
): Promise<unknown> {
|
||||
// Why: subscribe to chunk/end/error BEFORE awaiting the sentinel response so a
|
||||
// chunk that lands in the same dispatch tick as the response is not dropped
|
||||
// (mirrors readFileViaStream). streamIdRef stays SENTINEL until the sentinel
|
||||
// resolves; frames are queued until then and drained.
|
||||
const streamIdRef = { current: SENTINEL_STREAM_ID }
|
||||
const unsubscribers: (() => void)[] = []
|
||||
const cleanup = (): void => {
|
||||
while (unsubscribers.length > 0) {
|
||||
try {
|
||||
unsubscribers.pop()?.()
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new Promise<unknown>((resolve, reject) => {
|
||||
const parts: Buffer[] = []
|
||||
let expectedSeq = 0
|
||||
let receivedBytes = 0
|
||||
let totalBytes = 0
|
||||
let chunkCount = 0
|
||||
let settled = false
|
||||
let metadataReady = false
|
||||
const pending: PendingFrame[] = []
|
||||
|
||||
const inactivityMs = options?.inactivityTimeoutMs ?? STREAM_INACTIVITY_TIMEOUT_MS
|
||||
let inactivityTimer: ReturnType<typeof setTimeout> | null = null
|
||||
const clearInactivity = (): void => {
|
||||
if (inactivityTimer) {
|
||||
clearTimeout(inactivityTimer)
|
||||
inactivityTimer = null
|
||||
}
|
||||
}
|
||||
// Why: reset on every stream frame so a legitimately long stream is not
|
||||
// killed, but a wedged stream (no frames arriving) rejects instead of
|
||||
// hanging the caller forever.
|
||||
const armInactivity = (): void => {
|
||||
clearInactivity()
|
||||
inactivityTimer = setTimeout(() => {
|
||||
fail(
|
||||
new GitResponseStreamError(
|
||||
`Git response stream stalled (>${inactivityMs}ms without data)`
|
||||
)
|
||||
)
|
||||
}, inactivityMs)
|
||||
inactivityTimer.unref?.()
|
||||
}
|
||||
|
||||
const cancel = (): void => {
|
||||
if (streamIdRef.current !== SENTINEL_STREAM_ID && !mux.isDisposed()) {
|
||||
try {
|
||||
mux.notify('git.cancelResponseStream', { streamId: streamIdRef.current })
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
const fail = (err: Error): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
clearInactivity()
|
||||
cancel()
|
||||
cleanup()
|
||||
reject(err)
|
||||
}
|
||||
const succeed = (value: unknown): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
clearInactivity()
|
||||
cleanup()
|
||||
resolve(value)
|
||||
}
|
||||
|
||||
const handleChunk = (p: Record<string, unknown>): void => {
|
||||
if (settled || p.streamId !== streamIdRef.current) {
|
||||
return
|
||||
}
|
||||
const seq = p.seq as number
|
||||
const data = p.data as string
|
||||
if (typeof seq !== 'number' || typeof data !== 'string') {
|
||||
fail(new GitResponseStreamError(`Malformed chunk for git stream ${streamIdRef.current}`))
|
||||
return
|
||||
}
|
||||
if (seq !== expectedSeq) {
|
||||
fail(
|
||||
new GitResponseStreamError(
|
||||
`Out-of-order chunk for git stream ${streamIdRef.current}: expected ${expectedSeq}, got ${seq}`
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
const decoded = Buffer.from(data, 'base64')
|
||||
parts.push(decoded)
|
||||
receivedBytes += decoded.length
|
||||
expectedSeq += 1
|
||||
armInactivity()
|
||||
// Why: credit-based flow control — the relay caps unacked chunks so a big
|
||||
// response cannot queue unbounded ahead of interactive pty.data frames.
|
||||
if (!mux.isDisposed()) {
|
||||
try {
|
||||
mux.notify('git.responseAck', { streamId: streamIdRef.current, seq })
|
||||
} catch {
|
||||
// Disposal can race the check; the ACK is best-effort during teardown.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const handleEnd = (p: Record<string, unknown>): void => {
|
||||
if (settled || p.streamId !== streamIdRef.current) {
|
||||
return
|
||||
}
|
||||
if (expectedSeq !== chunkCount || receivedBytes !== totalBytes) {
|
||||
fail(
|
||||
new GitResponseStreamError(
|
||||
`Git stream ${streamIdRef.current} incomplete: chunks ${expectedSeq}/${chunkCount}, bytes ${receivedBytes}/${totalBytes}`
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
try {
|
||||
succeed(JSON.parse(Buffer.concat(parts).toString('utf-8')))
|
||||
} catch (err) {
|
||||
fail(
|
||||
new GitResponseStreamError(
|
||||
`Git stream ${streamIdRef.current} JSON parse failed: ${String(err)}`
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const handleStreamError = (p: Record<string, unknown>): void => {
|
||||
if (settled || p.streamId !== streamIdRef.current) {
|
||||
return
|
||||
}
|
||||
fail(new Error((p.message as string | undefined) ?? 'git response stream error'))
|
||||
}
|
||||
|
||||
const drainPending = (): void => {
|
||||
while (!settled && pending.length > 0) {
|
||||
const frame = pending.shift()!
|
||||
if (frame.kind === 'chunk') {
|
||||
handleChunk(frame.params)
|
||||
} else if (frame.kind === 'end') {
|
||||
handleEnd(frame.params)
|
||||
} else {
|
||||
handleStreamError(frame.params)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Why: pre-sentinel we cannot filter by streamId (our id is unknown yet), so
|
||||
// every concurrent reader transiently buffers all readers' chunks. Cap the
|
||||
// backlog by dropping the oldest; foreign frames are dropped on drain anyway,
|
||||
// and if our own seq-0 were ever dropped the seq check fails loudly rather
|
||||
// than corrupting. The sentinel normally resolves long before this cap.
|
||||
const pushPending = (frame: PendingFrame): void => {
|
||||
pending.push(frame)
|
||||
if (pending.length > MAX_PENDING_FRAMES) {
|
||||
pending.shift()
|
||||
}
|
||||
}
|
||||
|
||||
unsubscribers.push(
|
||||
mux.onNotificationByMethod('git.responseChunk', (p) => {
|
||||
if (!metadataReady) {
|
||||
pushPending({ kind: 'chunk', params: p })
|
||||
return
|
||||
}
|
||||
handleChunk(p)
|
||||
})
|
||||
)
|
||||
unsubscribers.push(
|
||||
mux.onNotificationByMethod('git.responseEnd', (p) => {
|
||||
if (!metadataReady) {
|
||||
pushPending({ kind: 'end', params: p })
|
||||
return
|
||||
}
|
||||
handleEnd(p)
|
||||
})
|
||||
)
|
||||
unsubscribers.push(
|
||||
mux.onNotificationByMethod('git.responseError', (p) => {
|
||||
if (!metadataReady) {
|
||||
pushPending({ kind: 'error', params: p })
|
||||
return
|
||||
}
|
||||
handleStreamError(p)
|
||||
})
|
||||
)
|
||||
unsubscribers.push(
|
||||
mux.onDispose((reason) => {
|
||||
const err = new Error(
|
||||
reason === 'connection_lost'
|
||||
? 'SSH connection lost, reconnecting...'
|
||||
: 'Multiplexer disposed'
|
||||
) as Error & { code: string }
|
||||
err.code = reason === 'connection_lost' ? 'CONNECTION_LOST' : 'DISPOSED'
|
||||
fail(err)
|
||||
})
|
||||
)
|
||||
|
||||
if (options?.signal) {
|
||||
const signal = options.signal
|
||||
if (signal.aborted) {
|
||||
const err = new Error('Request was cancelled') as Error & { name: string }
|
||||
err.name = 'AbortError'
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
const onAbort = (): void => {
|
||||
const err = new Error('Request was cancelled') as Error & { name: string }
|
||||
err.name = 'AbortError'
|
||||
fail(err)
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true })
|
||||
unsubscribers.push(() => signal.removeEventListener('abort', onAbort))
|
||||
}
|
||||
|
||||
// Why: forward only the mux-request options (signal/timeoutMs) and omit them
|
||||
// entirely when absent, so callers that previously issued a 2-arg
|
||||
// mux.request keep the same call shape (and their tests). inactivityTimeoutMs
|
||||
// governs reassembly here, not the sentinel request.
|
||||
const streamParams = { ...params, __streamResponse: true }
|
||||
const requestOptions =
|
||||
options?.signal !== undefined || options?.timeoutMs !== undefined
|
||||
? { signal: options.signal, timeoutMs: options.timeoutMs }
|
||||
: undefined
|
||||
const requestPromise = requestOptions
|
||||
? mux.request(method, streamParams, requestOptions)
|
||||
: mux.request(method, streamParams)
|
||||
void requestPromise
|
||||
.then((result) => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
// Old relay / small result: plain single-frame value, no stream follows.
|
||||
if (!isGitResponseStreamMarker(result)) {
|
||||
succeed(result)
|
||||
return
|
||||
}
|
||||
const marker = result.__orcaGitResponseStream
|
||||
totalBytes = marker.totalBytes
|
||||
chunkCount = marker.chunkCount
|
||||
streamIdRef.current = marker.streamId
|
||||
metadataReady = true
|
||||
// Why: start the inactivity deadline now — mux.request's timeout only
|
||||
// covered the sentinel; the reassembly phase needs its own guard.
|
||||
armInactivity()
|
||||
drainPending()
|
||||
})
|
||||
.catch((err) => fail(err as Error))
|
||||
})
|
||||
}
|
||||
|
|
@ -756,7 +756,12 @@ async function launchRelay(
|
|||
// Fire-and-forget via conn.exec: we don't need the output — the socket
|
||||
// poll below detects readiness.
|
||||
const logFile = `${remoteDir}/relay.log`
|
||||
const launchCmd = `cd ${escapedDir} && nohup ${escapedNode} relay.js --detached --grace-time ${graceTime} --sock-path ${shellEscape(sockFile)} > ${shellEscape(logFile)} 2>&1 </dev/null &`
|
||||
// Why: pass --log-file so the relay rotates relay.log in-process (size cap +
|
||||
// one archived generation). The shell redirect stays so pre-JS boot/crash
|
||||
// output is still captured; once JS starts, the in-process rotator owns all
|
||||
// subsequent logging (it wraps process.stderr/stdout), so the current log
|
||||
// stays at relay.log for the `tail relay.log` diagnostics workflow.
|
||||
const launchCmd = `cd ${escapedDir} && nohup ${escapedNode} relay.js --detached --grace-time ${graceTime} --sock-path ${shellEscape(sockFile)} --log-file ${shellEscape(logFile)} > ${shellEscape(logFile)} 2>&1 </dev/null &`
|
||||
const launchChannel = await conn.exec(launchCmd)
|
||||
launchChannel.on('data', () => {})
|
||||
launchChannel.on('error', () => {})
|
||||
|
|
@ -1052,6 +1057,10 @@ function windowsRelayLaunchCommand(
|
|||
quoted(sockPath),
|
||||
'--endpoint-dir',
|
||||
quoted(endpointDir),
|
||||
// Why: in-process rotation owns relay.log (the tail-diagnostics target);
|
||||
// the shell redirects remain for pre-JS boot/crash output.
|
||||
'--log-file',
|
||||
quoted(logFile),
|
||||
`1>${quoted(logFile)}`,
|
||||
`2>${quoted(errFile)}`
|
||||
].join(' ')
|
||||
|
|
|
|||
|
|
@ -396,6 +396,73 @@ describe('RelayDispatcher', () => {
|
|||
expect(listener).toHaveBeenCalledWith(1)
|
||||
})
|
||||
|
||||
it('detaches the primary client when its write throws (frame lost, trigger reconnect)', () => {
|
||||
// Regression: a primary-client write throw dropped the frame (possibly
|
||||
// pty.data/pty.exit) with no resend AND without notifying detach, so the
|
||||
// owning Orca's reconnect + PTY-reattach path never engaged until the ~20s
|
||||
// keepalive timeout — output/pane-death were silently lost in the meantime.
|
||||
let throwOnWrite = false
|
||||
const detachDispatcher = new RelayDispatcher((data) => {
|
||||
if (throwOnWrite) {
|
||||
throw new Error('socket closed')
|
||||
}
|
||||
written.push(Buffer.from(data))
|
||||
})
|
||||
try {
|
||||
const detachListener = vi.fn()
|
||||
detachDispatcher.onClientDetached(detachListener)
|
||||
|
||||
// A frame the owning Orca must not silently miss (e.g. a pane exit).
|
||||
throwOnWrite = true
|
||||
detachDispatcher.notify('pty.exit', { id: 'pty-1', code: 0 })
|
||||
|
||||
// Fix: the write failure detaches the primary so the reconnect/reattach
|
||||
// machinery runs promptly instead of waiting for keepalive timeout.
|
||||
expect(detachListener).toHaveBeenCalledWith(1)
|
||||
|
||||
// Recovery: a reconnecting socket swaps the write via setWrite; the client
|
||||
// is usable again and later frames flow to the new sink.
|
||||
throwOnWrite = false
|
||||
const recovered: Buffer[] = []
|
||||
detachDispatcher.setWrite((data) => {
|
||||
recovered.push(Buffer.from(data))
|
||||
})
|
||||
detachDispatcher.notify('pty.data', { id: 'pty-1', data: 'x' })
|
||||
expect(recovered.length).toBeGreaterThan(0)
|
||||
} finally {
|
||||
detachDispatcher.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('aborts in-flight primary requests when a client write throws', () => {
|
||||
let throwOnWrite = false
|
||||
const detachDispatcher = new RelayDispatcher(() => {
|
||||
if (throwOnWrite) {
|
||||
throw new Error('socket closed')
|
||||
}
|
||||
})
|
||||
let requestSignal: AbortSignal | undefined
|
||||
try {
|
||||
detachDispatcher.onRequest('slow.method', async (_params, context) => {
|
||||
requestSignal = context.signal
|
||||
await new Promise<void>((resolve) => {
|
||||
context.signal?.addEventListener('abort', () => resolve(), { once: true })
|
||||
})
|
||||
})
|
||||
detachDispatcher.feed(
|
||||
encodeJsonRpcFrame({ jsonrpc: '2.0', id: 9, method: 'slow.method' }, 1, 0)
|
||||
)
|
||||
|
||||
expect(requestSignal?.aborted).toBe(false)
|
||||
throwOnWrite = true
|
||||
detachDispatcher.notify('pty.exit', { id: 'pty-1', code: 0 })
|
||||
|
||||
expect(requestSignal?.aborted).toBe(true)
|
||||
} finally {
|
||||
detachDispatcher.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
describe('notifyBulk (bulk lane backpressure)', () => {
|
||||
it('resolves immediately when the sink accepts the frame', async () => {
|
||||
const frames: Buffer[] = []
|
||||
|
|
|
|||
|
|
@ -536,11 +536,21 @@ export class RelayDispatcher {
|
|||
} catch (err) {
|
||||
client.closed = true
|
||||
client.generation++
|
||||
this.requestAborts.abortClient(client.id)
|
||||
this.flushDrainWaiters(client)
|
||||
// Why: a write throw means this frame (possibly pty.data or pty.exit) was
|
||||
// lost with no resend — the framing carries seq/ack but no retransmit
|
||||
// buffer. Detach so the owning Orca's reconnect + PTY-reattach path runs
|
||||
// promptly (regenerating dropped pty.data from the replay buffer and pane
|
||||
// death via reattach-not-found), instead of silently dropping frames until
|
||||
// the ~20s keepalive timeout notices. The primary client stays in the map
|
||||
// (its object is reused across setWrite reconnects) but detach listeners
|
||||
// must still fire, exactly as invalidateClient() does for stdin/stdout
|
||||
// death — this makes the socket-write-throw path consistent with those.
|
||||
if (client !== this.primaryClient) {
|
||||
this.clients.delete(client.id)
|
||||
this.notifyClientDetached(client.id)
|
||||
}
|
||||
this.notifyClientDetached(client.id)
|
||||
process.stderr.write(
|
||||
`[relay] Client write failed: ${err instanceof Error ? err.message : String(err)}\n`
|
||||
)
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ import { getGitCloneFailureMessage } from '../shared/git-clone-failure-message'
|
|||
import { syncForkDefaultBranch, validateGitForkSyncExpectedUpstream } from '../shared/git-fork-sync'
|
||||
import { InFlightPromiseDedupe, stableInFlightKey } from '../shared/in-flight-promise-dedupe'
|
||||
import { GIT_FETCH_SKIP_AUTO_MAINTENANCE_CONFIG_ARGS } from '../shared/git-fetch-auto-maintenance'
|
||||
import { GitResponseStreamRegistry } from './git-response-stream'
|
||||
import { GIT_RESPONSE_STREAM_THRESHOLD } from './protocol'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const MAX_GIT_BUFFER = 10 * 1024 * 1024
|
||||
|
|
@ -171,6 +173,9 @@ function execFileWithStdin(
|
|||
export class GitHandler {
|
||||
private dispatcher: RelayDispatcher
|
||||
private readonly gitDiffReadDedupe = new InFlightPromiseDedupe<unknown>()
|
||||
// Why: large diff/exec responses are chunked onto the bulk lane so they do
|
||||
// not head-of-line-block interactive pty.data echo on the shared SSH channel.
|
||||
private readonly responseStreams = new GitResponseStreamRegistry()
|
||||
|
||||
// Why: configured submodule paths change rarely; an instance-level TTL cache
|
||||
// avoids re-reading `.gitmodules` on every diff click over SSH, and being
|
||||
|
|
@ -182,6 +187,13 @@ export class GitHandler {
|
|||
constructor(dispatcher: RelayDispatcher, _context: RelayContext) {
|
||||
this.dispatcher = dispatcher
|
||||
this.registerHandlers()
|
||||
// Why: a detached client's git.responseAck frames will never arrive; wake
|
||||
// any pump parked on the ack window so it re-checks staleness and exits.
|
||||
this.dispatcher.onClientDetached?.(() => this.responseStreams.wakeAll())
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.responseStreams.disposeAll()
|
||||
}
|
||||
|
||||
private registerHandlers(): void {
|
||||
|
|
@ -190,7 +202,7 @@ export class GitHandler {
|
|||
this.dispatcher.onRequest('git.checkIgnored', (p) => this.checkIgnored(p))
|
||||
this.dispatcher.onRequest('git.history', (p) => this.history(p))
|
||||
this.dispatcher.onRequest('git.commit', (p) => this.commit(p))
|
||||
this.dispatcher.onRequest('git.diff', (p) => this.getDiff(p))
|
||||
this.dispatcher.onRequest('git.diff', (p, context) => this.getDiff(p, context))
|
||||
this.dispatcher.onRequest('git.stage', (p) => this.stage(p))
|
||||
this.dispatcher.onRequest('git.unstage', (p) => this.unstage(p))
|
||||
this.dispatcher.onRequest('git.bulkStage', (p) => this.bulkStage(p))
|
||||
|
|
@ -215,8 +227,8 @@ export class GitHandler {
|
|||
this.dispatcher.onRequest('git.pull', (p) => this.pull(p))
|
||||
this.dispatcher.onRequest('git.fastForward', (p) => this.fastForward(p))
|
||||
this.dispatcher.onRequest('git.rebaseFromBase', (p) => this.rebaseFromBase(p))
|
||||
this.dispatcher.onRequest('git.branchDiff', (p) => this.branchDiff(p))
|
||||
this.dispatcher.onRequest('git.commitDiff', (p) => this.commitDiff(p))
|
||||
this.dispatcher.onRequest('git.branchDiff', (p, context) => this.branchDiff(p, context))
|
||||
this.dispatcher.onRequest('git.commitDiff', (p, context) => this.commitDiff(p, context))
|
||||
this.dispatcher.onRequest('git.listWorktrees', (p, context) => this.listWorktrees(p, context))
|
||||
this.dispatcher.onRequest('git.addWorktree', (p) => this.addWorktree(p))
|
||||
this.dispatcher.onRequest('git.removeWorktree', (p) => this.removeWorktree(p))
|
||||
|
|
@ -231,6 +243,45 @@ export class GitHandler {
|
|||
this.dispatcher.onRequest('git.exec', (p, context) => this.exec(p, context))
|
||||
this.dispatcher.onRequest('git.clone', (p, context) => this.clone(p, context))
|
||||
this.dispatcher.onRequest('git.isGitRepo', (p) => this.isGitRepo(p))
|
||||
this.dispatcher.onNotification('git.responseAck', (p, context) => this.responseAck(p, context))
|
||||
this.dispatcher.onNotification('git.cancelResponseStream', (p, context) =>
|
||||
this.cancelResponseStream(p, context)
|
||||
)
|
||||
}
|
||||
|
||||
private responseAck(params: Record<string, unknown>, context: RequestContext): void {
|
||||
const streamId = params.streamId
|
||||
const seq = params.seq
|
||||
if (typeof streamId === 'number' && typeof seq === 'number') {
|
||||
this.responseStreams.recordAck(streamId, seq, context.clientId)
|
||||
}
|
||||
}
|
||||
|
||||
private cancelResponseStream(params: Record<string, unknown>, context: RequestContext): void {
|
||||
const streamId = params.streamId
|
||||
if (typeof streamId === 'number') {
|
||||
this.responseStreams.abort(streamId, context.clientId)
|
||||
}
|
||||
}
|
||||
|
||||
// Why: when the client opted into response streaming and the serialized result
|
||||
// exceeds the threshold, chunk it onto the bulk lane and return a small
|
||||
// sentinel as the RPC result. Old clients omit the flag (single-frame, as
|
||||
// today); old relays never call this, so a new client falls back to the plain
|
||||
// result they return.
|
||||
private maybeStreamResponse(
|
||||
result: unknown,
|
||||
params: Record<string, unknown>,
|
||||
context: RequestContext | undefined
|
||||
): unknown {
|
||||
if (params.__streamResponse !== true || !context) {
|
||||
return result
|
||||
}
|
||||
const payload = Buffer.from(JSON.stringify(result ?? null), 'utf-8')
|
||||
if (payload.length <= GIT_RESPONSE_STREAM_THRESHOLD) {
|
||||
return result
|
||||
}
|
||||
return this.responseStreams.startStream(payload, this.dispatcher, context)
|
||||
}
|
||||
|
||||
private async runWithDiffDedupeClear<T>(run: () => Promise<T>): Promise<T> {
|
||||
|
|
@ -352,7 +403,7 @@ export class GitHandler {
|
|||
})
|
||||
}
|
||||
|
||||
private async getDiff(params: Record<string, unknown>) {
|
||||
private async getDiff(params: Record<string, unknown>, context?: RequestContext) {
|
||||
const worktreePath = params.worktreePath as string
|
||||
const filePath = params.filePath as string
|
||||
// Why: filePath is relative to worktreePath and used in readWorkingFile via
|
||||
|
|
@ -366,7 +417,7 @@ export class GitHandler {
|
|||
const compareAgainstHead = params.compareAgainstHead as boolean | undefined
|
||||
// Why: register the in-flight dedupe synchronously (before any await) so
|
||||
// concurrent identical reads coalesce; submodule routing happens inside.
|
||||
return this.gitDiffReadDedupe.run(
|
||||
const result = await this.gitDiffReadDedupe.run(
|
||||
stableInFlightKey(['diff', worktreePath, filePath, staged, compareAgainstHead]),
|
||||
async () => {
|
||||
// Why: gitlink paths can't be read as blobs and submodule working dirs
|
||||
|
|
@ -431,6 +482,7 @@ export class GitHandler {
|
|||
)
|
||||
}
|
||||
)
|
||||
return this.maybeStreamResponse(result, params, context)
|
||||
}
|
||||
|
||||
private async stage(params: Record<string, unknown>) {
|
||||
|
|
@ -1027,7 +1079,7 @@ export class GitHandler {
|
|||
}
|
||||
}
|
||||
|
||||
private async branchDiff(params: Record<string, unknown>) {
|
||||
private async branchDiff(params: Record<string, unknown>, context?: RequestContext) {
|
||||
const worktreePath = params.worktreePath as string
|
||||
const baseRef = params.baseRef as string
|
||||
if (baseRef.startsWith('-')) {
|
||||
|
|
@ -1038,7 +1090,7 @@ export class GitHandler {
|
|||
filePath: params.filePath as string | undefined,
|
||||
oldPath: params.oldPath as string | undefined
|
||||
}
|
||||
return this.gitDiffReadDedupe.run(
|
||||
const result = await this.gitDiffReadDedupe.run(
|
||||
stableInFlightKey([
|
||||
'branchDiff',
|
||||
worktreePath,
|
||||
|
|
@ -1056,9 +1108,10 @@ export class GitHandler {
|
|||
options
|
||||
)
|
||||
)
|
||||
return this.maybeStreamResponse(result, params, context)
|
||||
}
|
||||
|
||||
private async commitDiff(params: Record<string, unknown>) {
|
||||
private async commitDiff(params: Record<string, unknown>, context?: RequestContext) {
|
||||
const worktreePath = params.worktreePath as string
|
||||
const args = {
|
||||
commitOid: params.commitOid as string,
|
||||
|
|
@ -1066,7 +1119,7 @@ export class GitHandler {
|
|||
filePath: params.filePath as string,
|
||||
oldPath: params.oldPath as string | undefined
|
||||
}
|
||||
return this.gitDiffReadDedupe.run(
|
||||
const result = await this.gitDiffReadDedupe.run(
|
||||
stableInFlightKey([
|
||||
'commitDiff',
|
||||
worktreePath,
|
||||
|
|
@ -1077,6 +1130,7 @@ export class GitHandler {
|
|||
]),
|
||||
() => commitDiffEntry(this.gitBuffer.bind(this), worktreePath, args)
|
||||
)
|
||||
return this.maybeStreamResponse(result, params, context)
|
||||
}
|
||||
|
||||
private async exec(params: Record<string, unknown>, context?: RequestContext) {
|
||||
|
|
@ -1085,7 +1139,7 @@ export class GitHandler {
|
|||
|
||||
validateGitExecArgs(args)
|
||||
const { stdout, stderr } = await this.git(args, cwd, { signal: context?.signal })
|
||||
return { stdout, stderr }
|
||||
return this.maybeStreamResponse({ stdout, stderr }, params, context)
|
||||
}
|
||||
|
||||
private async clone(params: Record<string, unknown>, context?: RequestContext) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,309 @@
|
|||
/**
|
||||
* Regression: SSH typing latency under large git responses.
|
||||
*
|
||||
* The relay and the client share ONE ordered SSH channel. A large git diff/exec
|
||||
* response sent as a single JSON-RPC frame queues megabytes into the outbound
|
||||
* pipe at once; an interactive pty.data echo emitted mid-response then sits
|
||||
* behind all of it and typing feels seconds-slow.
|
||||
*
|
||||
* These tests model the SSH channel as a congestible in-memory pipe and assert
|
||||
* deterministic byte bounds instead of wall-clock latency:
|
||||
* - WITHOUT the fix (client does not opt in), the whole response queues ahead
|
||||
* of a pty echo;
|
||||
* - WITH the fix (client opts in), the response streams on the bulk lane and
|
||||
* at most ~1 chunk frame sits ahead of the echo;
|
||||
* - both paths reassemble the identical git result.
|
||||
*/
|
||||
import { describe, expect, it, beforeEach, afterEach } from 'vitest'
|
||||
import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'
|
||||
import { rm } from 'node:fs/promises'
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import * as path from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
|
||||
import {
|
||||
SshChannelMultiplexer,
|
||||
type MultiplexerTransport
|
||||
} from '../main/ssh/ssh-channel-multiplexer'
|
||||
import { requestGitStreamable } from '../main/ssh/ssh-git-response-stream-reader'
|
||||
|
||||
import { RelayDispatcher } from './dispatcher'
|
||||
import { RelayContext } from './context'
|
||||
import { GitHandler } from './git-handler'
|
||||
import { GIT_RESPONSE_CHUNK_SIZE } from './protocol'
|
||||
|
||||
// One framed git.responseChunk: base64 (4/3) + JSON envelope + header slack.
|
||||
const FRAMED_CHUNK_BYTES = Math.ceil((GIT_RESPONSE_CHUNK_SIZE * 4) / 3) + 512
|
||||
const SINK_HIGH_WATER_MARK = 64 * 1024
|
||||
|
||||
async function waitUntil(
|
||||
predicate: () => boolean,
|
||||
what: string,
|
||||
timeoutMs = 10_000
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (!predicate()) {
|
||||
if (Date.now() > deadline) {
|
||||
throw new Error(`waitUntil timed out: ${what}`)
|
||||
}
|
||||
await new Promise((r) => setImmediate(r))
|
||||
}
|
||||
}
|
||||
|
||||
async function waitUntilSettled(read: () => number, stableTurns = 25): Promise<void> {
|
||||
let last = read()
|
||||
let stable = 0
|
||||
while (stable < stableTurns) {
|
||||
await new Promise((r) => setImmediate(r))
|
||||
const current = read()
|
||||
if (current === last) {
|
||||
stable += 1
|
||||
} else {
|
||||
stable = 0
|
||||
last = current
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type Harness = {
|
||||
mux: SshChannelMultiplexer
|
||||
dispatcher: RelayDispatcher
|
||||
gitHandler: GitHandler
|
||||
queuedBytes: () => number
|
||||
deliverAll: () => void
|
||||
startAutoDeliver: () => void
|
||||
dispose: () => void
|
||||
}
|
||||
|
||||
function createHarness(opts: { congested: boolean }): Harness {
|
||||
let relayFeed: ((data: Buffer) => void) | null = null
|
||||
const clientDataCallbacks: ((data: Buffer) => void)[] = []
|
||||
|
||||
const clientTransport: MultiplexerTransport = {
|
||||
write: (data: Buffer) => {
|
||||
// Client → relay (keystrokes, acks) is the opposite duplex direction and
|
||||
// is not blocked by relay→client congestion.
|
||||
setImmediate(() => relayFeed?.(data))
|
||||
},
|
||||
onData: (cb) => {
|
||||
clientDataCallbacks.push(cb)
|
||||
},
|
||||
onClose: () => {}
|
||||
}
|
||||
|
||||
const outQueue: Buffer[] = []
|
||||
let queuedBytes = 0
|
||||
const drainWaiters = new Set<() => void>()
|
||||
const fireDrainIfIdle = (): void => {
|
||||
if (queuedBytes > 0) {
|
||||
return
|
||||
}
|
||||
for (const cb of Array.from(drainWaiters)) {
|
||||
drainWaiters.delete(cb)
|
||||
cb()
|
||||
}
|
||||
}
|
||||
|
||||
const dispatcher = new RelayDispatcher(
|
||||
(data: Buffer) => {
|
||||
outQueue.push(data)
|
||||
queuedBytes += data.length
|
||||
if (!opts.congested) {
|
||||
return true
|
||||
}
|
||||
return queuedBytes < SINK_HIGH_WATER_MARK
|
||||
},
|
||||
{
|
||||
waitWriteDrain: (cb: () => void) => {
|
||||
drainWaiters.add(cb)
|
||||
fireDrainIfIdle()
|
||||
}
|
||||
}
|
||||
)
|
||||
relayFeed = (data: Buffer) => dispatcher.feed(data)
|
||||
|
||||
const deliverAll = (): void => {
|
||||
while (outQueue.length > 0) {
|
||||
const buf = outQueue.shift()!
|
||||
queuedBytes -= buf.length
|
||||
for (const cb of clientDataCallbacks) {
|
||||
cb(buf)
|
||||
}
|
||||
}
|
||||
fireDrainIfIdle()
|
||||
}
|
||||
|
||||
let autoDeliverTimer: ReturnType<typeof setInterval> | null = null
|
||||
const startAutoDeliver = (): void => {
|
||||
if (autoDeliverTimer) {
|
||||
return
|
||||
}
|
||||
autoDeliverTimer = setInterval(deliverAll, 1)
|
||||
}
|
||||
|
||||
const context = new RelayContext()
|
||||
const gitHandler = new GitHandler(dispatcher, context)
|
||||
const mux = new SshChannelMultiplexer(clientTransport)
|
||||
|
||||
return {
|
||||
mux,
|
||||
dispatcher,
|
||||
gitHandler,
|
||||
queuedBytes: () => queuedBytes,
|
||||
deliverAll,
|
||||
startAutoDeliver,
|
||||
dispose: () => {
|
||||
if (autoDeliverTimer) {
|
||||
clearInterval(autoDeliverTimer)
|
||||
}
|
||||
mux.dispose()
|
||||
dispatcher.dispose()
|
||||
gitHandler.dispose()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build a repo whose staged diff for `big.txt` is several MB so git.diff
|
||||
// returns a payload well over the stream threshold.
|
||||
function makeRepoWithLargeStagedDiff(dir: string): void {
|
||||
const env = { ...process.env }
|
||||
const run = (args: string[]): void => {
|
||||
execFileSync('git', args, { cwd: dir, stdio: 'pipe', env })
|
||||
}
|
||||
run(['init'])
|
||||
run(['config', 'user.email', 'test@test.com'])
|
||||
run(['config', 'user.name', 'Test'])
|
||||
// Distinct non-repeating lines keep the diff from compressing to nothing.
|
||||
// Stay under the render limits (120k lines / 6M chars) so the diff result
|
||||
// carries the full ~4MB content and exceeds the stream threshold.
|
||||
const lines: string[] = []
|
||||
for (let i = 0; i < 12_000; i += 1) {
|
||||
lines.push(`line ${i} ${'x'.repeat(100)}`)
|
||||
}
|
||||
writeFileSync(path.join(dir, 'big.txt'), lines.join('\n'))
|
||||
run(['add', 'big.txt'])
|
||||
}
|
||||
|
||||
describe('large git response vs pty.data echo head-of-line blocking', () => {
|
||||
let tmpDir: string
|
||||
let repoDir: string
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = mkdtempSync(path.join(tmpdir(), 'relay-git-hol-'))
|
||||
repoDir = path.join(tmpDir, 'repo')
|
||||
mkdirSync(repoDir, { recursive: true })
|
||||
makeRepoWithLargeStagedDiff(repoDir)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('WITHOUT opt-in: the whole diff queues ahead of a pty echo (single-frame HOL)', async () => {
|
||||
const harness = createHarness({ congested: true })
|
||||
try {
|
||||
let queuedBytesAheadOfEcho = -1
|
||||
harness.dispatcher.onNotification('pty.data', (params) => {
|
||||
queuedBytesAheadOfEcho = harness.queuedBytes()
|
||||
harness.dispatcher.notify('pty.data', { id: params.id, data: params.data })
|
||||
})
|
||||
|
||||
// Plain request WITHOUT __streamResponse: single-frame response, as today.
|
||||
const diffPromise = harness.mux.request('git.diff', {
|
||||
worktreePath: repoDir,
|
||||
filePath: 'big.txt',
|
||||
staged: true
|
||||
})
|
||||
// Let the whole single JSON-RPC frame land in the congested pipe.
|
||||
await waitUntil(() => harness.queuedBytes() > SINK_HIGH_WATER_MARK, 'diff frame queued')
|
||||
await waitUntilSettled(() => harness.queuedBytes())
|
||||
|
||||
harness.mux.notify('pty.data', { id: 'pty-1', data: 'x' })
|
||||
await waitUntil(() => queuedBytesAheadOfEcho >= 0, 'echo emitted by relay')
|
||||
|
||||
// The echo sits behind the entire (multi-hundred-KB) diff frame — far
|
||||
// more than a single bulk chunk would be.
|
||||
expect(queuedBytesAheadOfEcho).toBeGreaterThan(512 * 1024)
|
||||
|
||||
harness.startAutoDeliver()
|
||||
const result = (await diffPromise) as { diff?: string }
|
||||
expect(typeof result).toBe('object')
|
||||
} finally {
|
||||
harness.dispose()
|
||||
}
|
||||
}, 30_000)
|
||||
|
||||
it('WITH opt-in: the diff streams on the bulk lane and the echo stays bounded', async () => {
|
||||
const harness = createHarness({ congested: true })
|
||||
try {
|
||||
let queuedBytesAheadOfEcho = -1
|
||||
harness.dispatcher.onNotification('pty.data', (params) => {
|
||||
queuedBytesAheadOfEcho = harness.queuedBytes()
|
||||
harness.dispatcher.notify('pty.data', { id: params.id, data: params.data })
|
||||
})
|
||||
|
||||
const diffPromise = requestGitStreamable(harness.mux, 'git.diff', {
|
||||
worktreePath: repoDir,
|
||||
filePath: 'big.txt',
|
||||
staged: true
|
||||
})
|
||||
// Deliver the sentinel response, then let the pump run into congestion.
|
||||
await waitUntil(() => harness.queuedBytes() > 0, 'sentinel queued')
|
||||
harness.deliverAll()
|
||||
await waitUntil(() => harness.queuedBytes() > 0, 'first chunk queued')
|
||||
await waitUntilSettled(() => harness.queuedBytes())
|
||||
|
||||
harness.mux.notify('pty.data', { id: 'pty-1', data: 'x' })
|
||||
await waitUntil(() => queuedBytesAheadOfEcho >= 0, 'echo emitted by relay')
|
||||
|
||||
// At most one in-flight bulk frame (the write that saturated the sink)
|
||||
// plus slack sits ahead of the echo.
|
||||
expect(queuedBytesAheadOfEcho).toBeLessThan(2 * FRAMED_CHUNK_BYTES)
|
||||
|
||||
harness.startAutoDeliver()
|
||||
const streamed = (await diffPromise) as Record<string, unknown>
|
||||
expect(typeof streamed).toBe('object')
|
||||
expect(streamed).not.toHaveProperty('__orcaGitResponseStream')
|
||||
} finally {
|
||||
harness.dispose()
|
||||
}
|
||||
}, 30_000)
|
||||
|
||||
it('streamed result equals the single-frame result', async () => {
|
||||
const harness = createHarness({ congested: false })
|
||||
try {
|
||||
harness.startAutoDeliver()
|
||||
const params = { worktreePath: repoDir, filePath: 'big.txt', staged: true }
|
||||
// Why: request both concurrently so the shared 1ms delivery pump drives
|
||||
// the plain frame and the streamed chunk/ack round-trips without either
|
||||
// starving the other on a slow CI event loop.
|
||||
const [plain, streamed] = await Promise.all([
|
||||
harness.mux.request('git.diff', params),
|
||||
requestGitStreamable(harness.mux, 'git.diff', params)
|
||||
])
|
||||
expect(streamed).toEqual(plain)
|
||||
} finally {
|
||||
harness.dispose()
|
||||
}
|
||||
}, 30_000)
|
||||
|
||||
it('rejects with an inactivity timeout when the stream stalls after the sentinel', async () => {
|
||||
const harness = createHarness({ congested: false })
|
||||
try {
|
||||
// Deliver the sentinel so reassembly begins, then stop delivering chunks
|
||||
// to model a relay pump that wedged while the channel stayed up. Without
|
||||
// the client-side inactivity deadline this promise would hang forever.
|
||||
const streamedPromise = requestGitStreamable(
|
||||
harness.mux,
|
||||
'git.diff',
|
||||
{ worktreePath: repoDir, filePath: 'big.txt', staged: true },
|
||||
{ inactivityTimeoutMs: 250 }
|
||||
)
|
||||
await waitUntil(() => harness.queuedBytes() > 0, 'sentinel queued')
|
||||
harness.deliverAll() // sentinel only; chunks stay undelivered
|
||||
await expect(streamedPromise).rejects.toThrow(/stalled/)
|
||||
} finally {
|
||||
harness.dispose()
|
||||
}
|
||||
}, 30_000)
|
||||
})
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RelayDispatcher, RequestContext } from './dispatcher'
|
||||
import { GitResponseStreamRegistry } from './git-response-stream'
|
||||
import { GIT_RESPONSE_CHUNK_SIZE, STREAM_ACK_WINDOW_CHUNKS } from './protocol'
|
||||
|
||||
async function flushPump(): Promise<void> {
|
||||
await new Promise<void>((resolve) => setImmediate(resolve))
|
||||
}
|
||||
|
||||
describe('GitResponseStreamRegistry client ownership', () => {
|
||||
const registries: GitResponseStreamRegistry[] = []
|
||||
|
||||
afterEach(() => {
|
||||
for (const registry of registries) {
|
||||
registry.disposeAll()
|
||||
}
|
||||
registries.length = 0
|
||||
})
|
||||
|
||||
it('ignores acknowledgements and cancellation from a different relay client', async () => {
|
||||
const ownerClientId = 7
|
||||
const notifyBulk = vi.fn().mockResolvedValue(undefined)
|
||||
const dispatcher = {
|
||||
notifyBulk,
|
||||
notify: vi.fn()
|
||||
} as unknown as RelayDispatcher
|
||||
const context: RequestContext = {
|
||||
clientId: ownerClientId,
|
||||
isStale: () => false
|
||||
}
|
||||
const registry = new GitResponseStreamRegistry()
|
||||
registries.push(registry)
|
||||
const payload = Buffer.alloc(GIT_RESPONSE_CHUNK_SIZE * (STREAM_ACK_WINDOW_CHUNKS * 3))
|
||||
const marker = registry.startStream(payload, dispatcher, context)
|
||||
const streamId = marker.__orcaGitResponseStream.streamId
|
||||
|
||||
await flushPump()
|
||||
expect(notifyBulk).toHaveBeenCalledTimes(STREAM_ACK_WINDOW_CHUNKS)
|
||||
|
||||
registry.recordAck(streamId, 10_000, ownerClientId + 1)
|
||||
await flushPump()
|
||||
expect(notifyBulk).toHaveBeenCalledTimes(STREAM_ACK_WINDOW_CHUNKS)
|
||||
|
||||
registry.abort(streamId, ownerClientId + 1)
|
||||
registry.recordAck(streamId, STREAM_ACK_WINDOW_CHUNKS - 1, ownerClientId)
|
||||
await flushPump()
|
||||
expect(notifyBulk.mock.calls.length).toBeGreaterThan(STREAM_ACK_WINDOW_CHUNKS)
|
||||
})
|
||||
|
||||
it('contains a secondary failure while reporting a stream error', async () => {
|
||||
const notifyBulk = vi.fn().mockRejectedValue(new Error('socket closed'))
|
||||
const dispatcher = { notifyBulk, notify: vi.fn() } as unknown as RelayDispatcher
|
||||
const registry = new GitResponseStreamRegistry()
|
||||
registries.push(registry)
|
||||
|
||||
registry.startStream(Buffer.from('payload'), dispatcher, {
|
||||
clientId: 7,
|
||||
isStale: () => false
|
||||
})
|
||||
|
||||
await flushPump()
|
||||
await flushPump()
|
||||
|
||||
expect(notifyBulk).toHaveBeenCalledTimes(2)
|
||||
expect(notifyBulk.mock.calls[0]?.[0]).toBe('git.responseChunk')
|
||||
expect(notifyBulk.mock.calls[1]?.[0]).toBe('git.responseError')
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,211 @@
|
|||
// Streams large git RPC responses (diff family + exec) onto the bulk lane in
|
||||
// chunks instead of one JSON-RPC frame, so a big diff cannot head-of-line-block
|
||||
// interactive pty.data echo on the shared SSH channel. Mirrors the fs
|
||||
// read-stream credit-window pattern (see fs-handler-file-read.ts) but the
|
||||
// payload is an in-memory serialized string rather than a file handle.
|
||||
import type { RelayDispatcher, RequestContext } from './dispatcher'
|
||||
import {
|
||||
GIT_RESPONSE_CHUNK_SIZE,
|
||||
STREAM_ACK_WINDOW_CHUNKS,
|
||||
STREAM_ACK_STALL_RECHECK_MS,
|
||||
type GitResponseStreamMarker
|
||||
} from './protocol'
|
||||
|
||||
type GitResponseStreamEntry = {
|
||||
ownerClientId: number
|
||||
aborted: boolean
|
||||
/** Highest chunk seq the client acknowledged (in-order; -1 = none yet). */
|
||||
ackedThroughSeq: number
|
||||
ackWaiters: Set<() => void>
|
||||
}
|
||||
|
||||
/** Serialized git responses are chunked as base64 so multi-byte UTF-8
|
||||
* sequences never split across a chunk boundary (the client concatenates the
|
||||
* decoded bytes and parses once). */
|
||||
function encodeChunks(payload: Buffer): string[] {
|
||||
const chunks: string[] = []
|
||||
for (let offset = 0; offset < payload.length; offset += GIT_RESPONSE_CHUNK_SIZE) {
|
||||
chunks.push(payload.subarray(offset, offset + GIT_RESPONSE_CHUNK_SIZE).toString('base64'))
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
export class GitResponseStreamRegistry {
|
||||
private streams = new Map<number, GitResponseStreamEntry>()
|
||||
private nextId = 1
|
||||
|
||||
private register(ownerClientId: number): number {
|
||||
const streamId = this.nextId++
|
||||
this.streams.set(streamId, {
|
||||
ownerClientId,
|
||||
aborted: false,
|
||||
ackedThroughSeq: -1,
|
||||
ackWaiters: new Set()
|
||||
})
|
||||
return streamId
|
||||
}
|
||||
|
||||
recordAck(streamId: number, seq: number, clientId: number): void {
|
||||
const entry = this.streams.get(streamId)
|
||||
if (
|
||||
!entry ||
|
||||
entry.ownerClientId !== clientId ||
|
||||
typeof seq !== 'number' ||
|
||||
!Number.isFinite(seq)
|
||||
) {
|
||||
return
|
||||
}
|
||||
if (seq > entry.ackedThroughSeq) {
|
||||
entry.ackedThroughSeq = seq
|
||||
}
|
||||
this.wake(entry)
|
||||
}
|
||||
|
||||
abort(streamId: number, clientId: number): void {
|
||||
const entry = this.streams.get(streamId)
|
||||
if (entry?.ownerClientId === clientId) {
|
||||
entry.aborted = true
|
||||
this.wake(entry)
|
||||
}
|
||||
}
|
||||
|
||||
/** Wake every parked pump so it re-checks staleness — used when a client
|
||||
* detaches and its acks will never arrive. */
|
||||
wakeAll(): void {
|
||||
for (const entry of this.streams.values()) {
|
||||
this.wake(entry)
|
||||
}
|
||||
}
|
||||
|
||||
private wake(entry: GitResponseStreamEntry): void {
|
||||
for (const waiter of Array.from(entry.ackWaiters)) {
|
||||
waiter()
|
||||
}
|
||||
}
|
||||
|
||||
private waitForAck(streamId: number): Promise<void> {
|
||||
const entry = this.streams.get(streamId)
|
||||
if (!entry || entry.aborted) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
let settled = false
|
||||
const finish = (): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
clearTimeout(timer)
|
||||
entry.ackWaiters.delete(finish)
|
||||
resolve()
|
||||
}
|
||||
const timer = setTimeout(finish, STREAM_ACK_STALL_RECHECK_MS)
|
||||
timer.unref?.()
|
||||
entry.ackWaiters.add(finish)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a stream for `payload`, kick off the bulk-lane pump on a later
|
||||
* task (so the sentinel response reaches the client first), and return the
|
||||
* sentinel marker to send as the RPC result.
|
||||
*/
|
||||
startStream(
|
||||
payload: Buffer,
|
||||
dispatcher: RelayDispatcher,
|
||||
context: RequestContext
|
||||
): GitResponseStreamMarker {
|
||||
const streamId = this.register(context.clientId)
|
||||
const chunks = encodeChunks(payload)
|
||||
// Why: kick the pump off the response task so the client sees the sentinel
|
||||
// (and can subscribe/reassemble) before the first chunk frame arrives.
|
||||
setImmediate(() => {
|
||||
void this.pump(streamId, chunks, dispatcher, context)
|
||||
})
|
||||
return {
|
||||
__orcaGitResponseStream: { streamId, totalBytes: payload.length, chunkCount: chunks.length }
|
||||
}
|
||||
}
|
||||
|
||||
private async pump(
|
||||
streamId: number,
|
||||
chunks: string[],
|
||||
dispatcher: RelayDispatcher,
|
||||
context: RequestContext
|
||||
): Promise<void> {
|
||||
const entry = this.streams.get(streamId)
|
||||
if (!entry) {
|
||||
return
|
||||
}
|
||||
const clientId = context.clientId
|
||||
let seq = 0
|
||||
let endReason: 'end' | 'aborted' | 'stale' = 'end'
|
||||
try {
|
||||
for (seq = 0; seq < chunks.length; seq += 1) {
|
||||
if (context.isStale()) {
|
||||
endReason = 'stale'
|
||||
break
|
||||
}
|
||||
if (entry.aborted) {
|
||||
endReason = 'aborted'
|
||||
break
|
||||
}
|
||||
// Why: credit window — the client acks each chunk, bounding how many
|
||||
// bulk bytes a keystroke echo can queue behind on the shared channel.
|
||||
while (
|
||||
seq - entry.ackedThroughSeq > STREAM_ACK_WINDOW_CHUNKS &&
|
||||
!context.isStale() &&
|
||||
!entry.aborted
|
||||
) {
|
||||
await this.waitForAck(streamId)
|
||||
}
|
||||
if (context.isStale()) {
|
||||
endReason = 'stale'
|
||||
break
|
||||
}
|
||||
if (entry.aborted) {
|
||||
endReason = 'aborted'
|
||||
break
|
||||
}
|
||||
// Why: notifyBulk waits out sink saturation so chunk frames never pile
|
||||
// up in the outbound pipe ahead of interactive pty.data frames.
|
||||
await dispatcher.notifyBulk(
|
||||
'git.responseChunk',
|
||||
{ streamId, seq, data: chunks[seq] },
|
||||
{
|
||||
clientId
|
||||
}
|
||||
)
|
||||
}
|
||||
if (endReason === 'end') {
|
||||
await dispatcher.notifyBulk('git.responseEnd', { streamId }, { clientId })
|
||||
}
|
||||
} catch (err) {
|
||||
if (!context.isStale() && !entry.aborted) {
|
||||
try {
|
||||
await dispatcher.notifyBulk(
|
||||
'git.responseError',
|
||||
{
|
||||
streamId,
|
||||
message: err instanceof Error ? err.message : String(err)
|
||||
},
|
||||
{ clientId }
|
||||
)
|
||||
} catch {
|
||||
// Why: the original failure may be the owning channel closing; a
|
||||
// second send failure must not escape this detached pump.
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.streams.delete(streamId)
|
||||
}
|
||||
}
|
||||
|
||||
disposeAll(): void {
|
||||
for (const entry of this.streams.values()) {
|
||||
entry.aborted = true
|
||||
this.wake(entry)
|
||||
}
|
||||
this.streams.clear()
|
||||
}
|
||||
}
|
||||
|
|
@ -59,6 +59,27 @@ export const STREAM_ACK_WINDOW_CHUNKS = 4
|
|||
* (and its open file handle) forever. */
|
||||
export const STREAM_ACK_STALL_RECHECK_MS = 1_000
|
||||
|
||||
// ── Git response streaming (see docs/relay-git-response-stream-design.md) ──
|
||||
|
||||
/** Serialized-JSON size above which a streamable git response (diff family +
|
||||
* exec) is chunked onto the bulk lane instead of one JSON-RPC frame, so a large
|
||||
* diff cannot head-of-line-block interactive pty.data echo on the shared SSH
|
||||
* channel. Below this, single-frame is cheaper and avoids stream overhead. */
|
||||
export const GIT_RESPONSE_STREAM_THRESHOLD = 256 * 1024
|
||||
|
||||
/** Per-chunk size (UTF-8 bytes of the serialized result) for git response
|
||||
* streaming. Independent from STREAM_CHUNK_SIZE — this offset math is not
|
||||
* shared with fs streams, so tuning it here is cross-version safe as long as
|
||||
* the client reassembles by concatenation (it does not depend on chunk size). */
|
||||
export const GIT_RESPONSE_CHUNK_SIZE = 128 * 1024
|
||||
|
||||
/** Sentinel result returned in place of a large git response: the real payload
|
||||
* follows as git.responseChunk frames on the bulk lane. Old relays never emit
|
||||
* this, so a new client falls back to the plain result they return. */
|
||||
export type GitResponseStreamMarker = {
|
||||
__orcaGitResponseStream: { streamId: number; totalBytes: number; chunkCount: number }
|
||||
}
|
||||
|
||||
export const RelayErrorCode = {
|
||||
TooManyStreams: -33006,
|
||||
StreamProtocolError: -33007
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ import { RelayDispatcher } from './dispatcher'
|
|||
import { RelayContext } from './context'
|
||||
import { PtyHandler } from './pty-handler'
|
||||
import { FsHandler } from './fs-handler'
|
||||
import { installRelayLogRotation } from './rotating-log-writer'
|
||||
import { GitHandler } from './git-handler'
|
||||
import { PreflightHandler } from './preflight-handler'
|
||||
import { ExternalAutomationsHandler } from './external-automations-handler'
|
||||
|
|
@ -114,6 +115,7 @@ function parseArgs(argv: string[]): {
|
|||
cliMode: boolean
|
||||
sockPath: string
|
||||
endpointDir?: string
|
||||
logFile?: string
|
||||
} {
|
||||
let graceTimeMs = DEFAULT_GRACE_MS
|
||||
let connectMode = false
|
||||
|
|
@ -121,6 +123,7 @@ function parseArgs(argv: string[]): {
|
|||
let cliMode = false
|
||||
let sockPath = ''
|
||||
let endpointDir: string | undefined
|
||||
let logFile: string | undefined
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
if (argv[i] === '--grace-time' && argv[i + 1]) {
|
||||
const parsed = Number.parseInt(argv[i + 1], 10)
|
||||
|
|
@ -143,12 +146,15 @@ function parseArgs(argv: string[]): {
|
|||
} else if (argv[i] === '--endpoint-dir' && argv[i + 1]) {
|
||||
endpointDir = argv[i + 1]
|
||||
i++
|
||||
} else if (argv[i] === '--log-file' && argv[i + 1]) {
|
||||
logFile = argv[i + 1]
|
||||
i++
|
||||
}
|
||||
}
|
||||
if (!sockPath) {
|
||||
sockPath = join(process.cwd(), SOCK_NAME)
|
||||
}
|
||||
return { graceTimeMs, connectMode, detached, cliMode, sockPath, endpointDir }
|
||||
return { graceTimeMs, connectMode, detached, cliMode, sockPath, endpointDir, logFile }
|
||||
}
|
||||
|
||||
// ── Connect mode ─────────────────────────────────────────────────────
|
||||
|
|
@ -318,7 +324,7 @@ async function readOrcaCliStdin(): Promise<string | undefined> {
|
|||
// ── Normal mode ──────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const { graceTimeMs, connectMode, detached, cliMode, sockPath, endpointDir } = parseArgs(
|
||||
const { graceTimeMs, connectMode, detached, cliMode, sockPath, endpointDir, logFile } = parseArgs(
|
||||
process.argv
|
||||
)
|
||||
|
||||
|
|
@ -332,6 +338,13 @@ async function main(): Promise<void> {
|
|||
return
|
||||
}
|
||||
|
||||
// Why: only the long-lived detached daemon accumulates relay.log; --connect
|
||||
// bridges and --orca-cli are short-lived and already returned above. Route all
|
||||
// relay logging through a size-capped rotator so relay.log can't grow forever.
|
||||
if (detached && logFile) {
|
||||
installRelayLogRotation(logFile)
|
||||
}
|
||||
|
||||
let ownsSocketPath = false
|
||||
let ownedSocketIdentity: SocketIdentity | null = null
|
||||
const ownsCurrentSocketPath = (): boolean => {
|
||||
|
|
@ -454,10 +467,7 @@ async function main(): Promise<void> {
|
|||
|
||||
const ptyHandler = new PtyHandler(dispatcher, graceTimeMs)
|
||||
const fsHandler = new FsHandler(dispatcher, context)
|
||||
// Why: GitHandler registers its own request handlers on construction,
|
||||
// so we hold the reference only for potential future disposal.
|
||||
const _gitHandler = new GitHandler(dispatcher, context)
|
||||
void _gitHandler
|
||||
const gitHandler = new GitHandler(dispatcher, context)
|
||||
|
||||
const _preflightHandler = new PreflightHandler(dispatcher)
|
||||
const _externalAutomationsHandler = new ExternalAutomationsHandler(dispatcher)
|
||||
|
|
@ -1013,6 +1023,7 @@ async function main(): Promise<void> {
|
|||
dispatcher.dispose()
|
||||
ptyHandler.dispose()
|
||||
fsHandler.dispose()
|
||||
gitHandler.dispose()
|
||||
hookServer.stop()
|
||||
// Why: Node's Unix server.close() can unlink the listen path. If the path
|
||||
// was externally removed and rebound by a newer relay, closing this older
|
||||
|
|
|
|||
|
|
@ -0,0 +1,161 @@
|
|||
/**
|
||||
* Regression: relay.log grows unbounded on long-lived relays.
|
||||
*
|
||||
* The relay is launched detached with `> relay.log 2>&1`, which truncates only
|
||||
* at relaunch; a relay that stays up for days accumulates per-stream stderr
|
||||
* lines forever. These tests assert the in-process rotator caps size, keeps one
|
||||
* archived generation, and always leaves the CURRENT log at relay.log so the
|
||||
* `tail -100 relay.log` diagnostics workflow keeps working.
|
||||
*/
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { mkdtempSync, mkdirSync, readFileSync, existsSync, statSync, writeFileSync } from 'node:fs'
|
||||
import { rm } from 'node:fs/promises'
|
||||
import * as path from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
|
||||
import { RotatingLogWriter, installRelayLogRotation } from './rotating-log-writer'
|
||||
|
||||
describe('RotatingLogWriter', () => {
|
||||
let dir: string
|
||||
let logPath: string
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(path.join(tmpdir(), 'relay-log-rot-'))
|
||||
logPath = path.join(dir, 'relay.log')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('rotates relay.log -> relay.log.1 at the cap and keeps the current log tail-able', () => {
|
||||
const cap = 4 * 1024
|
||||
const writer = new RotatingLogWriter(logPath, cap)
|
||||
try {
|
||||
const line = `${'a'.repeat(200)}\n`
|
||||
// Write well past the cap so at least one rotation happens.
|
||||
for (let i = 0; i < 60; i += 1) {
|
||||
writer.write(line)
|
||||
}
|
||||
|
||||
// Current log exists at relay.log (tail target) and is under the cap.
|
||||
expect(existsSync(logPath)).toBe(true)
|
||||
expect(statSync(logPath).size).toBeLessThanOrEqual(cap)
|
||||
// Exactly one archived generation.
|
||||
expect(existsSync(`${logPath}.1`)).toBe(true)
|
||||
expect(existsSync(`${logPath}.2`)).toBe(false)
|
||||
|
||||
// The most recent lines are in the current log (tail-ability).
|
||||
writer.write('MARKER-LAST\n')
|
||||
expect(readFileSync(logPath, 'utf-8')).toContain('MARKER-LAST')
|
||||
} finally {
|
||||
writer.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('preserves pre-existing boot output already in relay.log (append, not truncate)', () => {
|
||||
writeFileSync(logPath, 'BOOT-LINE-FROM-SHELL-REDIRECT\n')
|
||||
const writer = new RotatingLogWriter(logPath, 1024 * 1024)
|
||||
try {
|
||||
writer.write('runtime line\n')
|
||||
const contents = readFileSync(logPath, 'utf-8')
|
||||
expect(contents).toContain('BOOT-LINE-FROM-SHELL-REDIRECT')
|
||||
expect(contents).toContain('runtime line')
|
||||
} finally {
|
||||
writer.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('bounds a single oversized log write and keeps its newest tail', () => {
|
||||
const cap = 1024
|
||||
const writer = new RotatingLogWriter(logPath, cap)
|
||||
try {
|
||||
writer.write(`${'old'.repeat(1000)}LATEST-CONTEXT`)
|
||||
expect(statSync(logPath).size).toBeLessThanOrEqual(cap)
|
||||
expect(readFileSync(logPath, 'utf-8')).toContain('LATEST-CONTEXT')
|
||||
} finally {
|
||||
writer.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('caps total footprint to ~2x maxBytes (current + one archive)', () => {
|
||||
const cap = 8 * 1024
|
||||
const writer = new RotatingLogWriter(logPath, cap)
|
||||
try {
|
||||
const line = `${'z'.repeat(256)}\n`
|
||||
for (let i = 0; i < 500; i += 1) {
|
||||
writer.write(line)
|
||||
}
|
||||
const currentSize = statSync(logPath).size
|
||||
const archiveSize = existsSync(`${logPath}.1`) ? statSync(`${logPath}.1`).size : 0
|
||||
// Never more than the current file + a single archived generation.
|
||||
expect(currentSize).toBeLessThanOrEqual(cap)
|
||||
expect(archiveSize).toBeLessThanOrEqual(cap)
|
||||
expect(existsSync(`${logPath}.2`)).toBe(false)
|
||||
} finally {
|
||||
writer.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('caps size via truncate-in-place when rename cannot succeed (Windows shell-handle case)', () => {
|
||||
// Model the platform where renameSync fails (e.g. Windows, where the launch
|
||||
// shell's own `1>relay.log` handle blocks renaming the live file): make the
|
||||
// archive path a directory so renameSync(logPath, `${logPath}.1`) throws.
|
||||
mkdirSync(`${logPath}.1`, { recursive: true })
|
||||
const cap = 4 * 1024
|
||||
const writer = new RotatingLogWriter(logPath, cap)
|
||||
try {
|
||||
const line = `${'q'.repeat(200)}\n`
|
||||
for (let i = 0; i < 100; i += 1) {
|
||||
writer.write(line)
|
||||
}
|
||||
// The cap still holds via truncate-in-place even though no archive was made.
|
||||
expect(statSync(logPath).size).toBeLessThanOrEqual(cap)
|
||||
writer.write('MARKER-LAST\n')
|
||||
expect(readFileSync(logPath, 'utf-8')).toContain('MARKER-LAST')
|
||||
} finally {
|
||||
writer.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
it('installRelayLogRotation routes stdout/stderr through the rotator and restores', () => {
|
||||
const cap = 2 * 1024
|
||||
const { restore } = installRelayLogRotation(logPath, cap)
|
||||
try {
|
||||
process.stderr.write('via-stderr-line\n')
|
||||
process.stdout.write('via-stdout-line\n')
|
||||
expect(readFileSync(logPath, 'utf-8')).toContain('via-stderr-line')
|
||||
expect(readFileSync(logPath, 'utf-8')).toContain('via-stdout-line')
|
||||
} finally {
|
||||
restore()
|
||||
}
|
||||
// After restore, process.stderr no longer targets the rotator file. Spy so
|
||||
// the assertion write does not leak to the real test-runner stderr.
|
||||
const sizeAfterRestore = statSync(logPath).size
|
||||
const spy = vi.spyOn(process.stderr, 'write').mockReturnValue(true)
|
||||
try {
|
||||
process.stderr.write('should-not-be-in-relay-log\n')
|
||||
} finally {
|
||||
spy.mockRestore()
|
||||
}
|
||||
expect(statSync(logPath).size).toBe(sizeAfterRestore)
|
||||
})
|
||||
|
||||
it('leaves the original streams active when the log cannot be opened', () => {
|
||||
mkdirSync(logPath)
|
||||
const stdout = vi.spyOn(process.stdout, 'write').mockReturnValue(true)
|
||||
const stderr = vi.spyOn(process.stderr, 'write').mockReturnValue(true)
|
||||
try {
|
||||
const { writer, restore } = installRelayLogRotation(logPath)
|
||||
expect(writer.active).toBe(false)
|
||||
process.stdout.write('stdout fallback\n')
|
||||
process.stderr.write('stderr fallback\n')
|
||||
expect(stdout).toHaveBeenCalledWith('stdout fallback\n')
|
||||
expect(stderr).toHaveBeenCalledWith('stderr fallback\n')
|
||||
restore()
|
||||
} finally {
|
||||
stdout.mockRestore()
|
||||
stderr.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
// Size-capped, crash-safe rotation for the relay's diagnostic log.
|
||||
//
|
||||
// Why: the relay is launched detached with `> relay.log 2>&1`, which only
|
||||
// truncates at relaunch — a long-lived relay's relay.log grows unbounded
|
||||
// (per-stream stderr lines, reconnect flaps, etc.). Rotation must live in the
|
||||
// relay process because it owns its own stderr; the shell redirect cannot cap
|
||||
// size. Constraints honored:
|
||||
// - keeps working when the launch fd is a pipe/redirect on Linux/macOS/Windows
|
||||
// (append-only, no fd tricks; a rotation failure falls back to the prior fd);
|
||||
// - the diagnostics tail (`tail -100 ~/.orca-remote/relay-*/relay.log`) keeps
|
||||
// working because the CURRENT log is always at relay.log;
|
||||
// - crash-safe: writes and rotation are guarded so logging never throws, and
|
||||
// rotation renames (never deletes the live file) so no window loses logging.
|
||||
import { closeSync, openSync, renameSync, statSync, writeSync } from 'node:fs'
|
||||
|
||||
/** Default size cap before rotating relay.log → relay.log.1. 10 MB balances
|
||||
* keeping enough history for diagnosis against bounding disk use on small
|
||||
* remote hosts (e.g. a Raspberry Pi); one archived generation is kept. */
|
||||
export const DEFAULT_RELAY_LOG_MAX_BYTES = 10 * 1024 * 1024
|
||||
|
||||
type StreamWrite = typeof process.stderr.write
|
||||
|
||||
export class RotatingLogWriter {
|
||||
private readonly logPath: string
|
||||
private readonly rotatedPath: string
|
||||
private readonly maxBytes: number
|
||||
private fd: number | null = null
|
||||
private currentBytes = 0
|
||||
private failed = false
|
||||
|
||||
constructor(logPath: string, maxBytes: number = DEFAULT_RELAY_LOG_MAX_BYTES) {
|
||||
this.logPath = logPath
|
||||
this.rotatedPath = `${logPath}.1`
|
||||
this.maxBytes = maxBytes
|
||||
this.open()
|
||||
}
|
||||
|
||||
private open(mode: 'a' | 'w' = 'a'): void {
|
||||
try {
|
||||
// 'a' preserves pre-JS boot output already in relay.log and keeps
|
||||
// concurrent appends atomic; 'w' truncates in place (used as the rotation
|
||||
// fallback when a rename cannot succeed — e.g. Windows, where the launch
|
||||
// shell's own redirect handle blocks renaming the live file).
|
||||
this.fd = openSync(this.logPath, mode)
|
||||
try {
|
||||
this.currentBytes = mode === 'w' ? 0 : statSync(this.logPath).size
|
||||
} catch {
|
||||
this.currentBytes = 0
|
||||
}
|
||||
} catch {
|
||||
// Cannot open the log file (permission/full disk): disable rotation and
|
||||
// let callers fall back to the original stream.
|
||||
this.failed = true
|
||||
this.fd = null
|
||||
}
|
||||
}
|
||||
|
||||
/** True when the writer is usable; false means fall back to raw stderr. */
|
||||
get active(): boolean {
|
||||
return !this.failed && this.fd !== null
|
||||
}
|
||||
|
||||
write(chunk: string | Uint8Array): void {
|
||||
if (!this.active || this.fd === null) {
|
||||
return
|
||||
}
|
||||
const encoded = typeof chunk === 'string' ? Buffer.from(chunk, 'utf-8') : Buffer.from(chunk)
|
||||
// Why: one pathological log write must not bypass the disk cap. Preserve
|
||||
// the newest tail because it carries the most useful failure context.
|
||||
const buf =
|
||||
encoded.length > this.maxBytes ? encoded.subarray(encoded.length - this.maxBytes) : encoded
|
||||
try {
|
||||
// Rotate BEFORE writing when the incoming write would cross the cap, so a
|
||||
// single large line still lands wholly in the fresh file.
|
||||
if (this.currentBytes > 0 && this.currentBytes + buf.length > this.maxBytes) {
|
||||
this.rotate()
|
||||
}
|
||||
writeSync(this.fd, buf)
|
||||
this.currentBytes += buf.length
|
||||
} catch {
|
||||
// A write failure must never crash the relay; disable and fall back.
|
||||
this.failed = true
|
||||
this.closeQuietly()
|
||||
}
|
||||
}
|
||||
|
||||
private rotate(): void {
|
||||
if (this.fd !== null) {
|
||||
closeSync(this.fd)
|
||||
this.fd = null
|
||||
}
|
||||
let renamed = false
|
||||
try {
|
||||
// Preferred path: archive one generation. rename() replaces any existing
|
||||
// relay.log.1 atomically on POSIX; the live file is renamed (never
|
||||
// deleted) so no log window is lost, then we reopen a fresh relay.log.
|
||||
renameSync(this.logPath, this.rotatedPath)
|
||||
renamed = true
|
||||
} catch {
|
||||
// Why: on Windows the launch shell holds its own `1>relay.log` handle
|
||||
// WITHOUT FILE_SHARE_DELETE, so renaming the live file fails (EPERM/EBUSY)
|
||||
// — and a cross-device relay.log.1 would fail too. Fall back to truncating
|
||||
// relay.log in place ('w') so the size cap STILL holds and we don't churn a
|
||||
// failing rename on every subsequent line. The shell's fd keeps appending
|
||||
// at its own offset, but the truncation bounds total growth. Trade-off: no
|
||||
// archived generation on this platform/path.
|
||||
}
|
||||
// Successful rename → fresh append file (size 0). Failed rename → truncate.
|
||||
this.open(renamed ? 'a' : 'w')
|
||||
}
|
||||
|
||||
private closeQuietly(): void {
|
||||
if (this.fd !== null) {
|
||||
try {
|
||||
closeSync(this.fd)
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
this.fd = null
|
||||
}
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.closeQuietly()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Route process.stdout/stderr writes through a RotatingLogWriter that owns
|
||||
* `logPath`. Returns a restore function. If the writer cannot open the file,
|
||||
* the original streams are left untouched (logging still works, just uncapped).
|
||||
*/
|
||||
export function installRelayLogRotation(
|
||||
logPath: string,
|
||||
maxBytes: number = DEFAULT_RELAY_LOG_MAX_BYTES
|
||||
): { writer: RotatingLogWriter; restore: () => void } {
|
||||
const writer = new RotatingLogWriter(logPath, maxBytes)
|
||||
const originalStdout = process.stdout.write.bind(process.stdout)
|
||||
const originalStderr = process.stderr.write.bind(process.stderr)
|
||||
|
||||
if (!writer.active) {
|
||||
return { writer, restore: () => {} }
|
||||
}
|
||||
|
||||
const wrap =
|
||||
(original: StreamWrite): StreamWrite =>
|
||||
(chunk: string | Uint8Array, encodingOrCb?: unknown, cb?: unknown): boolean => {
|
||||
writer.write(chunk)
|
||||
// Why: preserve the Writable.write callback contract so callers awaiting
|
||||
// the write (rare, but e.g. flush-before-exit) are not left hanging.
|
||||
const callback = typeof encodingOrCb === 'function' ? encodingOrCb : cb
|
||||
if (typeof callback === 'function') {
|
||||
;(callback as (err?: Error | null) => void)(null)
|
||||
}
|
||||
// If the writer went inactive mid-run (write failure), fall back so logs
|
||||
// are not silently dropped for the rest of the session.
|
||||
if (!writer.active) {
|
||||
return (original as (c: string | Uint8Array) => boolean)(chunk)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
process.stdout.write = wrap(originalStdout as StreamWrite) as typeof process.stdout.write
|
||||
process.stderr.write = wrap(originalStderr as StreamWrite) as typeof process.stderr.write
|
||||
|
||||
return {
|
||||
writer,
|
||||
restore: () => {
|
||||
process.stdout.write = originalStdout as typeof process.stdout.write
|
||||
process.stderr.write = originalStderr as typeof process.stderr.write
|
||||
writer.dispose()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,10 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { CLIPBOARD_TEXT_MEASURE_YIELD_CODE_UNITS } from '../../../../shared/clipboard-text'
|
||||
import { createRemoteRuntimePtyTextBatcher } from './remote-runtime-pty-batching'
|
||||
import {
|
||||
createRemoteRuntimePtyTextBatcher,
|
||||
createRemoteRuntimeViewportBatcher
|
||||
} from './remote-runtime-pty-batching'
|
||||
|
||||
describe('createRemoteRuntimePtyTextBatcher', () => {
|
||||
it('coalesces small input until the debounce flush', async () => {
|
||||
|
|
@ -139,3 +142,42 @@ describe('createRemoteRuntimePtyTextBatcher', () => {
|
|||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('createRemoteRuntimeViewportBatcher', () => {
|
||||
it('drops the queued viewport on clear so a later flush emits nothing', () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const resizes: { cols: number; rows: number }[] = []
|
||||
const batcher = createRemoteRuntimeViewportBatcher(33, (cols, rows) => {
|
||||
resizes.push({ cols, rows })
|
||||
})
|
||||
|
||||
batcher.queue(120, 40)
|
||||
batcher.clear()
|
||||
// A stale pending viewport left behind by clear() would leak out here.
|
||||
batcher.flush()
|
||||
|
||||
expect(resizes).toEqual([])
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not emit a cleared viewport when the debounce timer would have fired', () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const resizes: { cols: number; rows: number }[] = []
|
||||
const batcher = createRemoteRuntimeViewportBatcher(33, (cols, rows) => {
|
||||
resizes.push({ cols, rows })
|
||||
})
|
||||
|
||||
batcher.queue(90, 30)
|
||||
batcher.clear()
|
||||
vi.advanceTimersByTime(100)
|
||||
|
||||
expect(resizes).toEqual([])
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -173,6 +173,9 @@ export function createRemoteRuntimeViewportBatcher(
|
|||
clearTimeout(timer)
|
||||
timer = null
|
||||
}
|
||||
// Why: also drop the queued viewport so a later flush()/reuse can't emit a
|
||||
// stale resize after the batcher was cleared on teardown/resubscribe.
|
||||
pending = null
|
||||
}
|
||||
|
||||
const flush = (): void => {
|
||||
|
|
|
|||
|
|
@ -1990,9 +1990,9 @@ describe('createRemoteRuntimePtyTransport', () => {
|
|||
emitOutput(streamId, 'live-after-overflow')
|
||||
|
||||
expect(onReplayData).not.toHaveBeenCalled()
|
||||
expect(onError).toHaveBeenCalledWith(
|
||||
'Remote terminal snapshot exceeded the 2 MiB replay limit; live output will continue.'
|
||||
)
|
||||
// Why: an oversized snapshot is skipped but live output continues, so the
|
||||
// transport classifies it as benign and never surfaces a fatal red banner.
|
||||
expect(onError).not.toHaveBeenCalled()
|
||||
expect(onConnect).toHaveBeenCalled()
|
||||
expect(onData).toHaveBeenCalledWith('live-after-overflow', expect.objectContaining({ seq: 1 }))
|
||||
})
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import {
|
|||
} from '../../runtime/runtime-terminal-stream'
|
||||
import {
|
||||
getRemoteRuntimeTerminalMultiplexer,
|
||||
REMOTE_TERMINAL_SNAPSHOT_TOO_LARGE,
|
||||
type RemoteRuntimeMultiplexedTerminal
|
||||
} from '../../runtime/remote-runtime-terminal-multiplexer'
|
||||
import {
|
||||
|
|
@ -373,6 +374,11 @@ export function createRemoteRuntimePtyTransport(
|
|||
|
||||
function handleRemoteTerminalError(error: unknown): void {
|
||||
const message = runtimeTerminalErrorMessage(error)
|
||||
if (message === REMOTE_TERMINAL_SNAPSHOT_TOO_LARGE) {
|
||||
// Why: an oversized initial snapshot is skipped but live output keeps
|
||||
// flowing — informational, not fatal, so never surface a red xterm banner.
|
||||
return
|
||||
}
|
||||
if (isRemoteTerminalGoneMessage(message)) {
|
||||
// Why: paired web clients consume host-published PTY handles. If the host
|
||||
// retires one between snapshots, clear this mirror and wait for the next
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { sendFollowupPromptWhenAgentReady } from './agent-followup-delivery'
|
||||
import {
|
||||
inspectRuntimeTerminalProcess,
|
||||
sendRuntimePtyInputVerified
|
||||
} from '@/runtime/runtime-terminal-inspection'
|
||||
import { TUI_AGENT_CONFIG } from '../../../shared/tui-agent-config'
|
||||
|
||||
vi.mock('@/runtime/runtime-terminal-inspection', () => ({
|
||||
inspectRuntimeTerminalProcess: vi.fn(),
|
||||
sendRuntimePtyInputVerified: vi.fn()
|
||||
}))
|
||||
|
||||
// The interpreter-wrapped agents that deliver their prompt over stdin after the
|
||||
// process starts. These are pip console-scripts, so the PTY foreground comm is
|
||||
// python/python3 — never the agent's own name.
|
||||
const INTERPRETER_WRAPPED_AGENTS = [
|
||||
{ agent: 'aider', expectedProcess: TUI_AGENT_CONFIG.aider.expectedProcess },
|
||||
{ agent: 'mistral-vibe', expectedProcess: TUI_AGENT_CONFIG['mistral-vibe'].expectedProcess }
|
||||
] as const
|
||||
|
||||
describe('sendFollowupPromptWhenAgentReady — interpreter-wrapped agents', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
vi.stubGlobal('globalThis', globalThis)
|
||||
// Deliver the prompt write eagerly so the test does not depend on retries.
|
||||
vi.mocked(sendRuntimePtyInputVerified).mockResolvedValue(true)
|
||||
})
|
||||
|
||||
it('sanity: config keeps aider/vibe as stdin-after-start with python-style expected process', () => {
|
||||
expect(TUI_AGENT_CONFIG.aider.promptInjectionMode).toBe('stdin-after-start')
|
||||
expect(TUI_AGENT_CONFIG['mistral-vibe'].promptInjectionMode).toBe('stdin-after-start')
|
||||
})
|
||||
|
||||
for (const { agent, expectedProcess } of INTERPRETER_WRAPPED_AGENTS) {
|
||||
it(`types the prompt once ${agent} is up behind a python3 wrapper with a live child`, async () => {
|
||||
// The console-script agent is running: foreground comm is python3 and the
|
||||
// PTY has a non-shell child. The exact agent name never appears.
|
||||
vi.mocked(inspectRuntimeTerminalProcess).mockResolvedValue({
|
||||
foregroundProcess: 'python3',
|
||||
hasChildProcesses: true
|
||||
})
|
||||
|
||||
const delivered = await sendFollowupPromptWhenAgentReady({
|
||||
ptyId: 'pty-1',
|
||||
expectedProcess,
|
||||
prompt: 'ship it',
|
||||
settings: null
|
||||
})
|
||||
|
||||
expect(delivered).toBe(true)
|
||||
expect(sendRuntimePtyInputVerified).toHaveBeenCalledWith(null, 'pty-1', 'ship it\r')
|
||||
})
|
||||
|
||||
it(`still refuses to type into a bare ${agent} shell foreground`, async () => {
|
||||
// No agent yet: foreground is a plain shell with no non-shell child. The
|
||||
// guard must NOT write user text into an arbitrary shell.
|
||||
vi.mocked(inspectRuntimeTerminalProcess).mockResolvedValue({
|
||||
foregroundProcess: 'zsh',
|
||||
hasChildProcesses: false
|
||||
})
|
||||
|
||||
const delivered = await sendFollowupPromptWhenAgentReady({
|
||||
ptyId: 'pty-1',
|
||||
expectedProcess,
|
||||
prompt: 'ship it',
|
||||
settings: null
|
||||
})
|
||||
|
||||
expect(delivered).toBe(false)
|
||||
expect(sendRuntimePtyInputVerified).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it(`refuses to type into a ${agent} wrapper without a live child`, async () => {
|
||||
vi.mocked(inspectRuntimeTerminalProcess).mockResolvedValue({
|
||||
foregroundProcess: 'python3',
|
||||
hasChildProcesses: false
|
||||
})
|
||||
|
||||
const delivered = await sendFollowupPromptWhenAgentReady({
|
||||
ptyId: 'pty-1',
|
||||
expectedProcess,
|
||||
prompt: 'ship it',
|
||||
settings: null
|
||||
})
|
||||
|
||||
expect(delivered).toBe(false)
|
||||
expect(sendRuntimePtyInputVerified).not.toHaveBeenCalled()
|
||||
})
|
||||
}
|
||||
|
||||
it('types immediately when the resolver already returns the agent name (local ps path)', async () => {
|
||||
// On local desktop the ps-table resolver usually resolves python3 → aider
|
||||
// before we poll; the exact-match path must keep working.
|
||||
vi.mocked(inspectRuntimeTerminalProcess).mockResolvedValue({
|
||||
foregroundProcess: 'aider',
|
||||
hasChildProcesses: true
|
||||
})
|
||||
|
||||
const delivered = await sendFollowupPromptWhenAgentReady({
|
||||
ptyId: 'pty-1',
|
||||
expectedProcess: 'aider',
|
||||
prompt: 'ship it',
|
||||
settings: null
|
||||
})
|
||||
|
||||
expect(delivered).toBe(true)
|
||||
expect(sendRuntimePtyInputVerified).toHaveBeenCalledWith(null, 'pty-1', 'ship it\r')
|
||||
})
|
||||
})
|
||||
|
|
@ -2,7 +2,11 @@ import {
|
|||
inspectRuntimeTerminalProcess,
|
||||
sendRuntimePtyInputVerified
|
||||
} from '@/runtime/runtime-terminal-inspection'
|
||||
import { isExpectedAgentProcess } from '../../../shared/agent-process-recognition'
|
||||
import {
|
||||
isAgentForegroundWrapperProcess,
|
||||
isExpectedAgentProcess
|
||||
} from '../../../shared/agent-process-recognition'
|
||||
import { isShellProcess } from '../../../shared/shell-process-detection'
|
||||
import type { GlobalSettings } from '../../../shared/types'
|
||||
|
||||
type RuntimeOwnerSettings = Pick<GlobalSettings, 'activeRuntimeEnvironmentId'> | null | undefined
|
||||
|
|
@ -25,7 +29,7 @@ export async function sendFollowupPromptWhenAgentReady(args: {
|
|||
}
|
||||
|
||||
// Why: delayed follow-ups must not type into an arbitrary shell. Require a
|
||||
// positive expected-process match before writing user/task text to the PTY.
|
||||
// positive readiness signal before writing user/task text to the PTY.
|
||||
async function waitForAgentForeground(
|
||||
ptyId: string,
|
||||
expectedProcess: string,
|
||||
|
|
@ -41,6 +45,20 @@ async function waitForAgentForeground(
|
|||
if (isExpectedAgentProcess(foreground, expectedProcess)) {
|
||||
return true
|
||||
}
|
||||
// Why: interpreter-wrapped agents (aider, mistral-vibe are pip console
|
||||
// scripts) surface a python/node foreground comm, so the exact-name check
|
||||
// never matches — locally when the ps-table resolver can't pin the child,
|
||||
// and over SSH when the relay falls back to the bare interpreter name. If
|
||||
// the foreground is a known agent wrapper (not a shell) with a live
|
||||
// non-shell child, the agent has taken over the PTY and can accept input.
|
||||
if (
|
||||
attempt >= 4 &&
|
||||
isAgentForegroundWrapperProcess(foreground) &&
|
||||
!isShellProcess(foreground) &&
|
||||
process.hasChildProcesses
|
||||
) {
|
||||
return true
|
||||
}
|
||||
} catch {
|
||||
// Ignore transient PTY inspection failures and keep polling.
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ const {
|
|||
mockInspectRuntimeTerminalProcess,
|
||||
mockSendRuntimePtyInputVerified,
|
||||
mockPasteDraftToAgentPtyWhenReady,
|
||||
mockShowAutomationPromptNotSentToast,
|
||||
mockTrack,
|
||||
store,
|
||||
storeListeners,
|
||||
|
|
@ -12,6 +13,7 @@ const {
|
|||
mockInspectRuntimeTerminalProcess: vi.fn(),
|
||||
mockSendRuntimePtyInputVerified: vi.fn(),
|
||||
mockPasteDraftToAgentPtyWhenReady: vi.fn(),
|
||||
mockShowAutomationPromptNotSentToast: vi.fn(),
|
||||
mockTrack: vi.fn(),
|
||||
storeListeners: new Set<(state: unknown, previousState: unknown) => void>(),
|
||||
startupLeafId: '11111111-1111-4111-8111-111111111111',
|
||||
|
|
@ -88,6 +90,10 @@ vi.mock('@/lib/telemetry', () => ({
|
|||
track: mockTrack
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/agent-background-session-timeout-toast', () => ({
|
||||
showAutomationPromptNotSentToast: mockShowAutomationPromptNotSentToast
|
||||
}))
|
||||
|
||||
import {
|
||||
ensureAgentStartupInTerminal,
|
||||
getSetupConfig,
|
||||
|
|
@ -298,6 +304,65 @@ describe('ensureAgentStartupInTerminal prompt delivery', () => {
|
|||
expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything())
|
||||
})
|
||||
|
||||
it('surfaces the not-sent toast when a follow-up prompt is dropped', async () => {
|
||||
// Foreground never becomes a recognized agent and there is no live child,
|
||||
// so the readiness wait times out and the prompt is not delivered.
|
||||
mockInspectRuntimeTerminalProcess.mockResolvedValue({
|
||||
foregroundProcess: 'zsh',
|
||||
hasChildProcesses: false
|
||||
})
|
||||
|
||||
await ensureAgentStartupInTerminal({
|
||||
worktreeId: 'wt-1',
|
||||
startup: {
|
||||
agent: 'aider',
|
||||
launchCommand: 'aider',
|
||||
expectedProcess: 'aider',
|
||||
followupPrompt: 'fix the spinner',
|
||||
launchConfig: { agentArgs: '', agentEnv: {} }
|
||||
}
|
||||
})
|
||||
|
||||
expect(mockSendRuntimePtyInputVerified).not.toHaveBeenCalled()
|
||||
expect(mockShowAutomationPromptNotSentToast).toHaveBeenCalledWith('aider')
|
||||
})
|
||||
|
||||
it('does not toast when a follow-up prompt is delivered', async () => {
|
||||
await ensureAgentStartupInTerminal({
|
||||
worktreeId: 'wt-1',
|
||||
startup: {
|
||||
agent: 'aider',
|
||||
launchCommand: 'aider',
|
||||
expectedProcess: 'aider',
|
||||
followupPrompt: 'fix the spinner',
|
||||
launchConfig: { agentArgs: '', agentEnv: {} }
|
||||
}
|
||||
})
|
||||
|
||||
expect(mockShowAutomationPromptNotSentToast).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('passes an onTimeout that surfaces the not-sent toast to the draft paste path', async () => {
|
||||
await ensureAgentStartupInTerminal({
|
||||
worktreeId: 'wt-1',
|
||||
startup: {
|
||||
agent: 'claude',
|
||||
launchCommand: 'claude',
|
||||
expectedProcess: 'claude',
|
||||
followupPrompt: null,
|
||||
launchConfig: { agentArgs: '', agentEnv: {} },
|
||||
draftPrompt: 'review this before sending'
|
||||
}
|
||||
})
|
||||
|
||||
const call = mockPasteDraftToAgentPtyWhenReady.mock.calls.at(-1)?.[0] as
|
||||
| { onTimeout?: () => void }
|
||||
| undefined
|
||||
expect(call?.onTimeout).toBeTypeOf('function')
|
||||
call?.onTimeout?.()
|
||||
expect(mockShowAutomationPromptNotSentToast).toHaveBeenCalledWith('claude')
|
||||
})
|
||||
|
||||
it('does not track when follow-up prompt delivery rejects', async () => {
|
||||
mockSendRuntimePtyInputVerified.mockRejectedValue(new Error('runtime timeout'))
|
||||
|
||||
|
|
@ -335,7 +400,8 @@ describe('ensureAgentStartupInTerminal prompt delivery', () => {
|
|||
ptyId: 'pty-1',
|
||||
content: 'review this before sending',
|
||||
agent: 'claude',
|
||||
forcePaste: true
|
||||
forcePaste: true,
|
||||
onTimeout: expect.any(Function)
|
||||
})
|
||||
expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything())
|
||||
})
|
||||
|
|
@ -378,7 +444,8 @@ describe('ensureAgentStartupInTerminal prompt delivery', () => {
|
|||
ptyId: 'agent-pty',
|
||||
content: 'Linear context draft',
|
||||
agent: 'codex',
|
||||
forcePaste: true
|
||||
forcePaste: true,
|
||||
onTimeout: expect.any(Function)
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -434,7 +501,8 @@ describe('ensureAgentStartupInTerminal prompt delivery', () => {
|
|||
ptyId: 'pty-delayed',
|
||||
content: 'https://github.com/stablyai/orca/pull/2051',
|
||||
agent: 'codex',
|
||||
forcePaste: true
|
||||
forcePaste: true,
|
||||
onTimeout: expect.any(Function)
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -515,7 +583,8 @@ describe('ensureAgentStartupInTerminal prompt delivery', () => {
|
|||
ptyId: 'startup-pty',
|
||||
content: 'linked draft',
|
||||
agent: 'codex',
|
||||
forcePaste: true
|
||||
forcePaste: true,
|
||||
onTimeout: expect.any(Function)
|
||||
})
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
pasteDraftToAgentPtyWhenReady
|
||||
} from '@/lib/agent-paste-draft'
|
||||
import { sendFollowupPromptWhenAgentReady } from '@/lib/agent-followup-delivery'
|
||||
import { showAutomationPromptNotSentToast } from '@/lib/agent-background-session-timeout-toast'
|
||||
import type { AgentStartupPlan } from '@/lib/tui-agent-startup'
|
||||
import type { LinkedWorkItemContext } from '@/lib/linked-work-item-context'
|
||||
import {
|
||||
|
|
@ -302,12 +303,17 @@ async function deliverAgentStartupToTerminal(
|
|||
// (aider, goose, etc.) that need their initial prompt typed into the live
|
||||
// session and submitted. Wait until the agent owns the PTY before writing.
|
||||
if (startup.followupPrompt) {
|
||||
await sendFollowupPromptWhenAgentReady({
|
||||
const delivered = await sendFollowupPromptWhenAgentReady({
|
||||
ptyId,
|
||||
expectedProcess: startup.expectedProcess,
|
||||
prompt: startup.followupPrompt,
|
||||
settings: runtimeSettings
|
||||
})
|
||||
// Why: a dropped follow-up is otherwise silent — surface the same toast the
|
||||
// draft path uses so the user knows to open the workspace and paste it.
|
||||
if (!delivered) {
|
||||
showAutomationPromptNotSentToast(startup.agent)
|
||||
}
|
||||
}
|
||||
|
||||
// Why: draftPrompt uses bracketed-paste so the URL lands atomically in the
|
||||
|
|
@ -321,7 +327,9 @@ async function deliverAgentStartupToTerminal(
|
|||
agent: startup.agent,
|
||||
// Why: startup.draftPrompt is only attached after native draft launch
|
||||
// planning is unavailable, so this paste is the first delivery attempt.
|
||||
forcePaste: true
|
||||
forcePaste: true,
|
||||
// Why: surface a dropped draft instead of silently losing it.
|
||||
onTimeout: () => showAutomationPromptNotSentToast(startup.agent)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,236 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
TerminalStreamOpcode,
|
||||
decodeTerminalStreamFrame,
|
||||
decodeTerminalStreamJson,
|
||||
encodeTerminalStreamFrame,
|
||||
encodeTerminalStreamJson,
|
||||
encodeTerminalStreamText
|
||||
} from '../../../shared/terminal-stream-protocol'
|
||||
import {
|
||||
getRemoteRuntimeTerminalMultiplexer,
|
||||
resetRemoteRuntimeTerminalMultiplexersForTests,
|
||||
type RemoteRuntimeMultiplexedTerminal
|
||||
} from './remote-runtime-terminal-multiplexer'
|
||||
|
||||
// Why: reproduces the silent frame-drop corruption. The server multiplex path
|
||||
// drops Output frames when the websocket buffer is over its cap
|
||||
// (encryptedBinaryReply returns false); the wire `seq` is an output high-water, so
|
||||
// a drop leaves a detectable gap. This harness drives the real client
|
||||
// multiplexer through the same subscribe transport the app uses and forces a
|
||||
// drop, asserting the client resyncs instead of rendering a corrupt tail.
|
||||
|
||||
type SubscribeCallbacks = {
|
||||
onResponse: (response: unknown) => void
|
||||
onBinary?: (bytes: Uint8Array<ArrayBufferLike>) => void
|
||||
onError?: (error: { message: string }) => void
|
||||
onClose?: () => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal server that mimics src/main/runtime/rpc/methods/terminal.ts's multiplex
|
||||
* output path: Output frames carry a monotonic UTF-16 high-water `seq`, and a
|
||||
* SnapshotRequest is answered with an initial-style snapshot (no requestId).
|
||||
*/
|
||||
class FakeMultiplexServer {
|
||||
private cursorUnits = 0
|
||||
private streamId = 0
|
||||
dropNextOutput = false
|
||||
droppedFrames = 0
|
||||
holdNextManualSnapshot = false
|
||||
snapshotRequests: (number | undefined)[] = []
|
||||
private heldManualRequestId: number | null = null
|
||||
private snapshotData = 'INITIAL'
|
||||
|
||||
constructor(
|
||||
private readonly toClient: (bytes: Uint8Array<ArrayBufferLike>) => void,
|
||||
private readonly onServerSideDrop?: () => void
|
||||
) {}
|
||||
|
||||
/** Client -> server frames arrive here (Subscribe / SnapshotRequest / Input). */
|
||||
receive(bytes: Uint8Array<ArrayBufferLike>): void {
|
||||
const frame = decodeTerminalStreamFrame(bytes)
|
||||
if (!frame) {
|
||||
return
|
||||
}
|
||||
if (frame.opcode === TerminalStreamOpcode.Subscribe) {
|
||||
const payload = decodeTerminalStreamJson<{ streamId: number }>(frame.payload)
|
||||
this.streamId = payload?.streamId ?? 0
|
||||
this.sendSnapshot()
|
||||
return
|
||||
}
|
||||
if (frame.opcode === TerminalStreamOpcode.SnapshotRequest) {
|
||||
const payload = decodeTerminalStreamJson<{ requestId?: number }>(frame.payload)
|
||||
this.snapshotRequests.push(payload?.requestId)
|
||||
if (typeof payload?.requestId === 'number' && this.holdNextManualSnapshot) {
|
||||
this.holdNextManualSnapshot = false
|
||||
this.heldManualRequestId = payload.requestId
|
||||
return
|
||||
}
|
||||
// Resync request: the server serializes the *current* buffer, so recovery
|
||||
// includes everything the client missed.
|
||||
this.snapshotData = 'RECOVERED'
|
||||
this.sendSnapshot(payload?.requestId)
|
||||
}
|
||||
}
|
||||
|
||||
private send(opcode: TerminalStreamOpcode, payload: Uint8Array, seq: number): void {
|
||||
this.toClient(encodeTerminalStreamFrame({ opcode, streamId: this.streamId, seq, payload }))
|
||||
}
|
||||
|
||||
private sendSnapshot(requestId?: number): void {
|
||||
this.send(
|
||||
TerminalStreamOpcode.SnapshotStart,
|
||||
encodeTerminalStreamJson({ cols: 80, rows: 24, seq: this.cursorUnits, requestId }),
|
||||
0
|
||||
)
|
||||
this.send(TerminalStreamOpcode.SnapshotChunk, encodeTerminalStreamText(this.snapshotData), 0)
|
||||
this.send(TerminalStreamOpcode.SnapshotEnd, new Uint8Array(), 0)
|
||||
}
|
||||
|
||||
/** Emit an Output chunk, honoring simulated websocket backpressure. */
|
||||
output(text: string): void {
|
||||
const startSeq = this.cursorUnits
|
||||
this.cursorUnits += text.length
|
||||
if (this.dropNextOutput) {
|
||||
// encryptedBinaryReply returned false: frame is NOT sent. The byte
|
||||
// high-water still advances (server keeps producing), so the next frame's
|
||||
// seq jumps past what the client last saw.
|
||||
this.dropNextOutput = false
|
||||
this.droppedFrames += 1
|
||||
this.onServerSideDrop?.()
|
||||
return
|
||||
}
|
||||
void startSeq
|
||||
this.send(TerminalStreamOpcode.Output, encodeTerminalStreamText(text), this.cursorUnits)
|
||||
}
|
||||
|
||||
flushHeldManualSnapshot(): void {
|
||||
if (this.heldManualRequestId === null) {
|
||||
throw new Error('No manual snapshot is held')
|
||||
}
|
||||
const requestId = this.heldManualRequestId
|
||||
this.heldManualRequestId = null
|
||||
this.snapshotData = 'MANUAL'
|
||||
this.sendSnapshot(requestId)
|
||||
}
|
||||
}
|
||||
|
||||
describe('remote terminal frame-drop resync', () => {
|
||||
const unsubscribe = vi.fn()
|
||||
let server: FakeMultiplexServer
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetRemoteRuntimeTerminalMultiplexersForTests()
|
||||
|
||||
const subscribe = vi.fn(async (_args: unknown, callbacks: SubscribeCallbacks) => {
|
||||
server = new FakeMultiplexServer((bytes) => callbacks.onBinary?.(bytes))
|
||||
queueMicrotask(() => callbacks.onResponse({ ok: true, result: { type: 'ready' } }))
|
||||
return {
|
||||
unsubscribe,
|
||||
sendBinary: (bytes: Uint8Array<ArrayBufferLike>) => server.receive(bytes)
|
||||
}
|
||||
})
|
||||
|
||||
vi.stubGlobal('window', {
|
||||
api: { runtimeEnvironments: { subscribe } }
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
async function subscribeClient(): Promise<{
|
||||
data: string[]
|
||||
snapshots: string[]
|
||||
stream: RemoteRuntimeMultiplexedTerminal
|
||||
}> {
|
||||
const data: string[] = []
|
||||
const snapshots: string[] = []
|
||||
const multiplexer = getRemoteRuntimeTerminalMultiplexer('env-1')
|
||||
const stream = await multiplexer.subscribeTerminal({
|
||||
terminal: 'terminal-1',
|
||||
client: { id: 'desktop-1', type: 'desktop' },
|
||||
callbacks: {
|
||||
onData: (chunk) => data.push(chunk),
|
||||
onSnapshot: (chunk) => snapshots.push(chunk)
|
||||
}
|
||||
})
|
||||
// Let the initial snapshot round-trip settle.
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
return { data, snapshots, stream }
|
||||
}
|
||||
|
||||
it('detects a dropped Output frame via the seq gap and resyncs', async () => {
|
||||
const { data, snapshots } = await subscribeClient()
|
||||
expect(snapshots).toEqual(['INITIAL'])
|
||||
|
||||
server.output('aaa')
|
||||
server.dropNextOutput = true
|
||||
server.output('bbb') // dropped under backpressure — never reaches the client
|
||||
server.output('ccc') // seq jumps past 'bbb', exposing the gap
|
||||
|
||||
// Flush the client's resync SnapshotRequest -> server snapshot round-trip.
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
|
||||
// The corrupt tail ('ccc', which followed a gap) is NOT rendered as live data.
|
||||
expect(data).toEqual(['aaa'])
|
||||
expect(server.droppedFrames).toBe(1)
|
||||
// Instead, a fresh authoritative snapshot recovers the terminal.
|
||||
expect(snapshots).toEqual(['INITIAL', 'RECOVERED'])
|
||||
})
|
||||
|
||||
it('passes contiguous output straight through without resyncing', async () => {
|
||||
const { data, snapshots } = await subscribeClient()
|
||||
|
||||
server.output('one')
|
||||
server.output('two')
|
||||
server.output('three')
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
|
||||
expect(data).toEqual(['one', 'two', 'three'])
|
||||
expect(snapshots).toEqual(['INITIAL'])
|
||||
})
|
||||
|
||||
it('uses UTF-16 sequence units when detecting gaps in multibyte output', async () => {
|
||||
const { data, snapshots } = await subscribeClient()
|
||||
|
||||
server.output('é')
|
||||
server.dropNextOutput = true
|
||||
server.output('🙂')
|
||||
server.output('界')
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
|
||||
expect(data).toEqual(['é'])
|
||||
expect(snapshots).toEqual(['INITIAL', 'RECOVERED'])
|
||||
})
|
||||
|
||||
it('defers recovery until an in-flight manual snapshot finishes', async () => {
|
||||
const { data, snapshots, stream } = await subscribeClient()
|
||||
server.holdNextManualSnapshot = true
|
||||
const manualSnapshot = stream.serializeBuffer({ scrollbackRows: 100 })
|
||||
await Promise.resolve()
|
||||
|
||||
server.output('aaa')
|
||||
server.dropNextOutput = true
|
||||
server.output('🙂')
|
||||
server.output('ccc')
|
||||
|
||||
expect(data).toEqual(['aaa'])
|
||||
expect(server.snapshotRequests).toHaveLength(1)
|
||||
|
||||
server.flushHeldManualSnapshot()
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
|
||||
await expect(manualSnapshot).resolves.toMatchObject({ data: 'MANUAL' })
|
||||
expect(server.snapshotRequests).toHaveLength(2)
|
||||
expect(snapshots).toEqual(['INITIAL', 'RECOVERED'])
|
||||
})
|
||||
})
|
||||
|
|
@ -77,6 +77,13 @@ type RemoteRuntimeMultiplexedTerminalState = {
|
|||
snapshotInfo: RemoteRuntimeSnapshotInfo | null
|
||||
initialSnapshotReceived: boolean
|
||||
pendingSnapshotRequest: RemoteRuntimeSnapshotRequest | null
|
||||
// Why: Output frames carry a UTF-16 offset high-water `seq`; a jump past the
|
||||
// expected next offset means the server dropped frames under backpressure.
|
||||
// Track it so a gap triggers a self-healing snapshot resync instead of
|
||||
// silently rendering corrupt/missing output (frame-drop resync).
|
||||
expectedSeq: number | undefined
|
||||
resyncInFlight: boolean
|
||||
resyncPendingSend: boolean
|
||||
}
|
||||
|
||||
type RemoteRuntimeSnapshotInfo = {
|
||||
|
|
@ -111,7 +118,9 @@ type RemoteRuntimeSnapshotRequest = {
|
|||
const CONTROL_STREAM_ID = 0
|
||||
const MAX_REMOTE_TERMINAL_SNAPSHOT_BYTES = 2 * 1024 * 1024
|
||||
const REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS = 10_000
|
||||
const REMOTE_TERMINAL_SNAPSHOT_TOO_LARGE =
|
||||
// Why: exported so the transport can classify it as benign — the snapshot was
|
||||
// skipped but live output continues, so it must not surface a fatal red banner.
|
||||
export const REMOTE_TERMINAL_SNAPSHOT_TOO_LARGE =
|
||||
'Remote terminal snapshot exceeded the 2 MiB replay limit; live output will continue.'
|
||||
|
||||
class RemoteRuntimeTerminalMultiplexer {
|
||||
|
|
@ -149,7 +158,10 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
snapshotTarget: 'initial',
|
||||
snapshotInfo: null,
|
||||
initialSnapshotReceived: false,
|
||||
pendingSnapshotRequest: null
|
||||
pendingSnapshotRequest: null,
|
||||
expectedSeq: undefined,
|
||||
resyncInFlight: false,
|
||||
resyncPendingSend: false
|
||||
}
|
||||
this.streams.set(streamId, state)
|
||||
|
||||
|
|
@ -334,10 +346,21 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
}
|
||||
if (frame.opcode === TerminalStreamOpcode.Output) {
|
||||
const data = decodeTerminalStreamText(frame.payload)
|
||||
stream.callbacks.onData(data, {
|
||||
seq: typeof frame.seq === 'number' && frame.seq > 0 ? frame.seq : undefined,
|
||||
rawLength: data.length
|
||||
})
|
||||
const rawLength = data.length
|
||||
// Why: a resync snapshot is authoritative; drop live output that arrives
|
||||
// while it is in flight so the corrupt post-gap tail is never rendered.
|
||||
if (stream.resyncInFlight) {
|
||||
return
|
||||
}
|
||||
const seq = typeof frame.seq === 'number' && frame.seq > 0 ? frame.seq : undefined
|
||||
if (this.detectOutputGap(stream, seq, rawLength)) {
|
||||
this.requestResyncSnapshot(stream)
|
||||
return
|
||||
}
|
||||
if (typeof seq === 'number') {
|
||||
stream.expectedSeq = seq
|
||||
}
|
||||
stream.callbacks.onData(data, { seq, rawLength })
|
||||
return
|
||||
}
|
||||
if (frame.opcode === TerminalStreamOpcode.SnapshotStart) {
|
||||
|
|
@ -400,9 +423,17 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
clearPendingSnapshotRequest(stream)
|
||||
}
|
||||
clearSnapshot(stream)
|
||||
// Why: the snapshot is the new authoritative output high-water; align the
|
||||
// gap detector to it and re-open the live path (used by both the initial
|
||||
// snapshot and a frame-drop resync, which reuses the 'initial' target).
|
||||
if (target === 'initial') {
|
||||
stream.expectedSeq = typeof info?.seq === 'number' ? info.seq : undefined
|
||||
stream.resyncInFlight = false
|
||||
stream.resyncPendingSend = false
|
||||
stream.initialSnapshotReceived = true
|
||||
stream.callbacks.onSubscribed?.()
|
||||
} else {
|
||||
this.sendDeferredResyncSnapshot(stream)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -412,12 +443,71 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
if (pendingSnapshotRequest) {
|
||||
clearPendingSnapshotRequest(stream)
|
||||
pendingSnapshotRequest.reject(new Error(decodeTerminalStreamText(frame.payload)))
|
||||
this.sendDeferredResyncSnapshot(stream)
|
||||
return
|
||||
}
|
||||
// Why: a failed resync must re-open the live path or output stalls forever.
|
||||
stream.resyncInFlight = false
|
||||
stream.resyncPendingSend = false
|
||||
stream.callbacks.onError?.(decodeTerminalStreamText(frame.payload))
|
||||
}
|
||||
}
|
||||
|
||||
// Why: Output `seq` is the UTF-16 high-water at the end of a chunk, so a chunk
|
||||
// that begins after the last high-water (startSeq > expectedSeq) means the
|
||||
// server dropped intervening frames under backpressure. Only flag a gap when
|
||||
// both offsets are known, and never on the first seq (nothing to compare to).
|
||||
private detectOutputGap(
|
||||
stream: RemoteRuntimeMultiplexedTerminalState,
|
||||
seq: number | undefined,
|
||||
rawLength: number
|
||||
): boolean {
|
||||
if (typeof seq !== 'number' || typeof stream.expectedSeq !== 'number') {
|
||||
return false
|
||||
}
|
||||
const startSeq = seq - rawLength
|
||||
return startSeq > stream.expectedSeq
|
||||
}
|
||||
|
||||
// Why: on a detected gap, discard the corrupt tail and pull a fresh
|
||||
// authoritative snapshot. The request carries no requestId so the server
|
||||
// reply renders through the initial-snapshot path (full reset), self-healing
|
||||
// without surfacing an error to the user.
|
||||
private requestResyncSnapshot(stream: RemoteRuntimeMultiplexedTerminalState): void {
|
||||
if (stream.resyncInFlight) {
|
||||
return
|
||||
}
|
||||
stream.resyncInFlight = true
|
||||
stream.expectedSeq = undefined
|
||||
if (stream.pendingSnapshotRequest) {
|
||||
// Why: snapshot frame groups are not multiplexed; wait for the manual
|
||||
// snapshot to finish so its response cannot be mistaken for recovery.
|
||||
stream.resyncPendingSend = true
|
||||
return
|
||||
}
|
||||
this.sendResyncSnapshot(stream)
|
||||
}
|
||||
|
||||
private sendDeferredResyncSnapshot(stream: RemoteRuntimeMultiplexedTerminalState): void {
|
||||
if (!stream.resyncInFlight || !stream.resyncPendingSend || stream.pendingSnapshotRequest) {
|
||||
return
|
||||
}
|
||||
this.sendResyncSnapshot(stream)
|
||||
}
|
||||
|
||||
private sendResyncSnapshot(stream: RemoteRuntimeMultiplexedTerminalState): void {
|
||||
stream.resyncPendingSend = false
|
||||
const sent = this.sendFrame(
|
||||
stream.streamId,
|
||||
TerminalStreamOpcode.SnapshotRequest,
|
||||
encodeTerminalStreamJson({ scrollbackRows: undefined })
|
||||
)
|
||||
if (!sent) {
|
||||
// Transport is down; the reconnect path re-subscribes from scratch.
|
||||
stream.resyncInFlight = false
|
||||
}
|
||||
}
|
||||
|
||||
private requestSnapshot(
|
||||
stream: RemoteRuntimeMultiplexedTerminalState,
|
||||
opts?: { scrollbackRows?: number }
|
||||
|
|
@ -431,6 +521,11 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
if (this.streams.get(stream.streamId) !== stream || !this.ready || !this.subscription) {
|
||||
return Promise.resolve(null)
|
||||
}
|
||||
// Recovery uses an untagged snapshot frame group; callers can retry after
|
||||
// it completes instead of racing another request onto the same frame lane.
|
||||
if (stream.resyncInFlight) {
|
||||
return Promise.resolve(null)
|
||||
}
|
||||
if (stream.pendingSnapshotRequest) {
|
||||
return Promise.reject(new Error('Remote terminal snapshot already in flight.'))
|
||||
}
|
||||
|
|
@ -438,8 +533,9 @@ class RemoteRuntimeTerminalMultiplexer {
|
|||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
if (stream.pendingSnapshotRequest?.timer === timer) {
|
||||
stream.pendingSnapshotRequest = null
|
||||
clearPendingSnapshotRequest(stream)
|
||||
reject(new Error('Remote terminal snapshot timed out.'))
|
||||
this.sendDeferredResyncSnapshot(stream)
|
||||
}
|
||||
}, REMOTE_TERMINAL_SNAPSHOT_REQUEST_TIMEOUT_MS)
|
||||
if (typeof timer.unref === 'function') {
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import {
|
|||
type RemoteRuntimeSocketLivenessMonitor,
|
||||
type RemoteRuntimeSocketLivenessOptions
|
||||
} from './remote-runtime-socket-liveness'
|
||||
import { createWsOutboundBackpressureQueue } from './ws-outbound-backpressure-queue'
|
||||
|
||||
export { RemoteRuntimeClientError } from './remote-runtime-client-error'
|
||||
|
||||
|
|
@ -375,6 +376,8 @@ export async function subscribeRemoteRuntimeRequest<TResult>(
|
|||
const cleanupSocketListeners = (): WebSocket | null => {
|
||||
liveness?.stop()
|
||||
liveness = null
|
||||
sendQueue?.dispose()
|
||||
sendQueue = null
|
||||
const socket = ws
|
||||
if (!socket) {
|
||||
return null
|
||||
|
|
@ -420,11 +423,37 @@ export async function subscribeRemoteRuntimeRequest<TResult>(
|
|||
}
|
||||
}
|
||||
|
||||
// Why: client input (keystrokes) must never be dropped under backpressure.
|
||||
// Hold encrypted frames in order while bufferedAmount is over the cap and
|
||||
// drain as it clears; a wedged link (hard cap) fails the socket so the
|
||||
// renderer resubscribes and replays a fresh snapshot.
|
||||
let sendQueue: ReturnType<typeof createWsOutboundBackpressureQueue<Buffer>> | null = null
|
||||
const ensureSendQueue = (
|
||||
socket: WebSocket
|
||||
): ReturnType<typeof createWsOutboundBackpressureQueue<Buffer>> => {
|
||||
if (!sendQueue) {
|
||||
sendQueue = createWsOutboundBackpressureQueue<Buffer>({
|
||||
send: (frame) => socket.send(frame, { binary: true }),
|
||||
byteLengthOf: (frame) => frame.byteLength,
|
||||
getBufferedAmount: () => socket.bufferedAmount,
|
||||
isWritable: () => socket.readyState === WebSocket.OPEN,
|
||||
onOverflow: () =>
|
||||
fail(
|
||||
new RemoteRuntimeClientError(
|
||||
'remote_runtime_unavailable',
|
||||
'Remote Orca runtime send buffer overflow; reconnecting.'
|
||||
)
|
||||
)
|
||||
})
|
||||
}
|
||||
return sendQueue
|
||||
}
|
||||
|
||||
const sendBinary = (bytes: Uint8Array<ArrayBufferLike>): boolean => {
|
||||
if (state !== 'ready' || !ws || ws.readyState !== WebSocket.OPEN) {
|
||||
return false
|
||||
}
|
||||
ws.send(Buffer.from(encryptBytes(bytes, sharedKey)), { binary: true })
|
||||
ensureSendQueue(ws).enqueue(Buffer.from(encryptBytes(bytes, sharedKey)))
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -346,18 +346,18 @@ describe('RemoteRuntimeSharedControlConnection', () => {
|
|||
connection.close()
|
||||
})
|
||||
|
||||
it('refreshes pending request timeouts when keepalive frames show server progress', async () => {
|
||||
it('times out a stuck short RPC on its absolute deadline despite keepalive frames', async () => {
|
||||
// Why: a keepalive on the shared socket is armed by an unrelated long-poll,
|
||||
// not by this request. It must NOT extend a stuck short RPC's deadline —
|
||||
// otherwise a hung server call hangs the caller forever (#7948).
|
||||
const server = await createServer({
|
||||
silentMethods: ['worktree.hang'],
|
||||
sendKeepaliveBeforeResponse: true,
|
||||
keepaliveDelayMs: 25,
|
||||
responseDelayMs: 60
|
||||
keepaliveDelayMs: 20
|
||||
})
|
||||
const connection = new RemoteRuntimeSharedControlConnection(server.pairing)
|
||||
|
||||
await expect(connection.request('worktree.ps', undefined, 50)).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { method: 'worktree.ps' }
|
||||
})
|
||||
await expect(connection.request('worktree.hang', undefined, 60)).rejects.toThrow('Timed out')
|
||||
|
||||
connection.close()
|
||||
})
|
||||
|
|
@ -618,6 +618,15 @@ function handleRequest(
|
|||
delayedResponses: (() => void)[]
|
||||
): void {
|
||||
requests.push(request)
|
||||
// Why: keepalives are armed by an unrelated long-poll and keep flowing even
|
||||
// while a method is deliberately silent — emit them before the silent return.
|
||||
if (options.sendKeepaliveBeforeResponse && options.keepaliveDelayMs !== undefined) {
|
||||
const timer = setInterval(
|
||||
() => sendEncrypted(ws, sharedKey, { _keepalive: true }),
|
||||
options.keepaliveDelayMs
|
||||
)
|
||||
ws.once('close', () => clearInterval(timer))
|
||||
}
|
||||
if (options.silentMethods?.includes(request.method)) {
|
||||
return
|
||||
}
|
||||
|
|
@ -647,13 +656,10 @@ function handleRequest(
|
|||
})
|
||||
}
|
||||
const closeAfterResponse = streaming && options.closeAfterStreamingResponse?.() === true
|
||||
if (options.sendKeepaliveBeforeResponse) {
|
||||
const sendKeepalive = (): void => sendEncrypted(ws, sharedKey, { _keepalive: true })
|
||||
if (options.keepaliveDelayMs !== undefined) {
|
||||
setTimeout(sendKeepalive, options.keepaliveDelayMs)
|
||||
} else {
|
||||
sendKeepalive()
|
||||
}
|
||||
// Delayed/periodic keepalives are handled by the interval above; here we only
|
||||
// cover the immediate single-keepalive-before-response case.
|
||||
if (options.sendKeepaliveBeforeResponse && options.keepaliveDelayMs === undefined) {
|
||||
sendEncrypted(ws, sharedKey, { _keepalive: true })
|
||||
}
|
||||
if (options.delaySubscriptionReady && streaming) {
|
||||
delayedResponses.push(sendResponse)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { requestSharedControl } from './remote-runtime-shared-control-requests'
|
||||
import {
|
||||
refreshSharedControlPendingRequestTimeouts,
|
||||
resolveSharedControlPendingResponse
|
||||
} from './remote-runtime-shared-control-state'
|
||||
import type { SharedControlPendingRequest } from './remote-runtime-shared-control-types'
|
||||
|
||||
// Why: a keepalive frame on the shared-control socket is armed by an unrelated
|
||||
// long-poll, not by any given pending short RPC. These fake-timer tests pin the
|
||||
// deadline semantics: keepalives must NOT keep a stuck short RPC alive forever,
|
||||
// but MAY extend a long-poll that opted into the short-RPC path.
|
||||
describe('shared control keepalive timeout refresh semantics', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
function startRequest(options: { refreshTimeoutOnKeepalive?: boolean } = {}): {
|
||||
pendingRequests: Map<string, SharedControlPendingRequest<unknown>>
|
||||
promise: Promise<unknown>
|
||||
onTimeout: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
const pendingRequests = new Map<string, SharedControlPendingRequest<unknown>>()
|
||||
const onTimeout = vi.fn()
|
||||
const promise = requestSharedControl({
|
||||
pendingRequests,
|
||||
method: 'git.status',
|
||||
params: undefined,
|
||||
timeoutMs: 1000,
|
||||
// ensureReady resolves immediately; the request is "in flight" but the
|
||||
// server never answers, modelling a genuinely stuck server-side call.
|
||||
ensureReady: () => Promise.resolve(),
|
||||
send: () => undefined,
|
||||
onTimeout,
|
||||
refreshTimeoutOnKeepalive: options.refreshTimeoutOnKeepalive
|
||||
})
|
||||
// Swallow the eventual rejection so unhandled-rejection noise doesn't leak.
|
||||
promise.catch(() => undefined)
|
||||
return { pendingRequests, promise, onTimeout }
|
||||
}
|
||||
|
||||
it('times out a stuck short RPC even while keepalive frames keep arriving', async () => {
|
||||
const { pendingRequests, promise, onTimeout } = startRequest()
|
||||
|
||||
// Periodic keepalives arrive faster than the 1000ms deadline — as they
|
||||
// would while a long-poll subscription streams over the same socket.
|
||||
for (let elapsed = 0; elapsed < 1000; elapsed += 200) {
|
||||
await vi.advanceTimersByTimeAsync(200)
|
||||
refreshSharedControlPendingRequestTimeouts(pendingRequests)
|
||||
}
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
|
||||
await expect(promise).rejects.toThrow()
|
||||
// The stuck-request path tears the connection down so reconnect+replay runs.
|
||||
expect(onTimeout).toHaveBeenCalledTimes(1)
|
||||
expect(pendingRequests.size).toBe(0)
|
||||
})
|
||||
|
||||
it('keeps refreshing a long-poll request that opted into keepalive refresh', async () => {
|
||||
const { pendingRequests, promise, onTimeout } = startRequest({
|
||||
refreshTimeoutOnKeepalive: true
|
||||
})
|
||||
|
||||
// Same keepalive cadence, but this request opted in, so each keepalive
|
||||
// pushes the deadline out and it never fires.
|
||||
for (let elapsed = 0; elapsed < 3000; elapsed += 200) {
|
||||
await vi.advanceTimersByTimeAsync(200)
|
||||
refreshSharedControlPendingRequestTimeouts(pendingRequests)
|
||||
}
|
||||
|
||||
expect(onTimeout).not.toHaveBeenCalled()
|
||||
expect(pendingRequests.size).toBe(1)
|
||||
|
||||
// It still resolves normally once the server finally answers.
|
||||
const [requestId] = pendingRequests.keys()
|
||||
resolveSharedControlPendingResponse(pendingRequests, requestId!, {
|
||||
id: requestId!,
|
||||
ok: true,
|
||||
result: { done: true },
|
||||
_meta: { runtimeId: 'runtime-test' }
|
||||
})
|
||||
await expect(promise).resolves.toMatchObject({ ok: true })
|
||||
})
|
||||
|
||||
it('fires the deadline for a short RPC when no keepalives arrive', async () => {
|
||||
const { pendingRequests, promise, onTimeout } = startRequest()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1001)
|
||||
|
||||
await expect(promise).rejects.toThrow()
|
||||
expect(onTimeout).toHaveBeenCalledTimes(1)
|
||||
expect(pendingRequests.size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
|
@ -17,6 +17,9 @@ export function requestSharedControl<TResult>(args: {
|
|||
ensureReady: () => Promise<void>
|
||||
send: (requestId: string, method: string, params: unknown) => void
|
||||
onTimeout?: (error: RemoteRuntimeClientError) => void
|
||||
// Why: default off — ordinary short RPCs keep an absolute deadline. Only
|
||||
// long-polls routed through this path opt in so keepalives extend them.
|
||||
refreshTimeoutOnKeepalive?: boolean
|
||||
}): Promise<RuntimeRpcResponse<TResult>> {
|
||||
const requestId = randomUUID()
|
||||
return new Promise<RuntimeRpcResponse<TResult>>((resolve, reject) => {
|
||||
|
|
@ -41,7 +44,8 @@ export function requestSharedControl<TResult>(args: {
|
|||
method: args.method,
|
||||
resolve: resolve as (response: RuntimeRpcResponse<unknown>) => void,
|
||||
reject,
|
||||
timeout
|
||||
timeout,
|
||||
refreshTimeoutOnKeepalive: args.refreshTimeoutOnKeepalive ?? false
|
||||
})
|
||||
void args.ensureReady().then(
|
||||
() => args.send(requestId, args.method, args.params),
|
||||
|
|
|
|||
|
|
@ -64,6 +64,11 @@ export function refreshSharedControlPendingRequestTimeouts(
|
|||
pendingRequests: Map<string, SharedControlPendingRequest<unknown>>
|
||||
): void {
|
||||
for (const pending of pendingRequests.values()) {
|
||||
// Why: only long-poll requests opted into keepalive refresh; refreshing an
|
||||
// ordinary short RPC would keep a genuinely-stuck server call alive forever.
|
||||
if (!pending.refreshTimeoutOnKeepalive) {
|
||||
continue
|
||||
}
|
||||
const timeout = pending.timeout as ReturnType<typeof setTimeout> & { refresh?: () => void }
|
||||
timeout.refresh?.()
|
||||
}
|
||||
|
|
@ -144,6 +149,13 @@ export function handleSharedControlSubscriptionResponse(
|
|||
subscription: SharedControlLogicalSubscription<unknown>,
|
||||
response: RuntimeRpcResponse<unknown>
|
||||
): void {
|
||||
// The replay window ends on either success or error. An error created no
|
||||
// remote subscription, so a later close must finish locally instead of
|
||||
// waiting forever for an id that will never arrive.
|
||||
subscription.awaitingResubscribe = false
|
||||
if (!response.ok) {
|
||||
subscription.sent = false
|
||||
}
|
||||
if (response.ok) {
|
||||
const subscriptionId = getSubscriptionId(response.result)
|
||||
if (subscriptionId) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,150 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RuntimeRpcResponse } from './runtime-rpc-envelope'
|
||||
import {
|
||||
closeSharedControlLogicalSubscription,
|
||||
createSharedControlSubscription,
|
||||
handleSharedControlLogicalResponse,
|
||||
replaySharedControlSubscriptions
|
||||
} from './remote-runtime-shared-control-subscriptions'
|
||||
import type { SharedControlLogicalSubscription } from './remote-runtime-shared-control-types'
|
||||
|
||||
function makeSubscriptions(): {
|
||||
subscriptions: Map<string, SharedControlLogicalSubscription<unknown>>
|
||||
subscription: SharedControlLogicalSubscription<unknown>
|
||||
} {
|
||||
const subscriptions = new Map<string, SharedControlLogicalSubscription<unknown>>()
|
||||
const subscription = createSharedControlSubscription({
|
||||
requestId: 'req-1',
|
||||
method: 'runtime.clientEvents.subscribe',
|
||||
params: null,
|
||||
callbacks: { onResponse: vi.fn(), onError: vi.fn() }
|
||||
})
|
||||
subscriptions.set(subscription.requestId, subscription)
|
||||
return { subscriptions, subscription }
|
||||
}
|
||||
|
||||
function okResponse(subscriptionId: string): RuntimeRpcResponse<unknown> {
|
||||
return {
|
||||
ok: true,
|
||||
id: 'req-1',
|
||||
result: { subscriptionId }
|
||||
} as unknown as RuntimeRpcResponse<unknown>
|
||||
}
|
||||
|
||||
describe('closeSharedControlLogicalSubscription — replay-window leak', () => {
|
||||
it('sends the unsubscribe when closed after an established subscribe replay completes', () => {
|
||||
const { subscriptions, subscription } = makeSubscriptions()
|
||||
// First establishment: server assigned a concrete subscription id.
|
||||
subscription.sent = true
|
||||
subscription.remoteSubscriptionId = 'server-sub-1'
|
||||
|
||||
const request = vi.fn()
|
||||
closeSharedControlLogicalSubscription({ subscriptions, subscription, request })
|
||||
|
||||
// Established subscription cleans up immediately by its known id.
|
||||
expect(request).toHaveBeenCalledWith('runtime.clientEvents.unsubscribe', {
|
||||
subscriptionId: 'server-sub-1'
|
||||
})
|
||||
expect(subscriptions.size).toBe(0)
|
||||
})
|
||||
|
||||
it('does NOT leak a server subscription when close races the replay resubscribe window', () => {
|
||||
const { subscriptions, subscription } = makeSubscriptions()
|
||||
// Subscription was previously established on the server.
|
||||
subscription.sent = true
|
||||
subscription.remoteSubscriptionId = 'server-sub-1'
|
||||
|
||||
// Reconnect replay: sent flips false and the old id is cleared right before
|
||||
// the resubscribe frame goes out. The server WILL assign a fresh id.
|
||||
const sent: SharedControlLogicalSubscription<unknown>[] = []
|
||||
replaySharedControlSubscriptions({
|
||||
subscriptions,
|
||||
send: (s) => {
|
||||
sent.push(s)
|
||||
},
|
||||
tagReplayedResponses: true
|
||||
})
|
||||
expect(subscription.sent).toBe(false)
|
||||
expect(subscription.remoteSubscriptionId).toBeNull()
|
||||
|
||||
// close() arrives during the window: no remoteSubscriptionId yet, so a
|
||||
// naive close finishes locally and never unsubscribes — leaking the server
|
||||
// subscription that the in-flight resubscribe is about to create.
|
||||
const request = vi.fn()
|
||||
closeSharedControlLogicalSubscription({ subscriptions, subscription, request })
|
||||
|
||||
// The close must be deferred, not finished with no cleanup.
|
||||
expect(subscriptions.size).toBe(1)
|
||||
expect(request).not.toHaveBeenCalled()
|
||||
|
||||
// When the resubscribe's ready response arrives with the new server id, the
|
||||
// deferred close fires the unsubscribe and only then finishes.
|
||||
handleSharedControlLogicalResponse({
|
||||
subscriptions,
|
||||
subscription,
|
||||
response: okResponse('server-sub-2'),
|
||||
request
|
||||
})
|
||||
|
||||
expect(request).toHaveBeenCalledWith('runtime.clientEvents.unsubscribe', {
|
||||
subscriptionId: 'server-sub-2'
|
||||
})
|
||||
expect(subscriptions.size).toBe(0)
|
||||
})
|
||||
|
||||
it('finishes locally without an unsubscribe for a never-sent subscription', () => {
|
||||
const { subscriptions, subscription } = makeSubscriptions()
|
||||
// Brand new: never sent, no server subscription exists.
|
||||
const request = vi.fn()
|
||||
closeSharedControlLogicalSubscription({ subscriptions, subscription, request })
|
||||
|
||||
expect(request).not.toHaveBeenCalled()
|
||||
expect(subscriptions.size).toBe(0)
|
||||
})
|
||||
|
||||
it('finishes a deferred close locally when replay returns an error', () => {
|
||||
const { subscriptions, subscription } = makeSubscriptions()
|
||||
subscription.sent = true
|
||||
subscription.remoteSubscriptionId = 'server-sub-1'
|
||||
replaySharedControlSubscriptions({ subscriptions, send: vi.fn(), tagReplayedResponses: true })
|
||||
|
||||
const request = vi.fn()
|
||||
closeSharedControlLogicalSubscription({ subscriptions, subscription, request })
|
||||
handleSharedControlLogicalResponse({
|
||||
subscriptions,
|
||||
subscription,
|
||||
response: {
|
||||
ok: false,
|
||||
id: 'req-1',
|
||||
error: { code: 'replay_failed', message: 'replay failed' }
|
||||
} as RuntimeRpcResponse<unknown>,
|
||||
request
|
||||
})
|
||||
|
||||
expect(request).not.toHaveBeenCalled()
|
||||
expect(subscriptions.size).toBe(0)
|
||||
})
|
||||
|
||||
it('does not leave a later close waiting after replay already failed', () => {
|
||||
const { subscriptions, subscription } = makeSubscriptions()
|
||||
subscription.sent = true
|
||||
subscription.remoteSubscriptionId = 'server-sub-1'
|
||||
replaySharedControlSubscriptions({ subscriptions, send: vi.fn(), tagReplayedResponses: true })
|
||||
|
||||
handleSharedControlLogicalResponse({
|
||||
subscriptions,
|
||||
subscription,
|
||||
response: {
|
||||
ok: false,
|
||||
id: 'req-1',
|
||||
error: { code: 'replay_failed', message: 'replay failed' }
|
||||
} as RuntimeRpcResponse<unknown>,
|
||||
request: vi.fn()
|
||||
})
|
||||
|
||||
const request = vi.fn()
|
||||
closeSharedControlLogicalSubscription({ subscriptions, subscription, request })
|
||||
expect(request).not.toHaveBeenCalled()
|
||||
expect(subscriptions.size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
|
@ -62,9 +62,14 @@ export function closeSharedControlLogicalSubscription(args: {
|
|||
args.request(cleanup.method, cleanup.params)
|
||||
return
|
||||
}
|
||||
if (args.subscription.sent && cleanupNeedsRemoteSubscriptionId(args.subscription.method)) {
|
||||
if (
|
||||
(args.subscription.sent || args.subscription.awaitingResubscribe) &&
|
||||
cleanupNeedsRemoteSubscriptionId(args.subscription.method)
|
||||
) {
|
||||
// Why: id-scoped server subscriptions can only be cleaned up after the
|
||||
// server returns its concrete subscription id in the ready response.
|
||||
// server returns its concrete subscription id in the ready response. This
|
||||
// also covers the reconnect replay window (sent===false, id cleared) where
|
||||
// a resubscribe is in flight — finishing locally there would leak it.
|
||||
args.subscription.closeAfterReady = true
|
||||
return
|
||||
}
|
||||
|
|
@ -100,6 +105,10 @@ export function replaySharedControlSubscriptions(args: {
|
|||
}
|
||||
subscription.sent = false
|
||||
subscription.remoteSubscriptionId = null
|
||||
// Why: mark the id-less window so a close() racing this resubscribe defers
|
||||
// to closeAfterReady instead of finishing locally and leaking the server
|
||||
// subscription the resubscribe is about to create.
|
||||
subscription.awaitingResubscribe = true
|
||||
if (args.tagReplayedResponses) {
|
||||
subscription.pendingReplayTag = true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,12 @@ export type SharedControlPendingRequest<TResult> = {
|
|||
resolve: (response: RuntimeRpcResponse<TResult>) => void
|
||||
reject: (error: Error) => void
|
||||
timeout: ReturnType<typeof setTimeout>
|
||||
// Why: keepalives on the shared socket are armed for an unrelated long-poll,
|
||||
// not this request. Only requests that opt in (long-polls issued via the
|
||||
// short-RPC path) may have their deadline refreshed by a keepalive; ordinary
|
||||
// short RPCs keep an absolute deadline so a stuck server call still times
|
||||
// out, tears the socket down, and reconnects/replays as designed.
|
||||
refreshTimeoutOnKeepalive: boolean
|
||||
}
|
||||
|
||||
export type SharedControlSubscriptionCallbacks<TResult> = {
|
||||
|
|
@ -34,6 +40,11 @@ export type SharedControlLogicalSubscription<TResult = unknown> = {
|
|||
// response is the authoritative re-emitted snapshot and gets tagged so
|
||||
// monotonic freshness gates don't drop it (#7718).
|
||||
pendingReplayTag?: boolean
|
||||
// Why: true from the moment a reconnect replay clears remoteSubscriptionId
|
||||
// until the resubscribe response arrives. A close() during this window has no
|
||||
// id to unsubscribe by yet, so it must defer instead of finishing locally —
|
||||
// otherwise the resubscribe the server is about to accept leaks.
|
||||
awaitingResubscribe?: boolean
|
||||
}
|
||||
|
||||
export type SharedControlReadyWaiter = {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,128 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { createWsOutboundBackpressureQueue } from './ws-outbound-backpressure-queue'
|
||||
|
||||
// Deterministic harness: bufferedAmount and the drain timer are both injected,
|
||||
// so no wall-clock races. `runTimers` fires the single parked drain callback.
|
||||
|
||||
function createHarness(overrides?: {
|
||||
softCapBytes?: number
|
||||
maxQueuedBytes?: number
|
||||
writable?: boolean
|
||||
}) {
|
||||
const sent: string[] = []
|
||||
let bufferedAmount = 0
|
||||
let writable = overrides?.writable ?? true
|
||||
const overflow = vi.fn()
|
||||
let pendingTimer: (() => void) | null = null
|
||||
|
||||
const queue = createWsOutboundBackpressureQueue<string>({
|
||||
send: (frame) => sent.push(frame),
|
||||
byteLengthOf: (frame) => frame.length,
|
||||
getBufferedAmount: () => bufferedAmount,
|
||||
isWritable: () => writable,
|
||||
onOverflow: overflow,
|
||||
softCapBytes: overrides?.softCapBytes ?? 100,
|
||||
maxQueuedBytes: overrides?.maxQueuedBytes ?? 1000,
|
||||
drainPollMs: 10,
|
||||
setTimer: (cb) => {
|
||||
pendingTimer = cb
|
||||
return 1 as unknown as ReturnType<typeof setTimeout>
|
||||
},
|
||||
clearTimer: () => {
|
||||
pendingTimer = null
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
queue,
|
||||
sent,
|
||||
overflow,
|
||||
setBuffered: (value: number) => {
|
||||
bufferedAmount = value
|
||||
},
|
||||
setWritable: (value: boolean) => {
|
||||
writable = value
|
||||
},
|
||||
runTimer: () => {
|
||||
const cb = pendingTimer
|
||||
pendingTimer = null
|
||||
cb?.()
|
||||
},
|
||||
hasTimer: () => pendingTimer !== null
|
||||
}
|
||||
}
|
||||
|
||||
describe('ws outbound backpressure queue', () => {
|
||||
it('sends straight through while under the soft cap', () => {
|
||||
const h = createHarness()
|
||||
h.queue.enqueue('a')
|
||||
h.queue.enqueue('b')
|
||||
expect(h.sent).toEqual(['a', 'b'])
|
||||
expect(h.hasTimer()).toBe(false)
|
||||
})
|
||||
|
||||
it('parks frames in order while over the cap and drains on recovery without loss', () => {
|
||||
const h = createHarness({ softCapBytes: 100 })
|
||||
h.setBuffered(200) // over cap
|
||||
h.queue.enqueue('one')
|
||||
h.queue.enqueue('two')
|
||||
h.queue.enqueue('three')
|
||||
// Nothing sent yet; all held in order.
|
||||
expect(h.sent).toEqual([])
|
||||
expect(h.queue.queuedBytes()).toBe('one'.length + 'two'.length + 'three'.length)
|
||||
|
||||
// Link recovers; the drain timer flushes everything in FIFO order.
|
||||
h.setBuffered(0)
|
||||
h.runTimer()
|
||||
expect(h.sent).toEqual(['one', 'two', 'three'])
|
||||
expect(h.queue.queuedBytes()).toBe(0)
|
||||
})
|
||||
|
||||
it('keeps ordering when a frame arrives while a backlog is parked', () => {
|
||||
const h = createHarness({ softCapBytes: 100 })
|
||||
h.setBuffered(200)
|
||||
h.queue.enqueue('first')
|
||||
h.setBuffered(0)
|
||||
// Even though the wire is now clear, an existing backlog means the new
|
||||
// frame must queue behind it, not jump the line.
|
||||
h.queue.enqueue('second')
|
||||
expect(h.sent).toEqual([])
|
||||
h.runTimer()
|
||||
expect(h.sent).toEqual(['first', 'second'])
|
||||
})
|
||||
|
||||
it('signals overflow (and drops backlog) when the hard cap is exceeded', () => {
|
||||
const h = createHarness({ softCapBytes: 10, maxQueuedBytes: 8 })
|
||||
h.setBuffered(100) // over soft cap: everything queues
|
||||
h.queue.enqueue('12345') // 5 bytes queued
|
||||
expect(h.overflow).not.toHaveBeenCalled()
|
||||
h.queue.enqueue('67890') // 10 bytes total > 8 -> overflow
|
||||
expect(h.overflow).toHaveBeenCalledTimes(1)
|
||||
// After overflow the queue is inert: no sends, no further overflow calls.
|
||||
h.setBuffered(0)
|
||||
h.runTimer()
|
||||
h.queue.enqueue('later')
|
||||
expect(h.sent).toEqual([])
|
||||
expect(h.overflow).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('drops the backlog if the socket becomes unwritable mid-park', () => {
|
||||
const h = createHarness({ softCapBytes: 10 })
|
||||
h.setBuffered(100)
|
||||
h.queue.enqueue('data')
|
||||
h.setWritable(false)
|
||||
h.runTimer()
|
||||
expect(h.sent).toEqual([])
|
||||
expect(h.queue.queuedBytes()).toBe(0)
|
||||
})
|
||||
|
||||
it('does not fast-path a frame while the socket is unwritable', () => {
|
||||
const h = createHarness({ writable: false })
|
||||
|
||||
h.queue.enqueue('data')
|
||||
|
||||
expect(h.sent).toEqual([])
|
||||
expect(h.queue.queuedBytes()).toBe('data'.length)
|
||||
expect(h.hasTimer()).toBe(true)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
// Why: both the server reply path (e2ee-channel) and the client send path
|
||||
// (remote-runtime-client) write to a ws with no backpressure handling. A fast
|
||||
// producer over a slow link balloons ws.bufferedAmount / RSS without bound, or
|
||||
// (binary path) silently drops frames. This queue holds outbound frames in
|
||||
// order while bufferedAmount is over a soft cap and flushes as it drains, so no
|
||||
// frame is dropped or reordered. It only signals overflow when a hard byte
|
||||
// bound is exceeded (the link is effectively dead), letting the caller force a
|
||||
// clean reconnect/resync instead of growing memory without limit.
|
||||
//
|
||||
// Generic over the frame type so it serves both the text reply path (encrypted
|
||||
// base64 strings) and the binary send path (Uint8Array frames).
|
||||
|
||||
export type WsOutboundBackpressureQueueOptions<TFrame> = {
|
||||
/** Send a frame on the wire. Called only when under the soft cap. */
|
||||
send: (frame: TFrame) => void
|
||||
/** Serialized byte length of a frame, for cap accounting. */
|
||||
byteLengthOf: (frame: TFrame) => number
|
||||
/** Current ws.bufferedAmount in bytes. */
|
||||
getBufferedAmount: () => number
|
||||
/** True when the socket can still accept sends (OPEN and keyed). */
|
||||
isWritable: () => boolean
|
||||
/**
|
||||
* Called once when queued bytes exceed maxQueuedBytes — the link is wedged.
|
||||
* The caller should tear the connection down so a fresh subscription can
|
||||
* replay an authoritative snapshot. The queue drops its backlog afterward.
|
||||
*/
|
||||
onOverflow: () => void
|
||||
/** Soft cap: stop draining onto the wire while bufferedAmount is above this. */
|
||||
softCapBytes?: number
|
||||
/** Hard cap on bytes held in this queue before onOverflow fires. */
|
||||
maxQueuedBytes?: number
|
||||
/** Poll interval used to re-check bufferedAmount while parked. */
|
||||
drainPollMs?: number
|
||||
/** Injectable scheduler for deterministic tests. */
|
||||
setTimer?: (cb: () => void, ms: number) => ReturnType<typeof setTimeout>
|
||||
clearTimer?: (timer: ReturnType<typeof setTimeout>) => void
|
||||
}
|
||||
|
||||
export type WsOutboundBackpressureQueue<TFrame> = {
|
||||
/** Queue-or-send a frame. Preserves order across all prior frames. */
|
||||
enqueue: (frame: TFrame) => void
|
||||
/** Bytes currently held (not yet handed to the wire). */
|
||||
queuedBytes: () => number
|
||||
/** Drop the backlog and stop the drain timer (call on close). */
|
||||
dispose: () => void
|
||||
}
|
||||
|
||||
const DEFAULT_SOFT_CAP_BYTES = 8 * 1024 * 1024
|
||||
// Why: tolerate a large transient burst (e.g. a build log spike) before
|
||||
// declaring the link dead; 64 MiB is ~8x the soft cap yet still bounds RSS.
|
||||
const DEFAULT_MAX_QUEUED_BYTES = 64 * 1024 * 1024
|
||||
const DEFAULT_DRAIN_POLL_MS = 25
|
||||
|
||||
export function createWsOutboundBackpressureQueue<TFrame>(
|
||||
options: WsOutboundBackpressureQueueOptions<TFrame>
|
||||
): WsOutboundBackpressureQueue<TFrame> {
|
||||
const softCapBytes = options.softCapBytes ?? DEFAULT_SOFT_CAP_BYTES
|
||||
const maxQueuedBytes = options.maxQueuedBytes ?? DEFAULT_MAX_QUEUED_BYTES
|
||||
const drainPollMs = options.drainPollMs ?? DEFAULT_DRAIN_POLL_MS
|
||||
const setTimer = options.setTimer ?? ((cb, ms) => setTimeout(cb, ms))
|
||||
const clearTimer = options.clearTimer ?? ((timer) => clearTimeout(timer))
|
||||
|
||||
// Why: a ws without a numeric bufferedAmount (some mocks/transports) must not
|
||||
// strand frames in the queue forever; treat unknown backpressure as "clear".
|
||||
const bufferedAmount = (): number => {
|
||||
const value = options.getBufferedAmount()
|
||||
return Number.isFinite(value) ? value : 0
|
||||
}
|
||||
|
||||
const queue: { frame: TFrame; bytes: number }[] = []
|
||||
let queueHead = 0
|
||||
let queued = 0
|
||||
let timer: ReturnType<typeof setTimeout> | null = null
|
||||
let overflowed = false
|
||||
let disposed = false
|
||||
|
||||
const stopTimer = (): void => {
|
||||
if (timer !== null) {
|
||||
clearTimer(timer)
|
||||
timer = null
|
||||
}
|
||||
}
|
||||
|
||||
const dropBacklog = (): void => {
|
||||
queue.length = 0
|
||||
queueHead = 0
|
||||
queued = 0
|
||||
stopTimer()
|
||||
}
|
||||
|
||||
// Drain as many queued frames as the wire will take without crossing the
|
||||
// soft cap; re-arm the poll timer if frames remain.
|
||||
const drain = (): void => {
|
||||
if (disposed || overflowed) {
|
||||
return
|
||||
}
|
||||
if (!options.isWritable()) {
|
||||
// Socket went away mid-park; let the transport's own close path clean up.
|
||||
dropBacklog()
|
||||
return
|
||||
}
|
||||
while (queueHead < queue.length && bufferedAmount() <= softCapBytes) {
|
||||
const entry = queue[queueHead++]
|
||||
queued -= entry.bytes
|
||||
options.send(entry.frame)
|
||||
}
|
||||
if (queueHead < queue.length) {
|
||||
timer = setTimer(drain, drainPollMs)
|
||||
} else {
|
||||
// Why: resetting the drained array keeps enqueue/drain O(1) per frame;
|
||||
// repeated Array.shift() would make recovery from a large backlog O(n²).
|
||||
queue.length = 0
|
||||
queueHead = 0
|
||||
stopTimer()
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
enqueue(frame: TFrame): void {
|
||||
if (disposed || overflowed) {
|
||||
return
|
||||
}
|
||||
// Fast path: nothing parked and the wire is under the cap — send directly.
|
||||
if (queueHead === queue.length && options.isWritable() && bufferedAmount() <= softCapBytes) {
|
||||
options.send(frame)
|
||||
return
|
||||
}
|
||||
const bytes = options.byteLengthOf(frame)
|
||||
queue.push({ frame, bytes })
|
||||
queued += bytes
|
||||
if (queued > maxQueuedBytes) {
|
||||
overflowed = true
|
||||
dropBacklog()
|
||||
options.onOverflow()
|
||||
return
|
||||
}
|
||||
if (timer === null) {
|
||||
timer = setTimer(drain, drainPollMs)
|
||||
}
|
||||
},
|
||||
queuedBytes: () => queued,
|
||||
dispose(): void {
|
||||
disposed = true
|
||||
dropBacklog()
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue