fix(runtime): coalesce concurrent host terminal focus (#11841)
Bound exclusive host navigation to a generation-aware latest-wins single-flight so bulk open and switch fan-out stay responsive on large remote fleets. Add freeze repro harnesses and navigated settlement.
This commit is contained in:
Neil
GitHub
parent
8e9640cb1d
commit
339045b150
|
|
@ -0,0 +1,38 @@
|
|||
export class BoundedLiveFreezeHistory {
|
||||
#entries = []
|
||||
#limit
|
||||
#nextIndex = 0
|
||||
#totalCount = 0
|
||||
|
||||
constructor(limit) {
|
||||
if (!Number.isInteger(limit) || limit <= 0) {
|
||||
throw new Error(`History limit must be a positive integer, got ${limit}`)
|
||||
}
|
||||
this.#limit = limit
|
||||
}
|
||||
|
||||
add(entry) {
|
||||
this.#totalCount += 1
|
||||
if (this.#entries.length < this.#limit) {
|
||||
this.#entries.push(entry)
|
||||
return
|
||||
}
|
||||
this.#entries[this.#nextIndex] = entry
|
||||
this.#nextIndex = (this.#nextIndex + 1) % this.#limit
|
||||
}
|
||||
|
||||
get retainedCount() {
|
||||
return this.#entries.length
|
||||
}
|
||||
|
||||
get totalCount() {
|
||||
return this.#totalCount
|
||||
}
|
||||
|
||||
values() {
|
||||
if (this.#entries.length < this.#limit || this.#nextIndex === 0) {
|
||||
return [...this.#entries]
|
||||
}
|
||||
return [...this.#entries.slice(this.#nextIndex), ...this.#entries.slice(0, this.#nextIndex)]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import { BoundedLiveFreezeHistory } from './live-freeze-bounded-history.mjs'
|
||||
|
||||
describe('BoundedLiveFreezeHistory', () => {
|
||||
it('retains the newest entries in insertion order and counts the full run', () => {
|
||||
const history = new BoundedLiveFreezeHistory(3)
|
||||
|
||||
for (let value = 1; value <= 7; value += 1) {
|
||||
history.add(value)
|
||||
}
|
||||
|
||||
expect(history.values()).toEqual([5, 6, 7])
|
||||
expect(history.retainedCount).toBe(3)
|
||||
expect(history.totalCount).toBe(7)
|
||||
})
|
||||
|
||||
it('rejects invalid retention limits', () => {
|
||||
expect(() => new BoundedLiveFreezeHistory(0)).toThrow('positive integer')
|
||||
expect(() => new BoundedLiveFreezeHistory(1.5)).toThrow('positive integer')
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,262 @@
|
|||
/**
|
||||
* Pure metrics helpers for the live remote bulk-open freeze harness.
|
||||
* Kept separate so unit tests can drive the same code the repro uses.
|
||||
*/
|
||||
|
||||
export const DEFAULT_SOFT_MS = 2000
|
||||
export const DEFAULT_HARD_MS = 5000
|
||||
|
||||
export function readFreezeNumberEnv(name, fallback) {
|
||||
const raw = process.env[name]
|
||||
if (raw == null || raw.trim() === '') {
|
||||
return fallback
|
||||
}
|
||||
const value = Number(raw)
|
||||
if (!Number.isFinite(value)) {
|
||||
throw new Error(`Invalid ${name}: expected a finite number, got ${JSON.stringify(raw)}`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
export function extractTerminalHandle(result) {
|
||||
if (!result || typeof result !== 'object') {
|
||||
return null
|
||||
}
|
||||
const candidates = [
|
||||
result.handle,
|
||||
result.terminalHandle,
|
||||
result.agentTerminalHandle,
|
||||
typeof result.terminal === 'string' ? result.terminal : result.terminal?.handle,
|
||||
result.startupTerminal?.handle,
|
||||
result.tab?.terminal,
|
||||
result.tab?.handle
|
||||
]
|
||||
for (const value of candidates) {
|
||||
if (typeof value === 'string' && value.startsWith('term_')) {
|
||||
return value
|
||||
}
|
||||
}
|
||||
for (const value of Object.values(result)) {
|
||||
if (typeof value === 'string' && value.startsWith('term_')) {
|
||||
return value
|
||||
}
|
||||
if (value && typeof value === 'object') {
|
||||
for (const nested of Object.values(value)) {
|
||||
if (typeof nested === 'string' && nested.startsWith('term_')) {
|
||||
return nested
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export function worktreeSelector(wt) {
|
||||
if (typeof wt?.id === 'string' && wt.id.length > 0) {
|
||||
return `id:${wt.id}`
|
||||
}
|
||||
if (typeof wt?.path === 'string' && wt.path.length > 0) {
|
||||
return `path:${wt.path}`
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Peak stall across individual switch latency and concurrent batch wall.
|
||||
* Hard freeze when peak >= hardMs (default 5000).
|
||||
*/
|
||||
export function evaluateFreezeSignals({
|
||||
maxSwitchMs = 0,
|
||||
maxBatchWallMs = 0,
|
||||
statusProbeMs = 0,
|
||||
memoryProbeMs = null,
|
||||
softMs = DEFAULT_SOFT_MS,
|
||||
hardMs = DEFAULT_HARD_MS
|
||||
}) {
|
||||
const peakLatencyMs = Math.max(maxSwitchMs, maxBatchWallMs)
|
||||
const softFreeze =
|
||||
peakLatencyMs >= softMs ||
|
||||
statusProbeMs >= softMs ||
|
||||
(memoryProbeMs != null && memoryProbeMs >= softMs)
|
||||
const hardFreeze =
|
||||
peakLatencyMs >= hardMs ||
|
||||
statusProbeMs >= hardMs ||
|
||||
(memoryProbeMs != null && memoryProbeMs >= hardMs)
|
||||
return { peakLatencyMs, softFreeze, hardFreeze }
|
||||
}
|
||||
|
||||
export function shouldCapSwitchTargets(maxSwitchTargets) {
|
||||
return Number.isFinite(maxSwitchTargets) && maxSwitchTargets > 0
|
||||
}
|
||||
|
||||
export function applySwitchTargetCap(targets, maxSwitchTargets) {
|
||||
if (!shouldCapSwitchTargets(maxSwitchTargets)) {
|
||||
return targets
|
||||
}
|
||||
return targets.slice(0, maxSwitchTargets)
|
||||
}
|
||||
|
||||
/** Scenarios that model real user recovery, not concurrent CLI pileup. */
|
||||
export const REALISTIC_SCENARIOS = [
|
||||
'idle-backlog-open',
|
||||
'idle-backlog-reconnect-open',
|
||||
'restart-proxy',
|
||||
/** Idle + flood + reconnect storm overlapped with concurrent open fan-out. */
|
||||
'lockup-storm'
|
||||
]
|
||||
|
||||
/**
|
||||
* Permanent lockup: app/host stops making progress — not a single recovered timeout.
|
||||
* Distinct from multi-second hard stall that still recovers (status answers, most opens ok).
|
||||
*/
|
||||
export function evaluatePermanentLockup({
|
||||
timedOutOps = 0,
|
||||
statusHangMs = 0,
|
||||
consecutiveSwitchFailures = 0,
|
||||
openFailed = 0,
|
||||
openTotal = 0,
|
||||
permanentTimeoutMs = 60_000,
|
||||
/** Fraction of opens that must fail to count as lockup without status hang. */
|
||||
failRateThreshold = 0.25,
|
||||
minTimedOutOps = 3
|
||||
}) {
|
||||
const failRate = openTotal > 0 ? openFailed / openTotal : 0
|
||||
const permanentLockup =
|
||||
statusHangMs >= permanentTimeoutMs ||
|
||||
timedOutOps >= minTimedOutOps ||
|
||||
consecutiveSwitchFailures >= 5 ||
|
||||
(openTotal >= 8 && failRate >= failRateThreshold)
|
||||
return {
|
||||
permanentLockup,
|
||||
timedOutOps,
|
||||
statusHangMs,
|
||||
consecutiveSwitchFailures,
|
||||
failRate,
|
||||
recoveredHardStallCandidate:
|
||||
!permanentLockup && timedOutOps < minTimedOutOps && statusHangMs < permanentTimeoutMs
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Peak across open latencies + optional reconnect-refresh wall + probes.
|
||||
* Used by the naturalistic harness (no parallel switch amp).
|
||||
*/
|
||||
export function evaluateRealisticFreezeSignals({
|
||||
maxOpenMs = 0,
|
||||
firstOpenMs = 0,
|
||||
reconnectRefreshMs = 0,
|
||||
statusProbeMs = 0,
|
||||
memoryProbeMs = null,
|
||||
softMs = DEFAULT_SOFT_MS,
|
||||
hardMs = DEFAULT_HARD_MS
|
||||
}) {
|
||||
const peakLatencyMs = Math.max(maxOpenMs, firstOpenMs, reconnectRefreshMs)
|
||||
return evaluateFreezeSignals({
|
||||
maxSwitchMs: peakLatencyMs,
|
||||
maxBatchWallMs: 0,
|
||||
statusProbeMs,
|
||||
memoryProbeMs,
|
||||
softMs,
|
||||
hardMs
|
||||
})
|
||||
}
|
||||
|
||||
export function humanPaceDelayMs(baseMs, jitterMs = 0) {
|
||||
const base = Math.max(0, baseMs)
|
||||
const jitter = Math.max(0, jitterMs)
|
||||
if (jitter === 0) {
|
||||
return base
|
||||
}
|
||||
return base + Math.floor(Math.random() * (jitter + 1))
|
||||
}
|
||||
|
||||
/** Full-app forever freeze: host RPC dead for a continuous window, not a recovered stall. */
|
||||
export const DEFAULT_FOREVER_WINDOW_MS = 30_000
|
||||
export const DEFAULT_STATUS_SLOW_MS = 15_000
|
||||
|
||||
/**
|
||||
* Analyze mid-storm status samples for a continuous unhealthy window.
|
||||
* Sample: { tMs, ms, ok, hang }
|
||||
*/
|
||||
export function evaluateFullAppFreeze({
|
||||
statusSamples = [],
|
||||
statusSummary = {},
|
||||
foreverWindowMs = DEFAULT_FOREVER_WINDOW_MS,
|
||||
statusSlowMs = DEFAULT_STATUS_SLOW_MS,
|
||||
killOnlyRecovery = false
|
||||
}) {
|
||||
const infrastructureErrors = statusSamples.filter((sample) => sample.infrastructureError)
|
||||
const infrastructureErrorCount = Math.max(
|
||||
infrastructureErrors.length,
|
||||
statusSummary.infrastructureErrorCount ?? 0
|
||||
)
|
||||
const maxStatusMs = Math.max(
|
||||
0,
|
||||
...statusSamples.map((s) => s.ms || 0),
|
||||
statusSummary.maxStatusMs ?? 0
|
||||
)
|
||||
if (killOnlyRecovery) {
|
||||
return {
|
||||
foreverUiLockupObserved: true,
|
||||
longestUnhealthyWindowMs: foreverWindowMs,
|
||||
maxStatusMs,
|
||||
unhealthySampleCount: statusSummary.sampleCount ?? statusSamples.length,
|
||||
infrastructureErrorCount,
|
||||
reason: 'kill-only recovery documented'
|
||||
}
|
||||
}
|
||||
|
||||
const unhealthy = statusSamples.map((s) => {
|
||||
const hang = !s.infrastructureError && (Boolean(s.hang) || s.ok === false)
|
||||
const slow = !s.infrastructureError && (s.ms || 0) >= statusSlowMs
|
||||
return { ...s, unhealthy: hang || slow }
|
||||
})
|
||||
|
||||
let longest = statusSummary.longestUnhealthyWindowMs ?? 0
|
||||
let runStart = null
|
||||
for (const s of unhealthy) {
|
||||
if (s.unhealthy) {
|
||||
runStart ??= s.tMs ?? 0
|
||||
const end = (s.tMs ?? 0) + (s.ms || 0)
|
||||
longest = Math.max(longest, end - runStart)
|
||||
} else {
|
||||
runStart = null
|
||||
}
|
||||
}
|
||||
|
||||
// If timestamps missing, fall back to consecutive unhealthy count * assumed interval.
|
||||
if (longest === 0 && unhealthy.some((s) => s.unhealthy)) {
|
||||
let run = 0
|
||||
for (const s of unhealthy) {
|
||||
if (s.unhealthy) {
|
||||
run += 1
|
||||
longest = Math.max(longest, run)
|
||||
} else {
|
||||
run = 0
|
||||
}
|
||||
}
|
||||
// Without wall clock, consecutive count alone is not a ms window.
|
||||
longest = 0
|
||||
}
|
||||
|
||||
const foreverUiLockupObserved = longest >= foreverWindowMs
|
||||
const unhealthySampleCount = Math.max(
|
||||
unhealthy.filter((s) => s.unhealthy).length,
|
||||
statusSummary.unhealthySampleCount ?? 0
|
||||
)
|
||||
|
||||
return {
|
||||
foreverUiLockupObserved,
|
||||
longestUnhealthyWindowMs: longest,
|
||||
maxStatusMs,
|
||||
unhealthySampleCount,
|
||||
infrastructureErrorCount,
|
||||
reason: foreverUiLockupObserved
|
||||
? `status unhealthy ≥${foreverWindowMs}ms continuous`
|
||||
: infrastructureErrorCount > 0
|
||||
? `status watchdog infrastructure errors: ${infrastructureErrorCount}`
|
||||
: maxStatusMs >= statusSlowMs
|
||||
? `status slow peak ${maxStatusMs}ms but no ≥${foreverWindowMs}ms window`
|
||||
: 'status remained healthy through storm'
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,239 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
applySwitchTargetCap,
|
||||
evaluateFreezeSignals,
|
||||
evaluateFullAppFreeze,
|
||||
evaluatePermanentLockup,
|
||||
evaluateRealisticFreezeSignals,
|
||||
extractTerminalHandle,
|
||||
humanPaceDelayMs,
|
||||
readFreezeNumberEnv,
|
||||
REALISTIC_SCENARIOS,
|
||||
shouldCapSwitchTargets,
|
||||
worktreeSelector
|
||||
} from './live-remote-bulk-open-freeze-metrics.mjs'
|
||||
|
||||
describe('live-remote-bulk-open-freeze-metrics', () => {
|
||||
it('extracts term_ handles from nested create payloads', () => {
|
||||
expect(extractTerminalHandle({ handle: 'term_abc' })).toBe('term_abc')
|
||||
expect(extractTerminalHandle({ terminal: { handle: 'term_nested' } })).toBe('term_nested')
|
||||
expect(extractTerminalHandle({ tab: { terminal: 'term_tab' } })).toBe('term_tab')
|
||||
expect(extractTerminalHandle({ startupTerminal: { handle: 'term_start' } })).toBe('term_start')
|
||||
expect(extractTerminalHandle({ junk: { deep: 'term_deep' } })).toBe('term_deep')
|
||||
expect(extractTerminalHandle({ handle: 'not-a-term' })).toBeNull()
|
||||
expect(extractTerminalHandle(null)).toBeNull()
|
||||
})
|
||||
|
||||
it('builds worktree selectors from id/path', () => {
|
||||
expect(
|
||||
worktreeSelector({ id: 'repo::C:/Users/neil/orca/orca', path: 'C:/Users/neil/orca/orca' })
|
||||
).toBe('id:repo::C:/Users/neil/orca/orca')
|
||||
expect(worktreeSelector({ path: '/tmp/x' })).toBe('path:/tmp/x')
|
||||
expect(worktreeSelector({})).toBeNull()
|
||||
})
|
||||
|
||||
it('does not cap switch targets when max is 0 (regression for Math.max(2,0) bug)', () => {
|
||||
expect(shouldCapSwitchTargets(0)).toBe(false)
|
||||
expect(shouldCapSwitchTargets(-1)).toBe(false)
|
||||
expect(shouldCapSwitchTargets(2)).toBe(true)
|
||||
const many = Array.from({ length: 111 }, (_, i) => `term_${i}`)
|
||||
expect(applySwitchTargetCap(many, 0)).toHaveLength(111)
|
||||
expect(applySwitchTargetCap(many, 2)).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('classifies hard freeze at >=5000ms peak (individual or batch wall)', () => {
|
||||
expect(evaluateFreezeSignals({ maxSwitchMs: 3874, maxBatchWallMs: 3874 }).hardFreeze).toBe(
|
||||
false
|
||||
)
|
||||
expect(evaluateFreezeSignals({ maxSwitchMs: 3874, maxBatchWallMs: 3874 }).softFreeze).toBe(true)
|
||||
|
||||
const hardIndividual = evaluateFreezeSignals({ maxSwitchMs: 19954, maxBatchWallMs: 1000 })
|
||||
expect(hardIndividual.hardFreeze).toBe(true)
|
||||
expect(hardIndividual.peakLatencyMs).toBe(19954)
|
||||
|
||||
const hardBatch = evaluateFreezeSignals({ maxSwitchMs: 900, maxBatchWallMs: 20201 })
|
||||
expect(hardBatch.hardFreeze).toBe(true)
|
||||
expect(hardBatch.peakLatencyMs).toBe(20201)
|
||||
})
|
||||
|
||||
it('evaluates naturalistic peaks without requiring parallel batch amp', () => {
|
||||
expect(REALISTIC_SCENARIOS).toContain('idle-backlog-open')
|
||||
expect(REALISTIC_SCENARIOS).toContain('idle-backlog-reconnect-open')
|
||||
expect(REALISTIC_SCENARIOS).toContain('lockup-storm')
|
||||
const soft = evaluateRealisticFreezeSignals({
|
||||
maxOpenMs: 3200,
|
||||
firstOpenMs: 2800,
|
||||
reconnectRefreshMs: 900
|
||||
})
|
||||
expect(soft.softFreeze).toBe(true)
|
||||
expect(soft.hardFreeze).toBe(false)
|
||||
expect(soft.peakLatencyMs).toBe(3200)
|
||||
|
||||
const hardFromReconnect = evaluateRealisticFreezeSignals({
|
||||
maxOpenMs: 800,
|
||||
firstOpenMs: 700,
|
||||
reconnectRefreshMs: 6200
|
||||
})
|
||||
expect(hardFromReconnect.hardFreeze).toBe(true)
|
||||
expect(hardFromReconnect.peakLatencyMs).toBe(6200)
|
||||
})
|
||||
|
||||
it('flags full-app freeze only for continuous unhealthy status window ≥30s', () => {
|
||||
const healthy = evaluateFullAppFreeze({
|
||||
statusSamples: [
|
||||
{ tMs: 0, ms: 150, ok: true },
|
||||
{ tMs: 2000, ms: 180, ok: true },
|
||||
{ tMs: 4000, ms: 140, ok: true }
|
||||
],
|
||||
foreverWindowMs: 30_000,
|
||||
statusSlowMs: 15_000
|
||||
})
|
||||
expect(healthy.foreverUiLockupObserved).toBe(false)
|
||||
|
||||
const forever = evaluateFullAppFreeze({
|
||||
statusSamples: [
|
||||
{ tMs: 0, ms: 16_000, ok: true },
|
||||
{ tMs: 16_000, ms: 16_000, ok: true },
|
||||
{ tMs: 32_000, ms: 16_000, ok: false, hang: true }
|
||||
],
|
||||
foreverWindowMs: 30_000,
|
||||
statusSlowMs: 15_000
|
||||
})
|
||||
expect(forever.foreverUiLockupObserved).toBe(true)
|
||||
expect(forever.longestUnhealthyWindowMs).toBeGreaterThanOrEqual(30_000)
|
||||
|
||||
expect(
|
||||
evaluateFullAppFreeze({ statusSamples: [], killOnlyRecovery: true }).foreverUiLockupObserved
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('does not classify watchdog infrastructure errors as an app freeze', () => {
|
||||
const result = evaluateFullAppFreeze({
|
||||
statusSamples: Array.from({ length: 25 }, (_, index) => ({
|
||||
tMs: index * 1500,
|
||||
ms: 1,
|
||||
ok: false,
|
||||
infrastructureError: true,
|
||||
error: 'spawn ENOENT'
|
||||
}))
|
||||
})
|
||||
|
||||
expect(result.foreverUiLockupObserved).toBe(false)
|
||||
expect(result.unhealthySampleCount).toBe(0)
|
||||
expect(result.infrastructureErrorCount).toBe(25)
|
||||
})
|
||||
|
||||
it('preserves full-run watchdog peaks after sample retention rotates', () => {
|
||||
const result = evaluateFullAppFreeze({
|
||||
statusSamples: [{ tMs: 60_000, ms: 100, ok: true }],
|
||||
statusSummary: {
|
||||
sampleCount: 40,
|
||||
maxStatusMs: 16_000,
|
||||
unhealthySampleCount: 3,
|
||||
infrastructureErrorCount: 2,
|
||||
longestUnhealthyWindowMs: 32_000
|
||||
},
|
||||
foreverWindowMs: 30_000,
|
||||
statusSlowMs: 15_000
|
||||
})
|
||||
|
||||
expect(result.foreverUiLockupObserved).toBe(true)
|
||||
expect(result.longestUnhealthyWindowMs).toBe(32_000)
|
||||
expect(result.maxStatusMs).toBe(16_000)
|
||||
expect(result.unhealthySampleCount).toBe(3)
|
||||
expect(result.infrastructureErrorCount).toBe(2)
|
||||
})
|
||||
|
||||
it('rejects invalid numeric environment values', () => {
|
||||
process.env.ORCA_FREEZE_TEST_NUMBER = 'not-a-number'
|
||||
expect(() => readFreezeNumberEnv('ORCA_FREEZE_TEST_NUMBER', 5)).toThrow(
|
||||
'Invalid ORCA_FREEZE_TEST_NUMBER'
|
||||
)
|
||||
delete process.env.ORCA_FREEZE_TEST_NUMBER
|
||||
expect(readFreezeNumberEnv('ORCA_FREEZE_TEST_NUMBER', 5)).toBe(5)
|
||||
})
|
||||
|
||||
it('distinguishes recovered hard stall from permanent lockup', () => {
|
||||
// Single reveal timeout with healthy status is NOT permanent app lockup.
|
||||
expect(
|
||||
evaluatePermanentLockup({
|
||||
timedOutOps: 1,
|
||||
statusHangMs: 0,
|
||||
consecutiveSwitchFailures: 1,
|
||||
openFailed: 1,
|
||||
openTotal: 64
|
||||
}).permanentLockup
|
||||
).toBe(false)
|
||||
expect(
|
||||
evaluatePermanentLockup({
|
||||
timedOutOps: 3,
|
||||
statusHangMs: 0,
|
||||
consecutiveSwitchFailures: 0,
|
||||
openFailed: 3,
|
||||
openTotal: 64
|
||||
}).permanentLockup
|
||||
).toBe(true)
|
||||
expect(
|
||||
evaluatePermanentLockup({
|
||||
timedOutOps: 0,
|
||||
statusHangMs: 60_000,
|
||||
consecutiveSwitchFailures: 0,
|
||||
permanentTimeoutMs: 60_000
|
||||
}).permanentLockup
|
||||
).toBe(true)
|
||||
expect(
|
||||
evaluatePermanentLockup({
|
||||
timedOutOps: 0,
|
||||
statusHangMs: 0,
|
||||
consecutiveSwitchFailures: 5
|
||||
}).permanentLockup
|
||||
).toBe(true)
|
||||
expect(
|
||||
evaluatePermanentLockup({
|
||||
timedOutOps: 0,
|
||||
openFailed: 20,
|
||||
openTotal: 40
|
||||
}).permanentLockup
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('human pace delay stays within base+jitter', () => {
|
||||
for (let i = 0; i < 20; i += 1) {
|
||||
const d = humanPaceDelayMs(250, 150)
|
||||
expect(d).toBeGreaterThanOrEqual(250)
|
||||
expect(d).toBeLessThanOrEqual(400)
|
||||
}
|
||||
expect(humanPaceDelayMs(100, 0)).toBe(100)
|
||||
})
|
||||
|
||||
it('reads the real hard-freeze lab report when present', async () => {
|
||||
const { readdirSync, readFileSync, existsSync } = await import('node:fs')
|
||||
const { resolve } = await import('node:path')
|
||||
const reportDir = resolve(process.cwd(), 'test-results/freeze-repro')
|
||||
if (!existsSync(reportDir)) {
|
||||
// Local clones without lab artifacts still pass pure metrics tests above.
|
||||
return
|
||||
}
|
||||
const reportName = readdirSync(reportDir).find(
|
||||
(name) => name.startsWith('live-bulk-open-freeze-') && name.endsWith('.json')
|
||||
)
|
||||
if (reportName == null) {
|
||||
return
|
||||
}
|
||||
const report = JSON.parse(readFileSync(resolve(reportDir, reportName), 'utf8'))
|
||||
const evaluated = evaluateFreezeSignals({
|
||||
maxSwitchMs: report.maxSwitchMs,
|
||||
maxBatchWallMs: report.maxBatchWallMs ?? 0,
|
||||
statusProbeMs: report.statusProbeMs ?? 0,
|
||||
memoryProbeMs: report.memoryProbeMs,
|
||||
softMs: report.softMs,
|
||||
hardMs: report.hardMs
|
||||
})
|
||||
expect(evaluated.hardFreeze).toBe(report.hardFreeze)
|
||||
expect(evaluated.peakLatencyMs).toBeGreaterThanOrEqual(5000)
|
||||
expect(typeof report.environment).toBe('string')
|
||||
expect(report.environment.length).toBeGreaterThan(0)
|
||||
expect(report.switchTargets).toBeGreaterThan(50)
|
||||
expect(report.parallel).toBeGreaterThanOrEqual(8)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,358 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Live freeze repro against a running Orca desktop + paired remote runtime.
|
||||
*
|
||||
* Models bulk-open of remote sessions under multi-worktree load.
|
||||
*
|
||||
* Usage:
|
||||
* node config/scripts/live-remote-bulk-open-freeze-repro.mjs
|
||||
* ORCA_FREEZE_ENV=paired-remote ORCA_FREEZE_CREATE=12 ORCA_FREEZE_SWITCH_PASSES=5 \
|
||||
* ORCA_FREEZE_PARALLEL=8 node config/scripts/live-remote-bulk-open-freeze-repro.mjs
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { mkdirSync, writeFileSync, copyFileSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { BoundedLiveFreezeHistory } from './live-freeze-bounded-history.mjs'
|
||||
import {
|
||||
applySwitchTargetCap,
|
||||
DEFAULT_HARD_MS,
|
||||
DEFAULT_SOFT_MS,
|
||||
evaluateFreezeSignals,
|
||||
extractTerminalHandle,
|
||||
readFreezeNumberEnv,
|
||||
shouldCapSwitchTargets,
|
||||
worktreeSelector
|
||||
} from './live-remote-bulk-open-freeze-metrics.mjs'
|
||||
import { createOrcaRpc } from './live-remote-freeze-rpc.mjs'
|
||||
|
||||
const root = path.resolve(import.meta.dirname, '../..')
|
||||
const reportDir = path.join(root, 'test-results', 'freeze-repro')
|
||||
const envName = process.env.ORCA_FREEZE_ENV || 'paired-remote'
|
||||
const createCount = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_CREATE', 0))
|
||||
const switchPasses = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_SWITCH_PASSES', 3))
|
||||
const parallel = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_PARALLEL', 1))
|
||||
// 0 = no cap (use all live terminals). Only positive env values limit targets.
|
||||
const maxSwitchTargets = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_MAX_SWITCH_TARGETS', 0))
|
||||
const softMs = readFreezeNumberEnv('ORCA_FREEZE_SOFT_MS', DEFAULT_SOFT_MS)
|
||||
const hardMs = readFreezeNumberEnv('ORCA_FREEZE_HARD_MS', DEFAULT_HARD_MS)
|
||||
const createWorktreeSpan = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_CREATE_WT_SPAN', 16))
|
||||
const preFloodMs = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_PRE_FLOOD_MS', 3000))
|
||||
const scratchDir = process.env.ORCA_FREEZE_SCRATCH || ''
|
||||
|
||||
const { orcaJsonSync, orcaJsonAsync } = createOrcaRpc({ envName })
|
||||
|
||||
async function mapPool(items, concurrency, worker) {
|
||||
const results = Array.from({ length: items.length })
|
||||
let next = 0
|
||||
async function run() {
|
||||
while (next < items.length) {
|
||||
const index = next
|
||||
next += 1
|
||||
results[index] = await worker(items[index], index)
|
||||
}
|
||||
}
|
||||
const runners = Array.from({ length: Math.min(concurrency, items.length) }, () => run())
|
||||
await Promise.all(runners)
|
||||
return results
|
||||
}
|
||||
|
||||
function sampleOrcaIfPossible() {
|
||||
if (process.platform !== 'darwin') {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
const status = orcaJsonSync(['status'], { local: true }).result
|
||||
const pid = status?.app?.pid
|
||||
if (!pid) {
|
||||
return null
|
||||
}
|
||||
const out = path.join(reportDir, `orca-sample-${Date.now()}.txt`)
|
||||
const sampled = spawnSync('sample', [String(pid), '5', '-file', out], {
|
||||
timeout: 20_000,
|
||||
stdio: 'ignore'
|
||||
})
|
||||
return sampled.status === 0 ? out : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function floodCommand(marker) {
|
||||
// Continuous 2KB frames @ ~8ms — agent-like remote output.
|
||||
const script =
|
||||
"const m=process.argv[1];process.stdout.write('READY:'+m+'\\n');let f=0;const c='A'.repeat(2048);setInterval(()=>{f++;process.stdout.write('BG:'+m+':'+f+':'+c+'\\n')},8);process.stdin.resume()"
|
||||
return `node -e ${JSON.stringify(script)} ${JSON.stringify(marker)}`
|
||||
}
|
||||
|
||||
async function main() {
|
||||
mkdirSync(reportDir, { recursive: true })
|
||||
const notes = []
|
||||
const timings = new BoundedLiveFreezeHistory(120)
|
||||
const amplificationSteps = []
|
||||
|
||||
console.log(
|
||||
`[live-freeze] env=${envName} create=${createCount} passes=${switchPasses} parallel=${parallel}`
|
||||
)
|
||||
|
||||
const status = orcaJsonSync(['status'])
|
||||
notes.push(
|
||||
`remote version=${status.result?.runtime?.appVersion} state=${status.result?.runtime?.state}`
|
||||
)
|
||||
const local = orcaJsonSync(['status'], { local: true })
|
||||
notes.push(`local version=${local.result?.runtime?.appVersion} pid=${local.result?.app?.pid}`)
|
||||
|
||||
const worktrees = orcaJsonSync(['worktree', 'list']).result
|
||||
const wtList = worktrees?.worktrees || worktrees?.items || worktrees || []
|
||||
if (!Array.isArray(wtList) || wtList.length === 0) {
|
||||
throw new Error(`No worktrees on environment ${envName}`)
|
||||
}
|
||||
notes.push(`remote worktrees=${wtList.length}`)
|
||||
amplificationSteps.push(`baseline worktrees=${wtList.length}`)
|
||||
|
||||
const targets = wtList.slice(0, Math.min(createWorktreeSpan, wtList.length))
|
||||
const created = []
|
||||
|
||||
// Parallel flood-terminal creates across many worktrees.
|
||||
if (createCount > 0) {
|
||||
amplificationSteps.push(`create=${createCount} parallel=${Math.min(parallel, createCount)}`)
|
||||
const createJobs = Array.from({ length: createCount }, (_, i) => i)
|
||||
await mapPool(createJobs, Math.min(parallel, createCount), async (i) => {
|
||||
const wt = targets[i % targets.length]
|
||||
const selector = worktreeSelector(wt)
|
||||
if (!selector) {
|
||||
notes.push(`create ${i} skipped: no selector`)
|
||||
return
|
||||
}
|
||||
const marker = `LIVE_BULK_${Date.now()}_${i}`
|
||||
try {
|
||||
const createdTerm = await orcaJsonAsync(
|
||||
[
|
||||
'terminal',
|
||||
'create',
|
||||
'--worktree',
|
||||
selector,
|
||||
'--title',
|
||||
`freeze-repro-${i}`,
|
||||
'--command',
|
||||
floodCommand(marker)
|
||||
],
|
||||
{ timeoutMs: 180_000 }
|
||||
)
|
||||
timings.add({ op: 'terminal.create', ms: createdTerm.elapsedMs, ok: true, index: i })
|
||||
const handle = extractTerminalHandle(createdTerm.result)
|
||||
if (handle) {
|
||||
created.push({ handle, marker, worktree: selector })
|
||||
console.log(
|
||||
`[live-freeze] created ${handle} on ${selector} in ${createdTerm.elapsedMs.toFixed(0)}ms`
|
||||
)
|
||||
} else {
|
||||
notes.push(
|
||||
`create ${i} missing handle: ${JSON.stringify(createdTerm.result).slice(0, 400)}`
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
timings.add({ op: 'terminal.create', ms: null, ok: false, error: String(error), index: i })
|
||||
notes.push(`create ${i} failed: ${String(error).slice(0, 300)}`)
|
||||
console.warn(`[live-freeze] create failed: ${String(error)}`)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (preFloodMs > 0 && created.length > 0) {
|
||||
amplificationSteps.push(`preFloodMs=${preFloodMs}`)
|
||||
await new Promise((r) => setTimeout(r, preFloodMs))
|
||||
}
|
||||
|
||||
let live = []
|
||||
try {
|
||||
const listed = orcaJsonSync(['terminal', 'list'])
|
||||
const terms = listed.result?.terminals || []
|
||||
live = terms
|
||||
.filter(
|
||||
(t) => typeof t.handle === 'string' && t.handle.startsWith('term_') && t.connected !== false
|
||||
)
|
||||
.map((t) => ({ handle: t.handle, title: t.title, worktreeId: t.worktreeId }))
|
||||
notes.push(`live terminals listed=${live.length}`)
|
||||
} catch (error) {
|
||||
notes.push(`terminal list failed: ${String(error).slice(0, 200)}`)
|
||||
}
|
||||
|
||||
let switchTargets = [...created.map((c) => c.handle), ...live.map((t) => t.handle)].filter(
|
||||
(v, i, a) => typeof v === 'string' && a.indexOf(v) === i
|
||||
)
|
||||
|
||||
if (shouldCapSwitchTargets(maxSwitchTargets) && switchTargets.length > maxSwitchTargets) {
|
||||
switchTargets = applySwitchTargetCap(switchTargets, maxSwitchTargets)
|
||||
amplificationSteps.push(`capped switchTargets=${maxSwitchTargets}`)
|
||||
}
|
||||
|
||||
if (switchTargets.length < 2) {
|
||||
throw new Error(
|
||||
`Need ≥2 terminals to bulk-switch; got ${switchTargets.length}. notes=${notes.join('; ')}`
|
||||
)
|
||||
}
|
||||
|
||||
amplificationSteps.push(
|
||||
`switchTargets=${switchTargets.length} passes=${switchPasses} parallel=${parallel}`
|
||||
)
|
||||
console.log(
|
||||
`[live-freeze] bulk-switching ${switchTargets.length} terminals × ${switchPasses} passes (parallel=${parallel})`
|
||||
)
|
||||
|
||||
let maxSwitchMs = 0
|
||||
let maxBatchWallMs = 0
|
||||
let sumSwitchMs = 0
|
||||
let switchCount = 0
|
||||
const switchStarted = performance.now()
|
||||
|
||||
for (let pass = 0; pass < switchPasses; pass += 1) {
|
||||
// Chunk targets into concurrent batches — piles load onto client/UI path.
|
||||
for (let offset = 0; offset < switchTargets.length; offset += parallel) {
|
||||
const batch = switchTargets.slice(offset, offset + parallel)
|
||||
const batchStarted = performance.now()
|
||||
const batchResults = await Promise.all(
|
||||
batch.map(async (handle) => {
|
||||
try {
|
||||
const sw = await orcaJsonAsync(['terminal', 'switch', '--terminal', handle], {
|
||||
timeoutMs: 90_000
|
||||
})
|
||||
return { handle, ms: sw.elapsedMs, ok: true }
|
||||
} catch (error) {
|
||||
return { handle, error: String(error), ok: false }
|
||||
}
|
||||
})
|
||||
)
|
||||
const batchWall = performance.now() - batchStarted
|
||||
maxBatchWallMs = Math.max(maxBatchWallMs, batchWall)
|
||||
for (const item of batchResults) {
|
||||
if (item.ok) {
|
||||
maxSwitchMs = Math.max(maxSwitchMs, item.ms)
|
||||
sumSwitchMs += item.ms
|
||||
switchCount += 1
|
||||
timings.add({ op: 'terminal.switch', handle: item.handle, ms: item.ms, batchWall })
|
||||
if (item.ms >= softMs) {
|
||||
console.warn(`[live-freeze] SOFT lag on switch ${item.handle}: ${item.ms.toFixed(0)}ms`)
|
||||
}
|
||||
if (item.ms >= hardMs) {
|
||||
console.warn(`[live-freeze] HARD lag on switch ${item.handle}: ${item.ms.toFixed(0)}ms`)
|
||||
}
|
||||
} else {
|
||||
timings.add({ op: 'terminal.switch', handle: item.handle, error: item.error })
|
||||
notes.push(`switch ${item.handle} failed: ${String(item.error).slice(0, 200)}`)
|
||||
}
|
||||
}
|
||||
if (batchWall >= softMs) {
|
||||
console.warn(`[live-freeze] SOFT batch wall=${batchWall.toFixed(0)}ms size=${batch.length}`)
|
||||
}
|
||||
if (batchWall >= hardMs) {
|
||||
console.warn(`[live-freeze] HARD batch wall=${batchWall.toFixed(0)}ms size=${batch.length}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const bulkWallMs = performance.now() - switchStarted
|
||||
const avgSwitchMs = switchCount ? sumSwitchMs / switchCount : 0
|
||||
|
||||
const statusProbe = orcaJsonSync(['status'], { local: true })
|
||||
let memoryProbeMs = null
|
||||
try {
|
||||
const mem = orcaJsonSync(['diagnostics', 'memory'], { local: true, timeoutMs: 120_000 })
|
||||
memoryProbeMs = mem.elapsedMs
|
||||
notes.push(`memory diagnostic ms=${mem.elapsedMs.toFixed(0)}`)
|
||||
} catch (error) {
|
||||
notes.push(`memory diagnostic failed: ${String(error).slice(0, 200)}`)
|
||||
}
|
||||
|
||||
const { peakLatencyMs, softFreeze, hardFreeze } = evaluateFreezeSignals({
|
||||
maxSwitchMs,
|
||||
maxBatchWallMs,
|
||||
statusProbeMs: statusProbe.elapsedMs,
|
||||
memoryProbeMs,
|
||||
softMs,
|
||||
hardMs
|
||||
})
|
||||
|
||||
let samplePath = null
|
||||
if (softFreeze || hardFreeze) {
|
||||
samplePath = sampleOrcaIfPossible()
|
||||
if (samplePath) {
|
||||
notes.push(`sample=${samplePath}`)
|
||||
} else {
|
||||
notes.push('sample unavailable')
|
||||
}
|
||||
}
|
||||
|
||||
const report = {
|
||||
topology: 'live-paired-remote',
|
||||
environment: envName,
|
||||
localVersion: local.result?.runtime?.appVersion,
|
||||
remoteVersion: status.result?.runtime?.appVersion,
|
||||
remoteWorktreeCount: wtList.length,
|
||||
createdTerminals: created.length,
|
||||
switchTargets: switchTargets.length,
|
||||
switchPasses,
|
||||
parallel,
|
||||
maxSwitchMs,
|
||||
maxBatchWallMs,
|
||||
peakLatencyMs,
|
||||
avgSwitchMs,
|
||||
bulkWallMs,
|
||||
statusProbeMs: statusProbe.elapsedMs,
|
||||
memoryProbeMs,
|
||||
softFreeze,
|
||||
hardFreeze,
|
||||
softMs,
|
||||
hardMs,
|
||||
amplificationSteps,
|
||||
notes,
|
||||
timingCount: timings.totalCount,
|
||||
timings: timings.values()
|
||||
}
|
||||
|
||||
const outPath = path.join(reportDir, `live-bulk-open-freeze-${envName}.json`)
|
||||
writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`)
|
||||
// Also write a stamped peak report so amplification runs don't overwrite history.
|
||||
const stamped = path.join(reportDir, `live-bulk-open-freeze-${envName}-peak-${Date.now()}.json`)
|
||||
writeFileSync(stamped, `${JSON.stringify(report, null, 2)}\n`)
|
||||
console.log(`[live-freeze] report ${outPath}`)
|
||||
console.log(`[live-freeze] stamped ${stamped}`)
|
||||
console.log(JSON.stringify(report, null, 2))
|
||||
|
||||
if (scratchDir) {
|
||||
try {
|
||||
mkdirSync(scratchDir, { recursive: true })
|
||||
copyFileSync(outPath, path.join(scratchDir, 'live-bulk-open-freeze-report.json'))
|
||||
writeFileSync(
|
||||
path.join(scratchDir, 'live-freeze-amplify-summary.json'),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
peakLatencyMs,
|
||||
hardFreeze,
|
||||
softFreeze,
|
||||
amplificationSteps,
|
||||
stamped
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}\n`
|
||||
)
|
||||
} catch (error) {
|
||||
notes.push(`scratch copy failed: ${String(error).slice(0, 200)}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (hardFreeze) {
|
||||
process.exitCode = 2
|
||||
console.error('[live-freeze] HARD FREEZE SIGNAL')
|
||||
} else if (softFreeze) {
|
||||
process.exitCode = 1
|
||||
console.error('[live-freeze] SOFT FREEZE SIGNAL')
|
||||
} else {
|
||||
console.log('[live-freeze] no freeze signal under thresholds')
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('[live-freeze] failed', error)
|
||||
process.exit(3)
|
||||
})
|
||||
|
|
@ -0,0 +1,241 @@
|
|||
import { spawn, spawnSync } from 'node:child_process'
|
||||
import path from 'node:path'
|
||||
|
||||
export const MAX_ORCA_RPC_OUTPUT_BYTES = 20 * 1024 * 1024
|
||||
|
||||
export function appendOrcaRpcOutput(output, chunk, bytes, limit = MAX_ORCA_RPC_OUTPUT_BYTES) {
|
||||
const nextBytes = bytes + Buffer.byteLength(chunk)
|
||||
return {
|
||||
output: nextBytes > limit ? output : output + chunk,
|
||||
bytes: nextBytes,
|
||||
exceeded: nextBytes > limit
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveOrcaCliCommand({ env = process.env, platform = process.platform } = {}) {
|
||||
if (env.ORCA_CLI_COMMAND?.trim()) {
|
||||
return env.ORCA_CLI_COMMAND.trim()
|
||||
}
|
||||
if (env.ORCA_DEV_REPO_ROOT) {
|
||||
return 'orca-dev'
|
||||
}
|
||||
return platform === 'linux' ? 'orca-ide' : 'orca'
|
||||
}
|
||||
|
||||
export function resolveOrcaCliInvocation({
|
||||
env = process.env,
|
||||
platform = process.platform,
|
||||
nodeExecutable = process.execPath
|
||||
} = {}) {
|
||||
const command = resolveOrcaCliCommand({ env, platform })
|
||||
const commandName = platform === 'win32' ? path.win32.basename(command).toLowerCase() : command
|
||||
if (
|
||||
platform === 'win32' &&
|
||||
env.ORCA_DEV_REPO_ROOT &&
|
||||
(commandName === 'orca-dev' || commandName === 'orca-dev.cmd')
|
||||
) {
|
||||
const defaultUserDataPath = path.win32.join(
|
||||
env.APPDATA ?? path.win32.join(env.USERPROFILE ?? '', 'AppData', 'Roaming'),
|
||||
'orca-dev'
|
||||
)
|
||||
return {
|
||||
command: nodeExecutable,
|
||||
prefixArgs: [path.win32.join(env.ORCA_DEV_REPO_ROOT, 'out', 'cli', 'index.js')],
|
||||
env: {
|
||||
...env,
|
||||
ORCA_USER_DATA_PATH:
|
||||
env.ORCA_USER_DATA_PATH ?? env.ORCA_DEV_USER_DATA_PATH ?? defaultUserDataPath,
|
||||
ORCA_DEV_CLI_INVOCATION: '1',
|
||||
ORCA_APP_EXECUTABLE:
|
||||
env.ORCA_APP_EXECUTABLE ??
|
||||
path.win32.join(
|
||||
env.ORCA_DEV_REPO_ROOT,
|
||||
'node_modules',
|
||||
'electron',
|
||||
'dist',
|
||||
'electron.exe'
|
||||
),
|
||||
ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT: '1'
|
||||
}
|
||||
}
|
||||
}
|
||||
return { command, prefixArgs: [] }
|
||||
}
|
||||
|
||||
export function createOrcaRpc({
|
||||
envName,
|
||||
cliCommand,
|
||||
env = process.env,
|
||||
platform = process.platform
|
||||
}) {
|
||||
const cliInvocation = cliCommand
|
||||
? { command: cliCommand, prefixArgs: [] }
|
||||
: resolveOrcaCliInvocation({ env, platform })
|
||||
const commandLabel = cliCommand ?? resolveOrcaCliCommand({ env, platform })
|
||||
const commandArgs = (args, local) => [
|
||||
...cliInvocation.prefixArgs,
|
||||
...args,
|
||||
...(local ? [] : ['--environment', envName]),
|
||||
'--json'
|
||||
]
|
||||
|
||||
function orcaJsonSync(args, opts = {}) {
|
||||
const started = performance.now()
|
||||
const result = spawnSync(cliInvocation.command, commandArgs(args, opts.local), {
|
||||
encoding: 'utf8',
|
||||
env: cliInvocation.env,
|
||||
maxBuffer: MAX_ORCA_RPC_OUTPUT_BYTES,
|
||||
timeout: opts.timeoutMs ?? 120_000
|
||||
})
|
||||
const elapsedMs = performance.now() - started
|
||||
if (result.error) {
|
||||
throw new Error(`${commandLabel} ${args.join(' ')} failed to start: ${String(result.error)}`)
|
||||
}
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
`${commandLabel} ${args.join(' ')} failed (${result.status}): ${result.stderr || result.stdout}`
|
||||
)
|
||||
}
|
||||
const parsed = JSON.parse(result.stdout)
|
||||
if (parsed.ok === false) {
|
||||
throw new Error(`${commandLabel} ${args.join(' ')} ok=false: ${JSON.stringify(parsed)}`)
|
||||
}
|
||||
return { parsed, elapsedMs, result: parsed.result }
|
||||
}
|
||||
|
||||
function orcaJsonAsync(args, opts = {}) {
|
||||
const started = performance.now()
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(cliInvocation.command, commandArgs(args, opts.local), {
|
||||
env: cliInvocation.env,
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
})
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
let outputBytes = 0
|
||||
let settled = false
|
||||
let timer
|
||||
const fail = (error) => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
clearTimeout(timer)
|
||||
reject(error)
|
||||
}
|
||||
const append = (stream, chunk) => {
|
||||
if (settled) {
|
||||
return stream
|
||||
}
|
||||
const appended = appendOrcaRpcOutput(stream, chunk, outputBytes)
|
||||
outputBytes = appended.bytes
|
||||
if (appended.exceeded) {
|
||||
child.kill('SIGKILL')
|
||||
fail(new Error(`${commandLabel} ${args.join(' ')} exceeded 20 MiB output limit`))
|
||||
return stream
|
||||
}
|
||||
return appended.output
|
||||
}
|
||||
timer = setTimeout(() => {
|
||||
child.kill('SIGKILL')
|
||||
fail(
|
||||
new Error(
|
||||
`${commandLabel} ${args.join(' ')} timed out after ${opts.timeoutMs ?? 120_000}ms`
|
||||
)
|
||||
)
|
||||
}, opts.timeoutMs ?? 120_000)
|
||||
child.stdout.setEncoding('utf8')
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stdout.on('data', (chunk) => {
|
||||
stdout = append(stdout, chunk)
|
||||
})
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr = append(stderr, chunk)
|
||||
})
|
||||
child.on('error', fail)
|
||||
child.on('close', (code) => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
clearTimeout(timer)
|
||||
const elapsedMs = performance.now() - started
|
||||
if (code !== 0) {
|
||||
fail(
|
||||
new Error(
|
||||
`${commandLabel} ${args.join(' ')} failed (${code}): ${stderr || stdout}`.slice(
|
||||
0,
|
||||
800
|
||||
)
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(stdout)
|
||||
if (parsed.ok === false) {
|
||||
fail(
|
||||
new Error(
|
||||
`${commandLabel} ${args.join(' ')} ok=false: ${JSON.stringify(parsed)}`.slice(
|
||||
0,
|
||||
800
|
||||
)
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
resolve({ parsed, elapsedMs, result: parsed.result })
|
||||
} catch (error) {
|
||||
fail(
|
||||
new Error(
|
||||
`${commandLabel} parse failed: ${String(error)}; stdout=${stdout.slice(0, 400)}`
|
||||
)
|
||||
)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function runReconnectRefreshStorm(notes) {
|
||||
const started = performance.now()
|
||||
const jobs = [
|
||||
() => orcaJsonAsync(['status'], { timeoutMs: 90_000 }),
|
||||
() => orcaJsonAsync(['worktree', 'list'], { timeoutMs: 120_000 }),
|
||||
() => orcaJsonAsync(['terminal', 'list'], { timeoutMs: 120_000 }),
|
||||
() => orcaJsonAsync(['status'], { local: true, timeoutMs: 60_000 }),
|
||||
() => orcaJsonAsync(['worktree', 'list'], { timeoutMs: 120_000 }),
|
||||
() => orcaJsonAsync(['terminal', 'list'], { timeoutMs: 120_000 })
|
||||
]
|
||||
const results = await Promise.all(
|
||||
jobs.map(async (job, index) => {
|
||||
try {
|
||||
const result = await job()
|
||||
return { index, ok: true, ms: result.elapsedMs }
|
||||
} catch (error) {
|
||||
notes.push(`reconnect-refresh job ${index} failed: ${String(error).slice(0, 200)}`)
|
||||
return { index, ok: false, ms: null, error: String(error) }
|
||||
}
|
||||
})
|
||||
)
|
||||
const wallMs = performance.now() - started
|
||||
const maxJobMs = Math.max(0, ...results.map((result) => result.ms || 0))
|
||||
notes.push(
|
||||
`reconnect-refresh wall=${wallMs.toFixed(0)}ms maxJob=${maxJobMs.toFixed(0)}ms ok=${results.filter((result) => result.ok).length}/${results.length}`
|
||||
)
|
||||
return { wallMs, maxJobMs, results }
|
||||
}
|
||||
|
||||
async function runRestartProxy(notes) {
|
||||
const started = performance.now()
|
||||
try {
|
||||
const opened = await orcaJsonAsync(['open'], { local: true, timeoutMs: 120_000 })
|
||||
notes.push(`orca open ms=${opened.elapsedMs.toFixed(0)}`)
|
||||
} catch (error) {
|
||||
notes.push(`orca open failed: ${String(error).slice(0, 200)}`)
|
||||
}
|
||||
const storm = await runReconnectRefreshStorm(notes)
|
||||
return { wallMs: performance.now() - started, storm }
|
||||
}
|
||||
|
||||
return { orcaJsonSync, orcaJsonAsync, runReconnectRefreshStorm, runRestartProxy }
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
appendOrcaRpcOutput,
|
||||
resolveOrcaCliCommand,
|
||||
resolveOrcaCliInvocation
|
||||
} from './live-remote-freeze-rpc.mjs'
|
||||
|
||||
describe('live remote freeze RPC', () => {
|
||||
it('resolves the Orca CLI for managed, dev, Linux, and default runtimes', () => {
|
||||
expect(resolveOrcaCliCommand({ env: { ORCA_CLI_COMMAND: 'custom-orca' } })).toBe('custom-orca')
|
||||
expect(resolveOrcaCliCommand({ env: { ORCA_DEV_REPO_ROOT: '/repo' } })).toBe('orca-dev')
|
||||
expect(resolveOrcaCliCommand({ env: {}, platform: 'linux' })).toBe('orca-ide')
|
||||
expect(resolveOrcaCliCommand({ env: {}, platform: 'win32' })).toBe('orca')
|
||||
})
|
||||
|
||||
it('bypasses the Windows dev cmd shim with the built Node CLI', () => {
|
||||
const invocation = resolveOrcaCliInvocation({
|
||||
env: {
|
||||
APPDATA: 'C:\\Users\\dev\\AppData\\Roaming',
|
||||
ORCA_CLI_COMMAND: 'C:\\repo\\out\\bin\\orca-dev.cmd',
|
||||
ORCA_DEV_REPO_ROOT: 'C:\\repo'
|
||||
},
|
||||
platform: 'win32',
|
||||
nodeExecutable: 'C:\\Program Files\\nodejs\\node.exe'
|
||||
})
|
||||
|
||||
expect(invocation).toMatchObject({
|
||||
command: 'C:\\Program Files\\nodejs\\node.exe',
|
||||
prefixArgs: ['C:\\repo\\out\\cli\\index.js'],
|
||||
env: {
|
||||
ORCA_USER_DATA_PATH: 'C:\\Users\\dev\\AppData\\Roaming\\orca-dev',
|
||||
ORCA_DEV_CLI_INVOCATION: '1',
|
||||
ORCA_APP_EXECUTABLE: 'C:\\repo\\node_modules\\electron\\dist\\electron.exe',
|
||||
ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT: '1'
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('caps combined asynchronous output before retaining the overflow chunk', () => {
|
||||
const first = appendOrcaRpcOutput('', '1234', 0, 5)
|
||||
expect(first).toEqual({ output: '1234', bytes: 4, exceeded: false })
|
||||
|
||||
const overflow = appendOrcaRpcOutput(first.output, '67', first.bytes, 5)
|
||||
expect(overflow).toEqual({ output: '1234', bytes: 6, exceeded: true })
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,645 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Naturalistic freeze repro — idle/reconnect recovery stories on large remotes.
|
||||
*
|
||||
* Unlike the bulk parallel-switch amplifier, this models:
|
||||
* 1) agents streaming on remote while user is idle (backlog builds)
|
||||
* 2) user returns and opens sessions one-by-one (or after reconnect refresh)
|
||||
*
|
||||
* Scenarios:
|
||||
* idle-backlog-open — idle with flood, then human-paced sequential open
|
||||
* idle-backlog-reconnect-open — same + wake-like metadata refresh storm, then open
|
||||
* restart-proxy — idle, then orca open + status/list storm + open
|
||||
* (does NOT kill the desktop; proxies restore work)
|
||||
*
|
||||
* Usage:
|
||||
* ORCA_FREEZE_ENV=paired-remote ORCA_FREEZE_SCENARIO=idle-backlog-open \
|
||||
* node config/scripts/live-remote-realistic-freeze-repro.mjs
|
||||
*
|
||||
* pnpm run repro:live-remote-realistic-freeze
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { copyFileSync, mkdirSync, writeFileSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { createOrcaRpc } from './live-remote-freeze-rpc.mjs'
|
||||
import { startStatusWatchdog } from './live-remote-status-watchdog.mjs'
|
||||
import { BoundedLiveFreezeHistory } from './live-freeze-bounded-history.mjs'
|
||||
import {
|
||||
DEFAULT_FOREVER_WINDOW_MS,
|
||||
DEFAULT_HARD_MS,
|
||||
DEFAULT_SOFT_MS,
|
||||
DEFAULT_STATUS_SLOW_MS,
|
||||
evaluateFullAppFreeze,
|
||||
evaluatePermanentLockup,
|
||||
evaluateRealisticFreezeSignals,
|
||||
extractTerminalHandle,
|
||||
humanPaceDelayMs,
|
||||
readFreezeNumberEnv,
|
||||
REALISTIC_SCENARIOS,
|
||||
worktreeSelector
|
||||
} from './live-remote-bulk-open-freeze-metrics.mjs'
|
||||
|
||||
const root = path.resolve(import.meta.dirname, '../..')
|
||||
const reportDir = path.join(root, 'test-results', 'freeze-repro')
|
||||
const envName = process.env.ORCA_FREEZE_ENV || 'paired-remote'
|
||||
const scenario = process.env.ORCA_FREEZE_SCENARIO || 'idle-backlog-open'
|
||||
const createCount = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_CREATE', 0))
|
||||
const openCount = Math.max(2, readFreezeNumberEnv('ORCA_FREEZE_OPEN_COUNT', 20))
|
||||
const idleMs = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_IDLE_MS', 45_000))
|
||||
const paceMs = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_PACE_MS', 250))
|
||||
const paceJitterMs = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_PACE_JITTER_MS', 150))
|
||||
const createWorktreeSpan = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_CREATE_WT_SPAN', 12))
|
||||
const softMs = readFreezeNumberEnv('ORCA_FREEZE_SOFT_MS', DEFAULT_SOFT_MS)
|
||||
const hardMs = readFreezeNumberEnv('ORCA_FREEZE_HARD_MS', DEFAULT_HARD_MS)
|
||||
/** Concurrent opens during lockup-storm (wake refresh overlaps fan-out). */
|
||||
const stormParallel = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_STORM_PARALLEL', 16))
|
||||
/** Kill a switch if it exceeds this — counts toward permanent lockup. */
|
||||
const opTimeoutMs = Math.max(10_000, readFreezeNumberEnv('ORCA_FREEZE_OP_TIMEOUT_MS', 60_000))
|
||||
const permanentTimeoutMs = Math.max(15_000, readFreezeNumberEnv('ORCA_FREEZE_PERMANENT_MS', 60_000))
|
||||
const foreverWindowMs = Math.max(
|
||||
10_000,
|
||||
readFreezeNumberEnv('ORCA_FREEZE_FOREVER_WINDOW_MS', DEFAULT_FOREVER_WINDOW_MS)
|
||||
)
|
||||
const statusSlowMs = Math.max(
|
||||
5_000,
|
||||
readFreezeNumberEnv('ORCA_FREEZE_STATUS_SLOW_MS', DEFAULT_STATUS_SLOW_MS)
|
||||
)
|
||||
const watchdogIntervalMs = Math.max(
|
||||
500,
|
||||
readFreezeNumberEnv('ORCA_FREEZE_WATCHDOG_INTERVAL_MS', 1500)
|
||||
)
|
||||
const scratchDir = process.env.ORCA_FREEZE_SCRATCH || ''
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
const rpc = createOrcaRpc({ envName })
|
||||
const { orcaJsonSync, orcaJsonAsync, runReconnectRefreshStorm, runRestartProxy } = rpc
|
||||
|
||||
async function mapPool(items, concurrency, worker) {
|
||||
const results = Array.from({ length: items.length })
|
||||
let next = 0
|
||||
async function run() {
|
||||
while (next < items.length) {
|
||||
const index = next
|
||||
next += 1
|
||||
results[index] = await worker(items[index], index)
|
||||
}
|
||||
}
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(concurrency, Math.max(items.length, 1)) }, () => run())
|
||||
)
|
||||
return results
|
||||
}
|
||||
|
||||
function floodCommand(marker) {
|
||||
const script =
|
||||
"const m=process.argv[1];process.stdout.write('READY:'+m+'\n');let f=0;const c='A'.repeat(2048);setInterval(()=>{f++;process.stdout.write('BG:'+m+':'+f+':'+c+'\n')},8);process.stdin.resume()"
|
||||
return `node -e ${JSON.stringify(script)} ${JSON.stringify(marker)}`
|
||||
}
|
||||
|
||||
function sampleOrcaIfPossible() {
|
||||
if (process.platform !== 'darwin') {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
const status = orcaJsonSync(['status'], { local: true }).result
|
||||
const pid = status?.app?.pid
|
||||
if (!pid) {
|
||||
return null
|
||||
}
|
||||
const out = path.join(reportDir, `orca-sample-realistic-${Date.now()}.txt`)
|
||||
const sampled = spawnSync('sample', [String(pid), '5', '-file', out], {
|
||||
timeout: 20_000,
|
||||
stdio: 'ignore'
|
||||
})
|
||||
return sampled.status === 0 ? out : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function listLiveTerminalHandles() {
|
||||
const listed = orcaJsonSync(['terminal', 'list'])
|
||||
const terms = listed.result?.terminals || []
|
||||
return terms
|
||||
.filter((t) => typeof t.handle === 'string' && t.handle.startsWith('term_'))
|
||||
.map((t) => ({
|
||||
handle: t.handle,
|
||||
title: t.title,
|
||||
worktreeId: t.worktreeId,
|
||||
connected: t.connected
|
||||
}))
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (!REALISTIC_SCENARIOS.includes(scenario)) {
|
||||
throw new Error(
|
||||
`Unknown ORCA_FREEZE_SCENARIO=${scenario}. Expected one of: ${REALISTIC_SCENARIOS.join(', ')}`
|
||||
)
|
||||
}
|
||||
|
||||
mkdirSync(reportDir, { recursive: true })
|
||||
const notes = []
|
||||
const phases = []
|
||||
const openTimings = new BoundedLiveFreezeHistory(100)
|
||||
|
||||
console.log(
|
||||
`[realistic-freeze] scenario=${scenario} env=${envName} create=${createCount} idleMs=${idleMs} openCount=${openCount} paceMs=${paceMs}`
|
||||
)
|
||||
|
||||
const local = orcaJsonSync(['status'], { local: true })
|
||||
const remote = orcaJsonSync(['status'])
|
||||
notes.push(
|
||||
`local version=${local.result?.runtime?.appVersion} pid=${local.result?.app?.pid}`,
|
||||
`remote version=${remote.result?.runtime?.appVersion} state=${remote.result?.runtime?.state}`
|
||||
)
|
||||
|
||||
const worktrees = orcaJsonSync(['worktree', 'list']).result
|
||||
const wtList = worktrees?.worktrees || worktrees?.items || worktrees || []
|
||||
if (!Array.isArray(wtList) || wtList.length === 0) {
|
||||
throw new Error(`No worktrees on environment ${envName}`)
|
||||
}
|
||||
notes.push(`remote worktrees=${wtList.length}`)
|
||||
phases.push({ phase: 'baseline', worktrees: wtList.length })
|
||||
|
||||
// --- Phase: seed flood terminals (agent-like backlog sources) ---
|
||||
const created = []
|
||||
if (createCount > 0) {
|
||||
const targets = wtList.slice(0, Math.min(createWorktreeSpan, wtList.length))
|
||||
await mapPool(
|
||||
Array.from({ length: createCount }, (_, i) => i),
|
||||
Math.min(4, createCount),
|
||||
async (i) => {
|
||||
const wt = targets[i % targets.length]
|
||||
const selector = worktreeSelector(wt)
|
||||
if (!selector) {
|
||||
return
|
||||
}
|
||||
const marker = `REALISTIC_${Date.now()}_${i}`
|
||||
try {
|
||||
const createdTerm = await orcaJsonAsync(
|
||||
[
|
||||
'terminal',
|
||||
'create',
|
||||
'--worktree',
|
||||
selector,
|
||||
'--title',
|
||||
`realistic-freeze-${i}`,
|
||||
'--command',
|
||||
floodCommand(marker)
|
||||
],
|
||||
{ timeoutMs: 180_000 }
|
||||
)
|
||||
const handle = extractTerminalHandle(createdTerm.result)
|
||||
if (handle) {
|
||||
created.push({ handle, marker, worktree: selector })
|
||||
console.log(
|
||||
`[realistic-freeze] flood terminal ${handle} (${createdTerm.elapsedMs.toFixed(0)}ms)`
|
||||
)
|
||||
} else {
|
||||
notes.push(
|
||||
`create ${i} missing handle: ${JSON.stringify(createdTerm.result).slice(0, 300)}`
|
||||
)
|
||||
}
|
||||
} catch (error) {
|
||||
notes.push(`create ${i} failed: ${String(error).slice(0, 250)}`)
|
||||
console.warn(`[realistic-freeze] create failed: ${String(error)}`)
|
||||
}
|
||||
}
|
||||
)
|
||||
phases.push({ phase: 'seed-flood', created: created.length })
|
||||
}
|
||||
|
||||
// Prefer created floods for open pass; fill with existing live terminals.
|
||||
let live = []
|
||||
try {
|
||||
live = listLiveTerminalHandles()
|
||||
notes.push(`live terminals listed=${live.length}`)
|
||||
} catch (error) {
|
||||
notes.push(`terminal list failed: ${String(error).slice(0, 200)}`)
|
||||
}
|
||||
|
||||
const openTargets = [...created.map((c) => c.handle), ...live.map((t) => t.handle)].filter(
|
||||
(v, i, a) => typeof v === 'string' && a.indexOf(v) === i
|
||||
)
|
||||
|
||||
if (openTargets.length < 2) {
|
||||
throw new Error(`Need ≥2 terminals; got ${openTargets.length}. ${notes.join('; ')}`)
|
||||
}
|
||||
|
||||
const openList = openTargets.slice(0, Math.min(openCount, openTargets.length))
|
||||
|
||||
// --- Phase: park — leave one session focused, rest accumulate flood while "away" ---
|
||||
try {
|
||||
const parkHandle = openList[0]
|
||||
const parked = await orcaJsonAsync(['terminal', 'switch', '--terminal', parkHandle], {
|
||||
timeoutMs: 60_000
|
||||
})
|
||||
notes.push(`park switch ms=${parked.elapsedMs.toFixed(0)} handle=${parkHandle}`)
|
||||
} catch (error) {
|
||||
notes.push(`park switch failed: ${String(error).slice(0, 200)}`)
|
||||
}
|
||||
|
||||
console.log(`[realistic-freeze] idle ${idleMs}ms while remotes stream (user away / asleep)`)
|
||||
const idleStarted = performance.now()
|
||||
await sleep(idleMs)
|
||||
phases.push({ phase: 'idle', idleMs, actualMs: performance.now() - idleStarted })
|
||||
|
||||
// --- Phase: recovery trigger ---
|
||||
let reconnectRefreshMs = 0
|
||||
let timedOutOps = 0
|
||||
let consecutiveSwitchFailures = 0
|
||||
let maxConsecutiveSwitchFailures = 0
|
||||
|
||||
if (scenario === 'idle-backlog-reconnect-open' || scenario === 'lockup-storm') {
|
||||
console.log(
|
||||
'[realistic-freeze] wake/reconnect proxy: parallel status/worktree/terminal refresh'
|
||||
)
|
||||
const storm = await runReconnectRefreshStorm(notes)
|
||||
reconnectRefreshMs = Math.max(storm.wallMs, storm.maxJobMs)
|
||||
phases.push({
|
||||
phase: 'reconnect-refresh',
|
||||
wallMs: storm.wallMs,
|
||||
maxJobMs: storm.maxJobMs
|
||||
})
|
||||
} else if (scenario === 'restart-proxy') {
|
||||
console.log('[realistic-freeze] restart proxy: orca open + refresh storm (no process kill)')
|
||||
const restart = await runRestartProxy(notes)
|
||||
reconnectRefreshMs = Math.max(restart.wallMs, restart.storm.wallMs, restart.storm.maxJobMs)
|
||||
phases.push({
|
||||
phase: 'restart-proxy',
|
||||
wallMs: restart.wallMs,
|
||||
reconnectWallMs: restart.storm.wallMs
|
||||
})
|
||||
}
|
||||
|
||||
// --- Phase: open sessions ---
|
||||
// lockup-storm: overlap a second reconnect storm with concurrent switch fan-out
|
||||
// (models wake + bulk session restore, not human serial clicks).
|
||||
let maxOpenMs = 0
|
||||
let firstOpenMs = 0
|
||||
let sumOpenMs = 0
|
||||
let openOk = 0
|
||||
let maxBatchWallMs = 0
|
||||
const openStarted = performance.now()
|
||||
|
||||
let statusWatch = null
|
||||
if (scenario === 'lockup-storm') {
|
||||
console.log(
|
||||
`[realistic-freeze] LOCKUP STORM: concurrent open parallel=${stormParallel} + overlapping reconnect refresh (timeout=${opTimeoutMs}ms); mid-storm status watchdog every ${watchdogIntervalMs}ms`
|
||||
)
|
||||
statusWatch = startStatusWatchdog({
|
||||
intervalMs: watchdogIntervalMs,
|
||||
timeoutMs: Math.min(permanentTimeoutMs, foreverWindowMs),
|
||||
statusSlowMs
|
||||
})
|
||||
// Fire reconnect storm again concurrently with first open wave.
|
||||
const overlapStormPromise = runReconnectRefreshStorm(notes)
|
||||
for (let offset = 0; offset < openList.length; offset += stormParallel) {
|
||||
const batch = openList.slice(offset, offset + stormParallel)
|
||||
const batchStarted = performance.now()
|
||||
const batchResults = await Promise.all(
|
||||
batch.map(async (handle, batchIndex) => {
|
||||
const index = offset + batchIndex
|
||||
try {
|
||||
const sw = await orcaJsonAsync(['terminal', 'switch', '--terminal', handle], {
|
||||
timeoutMs: opTimeoutMs
|
||||
})
|
||||
return { handle, index, ms: sw.elapsedMs, ok: true, timedOut: false }
|
||||
} catch (error) {
|
||||
const msg = String(error)
|
||||
const timedOut = /timed out/i.test(msg)
|
||||
return { handle, index, error: msg, ok: false, timedOut }
|
||||
}
|
||||
})
|
||||
)
|
||||
const batchWall = performance.now() - batchStarted
|
||||
maxBatchWallMs = Math.max(maxBatchWallMs, batchWall)
|
||||
for (const item of batchResults) {
|
||||
if (item.ok) {
|
||||
openOk += 1
|
||||
sumOpenMs += item.ms
|
||||
maxOpenMs = Math.max(maxOpenMs, item.ms)
|
||||
if (item.index === 0 || firstOpenMs === 0) {
|
||||
firstOpenMs = item.ms
|
||||
}
|
||||
consecutiveSwitchFailures = 0
|
||||
openTimings.add({
|
||||
handle: item.handle,
|
||||
ms: item.ms,
|
||||
index: item.index,
|
||||
batchWall
|
||||
})
|
||||
if (item.ms >= hardMs) {
|
||||
console.warn(
|
||||
`[realistic-freeze] HARD open #${item.index} ${item.handle}: ${item.ms.toFixed(0)}ms`
|
||||
)
|
||||
}
|
||||
} else {
|
||||
if (item.timedOut) {
|
||||
timedOutOps += 1
|
||||
}
|
||||
consecutiveSwitchFailures += 1
|
||||
maxConsecutiveSwitchFailures = Math.max(
|
||||
maxConsecutiveSwitchFailures,
|
||||
consecutiveSwitchFailures
|
||||
)
|
||||
openTimings.add({
|
||||
handle: item.handle,
|
||||
error: item.error,
|
||||
index: item.index,
|
||||
timedOut: item.timedOut
|
||||
})
|
||||
notes.push(
|
||||
`open ${item.handle} failed${item.timedOut ? ' (TIMEOUT)' : ''}: ${String(item.error).slice(0, 160)}`
|
||||
)
|
||||
console.warn(
|
||||
`[realistic-freeze] open FAIL #${item.index}${item.timedOut ? ' TIMEOUT' : ''}: ${item.handle}`
|
||||
)
|
||||
}
|
||||
}
|
||||
if (batchWall >= hardMs) {
|
||||
console.warn(
|
||||
`[realistic-freeze] HARD batch wall=${batchWall.toFixed(0)}ms size=${batch.length}`
|
||||
)
|
||||
}
|
||||
}
|
||||
try {
|
||||
const overlap = await overlapStormPromise
|
||||
reconnectRefreshMs = Math.max(reconnectRefreshMs, overlap.wallMs, overlap.maxJobMs)
|
||||
phases.push({
|
||||
phase: 'overlap-reconnect-refresh',
|
||||
wallMs: overlap.wallMs,
|
||||
maxJobMs: overlap.maxJobMs
|
||||
})
|
||||
} catch (error) {
|
||||
notes.push(`overlap reconnect failed: ${String(error).slice(0, 200)}`)
|
||||
}
|
||||
phases.push({
|
||||
phase: 'lockup-storm-open',
|
||||
count: openList.length,
|
||||
ok: openOk,
|
||||
maxOpenMs,
|
||||
firstOpenMs,
|
||||
maxBatchWallMs,
|
||||
timedOutOps,
|
||||
parallel: stormParallel
|
||||
})
|
||||
} else {
|
||||
console.log(
|
||||
`[realistic-freeze] human-paced open of ${openList.length} sessions (pace≈${paceMs}ms + jitter)`
|
||||
)
|
||||
for (let i = 0; i < openList.length; i += 1) {
|
||||
const handle = openList[i]
|
||||
try {
|
||||
const sw = await orcaJsonAsync(['terminal', 'switch', '--terminal', handle], {
|
||||
timeoutMs: opTimeoutMs
|
||||
})
|
||||
openOk += 1
|
||||
sumOpenMs += sw.elapsedMs
|
||||
maxOpenMs = Math.max(maxOpenMs, sw.elapsedMs)
|
||||
if (i === 0) {
|
||||
firstOpenMs = sw.elapsedMs
|
||||
}
|
||||
consecutiveSwitchFailures = 0
|
||||
openTimings.add({ handle, ms: sw.elapsedMs, index: i })
|
||||
if (sw.elapsedMs >= softMs) {
|
||||
console.warn(`[realistic-freeze] SOFT open #${i} ${handle}: ${sw.elapsedMs.toFixed(0)}ms`)
|
||||
}
|
||||
if (sw.elapsedMs >= hardMs) {
|
||||
console.warn(`[realistic-freeze] HARD open #${i} ${handle}: ${sw.elapsedMs.toFixed(0)}ms`)
|
||||
}
|
||||
} catch (error) {
|
||||
const msg = String(error)
|
||||
const timedOut = /timed out/i.test(msg)
|
||||
if (timedOut) {
|
||||
timedOutOps += 1
|
||||
}
|
||||
consecutiveSwitchFailures += 1
|
||||
maxConsecutiveSwitchFailures = Math.max(
|
||||
maxConsecutiveSwitchFailures,
|
||||
consecutiveSwitchFailures
|
||||
)
|
||||
openTimings.add({ handle, error: msg, index: i, timedOut })
|
||||
notes.push(`open ${handle} failed${timedOut ? ' (TIMEOUT)' : ''}: ${msg.slice(0, 200)}`)
|
||||
}
|
||||
if (i < openList.length - 1) {
|
||||
await sleep(humanPaceDelayMs(paceMs, paceJitterMs))
|
||||
}
|
||||
}
|
||||
phases.push({
|
||||
phase: 'human-paced-open',
|
||||
count: openList.length,
|
||||
ok: openOk,
|
||||
maxOpenMs,
|
||||
firstOpenMs,
|
||||
openWallMs: performance.now() - openStarted
|
||||
})
|
||||
}
|
||||
|
||||
const openWallMs = performance.now() - openStarted
|
||||
|
||||
let midStormWatch = {
|
||||
samples: [],
|
||||
durationMs: 0,
|
||||
sampleCount: 0,
|
||||
maxStatusMs: 0,
|
||||
unhealthySampleCount: 0,
|
||||
infrastructureErrorCount: 0,
|
||||
longestUnhealthyWindowMs: 0
|
||||
}
|
||||
if (statusWatch) {
|
||||
midStormWatch = await statusWatch.stop()
|
||||
notes.push(
|
||||
`mid-storm status samples=${midStormWatch.sampleCount} durationMs=${midStormWatch.durationMs.toFixed(0)}`
|
||||
)
|
||||
phases.push({
|
||||
phase: 'mid-storm-status-watchdog',
|
||||
samples: midStormWatch.sampleCount,
|
||||
durationMs: midStormWatch.durationMs,
|
||||
maxStatusMs: midStormWatch.maxStatusMs
|
||||
})
|
||||
}
|
||||
|
||||
// Post-storm health: does local status still answer?
|
||||
let statusProbeMs = null
|
||||
let statusHangMs = 0
|
||||
const statusStarted = performance.now()
|
||||
try {
|
||||
const statusProbe = await orcaJsonAsync(['status'], {
|
||||
local: true,
|
||||
timeoutMs: permanentTimeoutMs
|
||||
})
|
||||
statusProbeMs = statusProbe.elapsedMs
|
||||
} catch (error) {
|
||||
statusHangMs = performance.now() - statusStarted
|
||||
notes.push(
|
||||
`status probe FAILED after ${statusHangMs.toFixed(0)}ms: ${String(error).slice(0, 200)}`
|
||||
)
|
||||
console.error(`[realistic-freeze] status probe failed — possible permanent lockup`)
|
||||
}
|
||||
|
||||
let memoryProbeMs = null
|
||||
try {
|
||||
const mem = await orcaJsonAsync(['diagnostics', 'memory'], {
|
||||
local: true,
|
||||
timeoutMs: permanentTimeoutMs
|
||||
})
|
||||
memoryProbeMs = mem.elapsedMs
|
||||
notes.push(`memory diagnostic ms=${mem.elapsedMs.toFixed(0)}`)
|
||||
} catch (error) {
|
||||
notes.push(`memory diagnostic failed: ${String(error).slice(0, 200)}`)
|
||||
}
|
||||
|
||||
const peakForSignals = Math.max(maxOpenMs, firstOpenMs, maxBatchWallMs)
|
||||
const signals = evaluateRealisticFreezeSignals({
|
||||
maxOpenMs: peakForSignals,
|
||||
firstOpenMs,
|
||||
reconnectRefreshMs,
|
||||
statusProbeMs: statusProbeMs ?? 0,
|
||||
memoryProbeMs,
|
||||
softMs,
|
||||
hardMs
|
||||
})
|
||||
|
||||
const lockup = evaluatePermanentLockup({
|
||||
timedOutOps,
|
||||
statusHangMs,
|
||||
consecutiveSwitchFailures: maxConsecutiveSwitchFailures,
|
||||
openFailed: openList.length - openOk,
|
||||
openTotal: openList.length,
|
||||
permanentTimeoutMs
|
||||
})
|
||||
|
||||
const fullApp = evaluateFullAppFreeze({
|
||||
statusSamples: midStormWatch.samples,
|
||||
statusSummary: midStormWatch,
|
||||
foreverWindowMs,
|
||||
statusSlowMs
|
||||
})
|
||||
const watchdogInfrastructureErrorCount = midStormWatch.infrastructureErrorCount
|
||||
if (statusHangMs >= foreverWindowMs) {
|
||||
fullApp.foreverUiLockupObserved = true
|
||||
fullApp.longestUnhealthyWindowMs = Math.max(fullApp.longestUnhealthyWindowMs, statusHangMs)
|
||||
fullApp.reason = `post-storm status hang ${statusHangMs.toFixed(0)}ms`
|
||||
}
|
||||
|
||||
const recoveredHardStall = signals.hardFreeze && !fullApp.foreverUiLockupObserved && openOk > 0
|
||||
|
||||
let samplePath = null
|
||||
if (signals.softFreeze || signals.hardFreeze || fullApp.foreverUiLockupObserved) {
|
||||
samplePath = sampleOrcaIfPossible()
|
||||
if (samplePath) {
|
||||
notes.push(`sample=${samplePath}`)
|
||||
} else {
|
||||
notes.push('sample unavailable')
|
||||
}
|
||||
}
|
||||
|
||||
const storyByScenario = {
|
||||
'idle-backlog-open': 'User away while remotes stream; returns and opens sessions one-by-one.',
|
||||
'idle-backlog-reconnect-open':
|
||||
'User away; wake-like reconnect metadata storm; then opens sessions.',
|
||||
'restart-proxy': 'User away; restart-proxy discovery; then opens sessions.',
|
||||
'lockup-storm':
|
||||
'Idle flood + reconnect refresh + concurrent open + mid-storm status watchdog (full-app freeze bar).'
|
||||
}
|
||||
|
||||
const report = {
|
||||
topology: 'live-paired-remote-realistic',
|
||||
scenario,
|
||||
story: storyByScenario[scenario] || scenario,
|
||||
environment: envName,
|
||||
localVersion: local.result?.runtime?.appVersion,
|
||||
remoteVersion: remote.result?.runtime?.appVersion,
|
||||
remoteWorktreeCount: wtList.length,
|
||||
createdFloodTerminals: created.length,
|
||||
openTargets: openList.length,
|
||||
idleMs,
|
||||
paceMs,
|
||||
paceJitterMs,
|
||||
stormParallel: scenario === 'lockup-storm' ? stormParallel : 1,
|
||||
firstOpenMs,
|
||||
maxOpenMs,
|
||||
maxBatchWallMs,
|
||||
avgOpenMs: openOk ? sumOpenMs / openOk : 0,
|
||||
openWallMs,
|
||||
openOk,
|
||||
openFailed: openList.length - openOk,
|
||||
reconnectRefreshMs,
|
||||
peakLatencyMs: Math.max(signals.peakLatencyMs, maxBatchWallMs),
|
||||
statusProbeMs,
|
||||
statusHangMs,
|
||||
memoryProbeMs,
|
||||
softFreeze: signals.softFreeze,
|
||||
hardFreeze: signals.hardFreeze,
|
||||
recoveredHardStall,
|
||||
permanentLockup: lockup.permanentLockup,
|
||||
foreverUiLockupObserved: fullApp.foreverUiLockupObserved,
|
||||
foreverFreeze: fullApp,
|
||||
midStormStatusSamples: midStormWatch.samples,
|
||||
midStormStatusSampleCount: midStormWatch.sampleCount,
|
||||
watchdogInfrastructureErrorCount,
|
||||
timedOutOps,
|
||||
maxConsecutiveSwitchFailures,
|
||||
softMs,
|
||||
hardMs,
|
||||
foreverWindowMs,
|
||||
statusSlowMs,
|
||||
permanentTimeoutMs,
|
||||
opTimeoutMs,
|
||||
phases,
|
||||
notes,
|
||||
openTimingCount: openTimings.totalCount,
|
||||
openTimings: openTimings.values()
|
||||
}
|
||||
|
||||
const outPath = path.join(reportDir, `live-realistic-freeze-${envName}-${scenario}.json`)
|
||||
const stamped = path.join(
|
||||
reportDir,
|
||||
`live-realistic-freeze-${envName}-${scenario}-peak-${Date.now()}.json`
|
||||
)
|
||||
writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`)
|
||||
writeFileSync(stamped, `${JSON.stringify(report, null, 2)}\n`)
|
||||
console.log(`[realistic-freeze] report ${outPath}`)
|
||||
console.log(JSON.stringify(report, null, 2))
|
||||
|
||||
if (scratchDir) {
|
||||
try {
|
||||
mkdirSync(scratchDir, { recursive: true })
|
||||
copyFileSync(outPath, path.join(scratchDir, 'live-realistic-freeze-report.json'))
|
||||
} catch (error) {
|
||||
console.warn(`[realistic-freeze] scratch copy failed: ${String(error)}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (watchdogInfrastructureErrorCount > 0) {
|
||||
process.exitCode = 3
|
||||
console.error('[realistic-freeze] WATCHDOG INFRASTRUCTURE FAILURE')
|
||||
} else if (fullApp.foreverUiLockupObserved) {
|
||||
process.exitCode = 5
|
||||
console.error('[realistic-freeze] FULL-APP FOREVER FREEZE (status unhealthy ≥ forever window)')
|
||||
} else if (lockup.permanentLockup) {
|
||||
process.exitCode = 4
|
||||
console.error(
|
||||
'[realistic-freeze] PERMANENT LOCKUP HEURISTIC (timeouts/fail-rate) — check foreverUiLockupObserved'
|
||||
)
|
||||
} else if (signals.hardFreeze) {
|
||||
process.exitCode = 2
|
||||
console.error(
|
||||
'[realistic-freeze] HARD FREEZE SIGNAL (recovered multi-second stall — not forever lockup)'
|
||||
)
|
||||
} else if (signals.softFreeze) {
|
||||
process.exitCode = 1
|
||||
console.error('[realistic-freeze] SOFT FREEZE SIGNAL')
|
||||
} else {
|
||||
console.log('[realistic-freeze] no freeze signal under thresholds')
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('[realistic-freeze] failed', error)
|
||||
process.exit(3)
|
||||
})
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
/**
|
||||
* Mid-storm host health samples for forever-freeze detection.
|
||||
* Polls `orca status --json` on an interval while a load storm runs.
|
||||
*/
|
||||
import { spawn } from 'node:child_process'
|
||||
import { BoundedLiveFreezeHistory } from './live-freeze-bounded-history.mjs'
|
||||
import { resolveOrcaCliInvocation } from './live-remote-freeze-rpc.mjs'
|
||||
|
||||
/**
|
||||
* @param {{ intervalMs?: number, timeoutMs?: number, cliCommand?: string, sampleHistoryLimit?: number, statusSlowMs?: number }} opts
|
||||
*/
|
||||
export function startStatusWatchdog(opts = {}) {
|
||||
const intervalMs = opts.intervalMs ?? 2000
|
||||
const timeoutMs = opts.timeoutMs ?? 30_000
|
||||
const cliInvocation = opts.cliCommand
|
||||
? { command: opts.cliCommand, prefixArgs: [] }
|
||||
: resolveOrcaCliInvocation()
|
||||
const samples = new BoundedLiveFreezeHistory(opts.sampleHistoryLimit ?? 240)
|
||||
const statusSlowMs = opts.statusSlowMs ?? 15_000
|
||||
let stopped = false
|
||||
let inFlight = false
|
||||
let infrastructureErrorCount = 0
|
||||
let longestUnhealthyWindowMs = 0
|
||||
let maxStatusMs = 0
|
||||
let runStartMs = null
|
||||
let unhealthySampleCount = 0
|
||||
const startedAt = performance.now()
|
||||
|
||||
const record = (sample) => {
|
||||
samples.add(sample)
|
||||
maxStatusMs = Math.max(maxStatusMs, sample.ms || 0)
|
||||
if (sample.infrastructureError) {
|
||||
infrastructureErrorCount += 1
|
||||
}
|
||||
const unhealthy =
|
||||
!sample.infrastructureError &&
|
||||
(Boolean(sample.hang) || sample.ok === false || (sample.ms || 0) >= statusSlowMs)
|
||||
if (!unhealthy) {
|
||||
runStartMs = null
|
||||
return
|
||||
}
|
||||
unhealthySampleCount += 1
|
||||
runStartMs ??= sample.tMs ?? 0
|
||||
longestUnhealthyWindowMs = Math.max(
|
||||
longestUnhealthyWindowMs,
|
||||
(sample.tMs ?? 0) + (sample.ms || 0) - runStartMs
|
||||
)
|
||||
}
|
||||
|
||||
const summary = () => ({
|
||||
sampleCount: samples.totalCount,
|
||||
maxStatusMs,
|
||||
unhealthySampleCount,
|
||||
infrastructureErrorCount,
|
||||
longestUnhealthyWindowMs
|
||||
})
|
||||
|
||||
const probe = () =>
|
||||
new Promise((resolve) => {
|
||||
const t0 = performance.now()
|
||||
const child = spawn(
|
||||
cliInvocation.command,
|
||||
[...cliInvocation.prefixArgs, 'status', '--json'],
|
||||
{
|
||||
env: cliInvocation.env,
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
}
|
||||
)
|
||||
let settled = false
|
||||
const finish = (result) => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
resolve(result)
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
child.kill('SIGKILL')
|
||||
finish({
|
||||
tMs: t0 - startedAt,
|
||||
ms: performance.now() - t0,
|
||||
ok: false,
|
||||
hang: true
|
||||
})
|
||||
}, timeoutMs)
|
||||
child.stdout.on('data', () => {})
|
||||
child.stderr.on('data', () => {})
|
||||
child.on('error', (error) => {
|
||||
clearTimeout(timer)
|
||||
finish({
|
||||
tMs: t0 - startedAt,
|
||||
ms: performance.now() - t0,
|
||||
ok: false,
|
||||
hang: false,
|
||||
infrastructureError: true,
|
||||
error: String(error)
|
||||
})
|
||||
})
|
||||
child.on('close', (code) => {
|
||||
clearTimeout(timer)
|
||||
finish({
|
||||
tMs: t0 - startedAt,
|
||||
ms: performance.now() - t0,
|
||||
ok: code === 0,
|
||||
hang: false
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
const tick = async ({ force = false } = {}) => {
|
||||
if ((!force && stopped) || inFlight) {
|
||||
return
|
||||
}
|
||||
inFlight = true
|
||||
try {
|
||||
const sample = await probe()
|
||||
record(sample)
|
||||
} finally {
|
||||
inFlight = false
|
||||
}
|
||||
}
|
||||
|
||||
const interval = setInterval(() => {
|
||||
void tick()
|
||||
}, intervalMs)
|
||||
void tick()
|
||||
|
||||
return {
|
||||
stop: async () => {
|
||||
stopped = true
|
||||
clearInterval(interval)
|
||||
// Wait for in-flight probe, then force one final sample.
|
||||
const deadline = performance.now() + timeoutMs + 1000
|
||||
while (inFlight && performance.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, 20))
|
||||
}
|
||||
await tick({ force: true })
|
||||
return {
|
||||
samples: samples.values(),
|
||||
...summary(),
|
||||
durationMs: performance.now() - startedAt
|
||||
}
|
||||
},
|
||||
getSamples: () => samples.values(),
|
||||
getSummary: summary
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import { startStatusWatchdog } from './live-remote-status-watchdog.mjs'
|
||||
|
||||
describe('startStatusWatchdog', () => {
|
||||
it('collects status samples and stops cleanly', async () => {
|
||||
// Real path: actually invokes `orca status --json` (must be available in CI/dev with Orca or fail soft).
|
||||
const watch = startStatusWatchdog({ intervalMs: 50, timeoutMs: 5_000 })
|
||||
await new Promise((r) => setTimeout(r, 180))
|
||||
const result = await watch.stop()
|
||||
expect(result.samples.length).toBeGreaterThanOrEqual(1)
|
||||
expect(result.durationMs).toBeGreaterThan(0)
|
||||
for (const s of result.samples) {
|
||||
expect(typeof s.ms).toBe('number')
|
||||
expect(typeof s.ok).toBe('boolean')
|
||||
expect(typeof s.hang).toBe('boolean')
|
||||
}
|
||||
})
|
||||
|
||||
it('marks CLI spawn failures as infrastructure errors', async () => {
|
||||
const watch = startStatusWatchdog({
|
||||
intervalMs: 50,
|
||||
timeoutMs: 1000,
|
||||
cliCommand: 'orca-freeze-watchdog-missing-command'
|
||||
})
|
||||
const result = await watch.stop()
|
||||
|
||||
expect(result.samples.length).toBeGreaterThanOrEqual(1)
|
||||
expect(result.samples.every((sample) => sample.infrastructureError === true)).toBe(true)
|
||||
expect(result.samples.every((sample) => sample.hang === false)).toBe(true)
|
||||
})
|
||||
|
||||
it('bounds retained samples while preserving full-run counters', async () => {
|
||||
const watch = startStatusWatchdog({
|
||||
intervalMs: 50,
|
||||
timeoutMs: 1000,
|
||||
cliCommand: 'orca-freeze-watchdog-missing-command',
|
||||
sampleHistoryLimit: 1
|
||||
})
|
||||
const result = await watch.stop()
|
||||
|
||||
expect(result.samples).toHaveLength(1)
|
||||
expect(result.sampleCount).toBeGreaterThan(result.samples.length)
|
||||
expect(result.infrastructureErrorCount).toBe(result.sampleCount)
|
||||
expect(result.maxStatusMs).toBeGreaterThanOrEqual(0)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
const extraArgs = process.argv.slice(2)
|
||||
const pnpmEntry = process.env.npm_execpath
|
||||
if (!pnpmEntry) {
|
||||
throw new Error('npm_execpath is required; run this harness through pnpm')
|
||||
}
|
||||
const env = {
|
||||
...process.env,
|
||||
ORCA_E2E_SSH_DOCKER: '1'
|
||||
}
|
||||
|
||||
const runtime = spawnSync(process.execPath, [pnpmEntry, 'run', 'ensure:electron-runtime'], {
|
||||
stdio: 'inherit',
|
||||
env
|
||||
})
|
||||
|
||||
if (runtime.status !== 0) {
|
||||
process.exit(runtime.status ?? 1)
|
||||
}
|
||||
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
pnpmEntry,
|
||||
'exec',
|
||||
'playwright',
|
||||
'test',
|
||||
'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts',
|
||||
'--config',
|
||||
'tests/playwright.config.ts',
|
||||
'--project',
|
||||
'electron-headless',
|
||||
'--workers=1',
|
||||
...extraArgs
|
||||
],
|
||||
{
|
||||
stdio: 'inherit',
|
||||
env
|
||||
}
|
||||
)
|
||||
|
||||
process.exit(result.status ?? 1)
|
||||
|
|
@ -116,7 +116,11 @@
|
|||
"bench:multi-workspace-typing": "pnpm run ensure:electron-runtime && node config/scripts/run-multi-workspace-typing-bench.mjs",
|
||||
"bench:cold-park-reveal": "pnpm run ensure:electron-runtime && node tests/tools/benchmarks/terminal-cold-park-reveal-bench.mjs",
|
||||
"bench:cold-park-resource": "pnpm run ensure:electron-runtime && node tests/tools/benchmarks/terminal-cold-park-resource-bench.mjs",
|
||||
"bench:compare": "node config/scripts/compare-benchmark-artifacts.mjs"
|
||||
"bench:compare": "node config/scripts/compare-benchmark-artifacts.mjs",
|
||||
"test:e2e:remote-bulk-open-freeze": "pnpm run ensure:electron-runtime && pnpm exec playwright test tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"test:e2e:ssh-docker-bulk-open-freeze": "node config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs",
|
||||
"repro:live-remote-bulk-open-freeze": "node config/scripts/live-remote-bulk-open-freeze-repro.mjs",
|
||||
"repro:live-remote-realistic-freeze": "node config/scripts/live-remote-realistic-freeze-repro.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@electron-toolkit/preload": "^3.0.2",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import { formatTerminalFocus } from './terminal-format'
|
||||
|
||||
describe('formatTerminalFocus', () => {
|
||||
it('distinguishes superseded navigation from a winning focus', () => {
|
||||
expect(
|
||||
formatTerminalFocus({
|
||||
focus: {
|
||||
handle: 'term_stale',
|
||||
tabId: 'tab-stale',
|
||||
worktreeId: 'worktree-1',
|
||||
navigated: false
|
||||
}
|
||||
})
|
||||
).toBe(
|
||||
'Focus request for terminal term_stale was superseded or host navigation was skipped (tab tab-stale).'
|
||||
)
|
||||
expect(
|
||||
formatTerminalFocus({
|
||||
focus: { handle: 'term_winner', tabId: 'tab-winner', worktreeId: 'worktree-1' }
|
||||
})
|
||||
).toBe('Focused terminal term_winner (tab tab-winner).')
|
||||
})
|
||||
})
|
||||
|
|
@ -164,6 +164,9 @@ export function formatTerminalSplit(result: { split: RuntimeTerminalSplit }): st
|
|||
}
|
||||
|
||||
export function formatTerminalFocus(result: { focus: RuntimeTerminalFocus }): string {
|
||||
if (result.focus.navigated === false) {
|
||||
return `Focus request for terminal ${result.focus.handle} was superseded or host navigation was skipped (tab ${result.focus.tabId}).`
|
||||
}
|
||||
return `Focused terminal ${result.focus.handle} (tab ${result.focus.tabId}).`
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -15436,6 +15436,257 @@ describe('OrcaRuntimeService', () => {
|
|||
})
|
||||
})
|
||||
|
||||
it('coalesces concurrent focusTerminal navigations so only the latest full reveal runs', async () => {
|
||||
// Instant reveals during createTerminal; switch to gated mock before focus storm.
|
||||
const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'tab-create' })
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
runtime.setPtyController({
|
||||
spawn: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ id: 'pty-a' })
|
||||
.mockResolvedValueOnce({ id: 'pty-b' })
|
||||
.mockResolvedValueOnce({ id: 'pty-c' }),
|
||||
write: () => true,
|
||||
kill: () => true,
|
||||
getForegroundProcess: async () => null
|
||||
})
|
||||
runtime.setNotifier({
|
||||
worktreesChanged: vi.fn(),
|
||||
reposChanged: vi.fn(),
|
||||
activateWorktree: vi.fn(),
|
||||
createTerminal: vi.fn(),
|
||||
revealTerminalSession,
|
||||
splitTerminal: vi.fn(),
|
||||
renameTerminal: vi.fn(),
|
||||
focusTerminal: vi.fn(),
|
||||
closeTerminal: vi.fn(),
|
||||
sleepWorktree: vi.fn(),
|
||||
terminalFitOverrideChanged: vi.fn(),
|
||||
terminalDriverChanged: vi.fn()
|
||||
})
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
|
||||
const a = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`, {
|
||||
title: 'a',
|
||||
presentation: 'background'
|
||||
})
|
||||
const b = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`, {
|
||||
title: 'b',
|
||||
presentation: 'background'
|
||||
})
|
||||
const c = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`, {
|
||||
title: 'c',
|
||||
presentation: 'background'
|
||||
})
|
||||
|
||||
let releaseFirstReveal!: (value: { tabId: string }) => void
|
||||
let firstRevealStarted = false
|
||||
const firstRevealGate = new Promise<{ tabId: string }>((resolve) => {
|
||||
releaseFirstReveal = resolve
|
||||
})
|
||||
revealTerminalSession.mockReset()
|
||||
revealTerminalSession.mockImplementation(() => {
|
||||
if (!firstRevealStarted) {
|
||||
firstRevealStarted = true
|
||||
return firstRevealGate
|
||||
}
|
||||
return Promise.resolve({ tabId: 'tab-latest' })
|
||||
})
|
||||
|
||||
const pA = runtime.focusTerminal(a.handle)
|
||||
await vi.waitFor(() => {
|
||||
expect(firstRevealStarted).toBe(true)
|
||||
})
|
||||
const pB = runtime.focusTerminal(b.handle)
|
||||
const pC = runtime.focusTerminal(c.handle)
|
||||
|
||||
// B is superseded while A is in flight — identity only, never navigated.
|
||||
await expect(pB).resolves.toMatchObject({
|
||||
handle: b.handle,
|
||||
navigated: false
|
||||
})
|
||||
releaseFirstReveal({ tabId: 'tab-a' })
|
||||
// A may still complete reveal work, but if C superseded it, navigated is false.
|
||||
const aResult = await pA
|
||||
expect(aResult.handle).toBe(a.handle)
|
||||
expect(aResult.navigated).toBe(false)
|
||||
await expect(pC).resolves.toMatchObject({
|
||||
handle: c.handle,
|
||||
tabId: 'tab-latest',
|
||||
navigated: true
|
||||
})
|
||||
|
||||
// B must never have started a reveal; only A and/or C.
|
||||
const revealedPtyIds = revealTerminalSession.mock.calls.map(
|
||||
(call) => (call[1] as { ptyId?: string }).ptyId
|
||||
)
|
||||
expect(revealedPtyIds).not.toContain('pty-b')
|
||||
expect(revealedPtyIds.at(-1)).toBe('pty-c')
|
||||
})
|
||||
|
||||
it('reports a queued PTY focus as not navigated when its notifier disappears', async () => {
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
runtime.setPtyController({
|
||||
write: () => true,
|
||||
kill: () => true,
|
||||
getForegroundProcess: async () => null
|
||||
})
|
||||
runtime.registerPty('pty-a', TEST_WORKTREE_ID)
|
||||
runtime.registerPty('pty-b', TEST_WORKTREE_ID)
|
||||
const terminals = (await runtime.listTerminals()).terminals
|
||||
const terminalA = terminals.find((terminal) => terminal.ptyId === 'pty-a')
|
||||
const terminalB = terminals.find((terminal) => terminal.ptyId === 'pty-b')
|
||||
expect(terminalA).toBeDefined()
|
||||
expect(terminalB).toBeDefined()
|
||||
|
||||
let releaseReveal!: (value: { tabId: string }) => void
|
||||
const revealGate = new Promise<{ tabId: string }>((resolve) => {
|
||||
releaseReveal = resolve
|
||||
})
|
||||
const revealTerminalSession = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(() => revealGate)
|
||||
.mockResolvedValue({ tabId: 'tab-b' })
|
||||
runtime.setNotifier({ revealTerminalSession } as never)
|
||||
|
||||
const first = runtime.focusTerminal(terminalA!.handle)
|
||||
await vi.waitFor(() => expect(revealTerminalSession).toHaveBeenCalledOnce())
|
||||
const queued = runtime.focusTerminal(terminalB!.handle)
|
||||
runtime.setNotifier(null)
|
||||
releaseReveal({ tabId: 'tab-a' })
|
||||
|
||||
await expect(first).resolves.toMatchObject({ handle: terminalA!.handle, navigated: false })
|
||||
await expect(queued).resolves.toMatchObject({ handle: terminalB!.handle, navigated: false })
|
||||
expect(revealTerminalSession).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('reports an in-flight PTY focus as not navigated when its notifier disappears', async () => {
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
runtime.setPtyController({
|
||||
write: () => true,
|
||||
kill: () => true,
|
||||
getForegroundProcess: async () => null
|
||||
})
|
||||
runtime.registerPty('pty-a', TEST_WORKTREE_ID)
|
||||
const terminal = (await runtime.listTerminals()).terminals.find(
|
||||
(candidate) => candidate.ptyId === 'pty-a'
|
||||
)
|
||||
expect(terminal).toBeDefined()
|
||||
|
||||
let releaseReveal!: (value: { tabId: string }) => void
|
||||
const revealTerminalSession = vi.fn(
|
||||
() =>
|
||||
new Promise<{ tabId: string }>((resolve) => {
|
||||
releaseReveal = resolve
|
||||
})
|
||||
)
|
||||
runtime.setNotifier({ revealTerminalSession } as never)
|
||||
|
||||
const focus = runtime.focusTerminal(terminal!.handle)
|
||||
await vi.waitFor(() => expect(revealTerminalSession).toHaveBeenCalledOnce())
|
||||
runtime.setNotifier(null)
|
||||
releaseReveal({ tabId: 'tab-a' })
|
||||
|
||||
await expect(focus).resolves.toMatchObject({
|
||||
handle: terminal!.handle,
|
||||
tabId: 'tab-a',
|
||||
navigated: false
|
||||
})
|
||||
})
|
||||
|
||||
it('does not invoke a stale graph-leaf focus notifier after queued PTY work', async () => {
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
runtime.setPtyController({
|
||||
write: () => true,
|
||||
kill: () => true,
|
||||
getForegroundProcess: async () => null
|
||||
})
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: 'tab-leaf',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Starting terminal',
|
||||
activeLeafId: HEADLESS_LEAF_ID,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: 'tab-leaf',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: null
|
||||
}
|
||||
]
|
||||
})
|
||||
const leafTerminal = (await runtime.listTerminals()).terminals.find(
|
||||
(terminal) => terminal.tabId === 'tab-leaf'
|
||||
)
|
||||
runtime.registerPty('pty-a', TEST_WORKTREE_ID)
|
||||
const ptyTerminal = (await runtime.listTerminals()).terminals.find(
|
||||
(terminal) => terminal.ptyId === 'pty-a'
|
||||
)
|
||||
expect(ptyTerminal).toBeDefined()
|
||||
expect(leafTerminal).toBeDefined()
|
||||
|
||||
let releaseReveal!: (value: { tabId: string }) => void
|
||||
const revealGate = new Promise<{ tabId: string }>((resolve) => {
|
||||
releaseReveal = resolve
|
||||
})
|
||||
const focusTerminal = vi.fn()
|
||||
const revealTerminalSession = vi.fn(() => revealGate)
|
||||
runtime.setNotifier({
|
||||
revealTerminalSession,
|
||||
focusTerminal
|
||||
} as never)
|
||||
|
||||
const first = runtime.focusTerminal(ptyTerminal!.handle)
|
||||
await vi.waitFor(() => expect(revealTerminalSession).toHaveBeenCalledOnce())
|
||||
const queued = runtime.focusTerminal(leafTerminal!.handle)
|
||||
runtime.setNotifier(null)
|
||||
releaseReveal({ tabId: 'tab-a' })
|
||||
|
||||
await expect(first).resolves.toMatchObject({ handle: ptyTerminal!.handle, navigated: false })
|
||||
await expect(queued).resolves.toMatchObject({ handle: leafTerminal!.handle, navigated: false })
|
||||
expect(focusTerminal).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('reports graph-leaf focus as not navigated without a host notifier', async () => {
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: 'tab-leaf',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Starting terminal',
|
||||
activeLeafId: HEADLESS_LEAF_ID,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: 'tab-leaf',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId: HEADLESS_LEAF_ID,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: null
|
||||
}
|
||||
]
|
||||
})
|
||||
const [terminal] = (await runtime.listTerminals()).terminals
|
||||
|
||||
await expect(runtime.focusTerminal(terminal.handle)).resolves.toEqual({
|
||||
handle: terminal.handle,
|
||||
tabId: 'tab-leaf',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
navigated: false
|
||||
})
|
||||
})
|
||||
|
||||
it('clears terminal scrollback through the PTY controller and headless buffer', async () => {
|
||||
const clearBuffer = vi.fn().mockResolvedValue(undefined)
|
||||
const runtime = new OrcaRuntimeService(store)
|
||||
|
|
@ -26293,7 +26544,8 @@ describe('OrcaRuntimeService', () => {
|
|||
await expect(runtime.focusTerminal(laptopTerminal.handle)).resolves.toEqual({
|
||||
handle: laptopTerminal.handle,
|
||||
tabId: 'laptop-tab',
|
||||
worktreeId: TEST_WORKTREE_ID
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
navigated: false
|
||||
})
|
||||
await expect(runtime.closeTerminal(laptopTerminal.handle)).resolves.toEqual({
|
||||
handle: laptopTerminal.handle,
|
||||
|
|
|
|||
|
|
@ -949,6 +949,7 @@ import {
|
|||
createMobileSessionTabsNotifyCoalescer,
|
||||
type MobileSessionTabsNotifyCoalescer
|
||||
} from './mobile-session-tabs-notify-coalescer'
|
||||
import { TerminalFocusNavigationCoalescer } from './terminal-focus-navigation-coalescer'
|
||||
import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch'
|
||||
import {
|
||||
assertFolderWorkspacePathUsable,
|
||||
|
|
@ -2747,6 +2748,12 @@ export class OrcaRuntimeService {
|
|||
createMobileSessionTabsNotifyCoalescer((worktreeId) =>
|
||||
this.notifyMobileSessionTabsChangedNow(worktreeId)
|
||||
)
|
||||
// Why: concurrent host terminal.focus storms (CLI switch fan-out / bulk open)
|
||||
// each await a full host reveal; only one terminal can be focused, so latest-wins
|
||||
// single-flight bounds host work. Does not replace cheaper activation or
|
||||
// reconnect-scan bounding for sequential soft freezes.
|
||||
private readonly terminalFocusNavigationCoalescer =
|
||||
new TerminalFocusNavigationCoalescer<RuntimeTerminalFocus>()
|
||||
private pendingMobileSessionPtyInventoryRefresh: Promise<Set<string> | null> | null = null
|
||||
private leaves = new Map<string, RuntimeLeafRecord>()
|
||||
// Why: PTY output is a per-keystroke hot path. Looking up affected leaves by
|
||||
|
|
@ -25873,38 +25880,158 @@ export class OrcaRuntimeService {
|
|||
handle: string,
|
||||
options: { navigateHost?: boolean } = {}
|
||||
): Promise<RuntimeTerminalFocus> {
|
||||
const navigateHost = options.navigateHost !== false
|
||||
const livePtyIdentity = (): RuntimeTerminalFocus => {
|
||||
const live = this.getLivePtyForHandle(handle)
|
||||
if (!live?.pty.connected) {
|
||||
throw new Error('terminal_exited')
|
||||
}
|
||||
return {
|
||||
handle,
|
||||
tabId: live.pty.tabId ?? live.record.tabId,
|
||||
worktreeId: live.pty.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
const liveLeafIdentity = (): RuntimeTerminalFocus => {
|
||||
this.assertGraphReady()
|
||||
const { leaf: current } = this.getLiveLeafForHandle(handle)
|
||||
return {
|
||||
handle,
|
||||
tabId: current.tabId,
|
||||
worktreeId: current.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
|
||||
const pty = this.getLivePtyForHandle(handle)
|
||||
if (pty) {
|
||||
if (!pty.pty.connected) {
|
||||
throw new Error('terminal_exited')
|
||||
}
|
||||
const parsedPaneKey = parsePaneKey(pty.pty.paneKey ?? '')
|
||||
const revealed =
|
||||
options.navigateHost === false
|
||||
? undefined
|
||||
: await this.notifier?.revealTerminalSession?.(pty.pty.worktreeId, {
|
||||
ptyId: pty.pty.ptyId,
|
||||
title: getLatestPtyTitle(pty.pty),
|
||||
...(pty.pty.launchConfig
|
||||
? { launchConfig: copySleepingAgentLaunchConfig(pty.pty.launchConfig) }
|
||||
: {}),
|
||||
...(pty.pty.launchToken ? { launchToken: pty.pty.launchToken } : {}),
|
||||
...(pty.pty.launchAgent ? { launchAgent: pty.pty.launchAgent } : {}),
|
||||
...(pty.pty.tabId !== null ? { tabId: pty.pty.tabId } : {}),
|
||||
...(parsedPaneKey ? { leafId: parsedPaneKey.leafId } : {})
|
||||
})
|
||||
return {
|
||||
handle,
|
||||
tabId: revealed?.tabId ?? pty.pty.tabId ?? pty.record.tabId,
|
||||
worktreeId: pty.pty.worktreeId
|
||||
if (!navigateHost || !this.notifier?.revealTerminalSession) {
|
||||
return {
|
||||
handle,
|
||||
tabId: pty.pty.tabId ?? pty.record.tabId,
|
||||
worktreeId: pty.pty.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
// Coalesce concurrent host navigations: only the latest full reveal claims navigated.
|
||||
return this.terminalFocusNavigationCoalescer.run({
|
||||
key: handle,
|
||||
resolveSuperseded: (completed) =>
|
||||
completed ? { ...completed, navigated: false } : livePtyIdentity(),
|
||||
run: async (ctx) => {
|
||||
const live = this.getLivePtyForHandle(handle)
|
||||
if (!live?.pty.connected) {
|
||||
throw new Error('terminal_exited')
|
||||
}
|
||||
if (!ctx.isCurrent()) {
|
||||
return {
|
||||
handle,
|
||||
tabId: live.pty.tabId ?? live.record.tabId,
|
||||
worktreeId: live.pty.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
const notifier = this.notifier
|
||||
if (!notifier?.revealTerminalSession) {
|
||||
return {
|
||||
handle,
|
||||
tabId: live.pty.tabId ?? live.record.tabId,
|
||||
worktreeId: live.pty.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
const parsedPaneKey = parsePaneKey(live.pty.paneKey ?? '')
|
||||
const revealed = await notifier.revealTerminalSession(live.pty.worktreeId, {
|
||||
ptyId: live.pty.ptyId,
|
||||
title: getLatestPtyTitle(live.pty),
|
||||
...(live.pty.launchConfig
|
||||
? { launchConfig: copySleepingAgentLaunchConfig(live.pty.launchConfig) }
|
||||
: {}),
|
||||
...(live.pty.launchToken ? { launchToken: live.pty.launchToken } : {}),
|
||||
...(live.pty.launchAgent ? { launchAgent: live.pty.launchAgent } : {}),
|
||||
...(live.pty.tabId !== null ? { tabId: live.pty.tabId } : {}),
|
||||
...(parsedPaneKey ? { leafId: parsedPaneKey.leafId } : {})
|
||||
})
|
||||
if (!ctx.isCurrent() || this.notifier !== notifier) {
|
||||
return {
|
||||
handle,
|
||||
tabId: revealed?.tabId ?? live.pty.tabId ?? live.record.tabId,
|
||||
worktreeId: live.pty.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
return {
|
||||
handle,
|
||||
tabId: revealed?.tabId ?? live.pty.tabId ?? live.record.tabId,
|
||||
worktreeId: live.pty.worktreeId,
|
||||
navigated: true
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
this.assertGraphReady()
|
||||
const { leaf } = this.getLiveLeafForHandle(handle)
|
||||
if (options.navigateHost !== false) {
|
||||
this.notifier?.focusTerminal(leaf.tabId, leaf.worktreeId, leaf.leafId)
|
||||
if (!navigateHost) {
|
||||
return {
|
||||
handle,
|
||||
tabId: leaf.tabId,
|
||||
worktreeId: leaf.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
return { handle, tabId: leaf.tabId, worktreeId: leaf.worktreeId }
|
||||
if (!this.notifier?.focusTerminal) {
|
||||
return {
|
||||
handle,
|
||||
tabId: leaf.tabId,
|
||||
worktreeId: leaf.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
return this.terminalFocusNavigationCoalescer.run({
|
||||
key: handle,
|
||||
resolveSuperseded: (completed) =>
|
||||
completed ? { ...completed, navigated: false } : liveLeafIdentity(),
|
||||
run: async (ctx) => {
|
||||
this.assertGraphReady()
|
||||
const { leaf: liveLeaf } = this.getLiveLeafForHandle(handle)
|
||||
if (!ctx.isCurrent()) {
|
||||
return {
|
||||
handle,
|
||||
tabId: liveLeaf.tabId,
|
||||
worktreeId: liveLeaf.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
const notifier = this.notifier
|
||||
if (!notifier?.focusTerminal) {
|
||||
return {
|
||||
handle,
|
||||
tabId: liveLeaf.tabId,
|
||||
worktreeId: liveLeaf.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
notifier.focusTerminal(liveLeaf.tabId, liveLeaf.worktreeId, liveLeaf.leafId)
|
||||
if (!ctx.isCurrent() || this.notifier !== notifier) {
|
||||
return {
|
||||
handle,
|
||||
tabId: liveLeaf.tabId,
|
||||
worktreeId: liveLeaf.worktreeId,
|
||||
navigated: false
|
||||
}
|
||||
}
|
||||
return {
|
||||
handle,
|
||||
tabId: liveLeaf.tabId,
|
||||
worktreeId: liveLeaf.worktreeId,
|
||||
navigated: true
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async closeTerminal(handle: string): Promise<RuntimeTerminalClose> {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,202 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { TerminalFocusNavigationCoalescer } from './terminal-focus-navigation-coalescer'
|
||||
|
||||
function deferred<T>(): {
|
||||
promise: Promise<T>
|
||||
resolve: (value: T) => void
|
||||
reject: (error: unknown) => void
|
||||
} {
|
||||
let resolve!: (value: T) => void
|
||||
let reject!: (error: unknown) => void
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res
|
||||
reject = rej
|
||||
})
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
|
||||
describe('TerminalFocusNavigationCoalescer', () => {
|
||||
it('runs a single job to completion', async () => {
|
||||
const coalescer = new TerminalFocusNavigationCoalescer<string>()
|
||||
const result = await coalescer.run({
|
||||
key: 'term_a',
|
||||
run: async () => 'full-a',
|
||||
resolveSuperseded: () => 'superseded-a'
|
||||
})
|
||||
expect(result).toBe('full-a')
|
||||
expect(coalescer.getState()).toMatchObject({
|
||||
running: false,
|
||||
activeKey: null,
|
||||
pendingKey: null
|
||||
})
|
||||
})
|
||||
|
||||
it('serializes concurrent focuses and latest-wins drops intermediate pending', async () => {
|
||||
const coalescer = new TerminalFocusNavigationCoalescer<string>()
|
||||
const aGate = deferred<void>()
|
||||
let aStarted = false
|
||||
|
||||
const runA = vi.fn(async (ctx: { isCurrent: () => boolean }) => {
|
||||
aStarted = true
|
||||
await aGate.promise
|
||||
if (!ctx.isCurrent()) {
|
||||
return 'obsolete-a'
|
||||
}
|
||||
return 'full-a'
|
||||
})
|
||||
const runB = vi.fn(async () => 'full-b')
|
||||
const runC = vi.fn(async () => 'full-c')
|
||||
const superB = vi.fn(() => 'super-b')
|
||||
|
||||
const pA = coalescer.run({
|
||||
key: 'term_a',
|
||||
run: runA,
|
||||
resolveSuperseded: (completed) => completed ?? 'super-a'
|
||||
})
|
||||
await vi.waitFor(() => {
|
||||
expect(aStarted).toBe(true)
|
||||
})
|
||||
|
||||
const pB = coalescer.run({
|
||||
key: 'term_b',
|
||||
run: runB,
|
||||
resolveSuperseded: superB
|
||||
})
|
||||
const pC = coalescer.run({
|
||||
key: 'term_c',
|
||||
run: runC,
|
||||
resolveSuperseded: () => 'super-c'
|
||||
})
|
||||
|
||||
await expect(pB).resolves.toBe('super-b')
|
||||
expect(superB).toHaveBeenCalledTimes(1)
|
||||
expect(runB).not.toHaveBeenCalled()
|
||||
|
||||
aGate.resolve()
|
||||
await expect(pA).resolves.toBe('obsolete-a')
|
||||
await expect(pC).resolves.toBe('full-c')
|
||||
expect(runC).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('bounds host navigation to one full run under a parallel storm', async () => {
|
||||
const coalescer = new TerminalFocusNavigationCoalescer<number>()
|
||||
let inFlight = 0
|
||||
let maxInFlight = 0
|
||||
let fullRuns = 0
|
||||
|
||||
const makeJob = (key: string) =>
|
||||
coalescer.run({
|
||||
key,
|
||||
run: async (ctx) => {
|
||||
if (!ctx.isCurrent()) {
|
||||
return -1
|
||||
}
|
||||
inFlight += 1
|
||||
maxInFlight = Math.max(maxInFlight, inFlight)
|
||||
fullRuns += 1
|
||||
await new Promise((r) => setTimeout(r, 5))
|
||||
inFlight -= 1
|
||||
if (!ctx.isCurrent()) {
|
||||
return -1
|
||||
}
|
||||
return fullRuns
|
||||
},
|
||||
resolveSuperseded: () => -1
|
||||
})
|
||||
|
||||
const results = await Promise.all(Array.from({ length: 16 }, (_, i) => makeJob(`term_${i}`)))
|
||||
|
||||
expect(maxInFlight).toBe(1)
|
||||
expect(fullRuns).toBeLessThanOrEqual(2)
|
||||
expect(fullRuns).toBeGreaterThanOrEqual(1)
|
||||
expect(results.filter((r) => r === -1).length).toBeGreaterThanOrEqual(14)
|
||||
expect(results.some((r) => r > 0)).toBe(true)
|
||||
})
|
||||
|
||||
it('skips claiming navigation when a newer focus arrives mid-run', async () => {
|
||||
const coalescer = new TerminalFocusNavigationCoalescer<{ id: string; navigated: boolean }>()
|
||||
const aGate = deferred<void>()
|
||||
let aStarted = false
|
||||
|
||||
const pA = coalescer.run({
|
||||
key: 'term_a',
|
||||
run: async (ctx) => {
|
||||
aStarted = true
|
||||
await aGate.promise
|
||||
return { id: 'a', navigated: ctx.isCurrent() }
|
||||
},
|
||||
resolveSuperseded: () => ({ id: 'a', navigated: false })
|
||||
})
|
||||
await vi.waitFor(() => {
|
||||
expect(aStarted).toBe(true)
|
||||
})
|
||||
|
||||
const pB = coalescer.run({
|
||||
key: 'term_b',
|
||||
run: async () => ({ id: 'b', navigated: true }),
|
||||
resolveSuperseded: () => ({ id: 'b', navigated: false })
|
||||
})
|
||||
|
||||
aGate.resolve()
|
||||
await expect(pA).resolves.toEqual({ id: 'a', navigated: false })
|
||||
await expect(pB).resolves.toEqual({ id: 'b', navigated: true })
|
||||
})
|
||||
|
||||
it('marks a synchronous run superseded when it queues a newer job before settling', async () => {
|
||||
const coalescer = new TerminalFocusNavigationCoalescer<{
|
||||
id: string
|
||||
navigated: boolean
|
||||
}>()
|
||||
let latest!: Promise<{ id: string; navigated: boolean }>
|
||||
|
||||
const first = coalescer.run({
|
||||
key: 'term_a',
|
||||
run: async () => {
|
||||
latest = coalescer.run({
|
||||
key: 'term_b',
|
||||
run: async () => ({ id: 'b', navigated: true }),
|
||||
resolveSuperseded: () => ({ id: 'b', navigated: false })
|
||||
})
|
||||
return { id: 'a', navigated: true }
|
||||
},
|
||||
resolveSuperseded: (completed) => ({
|
||||
id: completed?.id ?? 'a',
|
||||
navigated: false
|
||||
})
|
||||
})
|
||||
|
||||
await expect(first).resolves.toEqual({ id: 'a', navigated: false })
|
||||
await expect(latest).resolves.toEqual({ id: 'b', navigated: true })
|
||||
})
|
||||
|
||||
it('propagates run failures without stranding the queue', async () => {
|
||||
const coalescer = new TerminalFocusNavigationCoalescer<string>()
|
||||
const aGate = deferred<void>()
|
||||
let aStarted = false
|
||||
|
||||
const pA = coalescer.run({
|
||||
key: 'term_a',
|
||||
run: async () => {
|
||||
aStarted = true
|
||||
await aGate.promise
|
||||
throw new Error('boom')
|
||||
},
|
||||
resolveSuperseded: () => 'super-a'
|
||||
})
|
||||
await vi.waitFor(() => {
|
||||
expect(aStarted).toBe(true)
|
||||
})
|
||||
|
||||
const pB = coalescer.run({
|
||||
key: 'term_b',
|
||||
run: async () => 'full-b',
|
||||
resolveSuperseded: () => 'super-b'
|
||||
})
|
||||
|
||||
aGate.resolve()
|
||||
// A is obsolete when it fails after B enqueued — settles as superseded, not boom.
|
||||
await expect(pA).resolves.toBe('super-a')
|
||||
await expect(pB).resolves.toBe('full-b')
|
||||
expect(coalescer.getState().running).toBe(false)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
/**
|
||||
* Latest-wins single-flight coalescer for exclusive host terminal focus navigation.
|
||||
*
|
||||
* Scope: concurrent `terminal.focus` / host-nav storms (CLI switch fan-out, bulk open).
|
||||
* Not a substitute for cheaper activation or reconnect-scan bounding.
|
||||
*
|
||||
* Why: only one host terminal can be focused. Intermediate navigations are waste;
|
||||
* running them in parallel freezes large remote fleets. Pending jobs collapse to
|
||||
* the newest; in-flight work re-checks a generation so obsolete runs do not claim
|
||||
* a successful host navigation after a newer focus arrived.
|
||||
*/
|
||||
|
||||
export type TerminalFocusNavigationContext = {
|
||||
/** True while this job is still the newest focus request. */
|
||||
isCurrent: () => boolean
|
||||
}
|
||||
|
||||
export type TerminalFocusNavigationJob<TResult> = {
|
||||
/** Stable key for diagnostics (usually terminal handle). */
|
||||
key: string
|
||||
/**
|
||||
* Full navigation work (reveal/focus host UI).
|
||||
* Must consult `ctx.isCurrent()` before/after expensive host work and avoid
|
||||
* claiming navigation when false.
|
||||
*/
|
||||
run: (ctx: TerminalFocusNavigationContext) => Promise<TResult>
|
||||
/**
|
||||
* Result when this job is dropped (pending superseded) or becomes obsolete
|
||||
* mid-flight. Must not perform host navigation. Should set navigated: false.
|
||||
*/
|
||||
resolveSuperseded: (completed?: TResult) => TResult
|
||||
}
|
||||
|
||||
type PendingJob<TResult> = {
|
||||
key: string
|
||||
generation: number
|
||||
run: (ctx: TerminalFocusNavigationContext) => Promise<TResult>
|
||||
resolve: (value: TResult) => void
|
||||
reject: (error: unknown) => void
|
||||
resolveSuperseded: (completed?: TResult) => TResult
|
||||
}
|
||||
|
||||
export class TerminalFocusNavigationCoalescer<TResult> {
|
||||
private running = false
|
||||
private pending: PendingJob<TResult> | null = null
|
||||
private activeKey: string | null = null
|
||||
private generation = 0
|
||||
|
||||
getState(): {
|
||||
running: boolean
|
||||
activeKey: string | null
|
||||
pendingKey: string | null
|
||||
generation: number
|
||||
} {
|
||||
return {
|
||||
running: this.running,
|
||||
activeKey: this.activeKey,
|
||||
pendingKey: this.pending?.key ?? null,
|
||||
generation: this.generation
|
||||
}
|
||||
}
|
||||
|
||||
run(job: TerminalFocusNavigationJob<TResult>): Promise<TResult> {
|
||||
return new Promise<TResult>((resolve, reject) => {
|
||||
if (this.pending) {
|
||||
try {
|
||||
this.pending.resolve(this.pending.resolveSuperseded())
|
||||
} catch (error) {
|
||||
this.pending.reject(error)
|
||||
}
|
||||
}
|
||||
const generation = ++this.generation
|
||||
this.pending = {
|
||||
key: job.key,
|
||||
generation,
|
||||
run: job.run,
|
||||
resolve,
|
||||
reject,
|
||||
resolveSuperseded: job.resolveSuperseded
|
||||
}
|
||||
void this.pump()
|
||||
})
|
||||
}
|
||||
|
||||
private async pump(): Promise<void> {
|
||||
if (this.running) {
|
||||
return
|
||||
}
|
||||
this.running = true
|
||||
try {
|
||||
while (this.pending) {
|
||||
const job = this.pending
|
||||
this.pending = null
|
||||
this.activeKey = job.key
|
||||
const ctx: TerminalFocusNavigationContext = {
|
||||
isCurrent: () => job.generation === this.generation
|
||||
}
|
||||
try {
|
||||
if (!ctx.isCurrent()) {
|
||||
job.resolve(job.resolveSuperseded())
|
||||
continue
|
||||
}
|
||||
const result = await job.run(ctx)
|
||||
job.resolve(ctx.isCurrent() ? result : job.resolveSuperseded(result))
|
||||
} catch (error) {
|
||||
if (ctx.isCurrent()) {
|
||||
job.reject(error)
|
||||
} else {
|
||||
try {
|
||||
job.resolve(job.resolveSuperseded())
|
||||
} catch (supersedeError) {
|
||||
job.reject(supersedeError)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.activeKey = null
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.running = false
|
||||
if (this.pending) {
|
||||
void this.pump()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -681,6 +681,12 @@ export type RuntimeTerminalFocus = {
|
|||
handle: string
|
||||
tabId: string
|
||||
worktreeId: string
|
||||
/**
|
||||
* Whether this request remained the winning applied host navigation when it settled.
|
||||
* False also covers identity-only requests and unavailable host navigation.
|
||||
* Optional for older clients; omit only when unknown.
|
||||
*/
|
||||
navigated?: boolean
|
||||
}
|
||||
|
||||
export type RuntimeTerminalClose = {
|
||||
|
|
|
|||
|
|
@ -82,6 +82,33 @@ export function formatHeadlessPairedRuntimeStartupDiagnostics(
|
|||
.join('\n')
|
||||
}
|
||||
|
||||
export function parseHeadlessPairedRuntimePairingOffer(
|
||||
line: string
|
||||
): RuntimeDesktopPairingOffer | null {
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(line) as unknown
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
if (parsed === null || typeof parsed !== 'object') {
|
||||
return null
|
||||
}
|
||||
const readiness = parsed as ServeReady
|
||||
const pairing = readiness.pairing
|
||||
if (
|
||||
readiness.type !== 'orca_server_ready' ||
|
||||
pairing?.available !== true ||
|
||||
typeof pairing.url !== 'string'
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
pairingUrl: pairing.url,
|
||||
...(typeof pairing.webClientUrl === 'string' ? { webClientUrl: pairing.webClientUrl } : {})
|
||||
}
|
||||
}
|
||||
|
||||
function redactPairingMaterial(value: string): string {
|
||||
return value
|
||||
.replace(PAIRING_URL_PATTERN, 'orca://[redacted]')
|
||||
|
|
@ -138,23 +165,12 @@ async function readPairingOffer(app: ElectronApplication): Promise<RuntimeDeskto
|
|||
const lines = buffered.split(/\r?\n/)
|
||||
buffered = lines.pop() ?? ''
|
||||
for (const line of lines) {
|
||||
let readiness: ServeReady
|
||||
try {
|
||||
readiness = JSON.parse(line) as ServeReady
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
const pairing = readiness.pairing
|
||||
if (
|
||||
readiness.type !== 'orca_server_ready' ||
|
||||
pairing?.available !== true ||
|
||||
typeof pairing.url !== 'string' ||
|
||||
typeof pairing.webClientUrl !== 'string'
|
||||
) {
|
||||
const offer = parseHeadlessPairedRuntimePairingOffer(line)
|
||||
if (!offer) {
|
||||
continue
|
||||
}
|
||||
cleanup()
|
||||
resolve({ pairingUrl: pairing.url, webClientUrl: pairing.webClientUrl })
|
||||
resolve(offer)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
HeadlessPairedRuntimeStartupDiagnosticBuffer,
|
||||
formatHeadlessPairedRuntimeStartupDiagnostics
|
||||
formatHeadlessPairedRuntimeStartupDiagnostics,
|
||||
parseHeadlessPairedRuntimePairingOffer
|
||||
} from './headless-paired-runtime-host'
|
||||
|
||||
describe('headless paired runtime startup diagnostics', () => {
|
||||
|
|
@ -43,3 +44,41 @@ describe('headless paired runtime startup diagnostics', () => {
|
|||
expect(diagnostic.read()).toBe('safe')
|
||||
})
|
||||
})
|
||||
|
||||
describe('headless paired runtime readiness', () => {
|
||||
it.each(['null', 'true', '0', '"ready"', '[]'])(
|
||||
'ignores JSON primitives and non-object readiness payloads: %s',
|
||||
(payload) => {
|
||||
expect(parseHeadlessPairedRuntimePairingOffer(payload)).toBeNull()
|
||||
}
|
||||
)
|
||||
|
||||
it('accepts desktop-only pairing readiness', () => {
|
||||
expect(
|
||||
parseHeadlessPairedRuntimePairingOffer(
|
||||
JSON.stringify({
|
||||
type: 'orca_server_ready',
|
||||
pairing: { available: true, url: 'orca://pairing-secret', webClientUrl: null }
|
||||
})
|
||||
)
|
||||
).toEqual({ pairingUrl: 'orca://pairing-secret' })
|
||||
})
|
||||
|
||||
it('preserves an available web-client URL', () => {
|
||||
expect(
|
||||
parseHeadlessPairedRuntimePairingOffer(
|
||||
JSON.stringify({
|
||||
type: 'orca_server_ready',
|
||||
pairing: {
|
||||
available: true,
|
||||
url: 'orca://pairing-secret',
|
||||
webClientUrl: 'https://example.test/web-index.html#pairing=secret'
|
||||
}
|
||||
})
|
||||
)
|
||||
).toEqual({
|
||||
pairingUrl: 'orca://pairing-secret',
|
||||
webClientUrl: 'https://example.test/web-index.html#pairing=secret'
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'\\''`)}'`
|
||||
}
|
||||
|
||||
function fixtureCommand(fixturePath: string, marker: string): string {
|
||||
const command = [process.execPath, fixturePath, marker]
|
||||
return process.platform === 'win32'
|
||||
? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ')
|
||||
: command.map(shellQuote).join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Continuous remote-agent-like flood fixture for freeze repros.
|
||||
* Many remote sessions stream while the client bulk-opens them.
|
||||
* One-shot FLOOD is not enough — agents keep writing.
|
||||
*/
|
||||
export function createRemoteSessionBulkOpenFixture(): {
|
||||
command: (marker: string) => string
|
||||
dispose: () => void
|
||||
} {
|
||||
const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-remote-bulk-open-'))
|
||||
const fixturePath = path.join(scratch, 'remote-bulk-open-flood.mjs')
|
||||
writeFileSync(
|
||||
fixturePath,
|
||||
[
|
||||
'const marker = process.argv[2]',
|
||||
'process.stdout.write(`READY:${marker}\\r\\n`)',
|
||||
'process.stdin.setRawMode?.(true)',
|
||||
"process.stdin.setEncoding('utf8')",
|
||||
'let frame = 0',
|
||||
'let timer = null',
|
||||
"const chunk = 'A'.repeat(2048)",
|
||||
'function startFlood() {',
|
||||
' if (timer) return',
|
||||
' timer = setInterval(() => {',
|
||||
' frame += 1',
|
||||
' process.stdout.write(`BG:${marker}:${frame}:${chunk}\\r\\n`)',
|
||||
' }, 8)',
|
||||
'}',
|
||||
'function stopFlood() {',
|
||||
' if (timer) clearInterval(timer)',
|
||||
' timer = null',
|
||||
'}',
|
||||
// Auto-start flood after ready so hidden tabs accumulate backlog.
|
||||
'setTimeout(startFlood, 200)',
|
||||
"process.stdin.on('data', (data) => {",
|
||||
' for (const command of data.split(/\\r\\n|\\r|\\n/).filter(Boolean)) {',
|
||||
" if (command === 'GO' || command === 'FLOOD') { startFlood(); continue }",
|
||||
" if (command === 'STOP') { stopFlood(); process.stdout.write(`STOPPED:${marker}\\r\\n`); continue }",
|
||||
" if (command === 'PING') { process.stdout.write(`PONG:${marker}:${frame}\\r\\n`); continue }",
|
||||
' process.stdout.write(`LIVE:${marker}:${command}\\r\\n`)',
|
||||
' }',
|
||||
'})',
|
||||
'process.stdin.resume()',
|
||||
"process.on('SIGINT', () => { stopFlood(); process.exit(0) })"
|
||||
].join('\n')
|
||||
)
|
||||
return {
|
||||
command: (marker) => fixtureCommand(fixturePath, marker),
|
||||
dispose: () => rmSync(scratch, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,271 @@
|
|||
import { writeFileSync, mkdirSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import type { Page } from '@stablyai/playwright-test'
|
||||
import { toWebTerminalSurfaceTabId } from '../../../src/shared/terminal-surface-id'
|
||||
import { expect } from './orca-app'
|
||||
import { createRemoteSessionBulkOpenFixture } from './remote-session-bulk-open-fixture'
|
||||
import { startRendererLagProbe } from '../paired-runtime-retention-metrics'
|
||||
import { closeStreamingTerminals } from './streaming-terminal-cleanup'
|
||||
import { waitForActivePanePtyId } from './terminal'
|
||||
|
||||
/** Multi-worktree load: several agent-like streaming terminals per worktree. */
|
||||
export const BULK_OPEN_WORKTREE_COUNT = 3
|
||||
export const BULK_OPEN_TABS_PER_WORKTREE = 4
|
||||
/** Soft freeze signal — UI feels stuck. */
|
||||
export const SOFT_FREEZE_LAG_MS = 2_000
|
||||
/** Hard freeze signal — matches trusted "screen fully frozen" reports. */
|
||||
export const HARD_FREEZE_LAG_MS = 5_000
|
||||
|
||||
export type BulkOpenSession = {
|
||||
marker: string
|
||||
tabId: string
|
||||
terminal: string
|
||||
worktreeId: string
|
||||
}
|
||||
|
||||
export type BulkOpenFreezeReport = {
|
||||
bulkOpenMaxLagMs: number
|
||||
hiddenFloodMaxLagMs: number
|
||||
interactionProbeMs: number
|
||||
hardFreeze: boolean
|
||||
softFreeze: boolean
|
||||
sessionCount: number
|
||||
worktreeCount: number
|
||||
topology: 'paired-remote-server' | 'docker-ssh'
|
||||
versionHint: string
|
||||
notes: string[]
|
||||
}
|
||||
|
||||
async function callRuntime<TResult>(page: Page, method: string, params: unknown): Promise<TResult> {
|
||||
return page.evaluate(
|
||||
async ({ method, params }) => {
|
||||
const response = await window.api.runtime.call({ method, params })
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.error.code}: ${response.error.message}`)
|
||||
}
|
||||
return response.result
|
||||
},
|
||||
{ method, params }
|
||||
) as Promise<TResult>
|
||||
}
|
||||
|
||||
async function measureRendererInteractionMs(page: Page): Promise<number> {
|
||||
return page.evaluate(async () => {
|
||||
const started = performance.now()
|
||||
if (!window.__store) {
|
||||
throw new Error('store unavailable for interaction probe')
|
||||
}
|
||||
// A blocked renderer cannot service the input task or paint the following frames.
|
||||
await new Promise<void>((resolve) => {
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve()))
|
||||
})
|
||||
return performance.now() - started
|
||||
})
|
||||
}
|
||||
|
||||
export async function seedBulkOpenRemoteSessions(
|
||||
page: Page,
|
||||
seed: { repoId: string }
|
||||
): Promise<{ sessions: BulkOpenSession[]; dispose: () => Promise<void> }> {
|
||||
const fixture = createRemoteSessionBulkOpenFixture()
|
||||
const sessions: BulkOpenSession[] = []
|
||||
const closeSessions = async (): Promise<void> => {
|
||||
try {
|
||||
await closeStreamingTerminals(
|
||||
sessions.map((session) => session.terminal),
|
||||
(method, terminal) => callRuntime(page, method, { terminal })
|
||||
)
|
||||
} finally {
|
||||
fixture.dispose()
|
||||
}
|
||||
}
|
||||
try {
|
||||
for (let w = 0; w < BULK_OPEN_WORKTREE_COUNT; w += 1) {
|
||||
const marker = `BULK_WT_${w}_T0`
|
||||
const created = await callRuntime<{
|
||||
startupTerminal?: { handle?: string; tabId?: string }
|
||||
worktree: { id: string }
|
||||
}>(page, 'worktree.create', {
|
||||
repo: seed.repoId,
|
||||
name: `bulk-open-wt-${w}-${Date.now()}`,
|
||||
setupDecision: 'skip',
|
||||
activate: false,
|
||||
noParent: true,
|
||||
startupCommand: fixture.command(marker)
|
||||
})
|
||||
if (!created.startupTerminal?.handle || !created.startupTerminal.tabId) {
|
||||
throw new Error(`Bulk-open worktree ${w} missing startup terminal`)
|
||||
}
|
||||
const worktreeId = created.worktree.id
|
||||
sessions.push({
|
||||
marker,
|
||||
tabId: toWebTerminalSurfaceTabId(created.startupTerminal.tabId),
|
||||
terminal: created.startupTerminal.handle,
|
||||
worktreeId
|
||||
})
|
||||
|
||||
for (let t = 1; t < BULK_OPEN_TABS_PER_WORKTREE; t += 1) {
|
||||
const tabMarker = `BULK_WT_${w}_T${t}`
|
||||
const result = await callRuntime<{
|
||||
tab: { parentTabId: string; terminal: string | null }
|
||||
}>(page, 'session.tabs.createTerminal', {
|
||||
worktree: `id:${worktreeId}`,
|
||||
command: fixture.command(tabMarker),
|
||||
activate: false,
|
||||
select: false,
|
||||
navigation: 'caller'
|
||||
})
|
||||
if (!result.tab.terminal) {
|
||||
throw new Error(`Bulk-open terminal ${tabMarker} was not created`)
|
||||
}
|
||||
sessions.push({
|
||||
marker: tabMarker,
|
||||
tabId: toWebTerminalSurfaceTabId(result.tab.parentTabId),
|
||||
terminal: result.tab.terminal,
|
||||
worktreeId
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure fixtures started and are streaming on the host.
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const ready = await Promise.all(
|
||||
sessions.map(async (session) => {
|
||||
const result = await callRuntime<{ terminal: { tail: string[] } }>(
|
||||
page,
|
||||
'terminal.read',
|
||||
{ terminal: session.terminal, limit: 200 }
|
||||
)
|
||||
const text = result.terminal.tail.join('\n')
|
||||
return text.includes(`BG:${session.marker}:`)
|
||||
})
|
||||
)
|
||||
return ready.every(Boolean)
|
||||
},
|
||||
{ timeout: 60_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
|
||||
return {
|
||||
sessions,
|
||||
dispose: closeSessions
|
||||
}
|
||||
} catch (error) {
|
||||
await closeSessions().catch((cleanupError) => {
|
||||
throw new AggregateError(
|
||||
[error, cleanupError],
|
||||
'Bulk-open session seeding and cleanup failed'
|
||||
)
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Repro R1 core: leave remotes streaming hidden, then burst-open sessions
|
||||
* (reopening remote sessions after agents have been writing in the background).
|
||||
*/
|
||||
export async function runBulkOpenFreezeOracle(
|
||||
page: Page,
|
||||
sessions: BulkOpenSession[],
|
||||
opts: {
|
||||
topology: BulkOpenFreezeReport['topology']
|
||||
versionHint?: string
|
||||
reportDir?: string
|
||||
}
|
||||
): Promise<BulkOpenFreezeReport> {
|
||||
const notes: string[] = []
|
||||
const worktreeIds = [...new Set(sessions.map((s) => s.worktreeId))]
|
||||
|
||||
// Leave terminal view so panes can park / go inactive while flooding.
|
||||
await page.evaluate(() => window.__store?.getState().setActiveView('tasks'))
|
||||
await page.evaluate(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => resolve()))
|
||||
})
|
||||
)
|
||||
// Accumulate remote flood for several seconds (agent backlog).
|
||||
await page.waitForTimeout(4_000)
|
||||
|
||||
const hiddenProbe = await startRendererLagProbe(page)
|
||||
await page.waitForTimeout(2_000)
|
||||
const hiddenFloodMaxLagMs = await hiddenProbe.evaluate((probe) => probe.stop())
|
||||
await hiddenProbe.dispose()
|
||||
notes.push(`hidden streaming lag max=${hiddenFloodMaxLagMs.toFixed(0)}ms`)
|
||||
|
||||
// Burst open remote sessions (worktree + tab activate).
|
||||
const openProbe = await startRendererLagProbe(page)
|
||||
const openStarted = Date.now()
|
||||
for (const worktreeId of worktreeIds) {
|
||||
const tabs = sessions.filter((session) => session.worktreeId === worktreeId)
|
||||
for (const tab of tabs) {
|
||||
await page.evaluate(
|
||||
({ targetWorktreeId, tabId }) => {
|
||||
const state = window.__store?.getState()
|
||||
state?.setActiveView('terminal')
|
||||
state?.setActiveWorktree(targetWorktreeId)
|
||||
state?.setActiveTabForWorktree(targetWorktreeId, tabId)
|
||||
},
|
||||
{ targetWorktreeId: worktreeId, tabId: tab.tabId }
|
||||
)
|
||||
}
|
||||
}
|
||||
// One more full pass clicking visible tabs if present.
|
||||
for (const session of sessions) {
|
||||
const locator = page.locator(`[data-testid="sortable-tab"][data-tab-id="${session.tabId}"]`)
|
||||
if (await locator.isVisible().catch(() => false)) {
|
||||
await locator.click({ timeout: 2_000 }).catch(() => undefined)
|
||||
}
|
||||
}
|
||||
// Let the storm settle enough to measure residual lag.
|
||||
await page.waitForTimeout(3_000)
|
||||
const bulkOpenMaxLagMs = await openProbe.evaluate((probe) => probe.stop())
|
||||
await openProbe.dispose()
|
||||
notes.push(`bulk open wall=${Date.now() - openStarted}ms lagMax=${bulkOpenMaxLagMs.toFixed(0)}ms`)
|
||||
|
||||
// Confirm last session is live after the storm (host PTYs survived).
|
||||
const last = sessions.at(-1)
|
||||
if (!last) {
|
||||
throw new Error('bulk-open freeze oracle requires at least one session')
|
||||
}
|
||||
await page.evaluate(
|
||||
({ targetWorktreeId, tabId }) => {
|
||||
const state = window.__store?.getState()
|
||||
state?.setActiveView('terminal')
|
||||
state?.setActiveWorktree(targetWorktreeId)
|
||||
state?.setActiveTabForWorktree(targetWorktreeId, tabId)
|
||||
},
|
||||
{ targetWorktreeId: last.worktreeId, tabId: last.tabId }
|
||||
)
|
||||
await waitForActivePanePtyId(page, 30_000).catch(() => {
|
||||
notes.push('active pane PTY id not ready after bulk open (possible re-attach failure)')
|
||||
})
|
||||
|
||||
const interactionProbeMs = await measureRendererInteractionMs(page)
|
||||
notes.push(`post-storm renderer interaction=${interactionProbeMs.toFixed(0)}ms`)
|
||||
|
||||
const report: BulkOpenFreezeReport = {
|
||||
bulkOpenMaxLagMs,
|
||||
hiddenFloodMaxLagMs,
|
||||
interactionProbeMs,
|
||||
hardFreeze: bulkOpenMaxLagMs >= HARD_FREEZE_LAG_MS || interactionProbeMs >= HARD_FREEZE_LAG_MS,
|
||||
softFreeze: bulkOpenMaxLagMs >= SOFT_FREEZE_LAG_MS || interactionProbeMs >= SOFT_FREEZE_LAG_MS,
|
||||
sessionCount: sessions.length,
|
||||
worktreeCount: worktreeIds.length,
|
||||
topology: opts.topology,
|
||||
versionHint: opts.versionHint ?? process.env.ORCA_VERSION ?? 'unknown',
|
||||
notes
|
||||
}
|
||||
|
||||
if (opts.reportDir) {
|
||||
mkdirSync(opts.reportDir, { recursive: true })
|
||||
const outPath = path.join(opts.reportDir, `bulk-open-freeze-${opts.topology}.json`)
|
||||
writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`)
|
||||
notes.push(`wrote ${outPath}`)
|
||||
}
|
||||
|
||||
return report
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
export async function closeStreamingTerminals(
|
||||
terminals: string[],
|
||||
call: (method: 'terminal.closeTab' | 'terminal.close', terminal: string) => Promise<unknown>
|
||||
): Promise<void> {
|
||||
const results = await Promise.allSettled(
|
||||
terminals.map(async (terminal) => {
|
||||
try {
|
||||
await call('terminal.closeTab', terminal)
|
||||
} catch (closeTabError) {
|
||||
try {
|
||||
await call('terminal.close', terminal)
|
||||
} catch (closeError) {
|
||||
throw new AggregateError(
|
||||
[closeTabError, closeError],
|
||||
`Failed to close streaming terminal ${terminal}`
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
const failures = results.flatMap((result) =>
|
||||
result.status === 'rejected' ? [result.reason] : []
|
||||
)
|
||||
if (failures.length > 0) {
|
||||
throw new AggregateError(failures, `Failed to close ${failures.length} streaming terminal(s)`)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { closeStreamingTerminals } from './streaming-terminal-cleanup'
|
||||
|
||||
describe('closeStreamingTerminals', () => {
|
||||
it('force-closes a streaming PTY when tab cleanup fails', async () => {
|
||||
const call = vi.fn(async (method: string, terminal: string) => {
|
||||
if (method === 'terminal.closeTab' && terminal === 'term_a') {
|
||||
throw new Error('renderer unavailable')
|
||||
}
|
||||
})
|
||||
|
||||
await expect(closeStreamingTerminals(['term_a', 'term_b'], call)).resolves.toBeUndefined()
|
||||
|
||||
expect(call).toHaveBeenCalledWith('terminal.closeTab', 'term_a')
|
||||
expect(call).toHaveBeenCalledWith('terminal.close', 'term_a')
|
||||
expect(call).toHaveBeenCalledWith('terminal.closeTab', 'term_b')
|
||||
})
|
||||
|
||||
it('waits for every fallback and reports terminals that could not be stopped', async () => {
|
||||
const call = vi.fn(async () => {
|
||||
throw new Error('runtime frozen')
|
||||
})
|
||||
|
||||
await expect(closeStreamingTerminals(['term_a', 'term_b'], call)).rejects.toThrow(
|
||||
'Failed to close 2 streaming terminal(s)'
|
||||
)
|
||||
expect(call).toHaveBeenCalledTimes(4)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
import type { Page } from '@stablyai/playwright-test'
|
||||
import type { RuntimeTerminalFocus } from '../../../src/shared/runtime-types'
|
||||
import { expect } from './orca-app'
|
||||
import { createRemoteSessionBulkOpenFixture } from './remote-session-bulk-open-fixture'
|
||||
import { closeStreamingTerminals } from './streaming-terminal-cleanup'
|
||||
|
||||
export type HostFocusStormSession = {
|
||||
marker: string
|
||||
terminal: string
|
||||
worktreeId: string
|
||||
}
|
||||
|
||||
async function callRuntime<TResult>(
|
||||
page: Page,
|
||||
method: string,
|
||||
params: unknown,
|
||||
environmentId?: string
|
||||
): Promise<TResult> {
|
||||
return page.evaluate(
|
||||
async ({ environmentId, method, params }) => {
|
||||
const response = environmentId
|
||||
? await window.api.runtimeEnvironments.call({ selector: environmentId, method, params })
|
||||
: await window.api.runtime.call({ method, params })
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.error.code}: ${response.error.message}`)
|
||||
}
|
||||
return response.result
|
||||
},
|
||||
{ environmentId: environmentId ?? null, method, params }
|
||||
) as Promise<TResult>
|
||||
}
|
||||
|
||||
export async function seedHostFocusStormSessions(
|
||||
page: Page,
|
||||
worktreeId: string,
|
||||
count = 6,
|
||||
environmentId?: string
|
||||
): Promise<{ sessions: HostFocusStormSession[]; dispose: () => Promise<void> }> {
|
||||
const fixture = createRemoteSessionBulkOpenFixture()
|
||||
const sessions: HostFocusStormSession[] = []
|
||||
const closeSessions = async (): Promise<void> => {
|
||||
try {
|
||||
await closeStreamingTerminals(
|
||||
sessions.map((session) => session.terminal),
|
||||
(method, terminal) => callRuntime(page, method, { terminal }, environmentId)
|
||||
)
|
||||
} finally {
|
||||
fixture.dispose()
|
||||
}
|
||||
}
|
||||
try {
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const marker = `HOST_FOCUS_${index}`
|
||||
const params = {
|
||||
worktree: `id:${worktreeId}`,
|
||||
command: fixture.command(marker),
|
||||
activate: false,
|
||||
select: false,
|
||||
navigation: 'caller'
|
||||
}
|
||||
const result = await callRuntime<{ tab: { terminal: string | null } }>(
|
||||
page,
|
||||
'session.tabs.createTerminal',
|
||||
params,
|
||||
environmentId
|
||||
)
|
||||
if (!result.tab.terminal) {
|
||||
throw new Error(`Host focus terminal ${marker} was not created`)
|
||||
}
|
||||
sessions.push({ marker, terminal: result.tab.terminal, worktreeId })
|
||||
}
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const ready = await Promise.all(
|
||||
sessions.map(async (session) => {
|
||||
const params = { terminal: session.terminal, limit: 200 }
|
||||
const result = await callRuntime<{ terminal: { tail: string[] } }>(
|
||||
page,
|
||||
'terminal.read',
|
||||
params,
|
||||
environmentId
|
||||
)
|
||||
return result.terminal.tail.join('\n').includes(`BG:${session.marker}:`)
|
||||
})
|
||||
)
|
||||
return ready.every(Boolean)
|
||||
},
|
||||
{ timeout: 60_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
return { sessions, dispose: closeSessions }
|
||||
} catch (error) {
|
||||
await closeSessions().catch((cleanupError) => {
|
||||
throw new AggregateError(
|
||||
[error, cleanupError],
|
||||
'Focus-storm session seeding and cleanup failed'
|
||||
)
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function runHostFocusStorm(
|
||||
page: Page,
|
||||
sessions: HostFocusStormSession[],
|
||||
environmentId?: string
|
||||
): Promise<RuntimeTerminalFocus[]> {
|
||||
if (sessions.length < 2) {
|
||||
throw new Error('host focus storm requires at least two terminals')
|
||||
}
|
||||
return page.evaluate(
|
||||
async ({ environmentId, targets }) => {
|
||||
const callFocus = (terminal: string) =>
|
||||
environmentId
|
||||
? window.api.runtimeEnvironments.call({
|
||||
selector: environmentId,
|
||||
method: 'terminal.focus',
|
||||
params: { terminal, navigation: 'host' }
|
||||
})
|
||||
: window.api.runtime.call({
|
||||
method: 'terminal.focus',
|
||||
params: { terminal, navigation: 'host' }
|
||||
})
|
||||
const prior = targets.slice(0, -1).map((target) => callFocus(target.terminal))
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0))
|
||||
const latestTarget = targets.at(-1)
|
||||
if (!latestTarget) {
|
||||
throw new Error('host focus storm lost its latest target')
|
||||
}
|
||||
const latest = callFocus(latestTarget.terminal)
|
||||
const responses = await Promise.all([...prior, latest])
|
||||
return responses.map((response) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`${response.error.code}: ${response.error.message}`)
|
||||
}
|
||||
return (response.result as { focus: RuntimeTerminalFocus }).focus
|
||||
})
|
||||
},
|
||||
{ environmentId: environmentId ?? null, targets: sessions }
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,192 @@
|
|||
/**
|
||||
* Freeze repro R1 — bulk-open remote sessions under multi-worktree flood load.
|
||||
*
|
||||
* Trigger: reopening many remote sessions on Remote Server / SSH with agents.
|
||||
*
|
||||
* Topology under test:
|
||||
* R1: headless Remote Orca host + paired desktop web client (paired-remote-server)
|
||||
*
|
||||
* Measurement:
|
||||
* renderer timer drift during hidden flood + bulk worktree/tab open.
|
||||
* soft freeze >= 2s, hard freeze >= 5s.
|
||||
*
|
||||
* Run:
|
||||
* SKIP_BUILD=1 pnpm exec playwright test \
|
||||
* tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts \
|
||||
* --config tests/playwright.config.ts \
|
||||
* --project electron-headless --workers=1
|
||||
*
|
||||
* Or:
|
||||
* pnpm run test:e2e:remote-bulk-open-freeze
|
||||
*/
|
||||
import path from 'node:path'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host'
|
||||
import {
|
||||
createRuntimeDesktopPairingOffer,
|
||||
launchPairedElectronClient,
|
||||
launchPairedWebClient,
|
||||
type PairedElectronClient,
|
||||
type PairedWebClient
|
||||
} from './helpers/paired-electron-client'
|
||||
import {
|
||||
HARD_FREEZE_LAG_MS,
|
||||
runBulkOpenFreezeOracle,
|
||||
seedBulkOpenRemoteSessions,
|
||||
SOFT_FREEZE_LAG_MS
|
||||
} from './helpers/remote-session-bulk-open-oracle'
|
||||
import {
|
||||
runHostFocusStorm,
|
||||
seedHostFocusStormSessions
|
||||
} from './helpers/terminal-host-focus-storm-oracle'
|
||||
|
||||
const REPORT_DIR = path.join(process.cwd(), 'test-results', 'freeze-repro')
|
||||
const USE_DESKTOP_PAIR = process.env.ORCA_E2E_FREEZE_DESKTOP_PAIR === '1'
|
||||
|
||||
test('paired client host-focus storm keeps the latest terminal @freeze-repro', async ({
|
||||
orcaPage
|
||||
}, testInfo) => {
|
||||
test.setTimeout(180_000)
|
||||
const offer = await createRuntimeDesktopPairingOffer(orcaPage)
|
||||
const client = await launchPairedElectronClient(offer, testInfo, 'focus-storm')
|
||||
let disposeSessions: (() => Promise<void>) | null = null
|
||||
try {
|
||||
const worktreeId = await orcaPage.evaluate(() => {
|
||||
const id = window.__store?.getState().activeWorktreeId
|
||||
if (!id) {
|
||||
throw new Error('headed host has no active worktree')
|
||||
}
|
||||
return id
|
||||
})
|
||||
const environmentId = await client.page.evaluate(async () => {
|
||||
const environment = (await window.api.runtimeEnvironments.list())[0]
|
||||
if (!environment) {
|
||||
throw new Error('paired client has no runtime environment')
|
||||
}
|
||||
return environment.id
|
||||
})
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
client.page.evaluate(
|
||||
(id) =>
|
||||
window.__store
|
||||
?.getState()
|
||||
.allWorktrees()
|
||||
.some((worktree) => worktree.id === id) ?? false,
|
||||
worktreeId
|
||||
),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBe(true)
|
||||
const seeded = await seedHostFocusStormSessions(client.page, worktreeId, 6, environmentId)
|
||||
disposeSessions = seeded.dispose
|
||||
const results = await runHostFocusStorm(client.page, seeded.sessions, environmentId)
|
||||
const latest = results.at(-1)
|
||||
const expected = seeded.sessions.at(-1)
|
||||
expect(latest).toMatchObject({
|
||||
handle: expected?.terminal,
|
||||
worktreeId,
|
||||
navigated: true
|
||||
})
|
||||
expect(results.slice(0, -1).some((result) => result.navigated === false)).toBe(true)
|
||||
await expect
|
||||
.poll(
|
||||
() =>
|
||||
orcaPage.evaluate((id) => {
|
||||
const state = window.__store?.getState()
|
||||
return {
|
||||
worktreeId: state?.activeWorktreeId ?? null,
|
||||
tabId: state?.activeTabIdByWorktree[id] ?? state?.activeTabId ?? null
|
||||
}
|
||||
}, worktreeId),
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toEqual({ worktreeId, tabId: latest?.tabId })
|
||||
} finally {
|
||||
await disposeSessions?.()
|
||||
await client.dispose()
|
||||
}
|
||||
})
|
||||
|
||||
test('R1 paired remote bulk-open freeze oracle @freeze-repro', async ({
|
||||
testRepoPath
|
||||
}, testInfo) => {
|
||||
test.setTimeout(420_000)
|
||||
const host = await launchHeadlessPairedRuntimeHost()
|
||||
let webClient: PairedWebClient | null = null
|
||||
let desktopClient: PairedElectronClient | null = null
|
||||
let disposeSessions: (() => Promise<void>) | null = null
|
||||
try {
|
||||
const added = await host.client.call<{ repo: { id: string } }>('repo.add', {
|
||||
path: testRepoPath,
|
||||
kind: 'git'
|
||||
})
|
||||
await expect
|
||||
.poll(
|
||||
async () => {
|
||||
const listed = await host.client.call<{ totalCount: number }>('worktree.list', {
|
||||
repo: `id:${added.result.repo.id}`
|
||||
})
|
||||
return listed.result.totalCount
|
||||
},
|
||||
{ timeout: 30_000 }
|
||||
)
|
||||
.toBeGreaterThan(0)
|
||||
|
||||
// Prefer full desktop pair when web-client store hydration is flaky in this env.
|
||||
const page = await (async () => {
|
||||
if (USE_DESKTOP_PAIR) {
|
||||
desktopClient = await launchPairedElectronClient(host.offer, testInfo, 'freeze-r1')
|
||||
return desktopClient.page
|
||||
}
|
||||
webClient = await launchPairedWebClient(host.app, host.offer, {
|
||||
terminalParkingDelayMs: 500
|
||||
})
|
||||
return webClient.page
|
||||
})()
|
||||
|
||||
await page.waitForFunction(() => Boolean(window.__store), null, { timeout: 60_000 })
|
||||
await expect
|
||||
.poll(() => page.evaluate(() => window.__store?.getState().allWorktrees().length ?? 0), {
|
||||
timeout: 90_000
|
||||
})
|
||||
.toBeGreaterThan(0)
|
||||
|
||||
const seeded = await seedBulkOpenRemoteSessions(page, {
|
||||
repoId: added.result.repo.id
|
||||
})
|
||||
disposeSessions = seeded.dispose
|
||||
|
||||
const report = await runBulkOpenFreezeOracle(page, seeded.sessions, {
|
||||
topology: 'paired-remote-server',
|
||||
versionHint: process.env.npm_package_version ?? '1.4.163-rc.3',
|
||||
reportDir: REPORT_DIR
|
||||
})
|
||||
|
||||
console.log('[freeze-repro R1]', JSON.stringify(report, null, 2))
|
||||
|
||||
if (report.hardFreeze) {
|
||||
throw new Error(
|
||||
`HARD FREEZE signal: bulkOpenMaxLagMs=${report.bulkOpenMaxLagMs.toFixed(0)} ` +
|
||||
`interactionProbeMs=${report.interactionProbeMs.toFixed(0)} ` +
|
||||
`(threshold ${HARD_FREEZE_LAG_MS}ms). notes=${report.notes.join('; ')}`
|
||||
)
|
||||
}
|
||||
if (report.softFreeze) {
|
||||
throw new Error(
|
||||
`SOFT FREEZE signal: bulkOpenMaxLagMs=${report.bulkOpenMaxLagMs.toFixed(0)} ` +
|
||||
`interactionProbeMs=${report.interactionProbeMs.toFixed(0)} ` +
|
||||
`(threshold ${SOFT_FREEZE_LAG_MS}ms). notes=${report.notes.join('; ')}`
|
||||
)
|
||||
}
|
||||
|
||||
expect(report.sessionCount).toBeGreaterThanOrEqual(8)
|
||||
expect(report.worktreeCount).toBe(3)
|
||||
} finally {
|
||||
await disposeSessions?.()
|
||||
await webClient?.dispose()
|
||||
await desktopClient?.dispose()
|
||||
await host.dispose()
|
||||
}
|
||||
})
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
/**
|
||||
* Freeze repro R2 — direct SSH topology via Docker SSH relay.
|
||||
*
|
||||
* Requires: ORCA_E2E_SSH_DOCKER=1 and Docker available.
|
||||
*
|
||||
* Run:
|
||||
* ORCA_E2E_SSH_DOCKER=1 pnpm run test:e2e:ssh-docker-bulk-open-freeze
|
||||
*/
|
||||
import path from 'node:path'
|
||||
import { expect, test } from './helpers/orca-app'
|
||||
import {
|
||||
cleanupDockerSshRelayTarget,
|
||||
DOCKER_SSH_RELAY_REMOTE_REPO_PATH,
|
||||
execDockerSshRelayTargetCommand,
|
||||
startDockerSshRelayTarget,
|
||||
type DockerSshRelayTarget
|
||||
} from './helpers/docker-ssh-relay-target'
|
||||
import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection'
|
||||
import { waitForActiveWorktree, waitForSessionReady } from './helpers/store'
|
||||
import {
|
||||
execInTerminal,
|
||||
focusLastTerminalPane,
|
||||
splitActiveTerminalPane,
|
||||
waitForActivePanePtyId,
|
||||
waitForActiveTerminalManager,
|
||||
waitForTerminalOutput
|
||||
} from './helpers/terminal'
|
||||
import { startRendererLagProbe } from './paired-runtime-retention-metrics'
|
||||
import { HARD_FREEZE_LAG_MS, SOFT_FREEZE_LAG_MS } from './helpers/remote-session-bulk-open-oracle'
|
||||
|
||||
const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1'
|
||||
const REPORT_DIR = path.join(process.cwd(), 'test-results', 'freeze-repro')
|
||||
const SESSION_SPLITS = 5
|
||||
|
||||
function shellQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", "'\\''")}'`
|
||||
}
|
||||
|
||||
function continuousFloodCommand(runId: string, index: number): string {
|
||||
// Node one-liner: continuous 2KB frames @ ~8ms like agent output.
|
||||
const script = [
|
||||
`const id='SSH_BULK_${runId}_${index}'`,
|
||||
"process.stdout.write('READY:'+id+'\\n')",
|
||||
'let f=0',
|
||||
"const c='S'.repeat(2048)",
|
||||
"setInterval(()=>{f++;process.stdout.write('BG:'+id+':'+f+':'+c+'\\n')},8)",
|
||||
'process.stdin.resume()'
|
||||
].join(';')
|
||||
return `node -e ${shellQuote(script)}`
|
||||
}
|
||||
|
||||
test.describe('R2 Docker SSH bulk-open freeze', () => {
|
||||
test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker SSH freeze repro')
|
||||
|
||||
test('bulk-open many flooding SSH terminals and measure renderer lag @freeze-repro', async ({
|
||||
orcaPage,
|
||||
registerPostElectronShutdownCleanup
|
||||
}) => {
|
||||
test.setTimeout(420_000)
|
||||
let target: DockerSshRelayTarget | null = null
|
||||
try {
|
||||
target = startDockerSshRelayTarget()
|
||||
registerPostElectronShutdownCleanup(async () => {
|
||||
if (target) {
|
||||
cleanupDockerSshRelayTarget(target)
|
||||
}
|
||||
})
|
||||
|
||||
await connectDockerSshRelayTarget(orcaPage, target, {
|
||||
remotePath: DOCKER_SSH_RELAY_REMOTE_REPO_PATH
|
||||
})
|
||||
await waitForSessionReady(orcaPage)
|
||||
await waitForActiveWorktree(orcaPage)
|
||||
|
||||
const runId = `${Date.now()}`
|
||||
// First terminal on the SSH worktree.
|
||||
await waitForActiveTerminalManager(orcaPage)
|
||||
await execInTerminal(orcaPage, continuousFloodCommand(runId, 0))
|
||||
await waitForTerminalOutput(orcaPage, `READY:SSH_BULK_${runId}_0`, 60_000)
|
||||
|
||||
for (let i = 1; i < SESSION_SPLITS; i += 1) {
|
||||
await splitActiveTerminalPane(orcaPage)
|
||||
await focusLastTerminalPane(orcaPage)
|
||||
await waitForActivePanePtyId(orcaPage, 30_000)
|
||||
await execInTerminal(orcaPage, continuousFloodCommand(runId, i))
|
||||
await waitForTerminalOutput(orcaPage, `READY:SSH_BULK_${runId}_${i}`, 60_000)
|
||||
}
|
||||
|
||||
// Leave the workspace view so panes go inactive while flooding.
|
||||
await orcaPage.evaluate(() => window.__store?.getState().setActiveView('tasks'))
|
||||
await orcaPage.waitForTimeout(4_000)
|
||||
|
||||
const hiddenProbe = await startRendererLagProbe(orcaPage)
|
||||
await orcaPage.waitForTimeout(2_000)
|
||||
const hiddenFloodMaxLagMs = await hiddenProbe.evaluate((probe) => probe.stop())
|
||||
await hiddenProbe.dispose()
|
||||
|
||||
// Burst open: return to terminal and cycle panes rapidly.
|
||||
const openProbe = await startRendererLagProbe(orcaPage)
|
||||
await orcaPage.evaluate(() => window.__store?.getState().setActiveView('terminal'))
|
||||
for (let pass = 0; pass < 3; pass += 1) {
|
||||
for (let i = 0; i < SESSION_SPLITS; i += 1) {
|
||||
await orcaPage.keyboard.press(process.platform === 'darwin' ? 'Meta+]' : 'Control+]')
|
||||
await orcaPage.waitForTimeout(50)
|
||||
}
|
||||
}
|
||||
await orcaPage.waitForTimeout(3_000)
|
||||
const bulkOpenMaxLagMs = await openProbe.evaluate((probe) => probe.stop())
|
||||
await openProbe.dispose()
|
||||
|
||||
const interactionProbeMs = await orcaPage.evaluate(async () => {
|
||||
const started = performance.now()
|
||||
const state = window.__store?.getState()
|
||||
const view = state?.activeView
|
||||
state?.setActiveView(view === 'tasks' ? 'terminal' : 'tasks')
|
||||
await new Promise<void>((r) =>
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => r()))
|
||||
)
|
||||
state?.setActiveView(view ?? 'terminal')
|
||||
await new Promise<void>((r) =>
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => r()))
|
||||
)
|
||||
return performance.now() - started
|
||||
})
|
||||
|
||||
const report = {
|
||||
topology: 'docker-ssh' as const,
|
||||
sessionCount: SESSION_SPLITS,
|
||||
hiddenFloodMaxLagMs,
|
||||
bulkOpenMaxLagMs,
|
||||
interactionProbeMs,
|
||||
softFreeze:
|
||||
bulkOpenMaxLagMs >= SOFT_FREEZE_LAG_MS || interactionProbeMs >= SOFT_FREEZE_LAG_MS,
|
||||
hardFreeze:
|
||||
bulkOpenMaxLagMs >= HARD_FREEZE_LAG_MS || interactionProbeMs >= HARD_FREEZE_LAG_MS,
|
||||
container: target.containerName,
|
||||
remoteHostStillStreaming: true
|
||||
}
|
||||
|
||||
const { mkdirSync, writeFileSync } = await import('node:fs')
|
||||
mkdirSync(REPORT_DIR, { recursive: true })
|
||||
writeFileSync(
|
||||
path.join(REPORT_DIR, 'bulk-open-freeze-docker-ssh.json'),
|
||||
`${JSON.stringify(report, null, 2)}\n`
|
||||
)
|
||||
console.log('[freeze-repro R2]', JSON.stringify(report, null, 2))
|
||||
|
||||
// Host still producing frames (host alive, client stuck).
|
||||
const hostFrames = execDockerSshRelayTargetCommand(
|
||||
target,
|
||||
`ps aux | grep -c '[n]ode -e' || true`
|
||||
)
|
||||
expect(Number(hostFrames) || 0).toBeGreaterThan(0)
|
||||
|
||||
if (report.hardFreeze) {
|
||||
throw new Error(
|
||||
`HARD FREEZE on Docker SSH: lag=${bulkOpenMaxLagMs.toFixed(0)}ms interaction=${interactionProbeMs.toFixed(0)}ms`
|
||||
)
|
||||
}
|
||||
if (report.softFreeze) {
|
||||
throw new Error(
|
||||
`SOFT FREEZE on Docker SSH: lag=${bulkOpenMaxLagMs.toFixed(0)}ms interaction=${interactionProbeMs.toFixed(0)}ms`
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
if (target) {
|
||||
cleanupDockerSshRelayTarget(target)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,203 @@
|
|||
# Live remote freeze repros
|
||||
|
||||
Two harnesses for bulk-open / reconnect freeze repros on large paired remotes:
|
||||
|
||||
| Harness | Realism | Purpose |
|
||||
| ------------------------------------- | --------------------------------------------------------------------- | ------------------------------------ |
|
||||
| **Realistic** (preferred for stories) | Idle + flood backlog → wake/reconnect-like refresh → human-paced open | Models overnight/return/restart |
|
||||
| **Bulk parallel** (stress amp) | Concurrent `terminal switch` | Forces hard freeze for load ceilings |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Desktop Orca running** (`orca status --json`).
|
||||
2. A **large paired remote** (many worktrees / agent terminals). Lab fleets often have ~60 worktrees and 100+ terminals.
|
||||
3. Repo checkout with these scripts.
|
||||
|
||||
```bash
|
||||
orca environment list --json
|
||||
orca worktree list --environment <name> --json | head
|
||||
orca terminal list --environment <name> --json | head
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## A. Realistic repro (preferred)
|
||||
|
||||
Story: remotes keep streaming while the user is away; user returns (optionally after wake/reconnect-like refresh) and opens sessions one-by-one.
|
||||
|
||||
```bash
|
||||
# Idle + human-paced open
|
||||
ORCA_FREEZE_ENV=paired-remote \
|
||||
ORCA_FREEZE_SCENARIO=idle-backlog-open \
|
||||
ORCA_FREEZE_CREATE=8 \
|
||||
ORCA_FREEZE_IDLE_MS=45000 \
|
||||
ORCA_FREEZE_OPEN_COUNT=24 \
|
||||
pnpm run repro:live-remote-realistic-freeze
|
||||
|
||||
# Wake-like: idle + reconnect metadata storm + open ← hard freeze in lab
|
||||
ORCA_FREEZE_ENV=paired-remote \
|
||||
ORCA_FREEZE_SCENARIO=idle-backlog-reconnect-open \
|
||||
ORCA_FREEZE_CREATE=10 \
|
||||
ORCA_FREEZE_IDLE_MS=60000 \
|
||||
ORCA_FREEZE_OPEN_COUNT=40 \
|
||||
pnpm run repro:live-remote-realistic-freeze
|
||||
|
||||
# Restart-proxy: idle + orca open + refresh storm + open (does not kill desktop)
|
||||
ORCA_FREEZE_ENV=paired-remote \
|
||||
ORCA_FREEZE_SCENARIO=restart-proxy \
|
||||
ORCA_FREEZE_CREATE=0 \
|
||||
ORCA_FREEZE_IDLE_MS=20000 \
|
||||
ORCA_FREEZE_OPEN_COUNT=30 \
|
||||
pnpm run repro:live-remote-realistic-freeze
|
||||
```
|
||||
|
||||
Or: `node config/scripts/live-remote-realistic-freeze-repro.mjs`
|
||||
|
||||
### Scenarios
|
||||
|
||||
| `ORCA_FREEZE_SCENARIO` | Models |
|
||||
| ----------------------------- | --------------------------------------------------------------------------------------------- |
|
||||
| `idle-backlog-open` | User away while agents stream; returns and opens sessions |
|
||||
| `idle-backlog-reconnect-open` | Same + parallel status/worktree/terminal refresh (wake/reconnect client storm) |
|
||||
| `restart-proxy` | `orca open` + refresh storm + open (post-restart discovery; no process kill) |
|
||||
| `lockup-storm` | Idle + flood + reconnect + **concurrent** open fan-out + **mid-storm `orca status` watchdog** |
|
||||
|
||||
### Realistic knobs
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
| --------------------------------- | ------------------- | ----------------------------------------------------------------- |
|
||||
| `ORCA_FREEZE_ENV` | `paired-remote` | Paired remote environment name |
|
||||
| `ORCA_FREEZE_SCENARIO` | `idle-backlog-open` | See table above |
|
||||
| `ORCA_FREEZE_CREATE` | `0` | New flood terminals; mutation requires an explicit positive value |
|
||||
| `ORCA_FREEZE_IDLE_MS` | `45000` | Time “away” while floods run |
|
||||
| `ORCA_FREEZE_OPEN_COUNT` | `20` | Sessions to open after return |
|
||||
| `ORCA_FREEZE_PACE_MS` | `250` | Base delay between opens (human pace) |
|
||||
| `ORCA_FREEZE_PACE_JITTER_MS` | `150` | Random extra delay |
|
||||
| `ORCA_FREEZE_SOFT_MS` / `HARD_MS` | 2000 / 5000 | Thresholds |
|
||||
|
||||
### Lab results (2026-07-31, client 1.4.163 / remote 1.4.163-rc.0)
|
||||
|
||||
| Scenario | create | idle | open | peak | Signal |
|
||||
| -------------------------------------------------- | ------ | ------ | -------------- | ---------------------------------------------------- | ------------------------------------------------------------------------- |
|
||||
| idle-backlog-open | 6 | 45s | 24 | **1.7s** max open | none (< soft) |
|
||||
| **idle-backlog-reconnect-open** | 10 | 60s | 40 | **11.0s** max open; reconnect refresh **3.6s** | **HARD (recovered)** |
|
||||
| **restart-proxy** | 0 | 20s | 30 | **11.2s** max open | **HARD (recovered)** |
|
||||
| **lockup-storm** (parallel open + overlap refresh) | 12–16 | 45–60s | 64–80 @ p20–32 | **27–35s** batches; some `Terminal reveal timed out` | **HARD stalls + reveal timeouts; app still answers `orca status` ~150ms** |
|
||||
|
||||
### Full-app forever freeze?
|
||||
|
||||
**Not observed** under CLI-driven escalation (including mid-storm status watchdog).
|
||||
|
||||
Latest lockup-storm with watchdog (2026-07-31):
|
||||
|
||||
| Field | Value |
|
||||
| --------------------------- | ----------------------------------- |
|
||||
| `foreverUiLockupObserved` | **false** |
|
||||
| Mid-storm status samples | **95**, max **~631ms**, **0 hangs** |
|
||||
| Peak open/batch | **~34s** (recovered hard stall) |
|
||||
| `Terminal reveal timed out` | yes (under fan-out) |
|
||||
| Post-storm `orca status` | **~113ms** |
|
||||
| Force Quit required | **no** |
|
||||
|
||||
Bar for full-app freeze in the harness: continuous **≥30s** window where `orca status` hangs/fails or stays ≥15s slow (`evaluateFullAppFreeze` / `foreverUiLockupObserved`).
|
||||
CLI spawn failures are reported as harness infrastructure errors, not product freezes.
|
||||
|
||||
What we **do** reproduce: severe multi-second / multi-tens-of-seconds stalls + flaky reveal.
|
||||
What we **do not**: UI dead forever until Force Quit. That likely needs **real OS sleep/wake**, **renderer React #185**, or a path status RPC does not share with the frozen surface.
|
||||
|
||||
| Exit | Meaning |
|
||||
| ---- | ------------------------------------------------------------------------------- |
|
||||
| 0 | no freeze |
|
||||
| 1 | soft (≥2s recovered) |
|
||||
| 2 | hard stall ≥5s **but recovered** |
|
||||
| 4 | permanentLockup heuristic (timeouts/fail-rate; check `foreverUiLockupObserved`) |
|
||||
| 5 | **full-app forever freeze** (status unhealthy ≥ forever window) |
|
||||
| 3 | harness error |
|
||||
|
||||
```bash
|
||||
# Full-app freeze attempt (watchdog on)
|
||||
ORCA_FREEZE_ENV=paired-remote ORCA_FREEZE_SCENARIO=lockup-storm \
|
||||
ORCA_FREEZE_CREATE=12 ORCA_FREEZE_IDLE_MS=30000 ORCA_FREEZE_OPEN_COUNT=80 \
|
||||
ORCA_FREEZE_STORM_PARALLEL=28 ORCA_FREEZE_FOREVER_WINDOW_MS=30000 \
|
||||
pnpm run repro:live-remote-realistic-freeze
|
||||
# Expect exit 2 (recovered hard) unless foreverUiLockupObserved becomes true
|
||||
```
|
||||
|
||||
**Interpretation:** Pure sequential open after idle stays under 2s. **Wake/reconnect-style refresh + open** (or concurrent fan-out) produces **recovered hard stalls**. True permanent lockup remains unproven with CLI-only levers.
|
||||
|
||||
Reports: `test-results/freeze-repro/live-realistic-freeze-<env>-<scenario>.json`
|
||||
|
||||
---
|
||||
|
||||
## B. Stress amp (bulk parallel)
|
||||
|
||||
Artificial concurrency lever; still useful for ceilings / CI stress.
|
||||
|
||||
```bash
|
||||
ORCA_FREEZE_ENV=paired-remote \
|
||||
ORCA_FREEZE_CREATE=0 \
|
||||
ORCA_FREEZE_SWITCH_PASSES=3 \
|
||||
ORCA_FREEZE_PARALLEL=16 \
|
||||
pnpm run repro:live-remote-bulk-open-freeze
|
||||
```
|
||||
|
||||
Lab: sequential soft ~3.3–3.9s; **parallel=16 → ~20s HARD**.
|
||||
|
||||
---
|
||||
|
||||
## Exit codes (both harnesses)
|
||||
|
||||
| Code | Meaning |
|
||||
| ---- | --------------------------------- |
|
||||
| 0 | No freeze signal under thresholds |
|
||||
| 1 | Soft freeze (peak ≥ 2s) |
|
||||
| 2 | **Hard freeze (peak ≥ 5s)** |
|
||||
| 3 | Harness failure |
|
||||
|
||||
---
|
||||
|
||||
## Product fix (concurrent host-focus storms)
|
||||
|
||||
Generation-aware **latest-wins single-flight** for exclusive host focus:
|
||||
|
||||
| Layer | Module |
|
||||
| -------- | ---------------------------------------------------------------------------------- |
|
||||
| Runtime | `TerminalFocusNavigationCoalescer` via `OrcaRuntimeService.focusTerminal` |
|
||||
| Contract | `RuntimeTerminalFocus.navigated?: boolean` — `false` when superseded / nav skipped |
|
||||
|
||||
**In scope:** concurrent `terminal.focus` / bulk-switch storms.
|
||||
**Residual:** sequential soft freezes; reconnect/wake metadata storms — need cheaper activation + scan bounding, not only focus coalescing.
|
||||
|
||||
## Files
|
||||
|
||||
| Path | Role |
|
||||
| -------------------------------------------------------------- | ---------------------------------- |
|
||||
| `config/scripts/live-remote-realistic-freeze-repro.mjs` | Naturalistic harness |
|
||||
| `config/scripts/live-remote-bulk-open-freeze-repro.mjs` | Parallel stress harness |
|
||||
| `config/scripts/live-remote-freeze-rpc.mjs` | Cross-platform bounded CLI runner |
|
||||
| `config/scripts/live-remote-bulk-open-freeze-metrics.mjs` | Shared thresholds / handle extract |
|
||||
| `config/scripts/live-remote-bulk-open-freeze-metrics.test.mjs` | Unit tests |
|
||||
| `src/main/runtime/terminal-focus-navigation-coalescer.ts` | Host focus single-flight |
|
||||
| `pnpm run repro:live-remote-realistic-freeze` | package entry |
|
||||
| `pnpm run repro:live-remote-bulk-open-freeze` | package entry |
|
||||
|
||||
---
|
||||
|
||||
## Safety
|
||||
|
||||
- Both harnesses default to `ORCA_FREEZE_CREATE=0`. A positive value creates persistent, high-output remote terminals; use it only on an isolated target you can clean up.
|
||||
- `restart-proxy` does **not** kill Orca; it runs `orca open` + refresh RPCs only.
|
||||
- Manual capture if UI fully freezes: `sample Orca 5 -file ~/Desktop/orca-freeze-sample.txt`
|
||||
|
||||
The scripts honor `ORCA_CLI_COMMAND`, then use `orca-dev` in a dev runtime, `orca-ide` on Linux, and `orca` elsewhere.
|
||||
|
||||
PowerShell equivalent for the first example:
|
||||
|
||||
```powershell
|
||||
$env:ORCA_FREEZE_ENV = 'paired-remote'
|
||||
$env:ORCA_FREEZE_SCENARIO = 'idle-backlog-open'
|
||||
$env:ORCA_FREEZE_CREATE = '8'
|
||||
$env:ORCA_FREEZE_IDLE_MS = '45000'
|
||||
$env:ORCA_FREEZE_OPEN_COUNT = '24'
|
||||
pnpm run repro:live-remote-realistic-freeze
|
||||
```
|
||||
Loading…
Reference in New Issue