diff --git a/config/scripts/live-freeze-bounded-history.mjs b/config/scripts/live-freeze-bounded-history.mjs new file mode 100644 index 000000000..7b64b7d52 --- /dev/null +++ b/config/scripts/live-freeze-bounded-history.mjs @@ -0,0 +1,38 @@ +export class BoundedLiveFreezeHistory { + #entries = [] + #limit + #nextIndex = 0 + #totalCount = 0 + + constructor(limit) { + if (!Number.isInteger(limit) || limit <= 0) { + throw new Error(`History limit must be a positive integer, got ${limit}`) + } + this.#limit = limit + } + + add(entry) { + this.#totalCount += 1 + if (this.#entries.length < this.#limit) { + this.#entries.push(entry) + return + } + this.#entries[this.#nextIndex] = entry + this.#nextIndex = (this.#nextIndex + 1) % this.#limit + } + + get retainedCount() { + return this.#entries.length + } + + get totalCount() { + return this.#totalCount + } + + values() { + if (this.#entries.length < this.#limit || this.#nextIndex === 0) { + return [...this.#entries] + } + return [...this.#entries.slice(this.#nextIndex), ...this.#entries.slice(0, this.#nextIndex)] + } +} diff --git a/config/scripts/live-freeze-bounded-history.test.mjs b/config/scripts/live-freeze-bounded-history.test.mjs new file mode 100644 index 000000000..48d1cfb5b --- /dev/null +++ b/config/scripts/live-freeze-bounded-history.test.mjs @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { BoundedLiveFreezeHistory } from './live-freeze-bounded-history.mjs' + +describe('BoundedLiveFreezeHistory', () => { + it('retains the newest entries in insertion order and counts the full run', () => { + const history = new BoundedLiveFreezeHistory(3) + + for (let value = 1; value <= 7; value += 1) { + history.add(value) + } + + expect(history.values()).toEqual([5, 6, 7]) + expect(history.retainedCount).toBe(3) + expect(history.totalCount).toBe(7) + }) + + it('rejects invalid retention limits', () => { + expect(() => new BoundedLiveFreezeHistory(0)).toThrow('positive integer') + expect(() => new BoundedLiveFreezeHistory(1.5)).toThrow('positive integer') + }) +}) diff --git a/config/scripts/live-remote-bulk-open-freeze-metrics.mjs b/config/scripts/live-remote-bulk-open-freeze-metrics.mjs new file mode 100644 index 000000000..4f24e59ff --- /dev/null +++ b/config/scripts/live-remote-bulk-open-freeze-metrics.mjs @@ -0,0 +1,262 @@ +/** + * Pure metrics helpers for the live remote bulk-open freeze harness. + * Kept separate so unit tests can drive the same code the repro uses. + */ + +export const DEFAULT_SOFT_MS = 2000 +export const DEFAULT_HARD_MS = 5000 + +export function readFreezeNumberEnv(name, fallback) { + const raw = process.env[name] + if (raw == null || raw.trim() === '') { + return fallback + } + const value = Number(raw) + if (!Number.isFinite(value)) { + throw new Error(`Invalid ${name}: expected a finite number, got ${JSON.stringify(raw)}`) + } + return value +} + +export function extractTerminalHandle(result) { + if (!result || typeof result !== 'object') { + return null + } + const candidates = [ + result.handle, + result.terminalHandle, + result.agentTerminalHandle, + typeof result.terminal === 'string' ? result.terminal : result.terminal?.handle, + result.startupTerminal?.handle, + result.tab?.terminal, + result.tab?.handle + ] + for (const value of candidates) { + if (typeof value === 'string' && value.startsWith('term_')) { + return value + } + } + for (const value of Object.values(result)) { + if (typeof value === 'string' && value.startsWith('term_')) { + return value + } + if (value && typeof value === 'object') { + for (const nested of Object.values(value)) { + if (typeof nested === 'string' && nested.startsWith('term_')) { + return nested + } + } + } + } + return null +} + +export function worktreeSelector(wt) { + if (typeof wt?.id === 'string' && wt.id.length > 0) { + return `id:${wt.id}` + } + if (typeof wt?.path === 'string' && wt.path.length > 0) { + return `path:${wt.path}` + } + return null +} + +/** + * Peak stall across individual switch latency and concurrent batch wall. + * Hard freeze when peak >= hardMs (default 5000). + */ +export function evaluateFreezeSignals({ + maxSwitchMs = 0, + maxBatchWallMs = 0, + statusProbeMs = 0, + memoryProbeMs = null, + softMs = DEFAULT_SOFT_MS, + hardMs = DEFAULT_HARD_MS +}) { + const peakLatencyMs = Math.max(maxSwitchMs, maxBatchWallMs) + const softFreeze = + peakLatencyMs >= softMs || + statusProbeMs >= softMs || + (memoryProbeMs != null && memoryProbeMs >= softMs) + const hardFreeze = + peakLatencyMs >= hardMs || + statusProbeMs >= hardMs || + (memoryProbeMs != null && memoryProbeMs >= hardMs) + return { peakLatencyMs, softFreeze, hardFreeze } +} + +export function shouldCapSwitchTargets(maxSwitchTargets) { + return Number.isFinite(maxSwitchTargets) && maxSwitchTargets > 0 +} + +export function applySwitchTargetCap(targets, maxSwitchTargets) { + if (!shouldCapSwitchTargets(maxSwitchTargets)) { + return targets + } + return targets.slice(0, maxSwitchTargets) +} + +/** Scenarios that model real user recovery, not concurrent CLI pileup. */ +export const REALISTIC_SCENARIOS = [ + 'idle-backlog-open', + 'idle-backlog-reconnect-open', + 'restart-proxy', + /** Idle + flood + reconnect storm overlapped with concurrent open fan-out. */ + 'lockup-storm' +] + +/** + * Permanent lockup: app/host stops making progress — not a single recovered timeout. + * Distinct from multi-second hard stall that still recovers (status answers, most opens ok). + */ +export function evaluatePermanentLockup({ + timedOutOps = 0, + statusHangMs = 0, + consecutiveSwitchFailures = 0, + openFailed = 0, + openTotal = 0, + permanentTimeoutMs = 60_000, + /** Fraction of opens that must fail to count as lockup without status hang. */ + failRateThreshold = 0.25, + minTimedOutOps = 3 +}) { + const failRate = openTotal > 0 ? openFailed / openTotal : 0 + const permanentLockup = + statusHangMs >= permanentTimeoutMs || + timedOutOps >= minTimedOutOps || + consecutiveSwitchFailures >= 5 || + (openTotal >= 8 && failRate >= failRateThreshold) + return { + permanentLockup, + timedOutOps, + statusHangMs, + consecutiveSwitchFailures, + failRate, + recoveredHardStallCandidate: + !permanentLockup && timedOutOps < minTimedOutOps && statusHangMs < permanentTimeoutMs + } +} + +/** + * Peak across open latencies + optional reconnect-refresh wall + probes. + * Used by the naturalistic harness (no parallel switch amp). + */ +export function evaluateRealisticFreezeSignals({ + maxOpenMs = 0, + firstOpenMs = 0, + reconnectRefreshMs = 0, + statusProbeMs = 0, + memoryProbeMs = null, + softMs = DEFAULT_SOFT_MS, + hardMs = DEFAULT_HARD_MS +}) { + const peakLatencyMs = Math.max(maxOpenMs, firstOpenMs, reconnectRefreshMs) + return evaluateFreezeSignals({ + maxSwitchMs: peakLatencyMs, + maxBatchWallMs: 0, + statusProbeMs, + memoryProbeMs, + softMs, + hardMs + }) +} + +export function humanPaceDelayMs(baseMs, jitterMs = 0) { + const base = Math.max(0, baseMs) + const jitter = Math.max(0, jitterMs) + if (jitter === 0) { + return base + } + return base + Math.floor(Math.random() * (jitter + 1)) +} + +/** Full-app forever freeze: host RPC dead for a continuous window, not a recovered stall. */ +export const DEFAULT_FOREVER_WINDOW_MS = 30_000 +export const DEFAULT_STATUS_SLOW_MS = 15_000 + +/** + * Analyze mid-storm status samples for a continuous unhealthy window. + * Sample: { tMs, ms, ok, hang } + */ +export function evaluateFullAppFreeze({ + statusSamples = [], + statusSummary = {}, + foreverWindowMs = DEFAULT_FOREVER_WINDOW_MS, + statusSlowMs = DEFAULT_STATUS_SLOW_MS, + killOnlyRecovery = false +}) { + const infrastructureErrors = statusSamples.filter((sample) => sample.infrastructureError) + const infrastructureErrorCount = Math.max( + infrastructureErrors.length, + statusSummary.infrastructureErrorCount ?? 0 + ) + const maxStatusMs = Math.max( + 0, + ...statusSamples.map((s) => s.ms || 0), + statusSummary.maxStatusMs ?? 0 + ) + if (killOnlyRecovery) { + return { + foreverUiLockupObserved: true, + longestUnhealthyWindowMs: foreverWindowMs, + maxStatusMs, + unhealthySampleCount: statusSummary.sampleCount ?? statusSamples.length, + infrastructureErrorCount, + reason: 'kill-only recovery documented' + } + } + + const unhealthy = statusSamples.map((s) => { + const hang = !s.infrastructureError && (Boolean(s.hang) || s.ok === false) + const slow = !s.infrastructureError && (s.ms || 0) >= statusSlowMs + return { ...s, unhealthy: hang || slow } + }) + + let longest = statusSummary.longestUnhealthyWindowMs ?? 0 + let runStart = null + for (const s of unhealthy) { + if (s.unhealthy) { + runStart ??= s.tMs ?? 0 + const end = (s.tMs ?? 0) + (s.ms || 0) + longest = Math.max(longest, end - runStart) + } else { + runStart = null + } + } + + // If timestamps missing, fall back to consecutive unhealthy count * assumed interval. + if (longest === 0 && unhealthy.some((s) => s.unhealthy)) { + let run = 0 + for (const s of unhealthy) { + if (s.unhealthy) { + run += 1 + longest = Math.max(longest, run) + } else { + run = 0 + } + } + // Without wall clock, consecutive count alone is not a ms window. + longest = 0 + } + + const foreverUiLockupObserved = longest >= foreverWindowMs + const unhealthySampleCount = Math.max( + unhealthy.filter((s) => s.unhealthy).length, + statusSummary.unhealthySampleCount ?? 0 + ) + + return { + foreverUiLockupObserved, + longestUnhealthyWindowMs: longest, + maxStatusMs, + unhealthySampleCount, + infrastructureErrorCount, + reason: foreverUiLockupObserved + ? `status unhealthy ≥${foreverWindowMs}ms continuous` + : infrastructureErrorCount > 0 + ? `status watchdog infrastructure errors: ${infrastructureErrorCount}` + : maxStatusMs >= statusSlowMs + ? `status slow peak ${maxStatusMs}ms but no ≥${foreverWindowMs}ms window` + : 'status remained healthy through storm' + } +} diff --git a/config/scripts/live-remote-bulk-open-freeze-metrics.test.mjs b/config/scripts/live-remote-bulk-open-freeze-metrics.test.mjs new file mode 100644 index 000000000..586364b43 --- /dev/null +++ b/config/scripts/live-remote-bulk-open-freeze-metrics.test.mjs @@ -0,0 +1,239 @@ +import { describe, expect, it } from 'vitest' +import { + applySwitchTargetCap, + evaluateFreezeSignals, + evaluateFullAppFreeze, + evaluatePermanentLockup, + evaluateRealisticFreezeSignals, + extractTerminalHandle, + humanPaceDelayMs, + readFreezeNumberEnv, + REALISTIC_SCENARIOS, + shouldCapSwitchTargets, + worktreeSelector +} from './live-remote-bulk-open-freeze-metrics.mjs' + +describe('live-remote-bulk-open-freeze-metrics', () => { + it('extracts term_ handles from nested create payloads', () => { + expect(extractTerminalHandle({ handle: 'term_abc' })).toBe('term_abc') + expect(extractTerminalHandle({ terminal: { handle: 'term_nested' } })).toBe('term_nested') + expect(extractTerminalHandle({ tab: { terminal: 'term_tab' } })).toBe('term_tab') + expect(extractTerminalHandle({ startupTerminal: { handle: 'term_start' } })).toBe('term_start') + expect(extractTerminalHandle({ junk: { deep: 'term_deep' } })).toBe('term_deep') + expect(extractTerminalHandle({ handle: 'not-a-term' })).toBeNull() + expect(extractTerminalHandle(null)).toBeNull() + }) + + it('builds worktree selectors from id/path', () => { + expect( + worktreeSelector({ id: 'repo::C:/Users/neil/orca/orca', path: 'C:/Users/neil/orca/orca' }) + ).toBe('id:repo::C:/Users/neil/orca/orca') + expect(worktreeSelector({ path: '/tmp/x' })).toBe('path:/tmp/x') + expect(worktreeSelector({})).toBeNull() + }) + + it('does not cap switch targets when max is 0 (regression for Math.max(2,0) bug)', () => { + expect(shouldCapSwitchTargets(0)).toBe(false) + expect(shouldCapSwitchTargets(-1)).toBe(false) + expect(shouldCapSwitchTargets(2)).toBe(true) + const many = Array.from({ length: 111 }, (_, i) => `term_${i}`) + expect(applySwitchTargetCap(many, 0)).toHaveLength(111) + expect(applySwitchTargetCap(many, 2)).toHaveLength(2) + }) + + it('classifies hard freeze at >=5000ms peak (individual or batch wall)', () => { + expect(evaluateFreezeSignals({ maxSwitchMs: 3874, maxBatchWallMs: 3874 }).hardFreeze).toBe( + false + ) + expect(evaluateFreezeSignals({ maxSwitchMs: 3874, maxBatchWallMs: 3874 }).softFreeze).toBe(true) + + const hardIndividual = evaluateFreezeSignals({ maxSwitchMs: 19954, maxBatchWallMs: 1000 }) + expect(hardIndividual.hardFreeze).toBe(true) + expect(hardIndividual.peakLatencyMs).toBe(19954) + + const hardBatch = evaluateFreezeSignals({ maxSwitchMs: 900, maxBatchWallMs: 20201 }) + expect(hardBatch.hardFreeze).toBe(true) + expect(hardBatch.peakLatencyMs).toBe(20201) + }) + + it('evaluates naturalistic peaks without requiring parallel batch amp', () => { + expect(REALISTIC_SCENARIOS).toContain('idle-backlog-open') + expect(REALISTIC_SCENARIOS).toContain('idle-backlog-reconnect-open') + expect(REALISTIC_SCENARIOS).toContain('lockup-storm') + const soft = evaluateRealisticFreezeSignals({ + maxOpenMs: 3200, + firstOpenMs: 2800, + reconnectRefreshMs: 900 + }) + expect(soft.softFreeze).toBe(true) + expect(soft.hardFreeze).toBe(false) + expect(soft.peakLatencyMs).toBe(3200) + + const hardFromReconnect = evaluateRealisticFreezeSignals({ + maxOpenMs: 800, + firstOpenMs: 700, + reconnectRefreshMs: 6200 + }) + expect(hardFromReconnect.hardFreeze).toBe(true) + expect(hardFromReconnect.peakLatencyMs).toBe(6200) + }) + + it('flags full-app freeze only for continuous unhealthy status window ≥30s', () => { + const healthy = evaluateFullAppFreeze({ + statusSamples: [ + { tMs: 0, ms: 150, ok: true }, + { tMs: 2000, ms: 180, ok: true }, + { tMs: 4000, ms: 140, ok: true } + ], + foreverWindowMs: 30_000, + statusSlowMs: 15_000 + }) + expect(healthy.foreverUiLockupObserved).toBe(false) + + const forever = evaluateFullAppFreeze({ + statusSamples: [ + { tMs: 0, ms: 16_000, ok: true }, + { tMs: 16_000, ms: 16_000, ok: true }, + { tMs: 32_000, ms: 16_000, ok: false, hang: true } + ], + foreverWindowMs: 30_000, + statusSlowMs: 15_000 + }) + expect(forever.foreverUiLockupObserved).toBe(true) + expect(forever.longestUnhealthyWindowMs).toBeGreaterThanOrEqual(30_000) + + expect( + evaluateFullAppFreeze({ statusSamples: [], killOnlyRecovery: true }).foreverUiLockupObserved + ).toBe(true) + }) + + it('does not classify watchdog infrastructure errors as an app freeze', () => { + const result = evaluateFullAppFreeze({ + statusSamples: Array.from({ length: 25 }, (_, index) => ({ + tMs: index * 1500, + ms: 1, + ok: false, + infrastructureError: true, + error: 'spawn ENOENT' + })) + }) + + expect(result.foreverUiLockupObserved).toBe(false) + expect(result.unhealthySampleCount).toBe(0) + expect(result.infrastructureErrorCount).toBe(25) + }) + + it('preserves full-run watchdog peaks after sample retention rotates', () => { + const result = evaluateFullAppFreeze({ + statusSamples: [{ tMs: 60_000, ms: 100, ok: true }], + statusSummary: { + sampleCount: 40, + maxStatusMs: 16_000, + unhealthySampleCount: 3, + infrastructureErrorCount: 2, + longestUnhealthyWindowMs: 32_000 + }, + foreverWindowMs: 30_000, + statusSlowMs: 15_000 + }) + + expect(result.foreverUiLockupObserved).toBe(true) + expect(result.longestUnhealthyWindowMs).toBe(32_000) + expect(result.maxStatusMs).toBe(16_000) + expect(result.unhealthySampleCount).toBe(3) + expect(result.infrastructureErrorCount).toBe(2) + }) + + it('rejects invalid numeric environment values', () => { + process.env.ORCA_FREEZE_TEST_NUMBER = 'not-a-number' + expect(() => readFreezeNumberEnv('ORCA_FREEZE_TEST_NUMBER', 5)).toThrow( + 'Invalid ORCA_FREEZE_TEST_NUMBER' + ) + delete process.env.ORCA_FREEZE_TEST_NUMBER + expect(readFreezeNumberEnv('ORCA_FREEZE_TEST_NUMBER', 5)).toBe(5) + }) + + it('distinguishes recovered hard stall from permanent lockup', () => { + // Single reveal timeout with healthy status is NOT permanent app lockup. + expect( + evaluatePermanentLockup({ + timedOutOps: 1, + statusHangMs: 0, + consecutiveSwitchFailures: 1, + openFailed: 1, + openTotal: 64 + }).permanentLockup + ).toBe(false) + expect( + evaluatePermanentLockup({ + timedOutOps: 3, + statusHangMs: 0, + consecutiveSwitchFailures: 0, + openFailed: 3, + openTotal: 64 + }).permanentLockup + ).toBe(true) + expect( + evaluatePermanentLockup({ + timedOutOps: 0, + statusHangMs: 60_000, + consecutiveSwitchFailures: 0, + permanentTimeoutMs: 60_000 + }).permanentLockup + ).toBe(true) + expect( + evaluatePermanentLockup({ + timedOutOps: 0, + statusHangMs: 0, + consecutiveSwitchFailures: 5 + }).permanentLockup + ).toBe(true) + expect( + evaluatePermanentLockup({ + timedOutOps: 0, + openFailed: 20, + openTotal: 40 + }).permanentLockup + ).toBe(true) + }) + + it('human pace delay stays within base+jitter', () => { + for (let i = 0; i < 20; i += 1) { + const d = humanPaceDelayMs(250, 150) + expect(d).toBeGreaterThanOrEqual(250) + expect(d).toBeLessThanOrEqual(400) + } + expect(humanPaceDelayMs(100, 0)).toBe(100) + }) + + it('reads the real hard-freeze lab report when present', async () => { + const { readdirSync, readFileSync, existsSync } = await import('node:fs') + const { resolve } = await import('node:path') + const reportDir = resolve(process.cwd(), 'test-results/freeze-repro') + if (!existsSync(reportDir)) { + // Local clones without lab artifacts still pass pure metrics tests above. + return + } + const reportName = readdirSync(reportDir).find( + (name) => name.startsWith('live-bulk-open-freeze-') && name.endsWith('.json') + ) + if (reportName == null) { + return + } + const report = JSON.parse(readFileSync(resolve(reportDir, reportName), 'utf8')) + const evaluated = evaluateFreezeSignals({ + maxSwitchMs: report.maxSwitchMs, + maxBatchWallMs: report.maxBatchWallMs ?? 0, + statusProbeMs: report.statusProbeMs ?? 0, + memoryProbeMs: report.memoryProbeMs, + softMs: report.softMs, + hardMs: report.hardMs + }) + expect(evaluated.hardFreeze).toBe(report.hardFreeze) + expect(evaluated.peakLatencyMs).toBeGreaterThanOrEqual(5000) + expect(typeof report.environment).toBe('string') + expect(report.environment.length).toBeGreaterThan(0) + expect(report.switchTargets).toBeGreaterThan(50) + expect(report.parallel).toBeGreaterThanOrEqual(8) + }) +}) diff --git a/config/scripts/live-remote-bulk-open-freeze-repro.mjs b/config/scripts/live-remote-bulk-open-freeze-repro.mjs new file mode 100644 index 000000000..9da369d71 --- /dev/null +++ b/config/scripts/live-remote-bulk-open-freeze-repro.mjs @@ -0,0 +1,358 @@ +#!/usr/bin/env node +/** + * Live freeze repro against a running Orca desktop + paired remote runtime. + * + * Models bulk-open of remote sessions under multi-worktree load. + * + * Usage: + * node config/scripts/live-remote-bulk-open-freeze-repro.mjs + * ORCA_FREEZE_ENV=paired-remote ORCA_FREEZE_CREATE=12 ORCA_FREEZE_SWITCH_PASSES=5 \ + * ORCA_FREEZE_PARALLEL=8 node config/scripts/live-remote-bulk-open-freeze-repro.mjs + */ +import { spawnSync } from 'node:child_process' +import { mkdirSync, writeFileSync, copyFileSync } from 'node:fs' +import path from 'node:path' +import { BoundedLiveFreezeHistory } from './live-freeze-bounded-history.mjs' +import { + applySwitchTargetCap, + DEFAULT_HARD_MS, + DEFAULT_SOFT_MS, + evaluateFreezeSignals, + extractTerminalHandle, + readFreezeNumberEnv, + shouldCapSwitchTargets, + worktreeSelector +} from './live-remote-bulk-open-freeze-metrics.mjs' +import { createOrcaRpc } from './live-remote-freeze-rpc.mjs' + +const root = path.resolve(import.meta.dirname, '../..') +const reportDir = path.join(root, 'test-results', 'freeze-repro') +const envName = process.env.ORCA_FREEZE_ENV || 'paired-remote' +const createCount = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_CREATE', 0)) +const switchPasses = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_SWITCH_PASSES', 3)) +const parallel = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_PARALLEL', 1)) +// 0 = no cap (use all live terminals). Only positive env values limit targets. +const maxSwitchTargets = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_MAX_SWITCH_TARGETS', 0)) +const softMs = readFreezeNumberEnv('ORCA_FREEZE_SOFT_MS', DEFAULT_SOFT_MS) +const hardMs = readFreezeNumberEnv('ORCA_FREEZE_HARD_MS', DEFAULT_HARD_MS) +const createWorktreeSpan = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_CREATE_WT_SPAN', 16)) +const preFloodMs = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_PRE_FLOOD_MS', 3000)) +const scratchDir = process.env.ORCA_FREEZE_SCRATCH || '' + +const { orcaJsonSync, orcaJsonAsync } = createOrcaRpc({ envName }) + +async function mapPool(items, concurrency, worker) { + const results = Array.from({ length: items.length }) + let next = 0 + async function run() { + while (next < items.length) { + const index = next + next += 1 + results[index] = await worker(items[index], index) + } + } + const runners = Array.from({ length: Math.min(concurrency, items.length) }, () => run()) + await Promise.all(runners) + return results +} + +function sampleOrcaIfPossible() { + if (process.platform !== 'darwin') { + return null + } + try { + const status = orcaJsonSync(['status'], { local: true }).result + const pid = status?.app?.pid + if (!pid) { + return null + } + const out = path.join(reportDir, `orca-sample-${Date.now()}.txt`) + const sampled = spawnSync('sample', [String(pid), '5', '-file', out], { + timeout: 20_000, + stdio: 'ignore' + }) + return sampled.status === 0 ? out : null + } catch { + return null + } +} + +function floodCommand(marker) { + // Continuous 2KB frames @ ~8ms — agent-like remote output. + const script = + "const m=process.argv[1];process.stdout.write('READY:'+m+'\\n');let f=0;const c='A'.repeat(2048);setInterval(()=>{f++;process.stdout.write('BG:'+m+':'+f+':'+c+'\\n')},8);process.stdin.resume()" + return `node -e ${JSON.stringify(script)} ${JSON.stringify(marker)}` +} + +async function main() { + mkdirSync(reportDir, { recursive: true }) + const notes = [] + const timings = new BoundedLiveFreezeHistory(120) + const amplificationSteps = [] + + console.log( + `[live-freeze] env=${envName} create=${createCount} passes=${switchPasses} parallel=${parallel}` + ) + + const status = orcaJsonSync(['status']) + notes.push( + `remote version=${status.result?.runtime?.appVersion} state=${status.result?.runtime?.state}` + ) + const local = orcaJsonSync(['status'], { local: true }) + notes.push(`local version=${local.result?.runtime?.appVersion} pid=${local.result?.app?.pid}`) + + const worktrees = orcaJsonSync(['worktree', 'list']).result + const wtList = worktrees?.worktrees || worktrees?.items || worktrees || [] + if (!Array.isArray(wtList) || wtList.length === 0) { + throw new Error(`No worktrees on environment ${envName}`) + } + notes.push(`remote worktrees=${wtList.length}`) + amplificationSteps.push(`baseline worktrees=${wtList.length}`) + + const targets = wtList.slice(0, Math.min(createWorktreeSpan, wtList.length)) + const created = [] + + // Parallel flood-terminal creates across many worktrees. + if (createCount > 0) { + amplificationSteps.push(`create=${createCount} parallel=${Math.min(parallel, createCount)}`) + const createJobs = Array.from({ length: createCount }, (_, i) => i) + await mapPool(createJobs, Math.min(parallel, createCount), async (i) => { + const wt = targets[i % targets.length] + const selector = worktreeSelector(wt) + if (!selector) { + notes.push(`create ${i} skipped: no selector`) + return + } + const marker = `LIVE_BULK_${Date.now()}_${i}` + try { + const createdTerm = await orcaJsonAsync( + [ + 'terminal', + 'create', + '--worktree', + selector, + '--title', + `freeze-repro-${i}`, + '--command', + floodCommand(marker) + ], + { timeoutMs: 180_000 } + ) + timings.add({ op: 'terminal.create', ms: createdTerm.elapsedMs, ok: true, index: i }) + const handle = extractTerminalHandle(createdTerm.result) + if (handle) { + created.push({ handle, marker, worktree: selector }) + console.log( + `[live-freeze] created ${handle} on ${selector} in ${createdTerm.elapsedMs.toFixed(0)}ms` + ) + } else { + notes.push( + `create ${i} missing handle: ${JSON.stringify(createdTerm.result).slice(0, 400)}` + ) + } + } catch (error) { + timings.add({ op: 'terminal.create', ms: null, ok: false, error: String(error), index: i }) + notes.push(`create ${i} failed: ${String(error).slice(0, 300)}`) + console.warn(`[live-freeze] create failed: ${String(error)}`) + } + }) + } + + if (preFloodMs > 0 && created.length > 0) { + amplificationSteps.push(`preFloodMs=${preFloodMs}`) + await new Promise((r) => setTimeout(r, preFloodMs)) + } + + let live = [] + try { + const listed = orcaJsonSync(['terminal', 'list']) + const terms = listed.result?.terminals || [] + live = terms + .filter( + (t) => typeof t.handle === 'string' && t.handle.startsWith('term_') && t.connected !== false + ) + .map((t) => ({ handle: t.handle, title: t.title, worktreeId: t.worktreeId })) + notes.push(`live terminals listed=${live.length}`) + } catch (error) { + notes.push(`terminal list failed: ${String(error).slice(0, 200)}`) + } + + let switchTargets = [...created.map((c) => c.handle), ...live.map((t) => t.handle)].filter( + (v, i, a) => typeof v === 'string' && a.indexOf(v) === i + ) + + if (shouldCapSwitchTargets(maxSwitchTargets) && switchTargets.length > maxSwitchTargets) { + switchTargets = applySwitchTargetCap(switchTargets, maxSwitchTargets) + amplificationSteps.push(`capped switchTargets=${maxSwitchTargets}`) + } + + if (switchTargets.length < 2) { + throw new Error( + `Need ≥2 terminals to bulk-switch; got ${switchTargets.length}. notes=${notes.join('; ')}` + ) + } + + amplificationSteps.push( + `switchTargets=${switchTargets.length} passes=${switchPasses} parallel=${parallel}` + ) + console.log( + `[live-freeze] bulk-switching ${switchTargets.length} terminals × ${switchPasses} passes (parallel=${parallel})` + ) + + let maxSwitchMs = 0 + let maxBatchWallMs = 0 + let sumSwitchMs = 0 + let switchCount = 0 + const switchStarted = performance.now() + + for (let pass = 0; pass < switchPasses; pass += 1) { + // Chunk targets into concurrent batches — piles load onto client/UI path. + for (let offset = 0; offset < switchTargets.length; offset += parallel) { + const batch = switchTargets.slice(offset, offset + parallel) + const batchStarted = performance.now() + const batchResults = await Promise.all( + batch.map(async (handle) => { + try { + const sw = await orcaJsonAsync(['terminal', 'switch', '--terminal', handle], { + timeoutMs: 90_000 + }) + return { handle, ms: sw.elapsedMs, ok: true } + } catch (error) { + return { handle, error: String(error), ok: false } + } + }) + ) + const batchWall = performance.now() - batchStarted + maxBatchWallMs = Math.max(maxBatchWallMs, batchWall) + for (const item of batchResults) { + if (item.ok) { + maxSwitchMs = Math.max(maxSwitchMs, item.ms) + sumSwitchMs += item.ms + switchCount += 1 + timings.add({ op: 'terminal.switch', handle: item.handle, ms: item.ms, batchWall }) + if (item.ms >= softMs) { + console.warn(`[live-freeze] SOFT lag on switch ${item.handle}: ${item.ms.toFixed(0)}ms`) + } + if (item.ms >= hardMs) { + console.warn(`[live-freeze] HARD lag on switch ${item.handle}: ${item.ms.toFixed(0)}ms`) + } + } else { + timings.add({ op: 'terminal.switch', handle: item.handle, error: item.error }) + notes.push(`switch ${item.handle} failed: ${String(item.error).slice(0, 200)}`) + } + } + if (batchWall >= softMs) { + console.warn(`[live-freeze] SOFT batch wall=${batchWall.toFixed(0)}ms size=${batch.length}`) + } + if (batchWall >= hardMs) { + console.warn(`[live-freeze] HARD batch wall=${batchWall.toFixed(0)}ms size=${batch.length}`) + } + } + } + + const bulkWallMs = performance.now() - switchStarted + const avgSwitchMs = switchCount ? sumSwitchMs / switchCount : 0 + + const statusProbe = orcaJsonSync(['status'], { local: true }) + let memoryProbeMs = null + try { + const mem = orcaJsonSync(['diagnostics', 'memory'], { local: true, timeoutMs: 120_000 }) + memoryProbeMs = mem.elapsedMs + notes.push(`memory diagnostic ms=${mem.elapsedMs.toFixed(0)}`) + } catch (error) { + notes.push(`memory diagnostic failed: ${String(error).slice(0, 200)}`) + } + + const { peakLatencyMs, softFreeze, hardFreeze } = evaluateFreezeSignals({ + maxSwitchMs, + maxBatchWallMs, + statusProbeMs: statusProbe.elapsedMs, + memoryProbeMs, + softMs, + hardMs + }) + + let samplePath = null + if (softFreeze || hardFreeze) { + samplePath = sampleOrcaIfPossible() + if (samplePath) { + notes.push(`sample=${samplePath}`) + } else { + notes.push('sample unavailable') + } + } + + const report = { + topology: 'live-paired-remote', + environment: envName, + localVersion: local.result?.runtime?.appVersion, + remoteVersion: status.result?.runtime?.appVersion, + remoteWorktreeCount: wtList.length, + createdTerminals: created.length, + switchTargets: switchTargets.length, + switchPasses, + parallel, + maxSwitchMs, + maxBatchWallMs, + peakLatencyMs, + avgSwitchMs, + bulkWallMs, + statusProbeMs: statusProbe.elapsedMs, + memoryProbeMs, + softFreeze, + hardFreeze, + softMs, + hardMs, + amplificationSteps, + notes, + timingCount: timings.totalCount, + timings: timings.values() + } + + const outPath = path.join(reportDir, `live-bulk-open-freeze-${envName}.json`) + writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`) + // Also write a stamped peak report so amplification runs don't overwrite history. + const stamped = path.join(reportDir, `live-bulk-open-freeze-${envName}-peak-${Date.now()}.json`) + writeFileSync(stamped, `${JSON.stringify(report, null, 2)}\n`) + console.log(`[live-freeze] report ${outPath}`) + console.log(`[live-freeze] stamped ${stamped}`) + console.log(JSON.stringify(report, null, 2)) + + if (scratchDir) { + try { + mkdirSync(scratchDir, { recursive: true }) + copyFileSync(outPath, path.join(scratchDir, 'live-bulk-open-freeze-report.json')) + writeFileSync( + path.join(scratchDir, 'live-freeze-amplify-summary.json'), + `${JSON.stringify( + { + peakLatencyMs, + hardFreeze, + softFreeze, + amplificationSteps, + stamped + }, + null, + 2 + )}\n` + ) + } catch (error) { + notes.push(`scratch copy failed: ${String(error).slice(0, 200)}`) + } + } + + if (hardFreeze) { + process.exitCode = 2 + console.error('[live-freeze] HARD FREEZE SIGNAL') + } else if (softFreeze) { + process.exitCode = 1 + console.error('[live-freeze] SOFT FREEZE SIGNAL') + } else { + console.log('[live-freeze] no freeze signal under thresholds') + } +} + +main().catch((error) => { + console.error('[live-freeze] failed', error) + process.exit(3) +}) diff --git a/config/scripts/live-remote-freeze-rpc.mjs b/config/scripts/live-remote-freeze-rpc.mjs new file mode 100644 index 000000000..cb018147b --- /dev/null +++ b/config/scripts/live-remote-freeze-rpc.mjs @@ -0,0 +1,241 @@ +import { spawn, spawnSync } from 'node:child_process' +import path from 'node:path' + +export const MAX_ORCA_RPC_OUTPUT_BYTES = 20 * 1024 * 1024 + +export function appendOrcaRpcOutput(output, chunk, bytes, limit = MAX_ORCA_RPC_OUTPUT_BYTES) { + const nextBytes = bytes + Buffer.byteLength(chunk) + return { + output: nextBytes > limit ? output : output + chunk, + bytes: nextBytes, + exceeded: nextBytes > limit + } +} + +export function resolveOrcaCliCommand({ env = process.env, platform = process.platform } = {}) { + if (env.ORCA_CLI_COMMAND?.trim()) { + return env.ORCA_CLI_COMMAND.trim() + } + if (env.ORCA_DEV_REPO_ROOT) { + return 'orca-dev' + } + return platform === 'linux' ? 'orca-ide' : 'orca' +} + +export function resolveOrcaCliInvocation({ + env = process.env, + platform = process.platform, + nodeExecutable = process.execPath +} = {}) { + const command = resolveOrcaCliCommand({ env, platform }) + const commandName = platform === 'win32' ? path.win32.basename(command).toLowerCase() : command + if ( + platform === 'win32' && + env.ORCA_DEV_REPO_ROOT && + (commandName === 'orca-dev' || commandName === 'orca-dev.cmd') + ) { + const defaultUserDataPath = path.win32.join( + env.APPDATA ?? path.win32.join(env.USERPROFILE ?? '', 'AppData', 'Roaming'), + 'orca-dev' + ) + return { + command: nodeExecutable, + prefixArgs: [path.win32.join(env.ORCA_DEV_REPO_ROOT, 'out', 'cli', 'index.js')], + env: { + ...env, + ORCA_USER_DATA_PATH: + env.ORCA_USER_DATA_PATH ?? env.ORCA_DEV_USER_DATA_PATH ?? defaultUserDataPath, + ORCA_DEV_CLI_INVOCATION: '1', + ORCA_APP_EXECUTABLE: + env.ORCA_APP_EXECUTABLE ?? + path.win32.join( + env.ORCA_DEV_REPO_ROOT, + 'node_modules', + 'electron', + 'dist', + 'electron.exe' + ), + ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT: '1' + } + } + } + return { command, prefixArgs: [] } +} + +export function createOrcaRpc({ + envName, + cliCommand, + env = process.env, + platform = process.platform +}) { + const cliInvocation = cliCommand + ? { command: cliCommand, prefixArgs: [] } + : resolveOrcaCliInvocation({ env, platform }) + const commandLabel = cliCommand ?? resolveOrcaCliCommand({ env, platform }) + const commandArgs = (args, local) => [ + ...cliInvocation.prefixArgs, + ...args, + ...(local ? [] : ['--environment', envName]), + '--json' + ] + + function orcaJsonSync(args, opts = {}) { + const started = performance.now() + const result = spawnSync(cliInvocation.command, commandArgs(args, opts.local), { + encoding: 'utf8', + env: cliInvocation.env, + maxBuffer: MAX_ORCA_RPC_OUTPUT_BYTES, + timeout: opts.timeoutMs ?? 120_000 + }) + const elapsedMs = performance.now() - started + if (result.error) { + throw new Error(`${commandLabel} ${args.join(' ')} failed to start: ${String(result.error)}`) + } + if (result.status !== 0) { + throw new Error( + `${commandLabel} ${args.join(' ')} failed (${result.status}): ${result.stderr || result.stdout}` + ) + } + const parsed = JSON.parse(result.stdout) + if (parsed.ok === false) { + throw new Error(`${commandLabel} ${args.join(' ')} ok=false: ${JSON.stringify(parsed)}`) + } + return { parsed, elapsedMs, result: parsed.result } + } + + function orcaJsonAsync(args, opts = {}) { + const started = performance.now() + return new Promise((resolve, reject) => { + const child = spawn(cliInvocation.command, commandArgs(args, opts.local), { + env: cliInvocation.env, + stdio: ['ignore', 'pipe', 'pipe'] + }) + let stdout = '' + let stderr = '' + let outputBytes = 0 + let settled = false + let timer + const fail = (error) => { + if (settled) { + return + } + settled = true + clearTimeout(timer) + reject(error) + } + const append = (stream, chunk) => { + if (settled) { + return stream + } + const appended = appendOrcaRpcOutput(stream, chunk, outputBytes) + outputBytes = appended.bytes + if (appended.exceeded) { + child.kill('SIGKILL') + fail(new Error(`${commandLabel} ${args.join(' ')} exceeded 20 MiB output limit`)) + return stream + } + return appended.output + } + timer = setTimeout(() => { + child.kill('SIGKILL') + fail( + new Error( + `${commandLabel} ${args.join(' ')} timed out after ${opts.timeoutMs ?? 120_000}ms` + ) + ) + }, opts.timeoutMs ?? 120_000) + child.stdout.setEncoding('utf8') + child.stderr.setEncoding('utf8') + child.stdout.on('data', (chunk) => { + stdout = append(stdout, chunk) + }) + child.stderr.on('data', (chunk) => { + stderr = append(stderr, chunk) + }) + child.on('error', fail) + child.on('close', (code) => { + if (settled) { + return + } + clearTimeout(timer) + const elapsedMs = performance.now() - started + if (code !== 0) { + fail( + new Error( + `${commandLabel} ${args.join(' ')} failed (${code}): ${stderr || stdout}`.slice( + 0, + 800 + ) + ) + ) + return + } + try { + const parsed = JSON.parse(stdout) + if (parsed.ok === false) { + fail( + new Error( + `${commandLabel} ${args.join(' ')} ok=false: ${JSON.stringify(parsed)}`.slice( + 0, + 800 + ) + ) + ) + return + } + settled = true + resolve({ parsed, elapsedMs, result: parsed.result }) + } catch (error) { + fail( + new Error( + `${commandLabel} parse failed: ${String(error)}; stdout=${stdout.slice(0, 400)}` + ) + ) + } + }) + }) + } + + async function runReconnectRefreshStorm(notes) { + const started = performance.now() + const jobs = [ + () => orcaJsonAsync(['status'], { timeoutMs: 90_000 }), + () => orcaJsonAsync(['worktree', 'list'], { timeoutMs: 120_000 }), + () => orcaJsonAsync(['terminal', 'list'], { timeoutMs: 120_000 }), + () => orcaJsonAsync(['status'], { local: true, timeoutMs: 60_000 }), + () => orcaJsonAsync(['worktree', 'list'], { timeoutMs: 120_000 }), + () => orcaJsonAsync(['terminal', 'list'], { timeoutMs: 120_000 }) + ] + const results = await Promise.all( + jobs.map(async (job, index) => { + try { + const result = await job() + return { index, ok: true, ms: result.elapsedMs } + } catch (error) { + notes.push(`reconnect-refresh job ${index} failed: ${String(error).slice(0, 200)}`) + return { index, ok: false, ms: null, error: String(error) } + } + }) + ) + const wallMs = performance.now() - started + const maxJobMs = Math.max(0, ...results.map((result) => result.ms || 0)) + notes.push( + `reconnect-refresh wall=${wallMs.toFixed(0)}ms maxJob=${maxJobMs.toFixed(0)}ms ok=${results.filter((result) => result.ok).length}/${results.length}` + ) + return { wallMs, maxJobMs, results } + } + + async function runRestartProxy(notes) { + const started = performance.now() + try { + const opened = await orcaJsonAsync(['open'], { local: true, timeoutMs: 120_000 }) + notes.push(`orca open ms=${opened.elapsedMs.toFixed(0)}`) + } catch (error) { + notes.push(`orca open failed: ${String(error).slice(0, 200)}`) + } + const storm = await runReconnectRefreshStorm(notes) + return { wallMs: performance.now() - started, storm } + } + + return { orcaJsonSync, orcaJsonAsync, runReconnectRefreshStorm, runRestartProxy } +} diff --git a/config/scripts/live-remote-freeze-rpc.test.mjs b/config/scripts/live-remote-freeze-rpc.test.mjs new file mode 100644 index 000000000..e68bd09df --- /dev/null +++ b/config/scripts/live-remote-freeze-rpc.test.mjs @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import { + appendOrcaRpcOutput, + resolveOrcaCliCommand, + resolveOrcaCliInvocation +} from './live-remote-freeze-rpc.mjs' + +describe('live remote freeze RPC', () => { + it('resolves the Orca CLI for managed, dev, Linux, and default runtimes', () => { + expect(resolveOrcaCliCommand({ env: { ORCA_CLI_COMMAND: 'custom-orca' } })).toBe('custom-orca') + expect(resolveOrcaCliCommand({ env: { ORCA_DEV_REPO_ROOT: '/repo' } })).toBe('orca-dev') + expect(resolveOrcaCliCommand({ env: {}, platform: 'linux' })).toBe('orca-ide') + expect(resolveOrcaCliCommand({ env: {}, platform: 'win32' })).toBe('orca') + }) + + it('bypasses the Windows dev cmd shim with the built Node CLI', () => { + const invocation = resolveOrcaCliInvocation({ + env: { + APPDATA: 'C:\\Users\\dev\\AppData\\Roaming', + ORCA_CLI_COMMAND: 'C:\\repo\\out\\bin\\orca-dev.cmd', + ORCA_DEV_REPO_ROOT: 'C:\\repo' + }, + platform: 'win32', + nodeExecutable: 'C:\\Program Files\\nodejs\\node.exe' + }) + + expect(invocation).toMatchObject({ + command: 'C:\\Program Files\\nodejs\\node.exe', + prefixArgs: ['C:\\repo\\out\\cli\\index.js'], + env: { + ORCA_USER_DATA_PATH: 'C:\\Users\\dev\\AppData\\Roaming\\orca-dev', + ORCA_DEV_CLI_INVOCATION: '1', + ORCA_APP_EXECUTABLE: 'C:\\repo\\node_modules\\electron\\dist\\electron.exe', + ORCA_APP_EXECUTABLE_NEEDS_APP_ROOT: '1' + } + }) + }) + + it('caps combined asynchronous output before retaining the overflow chunk', () => { + const first = appendOrcaRpcOutput('', '1234', 0, 5) + expect(first).toEqual({ output: '1234', bytes: 4, exceeded: false }) + + const overflow = appendOrcaRpcOutput(first.output, '67', first.bytes, 5) + expect(overflow).toEqual({ output: '1234', bytes: 6, exceeded: true }) + }) +}) diff --git a/config/scripts/live-remote-realistic-freeze-repro.mjs b/config/scripts/live-remote-realistic-freeze-repro.mjs new file mode 100644 index 000000000..40820f48a --- /dev/null +++ b/config/scripts/live-remote-realistic-freeze-repro.mjs @@ -0,0 +1,645 @@ +#!/usr/bin/env node +/** + * Naturalistic freeze repro — idle/reconnect recovery stories on large remotes. + * + * Unlike the bulk parallel-switch amplifier, this models: + * 1) agents streaming on remote while user is idle (backlog builds) + * 2) user returns and opens sessions one-by-one (or after reconnect refresh) + * + * Scenarios: + * idle-backlog-open — idle with flood, then human-paced sequential open + * idle-backlog-reconnect-open — same + wake-like metadata refresh storm, then open + * restart-proxy — idle, then orca open + status/list storm + open + * (does NOT kill the desktop; proxies restore work) + * + * Usage: + * ORCA_FREEZE_ENV=paired-remote ORCA_FREEZE_SCENARIO=idle-backlog-open \ + * node config/scripts/live-remote-realistic-freeze-repro.mjs + * + * pnpm run repro:live-remote-realistic-freeze + */ +import { spawnSync } from 'node:child_process' +import { copyFileSync, mkdirSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import { createOrcaRpc } from './live-remote-freeze-rpc.mjs' +import { startStatusWatchdog } from './live-remote-status-watchdog.mjs' +import { BoundedLiveFreezeHistory } from './live-freeze-bounded-history.mjs' +import { + DEFAULT_FOREVER_WINDOW_MS, + DEFAULT_HARD_MS, + DEFAULT_SOFT_MS, + DEFAULT_STATUS_SLOW_MS, + evaluateFullAppFreeze, + evaluatePermanentLockup, + evaluateRealisticFreezeSignals, + extractTerminalHandle, + humanPaceDelayMs, + readFreezeNumberEnv, + REALISTIC_SCENARIOS, + worktreeSelector +} from './live-remote-bulk-open-freeze-metrics.mjs' + +const root = path.resolve(import.meta.dirname, '../..') +const reportDir = path.join(root, 'test-results', 'freeze-repro') +const envName = process.env.ORCA_FREEZE_ENV || 'paired-remote' +const scenario = process.env.ORCA_FREEZE_SCENARIO || 'idle-backlog-open' +const createCount = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_CREATE', 0)) +const openCount = Math.max(2, readFreezeNumberEnv('ORCA_FREEZE_OPEN_COUNT', 20)) +const idleMs = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_IDLE_MS', 45_000)) +const paceMs = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_PACE_MS', 250)) +const paceJitterMs = Math.max(0, readFreezeNumberEnv('ORCA_FREEZE_PACE_JITTER_MS', 150)) +const createWorktreeSpan = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_CREATE_WT_SPAN', 12)) +const softMs = readFreezeNumberEnv('ORCA_FREEZE_SOFT_MS', DEFAULT_SOFT_MS) +const hardMs = readFreezeNumberEnv('ORCA_FREEZE_HARD_MS', DEFAULT_HARD_MS) +/** Concurrent opens during lockup-storm (wake refresh overlaps fan-out). */ +const stormParallel = Math.max(1, readFreezeNumberEnv('ORCA_FREEZE_STORM_PARALLEL', 16)) +/** Kill a switch if it exceeds this — counts toward permanent lockup. */ +const opTimeoutMs = Math.max(10_000, readFreezeNumberEnv('ORCA_FREEZE_OP_TIMEOUT_MS', 60_000)) +const permanentTimeoutMs = Math.max(15_000, readFreezeNumberEnv('ORCA_FREEZE_PERMANENT_MS', 60_000)) +const foreverWindowMs = Math.max( + 10_000, + readFreezeNumberEnv('ORCA_FREEZE_FOREVER_WINDOW_MS', DEFAULT_FOREVER_WINDOW_MS) +) +const statusSlowMs = Math.max( + 5_000, + readFreezeNumberEnv('ORCA_FREEZE_STATUS_SLOW_MS', DEFAULT_STATUS_SLOW_MS) +) +const watchdogIntervalMs = Math.max( + 500, + readFreezeNumberEnv('ORCA_FREEZE_WATCHDOG_INTERVAL_MS', 1500) +) +const scratchDir = process.env.ORCA_FREEZE_SCRATCH || '' + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +const rpc = createOrcaRpc({ envName }) +const { orcaJsonSync, orcaJsonAsync, runReconnectRefreshStorm, runRestartProxy } = rpc + +async function mapPool(items, concurrency, worker) { + const results = Array.from({ length: items.length }) + let next = 0 + async function run() { + while (next < items.length) { + const index = next + next += 1 + results[index] = await worker(items[index], index) + } + } + await Promise.all( + Array.from({ length: Math.min(concurrency, Math.max(items.length, 1)) }, () => run()) + ) + return results +} + +function floodCommand(marker) { + const script = + "const m=process.argv[1];process.stdout.write('READY:'+m+'\n');let f=0;const c='A'.repeat(2048);setInterval(()=>{f++;process.stdout.write('BG:'+m+':'+f+':'+c+'\n')},8);process.stdin.resume()" + return `node -e ${JSON.stringify(script)} ${JSON.stringify(marker)}` +} + +function sampleOrcaIfPossible() { + if (process.platform !== 'darwin') { + return null + } + try { + const status = orcaJsonSync(['status'], { local: true }).result + const pid = status?.app?.pid + if (!pid) { + return null + } + const out = path.join(reportDir, `orca-sample-realistic-${Date.now()}.txt`) + const sampled = spawnSync('sample', [String(pid), '5', '-file', out], { + timeout: 20_000, + stdio: 'ignore' + }) + return sampled.status === 0 ? out : null + } catch { + return null + } +} + +function listLiveTerminalHandles() { + const listed = orcaJsonSync(['terminal', 'list']) + const terms = listed.result?.terminals || [] + return terms + .filter((t) => typeof t.handle === 'string' && t.handle.startsWith('term_')) + .map((t) => ({ + handle: t.handle, + title: t.title, + worktreeId: t.worktreeId, + connected: t.connected + })) +} + +async function main() { + if (!REALISTIC_SCENARIOS.includes(scenario)) { + throw new Error( + `Unknown ORCA_FREEZE_SCENARIO=${scenario}. Expected one of: ${REALISTIC_SCENARIOS.join(', ')}` + ) + } + + mkdirSync(reportDir, { recursive: true }) + const notes = [] + const phases = [] + const openTimings = new BoundedLiveFreezeHistory(100) + + console.log( + `[realistic-freeze] scenario=${scenario} env=${envName} create=${createCount} idleMs=${idleMs} openCount=${openCount} paceMs=${paceMs}` + ) + + const local = orcaJsonSync(['status'], { local: true }) + const remote = orcaJsonSync(['status']) + notes.push( + `local version=${local.result?.runtime?.appVersion} pid=${local.result?.app?.pid}`, + `remote version=${remote.result?.runtime?.appVersion} state=${remote.result?.runtime?.state}` + ) + + const worktrees = orcaJsonSync(['worktree', 'list']).result + const wtList = worktrees?.worktrees || worktrees?.items || worktrees || [] + if (!Array.isArray(wtList) || wtList.length === 0) { + throw new Error(`No worktrees on environment ${envName}`) + } + notes.push(`remote worktrees=${wtList.length}`) + phases.push({ phase: 'baseline', worktrees: wtList.length }) + + // --- Phase: seed flood terminals (agent-like backlog sources) --- + const created = [] + if (createCount > 0) { + const targets = wtList.slice(0, Math.min(createWorktreeSpan, wtList.length)) + await mapPool( + Array.from({ length: createCount }, (_, i) => i), + Math.min(4, createCount), + async (i) => { + const wt = targets[i % targets.length] + const selector = worktreeSelector(wt) + if (!selector) { + return + } + const marker = `REALISTIC_${Date.now()}_${i}` + try { + const createdTerm = await orcaJsonAsync( + [ + 'terminal', + 'create', + '--worktree', + selector, + '--title', + `realistic-freeze-${i}`, + '--command', + floodCommand(marker) + ], + { timeoutMs: 180_000 } + ) + const handle = extractTerminalHandle(createdTerm.result) + if (handle) { + created.push({ handle, marker, worktree: selector }) + console.log( + `[realistic-freeze] flood terminal ${handle} (${createdTerm.elapsedMs.toFixed(0)}ms)` + ) + } else { + notes.push( + `create ${i} missing handle: ${JSON.stringify(createdTerm.result).slice(0, 300)}` + ) + } + } catch (error) { + notes.push(`create ${i} failed: ${String(error).slice(0, 250)}`) + console.warn(`[realistic-freeze] create failed: ${String(error)}`) + } + } + ) + phases.push({ phase: 'seed-flood', created: created.length }) + } + + // Prefer created floods for open pass; fill with existing live terminals. + let live = [] + try { + live = listLiveTerminalHandles() + notes.push(`live terminals listed=${live.length}`) + } catch (error) { + notes.push(`terminal list failed: ${String(error).slice(0, 200)}`) + } + + const openTargets = [...created.map((c) => c.handle), ...live.map((t) => t.handle)].filter( + (v, i, a) => typeof v === 'string' && a.indexOf(v) === i + ) + + if (openTargets.length < 2) { + throw new Error(`Need ≥2 terminals; got ${openTargets.length}. ${notes.join('; ')}`) + } + + const openList = openTargets.slice(0, Math.min(openCount, openTargets.length)) + + // --- Phase: park — leave one session focused, rest accumulate flood while "away" --- + try { + const parkHandle = openList[0] + const parked = await orcaJsonAsync(['terminal', 'switch', '--terminal', parkHandle], { + timeoutMs: 60_000 + }) + notes.push(`park switch ms=${parked.elapsedMs.toFixed(0)} handle=${parkHandle}`) + } catch (error) { + notes.push(`park switch failed: ${String(error).slice(0, 200)}`) + } + + console.log(`[realistic-freeze] idle ${idleMs}ms while remotes stream (user away / asleep)`) + const idleStarted = performance.now() + await sleep(idleMs) + phases.push({ phase: 'idle', idleMs, actualMs: performance.now() - idleStarted }) + + // --- Phase: recovery trigger --- + let reconnectRefreshMs = 0 + let timedOutOps = 0 + let consecutiveSwitchFailures = 0 + let maxConsecutiveSwitchFailures = 0 + + if (scenario === 'idle-backlog-reconnect-open' || scenario === 'lockup-storm') { + console.log( + '[realistic-freeze] wake/reconnect proxy: parallel status/worktree/terminal refresh' + ) + const storm = await runReconnectRefreshStorm(notes) + reconnectRefreshMs = Math.max(storm.wallMs, storm.maxJobMs) + phases.push({ + phase: 'reconnect-refresh', + wallMs: storm.wallMs, + maxJobMs: storm.maxJobMs + }) + } else if (scenario === 'restart-proxy') { + console.log('[realistic-freeze] restart proxy: orca open + refresh storm (no process kill)') + const restart = await runRestartProxy(notes) + reconnectRefreshMs = Math.max(restart.wallMs, restart.storm.wallMs, restart.storm.maxJobMs) + phases.push({ + phase: 'restart-proxy', + wallMs: restart.wallMs, + reconnectWallMs: restart.storm.wallMs + }) + } + + // --- Phase: open sessions --- + // lockup-storm: overlap a second reconnect storm with concurrent switch fan-out + // (models wake + bulk session restore, not human serial clicks). + let maxOpenMs = 0 + let firstOpenMs = 0 + let sumOpenMs = 0 + let openOk = 0 + let maxBatchWallMs = 0 + const openStarted = performance.now() + + let statusWatch = null + if (scenario === 'lockup-storm') { + console.log( + `[realistic-freeze] LOCKUP STORM: concurrent open parallel=${stormParallel} + overlapping reconnect refresh (timeout=${opTimeoutMs}ms); mid-storm status watchdog every ${watchdogIntervalMs}ms` + ) + statusWatch = startStatusWatchdog({ + intervalMs: watchdogIntervalMs, + timeoutMs: Math.min(permanentTimeoutMs, foreverWindowMs), + statusSlowMs + }) + // Fire reconnect storm again concurrently with first open wave. + const overlapStormPromise = runReconnectRefreshStorm(notes) + for (let offset = 0; offset < openList.length; offset += stormParallel) { + const batch = openList.slice(offset, offset + stormParallel) + const batchStarted = performance.now() + const batchResults = await Promise.all( + batch.map(async (handle, batchIndex) => { + const index = offset + batchIndex + try { + const sw = await orcaJsonAsync(['terminal', 'switch', '--terminal', handle], { + timeoutMs: opTimeoutMs + }) + return { handle, index, ms: sw.elapsedMs, ok: true, timedOut: false } + } catch (error) { + const msg = String(error) + const timedOut = /timed out/i.test(msg) + return { handle, index, error: msg, ok: false, timedOut } + } + }) + ) + const batchWall = performance.now() - batchStarted + maxBatchWallMs = Math.max(maxBatchWallMs, batchWall) + for (const item of batchResults) { + if (item.ok) { + openOk += 1 + sumOpenMs += item.ms + maxOpenMs = Math.max(maxOpenMs, item.ms) + if (item.index === 0 || firstOpenMs === 0) { + firstOpenMs = item.ms + } + consecutiveSwitchFailures = 0 + openTimings.add({ + handle: item.handle, + ms: item.ms, + index: item.index, + batchWall + }) + if (item.ms >= hardMs) { + console.warn( + `[realistic-freeze] HARD open #${item.index} ${item.handle}: ${item.ms.toFixed(0)}ms` + ) + } + } else { + if (item.timedOut) { + timedOutOps += 1 + } + consecutiveSwitchFailures += 1 + maxConsecutiveSwitchFailures = Math.max( + maxConsecutiveSwitchFailures, + consecutiveSwitchFailures + ) + openTimings.add({ + handle: item.handle, + error: item.error, + index: item.index, + timedOut: item.timedOut + }) + notes.push( + `open ${item.handle} failed${item.timedOut ? ' (TIMEOUT)' : ''}: ${String(item.error).slice(0, 160)}` + ) + console.warn( + `[realistic-freeze] open FAIL #${item.index}${item.timedOut ? ' TIMEOUT' : ''}: ${item.handle}` + ) + } + } + if (batchWall >= hardMs) { + console.warn( + `[realistic-freeze] HARD batch wall=${batchWall.toFixed(0)}ms size=${batch.length}` + ) + } + } + try { + const overlap = await overlapStormPromise + reconnectRefreshMs = Math.max(reconnectRefreshMs, overlap.wallMs, overlap.maxJobMs) + phases.push({ + phase: 'overlap-reconnect-refresh', + wallMs: overlap.wallMs, + maxJobMs: overlap.maxJobMs + }) + } catch (error) { + notes.push(`overlap reconnect failed: ${String(error).slice(0, 200)}`) + } + phases.push({ + phase: 'lockup-storm-open', + count: openList.length, + ok: openOk, + maxOpenMs, + firstOpenMs, + maxBatchWallMs, + timedOutOps, + parallel: stormParallel + }) + } else { + console.log( + `[realistic-freeze] human-paced open of ${openList.length} sessions (pace≈${paceMs}ms + jitter)` + ) + for (let i = 0; i < openList.length; i += 1) { + const handle = openList[i] + try { + const sw = await orcaJsonAsync(['terminal', 'switch', '--terminal', handle], { + timeoutMs: opTimeoutMs + }) + openOk += 1 + sumOpenMs += sw.elapsedMs + maxOpenMs = Math.max(maxOpenMs, sw.elapsedMs) + if (i === 0) { + firstOpenMs = sw.elapsedMs + } + consecutiveSwitchFailures = 0 + openTimings.add({ handle, ms: sw.elapsedMs, index: i }) + if (sw.elapsedMs >= softMs) { + console.warn(`[realistic-freeze] SOFT open #${i} ${handle}: ${sw.elapsedMs.toFixed(0)}ms`) + } + if (sw.elapsedMs >= hardMs) { + console.warn(`[realistic-freeze] HARD open #${i} ${handle}: ${sw.elapsedMs.toFixed(0)}ms`) + } + } catch (error) { + const msg = String(error) + const timedOut = /timed out/i.test(msg) + if (timedOut) { + timedOutOps += 1 + } + consecutiveSwitchFailures += 1 + maxConsecutiveSwitchFailures = Math.max( + maxConsecutiveSwitchFailures, + consecutiveSwitchFailures + ) + openTimings.add({ handle, error: msg, index: i, timedOut }) + notes.push(`open ${handle} failed${timedOut ? ' (TIMEOUT)' : ''}: ${msg.slice(0, 200)}`) + } + if (i < openList.length - 1) { + await sleep(humanPaceDelayMs(paceMs, paceJitterMs)) + } + } + phases.push({ + phase: 'human-paced-open', + count: openList.length, + ok: openOk, + maxOpenMs, + firstOpenMs, + openWallMs: performance.now() - openStarted + }) + } + + const openWallMs = performance.now() - openStarted + + let midStormWatch = { + samples: [], + durationMs: 0, + sampleCount: 0, + maxStatusMs: 0, + unhealthySampleCount: 0, + infrastructureErrorCount: 0, + longestUnhealthyWindowMs: 0 + } + if (statusWatch) { + midStormWatch = await statusWatch.stop() + notes.push( + `mid-storm status samples=${midStormWatch.sampleCount} durationMs=${midStormWatch.durationMs.toFixed(0)}` + ) + phases.push({ + phase: 'mid-storm-status-watchdog', + samples: midStormWatch.sampleCount, + durationMs: midStormWatch.durationMs, + maxStatusMs: midStormWatch.maxStatusMs + }) + } + + // Post-storm health: does local status still answer? + let statusProbeMs = null + let statusHangMs = 0 + const statusStarted = performance.now() + try { + const statusProbe = await orcaJsonAsync(['status'], { + local: true, + timeoutMs: permanentTimeoutMs + }) + statusProbeMs = statusProbe.elapsedMs + } catch (error) { + statusHangMs = performance.now() - statusStarted + notes.push( + `status probe FAILED after ${statusHangMs.toFixed(0)}ms: ${String(error).slice(0, 200)}` + ) + console.error(`[realistic-freeze] status probe failed — possible permanent lockup`) + } + + let memoryProbeMs = null + try { + const mem = await orcaJsonAsync(['diagnostics', 'memory'], { + local: true, + timeoutMs: permanentTimeoutMs + }) + memoryProbeMs = mem.elapsedMs + notes.push(`memory diagnostic ms=${mem.elapsedMs.toFixed(0)}`) + } catch (error) { + notes.push(`memory diagnostic failed: ${String(error).slice(0, 200)}`) + } + + const peakForSignals = Math.max(maxOpenMs, firstOpenMs, maxBatchWallMs) + const signals = evaluateRealisticFreezeSignals({ + maxOpenMs: peakForSignals, + firstOpenMs, + reconnectRefreshMs, + statusProbeMs: statusProbeMs ?? 0, + memoryProbeMs, + softMs, + hardMs + }) + + const lockup = evaluatePermanentLockup({ + timedOutOps, + statusHangMs, + consecutiveSwitchFailures: maxConsecutiveSwitchFailures, + openFailed: openList.length - openOk, + openTotal: openList.length, + permanentTimeoutMs + }) + + const fullApp = evaluateFullAppFreeze({ + statusSamples: midStormWatch.samples, + statusSummary: midStormWatch, + foreverWindowMs, + statusSlowMs + }) + const watchdogInfrastructureErrorCount = midStormWatch.infrastructureErrorCount + if (statusHangMs >= foreverWindowMs) { + fullApp.foreverUiLockupObserved = true + fullApp.longestUnhealthyWindowMs = Math.max(fullApp.longestUnhealthyWindowMs, statusHangMs) + fullApp.reason = `post-storm status hang ${statusHangMs.toFixed(0)}ms` + } + + const recoveredHardStall = signals.hardFreeze && !fullApp.foreverUiLockupObserved && openOk > 0 + + let samplePath = null + if (signals.softFreeze || signals.hardFreeze || fullApp.foreverUiLockupObserved) { + samplePath = sampleOrcaIfPossible() + if (samplePath) { + notes.push(`sample=${samplePath}`) + } else { + notes.push('sample unavailable') + } + } + + const storyByScenario = { + 'idle-backlog-open': 'User away while remotes stream; returns and opens sessions one-by-one.', + 'idle-backlog-reconnect-open': + 'User away; wake-like reconnect metadata storm; then opens sessions.', + 'restart-proxy': 'User away; restart-proxy discovery; then opens sessions.', + 'lockup-storm': + 'Idle flood + reconnect refresh + concurrent open + mid-storm status watchdog (full-app freeze bar).' + } + + const report = { + topology: 'live-paired-remote-realistic', + scenario, + story: storyByScenario[scenario] || scenario, + environment: envName, + localVersion: local.result?.runtime?.appVersion, + remoteVersion: remote.result?.runtime?.appVersion, + remoteWorktreeCount: wtList.length, + createdFloodTerminals: created.length, + openTargets: openList.length, + idleMs, + paceMs, + paceJitterMs, + stormParallel: scenario === 'lockup-storm' ? stormParallel : 1, + firstOpenMs, + maxOpenMs, + maxBatchWallMs, + avgOpenMs: openOk ? sumOpenMs / openOk : 0, + openWallMs, + openOk, + openFailed: openList.length - openOk, + reconnectRefreshMs, + peakLatencyMs: Math.max(signals.peakLatencyMs, maxBatchWallMs), + statusProbeMs, + statusHangMs, + memoryProbeMs, + softFreeze: signals.softFreeze, + hardFreeze: signals.hardFreeze, + recoveredHardStall, + permanentLockup: lockup.permanentLockup, + foreverUiLockupObserved: fullApp.foreverUiLockupObserved, + foreverFreeze: fullApp, + midStormStatusSamples: midStormWatch.samples, + midStormStatusSampleCount: midStormWatch.sampleCount, + watchdogInfrastructureErrorCount, + timedOutOps, + maxConsecutiveSwitchFailures, + softMs, + hardMs, + foreverWindowMs, + statusSlowMs, + permanentTimeoutMs, + opTimeoutMs, + phases, + notes, + openTimingCount: openTimings.totalCount, + openTimings: openTimings.values() + } + + const outPath = path.join(reportDir, `live-realistic-freeze-${envName}-${scenario}.json`) + const stamped = path.join( + reportDir, + `live-realistic-freeze-${envName}-${scenario}-peak-${Date.now()}.json` + ) + writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`) + writeFileSync(stamped, `${JSON.stringify(report, null, 2)}\n`) + console.log(`[realistic-freeze] report ${outPath}`) + console.log(JSON.stringify(report, null, 2)) + + if (scratchDir) { + try { + mkdirSync(scratchDir, { recursive: true }) + copyFileSync(outPath, path.join(scratchDir, 'live-realistic-freeze-report.json')) + } catch (error) { + console.warn(`[realistic-freeze] scratch copy failed: ${String(error)}`) + } + } + + if (watchdogInfrastructureErrorCount > 0) { + process.exitCode = 3 + console.error('[realistic-freeze] WATCHDOG INFRASTRUCTURE FAILURE') + } else if (fullApp.foreverUiLockupObserved) { + process.exitCode = 5 + console.error('[realistic-freeze] FULL-APP FOREVER FREEZE (status unhealthy ≥ forever window)') + } else if (lockup.permanentLockup) { + process.exitCode = 4 + console.error( + '[realistic-freeze] PERMANENT LOCKUP HEURISTIC (timeouts/fail-rate) — check foreverUiLockupObserved' + ) + } else if (signals.hardFreeze) { + process.exitCode = 2 + console.error( + '[realistic-freeze] HARD FREEZE SIGNAL (recovered multi-second stall — not forever lockup)' + ) + } else if (signals.softFreeze) { + process.exitCode = 1 + console.error('[realistic-freeze] SOFT FREEZE SIGNAL') + } else { + console.log('[realistic-freeze] no freeze signal under thresholds') + } +} + +main().catch((error) => { + console.error('[realistic-freeze] failed', error) + process.exit(3) +}) diff --git a/config/scripts/live-remote-status-watchdog.mjs b/config/scripts/live-remote-status-watchdog.mjs new file mode 100644 index 000000000..9e3951437 --- /dev/null +++ b/config/scripts/live-remote-status-watchdog.mjs @@ -0,0 +1,147 @@ +/** + * Mid-storm host health samples for forever-freeze detection. + * Polls `orca status --json` on an interval while a load storm runs. + */ +import { spawn } from 'node:child_process' +import { BoundedLiveFreezeHistory } from './live-freeze-bounded-history.mjs' +import { resolveOrcaCliInvocation } from './live-remote-freeze-rpc.mjs' + +/** + * @param {{ intervalMs?: number, timeoutMs?: number, cliCommand?: string, sampleHistoryLimit?: number, statusSlowMs?: number }} opts + */ +export function startStatusWatchdog(opts = {}) { + const intervalMs = opts.intervalMs ?? 2000 + const timeoutMs = opts.timeoutMs ?? 30_000 + const cliInvocation = opts.cliCommand + ? { command: opts.cliCommand, prefixArgs: [] } + : resolveOrcaCliInvocation() + const samples = new BoundedLiveFreezeHistory(opts.sampleHistoryLimit ?? 240) + const statusSlowMs = opts.statusSlowMs ?? 15_000 + let stopped = false + let inFlight = false + let infrastructureErrorCount = 0 + let longestUnhealthyWindowMs = 0 + let maxStatusMs = 0 + let runStartMs = null + let unhealthySampleCount = 0 + const startedAt = performance.now() + + const record = (sample) => { + samples.add(sample) + maxStatusMs = Math.max(maxStatusMs, sample.ms || 0) + if (sample.infrastructureError) { + infrastructureErrorCount += 1 + } + const unhealthy = + !sample.infrastructureError && + (Boolean(sample.hang) || sample.ok === false || (sample.ms || 0) >= statusSlowMs) + if (!unhealthy) { + runStartMs = null + return + } + unhealthySampleCount += 1 + runStartMs ??= sample.tMs ?? 0 + longestUnhealthyWindowMs = Math.max( + longestUnhealthyWindowMs, + (sample.tMs ?? 0) + (sample.ms || 0) - runStartMs + ) + } + + const summary = () => ({ + sampleCount: samples.totalCount, + maxStatusMs, + unhealthySampleCount, + infrastructureErrorCount, + longestUnhealthyWindowMs + }) + + const probe = () => + new Promise((resolve) => { + const t0 = performance.now() + const child = spawn( + cliInvocation.command, + [...cliInvocation.prefixArgs, 'status', '--json'], + { + env: cliInvocation.env, + stdio: ['ignore', 'pipe', 'pipe'] + } + ) + let settled = false + const finish = (result) => { + if (settled) { + return + } + settled = true + resolve(result) + } + const timer = setTimeout(() => { + child.kill('SIGKILL') + finish({ + tMs: t0 - startedAt, + ms: performance.now() - t0, + ok: false, + hang: true + }) + }, timeoutMs) + child.stdout.on('data', () => {}) + child.stderr.on('data', () => {}) + child.on('error', (error) => { + clearTimeout(timer) + finish({ + tMs: t0 - startedAt, + ms: performance.now() - t0, + ok: false, + hang: false, + infrastructureError: true, + error: String(error) + }) + }) + child.on('close', (code) => { + clearTimeout(timer) + finish({ + tMs: t0 - startedAt, + ms: performance.now() - t0, + ok: code === 0, + hang: false + }) + }) + }) + + const tick = async ({ force = false } = {}) => { + if ((!force && stopped) || inFlight) { + return + } + inFlight = true + try { + const sample = await probe() + record(sample) + } finally { + inFlight = false + } + } + + const interval = setInterval(() => { + void tick() + }, intervalMs) + void tick() + + return { + stop: async () => { + stopped = true + clearInterval(interval) + // Wait for in-flight probe, then force one final sample. + const deadline = performance.now() + timeoutMs + 1000 + while (inFlight && performance.now() < deadline) { + await new Promise((r) => setTimeout(r, 20)) + } + await tick({ force: true }) + return { + samples: samples.values(), + ...summary(), + durationMs: performance.now() - startedAt + } + }, + getSamples: () => samples.values(), + getSummary: summary + } +} diff --git a/config/scripts/live-remote-status-watchdog.test.mjs b/config/scripts/live-remote-status-watchdog.test.mjs new file mode 100644 index 000000000..570750aad --- /dev/null +++ b/config/scripts/live-remote-status-watchdog.test.mjs @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import { startStatusWatchdog } from './live-remote-status-watchdog.mjs' + +describe('startStatusWatchdog', () => { + it('collects status samples and stops cleanly', async () => { + // Real path: actually invokes `orca status --json` (must be available in CI/dev with Orca or fail soft). + const watch = startStatusWatchdog({ intervalMs: 50, timeoutMs: 5_000 }) + await new Promise((r) => setTimeout(r, 180)) + const result = await watch.stop() + expect(result.samples.length).toBeGreaterThanOrEqual(1) + expect(result.durationMs).toBeGreaterThan(0) + for (const s of result.samples) { + expect(typeof s.ms).toBe('number') + expect(typeof s.ok).toBe('boolean') + expect(typeof s.hang).toBe('boolean') + } + }) + + it('marks CLI spawn failures as infrastructure errors', async () => { + const watch = startStatusWatchdog({ + intervalMs: 50, + timeoutMs: 1000, + cliCommand: 'orca-freeze-watchdog-missing-command' + }) + const result = await watch.stop() + + expect(result.samples.length).toBeGreaterThanOrEqual(1) + expect(result.samples.every((sample) => sample.infrastructureError === true)).toBe(true) + expect(result.samples.every((sample) => sample.hang === false)).toBe(true) + }) + + it('bounds retained samples while preserving full-run counters', async () => { + const watch = startStatusWatchdog({ + intervalMs: 50, + timeoutMs: 1000, + cliCommand: 'orca-freeze-watchdog-missing-command', + sampleHistoryLimit: 1 + }) + const result = await watch.stop() + + expect(result.samples).toHaveLength(1) + expect(result.sampleCount).toBeGreaterThan(result.samples.length) + expect(result.infrastructureErrorCount).toBe(result.sampleCount) + expect(result.maxStatusMs).toBeGreaterThanOrEqual(0) + }) +}) diff --git a/config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs b/config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs new file mode 100644 index 000000000..36fbbf915 --- /dev/null +++ b/config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs @@ -0,0 +1,43 @@ +import { spawnSync } from 'node:child_process' + +const extraArgs = process.argv.slice(2) +const pnpmEntry = process.env.npm_execpath +if (!pnpmEntry) { + throw new Error('npm_execpath is required; run this harness through pnpm') +} +const env = { + ...process.env, + ORCA_E2E_SSH_DOCKER: '1' +} + +const runtime = spawnSync(process.execPath, [pnpmEntry, 'run', 'ensure:electron-runtime'], { + stdio: 'inherit', + env +}) + +if (runtime.status !== 0) { + process.exit(runtime.status ?? 1) +} + +const result = spawnSync( + process.execPath, + [ + pnpmEntry, + 'exec', + 'playwright', + 'test', + 'tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts', + '--config', + 'tests/playwright.config.ts', + '--project', + 'electron-headless', + '--workers=1', + ...extraArgs + ], + { + stdio: 'inherit', + env + } +) + +process.exit(result.status ?? 1) diff --git a/package.json b/package.json index 16bf76e37..21a74b4f8 100644 --- a/package.json +++ b/package.json @@ -116,7 +116,11 @@ "bench:multi-workspace-typing": "pnpm run ensure:electron-runtime && node config/scripts/run-multi-workspace-typing-bench.mjs", "bench:cold-park-reveal": "pnpm run ensure:electron-runtime && node tests/tools/benchmarks/terminal-cold-park-reveal-bench.mjs", "bench:cold-park-resource": "pnpm run ensure:electron-runtime && node tests/tools/benchmarks/terminal-cold-park-resource-bench.mjs", - "bench:compare": "node config/scripts/compare-benchmark-artifacts.mjs" + "bench:compare": "node config/scripts/compare-benchmark-artifacts.mjs", + "test:e2e:remote-bulk-open-freeze": "pnpm run ensure:electron-runtime && pnpm exec playwright test tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", + "test:e2e:ssh-docker-bulk-open-freeze": "node config/scripts/run-ssh-docker-bulk-open-freeze-e2e.mjs", + "repro:live-remote-bulk-open-freeze": "node config/scripts/live-remote-bulk-open-freeze-repro.mjs", + "repro:live-remote-realistic-freeze": "node config/scripts/live-remote-realistic-freeze-repro.mjs" }, "dependencies": { "@electron-toolkit/preload": "^3.0.2", diff --git a/src/cli/terminal-format.test.ts b/src/cli/terminal-format.test.ts new file mode 100644 index 000000000..014f63398 --- /dev/null +++ b/src/cli/terminal-format.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest' +import { formatTerminalFocus } from './terminal-format' + +describe('formatTerminalFocus', () => { + it('distinguishes superseded navigation from a winning focus', () => { + expect( + formatTerminalFocus({ + focus: { + handle: 'term_stale', + tabId: 'tab-stale', + worktreeId: 'worktree-1', + navigated: false + } + }) + ).toBe( + 'Focus request for terminal term_stale was superseded or host navigation was skipped (tab tab-stale).' + ) + expect( + formatTerminalFocus({ + focus: { handle: 'term_winner', tabId: 'tab-winner', worktreeId: 'worktree-1' } + }) + ).toBe('Focused terminal term_winner (tab tab-winner).') + }) +}) diff --git a/src/cli/terminal-format.ts b/src/cli/terminal-format.ts index a6fbd6a95..ae126fa60 100644 --- a/src/cli/terminal-format.ts +++ b/src/cli/terminal-format.ts @@ -164,6 +164,9 @@ export function formatTerminalSplit(result: { split: RuntimeTerminalSplit }): st } export function formatTerminalFocus(result: { focus: RuntimeTerminalFocus }): string { + if (result.focus.navigated === false) { + return `Focus request for terminal ${result.focus.handle} was superseded or host navigation was skipped (tab ${result.focus.tabId}).` + } return `Focused terminal ${result.focus.handle} (tab ${result.focus.tabId}).` } diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index fa187429e..4eb1a6e3b 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -15436,6 +15436,257 @@ describe('OrcaRuntimeService', () => { }) }) + it('coalesces concurrent focusTerminal navigations so only the latest full reveal runs', async () => { + // Instant reveals during createTerminal; switch to gated mock before focus storm. + const revealTerminalSession = vi.fn().mockResolvedValue({ tabId: 'tab-create' }) + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + spawn: vi + .fn() + .mockResolvedValueOnce({ id: 'pty-a' }) + .mockResolvedValueOnce({ id: 'pty-b' }) + .mockResolvedValueOnce({ id: 'pty-c' }), + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.setNotifier({ + worktreesChanged: vi.fn(), + reposChanged: vi.fn(), + activateWorktree: vi.fn(), + createTerminal: vi.fn(), + revealTerminalSession, + splitTerminal: vi.fn(), + renameTerminal: vi.fn(), + focusTerminal: vi.fn(), + closeTerminal: vi.fn(), + sleepWorktree: vi.fn(), + terminalFitOverrideChanged: vi.fn(), + terminalDriverChanged: vi.fn() + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + const a = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`, { + title: 'a', + presentation: 'background' + }) + const b = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`, { + title: 'b', + presentation: 'background' + }) + const c = await runtime.createTerminal(`path:${TEST_WORKTREE_PATH}`, { + title: 'c', + presentation: 'background' + }) + + let releaseFirstReveal!: (value: { tabId: string }) => void + let firstRevealStarted = false + const firstRevealGate = new Promise<{ tabId: string }>((resolve) => { + releaseFirstReveal = resolve + }) + revealTerminalSession.mockReset() + revealTerminalSession.mockImplementation(() => { + if (!firstRevealStarted) { + firstRevealStarted = true + return firstRevealGate + } + return Promise.resolve({ tabId: 'tab-latest' }) + }) + + const pA = runtime.focusTerminal(a.handle) + await vi.waitFor(() => { + expect(firstRevealStarted).toBe(true) + }) + const pB = runtime.focusTerminal(b.handle) + const pC = runtime.focusTerminal(c.handle) + + // B is superseded while A is in flight — identity only, never navigated. + await expect(pB).resolves.toMatchObject({ + handle: b.handle, + navigated: false + }) + releaseFirstReveal({ tabId: 'tab-a' }) + // A may still complete reveal work, but if C superseded it, navigated is false. + const aResult = await pA + expect(aResult.handle).toBe(a.handle) + expect(aResult.navigated).toBe(false) + await expect(pC).resolves.toMatchObject({ + handle: c.handle, + tabId: 'tab-latest', + navigated: true + }) + + // B must never have started a reveal; only A and/or C. + const revealedPtyIds = revealTerminalSession.mock.calls.map( + (call) => (call[1] as { ptyId?: string }).ptyId + ) + expect(revealedPtyIds).not.toContain('pty-b') + expect(revealedPtyIds.at(-1)).toBe('pty-c') + }) + + it('reports a queued PTY focus as not navigated when its notifier disappears', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.registerPty('pty-a', TEST_WORKTREE_ID) + runtime.registerPty('pty-b', TEST_WORKTREE_ID) + const terminals = (await runtime.listTerminals()).terminals + const terminalA = terminals.find((terminal) => terminal.ptyId === 'pty-a') + const terminalB = terminals.find((terminal) => terminal.ptyId === 'pty-b') + expect(terminalA).toBeDefined() + expect(terminalB).toBeDefined() + + let releaseReveal!: (value: { tabId: string }) => void + const revealGate = new Promise<{ tabId: string }>((resolve) => { + releaseReveal = resolve + }) + const revealTerminalSession = vi + .fn() + .mockImplementationOnce(() => revealGate) + .mockResolvedValue({ tabId: 'tab-b' }) + runtime.setNotifier({ revealTerminalSession } as never) + + const first = runtime.focusTerminal(terminalA!.handle) + await vi.waitFor(() => expect(revealTerminalSession).toHaveBeenCalledOnce()) + const queued = runtime.focusTerminal(terminalB!.handle) + runtime.setNotifier(null) + releaseReveal({ tabId: 'tab-a' }) + + await expect(first).resolves.toMatchObject({ handle: terminalA!.handle, navigated: false }) + await expect(queued).resolves.toMatchObject({ handle: terminalB!.handle, navigated: false }) + expect(revealTerminalSession).toHaveBeenCalledOnce() + }) + + it('reports an in-flight PTY focus as not navigated when its notifier disappears', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.registerPty('pty-a', TEST_WORKTREE_ID) + const terminal = (await runtime.listTerminals()).terminals.find( + (candidate) => candidate.ptyId === 'pty-a' + ) + expect(terminal).toBeDefined() + + let releaseReveal!: (value: { tabId: string }) => void + const revealTerminalSession = vi.fn( + () => + new Promise<{ tabId: string }>((resolve) => { + releaseReveal = resolve + }) + ) + runtime.setNotifier({ revealTerminalSession } as never) + + const focus = runtime.focusTerminal(terminal!.handle) + await vi.waitFor(() => expect(revealTerminalSession).toHaveBeenCalledOnce()) + runtime.setNotifier(null) + releaseReveal({ tabId: 'tab-a' }) + + await expect(focus).resolves.toMatchObject({ + handle: terminal!.handle, + tabId: 'tab-a', + navigated: false + }) + }) + + it('does not invoke a stale graph-leaf focus notifier after queued PTY work', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.setPtyController({ + write: () => true, + kill: () => true, + getForegroundProcess: async () => null + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-leaf', + worktreeId: TEST_WORKTREE_ID, + title: 'Starting terminal', + activeLeafId: HEADLESS_LEAF_ID, + layout: null + } + ], + leaves: [ + { + tabId: 'tab-leaf', + worktreeId: TEST_WORKTREE_ID, + leafId: HEADLESS_LEAF_ID, + paneRuntimeId: 1, + ptyId: null + } + ] + }) + const leafTerminal = (await runtime.listTerminals()).terminals.find( + (terminal) => terminal.tabId === 'tab-leaf' + ) + runtime.registerPty('pty-a', TEST_WORKTREE_ID) + const ptyTerminal = (await runtime.listTerminals()).terminals.find( + (terminal) => terminal.ptyId === 'pty-a' + ) + expect(ptyTerminal).toBeDefined() + expect(leafTerminal).toBeDefined() + + let releaseReveal!: (value: { tabId: string }) => void + const revealGate = new Promise<{ tabId: string }>((resolve) => { + releaseReveal = resolve + }) + const focusTerminal = vi.fn() + const revealTerminalSession = vi.fn(() => revealGate) + runtime.setNotifier({ + revealTerminalSession, + focusTerminal + } as never) + + const first = runtime.focusTerminal(ptyTerminal!.handle) + await vi.waitFor(() => expect(revealTerminalSession).toHaveBeenCalledOnce()) + const queued = runtime.focusTerminal(leafTerminal!.handle) + runtime.setNotifier(null) + releaseReveal({ tabId: 'tab-a' }) + + await expect(first).resolves.toMatchObject({ handle: ptyTerminal!.handle, navigated: false }) + await expect(queued).resolves.toMatchObject({ handle: leafTerminal!.handle, navigated: false }) + expect(focusTerminal).not.toHaveBeenCalled() + }) + + it('reports graph-leaf focus as not navigated without a host notifier', async () => { + const runtime = new OrcaRuntimeService(store) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-leaf', + worktreeId: TEST_WORKTREE_ID, + title: 'Starting terminal', + activeLeafId: HEADLESS_LEAF_ID, + layout: null + } + ], + leaves: [ + { + tabId: 'tab-leaf', + worktreeId: TEST_WORKTREE_ID, + leafId: HEADLESS_LEAF_ID, + paneRuntimeId: 1, + ptyId: null + } + ] + }) + const [terminal] = (await runtime.listTerminals()).terminals + + await expect(runtime.focusTerminal(terminal.handle)).resolves.toEqual({ + handle: terminal.handle, + tabId: 'tab-leaf', + worktreeId: TEST_WORKTREE_ID, + navigated: false + }) + }) + it('clears terminal scrollback through the PTY controller and headless buffer', async () => { const clearBuffer = vi.fn().mockResolvedValue(undefined) const runtime = new OrcaRuntimeService(store) @@ -26293,7 +26544,8 @@ describe('OrcaRuntimeService', () => { await expect(runtime.focusTerminal(laptopTerminal.handle)).resolves.toEqual({ handle: laptopTerminal.handle, tabId: 'laptop-tab', - worktreeId: TEST_WORKTREE_ID + worktreeId: TEST_WORKTREE_ID, + navigated: false }) await expect(runtime.closeTerminal(laptopTerminal.handle)).resolves.toEqual({ handle: laptopTerminal.handle, diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index ee6c31a52..d1abd0c8f 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -949,6 +949,7 @@ import { createMobileSessionTabsNotifyCoalescer, type MobileSessionTabsNotifyCoalescer } from './mobile-session-tabs-notify-coalescer' +import { TerminalFocusNavigationCoalescer } from './terminal-focus-navigation-coalescer' import { getSshFilesystemProvider } from '../providers/ssh-filesystem-dispatch' import { assertFolderWorkspacePathUsable, @@ -2747,6 +2748,12 @@ export class OrcaRuntimeService { createMobileSessionTabsNotifyCoalescer((worktreeId) => this.notifyMobileSessionTabsChangedNow(worktreeId) ) + // Why: concurrent host terminal.focus storms (CLI switch fan-out / bulk open) + // each await a full host reveal; only one terminal can be focused, so latest-wins + // single-flight bounds host work. Does not replace cheaper activation or + // reconnect-scan bounding for sequential soft freezes. + private readonly terminalFocusNavigationCoalescer = + new TerminalFocusNavigationCoalescer() private pendingMobileSessionPtyInventoryRefresh: Promise | null> | null = null private leaves = new Map() // Why: PTY output is a per-keystroke hot path. Looking up affected leaves by @@ -25873,38 +25880,158 @@ export class OrcaRuntimeService { handle: string, options: { navigateHost?: boolean } = {} ): Promise { + const navigateHost = options.navigateHost !== false + const livePtyIdentity = (): RuntimeTerminalFocus => { + const live = this.getLivePtyForHandle(handle) + if (!live?.pty.connected) { + throw new Error('terminal_exited') + } + return { + handle, + tabId: live.pty.tabId ?? live.record.tabId, + worktreeId: live.pty.worktreeId, + navigated: false + } + } + const liveLeafIdentity = (): RuntimeTerminalFocus => { + this.assertGraphReady() + const { leaf: current } = this.getLiveLeafForHandle(handle) + return { + handle, + tabId: current.tabId, + worktreeId: current.worktreeId, + navigated: false + } + } + const pty = this.getLivePtyForHandle(handle) if (pty) { if (!pty.pty.connected) { throw new Error('terminal_exited') } - const parsedPaneKey = parsePaneKey(pty.pty.paneKey ?? '') - const revealed = - options.navigateHost === false - ? undefined - : await this.notifier?.revealTerminalSession?.(pty.pty.worktreeId, { - ptyId: pty.pty.ptyId, - title: getLatestPtyTitle(pty.pty), - ...(pty.pty.launchConfig - ? { launchConfig: copySleepingAgentLaunchConfig(pty.pty.launchConfig) } - : {}), - ...(pty.pty.launchToken ? { launchToken: pty.pty.launchToken } : {}), - ...(pty.pty.launchAgent ? { launchAgent: pty.pty.launchAgent } : {}), - ...(pty.pty.tabId !== null ? { tabId: pty.pty.tabId } : {}), - ...(parsedPaneKey ? { leafId: parsedPaneKey.leafId } : {}) - }) - return { - handle, - tabId: revealed?.tabId ?? pty.pty.tabId ?? pty.record.tabId, - worktreeId: pty.pty.worktreeId + if (!navigateHost || !this.notifier?.revealTerminalSession) { + return { + handle, + tabId: pty.pty.tabId ?? pty.record.tabId, + worktreeId: pty.pty.worktreeId, + navigated: false + } } + // Coalesce concurrent host navigations: only the latest full reveal claims navigated. + return this.terminalFocusNavigationCoalescer.run({ + key: handle, + resolveSuperseded: (completed) => + completed ? { ...completed, navigated: false } : livePtyIdentity(), + run: async (ctx) => { + const live = this.getLivePtyForHandle(handle) + if (!live?.pty.connected) { + throw new Error('terminal_exited') + } + if (!ctx.isCurrent()) { + return { + handle, + tabId: live.pty.tabId ?? live.record.tabId, + worktreeId: live.pty.worktreeId, + navigated: false + } + } + const notifier = this.notifier + if (!notifier?.revealTerminalSession) { + return { + handle, + tabId: live.pty.tabId ?? live.record.tabId, + worktreeId: live.pty.worktreeId, + navigated: false + } + } + const parsedPaneKey = parsePaneKey(live.pty.paneKey ?? '') + const revealed = await notifier.revealTerminalSession(live.pty.worktreeId, { + ptyId: live.pty.ptyId, + title: getLatestPtyTitle(live.pty), + ...(live.pty.launchConfig + ? { launchConfig: copySleepingAgentLaunchConfig(live.pty.launchConfig) } + : {}), + ...(live.pty.launchToken ? { launchToken: live.pty.launchToken } : {}), + ...(live.pty.launchAgent ? { launchAgent: live.pty.launchAgent } : {}), + ...(live.pty.tabId !== null ? { tabId: live.pty.tabId } : {}), + ...(parsedPaneKey ? { leafId: parsedPaneKey.leafId } : {}) + }) + if (!ctx.isCurrent() || this.notifier !== notifier) { + return { + handle, + tabId: revealed?.tabId ?? live.pty.tabId ?? live.record.tabId, + worktreeId: live.pty.worktreeId, + navigated: false + } + } + return { + handle, + tabId: revealed?.tabId ?? live.pty.tabId ?? live.record.tabId, + worktreeId: live.pty.worktreeId, + navigated: true + } + } + }) } this.assertGraphReady() const { leaf } = this.getLiveLeafForHandle(handle) - if (options.navigateHost !== false) { - this.notifier?.focusTerminal(leaf.tabId, leaf.worktreeId, leaf.leafId) + if (!navigateHost) { + return { + handle, + tabId: leaf.tabId, + worktreeId: leaf.worktreeId, + navigated: false + } } - return { handle, tabId: leaf.tabId, worktreeId: leaf.worktreeId } + if (!this.notifier?.focusTerminal) { + return { + handle, + tabId: leaf.tabId, + worktreeId: leaf.worktreeId, + navigated: false + } + } + return this.terminalFocusNavigationCoalescer.run({ + key: handle, + resolveSuperseded: (completed) => + completed ? { ...completed, navigated: false } : liveLeafIdentity(), + run: async (ctx) => { + this.assertGraphReady() + const { leaf: liveLeaf } = this.getLiveLeafForHandle(handle) + if (!ctx.isCurrent()) { + return { + handle, + tabId: liveLeaf.tabId, + worktreeId: liveLeaf.worktreeId, + navigated: false + } + } + const notifier = this.notifier + if (!notifier?.focusTerminal) { + return { + handle, + tabId: liveLeaf.tabId, + worktreeId: liveLeaf.worktreeId, + navigated: false + } + } + notifier.focusTerminal(liveLeaf.tabId, liveLeaf.worktreeId, liveLeaf.leafId) + if (!ctx.isCurrent() || this.notifier !== notifier) { + return { + handle, + tabId: liveLeaf.tabId, + worktreeId: liveLeaf.worktreeId, + navigated: false + } + } + return { + handle, + tabId: liveLeaf.tabId, + worktreeId: liveLeaf.worktreeId, + navigated: true + } + } + }) } async closeTerminal(handle: string): Promise { diff --git a/src/main/runtime/terminal-focus-navigation-coalescer.test.ts b/src/main/runtime/terminal-focus-navigation-coalescer.test.ts new file mode 100644 index 000000000..cf3fb1aa4 --- /dev/null +++ b/src/main/runtime/terminal-focus-navigation-coalescer.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it, vi } from 'vitest' +import { TerminalFocusNavigationCoalescer } from './terminal-focus-navigation-coalescer' + +function deferred(): { + promise: Promise + resolve: (value: T) => void + reject: (error: unknown) => void +} { + let resolve!: (value: T) => void + let reject!: (error: unknown) => void + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve, reject } +} + +describe('TerminalFocusNavigationCoalescer', () => { + it('runs a single job to completion', async () => { + const coalescer = new TerminalFocusNavigationCoalescer() + const result = await coalescer.run({ + key: 'term_a', + run: async () => 'full-a', + resolveSuperseded: () => 'superseded-a' + }) + expect(result).toBe('full-a') + expect(coalescer.getState()).toMatchObject({ + running: false, + activeKey: null, + pendingKey: null + }) + }) + + it('serializes concurrent focuses and latest-wins drops intermediate pending', async () => { + const coalescer = new TerminalFocusNavigationCoalescer() + const aGate = deferred() + let aStarted = false + + const runA = vi.fn(async (ctx: { isCurrent: () => boolean }) => { + aStarted = true + await aGate.promise + if (!ctx.isCurrent()) { + return 'obsolete-a' + } + return 'full-a' + }) + const runB = vi.fn(async () => 'full-b') + const runC = vi.fn(async () => 'full-c') + const superB = vi.fn(() => 'super-b') + + const pA = coalescer.run({ + key: 'term_a', + run: runA, + resolveSuperseded: (completed) => completed ?? 'super-a' + }) + await vi.waitFor(() => { + expect(aStarted).toBe(true) + }) + + const pB = coalescer.run({ + key: 'term_b', + run: runB, + resolveSuperseded: superB + }) + const pC = coalescer.run({ + key: 'term_c', + run: runC, + resolveSuperseded: () => 'super-c' + }) + + await expect(pB).resolves.toBe('super-b') + expect(superB).toHaveBeenCalledTimes(1) + expect(runB).not.toHaveBeenCalled() + + aGate.resolve() + await expect(pA).resolves.toBe('obsolete-a') + await expect(pC).resolves.toBe('full-c') + expect(runC).toHaveBeenCalledTimes(1) + }) + + it('bounds host navigation to one full run under a parallel storm', async () => { + const coalescer = new TerminalFocusNavigationCoalescer() + let inFlight = 0 + let maxInFlight = 0 + let fullRuns = 0 + + const makeJob = (key: string) => + coalescer.run({ + key, + run: async (ctx) => { + if (!ctx.isCurrent()) { + return -1 + } + inFlight += 1 + maxInFlight = Math.max(maxInFlight, inFlight) + fullRuns += 1 + await new Promise((r) => setTimeout(r, 5)) + inFlight -= 1 + if (!ctx.isCurrent()) { + return -1 + } + return fullRuns + }, + resolveSuperseded: () => -1 + }) + + const results = await Promise.all(Array.from({ length: 16 }, (_, i) => makeJob(`term_${i}`))) + + expect(maxInFlight).toBe(1) + expect(fullRuns).toBeLessThanOrEqual(2) + expect(fullRuns).toBeGreaterThanOrEqual(1) + expect(results.filter((r) => r === -1).length).toBeGreaterThanOrEqual(14) + expect(results.some((r) => r > 0)).toBe(true) + }) + + it('skips claiming navigation when a newer focus arrives mid-run', async () => { + const coalescer = new TerminalFocusNavigationCoalescer<{ id: string; navigated: boolean }>() + const aGate = deferred() + let aStarted = false + + const pA = coalescer.run({ + key: 'term_a', + run: async (ctx) => { + aStarted = true + await aGate.promise + return { id: 'a', navigated: ctx.isCurrent() } + }, + resolveSuperseded: () => ({ id: 'a', navigated: false }) + }) + await vi.waitFor(() => { + expect(aStarted).toBe(true) + }) + + const pB = coalescer.run({ + key: 'term_b', + run: async () => ({ id: 'b', navigated: true }), + resolveSuperseded: () => ({ id: 'b', navigated: false }) + }) + + aGate.resolve() + await expect(pA).resolves.toEqual({ id: 'a', navigated: false }) + await expect(pB).resolves.toEqual({ id: 'b', navigated: true }) + }) + + it('marks a synchronous run superseded when it queues a newer job before settling', async () => { + const coalescer = new TerminalFocusNavigationCoalescer<{ + id: string + navigated: boolean + }>() + let latest!: Promise<{ id: string; navigated: boolean }> + + const first = coalescer.run({ + key: 'term_a', + run: async () => { + latest = coalescer.run({ + key: 'term_b', + run: async () => ({ id: 'b', navigated: true }), + resolveSuperseded: () => ({ id: 'b', navigated: false }) + }) + return { id: 'a', navigated: true } + }, + resolveSuperseded: (completed) => ({ + id: completed?.id ?? 'a', + navigated: false + }) + }) + + await expect(first).resolves.toEqual({ id: 'a', navigated: false }) + await expect(latest).resolves.toEqual({ id: 'b', navigated: true }) + }) + + it('propagates run failures without stranding the queue', async () => { + const coalescer = new TerminalFocusNavigationCoalescer() + const aGate = deferred() + let aStarted = false + + const pA = coalescer.run({ + key: 'term_a', + run: async () => { + aStarted = true + await aGate.promise + throw new Error('boom') + }, + resolveSuperseded: () => 'super-a' + }) + await vi.waitFor(() => { + expect(aStarted).toBe(true) + }) + + const pB = coalescer.run({ + key: 'term_b', + run: async () => 'full-b', + resolveSuperseded: () => 'super-b' + }) + + aGate.resolve() + // A is obsolete when it fails after B enqueued — settles as superseded, not boom. + await expect(pA).resolves.toBe('super-a') + await expect(pB).resolves.toBe('full-b') + expect(coalescer.getState().running).toBe(false) + }) +}) diff --git a/src/main/runtime/terminal-focus-navigation-coalescer.ts b/src/main/runtime/terminal-focus-navigation-coalescer.ts new file mode 100644 index 000000000..bf69ef6a4 --- /dev/null +++ b/src/main/runtime/terminal-focus-navigation-coalescer.ts @@ -0,0 +1,126 @@ +/** + * Latest-wins single-flight coalescer for exclusive host terminal focus navigation. + * + * Scope: concurrent `terminal.focus` / host-nav storms (CLI switch fan-out, bulk open). + * Not a substitute for cheaper activation or reconnect-scan bounding. + * + * Why: only one host terminal can be focused. Intermediate navigations are waste; + * running them in parallel freezes large remote fleets. Pending jobs collapse to + * the newest; in-flight work re-checks a generation so obsolete runs do not claim + * a successful host navigation after a newer focus arrived. + */ + +export type TerminalFocusNavigationContext = { + /** True while this job is still the newest focus request. */ + isCurrent: () => boolean +} + +export type TerminalFocusNavigationJob = { + /** Stable key for diagnostics (usually terminal handle). */ + key: string + /** + * Full navigation work (reveal/focus host UI). + * Must consult `ctx.isCurrent()` before/after expensive host work and avoid + * claiming navigation when false. + */ + run: (ctx: TerminalFocusNavigationContext) => Promise + /** + * Result when this job is dropped (pending superseded) or becomes obsolete + * mid-flight. Must not perform host navigation. Should set navigated: false. + */ + resolveSuperseded: (completed?: TResult) => TResult +} + +type PendingJob = { + key: string + generation: number + run: (ctx: TerminalFocusNavigationContext) => Promise + resolve: (value: TResult) => void + reject: (error: unknown) => void + resolveSuperseded: (completed?: TResult) => TResult +} + +export class TerminalFocusNavigationCoalescer { + private running = false + private pending: PendingJob | null = null + private activeKey: string | null = null + private generation = 0 + + getState(): { + running: boolean + activeKey: string | null + pendingKey: string | null + generation: number + } { + return { + running: this.running, + activeKey: this.activeKey, + pendingKey: this.pending?.key ?? null, + generation: this.generation + } + } + + run(job: TerminalFocusNavigationJob): Promise { + return new Promise((resolve, reject) => { + if (this.pending) { + try { + this.pending.resolve(this.pending.resolveSuperseded()) + } catch (error) { + this.pending.reject(error) + } + } + const generation = ++this.generation + this.pending = { + key: job.key, + generation, + run: job.run, + resolve, + reject, + resolveSuperseded: job.resolveSuperseded + } + void this.pump() + }) + } + + private async pump(): Promise { + if (this.running) { + return + } + this.running = true + try { + while (this.pending) { + const job = this.pending + this.pending = null + this.activeKey = job.key + const ctx: TerminalFocusNavigationContext = { + isCurrent: () => job.generation === this.generation + } + try { + if (!ctx.isCurrent()) { + job.resolve(job.resolveSuperseded()) + continue + } + const result = await job.run(ctx) + job.resolve(ctx.isCurrent() ? result : job.resolveSuperseded(result)) + } catch (error) { + if (ctx.isCurrent()) { + job.reject(error) + } else { + try { + job.resolve(job.resolveSuperseded()) + } catch (supersedeError) { + job.reject(supersedeError) + } + } + } finally { + this.activeKey = null + } + } + } finally { + this.running = false + if (this.pending) { + void this.pump() + } + } + } +} diff --git a/src/shared/runtime-types.ts b/src/shared/runtime-types.ts index c357ff63d..cbb8fa223 100644 --- a/src/shared/runtime-types.ts +++ b/src/shared/runtime-types.ts @@ -681,6 +681,12 @@ export type RuntimeTerminalFocus = { handle: string tabId: string worktreeId: string + /** + * Whether this request remained the winning applied host navigation when it settled. + * False also covers identity-only requests and unavailable host navigation. + * Optional for older clients; omit only when unknown. + */ + navigated?: boolean } export type RuntimeTerminalClose = { diff --git a/tests/e2e/helpers/headless-paired-runtime-host.ts b/tests/e2e/helpers/headless-paired-runtime-host.ts index 83186e5a7..9159154b2 100644 --- a/tests/e2e/helpers/headless-paired-runtime-host.ts +++ b/tests/e2e/helpers/headless-paired-runtime-host.ts @@ -82,6 +82,33 @@ export function formatHeadlessPairedRuntimeStartupDiagnostics( .join('\n') } +export function parseHeadlessPairedRuntimePairingOffer( + line: string +): RuntimeDesktopPairingOffer | null { + let parsed: unknown + try { + parsed = JSON.parse(line) as unknown + } catch { + return null + } + if (parsed === null || typeof parsed !== 'object') { + return null + } + const readiness = parsed as ServeReady + const pairing = readiness.pairing + if ( + readiness.type !== 'orca_server_ready' || + pairing?.available !== true || + typeof pairing.url !== 'string' + ) { + return null + } + return { + pairingUrl: pairing.url, + ...(typeof pairing.webClientUrl === 'string' ? { webClientUrl: pairing.webClientUrl } : {}) + } +} + function redactPairingMaterial(value: string): string { return value .replace(PAIRING_URL_PATTERN, 'orca://[redacted]') @@ -138,23 +165,12 @@ async function readPairingOffer(app: ElectronApplication): Promise { @@ -43,3 +44,41 @@ describe('headless paired runtime startup diagnostics', () => { expect(diagnostic.read()).toBe('safe') }) }) + +describe('headless paired runtime readiness', () => { + it.each(['null', 'true', '0', '"ready"', '[]'])( + 'ignores JSON primitives and non-object readiness payloads: %s', + (payload) => { + expect(parseHeadlessPairedRuntimePairingOffer(payload)).toBeNull() + } + ) + + it('accepts desktop-only pairing readiness', () => { + expect( + parseHeadlessPairedRuntimePairingOffer( + JSON.stringify({ + type: 'orca_server_ready', + pairing: { available: true, url: 'orca://pairing-secret', webClientUrl: null } + }) + ) + ).toEqual({ pairingUrl: 'orca://pairing-secret' }) + }) + + it('preserves an available web-client URL', () => { + expect( + parseHeadlessPairedRuntimePairingOffer( + JSON.stringify({ + type: 'orca_server_ready', + pairing: { + available: true, + url: 'orca://pairing-secret', + webClientUrl: 'https://example.test/web-index.html#pairing=secret' + } + }) + ) + ).toEqual({ + pairingUrl: 'orca://pairing-secret', + webClientUrl: 'https://example.test/web-index.html#pairing=secret' + }) + }) +}) diff --git a/tests/e2e/helpers/remote-session-bulk-open-fixture.ts b/tests/e2e/helpers/remote-session-bulk-open-fixture.ts new file mode 100644 index 000000000..359add80b --- /dev/null +++ b/tests/e2e/helpers/remote-session-bulk-open-fixture.ts @@ -0,0 +1,66 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'\\''`)}'` +} + +function fixtureCommand(fixturePath: string, marker: string): string { + const command = [process.execPath, fixturePath, marker] + return process.platform === 'win32' + ? command.map((value) => `"${value.replaceAll('"', '""')}"`).join(' ') + : command.map(shellQuote).join(' ') +} + +/** + * Continuous remote-agent-like flood fixture for freeze repros. + * Many remote sessions stream while the client bulk-opens them. + * One-shot FLOOD is not enough — agents keep writing. + */ +export function createRemoteSessionBulkOpenFixture(): { + command: (marker: string) => string + dispose: () => void +} { + const scratch = mkdtempSync(path.join(os.tmpdir(), 'orca-remote-bulk-open-')) + const fixturePath = path.join(scratch, 'remote-bulk-open-flood.mjs') + writeFileSync( + fixturePath, + [ + 'const marker = process.argv[2]', + 'process.stdout.write(`READY:${marker}\\r\\n`)', + 'process.stdin.setRawMode?.(true)', + "process.stdin.setEncoding('utf8')", + 'let frame = 0', + 'let timer = null', + "const chunk = 'A'.repeat(2048)", + 'function startFlood() {', + ' if (timer) return', + ' timer = setInterval(() => {', + ' frame += 1', + ' process.stdout.write(`BG:${marker}:${frame}:${chunk}\\r\\n`)', + ' }, 8)', + '}', + 'function stopFlood() {', + ' if (timer) clearInterval(timer)', + ' timer = null', + '}', + // Auto-start flood after ready so hidden tabs accumulate backlog. + 'setTimeout(startFlood, 200)', + "process.stdin.on('data', (data) => {", + ' for (const command of data.split(/\\r\\n|\\r|\\n/).filter(Boolean)) {', + " if (command === 'GO' || command === 'FLOOD') { startFlood(); continue }", + " if (command === 'STOP') { stopFlood(); process.stdout.write(`STOPPED:${marker}\\r\\n`); continue }", + " if (command === 'PING') { process.stdout.write(`PONG:${marker}:${frame}\\r\\n`); continue }", + ' process.stdout.write(`LIVE:${marker}:${command}\\r\\n`)', + ' }', + '})', + 'process.stdin.resume()', + "process.on('SIGINT', () => { stopFlood(); process.exit(0) })" + ].join('\n') + ) + return { + command: (marker) => fixtureCommand(fixturePath, marker), + dispose: () => rmSync(scratch, { recursive: true, force: true }) + } +} diff --git a/tests/e2e/helpers/remote-session-bulk-open-oracle.ts b/tests/e2e/helpers/remote-session-bulk-open-oracle.ts new file mode 100644 index 000000000..0278e9ea3 --- /dev/null +++ b/tests/e2e/helpers/remote-session-bulk-open-oracle.ts @@ -0,0 +1,271 @@ +import { writeFileSync, mkdirSync } from 'node:fs' +import path from 'node:path' +import type { Page } from '@stablyai/playwright-test' +import { toWebTerminalSurfaceTabId } from '../../../src/shared/terminal-surface-id' +import { expect } from './orca-app' +import { createRemoteSessionBulkOpenFixture } from './remote-session-bulk-open-fixture' +import { startRendererLagProbe } from '../paired-runtime-retention-metrics' +import { closeStreamingTerminals } from './streaming-terminal-cleanup' +import { waitForActivePanePtyId } from './terminal' + +/** Multi-worktree load: several agent-like streaming terminals per worktree. */ +export const BULK_OPEN_WORKTREE_COUNT = 3 +export const BULK_OPEN_TABS_PER_WORKTREE = 4 +/** Soft freeze signal — UI feels stuck. */ +export const SOFT_FREEZE_LAG_MS = 2_000 +/** Hard freeze signal — matches trusted "screen fully frozen" reports. */ +export const HARD_FREEZE_LAG_MS = 5_000 + +export type BulkOpenSession = { + marker: string + tabId: string + terminal: string + worktreeId: string +} + +export type BulkOpenFreezeReport = { + bulkOpenMaxLagMs: number + hiddenFloodMaxLagMs: number + interactionProbeMs: number + hardFreeze: boolean + softFreeze: boolean + sessionCount: number + worktreeCount: number + topology: 'paired-remote-server' | 'docker-ssh' + versionHint: string + notes: string[] +} + +async function callRuntime(page: Page, method: string, params: unknown): Promise { + return page.evaluate( + async ({ method, params }) => { + const response = await window.api.runtime.call({ method, params }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { method, params } + ) as Promise +} + +async function measureRendererInteractionMs(page: Page): Promise { + return page.evaluate(async () => { + const started = performance.now() + if (!window.__store) { + throw new Error('store unavailable for interaction probe') + } + // A blocked renderer cannot service the input task or paint the following frames. + await new Promise((resolve) => { + requestAnimationFrame(() => requestAnimationFrame(() => resolve())) + }) + return performance.now() - started + }) +} + +export async function seedBulkOpenRemoteSessions( + page: Page, + seed: { repoId: string } +): Promise<{ sessions: BulkOpenSession[]; dispose: () => Promise }> { + const fixture = createRemoteSessionBulkOpenFixture() + const sessions: BulkOpenSession[] = [] + const closeSessions = async (): Promise => { + try { + await closeStreamingTerminals( + sessions.map((session) => session.terminal), + (method, terminal) => callRuntime(page, method, { terminal }) + ) + } finally { + fixture.dispose() + } + } + try { + for (let w = 0; w < BULK_OPEN_WORKTREE_COUNT; w += 1) { + const marker = `BULK_WT_${w}_T0` + const created = await callRuntime<{ + startupTerminal?: { handle?: string; tabId?: string } + worktree: { id: string } + }>(page, 'worktree.create', { + repo: seed.repoId, + name: `bulk-open-wt-${w}-${Date.now()}`, + setupDecision: 'skip', + activate: false, + noParent: true, + startupCommand: fixture.command(marker) + }) + if (!created.startupTerminal?.handle || !created.startupTerminal.tabId) { + throw new Error(`Bulk-open worktree ${w} missing startup terminal`) + } + const worktreeId = created.worktree.id + sessions.push({ + marker, + tabId: toWebTerminalSurfaceTabId(created.startupTerminal.tabId), + terminal: created.startupTerminal.handle, + worktreeId + }) + + for (let t = 1; t < BULK_OPEN_TABS_PER_WORKTREE; t += 1) { + const tabMarker = `BULK_WT_${w}_T${t}` + const result = await callRuntime<{ + tab: { parentTabId: string; terminal: string | null } + }>(page, 'session.tabs.createTerminal', { + worktree: `id:${worktreeId}`, + command: fixture.command(tabMarker), + activate: false, + select: false, + navigation: 'caller' + }) + if (!result.tab.terminal) { + throw new Error(`Bulk-open terminal ${tabMarker} was not created`) + } + sessions.push({ + marker: tabMarker, + tabId: toWebTerminalSurfaceTabId(result.tab.parentTabId), + terminal: result.tab.terminal, + worktreeId + }) + } + } + + // Ensure fixtures started and are streaming on the host. + await expect + .poll( + async () => { + const ready = await Promise.all( + sessions.map(async (session) => { + const result = await callRuntime<{ terminal: { tail: string[] } }>( + page, + 'terminal.read', + { terminal: session.terminal, limit: 200 } + ) + const text = result.terminal.tail.join('\n') + return text.includes(`BG:${session.marker}:`) + }) + ) + return ready.every(Boolean) + }, + { timeout: 60_000 } + ) + .toBe(true) + + return { + sessions, + dispose: closeSessions + } + } catch (error) { + await closeSessions().catch((cleanupError) => { + throw new AggregateError( + [error, cleanupError], + 'Bulk-open session seeding and cleanup failed' + ) + }) + throw error + } +} + +/** + * Repro R1 core: leave remotes streaming hidden, then burst-open sessions + * (reopening remote sessions after agents have been writing in the background). + */ +export async function runBulkOpenFreezeOracle( + page: Page, + sessions: BulkOpenSession[], + opts: { + topology: BulkOpenFreezeReport['topology'] + versionHint?: string + reportDir?: string + } +): Promise { + const notes: string[] = [] + const worktreeIds = [...new Set(sessions.map((s) => s.worktreeId))] + + // Leave terminal view so panes can park / go inactive while flooding. + await page.evaluate(() => window.__store?.getState().setActiveView('tasks')) + await page.evaluate( + () => + new Promise((resolve) => { + requestAnimationFrame(() => requestAnimationFrame(() => resolve())) + }) + ) + // Accumulate remote flood for several seconds (agent backlog). + await page.waitForTimeout(4_000) + + const hiddenProbe = await startRendererLagProbe(page) + await page.waitForTimeout(2_000) + const hiddenFloodMaxLagMs = await hiddenProbe.evaluate((probe) => probe.stop()) + await hiddenProbe.dispose() + notes.push(`hidden streaming lag max=${hiddenFloodMaxLagMs.toFixed(0)}ms`) + + // Burst open remote sessions (worktree + tab activate). + const openProbe = await startRendererLagProbe(page) + const openStarted = Date.now() + for (const worktreeId of worktreeIds) { + const tabs = sessions.filter((session) => session.worktreeId === worktreeId) + for (const tab of tabs) { + await page.evaluate( + ({ targetWorktreeId, tabId }) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(targetWorktreeId) + state?.setActiveTabForWorktree(targetWorktreeId, tabId) + }, + { targetWorktreeId: worktreeId, tabId: tab.tabId } + ) + } + } + // One more full pass clicking visible tabs if present. + for (const session of sessions) { + const locator = page.locator(`[data-testid="sortable-tab"][data-tab-id="${session.tabId}"]`) + if (await locator.isVisible().catch(() => false)) { + await locator.click({ timeout: 2_000 }).catch(() => undefined) + } + } + // Let the storm settle enough to measure residual lag. + await page.waitForTimeout(3_000) + const bulkOpenMaxLagMs = await openProbe.evaluate((probe) => probe.stop()) + await openProbe.dispose() + notes.push(`bulk open wall=${Date.now() - openStarted}ms lagMax=${bulkOpenMaxLagMs.toFixed(0)}ms`) + + // Confirm last session is live after the storm (host PTYs survived). + const last = sessions.at(-1) + if (!last) { + throw new Error('bulk-open freeze oracle requires at least one session') + } + await page.evaluate( + ({ targetWorktreeId, tabId }) => { + const state = window.__store?.getState() + state?.setActiveView('terminal') + state?.setActiveWorktree(targetWorktreeId) + state?.setActiveTabForWorktree(targetWorktreeId, tabId) + }, + { targetWorktreeId: last.worktreeId, tabId: last.tabId } + ) + await waitForActivePanePtyId(page, 30_000).catch(() => { + notes.push('active pane PTY id not ready after bulk open (possible re-attach failure)') + }) + + const interactionProbeMs = await measureRendererInteractionMs(page) + notes.push(`post-storm renderer interaction=${interactionProbeMs.toFixed(0)}ms`) + + const report: BulkOpenFreezeReport = { + bulkOpenMaxLagMs, + hiddenFloodMaxLagMs, + interactionProbeMs, + hardFreeze: bulkOpenMaxLagMs >= HARD_FREEZE_LAG_MS || interactionProbeMs >= HARD_FREEZE_LAG_MS, + softFreeze: bulkOpenMaxLagMs >= SOFT_FREEZE_LAG_MS || interactionProbeMs >= SOFT_FREEZE_LAG_MS, + sessionCount: sessions.length, + worktreeCount: worktreeIds.length, + topology: opts.topology, + versionHint: opts.versionHint ?? process.env.ORCA_VERSION ?? 'unknown', + notes + } + + if (opts.reportDir) { + mkdirSync(opts.reportDir, { recursive: true }) + const outPath = path.join(opts.reportDir, `bulk-open-freeze-${opts.topology}.json`) + writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`) + notes.push(`wrote ${outPath}`) + } + + return report +} diff --git a/tests/e2e/helpers/streaming-terminal-cleanup.ts b/tests/e2e/helpers/streaming-terminal-cleanup.ts new file mode 100644 index 000000000..ceea47568 --- /dev/null +++ b/tests/e2e/helpers/streaming-terminal-cleanup.ts @@ -0,0 +1,27 @@ +export async function closeStreamingTerminals( + terminals: string[], + call: (method: 'terminal.closeTab' | 'terminal.close', terminal: string) => Promise +): Promise { + const results = await Promise.allSettled( + terminals.map(async (terminal) => { + try { + await call('terminal.closeTab', terminal) + } catch (closeTabError) { + try { + await call('terminal.close', terminal) + } catch (closeError) { + throw new AggregateError( + [closeTabError, closeError], + `Failed to close streaming terminal ${terminal}` + ) + } + } + }) + ) + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [] + ) + if (failures.length > 0) { + throw new AggregateError(failures, `Failed to close ${failures.length} streaming terminal(s)`) + } +} diff --git a/tests/e2e/helpers/streaming-terminal-cleanup.unit.test.ts b/tests/e2e/helpers/streaming-terminal-cleanup.unit.test.ts new file mode 100644 index 000000000..c38ed6185 --- /dev/null +++ b/tests/e2e/helpers/streaming-terminal-cleanup.unit.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it, vi } from 'vitest' +import { closeStreamingTerminals } from './streaming-terminal-cleanup' + +describe('closeStreamingTerminals', () => { + it('force-closes a streaming PTY when tab cleanup fails', async () => { + const call = vi.fn(async (method: string, terminal: string) => { + if (method === 'terminal.closeTab' && terminal === 'term_a') { + throw new Error('renderer unavailable') + } + }) + + await expect(closeStreamingTerminals(['term_a', 'term_b'], call)).resolves.toBeUndefined() + + expect(call).toHaveBeenCalledWith('terminal.closeTab', 'term_a') + expect(call).toHaveBeenCalledWith('terminal.close', 'term_a') + expect(call).toHaveBeenCalledWith('terminal.closeTab', 'term_b') + }) + + it('waits for every fallback and reports terminals that could not be stopped', async () => { + const call = vi.fn(async () => { + throw new Error('runtime frozen') + }) + + await expect(closeStreamingTerminals(['term_a', 'term_b'], call)).rejects.toThrow( + 'Failed to close 2 streaming terminal(s)' + ) + expect(call).toHaveBeenCalledTimes(4) + }) +}) diff --git a/tests/e2e/helpers/terminal-host-focus-storm-oracle.ts b/tests/e2e/helpers/terminal-host-focus-storm-oracle.ts new file mode 100644 index 000000000..846c252ab --- /dev/null +++ b/tests/e2e/helpers/terminal-host-focus-storm-oracle.ts @@ -0,0 +1,142 @@ +import type { Page } from '@stablyai/playwright-test' +import type { RuntimeTerminalFocus } from '../../../src/shared/runtime-types' +import { expect } from './orca-app' +import { createRemoteSessionBulkOpenFixture } from './remote-session-bulk-open-fixture' +import { closeStreamingTerminals } from './streaming-terminal-cleanup' + +export type HostFocusStormSession = { + marker: string + terminal: string + worktreeId: string +} + +async function callRuntime( + page: Page, + method: string, + params: unknown, + environmentId?: string +): Promise { + return page.evaluate( + async ({ environmentId, method, params }) => { + const response = environmentId + ? await window.api.runtimeEnvironments.call({ selector: environmentId, method, params }) + : await window.api.runtime.call({ method, params }) + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return response.result + }, + { environmentId: environmentId ?? null, method, params } + ) as Promise +} + +export async function seedHostFocusStormSessions( + page: Page, + worktreeId: string, + count = 6, + environmentId?: string +): Promise<{ sessions: HostFocusStormSession[]; dispose: () => Promise }> { + const fixture = createRemoteSessionBulkOpenFixture() + const sessions: HostFocusStormSession[] = [] + const closeSessions = async (): Promise => { + try { + await closeStreamingTerminals( + sessions.map((session) => session.terminal), + (method, terminal) => callRuntime(page, method, { terminal }, environmentId) + ) + } finally { + fixture.dispose() + } + } + try { + for (let index = 0; index < count; index += 1) { + const marker = `HOST_FOCUS_${index}` + const params = { + worktree: `id:${worktreeId}`, + command: fixture.command(marker), + activate: false, + select: false, + navigation: 'caller' + } + const result = await callRuntime<{ tab: { terminal: string | null } }>( + page, + 'session.tabs.createTerminal', + params, + environmentId + ) + if (!result.tab.terminal) { + throw new Error(`Host focus terminal ${marker} was not created`) + } + sessions.push({ marker, terminal: result.tab.terminal, worktreeId }) + } + await expect + .poll( + async () => { + const ready = await Promise.all( + sessions.map(async (session) => { + const params = { terminal: session.terminal, limit: 200 } + const result = await callRuntime<{ terminal: { tail: string[] } }>( + page, + 'terminal.read', + params, + environmentId + ) + return result.terminal.tail.join('\n').includes(`BG:${session.marker}:`) + }) + ) + return ready.every(Boolean) + }, + { timeout: 60_000 } + ) + .toBe(true) + return { sessions, dispose: closeSessions } + } catch (error) { + await closeSessions().catch((cleanupError) => { + throw new AggregateError( + [error, cleanupError], + 'Focus-storm session seeding and cleanup failed' + ) + }) + throw error + } +} + +export async function runHostFocusStorm( + page: Page, + sessions: HostFocusStormSession[], + environmentId?: string +): Promise { + if (sessions.length < 2) { + throw new Error('host focus storm requires at least two terminals') + } + return page.evaluate( + async ({ environmentId, targets }) => { + const callFocus = (terminal: string) => + environmentId + ? window.api.runtimeEnvironments.call({ + selector: environmentId, + method: 'terminal.focus', + params: { terminal, navigation: 'host' } + }) + : window.api.runtime.call({ + method: 'terminal.focus', + params: { terminal, navigation: 'host' } + }) + const prior = targets.slice(0, -1).map((target) => callFocus(target.terminal)) + await new Promise((resolve) => setTimeout(resolve, 0)) + const latestTarget = targets.at(-1) + if (!latestTarget) { + throw new Error('host focus storm lost its latest target') + } + const latest = callFocus(latestTarget.terminal) + const responses = await Promise.all([...prior, latest]) + return responses.map((response) => { + if (!response.ok) { + throw new Error(`${response.error.code}: ${response.error.message}`) + } + return (response.result as { focus: RuntimeTerminalFocus }).focus + }) + }, + { environmentId: environmentId ?? null, targets: sessions } + ) +} diff --git a/tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts b/tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts new file mode 100644 index 000000000..9ee26e9ac --- /dev/null +++ b/tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts @@ -0,0 +1,192 @@ +/** + * Freeze repro R1 — bulk-open remote sessions under multi-worktree flood load. + * + * Trigger: reopening many remote sessions on Remote Server / SSH with agents. + * + * Topology under test: + * R1: headless Remote Orca host + paired desktop web client (paired-remote-server) + * + * Measurement: + * renderer timer drift during hidden flood + bulk worktree/tab open. + * soft freeze >= 2s, hard freeze >= 5s. + * + * Run: + * SKIP_BUILD=1 pnpm exec playwright test \ + * tests/e2e/remote-session-bulk-open-freeze-repro.spec.ts \ + * --config tests/playwright.config.ts \ + * --project electron-headless --workers=1 + * + * Or: + * pnpm run test:e2e:remote-bulk-open-freeze + */ +import path from 'node:path' +import { expect, test } from './helpers/orca-app' +import { launchHeadlessPairedRuntimeHost } from './helpers/headless-paired-runtime-host' +import { + createRuntimeDesktopPairingOffer, + launchPairedElectronClient, + launchPairedWebClient, + type PairedElectronClient, + type PairedWebClient +} from './helpers/paired-electron-client' +import { + HARD_FREEZE_LAG_MS, + runBulkOpenFreezeOracle, + seedBulkOpenRemoteSessions, + SOFT_FREEZE_LAG_MS +} from './helpers/remote-session-bulk-open-oracle' +import { + runHostFocusStorm, + seedHostFocusStormSessions +} from './helpers/terminal-host-focus-storm-oracle' + +const REPORT_DIR = path.join(process.cwd(), 'test-results', 'freeze-repro') +const USE_DESKTOP_PAIR = process.env.ORCA_E2E_FREEZE_DESKTOP_PAIR === '1' + +test('paired client host-focus storm keeps the latest terminal @freeze-repro', async ({ + orcaPage +}, testInfo) => { + test.setTimeout(180_000) + const offer = await createRuntimeDesktopPairingOffer(orcaPage) + const client = await launchPairedElectronClient(offer, testInfo, 'focus-storm') + let disposeSessions: (() => Promise) | null = null + try { + const worktreeId = await orcaPage.evaluate(() => { + const id = window.__store?.getState().activeWorktreeId + if (!id) { + throw new Error('headed host has no active worktree') + } + return id + }) + const environmentId = await client.page.evaluate(async () => { + const environment = (await window.api.runtimeEnvironments.list())[0] + if (!environment) { + throw new Error('paired client has no runtime environment') + } + return environment.id + }) + await expect + .poll( + () => + client.page.evaluate( + (id) => + window.__store + ?.getState() + .allWorktrees() + .some((worktree) => worktree.id === id) ?? false, + worktreeId + ), + { timeout: 30_000 } + ) + .toBe(true) + const seeded = await seedHostFocusStormSessions(client.page, worktreeId, 6, environmentId) + disposeSessions = seeded.dispose + const results = await runHostFocusStorm(client.page, seeded.sessions, environmentId) + const latest = results.at(-1) + const expected = seeded.sessions.at(-1) + expect(latest).toMatchObject({ + handle: expected?.terminal, + worktreeId, + navigated: true + }) + expect(results.slice(0, -1).some((result) => result.navigated === false)).toBe(true) + await expect + .poll( + () => + orcaPage.evaluate((id) => { + const state = window.__store?.getState() + return { + worktreeId: state?.activeWorktreeId ?? null, + tabId: state?.activeTabIdByWorktree[id] ?? state?.activeTabId ?? null + } + }, worktreeId), + { timeout: 30_000 } + ) + .toEqual({ worktreeId, tabId: latest?.tabId }) + } finally { + await disposeSessions?.() + await client.dispose() + } +}) + +test('R1 paired remote bulk-open freeze oracle @freeze-repro', async ({ + testRepoPath +}, testInfo) => { + test.setTimeout(420_000) + const host = await launchHeadlessPairedRuntimeHost() + let webClient: PairedWebClient | null = null + let desktopClient: PairedElectronClient | null = null + let disposeSessions: (() => Promise) | null = null + try { + const added = await host.client.call<{ repo: { id: string } }>('repo.add', { + path: testRepoPath, + kind: 'git' + }) + await expect + .poll( + async () => { + const listed = await host.client.call<{ totalCount: number }>('worktree.list', { + repo: `id:${added.result.repo.id}` + }) + return listed.result.totalCount + }, + { timeout: 30_000 } + ) + .toBeGreaterThan(0) + + // Prefer full desktop pair when web-client store hydration is flaky in this env. + const page = await (async () => { + if (USE_DESKTOP_PAIR) { + desktopClient = await launchPairedElectronClient(host.offer, testInfo, 'freeze-r1') + return desktopClient.page + } + webClient = await launchPairedWebClient(host.app, host.offer, { + terminalParkingDelayMs: 500 + }) + return webClient.page + })() + + await page.waitForFunction(() => Boolean(window.__store), null, { timeout: 60_000 }) + await expect + .poll(() => page.evaluate(() => window.__store?.getState().allWorktrees().length ?? 0), { + timeout: 90_000 + }) + .toBeGreaterThan(0) + + const seeded = await seedBulkOpenRemoteSessions(page, { + repoId: added.result.repo.id + }) + disposeSessions = seeded.dispose + + const report = await runBulkOpenFreezeOracle(page, seeded.sessions, { + topology: 'paired-remote-server', + versionHint: process.env.npm_package_version ?? '1.4.163-rc.3', + reportDir: REPORT_DIR + }) + + console.log('[freeze-repro R1]', JSON.stringify(report, null, 2)) + + if (report.hardFreeze) { + throw new Error( + `HARD FREEZE signal: bulkOpenMaxLagMs=${report.bulkOpenMaxLagMs.toFixed(0)} ` + + `interactionProbeMs=${report.interactionProbeMs.toFixed(0)} ` + + `(threshold ${HARD_FREEZE_LAG_MS}ms). notes=${report.notes.join('; ')}` + ) + } + if (report.softFreeze) { + throw new Error( + `SOFT FREEZE signal: bulkOpenMaxLagMs=${report.bulkOpenMaxLagMs.toFixed(0)} ` + + `interactionProbeMs=${report.interactionProbeMs.toFixed(0)} ` + + `(threshold ${SOFT_FREEZE_LAG_MS}ms). notes=${report.notes.join('; ')}` + ) + } + + expect(report.sessionCount).toBeGreaterThanOrEqual(8) + expect(report.worktreeCount).toBe(3) + } finally { + await disposeSessions?.() + await webClient?.dispose() + await desktopClient?.dispose() + await host.dispose() + } +}) diff --git a/tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts b/tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts new file mode 100644 index 000000000..ee9f8c384 --- /dev/null +++ b/tests/e2e/ssh-docker-bulk-open-freeze-repro.spec.ts @@ -0,0 +1,171 @@ +/** + * Freeze repro R2 — direct SSH topology via Docker SSH relay. + * + * Requires: ORCA_E2E_SSH_DOCKER=1 and Docker available. + * + * Run: + * ORCA_E2E_SSH_DOCKER=1 pnpm run test:e2e:ssh-docker-bulk-open-freeze + */ +import path from 'node:path' +import { expect, test } from './helpers/orca-app' +import { + cleanupDockerSshRelayTarget, + DOCKER_SSH_RELAY_REMOTE_REPO_PATH, + execDockerSshRelayTargetCommand, + startDockerSshRelayTarget, + type DockerSshRelayTarget +} from './helpers/docker-ssh-relay-target' +import { connectDockerSshRelayTarget } from './helpers/docker-ssh-relay-connection' +import { waitForActiveWorktree, waitForSessionReady } from './helpers/store' +import { + execInTerminal, + focusLastTerminalPane, + splitActiveTerminalPane, + waitForActivePanePtyId, + waitForActiveTerminalManager, + waitForTerminalOutput +} from './helpers/terminal' +import { startRendererLagProbe } from './paired-runtime-retention-metrics' +import { HARD_FREEZE_LAG_MS, SOFT_FREEZE_LAG_MS } from './helpers/remote-session-bulk-open-oracle' + +const RUN_DOCKER_SSH = process.env.ORCA_E2E_SSH_DOCKER === '1' +const REPORT_DIR = path.join(process.cwd(), 'test-results', 'freeze-repro') +const SESSION_SPLITS = 5 + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'` +} + +function continuousFloodCommand(runId: string, index: number): string { + // Node one-liner: continuous 2KB frames @ ~8ms like agent output. + const script = [ + `const id='SSH_BULK_${runId}_${index}'`, + "process.stdout.write('READY:'+id+'\\n')", + 'let f=0', + "const c='S'.repeat(2048)", + "setInterval(()=>{f++;process.stdout.write('BG:'+id+':'+f+':'+c+'\\n')},8)", + 'process.stdin.resume()' + ].join(';') + return `node -e ${shellQuote(script)}` +} + +test.describe('R2 Docker SSH bulk-open freeze', () => { + test.skip(!RUN_DOCKER_SSH, 'Set ORCA_E2E_SSH_DOCKER=1 to run Docker SSH freeze repro') + + test('bulk-open many flooding SSH terminals and measure renderer lag @freeze-repro', async ({ + orcaPage, + registerPostElectronShutdownCleanup + }) => { + test.setTimeout(420_000) + let target: DockerSshRelayTarget | null = null + try { + target = startDockerSshRelayTarget() + registerPostElectronShutdownCleanup(async () => { + if (target) { + cleanupDockerSshRelayTarget(target) + } + }) + + await connectDockerSshRelayTarget(orcaPage, target, { + remotePath: DOCKER_SSH_RELAY_REMOTE_REPO_PATH + }) + await waitForSessionReady(orcaPage) + await waitForActiveWorktree(orcaPage) + + const runId = `${Date.now()}` + // First terminal on the SSH worktree. + await waitForActiveTerminalManager(orcaPage) + await execInTerminal(orcaPage, continuousFloodCommand(runId, 0)) + await waitForTerminalOutput(orcaPage, `READY:SSH_BULK_${runId}_0`, 60_000) + + for (let i = 1; i < SESSION_SPLITS; i += 1) { + await splitActiveTerminalPane(orcaPage) + await focusLastTerminalPane(orcaPage) + await waitForActivePanePtyId(orcaPage, 30_000) + await execInTerminal(orcaPage, continuousFloodCommand(runId, i)) + await waitForTerminalOutput(orcaPage, `READY:SSH_BULK_${runId}_${i}`, 60_000) + } + + // Leave the workspace view so panes go inactive while flooding. + await orcaPage.evaluate(() => window.__store?.getState().setActiveView('tasks')) + await orcaPage.waitForTimeout(4_000) + + const hiddenProbe = await startRendererLagProbe(orcaPage) + await orcaPage.waitForTimeout(2_000) + const hiddenFloodMaxLagMs = await hiddenProbe.evaluate((probe) => probe.stop()) + await hiddenProbe.dispose() + + // Burst open: return to terminal and cycle panes rapidly. + const openProbe = await startRendererLagProbe(orcaPage) + await orcaPage.evaluate(() => window.__store?.getState().setActiveView('terminal')) + for (let pass = 0; pass < 3; pass += 1) { + for (let i = 0; i < SESSION_SPLITS; i += 1) { + await orcaPage.keyboard.press(process.platform === 'darwin' ? 'Meta+]' : 'Control+]') + await orcaPage.waitForTimeout(50) + } + } + await orcaPage.waitForTimeout(3_000) + const bulkOpenMaxLagMs = await openProbe.evaluate((probe) => probe.stop()) + await openProbe.dispose() + + const interactionProbeMs = await orcaPage.evaluate(async () => { + const started = performance.now() + const state = window.__store?.getState() + const view = state?.activeView + state?.setActiveView(view === 'tasks' ? 'terminal' : 'tasks') + await new Promise((r) => + requestAnimationFrame(() => requestAnimationFrame(() => r())) + ) + state?.setActiveView(view ?? 'terminal') + await new Promise((r) => + requestAnimationFrame(() => requestAnimationFrame(() => r())) + ) + return performance.now() - started + }) + + const report = { + topology: 'docker-ssh' as const, + sessionCount: SESSION_SPLITS, + hiddenFloodMaxLagMs, + bulkOpenMaxLagMs, + interactionProbeMs, + softFreeze: + bulkOpenMaxLagMs >= SOFT_FREEZE_LAG_MS || interactionProbeMs >= SOFT_FREEZE_LAG_MS, + hardFreeze: + bulkOpenMaxLagMs >= HARD_FREEZE_LAG_MS || interactionProbeMs >= HARD_FREEZE_LAG_MS, + container: target.containerName, + remoteHostStillStreaming: true + } + + const { mkdirSync, writeFileSync } = await import('node:fs') + mkdirSync(REPORT_DIR, { recursive: true }) + writeFileSync( + path.join(REPORT_DIR, 'bulk-open-freeze-docker-ssh.json'), + `${JSON.stringify(report, null, 2)}\n` + ) + console.log('[freeze-repro R2]', JSON.stringify(report, null, 2)) + + // Host still producing frames (host alive, client stuck). + const hostFrames = execDockerSshRelayTargetCommand( + target, + `ps aux | grep -c '[n]ode -e' || true` + ) + expect(Number(hostFrames) || 0).toBeGreaterThan(0) + + if (report.hardFreeze) { + throw new Error( + `HARD FREEZE on Docker SSH: lag=${bulkOpenMaxLagMs.toFixed(0)}ms interaction=${interactionProbeMs.toFixed(0)}ms` + ) + } + if (report.softFreeze) { + throw new Error( + `SOFT FREEZE on Docker SSH: lag=${bulkOpenMaxLagMs.toFixed(0)}ms interaction=${interactionProbeMs.toFixed(0)}ms` + ) + } + } finally { + if (target) { + cleanupDockerSshRelayTarget(target) + } + } + }) +}) diff --git a/tests/tools/freeze-repro/README.md b/tests/tools/freeze-repro/README.md new file mode 100644 index 000000000..8fc611460 --- /dev/null +++ b/tests/tools/freeze-repro/README.md @@ -0,0 +1,203 @@ +# Live remote freeze repros + +Two harnesses for bulk-open / reconnect freeze repros on large paired remotes: + +| Harness | Realism | Purpose | +| ------------------------------------- | --------------------------------------------------------------------- | ------------------------------------ | +| **Realistic** (preferred for stories) | Idle + flood backlog → wake/reconnect-like refresh → human-paced open | Models overnight/return/restart | +| **Bulk parallel** (stress amp) | Concurrent `terminal switch` | Forces hard freeze for load ceilings | + +## Prerequisites + +1. **Desktop Orca running** (`orca status --json`). +2. A **large paired remote** (many worktrees / agent terminals). Lab fleets often have ~60 worktrees and 100+ terminals. +3. Repo checkout with these scripts. + +```bash +orca environment list --json +orca worktree list --environment --json | head +orca terminal list --environment --json | head +``` + +--- + +## A. Realistic repro (preferred) + +Story: remotes keep streaming while the user is away; user returns (optionally after wake/reconnect-like refresh) and opens sessions one-by-one. + +```bash +# Idle + human-paced open +ORCA_FREEZE_ENV=paired-remote \ +ORCA_FREEZE_SCENARIO=idle-backlog-open \ +ORCA_FREEZE_CREATE=8 \ +ORCA_FREEZE_IDLE_MS=45000 \ +ORCA_FREEZE_OPEN_COUNT=24 \ +pnpm run repro:live-remote-realistic-freeze + +# Wake-like: idle + reconnect metadata storm + open ← hard freeze in lab +ORCA_FREEZE_ENV=paired-remote \ +ORCA_FREEZE_SCENARIO=idle-backlog-reconnect-open \ +ORCA_FREEZE_CREATE=10 \ +ORCA_FREEZE_IDLE_MS=60000 \ +ORCA_FREEZE_OPEN_COUNT=40 \ +pnpm run repro:live-remote-realistic-freeze + +# Restart-proxy: idle + orca open + refresh storm + open (does not kill desktop) +ORCA_FREEZE_ENV=paired-remote \ +ORCA_FREEZE_SCENARIO=restart-proxy \ +ORCA_FREEZE_CREATE=0 \ +ORCA_FREEZE_IDLE_MS=20000 \ +ORCA_FREEZE_OPEN_COUNT=30 \ +pnpm run repro:live-remote-realistic-freeze +``` + +Or: `node config/scripts/live-remote-realistic-freeze-repro.mjs` + +### Scenarios + +| `ORCA_FREEZE_SCENARIO` | Models | +| ----------------------------- | --------------------------------------------------------------------------------------------- | +| `idle-backlog-open` | User away while agents stream; returns and opens sessions | +| `idle-backlog-reconnect-open` | Same + parallel status/worktree/terminal refresh (wake/reconnect client storm) | +| `restart-proxy` | `orca open` + refresh storm + open (post-restart discovery; no process kill) | +| `lockup-storm` | Idle + flood + reconnect + **concurrent** open fan-out + **mid-storm `orca status` watchdog** | + +### Realistic knobs + +| Variable | Default | Meaning | +| --------------------------------- | ------------------- | ----------------------------------------------------------------- | +| `ORCA_FREEZE_ENV` | `paired-remote` | Paired remote environment name | +| `ORCA_FREEZE_SCENARIO` | `idle-backlog-open` | See table above | +| `ORCA_FREEZE_CREATE` | `0` | New flood terminals; mutation requires an explicit positive value | +| `ORCA_FREEZE_IDLE_MS` | `45000` | Time “away” while floods run | +| `ORCA_FREEZE_OPEN_COUNT` | `20` | Sessions to open after return | +| `ORCA_FREEZE_PACE_MS` | `250` | Base delay between opens (human pace) | +| `ORCA_FREEZE_PACE_JITTER_MS` | `150` | Random extra delay | +| `ORCA_FREEZE_SOFT_MS` / `HARD_MS` | 2000 / 5000 | Thresholds | + +### Lab results (2026-07-31, client 1.4.163 / remote 1.4.163-rc.0) + +| Scenario | create | idle | open | peak | Signal | +| -------------------------------------------------- | ------ | ------ | -------------- | ---------------------------------------------------- | ------------------------------------------------------------------------- | +| idle-backlog-open | 6 | 45s | 24 | **1.7s** max open | none (< soft) | +| **idle-backlog-reconnect-open** | 10 | 60s | 40 | **11.0s** max open; reconnect refresh **3.6s** | **HARD (recovered)** | +| **restart-proxy** | 0 | 20s | 30 | **11.2s** max open | **HARD (recovered)** | +| **lockup-storm** (parallel open + overlap refresh) | 12–16 | 45–60s | 64–80 @ p20–32 | **27–35s** batches; some `Terminal reveal timed out` | **HARD stalls + reveal timeouts; app still answers `orca status` ~150ms** | + +### Full-app forever freeze? + +**Not observed** under CLI-driven escalation (including mid-storm status watchdog). + +Latest lockup-storm with watchdog (2026-07-31): + +| Field | Value | +| --------------------------- | ----------------------------------- | +| `foreverUiLockupObserved` | **false** | +| Mid-storm status samples | **95**, max **~631ms**, **0 hangs** | +| Peak open/batch | **~34s** (recovered hard stall) | +| `Terminal reveal timed out` | yes (under fan-out) | +| Post-storm `orca status` | **~113ms** | +| Force Quit required | **no** | + +Bar for full-app freeze in the harness: continuous **≥30s** window where `orca status` hangs/fails or stays ≥15s slow (`evaluateFullAppFreeze` / `foreverUiLockupObserved`). +CLI spawn failures are reported as harness infrastructure errors, not product freezes. + +What we **do** reproduce: severe multi-second / multi-tens-of-seconds stalls + flaky reveal. +What we **do not**: UI dead forever until Force Quit. That likely needs **real OS sleep/wake**, **renderer React #185**, or a path status RPC does not share with the frozen surface. + +| Exit | Meaning | +| ---- | ------------------------------------------------------------------------------- | +| 0 | no freeze | +| 1 | soft (≥2s recovered) | +| 2 | hard stall ≥5s **but recovered** | +| 4 | permanentLockup heuristic (timeouts/fail-rate; check `foreverUiLockupObserved`) | +| 5 | **full-app forever freeze** (status unhealthy ≥ forever window) | +| 3 | harness error | + +```bash +# Full-app freeze attempt (watchdog on) +ORCA_FREEZE_ENV=paired-remote ORCA_FREEZE_SCENARIO=lockup-storm \ + ORCA_FREEZE_CREATE=12 ORCA_FREEZE_IDLE_MS=30000 ORCA_FREEZE_OPEN_COUNT=80 \ + ORCA_FREEZE_STORM_PARALLEL=28 ORCA_FREEZE_FOREVER_WINDOW_MS=30000 \ + pnpm run repro:live-remote-realistic-freeze +# Expect exit 2 (recovered hard) unless foreverUiLockupObserved becomes true +``` + +**Interpretation:** Pure sequential open after idle stays under 2s. **Wake/reconnect-style refresh + open** (or concurrent fan-out) produces **recovered hard stalls**. True permanent lockup remains unproven with CLI-only levers. + +Reports: `test-results/freeze-repro/live-realistic-freeze--.json` + +--- + +## B. Stress amp (bulk parallel) + +Artificial concurrency lever; still useful for ceilings / CI stress. + +```bash +ORCA_FREEZE_ENV=paired-remote \ +ORCA_FREEZE_CREATE=0 \ +ORCA_FREEZE_SWITCH_PASSES=3 \ +ORCA_FREEZE_PARALLEL=16 \ +pnpm run repro:live-remote-bulk-open-freeze +``` + +Lab: sequential soft ~3.3–3.9s; **parallel=16 → ~20s HARD**. + +--- + +## Exit codes (both harnesses) + +| Code | Meaning | +| ---- | --------------------------------- | +| 0 | No freeze signal under thresholds | +| 1 | Soft freeze (peak ≥ 2s) | +| 2 | **Hard freeze (peak ≥ 5s)** | +| 3 | Harness failure | + +--- + +## Product fix (concurrent host-focus storms) + +Generation-aware **latest-wins single-flight** for exclusive host focus: + +| Layer | Module | +| -------- | ---------------------------------------------------------------------------------- | +| Runtime | `TerminalFocusNavigationCoalescer` via `OrcaRuntimeService.focusTerminal` | +| Contract | `RuntimeTerminalFocus.navigated?: boolean` — `false` when superseded / nav skipped | + +**In scope:** concurrent `terminal.focus` / bulk-switch storms. +**Residual:** sequential soft freezes; reconnect/wake metadata storms — need cheaper activation + scan bounding, not only focus coalescing. + +## Files + +| Path | Role | +| -------------------------------------------------------------- | ---------------------------------- | +| `config/scripts/live-remote-realistic-freeze-repro.mjs` | Naturalistic harness | +| `config/scripts/live-remote-bulk-open-freeze-repro.mjs` | Parallel stress harness | +| `config/scripts/live-remote-freeze-rpc.mjs` | Cross-platform bounded CLI runner | +| `config/scripts/live-remote-bulk-open-freeze-metrics.mjs` | Shared thresholds / handle extract | +| `config/scripts/live-remote-bulk-open-freeze-metrics.test.mjs` | Unit tests | +| `src/main/runtime/terminal-focus-navigation-coalescer.ts` | Host focus single-flight | +| `pnpm run repro:live-remote-realistic-freeze` | package entry | +| `pnpm run repro:live-remote-bulk-open-freeze` | package entry | + +--- + +## Safety + +- Both harnesses default to `ORCA_FREEZE_CREATE=0`. A positive value creates persistent, high-output remote terminals; use it only on an isolated target you can clean up. +- `restart-proxy` does **not** kill Orca; it runs `orca open` + refresh RPCs only. +- Manual capture if UI fully freezes: `sample Orca 5 -file ~/Desktop/orca-freeze-sample.txt` + +The scripts honor `ORCA_CLI_COMMAND`, then use `orca-dev` in a dev runtime, `orca-ide` on Linux, and `orca` elsewhere. + +PowerShell equivalent for the first example: + +```powershell +$env:ORCA_FREEZE_ENV = 'paired-remote' +$env:ORCA_FREEZE_SCENARIO = 'idle-backlog-open' +$env:ORCA_FREEZE_CREATE = '8' +$env:ORCA_FREEZE_IDLE_MS = '45000' +$env:ORCA_FREEZE_OPEN_COUNT = '24' +pnpm run repro:live-remote-realistic-freeze +```