fix(terminal): rate-cap WebGL atlas recovery (#12622)
This commit is contained in:
Brennan Benson
GitHub
parent
fe72eeb75c
commit
9507cbce0c
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"schemaVersion": 1,
|
||||
"updatedAt": "2026-08-03",
|
||||
"updatedAt": "2026-08-04",
|
||||
"policy": {
|
||||
"maturityLevels": ["experimental", "soak", "blocking", "accepted-gap", "deprecated"],
|
||||
"blockingPromotion": {
|
||||
|
|
@ -8080,21 +8080,24 @@
|
|||
"providers": ["local", "daemon", "ssh", "remote-runtime"],
|
||||
"coveredPlatforms": ["macos"],
|
||||
"coveredProviders": ["local"],
|
||||
"coverageNotes": "A deterministic renderer contract models one visible manager plus 64 mounted hidden managers and hidden synchronized/TUI output. A macOS Electron run verifies that a document visibility cycle preserves real WebGL atlases and terminal pixels. Production v1.4.163 evidence linked the same 49-manager fanout to paired traffic, but the candidate has not been rerun against an isolated live paired server.",
|
||||
"coverageNotes": "A deterministic renderer contract models one visible manager plus 64 mounted hidden managers, hidden synchronized/TUI output, and five minutes of sustained streaming recovery requests. A macOS Electron run verifies that a document visibility cycle preserves real WebGL atlases and terminal pixels. Production v1.4.163 evidence linked the same 49-manager fanout to paired traffic, but the candidate has not been rerun against an isolated live paired server.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/pull/7054",
|
||||
"https://github.com/stablyai/orca/pull/7604"
|
||||
"https://github.com/stablyai/orca/pull/7604",
|
||||
"https://github.com/stablyai/orca/issues/12094"
|
||||
],
|
||||
"invariant": "Ordinary document visibility transitions and hidden terminal output must not clear the shared WebGL glyph atlas. Heavy reset-and-refresh recovery may touch only managers with visible terminal surfaces; hidden managers recover when revealed. Genuine OS resume remains a heavy recovery trigger.",
|
||||
"oracle": "Dispatch a visible document visibilitychange and require atlas-preserving wake recovery; register one visible manager and 64 hidden managers and require exactly one reset and one refresh; parse hidden synchronized and high-confidence TUI output and require zero global atlas-recovery schedules. Separately, drive an Electron visibility cycle with two real WebGL panes, require zero atlas clears, and retain at least 85% of each pane's baseline ink pixels.",
|
||||
"invariant": "Ordinary document visibility transitions and hidden terminal output must not clear the shared WebGL glyph atlas. Heavy reset-and-refresh recovery may touch only managers with visible terminal surfaces; hidden managers recover when revealed. Sustained streaming recovery requests must keep resets rate-bounded without starving repair. Genuine OS resume remains a heavy recovery trigger.",
|
||||
"oracle": "Dispatch a visible document visibilitychange and require atlas-preserving wake recovery; register one visible manager and 64 hidden managers and require exactly one reset and one refresh; parse hidden synchronized and high-confidence TUI output and require zero global atlas-recovery schedules. Across five minutes of 300ms streaming recovery requests, require more than one but at most 75 resets, no internal or trailing repair gap above 6.5 seconds, atlas-preserving presentation while a reset is suppressed, and one delayed wipe for a final suppressed settle. Separately, drive an Electron visibility cycle with two real WebGL panes, require zero atlas clears, and retain at least 85% of each pane's baseline ink pixels.",
|
||||
"commands": [
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/use-terminal-window-wake-recovery.test.ts src/renderer/src/components/terminal-pane/use-terminal-pane-global-effects.test.ts src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts",
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts",
|
||||
"pnpm run ensure:electron-runtime && pnpm exec playwright test tests/e2e/terminal-document-visibility-webgl-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1",
|
||||
"pnpm run ensure:electron-runtime && pnpm exec playwright test tests/e2e/terminal-document-visibility-webgl-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1"
|
||||
],
|
||||
"testFiles": [
|
||||
"src/renderer/src/components/terminal-pane/use-terminal-window-wake-recovery.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/use-terminal-pane-global-effects.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts",
|
||||
"src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts",
|
||||
"src/renderer/src/components/terminal-pane/pty-connection.test.ts",
|
||||
"tests/e2e/terminal-document-visibility-webgl-recovery.spec.ts"
|
||||
|
|
@ -8110,6 +8113,13 @@
|
|||
"preserves WebGL texture atlases when the active terminal document becomes visible"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts",
|
||||
"assertions": [
|
||||
"sustained streaming requests keep atlas resets rate-bounded with no repair starvation",
|
||||
"suppressed resets present live buffers, cancel repair during resumed output, receive a trailing repair after settle, and clean up pending timers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts",
|
||||
"assertions": ["bounds atlas recovery to visible managers"]
|
||||
|
|
@ -8130,6 +8140,15 @@
|
|||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-08-04",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 0.5,
|
||||
"summary": "Seven deterministic tests passed for sustained reset cadence, bounded repair gaps, suppressed presentation with trailing repair, clock rollback, one-shot bypasses, and pending-timer cleanup."
|
||||
},
|
||||
{
|
||||
"date": "2026-08-01",
|
||||
"runner": "local",
|
||||
|
|
@ -8172,7 +8191,7 @@
|
|||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "A recovery with one visible and 64 hidden managers performs one reset and one refresh instead of 65 of each. Hidden synchronized/TUI output schedules zero global recovery work; no polling, cooldown, provider call, or parking change is added."
|
||||
"evidence": "A recovery with one visible and 64 hidden managers performs one reset and one refresh instead of 65 of each. Hidden synchronized/TUI output schedules zero global recovery work. Sustained 300ms recovery requests keep full atlas resets at or below 75 over five minutes while preserving a 6.5-second maximum repair gap. A final suppressed settle schedules one cancelable repair at the next budget boundary; resumed streaming cancels it. The rate budget adds no polling, recurring timer, provider call, or parking change."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Accumulate stable macOS Electron runs with real BrowserWindow visibility transitions.",
|
||||
|
|
|
|||
|
|
@ -316,6 +316,26 @@ describe('renderer breadcrumb IPC routing', () => {
|
|||
])
|
||||
})
|
||||
|
||||
it('coalesces atlas resets per trigger reason', () => {
|
||||
emitRendererBreadcrumb({
|
||||
name: 'terminal_webgl_diagnostic',
|
||||
data: { kind: 'webgl-atlas-reset', reason: 'terminal-output' }
|
||||
})
|
||||
emitRendererBreadcrumb({
|
||||
name: 'terminal_webgl_diagnostic',
|
||||
data: { kind: 'webgl-atlas-reset', reason: 'system-resume' }
|
||||
})
|
||||
|
||||
expect(
|
||||
recordCoalescedCrashBreadcrumbMock.mock.calls.map(
|
||||
(call) => (call[0] as { coalesceKey: string }).coalesceKey
|
||||
)
|
||||
).toEqual([
|
||||
'terminal_webgl_diagnostic:webgl-atlas-reset:terminal-output',
|
||||
'terminal_webgl_diagnostic:webgl-atlas-reset:system-resume'
|
||||
])
|
||||
})
|
||||
|
||||
// Why: the renderer guard is once per tab-id/verdict, so one stale worktree
|
||||
// map can emit enough crumbs to evict the pre-crash trail.
|
||||
it('coalesces duplicate-tab-owner notices across tabs', () => {
|
||||
|
|
|
|||
|
|
@ -362,12 +362,11 @@ function rendererBreadcrumbCoalesceKey(
|
|||
if (name === PARK_VERDICT_CHURN_BREADCRUMB) {
|
||||
return `${name}:${String(data?.trigger ?? '')}`
|
||||
}
|
||||
// Why kind and not name alone: a context loss (GPU/driver gave up on this
|
||||
// renderer) and an atlas reset (routine post-wake repaint) must never
|
||||
// suppress each other. Within one kind the count is the whole signal — every
|
||||
// live pane emits on a GPU death.
|
||||
// Preserve distinct GPU failures and atlas-reset triggers while coalescing each storm.
|
||||
if (name === TERMINAL_WEBGL_DIAGNOSTIC_BREADCRUMB) {
|
||||
return `${name}:${String(data?.kind ?? '')}`
|
||||
const kind = String(data?.kind ?? '')
|
||||
const reason = kind === 'webgl-atlas-reset' ? data?.reason : undefined
|
||||
return reason ? `${name}:${kind}:${String(reason)}` : `${name}:${kind}`
|
||||
}
|
||||
// Why: a stale map can emit once per tab-id/verdict; key by verdict so
|
||||
// last-write coalescing cannot erase the other signal while remaining bounded.
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ export function sampleRenderDesyncOnce(
|
|||
// Why: captures can contain full terminal canvases and buffer contents.
|
||||
// Keep recovery available after the per-session evidence budget is spent.
|
||||
console.warn(`[terminal] render desync detected on pane ${paneKey}; capture budget exhausted`)
|
||||
resetAndRefreshAllTerminalWebglAtlases()
|
||||
resetAndRefreshAllTerminalWebglAtlases('render-desync')
|
||||
stopSampleBurst()
|
||||
return
|
||||
}
|
||||
|
|
@ -273,7 +273,7 @@ async function persistEvidenceThenRecover(
|
|||
entry.bufferText = undefined
|
||||
}
|
||||
|
||||
resetAndRefreshAllTerminalWebglAtlases()
|
||||
resetAndRefreshAllTerminalWebglAtlases('render-desync')
|
||||
const timeoutId = setTimeout(() => {
|
||||
healedCaptureTimeoutIds.delete(timeoutId)
|
||||
void window.api.app
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ export function resumeTerminalVisibility({
|
|||
if (!shouldUseLightTabResume) {
|
||||
// Why: this clear wipes the glyph atlas shared with other same-config
|
||||
// terminals; refresh after reset so rebuilt atlases repaint from xterm.
|
||||
resetAndRefreshAllTerminalWebglAtlases()
|
||||
resetAndRefreshAllTerminalWebglAtlases('visibility-resume')
|
||||
}
|
||||
// Why: the synchronous recovery above can fire before the revealed pane is
|
||||
// attached and laid out. Follow up after layout with one shared-atlas-safe
|
||||
|
|
@ -168,7 +168,7 @@ export function recoverVisibleTerminalWindowWake({
|
|||
// every same-config pane re-rasterize at once, and xterm's atlas page-merge
|
||||
// clear-model flag is consumed by one renderer (xterm.js #4480), so panes
|
||||
// that lose that race paint garbled glyphs mid-stream.
|
||||
resetAndRefreshAllTerminalWebglAtlases()
|
||||
resetAndRefreshAllTerminalWebglAtlases('system-resume')
|
||||
manager.scheduleRevealRepaint()
|
||||
} else {
|
||||
// Why: the reveal repaint runs a shared-atlas reset, so a plain refocus
|
||||
|
|
|
|||
|
|
@ -0,0 +1,220 @@
|
|||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
registerLivePaneManager,
|
||||
unregisterLivePaneManager
|
||||
} from '@/lib/pane-manager/pane-manager-registry'
|
||||
import { setTerminalWebglDiagnosticRecorder } from '../../../../shared/terminal-webgl-diagnostics'
|
||||
import {
|
||||
resetTerminalWebglAtlasRecoveryBudgetForTesting,
|
||||
scheduleImagePasteWebglAtlasRecovery,
|
||||
scheduleTabRevealWebglAtlasRecovery,
|
||||
scheduleTerminalWebglAtlasRecovery,
|
||||
TERMINAL_OUTPUT_RECOVERY_QUIET_MS
|
||||
} from './terminal-webgl-atlas-recovery'
|
||||
|
||||
describe('terminal WebGL atlas recovery rate', () => {
|
||||
const managers: { resetWebglTextureAtlases(): void }[] = []
|
||||
|
||||
function registerManager() {
|
||||
const manager = {
|
||||
resetWebglTextureAtlases: vi.fn(),
|
||||
refreshAllPanes: vi.fn(),
|
||||
scheduleRevealPresent: vi.fn()
|
||||
}
|
||||
registerLivePaneManager(manager)
|
||||
managers.push(manager)
|
||||
return manager
|
||||
}
|
||||
|
||||
function useImmediateAnimationFrames(): void {
|
||||
vi.stubGlobal(
|
||||
'requestAnimationFrame',
|
||||
vi.fn((callback: FrameRequestCallback) => {
|
||||
callback(0)
|
||||
return 1
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const manager of managers.splice(0)) {
|
||||
unregisterLivePaneManager(manager)
|
||||
}
|
||||
setTerminalWebglDiagnosticRecorder(null)
|
||||
resetTerminalWebglAtlasRecoveryBudgetForTesting()
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('caps atlas wipes under a sustained redraw cadence', () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
useImmediateAnimationFrames()
|
||||
const wipeTimes: number[] = []
|
||||
const manager = {
|
||||
resetWebglTextureAtlases: vi.fn(() => wipeTimes.push(Date.now())),
|
||||
refreshAllPanes: vi.fn(),
|
||||
scheduleRevealPresent: vi.fn()
|
||||
}
|
||||
registerLivePaneManager(manager)
|
||||
managers.push(manager)
|
||||
|
||||
for (let elapsed = 0; elapsed < 300_000; elapsed += 300) {
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(300)
|
||||
}
|
||||
|
||||
expect(wipeTimes.length).toBeGreaterThan(1)
|
||||
const gaps = wipeTimes.slice(1).map((at, index) => at - wipeTimes[index]!)
|
||||
expect(wipeTimes.length).toBeLessThanOrEqual(75)
|
||||
expect(Math.max(...gaps)).toBeLessThanOrEqual(6_500)
|
||||
expect(Date.now() - wipeTimes.at(-1)!).toBeLessThanOrEqual(6_500)
|
||||
|
||||
const streamEndedAt = Date.now()
|
||||
const wipesBeforeTailRepair = wipeTimes.length
|
||||
vi.advanceTimersByTime(6_500)
|
||||
expect(wipeTimes).toHaveLength(wipesBeforeTailRepair + 1)
|
||||
expect(wipeTimes.at(-1)! - streamEndedAt).toBeLessThanOrEqual(6_500)
|
||||
})
|
||||
|
||||
it('cancels pending settle and deferred repair timers when recovery state resets', () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
useImmediateAnimationFrames()
|
||||
const manager = registerManager()
|
||||
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
resetTerminalWebglAtlasRecoveryBudgetForTesting()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
|
||||
expect(manager.resetWebglTextureAtlases).not.toHaveBeenCalled()
|
||||
expect(manager.scheduleRevealPresent).not.toHaveBeenCalled()
|
||||
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
resetTerminalWebglAtlasRecoveryBudgetForTesting()
|
||||
vi.advanceTimersByTime(3_000)
|
||||
|
||||
expect(manager.resetWebglTextureAtlases).toHaveBeenCalledOnce()
|
||||
expect(manager.scheduleRevealPresent).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('presents a suppressed wipe and cancels its repair while output resumes', () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
useImmediateAnimationFrames()
|
||||
const manager = registerManager()
|
||||
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
|
||||
expect(manager.resetWebglTextureAtlases).toHaveBeenCalledOnce()
|
||||
expect(manager.scheduleRevealPresent).toHaveBeenCalledOnce()
|
||||
|
||||
for (let elapsed = 0; elapsed < 3_000; elapsed += 50) {
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(50)
|
||||
}
|
||||
expect(manager.resetWebglTextureAtlases).toHaveBeenCalledOnce()
|
||||
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('repairs the final suppressed settle when the budget becomes available', () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
useImmediateAnimationFrames()
|
||||
const manager = registerManager()
|
||||
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
vi.advanceTimersByTime(2_799)
|
||||
|
||||
expect(manager.resetWebglTextureAtlases).toHaveBeenCalledOnce()
|
||||
vi.advanceTimersByTime(1)
|
||||
expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('reports suppressed reset counts on the next permitted wipe', () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
useImmediateAnimationFrames()
|
||||
registerManager()
|
||||
const diagnostics: { kind: string; detail?: Record<string, unknown> }[] = []
|
||||
setTerminalWebglDiagnosticRecorder((kind, detail) => diagnostics.push({ kind, detail }))
|
||||
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
vi.advanceTimersByTime(2_600)
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
|
||||
expect(diagnostics.filter(({ kind }) => kind === 'webgl-atlas-reset-rate')).toEqual([
|
||||
{
|
||||
kind: 'webgl-atlas-reset-rate',
|
||||
detail: {
|
||||
reason: 'terminal-output',
|
||||
attemptsSinceLastReset: 1,
|
||||
atlasResetsSuppressed: 0,
|
||||
intervalMs: 0
|
||||
}
|
||||
},
|
||||
{
|
||||
kind: 'webgl-atlas-reset-rate',
|
||||
detail: {
|
||||
reason: 'terminal-output',
|
||||
attemptsSinceLastReset: 2,
|
||||
atlasResetsSuppressed: 1,
|
||||
intervalMs: 3_000
|
||||
}
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('recovers when the wall clock steps backwards', () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(600_000)
|
||||
useImmediateAnimationFrames()
|
||||
const manager = registerManager()
|
||||
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
vi.setSystemTime(60_000)
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
|
||||
expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('keeps one-shot paste and reveal recovery outside the streaming budget', () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
useImmediateAnimationFrames()
|
||||
const manager = registerManager()
|
||||
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
scheduleTerminalWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
manager.resetWebglTextureAtlases.mockClear()
|
||||
|
||||
scheduleTabRevealWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(500)
|
||||
expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(3)
|
||||
|
||||
manager.resetWebglTextureAtlases.mockClear()
|
||||
scheduleImagePasteWebglAtlasRecovery()
|
||||
vi.advanceTimersByTime(500)
|
||||
expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
})
|
||||
|
|
@ -4,6 +4,7 @@ import {
|
|||
unregisterLivePaneManager
|
||||
} from '@/lib/pane-manager/pane-manager-registry'
|
||||
import {
|
||||
resetTerminalWebglAtlasRecoveryBudgetForTesting,
|
||||
scheduleImagePasteWebglAtlasRecovery,
|
||||
scheduleTabRevealWebglAtlasRecovery,
|
||||
scheduleTerminalWebglAtlasRecovery,
|
||||
|
|
@ -16,10 +17,12 @@ describe('terminal WebGL atlas recovery', () => {
|
|||
function registerManager(): {
|
||||
resetWebglTextureAtlases: Mock<() => void>
|
||||
refreshAllPanes: Mock<() => void>
|
||||
scheduleRevealPresent: Mock<() => void>
|
||||
} {
|
||||
const manager = {
|
||||
resetWebglTextureAtlases: vi.fn<() => void>(),
|
||||
refreshAllPanes: vi.fn<() => void>()
|
||||
refreshAllPanes: vi.fn<() => void>(),
|
||||
scheduleRevealPresent: vi.fn<() => void>()
|
||||
}
|
||||
registerLivePaneManager(manager)
|
||||
registeredManagers.push(manager)
|
||||
|
|
@ -30,9 +33,7 @@ describe('terminal WebGL atlas recovery', () => {
|
|||
for (const manager of registeredManagers.splice(0)) {
|
||||
unregisterLivePaneManager(manager)
|
||||
}
|
||||
// Why: a test can leave the module-global debounce timer armed; clear the
|
||||
// fake-timer queue before restoring real timers so no pending fire leaks
|
||||
// into a later test.
|
||||
resetTerminalWebglAtlasRecoveryBudgetForTesting()
|
||||
vi.clearAllTimers()
|
||||
vi.useRealTimers()
|
||||
vi.unstubAllGlobals()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
import { resetAndRefreshAllTerminalWebglAtlases } from '@/lib/pane-manager/pane-manager-registry'
|
||||
import {
|
||||
presentAllTerminalPanesWithoutAtlasClear,
|
||||
resetAndRefreshAllTerminalWebglAtlases
|
||||
} from '@/lib/pane-manager/pane-manager-registry'
|
||||
import { recordTerminalWebglDiagnostic } from '../../../../shared/terminal-webgl-diagnostics'
|
||||
|
||||
const ATLAS_RECOVERY_DELAYS_MS = [120, 500]
|
||||
|
||||
|
|
@ -7,7 +11,18 @@ const ATLAS_RECOVERY_DELAYS_MS = [120, 500]
|
|||
// (STA-1365). Wait for output to go quiet so recovery runs once, on settle.
|
||||
export const TERMINAL_OUTPUT_RECOVERY_QUIET_MS = 200
|
||||
|
||||
// Bound reset storms while preserving prompt recovery and steady repair attempts.
|
||||
const TERMINAL_OUTPUT_RECOVERY_MIN_INTERVAL_MS = 3_000
|
||||
const TERMINAL_OUTPUT_RECOVERY_BURST_WIPES = 10
|
||||
const TERMINAL_OUTPUT_RECOVERY_REFILL_INTERVAL_MS = 6_000
|
||||
|
||||
let terminalOutputRecoveryDebounceTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let terminalOutputRecoveryRetryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let terminalOutputRecoveryWipeTokens = TERMINAL_OUTPUT_RECOVERY_BURST_WIPES
|
||||
let terminalOutputRecoveryTokensRefilledAt: number | null = null
|
||||
let terminalOutputRecoveryLastWipeAt: number | null = null
|
||||
let terminalOutputRecoveryAttemptsSinceLastWipe = 0
|
||||
let terminalOutputRecoverySuppressedSinceLastWipe = 0
|
||||
|
||||
function scheduleNextFrame(callback: () => void): void {
|
||||
if (typeof globalThis.requestAnimationFrame === 'function') {
|
||||
|
|
@ -17,20 +32,28 @@ function scheduleNextFrame(callback: () => void): void {
|
|||
globalThis.setTimeout(callback, 0)
|
||||
}
|
||||
|
||||
function resetAtlasesAndRefreshPanes(): void {
|
||||
function resetAtlasesAndRefreshPanes(reason: string): void {
|
||||
try {
|
||||
// Why: the glyph atlas is shared across same-config terminals, so the
|
||||
// recovery reset must be followed by repainting each rebuilt render model.
|
||||
resetAndRefreshAllTerminalWebglAtlases()
|
||||
resetAndRefreshAllTerminalWebglAtlases(reason)
|
||||
} catch {
|
||||
/* ignore - terminal pane may have unmounted after scheduling recovery */
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleAtlasRecoveryBurst(): void {
|
||||
scheduleNextFrame(() => resetAtlasesAndRefreshPanes())
|
||||
function presentPanesWithoutAtlasClear(): void {
|
||||
try {
|
||||
presentAllTerminalPanesWithoutAtlasClear()
|
||||
} catch {
|
||||
/* ignore - terminal pane may have unmounted after scheduling recovery */
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleAtlasRecoveryBurst(reason: string): void {
|
||||
scheduleNextFrame(() => resetAtlasesAndRefreshPanes(reason))
|
||||
for (const delayMs of ATLAS_RECOVERY_DELAYS_MS) {
|
||||
globalThis.setTimeout(() => resetAtlasesAndRefreshPanes(), delayMs)
|
||||
globalThis.setTimeout(() => resetAtlasesAndRefreshPanes(reason), delayMs)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -38,13 +61,13 @@ export function scheduleImagePasteWebglAtlasRecovery(): void {
|
|||
// Why: image chips can redraw after bracketed paste parsing, so cover the
|
||||
// short post-paste paint window with a few cheap atlas rebuilds. Paste is a
|
||||
// one-shot event, so recover immediately rather than debouncing.
|
||||
scheduleAtlasRecoveryBurst()
|
||||
scheduleAtlasRecoveryBurst('image-paste')
|
||||
}
|
||||
|
||||
export function scheduleTabRevealWebglAtlasRecovery(): void {
|
||||
// Why: a tab reveal is one-shot, so recover immediately — decoupled from the
|
||||
// streaming debounce so a background stream can't defer a revealed tab's rebuild.
|
||||
scheduleAtlasRecoveryBurst()
|
||||
scheduleAtlasRecoveryBurst('tab-reveal')
|
||||
}
|
||||
|
||||
export function scheduleTerminalWebglAtlasRecovery(): void {
|
||||
|
|
@ -54,8 +77,101 @@ export function scheduleTerminalWebglAtlasRecovery(): void {
|
|||
if (terminalOutputRecoveryDebounceTimer != null) {
|
||||
globalThis.clearTimeout(terminalOutputRecoveryDebounceTimer)
|
||||
}
|
||||
if (terminalOutputRecoveryRetryTimer != null) {
|
||||
globalThis.clearTimeout(terminalOutputRecoveryRetryTimer)
|
||||
terminalOutputRecoveryRetryTimer = null
|
||||
}
|
||||
terminalOutputRecoveryDebounceTimer = globalThis.setTimeout(() => {
|
||||
terminalOutputRecoveryDebounceTimer = null
|
||||
resetAtlasesAndRefreshPanes()
|
||||
terminalOutputRecoveryAttemptsSinceLastWipe += 1
|
||||
const decision = consumeTerminalOutputRecoveryWipeBudget()
|
||||
if (!decision.allowed) {
|
||||
terminalOutputRecoverySuppressedSinceLastWipe += 1
|
||||
presentPanesWithoutAtlasClear()
|
||||
scheduleTerminalOutputRecoveryRetry(decision.retryAfterMs)
|
||||
return
|
||||
}
|
||||
resetTerminalOutputAtlases(decision.intervalMs)
|
||||
}, TERMINAL_OUTPUT_RECOVERY_QUIET_MS)
|
||||
}
|
||||
|
||||
function scheduleTerminalOutputRecoveryRetry(delayMs: number): void {
|
||||
terminalOutputRecoveryRetryTimer = globalThis.setTimeout(() => {
|
||||
terminalOutputRecoveryRetryTimer = null
|
||||
const decision = consumeTerminalOutputRecoveryWipeBudget()
|
||||
if (!decision.allowed) {
|
||||
scheduleTerminalOutputRecoveryRetry(decision.retryAfterMs)
|
||||
return
|
||||
}
|
||||
resetTerminalOutputAtlases(decision.intervalMs)
|
||||
}, delayMs)
|
||||
}
|
||||
|
||||
function resetTerminalOutputAtlases(intervalMs: number): void {
|
||||
recordTerminalWebglDiagnostic('webgl-atlas-reset-rate', {
|
||||
reason: 'terminal-output',
|
||||
attemptsSinceLastReset: terminalOutputRecoveryAttemptsSinceLastWipe,
|
||||
atlasResetsSuppressed: terminalOutputRecoverySuppressedSinceLastWipe,
|
||||
intervalMs
|
||||
})
|
||||
terminalOutputRecoveryAttemptsSinceLastWipe = 0
|
||||
terminalOutputRecoverySuppressedSinceLastWipe = 0
|
||||
resetAtlasesAndRefreshPanes('terminal-output')
|
||||
}
|
||||
|
||||
export function resetTerminalWebglAtlasRecoveryBudgetForTesting(): void {
|
||||
if (terminalOutputRecoveryDebounceTimer != null) {
|
||||
globalThis.clearTimeout(terminalOutputRecoveryDebounceTimer)
|
||||
}
|
||||
terminalOutputRecoveryDebounceTimer = null
|
||||
if (terminalOutputRecoveryRetryTimer != null) {
|
||||
globalThis.clearTimeout(terminalOutputRecoveryRetryTimer)
|
||||
}
|
||||
terminalOutputRecoveryRetryTimer = null
|
||||
terminalOutputRecoveryWipeTokens = TERMINAL_OUTPUT_RECOVERY_BURST_WIPES
|
||||
terminalOutputRecoveryTokensRefilledAt = null
|
||||
terminalOutputRecoveryLastWipeAt = null
|
||||
terminalOutputRecoveryAttemptsSinceLastWipe = 0
|
||||
terminalOutputRecoverySuppressedSinceLastWipe = 0
|
||||
}
|
||||
|
||||
function refillTerminalOutputRecoveryWipeTokens(now: number): void {
|
||||
const elapsedMs = now - (terminalOutputRecoveryTokensRefilledAt ?? now)
|
||||
terminalOutputRecoveryTokensRefilledAt = now
|
||||
// Sleep, NTP, and remote clock corrections must not wedge recovery.
|
||||
if (elapsedMs < 0) {
|
||||
terminalOutputRecoveryWipeTokens = TERMINAL_OUTPUT_RECOVERY_BURST_WIPES
|
||||
terminalOutputRecoveryLastWipeAt = null
|
||||
return
|
||||
}
|
||||
terminalOutputRecoveryWipeTokens = Math.min(
|
||||
TERMINAL_OUTPUT_RECOVERY_BURST_WIPES,
|
||||
terminalOutputRecoveryWipeTokens + elapsedMs / TERMINAL_OUTPUT_RECOVERY_REFILL_INTERVAL_MS
|
||||
)
|
||||
}
|
||||
|
||||
function consumeTerminalOutputRecoveryWipeBudget(): {
|
||||
allowed: boolean
|
||||
intervalMs: number
|
||||
retryAfterMs: number
|
||||
} {
|
||||
const now = Date.now()
|
||||
refillTerminalOutputRecoveryWipeTokens(now)
|
||||
const intervalMs =
|
||||
terminalOutputRecoveryLastWipeAt == null ? 0 : now - terminalOutputRecoveryLastWipeAt
|
||||
const intervalBudgetMs =
|
||||
terminalOutputRecoveryLastWipeAt == null
|
||||
? 0
|
||||
: Math.max(0, TERMINAL_OUTPUT_RECOVERY_MIN_INTERVAL_MS - intervalMs)
|
||||
const tokenBudgetMs =
|
||||
terminalOutputRecoveryWipeTokens >= 1
|
||||
? 0
|
||||
: (1 - terminalOutputRecoveryWipeTokens) * TERMINAL_OUTPUT_RECOVERY_REFILL_INTERVAL_MS
|
||||
const retryAfterMs = Math.ceil(Math.max(intervalBudgetMs, tokenBudgetMs))
|
||||
if (retryAfterMs > 0) {
|
||||
return { allowed: false, intervalMs, retryAfterMs }
|
||||
}
|
||||
terminalOutputRecoveryWipeTokens -= 1
|
||||
terminalOutputRecoveryLastWipeAt = now
|
||||
return { allowed: true, intervalMs, retryAfterMs: 0 }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { collectRendererMemoryProfileCounts } from '../renderer-memory-profile'
|
|||
import {
|
||||
forEachLivePaneForDesyncSentinel,
|
||||
getLivePaneCensus,
|
||||
presentAllTerminalPanesWithoutAtlasClear,
|
||||
getLivePaneMemoryProfileCounts,
|
||||
refitAndRefreshAllTerminalPanes,
|
||||
registerLivePaneManager,
|
||||
|
|
@ -136,6 +137,29 @@ describe('pane manager registry', () => {
|
|||
expect(healthy.refreshAllPanes).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('presents visible managers without resetting their atlases', () => {
|
||||
const visible = {
|
||||
resetWebglTextureAtlases: vi.fn<() => void>(),
|
||||
scheduleRevealPresent: vi.fn<() => void>(),
|
||||
isVisibleForAtlasRecovery: () => true
|
||||
}
|
||||
const hidden = {
|
||||
resetWebglTextureAtlases: vi.fn<() => void>(),
|
||||
scheduleRevealPresent: vi.fn<() => void>(),
|
||||
isVisibleForAtlasRecovery: () => false
|
||||
}
|
||||
registerLivePaneManager(visible)
|
||||
registeredManagers.push(visible)
|
||||
registerLivePaneManager(hidden)
|
||||
registeredManagers.push(hidden)
|
||||
|
||||
presentAllTerminalPanesWithoutAtlasClear()
|
||||
|
||||
expect(visible.scheduleRevealPresent).toHaveBeenCalledOnce()
|
||||
expect(visible.resetWebglTextureAtlases).not.toHaveBeenCalled()
|
||||
expect(hidden.scheduleRevealPresent).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('fits and refreshes every registered manager', () => {
|
||||
const first = {
|
||||
resetWebglTextureAtlases: vi.fn<() => void>(),
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ type RegisteredPaneManager = {
|
|||
getPanes?: (limit?: number) => { id: number; terminal: unknown }[]
|
||||
getPaneCount?: () => number
|
||||
isVisibleForAtlasRecovery?: () => boolean
|
||||
scheduleRevealPresent?: () => void
|
||||
}
|
||||
|
||||
const liveManagers = new Set<RegisteredPaneManager>()
|
||||
|
|
@ -78,6 +79,19 @@ export function resetAndRefreshAllTerminalWebglAtlases(reason?: string): void {
|
|||
}
|
||||
}
|
||||
|
||||
export function presentAllTerminalPanesWithoutAtlasClear(): void {
|
||||
for (const manager of liveManagers) {
|
||||
if (manager.isVisibleForAtlasRecovery?.() === false) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
manager.scheduleRevealPresent?.()
|
||||
} catch {
|
||||
// A disposing manager must not block sibling panes from presenting.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-pane WebGL renderer state across all live managers, for the one-paste
|
||||
* freeze report. Lets a post-wake garble report show, per pane, whether it
|
||||
|
|
|
|||
Loading…
Reference in New Issue