diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 4abdba712..ce371c681 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "updatedAt": "2026-08-03", + "updatedAt": "2026-08-04", "policy": { "maturityLevels": ["experimental", "soak", "blocking", "accepted-gap", "deprecated"], "blockingPromotion": { @@ -8080,21 +8080,24 @@ "providers": ["local", "daemon", "ssh", "remote-runtime"], "coveredPlatforms": ["macos"], "coveredProviders": ["local"], - "coverageNotes": "A deterministic renderer contract models one visible manager plus 64 mounted hidden managers and hidden synchronized/TUI output. A macOS Electron run verifies that a document visibility cycle preserves real WebGL atlases and terminal pixels. Production v1.4.163 evidence linked the same 49-manager fanout to paired traffic, but the candidate has not been rerun against an isolated live paired server.", + "coverageNotes": "A deterministic renderer contract models one visible manager plus 64 mounted hidden managers, hidden synchronized/TUI output, and five minutes of sustained streaming recovery requests. A macOS Electron run verifies that a document visibility cycle preserves real WebGL atlases and terminal pixels. Production v1.4.163 evidence linked the same 49-manager fanout to paired traffic, but the candidate has not been rerun against an isolated live paired server.", "motivatingLinks": [ "https://github.com/stablyai/orca/pull/7054", - "https://github.com/stablyai/orca/pull/7604" + "https://github.com/stablyai/orca/pull/7604", + "https://github.com/stablyai/orca/issues/12094" ], - "invariant": "Ordinary document visibility transitions and hidden terminal output must not clear the shared WebGL glyph atlas. Heavy reset-and-refresh recovery may touch only managers with visible terminal surfaces; hidden managers recover when revealed. Genuine OS resume remains a heavy recovery trigger.", - "oracle": "Dispatch a visible document visibilitychange and require atlas-preserving wake recovery; register one visible manager and 64 hidden managers and require exactly one reset and one refresh; parse hidden synchronized and high-confidence TUI output and require zero global atlas-recovery schedules. Separately, drive an Electron visibility cycle with two real WebGL panes, require zero atlas clears, and retain at least 85% of each pane's baseline ink pixels.", + "invariant": "Ordinary document visibility transitions and hidden terminal output must not clear the shared WebGL glyph atlas. Heavy reset-and-refresh recovery may touch only managers with visible terminal surfaces; hidden managers recover when revealed. Sustained streaming recovery requests must keep resets rate-bounded without starving repair. Genuine OS resume remains a heavy recovery trigger.", + "oracle": "Dispatch a visible document visibilitychange and require atlas-preserving wake recovery; register one visible manager and 64 hidden managers and require exactly one reset and one refresh; parse hidden synchronized and high-confidence TUI output and require zero global atlas-recovery schedules. Across five minutes of 300ms streaming recovery requests, require more than one but at most 75 resets, no internal or trailing repair gap above 6.5 seconds, atlas-preserving presentation while a reset is suppressed, and one delayed wipe for a final suppressed settle. Separately, drive an Electron visibility cycle with two real WebGL panes, require zero atlas clears, and retain at least 85% of each pane's baseline ink pixels.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/use-terminal-window-wake-recovery.test.ts src/renderer/src/components/terminal-pane/use-terminal-pane-global-effects.test.ts src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts", "pnpm run ensure:electron-runtime && pnpm exec playwright test tests/e2e/terminal-document-visibility-webgl-recovery.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1", "pnpm run ensure:electron-runtime && pnpm exec playwright test tests/e2e/terminal-document-visibility-webgl-recovery.spec.ts --config tests/playwright.config.ts --project electron-headful --workers=1" ], "testFiles": [ "src/renderer/src/components/terminal-pane/use-terminal-window-wake-recovery.test.ts", "src/renderer/src/components/terminal-pane/use-terminal-pane-global-effects.test.ts", + "src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts", "src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts", "src/renderer/src/components/terminal-pane/pty-connection.test.ts", "tests/e2e/terminal-document-visibility-webgl-recovery.spec.ts" @@ -8110,6 +8113,13 @@ "preserves WebGL texture atlases when the active terminal document becomes visible" ] }, + { + "file": "src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts", + "assertions": [ + "sustained streaming requests keep atlas resets rate-bounded with no repair starvation", + "suppressed resets present live buffers, cancel repair during resumed output, receive a trailing repair after settle, and clean up pending timers" + ] + }, { "file": "src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts", "assertions": ["bounds atlas recovery to visible managers"] @@ -8130,6 +8140,15 @@ } ], "evidenceRuns": [ + { + "date": "2026-08-04", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts", + "result": "passed", + "durationSeconds": 0.5, + "summary": "Seven deterministic tests passed for sustained reset cadence, bounded repair gaps, suppressed presentation with trailing repair, clock rollback, one-shot bypasses, and pending-timer cleanup." + }, { "date": "2026-08-01", "runner": "local", @@ -8172,7 +8191,7 @@ }, "performanceBudget": { "required": true, - "evidence": "A recovery with one visible and 64 hidden managers performs one reset and one refresh instead of 65 of each. Hidden synchronized/TUI output schedules zero global recovery work; no polling, cooldown, provider call, or parking change is added." + "evidence": "A recovery with one visible and 64 hidden managers performs one reset and one refresh instead of 65 of each. Hidden synchronized/TUI output schedules zero global recovery work. Sustained 300ms recovery requests keep full atlas resets at or below 75 over five minutes while preserving a 6.5-second maximum repair gap. A final suppressed settle schedules one cancelable repair at the next budget boundary; resumed streaming cancels it. The rate budget adds no polling, recurring timer, provider call, or parking change." }, "promotionCriteria": [ "Accumulate stable macOS Electron runs with real BrowserWindow visibility transitions.", diff --git a/src/main/ipc/crash-reporting-renderer-breadcrumbs.test.ts b/src/main/ipc/crash-reporting-renderer-breadcrumbs.test.ts index 9755facf4..f291580db 100644 --- a/src/main/ipc/crash-reporting-renderer-breadcrumbs.test.ts +++ b/src/main/ipc/crash-reporting-renderer-breadcrumbs.test.ts @@ -316,6 +316,26 @@ describe('renderer breadcrumb IPC routing', () => { ]) }) + it('coalesces atlas resets per trigger reason', () => { + emitRendererBreadcrumb({ + name: 'terminal_webgl_diagnostic', + data: { kind: 'webgl-atlas-reset', reason: 'terminal-output' } + }) + emitRendererBreadcrumb({ + name: 'terminal_webgl_diagnostic', + data: { kind: 'webgl-atlas-reset', reason: 'system-resume' } + }) + + expect( + recordCoalescedCrashBreadcrumbMock.mock.calls.map( + (call) => (call[0] as { coalesceKey: string }).coalesceKey + ) + ).toEqual([ + 'terminal_webgl_diagnostic:webgl-atlas-reset:terminal-output', + 'terminal_webgl_diagnostic:webgl-atlas-reset:system-resume' + ]) + }) + // Why: the renderer guard is once per tab-id/verdict, so one stale worktree // map can emit enough crumbs to evict the pre-crash trail. it('coalesces duplicate-tab-owner notices across tabs', () => { diff --git a/src/main/ipc/crash-reporting.ts b/src/main/ipc/crash-reporting.ts index 71c52446d..2094d0ee1 100644 --- a/src/main/ipc/crash-reporting.ts +++ b/src/main/ipc/crash-reporting.ts @@ -362,12 +362,11 @@ function rendererBreadcrumbCoalesceKey( if (name === PARK_VERDICT_CHURN_BREADCRUMB) { return `${name}:${String(data?.trigger ?? '')}` } - // Why kind and not name alone: a context loss (GPU/driver gave up on this - // renderer) and an atlas reset (routine post-wake repaint) must never - // suppress each other. Within one kind the count is the whole signal — every - // live pane emits on a GPU death. + // Preserve distinct GPU failures and atlas-reset triggers while coalescing each storm. if (name === TERMINAL_WEBGL_DIAGNOSTIC_BREADCRUMB) { - return `${name}:${String(data?.kind ?? '')}` + const kind = String(data?.kind ?? '') + const reason = kind === 'webgl-atlas-reset' ? data?.reason : undefined + return reason ? `${name}:${kind}:${String(reason)}` : `${name}:${kind}` } // Why: a stale map can emit once per tab-id/verdict; key by verdict so // last-write coalescing cannot erase the other signal while remaining bounded. diff --git a/src/renderer/src/components/terminal-pane/terminal-render-desync-sentinel.ts b/src/renderer/src/components/terminal-pane/terminal-render-desync-sentinel.ts index b6a9b0847..0fca0d767 100644 --- a/src/renderer/src/components/terminal-pane/terminal-render-desync-sentinel.ts +++ b/src/renderer/src/components/terminal-pane/terminal-render-desync-sentinel.ts @@ -128,7 +128,7 @@ export function sampleRenderDesyncOnce( // Why: captures can contain full terminal canvases and buffer contents. // Keep recovery available after the per-session evidence budget is spent. console.warn(`[terminal] render desync detected on pane ${paneKey}; capture budget exhausted`) - resetAndRefreshAllTerminalWebglAtlases() + resetAndRefreshAllTerminalWebglAtlases('render-desync') stopSampleBurst() return } @@ -273,7 +273,7 @@ async function persistEvidenceThenRecover( entry.bufferText = undefined } - resetAndRefreshAllTerminalWebglAtlases() + resetAndRefreshAllTerminalWebglAtlases('render-desync') const timeoutId = setTimeout(() => { healedCaptureTimeoutIds.delete(timeoutId) void window.api.app diff --git a/src/renderer/src/components/terminal-pane/terminal-visibility-resume.ts b/src/renderer/src/components/terminal-pane/terminal-visibility-resume.ts index 707872662..c9cf2a892 100644 --- a/src/renderer/src/components/terminal-pane/terminal-visibility-resume.ts +++ b/src/renderer/src/components/terminal-pane/terminal-visibility-resume.ts @@ -91,7 +91,7 @@ export function resumeTerminalVisibility({ if (!shouldUseLightTabResume) { // Why: this clear wipes the glyph atlas shared with other same-config // terminals; refresh after reset so rebuilt atlases repaint from xterm. - resetAndRefreshAllTerminalWebglAtlases() + resetAndRefreshAllTerminalWebglAtlases('visibility-resume') } // Why: the synchronous recovery above can fire before the revealed pane is // attached and laid out. Follow up after layout with one shared-atlas-safe @@ -168,7 +168,7 @@ export function recoverVisibleTerminalWindowWake({ // every same-config pane re-rasterize at once, and xterm's atlas page-merge // clear-model flag is consumed by one renderer (xterm.js #4480), so panes // that lose that race paint garbled glyphs mid-stream. - resetAndRefreshAllTerminalWebglAtlases() + resetAndRefreshAllTerminalWebglAtlases('system-resume') manager.scheduleRevealRepaint() } else { // Why: the reveal repaint runs a shared-atlas reset, so a plain refocus diff --git a/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts b/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts new file mode 100644 index 000000000..4add011e4 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery-rate.test.ts @@ -0,0 +1,220 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + registerLivePaneManager, + unregisterLivePaneManager +} from '@/lib/pane-manager/pane-manager-registry' +import { setTerminalWebglDiagnosticRecorder } from '../../../../shared/terminal-webgl-diagnostics' +import { + resetTerminalWebglAtlasRecoveryBudgetForTesting, + scheduleImagePasteWebglAtlasRecovery, + scheduleTabRevealWebglAtlasRecovery, + scheduleTerminalWebglAtlasRecovery, + TERMINAL_OUTPUT_RECOVERY_QUIET_MS +} from './terminal-webgl-atlas-recovery' + +describe('terminal WebGL atlas recovery rate', () => { + const managers: { resetWebglTextureAtlases(): void }[] = [] + + function registerManager() { + const manager = { + resetWebglTextureAtlases: vi.fn(), + refreshAllPanes: vi.fn(), + scheduleRevealPresent: vi.fn() + } + registerLivePaneManager(manager) + managers.push(manager) + return manager + } + + function useImmediateAnimationFrames(): void { + vi.stubGlobal( + 'requestAnimationFrame', + vi.fn((callback: FrameRequestCallback) => { + callback(0) + return 1 + }) + ) + } + + afterEach(() => { + for (const manager of managers.splice(0)) { + unregisterLivePaneManager(manager) + } + setTerminalWebglDiagnosticRecorder(null) + resetTerminalWebglAtlasRecoveryBudgetForTesting() + vi.clearAllTimers() + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + it('caps atlas wipes under a sustained redraw cadence', () => { + vi.useFakeTimers() + vi.setSystemTime(0) + useImmediateAnimationFrames() + const wipeTimes: number[] = [] + const manager = { + resetWebglTextureAtlases: vi.fn(() => wipeTimes.push(Date.now())), + refreshAllPanes: vi.fn(), + scheduleRevealPresent: vi.fn() + } + registerLivePaneManager(manager) + managers.push(manager) + + for (let elapsed = 0; elapsed < 300_000; elapsed += 300) { + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(300) + } + + expect(wipeTimes.length).toBeGreaterThan(1) + const gaps = wipeTimes.slice(1).map((at, index) => at - wipeTimes[index]!) + expect(wipeTimes.length).toBeLessThanOrEqual(75) + expect(Math.max(...gaps)).toBeLessThanOrEqual(6_500) + expect(Date.now() - wipeTimes.at(-1)!).toBeLessThanOrEqual(6_500) + + const streamEndedAt = Date.now() + const wipesBeforeTailRepair = wipeTimes.length + vi.advanceTimersByTime(6_500) + expect(wipeTimes).toHaveLength(wipesBeforeTailRepair + 1) + expect(wipeTimes.at(-1)! - streamEndedAt).toBeLessThanOrEqual(6_500) + }) + + it('cancels pending settle and deferred repair timers when recovery state resets', () => { + vi.useFakeTimers() + vi.setSystemTime(0) + useImmediateAnimationFrames() + const manager = registerManager() + + scheduleTerminalWebglAtlasRecovery() + resetTerminalWebglAtlasRecoveryBudgetForTesting() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + + expect(manager.resetWebglTextureAtlases).not.toHaveBeenCalled() + expect(manager.scheduleRevealPresent).not.toHaveBeenCalled() + + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + resetTerminalWebglAtlasRecoveryBudgetForTesting() + vi.advanceTimersByTime(3_000) + + expect(manager.resetWebglTextureAtlases).toHaveBeenCalledOnce() + expect(manager.scheduleRevealPresent).toHaveBeenCalledOnce() + }) + + it('presents a suppressed wipe and cancels its repair while output resumes', () => { + vi.useFakeTimers() + vi.setSystemTime(0) + useImmediateAnimationFrames() + const manager = registerManager() + + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + + expect(manager.resetWebglTextureAtlases).toHaveBeenCalledOnce() + expect(manager.scheduleRevealPresent).toHaveBeenCalledOnce() + + for (let elapsed = 0; elapsed < 3_000; elapsed += 50) { + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(50) + } + expect(manager.resetWebglTextureAtlases).toHaveBeenCalledOnce() + + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(2) + }) + + it('repairs the final suppressed settle when the budget becomes available', () => { + vi.useFakeTimers() + vi.setSystemTime(0) + useImmediateAnimationFrames() + const manager = registerManager() + + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + vi.advanceTimersByTime(2_799) + + expect(manager.resetWebglTextureAtlases).toHaveBeenCalledOnce() + vi.advanceTimersByTime(1) + expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(2) + }) + + it('reports suppressed reset counts on the next permitted wipe', () => { + vi.useFakeTimers() + vi.setSystemTime(0) + useImmediateAnimationFrames() + registerManager() + const diagnostics: { kind: string; detail?: Record }[] = [] + setTerminalWebglDiagnosticRecorder((kind, detail) => diagnostics.push({ kind, detail })) + + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + vi.advanceTimersByTime(2_600) + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + + expect(diagnostics.filter(({ kind }) => kind === 'webgl-atlas-reset-rate')).toEqual([ + { + kind: 'webgl-atlas-reset-rate', + detail: { + reason: 'terminal-output', + attemptsSinceLastReset: 1, + atlasResetsSuppressed: 0, + intervalMs: 0 + } + }, + { + kind: 'webgl-atlas-reset-rate', + detail: { + reason: 'terminal-output', + attemptsSinceLastReset: 2, + atlasResetsSuppressed: 1, + intervalMs: 3_000 + } + } + ]) + }) + + it('recovers when the wall clock steps backwards', () => { + vi.useFakeTimers() + vi.setSystemTime(600_000) + useImmediateAnimationFrames() + const manager = registerManager() + + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + vi.setSystemTime(60_000) + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + + expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(2) + }) + + it('keeps one-shot paste and reveal recovery outside the streaming budget', () => { + vi.useFakeTimers() + vi.setSystemTime(0) + useImmediateAnimationFrames() + const manager = registerManager() + + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + scheduleTerminalWebglAtlasRecovery() + vi.advanceTimersByTime(TERMINAL_OUTPUT_RECOVERY_QUIET_MS) + manager.resetWebglTextureAtlases.mockClear() + + scheduleTabRevealWebglAtlasRecovery() + vi.advanceTimersByTime(500) + expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(3) + + manager.resetWebglTextureAtlases.mockClear() + scheduleImagePasteWebglAtlasRecovery() + vi.advanceTimersByTime(500) + expect(manager.resetWebglTextureAtlases).toHaveBeenCalledTimes(3) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery.test.ts b/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery.test.ts index 93c696719..69d5df93f 100644 --- a/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery.test.ts +++ b/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery.test.ts @@ -4,6 +4,7 @@ import { unregisterLivePaneManager } from '@/lib/pane-manager/pane-manager-registry' import { + resetTerminalWebglAtlasRecoveryBudgetForTesting, scheduleImagePasteWebglAtlasRecovery, scheduleTabRevealWebglAtlasRecovery, scheduleTerminalWebglAtlasRecovery, @@ -16,10 +17,12 @@ describe('terminal WebGL atlas recovery', () => { function registerManager(): { resetWebglTextureAtlases: Mock<() => void> refreshAllPanes: Mock<() => void> + scheduleRevealPresent: Mock<() => void> } { const manager = { resetWebglTextureAtlases: vi.fn<() => void>(), - refreshAllPanes: vi.fn<() => void>() + refreshAllPanes: vi.fn<() => void>(), + scheduleRevealPresent: vi.fn<() => void>() } registerLivePaneManager(manager) registeredManagers.push(manager) @@ -30,9 +33,7 @@ describe('terminal WebGL atlas recovery', () => { for (const manager of registeredManagers.splice(0)) { unregisterLivePaneManager(manager) } - // Why: a test can leave the module-global debounce timer armed; clear the - // fake-timer queue before restoring real timers so no pending fire leaks - // into a later test. + resetTerminalWebglAtlasRecoveryBudgetForTesting() vi.clearAllTimers() vi.useRealTimers() vi.unstubAllGlobals() diff --git a/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery.ts b/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery.ts index 1912e0c0e..d7d54dc09 100644 --- a/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery.ts +++ b/src/renderer/src/components/terminal-pane/terminal-webgl-atlas-recovery.ts @@ -1,4 +1,8 @@ -import { resetAndRefreshAllTerminalWebglAtlases } from '@/lib/pane-manager/pane-manager-registry' +import { + presentAllTerminalPanesWithoutAtlasClear, + resetAndRefreshAllTerminalWebglAtlases +} from '@/lib/pane-manager/pane-manager-registry' +import { recordTerminalWebglDiagnostic } from '../../../../shared/terminal-webgl-diagnostics' const ATLAS_RECOVERY_DELAYS_MS = [120, 500] @@ -7,7 +11,18 @@ const ATLAS_RECOVERY_DELAYS_MS = [120, 500] // (STA-1365). Wait for output to go quiet so recovery runs once, on settle. export const TERMINAL_OUTPUT_RECOVERY_QUIET_MS = 200 +// Bound reset storms while preserving prompt recovery and steady repair attempts. +const TERMINAL_OUTPUT_RECOVERY_MIN_INTERVAL_MS = 3_000 +const TERMINAL_OUTPUT_RECOVERY_BURST_WIPES = 10 +const TERMINAL_OUTPUT_RECOVERY_REFILL_INTERVAL_MS = 6_000 + let terminalOutputRecoveryDebounceTimer: ReturnType | null = null +let terminalOutputRecoveryRetryTimer: ReturnType | null = null +let terminalOutputRecoveryWipeTokens = TERMINAL_OUTPUT_RECOVERY_BURST_WIPES +let terminalOutputRecoveryTokensRefilledAt: number | null = null +let terminalOutputRecoveryLastWipeAt: number | null = null +let terminalOutputRecoveryAttemptsSinceLastWipe = 0 +let terminalOutputRecoverySuppressedSinceLastWipe = 0 function scheduleNextFrame(callback: () => void): void { if (typeof globalThis.requestAnimationFrame === 'function') { @@ -17,20 +32,28 @@ function scheduleNextFrame(callback: () => void): void { globalThis.setTimeout(callback, 0) } -function resetAtlasesAndRefreshPanes(): void { +function resetAtlasesAndRefreshPanes(reason: string): void { try { // Why: the glyph atlas is shared across same-config terminals, so the // recovery reset must be followed by repainting each rebuilt render model. - resetAndRefreshAllTerminalWebglAtlases() + resetAndRefreshAllTerminalWebglAtlases(reason) } catch { /* ignore - terminal pane may have unmounted after scheduling recovery */ } } -function scheduleAtlasRecoveryBurst(): void { - scheduleNextFrame(() => resetAtlasesAndRefreshPanes()) +function presentPanesWithoutAtlasClear(): void { + try { + presentAllTerminalPanesWithoutAtlasClear() + } catch { + /* ignore - terminal pane may have unmounted after scheduling recovery */ + } +} + +function scheduleAtlasRecoveryBurst(reason: string): void { + scheduleNextFrame(() => resetAtlasesAndRefreshPanes(reason)) for (const delayMs of ATLAS_RECOVERY_DELAYS_MS) { - globalThis.setTimeout(() => resetAtlasesAndRefreshPanes(), delayMs) + globalThis.setTimeout(() => resetAtlasesAndRefreshPanes(reason), delayMs) } } @@ -38,13 +61,13 @@ export function scheduleImagePasteWebglAtlasRecovery(): void { // Why: image chips can redraw after bracketed paste parsing, so cover the // short post-paste paint window with a few cheap atlas rebuilds. Paste is a // one-shot event, so recover immediately rather than debouncing. - scheduleAtlasRecoveryBurst() + scheduleAtlasRecoveryBurst('image-paste') } export function scheduleTabRevealWebglAtlasRecovery(): void { // Why: a tab reveal is one-shot, so recover immediately — decoupled from the // streaming debounce so a background stream can't defer a revealed tab's rebuild. - scheduleAtlasRecoveryBurst() + scheduleAtlasRecoveryBurst('tab-reveal') } export function scheduleTerminalWebglAtlasRecovery(): void { @@ -54,8 +77,101 @@ export function scheduleTerminalWebglAtlasRecovery(): void { if (terminalOutputRecoveryDebounceTimer != null) { globalThis.clearTimeout(terminalOutputRecoveryDebounceTimer) } + if (terminalOutputRecoveryRetryTimer != null) { + globalThis.clearTimeout(terminalOutputRecoveryRetryTimer) + terminalOutputRecoveryRetryTimer = null + } terminalOutputRecoveryDebounceTimer = globalThis.setTimeout(() => { terminalOutputRecoveryDebounceTimer = null - resetAtlasesAndRefreshPanes() + terminalOutputRecoveryAttemptsSinceLastWipe += 1 + const decision = consumeTerminalOutputRecoveryWipeBudget() + if (!decision.allowed) { + terminalOutputRecoverySuppressedSinceLastWipe += 1 + presentPanesWithoutAtlasClear() + scheduleTerminalOutputRecoveryRetry(decision.retryAfterMs) + return + } + resetTerminalOutputAtlases(decision.intervalMs) }, TERMINAL_OUTPUT_RECOVERY_QUIET_MS) } + +function scheduleTerminalOutputRecoveryRetry(delayMs: number): void { + terminalOutputRecoveryRetryTimer = globalThis.setTimeout(() => { + terminalOutputRecoveryRetryTimer = null + const decision = consumeTerminalOutputRecoveryWipeBudget() + if (!decision.allowed) { + scheduleTerminalOutputRecoveryRetry(decision.retryAfterMs) + return + } + resetTerminalOutputAtlases(decision.intervalMs) + }, delayMs) +} + +function resetTerminalOutputAtlases(intervalMs: number): void { + recordTerminalWebglDiagnostic('webgl-atlas-reset-rate', { + reason: 'terminal-output', + attemptsSinceLastReset: terminalOutputRecoveryAttemptsSinceLastWipe, + atlasResetsSuppressed: terminalOutputRecoverySuppressedSinceLastWipe, + intervalMs + }) + terminalOutputRecoveryAttemptsSinceLastWipe = 0 + terminalOutputRecoverySuppressedSinceLastWipe = 0 + resetAtlasesAndRefreshPanes('terminal-output') +} + +export function resetTerminalWebglAtlasRecoveryBudgetForTesting(): void { + if (terminalOutputRecoveryDebounceTimer != null) { + globalThis.clearTimeout(terminalOutputRecoveryDebounceTimer) + } + terminalOutputRecoveryDebounceTimer = null + if (terminalOutputRecoveryRetryTimer != null) { + globalThis.clearTimeout(terminalOutputRecoveryRetryTimer) + } + terminalOutputRecoveryRetryTimer = null + terminalOutputRecoveryWipeTokens = TERMINAL_OUTPUT_RECOVERY_BURST_WIPES + terminalOutputRecoveryTokensRefilledAt = null + terminalOutputRecoveryLastWipeAt = null + terminalOutputRecoveryAttemptsSinceLastWipe = 0 + terminalOutputRecoverySuppressedSinceLastWipe = 0 +} + +function refillTerminalOutputRecoveryWipeTokens(now: number): void { + const elapsedMs = now - (terminalOutputRecoveryTokensRefilledAt ?? now) + terminalOutputRecoveryTokensRefilledAt = now + // Sleep, NTP, and remote clock corrections must not wedge recovery. + if (elapsedMs < 0) { + terminalOutputRecoveryWipeTokens = TERMINAL_OUTPUT_RECOVERY_BURST_WIPES + terminalOutputRecoveryLastWipeAt = null + return + } + terminalOutputRecoveryWipeTokens = Math.min( + TERMINAL_OUTPUT_RECOVERY_BURST_WIPES, + terminalOutputRecoveryWipeTokens + elapsedMs / TERMINAL_OUTPUT_RECOVERY_REFILL_INTERVAL_MS + ) +} + +function consumeTerminalOutputRecoveryWipeBudget(): { + allowed: boolean + intervalMs: number + retryAfterMs: number +} { + const now = Date.now() + refillTerminalOutputRecoveryWipeTokens(now) + const intervalMs = + terminalOutputRecoveryLastWipeAt == null ? 0 : now - terminalOutputRecoveryLastWipeAt + const intervalBudgetMs = + terminalOutputRecoveryLastWipeAt == null + ? 0 + : Math.max(0, TERMINAL_OUTPUT_RECOVERY_MIN_INTERVAL_MS - intervalMs) + const tokenBudgetMs = + terminalOutputRecoveryWipeTokens >= 1 + ? 0 + : (1 - terminalOutputRecoveryWipeTokens) * TERMINAL_OUTPUT_RECOVERY_REFILL_INTERVAL_MS + const retryAfterMs = Math.ceil(Math.max(intervalBudgetMs, tokenBudgetMs)) + if (retryAfterMs > 0) { + return { allowed: false, intervalMs, retryAfterMs } + } + terminalOutputRecoveryWipeTokens -= 1 + terminalOutputRecoveryLastWipeAt = now + return { allowed: true, intervalMs, retryAfterMs: 0 } +} diff --git a/src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts b/src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts index fa21b6ee8..10beddbf8 100644 --- a/src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts +++ b/src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts @@ -3,6 +3,7 @@ import { collectRendererMemoryProfileCounts } from '../renderer-memory-profile' import { forEachLivePaneForDesyncSentinel, getLivePaneCensus, + presentAllTerminalPanesWithoutAtlasClear, getLivePaneMemoryProfileCounts, refitAndRefreshAllTerminalPanes, registerLivePaneManager, @@ -136,6 +137,29 @@ describe('pane manager registry', () => { expect(healthy.refreshAllPanes).toHaveBeenCalledTimes(1) }) + it('presents visible managers without resetting their atlases', () => { + const visible = { + resetWebglTextureAtlases: vi.fn<() => void>(), + scheduleRevealPresent: vi.fn<() => void>(), + isVisibleForAtlasRecovery: () => true + } + const hidden = { + resetWebglTextureAtlases: vi.fn<() => void>(), + scheduleRevealPresent: vi.fn<() => void>(), + isVisibleForAtlasRecovery: () => false + } + registerLivePaneManager(visible) + registeredManagers.push(visible) + registerLivePaneManager(hidden) + registeredManagers.push(hidden) + + presentAllTerminalPanesWithoutAtlasClear() + + expect(visible.scheduleRevealPresent).toHaveBeenCalledOnce() + expect(visible.resetWebglTextureAtlases).not.toHaveBeenCalled() + expect(hidden.scheduleRevealPresent).not.toHaveBeenCalled() + }) + it('fits and refreshes every registered manager', () => { const first = { resetWebglTextureAtlases: vi.fn<() => void>(), diff --git a/src/renderer/src/lib/pane-manager/pane-manager-registry.ts b/src/renderer/src/lib/pane-manager/pane-manager-registry.ts index 832f506f5..588970436 100644 --- a/src/renderer/src/lib/pane-manager/pane-manager-registry.ts +++ b/src/renderer/src/lib/pane-manager/pane-manager-registry.ts @@ -10,6 +10,7 @@ type RegisteredPaneManager = { getPanes?: (limit?: number) => { id: number; terminal: unknown }[] getPaneCount?: () => number isVisibleForAtlasRecovery?: () => boolean + scheduleRevealPresent?: () => void } const liveManagers = new Set() @@ -78,6 +79,19 @@ export function resetAndRefreshAllTerminalWebglAtlases(reason?: string): void { } } +export function presentAllTerminalPanesWithoutAtlasClear(): void { + for (const manager of liveManagers) { + if (manager.isVisibleForAtlasRecovery?.() === false) { + continue + } + try { + manager.scheduleRevealPresent?.() + } catch { + // A disposing manager must not block sibling panes from presenting. + } + } +} + /** * Per-pane WebGL renderer state across all live managers, for the one-paste * freeze report. Lets a post-wake garble report show, per pane, whether it