fix(renderer): raise renderer V8 heap toward the 4GB pointer-compression cage (#7531)

* fix(renderer): raise renderer V8 heap toward the 4GB pointer-compression cage

Renderer OOM ('renderer crashed'/'oom', exit 5 / 0xE0000008 / SIGTRAP) is the
dominant crash in the crash channel: the renderer JS heap reaches Chromium's
default V8 old-space ceiling (~RAM/4) and V8 aborts. Two adversarial leak hunts
(13 agents across every renderer subsystem) found no unbounded GB-scale leak, so
this is a capacity ceiling, not a leak.

Chromium sizes the renderer heap at ~RAM/4, leaving 8-15GB machines well under
V8's ~4GB pointer-compression cage (an 8GB machine caps near 2.2GB). Reclaim that
unused headroom via --max-old-space-size in a focused startup module, gated on
physical RAM (>=8GB, ~40% of RAM, floor 3072MB, capped at the real 4096MB cage).
16GB+ machines are already at the cage so this is a no-op for them; low-RAM
machines keep the default to avoid trading a clean OOM for OS memory-pressure
kills.

Overridable with ORCA_RENDERER_HEAP_MB (number to force, default/off/0 to opt
out). Verified on Electron 42.3.3: the main-process js-flags switch propagates to
the renderer V8 and is honored up to the 4096MB cage (5000/12288 -> 4096).

Co-authored-by: Orca <help@stably.ai>

* fix(renderer): address CodeRabbit — floor-to-0 override + Linux 8GB gate

- parseRendererHeapOverrideMb: a fractional override in (0,1) floored to 0 and
  emitted an invalid --max-old-space-size=0; treat floored-to-0 as an opt-out.
- Lower the RAM gate from 8 to 7.5 GiB: os.totalmem() on Linux reports MemTotal
  (excludes kernel/firmware-reserved RAM), so a real 8 GB box reports ~7.7 GiB
  and was wrongly excluded from the headroom — the exact crashing population.
  7.5 still cleanly excludes 6 GB machines (report ~5.7 GiB).

Co-authored-by: Orca <help@stably.ai>

---------

Co-authored-by: Orca <help@stably.ai>

src/main/index.ts

@@ -62,6 +62,7 @@ import {
   patchPackagedProcessPath,
   shouldInstallManagedHooks
 } from './startup/configure-process'
+import { enableRendererHeapHeadroom } from './startup/renderer-heap-headroom'
 import { ensureVirtualDisplayForHeadlessServe } from './startup/ensure-virtual-display'
 import {
   readActiveGpuFallbackMarker,
@@ -585,6 +586,7 @@ if (hasSingleInstanceLock) {
     platform: process.platform
   })
   configureElectronNetworkCompatibility()
+  enableRendererHeapHeadroom()
   maybeApplyGpuFallbackForThisLaunch()
   if (!gpuFallbackActiveThisLaunch) {
     enableMainProcessGpuFeatures()

src/main/startup/renderer-heap-headroom.test.ts

@@ -0,0 +1,121 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { computeRendererHeapCeilingMb } from './renderer-heap-headroom'
+
+vi.mock('electron', () => ({
+  app: {
+    commandLine: {
+      appendSwitch: vi.fn(),
+      getSwitchValue: vi.fn(() => '')
+    }
+  }
+}))
+
+afterEach(() => {
+  vi.restoreAllMocks()
+})
+
+const GIB = 1024 * 1024 * 1024
+
+describe('computeRendererHeapCeilingMb', () => {
+  it('leaves Chromium default (null) below the ~8 GB gate to avoid OS memory pressure', () => {
+    expect(computeRendererHeapCeilingMb(4 * GIB)).toBeNull()
+    expect(computeRendererHeapCeilingMb(6 * GIB)).toBeNull() // 6 GB reports ~5.7 GiB
+    expect(computeRendererHeapCeilingMb(7 * GIB)).toBeNull() // below the 7.5 GiB gate
+  })
+
+  it('includes 8 GB machines that report below 8 GiB (Linux MemTotal excludes reserved RAM)', () => {
+    // A real 8 GB Linux box reports ~7.7 GiB; it must still get the headroom.
+    expect(computeRendererHeapCeilingMb(7.7 * GIB)).toBe(3072)
+    expect(computeRendererHeapCeilingMb(7.5 * GIB)).toBe(3072)
+  })
+
+  it('raises the ceiling toward the 4 GB pointer-compression cage, floored and capped', () => {
+    expect(computeRendererHeapCeilingMb(8 * GIB)).toBe(3072) // floor: 8 GB default ~2.2 GB -> 3072
+    expect(computeRendererHeapCeilingMb(12 * GIB)).toBe(4096) // 0.4*12 -> 4096 (cage)
+    expect(computeRendererHeapCeilingMb(16 * GIB)).toBe(4096) // cage cap
+    expect(computeRendererHeapCeilingMb(128 * GIB)).toBe(4096) // cage cap, never higher
+  })
+
+  it('honors a positive ORCA_RENDERER_HEAP_MB override regardless of RAM', () => {
+    expect(computeRendererHeapCeilingMb(4 * GIB, '5000')).toBe(5000)
+    expect(computeRendererHeapCeilingMb(128 * GIB, '4096')).toBe(4096)
+  })
+
+  it('opts out (null) for default/off/none/0/negative overrides', () => {
+    for (const value of ['default', 'off', 'none', '0', '-1']) {
+      expect(computeRendererHeapCeilingMb(16 * GIB, value)).toBeNull()
+    }
+  })
+
+  it('opts out (null) for a fractional override that would floor to 0 (never emits max-old-space-size=0)', () => {
+    for (const value of ['0.5', '0.9', '0.0001']) {
+      expect(computeRendererHeapCeilingMb(16 * GIB, value)).toBeNull()
+    }
+  })
+
+  it('falls through to RAM tiers for blank/invalid overrides', () => {
+    expect(computeRendererHeapCeilingMb(16 * GIB, '')).toBe(4096)
+    expect(computeRendererHeapCeilingMb(16 * GIB, 'abc')).toBe(4096)
+  })
+
+  it('returns null for a non-finite / non-positive RAM reading', () => {
+    expect(computeRendererHeapCeilingMb(Number.NaN)).toBeNull()
+    expect(computeRendererHeapCeilingMb(0)).toBeNull()
+  })
+})
+
+describe('enableRendererHeapHeadroom', () => {
+  it('appends --max-old-space-size as a js-flags switch on a RAM-capable machine', async () => {
+    const { app } = await import('electron')
+    const { enableRendererHeapHeadroom } = await import('./renderer-heap-headroom')
+
+    vi.mocked(app.commandLine.appendSwitch).mockClear()
+    vi.mocked(app.commandLine.getSwitchValue).mockReturnValue('')
+
+    enableRendererHeapHeadroom({ totalMemoryBytes: 16 * GIB, env: {} })
+
+    expect(app.commandLine.appendSwitch).toHaveBeenCalledWith(
+      'js-flags',
+      '--max-old-space-size=4096'
+    )
+  })
+
+  it('does not set a switch on low-RAM machines', async () => {
+    const { app } = await import('electron')
+    const { enableRendererHeapHeadroom } = await import('./renderer-heap-headroom')
+
+    vi.mocked(app.commandLine.appendSwitch).mockClear()
+    vi.mocked(app.commandLine.getSwitchValue).mockReturnValue('')
+
+    enableRendererHeapHeadroom({ totalMemoryBytes: 4 * GIB, env: {} })
+
+    expect(app.commandLine.appendSwitch).not.toHaveBeenCalledWith('js-flags', expect.anything())
+  })
+
+  it('preserves an explicit prior --max-old-space-size instead of stacking a second value', async () => {
+    const { app } = await import('electron')
+    const { enableRendererHeapHeadroom } = await import('./renderer-heap-headroom')
+
+    vi.mocked(app.commandLine.appendSwitch).mockClear()
+    vi.mocked(app.commandLine.getSwitchValue).mockReturnValue('--max-old-space-size=2048')
+
+    enableRendererHeapHeadroom({ totalMemoryBytes: 16 * GIB, env: {} })
+
+    expect(app.commandLine.appendSwitch).not.toHaveBeenCalledWith('js-flags', expect.anything())
+  })
+
+  it('merges with an unrelated existing js-flags value', async () => {
+    const { app } = await import('electron')
+    const { enableRendererHeapHeadroom } = await import('./renderer-heap-headroom')
+
+    vi.mocked(app.commandLine.appendSwitch).mockClear()
+    vi.mocked(app.commandLine.getSwitchValue).mockReturnValue('--no-opt')
+
+    enableRendererHeapHeadroom({ totalMemoryBytes: 16 * GIB, env: {} })
+
+    expect(app.commandLine.appendSwitch).toHaveBeenCalledWith(
+      'js-flags',
+      '--no-opt --max-old-space-size=4096'
+    )
+  })
+})

src/main/startup/renderer-heap-headroom.ts

@@ -0,0 +1,102 @@
+import { app } from 'electron'
+import { totalmem } from 'node:os'
+
+const RENDERER_HEAP_ENV_VAR = 'ORCA_RENDERER_HEAP_MB'
+const BYTES_PER_GIB = 1024 * 1024 * 1024
+// Why: Chromium sizes the renderer's V8 old-space heap from a physical-memory
+// heuristic (~RAM/4), so an 8 GB machine caps the renderer near ~2.2 GB even
+// though V8's pointer-compression cage allows up to ~4 GB. Heavy Orca sessions
+// (many agent terminals × scrollback, PR/git caches, React tree) legitimately
+// reach that low default and V8 aborts with an OOM — the dominant renderer
+// crash in the crash channel. Reclaim the unused headroom up to the 4 GB cage
+// on machines that have the RAM. Requests above 4096 are silently capped by the
+// pointer-compression cage, so 4096 is the real ceiling — we cannot go higher.
+// Machines below ~8 GB keep Chromium's default: raising their ceiling would
+// trade a clean OOM for OS memory-pressure kills / swap thrash.
+// Why 7.5 not 8: os.totalmem() on Linux reports MemTotal, which excludes
+// kernel/firmware-reserved RAM, so a real 8 GB machine reports ~7.7 GiB. Gating
+// at exactly 8 would wrongly exclude 8 GB Linux boxes (a crashing population)
+// while still cleanly excluding 6 GB machines (which report ~5.7 GiB).
+const RENDERER_HEAP_MIN_TOTAL_GIB = 7.5
+const RENDERER_HEAP_RAM_FRACTION = 0.4
+const RENDERER_HEAP_FLOOR_MB = 3072
+// V8 pointer-compression cage hard limit; --max-old-space-size above this is ignored.
+const RENDERER_HEAP_CAP_MB = 4096
+
+type HeapOverride = number | 'disable' | undefined
+
+function parseRendererHeapOverrideMb(value: string | undefined): HeapOverride {
+  if (value === undefined) {
+    return undefined
+  }
+  const normalized = value.trim().toLowerCase()
+  if (normalized === '') {
+    return undefined
+  }
+  // Why: give operators an explicit opt-out (and E2E a way to pin the default)
+  // without editing the RAM tiers.
+  if (normalized === 'default' || normalized === 'off' || normalized === 'none') {
+    return 'disable'
+  }
+  const parsed = Number(normalized)
+  // Why: ignore an unparseable value (typo) and fall through to the RAM tiers,
+  // but treat an explicit non-positive number as an opt-out.
+  if (!Number.isFinite(parsed)) {
+    return undefined
+  }
+  if (parsed <= 0) {
+    return 'disable'
+  }
+  // Why: a fractional value in (0,1) floors to 0, which would emit an invalid
+  // --max-old-space-size=0. Treat a floored-to-0 override as an opt-out too.
+  const flooredMb = Math.floor(parsed)
+  return flooredMb <= 0 ? 'disable' : flooredMb
+}
+
+/**
+ * Renderer V8 old-space ceiling (MB) to request via --max-old-space-size, or
+ * null to keep Chromium's physical-memory default. Pure so the RAM tiers and
+ * the env override are unit-testable without spawning Electron.
+ */
+export function computeRendererHeapCeilingMb(
+  totalMemoryBytes: number,
+  envOverride?: string
+): number | null {
+  const override = parseRendererHeapOverrideMb(envOverride)
+  if (override === 'disable') {
+    return null
+  }
+  if (typeof override === 'number') {
+    return override
+  }
+  if (!Number.isFinite(totalMemoryBytes) || totalMemoryBytes <= 0) {
+    return null
+  }
+  const totalGib = totalMemoryBytes / BYTES_PER_GIB
+  if (totalGib < RENDERER_HEAP_MIN_TOTAL_GIB) {
+    return null
+  }
+  const targetMb = Math.floor(totalGib * RENDERER_HEAP_RAM_FRACTION) * 1024
+  return Math.min(RENDERER_HEAP_CAP_MB, Math.max(RENDERER_HEAP_FLOOR_MB, targetMb))
+}
+
+export function enableRendererHeapHeadroom(
+  options: { totalMemoryBytes?: number; env?: NodeJS.ProcessEnv } = {}
+): void {
+  const totalMemoryBytes = options.totalMemoryBytes ?? totalmem()
+  const envOverride = (options.env ?? process.env)[RENDERER_HEAP_ENV_VAR]
+  const ceilingMb = computeRendererHeapCeilingMb(totalMemoryBytes, envOverride)
+  if (ceilingMb === null) {
+    return
+  }
+  const existing = app.commandLine.getSwitchValue('js-flags')
+  // Why: respect an explicit --max-old-space-size someone already set (e.g. via
+  // ELECTRON_EXTRA_LAUNCH_ARGS) instead of stacking a second, ignored value.
+  if (existing.includes('--max-old-space-size')) {
+    return
+  }
+  const flag = `--max-old-space-size=${ceilingMb}`
+  // Why: js-flags is process-wide and must be set before app 'ready' so it
+  // reaches renderer/utility V8 isolates when Chromium spawns them.
+  app.commandLine.appendSwitch('js-flags', existing ? `${existing} ${flag}` : flag)
+}