From f0443c326ad3e1e29b5ff5d5690b5a85138db513 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:06:29 -0700 Subject: [PATCH] fix(codex): recover interrupted state DB backfills (#12617) * fix(codex): recover interrupted state DB backfills * fix(codex): detect mixed-case backfill timeout * fix(codex): harden backfill recovery review findings * fix(codex): keep process identity retries safe --- config/tsconfig.cli.json | 2 + .../agent-hooks/managed-hook-install-lock.ts | 15 +- .../managed-hook-owner-identity.test.ts | 53 ++++ .../managed-hook-owner-identity.ts | 83 ++++- src/main/codex/codex-hook-trust-grant.test.ts | 16 + src/main/codex/codex-hook-trust-grant.ts | 9 +- .../codex-state-db-backfill-recovery.test.ts | 196 ++++++++++++ .../codex/codex-state-db-backfill-recovery.ts | 288 ++++++++++++++++++ src/main/codex/codex-state-db.test.ts | 76 +++++ src/main/codex/codex-state-db.ts | 115 +++++++ src/main/index.ts | 7 + src/main/ipc/pty.test.ts | 44 ++- src/main/ipc/pty.ts | 7 + src/main/rate-limits/codex-fetcher.test.ts | 36 ++- src/main/rate-limits/codex-fetcher.ts | 15 + .../codex-backfill-error-detector.test.ts | 21 ++ .../codex-backfill-error-detector.ts | 33 ++ .../terminal-pane/pty-connection.test.ts | 25 ++ .../terminal-pane/pty-connection.ts | 9 + 19 files changed, 1031 insertions(+), 19 deletions(-) create mode 100644 src/main/codex/codex-state-db-backfill-recovery.test.ts create mode 100644 src/main/codex/codex-state-db-backfill-recovery.ts create mode 100644 src/main/codex/codex-state-db.test.ts create mode 100644 src/main/codex/codex-state-db.ts create mode 100644 src/renderer/src/components/terminal-pane/codex-backfill-error-detector.test.ts create mode 100644 src/renderer/src/components/terminal-pane/codex-backfill-error-detector.ts diff --git a/config/tsconfig.cli.json b/config/tsconfig.cli.json index 386fb6a15..14295cb47 100644 --- a/config/tsconfig.cli.json +++ b/config/tsconfig.cli.json @@ -35,6 +35,7 @@ "../src/main/codex/codex-hook-trust-grant.ts", "../src/main/codex/codex-managed-trust-reconciliation.ts", "../src/main/codex/codex-process-exit-deadline.ts", + "../src/main/codex/codex-state-db.ts", "../src/main/codex/codex-trust-config-rollback.ts", "../src/main/codex/codex-trust-grant-telemetry.ts", "../src/main/codex/codex-trust-grant-host.ts", @@ -77,6 +78,7 @@ // project's serve spec; the module has no imports, so listing it pulls in nothing else. "../src/main/startup/serve-mode-argv.ts", "../src/main/runtime/runtime-metadata.ts", + "../src/main/sqlite/sync-database.ts", "../src/main/win32-utils.ts" ], "compilerOptions": { diff --git a/src/main/agent-hooks/managed-hook-install-lock.ts b/src/main/agent-hooks/managed-hook-install-lock.ts index c283b99c4..41dd21f9b 100644 --- a/src/main/agent-hooks/managed-hook-install-lock.ts +++ b/src/main/agent-hooks/managed-hook-install-lock.ts @@ -46,7 +46,8 @@ async function releaseInstallLock( async function acquireInstallLock( home: string, signal?: AbortSignal, - suppliedHostIdentity?: string + suppliedHostIdentity?: string, + waitTimeoutMs = LOCK_WAIT_TIMEOUT_MS ): Promise<() => Promise> { const lockParent = join(home, '.orca') const lockPath = join(lockParent, 'managed-hook-install.lock') @@ -57,7 +58,7 @@ async function acquireInstallLock( if (typeof processIdentity !== 'string') { throw new Error('Could not identify the managed-hook installer process') } - const deadline = Date.now() + LOCK_WAIT_TIMEOUT_MS + const deadline = Date.now() + waitTimeoutMs while (true) { signal?.throwIfAborted() @@ -130,9 +131,15 @@ export async function withManagedHookInstallLock( home: string, signal: AbortSignal | undefined, run: () => Promise, - hostIdentity?: string + hostIdentity?: string, + options?: { waitTimeoutMs?: number } ): Promise { - const release = await acquireInstallLock(home, signal, hostIdentity) + const release = await acquireInstallLock( + home, + signal, + hostIdentity, + options?.waitTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS + ) try { signal?.throwIfAborted() return await run() diff --git a/src/main/agent-hooks/managed-hook-owner-identity.test.ts b/src/main/agent-hooks/managed-hook-owner-identity.test.ts index ae8ac15dd..9e8d5cc9b 100644 --- a/src/main/agent-hooks/managed-hook-owner-identity.test.ts +++ b/src/main/agent-hooks/managed-hook-owner-identity.test.ts @@ -67,6 +67,33 @@ async function loadLinuxIdentity(fixture: LinuxIdentityFixture) { return { identity: await import('./managed-hook-owner-identity'), readFile } } +async function loadWindowsIdentity() { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const execFileAsync = vi.fn(async (_file: string, args: string[]) => ({ + stdout: args.join(' ').includes('MachineGuid') + ? '\r\n MachineGuid REG_SZ AAAAAAAA-BBBB-4CCC-8DDD-EEEEEEEEEEEE\r\n' + : '1777777777000\r\n' + })) + vi.doMock('node:util', () => ({ promisify: () => execFileAsync })) + return { identity: await import('./managed-hook-owner-identity'), execFileAsync } +} + +async function loadDarwinIdentity() { + Object.defineProperty(process, 'platform', { configurable: true, value: 'darwin' }) + let psCalls = 0 + const execFileAsync = vi.fn(async (file: string) => { + if (file === 'sysctl') { + return { stdout: 'boot-session\n' } + } + if (file === 'ps' && ++psCalls === 1) { + throw new Error('transient ps failure') + } + return { stdout: 'Wed Aug 5 12:00:00 2026 node app\n' } + }) + vi.doMock('node:util', () => ({ promisify: () => execFileAsync })) + return await import('./managed-hook-owner-identity') +} + afterEach(() => { Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform }) if (originalGetuidDescriptor) { @@ -76,6 +103,8 @@ afterEach(() => { } vi.unstubAllEnvs() vi.doUnmock('node:fs/promises') + vi.doUnmock('node:child_process') + vi.doUnmock('node:util') vi.resetModules() }) @@ -178,4 +207,28 @@ describe('managed hook owner identity', () => { 'linux:pid:[4026533001]:current-boot-id:4242' ) }) + + it('uses machine and process creation identities on Windows', async () => { + const { identity, execFileAsync } = await loadWindowsIdentity() + + await expect(identity.readManagedHookHostIdentity()).resolves.toBe( + 'win32:aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee' + ) + await expect(identity.readManagedHookProcessIdentity(process.pid)).resolves.toBe( + `win32:${process.pid}:1777777777000` + ) + await expect(identity.readManagedHookProcessIdentity(process.pid)).resolves.toBe( + `win32:${process.pid}:1777777777000` + ) + expect(execFileAsync).toHaveBeenCalledTimes(2) + }) + + it('retries an unverified macOS process identity', async () => { + const identity = await loadDarwinIdentity() + + await expect(identity.readManagedHookProcessIdentity(process.pid)).resolves.toBeUndefined() + await expect(identity.readManagedHookProcessIdentity(process.pid)).resolves.toBe( + 'darwin:boot-session:Wed Aug 5 12:00:00 2026 node app' + ) + }) }) diff --git a/src/main/agent-hooks/managed-hook-owner-identity.ts b/src/main/agent-hooks/managed-hook-owner-identity.ts index 85941007e..7f60956e6 100644 --- a/src/main/agent-hooks/managed-hook-owner-identity.ts +++ b/src/main/agent-hooks/managed-hook-owner-identity.ts @@ -1,15 +1,30 @@ import { execFile } from 'node:child_process' import { randomUUID } from 'node:crypto' import { link, lstat, mkdir, readFile, readlink, unlink, writeFile } from 'node:fs/promises' -import { join } from 'node:path' +import { join, win32 } from 'node:path' import { promisify } from 'node:util' -const execFileAsync = promisify(execFile) const runtimeHostIdentity = `runtime:${randomUUID()}` const runtimeProcessIdentity = `runtime:${randomUUID()}` let hostIdentityPromise: Promise | undefined let bootIdentityPromise: Promise | undefined +let selfProcessIdentityPromise: Promise | undefined const HOST_TOKEN_PATTERN = /^[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/i +const WINDOWS_PROCESS_IDENTITY_TIMEOUT_MS = 5_000 + +function getWindowsPowerShellPath(): string { + return win32.join( + process.env.SystemRoot ?? 'C:\\Windows', + 'System32', + 'WindowsPowerShell', + 'v1.0', + 'powershell.exe' + ) +} + +function getWindowsRegistryPath(): string { + return win32.join(process.env.SystemRoot ?? 'C:\\Windows', 'System32', 'reg.exe') +} function hasCode(error: unknown, code: string): boolean { return error instanceof Error && 'code' in error && error.code === code @@ -100,12 +115,29 @@ async function readHostIdentity(): Promise { // but a later process gets a different identity and cannot steal its residue. return runtimeHostIdentity } + if (process.platform === 'win32') { + try { + const { stdout } = await promisify(execFile)( + getWindowsRegistryPath(), + ['query', 'HKLM\\SOFTWARE\\Microsoft\\Cryptography', '/v', 'MachineGuid'], + { encoding: 'utf8', timeout: 1_000, windowsHide: true } + ) + const machineGuid = /^\s*MachineGuid\s+REG_\w+\s+(.+?)\s*$/im.exec(stdout)?.[1] + return machineGuid ? `win32:${machineGuid.toLowerCase()}` : runtimeHostIdentity + } catch { + return runtimeHostIdentity + } + } if (process.platform === 'darwin') { try { - const { stdout } = await execFileAsync('ioreg', ['-rd1', '-c', 'IOPlatformExpertDevice'], { - encoding: 'utf8', - timeout: 1_000 - }) + const { stdout } = await promisify(execFile)( + 'ioreg', + ['-rd1', '-c', 'IOPlatformExpertDevice'], + { + encoding: 'utf8', + timeout: 1_000 + } + ) const platformId = /"IOPlatformUUID"\s*=\s*"([^"]+)"/.exec(stdout)?.[1] return platformId ? `darwin:${platformId}` : runtimeHostIdentity } catch { @@ -127,7 +159,7 @@ export async function readBootIdentity(): Promise { if (process.platform === 'darwin') { try { - const { stdout } = await execFileAsync('sysctl', ['-n', 'kern.bootsessionuuid'], { + const { stdout } = await promisify(execFile)('sysctl', ['-n', 'kern.bootsessionuuid'], { encoding: 'utf8', timeout: 1_000 }) @@ -160,6 +192,14 @@ export function scopeManagedHookHostIdentity( export async function readManagedHookProcessIdentity( pid: number ): Promise { + if (process.platform === 'win32' && pid === process.pid) { + selfProcessIdentityPromise ??= readProcessIdentity(pid) + return await selfProcessIdentityPromise + } + return await readProcessIdentity(pid) +} + +async function readProcessIdentity(pid: number): Promise { if (process.platform === 'linux') { try { const [statLine, pidNamespace, bootIdentity] = await Promise.all([ @@ -179,10 +219,37 @@ export async function readManagedHookProcessIdentity( } } + if (process.platform === 'win32') { + try { + const { stdout } = await promisify(execFile)( + getWindowsPowerShellPath(), + [ + '-NoProfile', + '-NonInteractive', + '-Command', + `$ErrorActionPreference = 'Stop'; ` + + `$p = Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}"; ` + + `if (!$p) { 'missing'; exit 0 }; ` + + `[string]([DateTimeOffset]$p.CreationDate).ToUnixTimeMilliseconds()` + ], + { encoding: 'utf8', timeout: WINDOWS_PROCESS_IDENTITY_TIMEOUT_MS, windowsHide: true } + ) + const startedAt = stdout.trim() + return startedAt === 'missing' ? null : startedAt ? `win32:${pid}:${startedAt}` : undefined + } catch { + try { + process.kill(pid, 0) + return pid === process.pid ? runtimeProcessIdentity : undefined + } catch (error) { + return hasCode(error, 'ESRCH') ? null : undefined + } + } + } + bootIdentityPromise ??= readBootIdentity() const bootIdentity = await bootIdentityPromise try { - const { stdout } = await execFileAsync( + const { stdout } = await promisify(execFile)( 'ps', ['-o', 'lstart=', '-o', 'command=', '-p', String(pid)], { diff --git a/src/main/codex/codex-hook-trust-grant.test.ts b/src/main/codex/codex-hook-trust-grant.test.ts index 9f065f9db..e1ca7c410 100644 --- a/src/main/codex/codex-hook-trust-grant.test.ts +++ b/src/main/codex/codex-hook-trust-grant.test.ts @@ -97,6 +97,22 @@ function grantedSessionResult(entries: CodexTrustEntry[], hashPrefix = 'sha256:c } describe('grantManagedCodexHookTrust', () => { + it('does not let a short trust RPC claim an incomplete session index', () => { + const sessions = join(runtimeHomeDir, 'sessions') + mkdirSync(sessions, { recursive: true }) + for (let index = 0; index < 100; index += 1) { + writeFileSync(join(sessions, `${index}.jsonl`), '{}\n') + } + const runner = vi.fn() + _internals.setGrantSessionRunnerSync(runner) + + expect(grantManagedCodexHookTrust(buildPlan([managedEntry('stop')]))).toMatchObject({ + lane: 'fallback', + reason: 'retry-cached' + }) + expect(runner).not.toHaveBeenCalled() + }) + it('returns granted entries with codex-verbatim hashes and records the ledger', () => { const entries = [managedEntry('session_start'), managedEntry('stop')] const runner = vi.fn((_request: CodexHookTrustGrantRequest) => grantedSessionResult(entries)) diff --git a/src/main/codex/codex-hook-trust-grant.ts b/src/main/codex/codex-hook-trust-grant.ts index 94eb3117b..9142a590a 100644 --- a/src/main/codex/codex-hook-trust-grant.ts +++ b/src/main/codex/codex-hook-trust-grant.ts @@ -35,6 +35,7 @@ import { resolveCodexTrustGrantHost, type CodexTrustGrantHost } from './codex-trust-grant-host' +import { isCodexStateDbBackfillPending } from './codex-state-db' // Why: a transiently hung app-server must not block launch prep on every pane. // The legacy lane remains available while a short, host-scoped cooldown runs. @@ -74,9 +75,7 @@ const diagnostics = { export type CodexTrustGrantDiagnostics = typeof diagnostics const transientRetryAfterByHost = new Map() -export function getCodexTrustGrantDiagnostics(): CodexTrustGrantDiagnostics { - return { ...diagnostics } -} +export const getCodexTrustGrantDiagnostics = (): CodexTrustGrantDiagnostics => ({ ...diagnostics }) type GrantSessionRunnerSync = ( request: CodexHookTrustGrantRequest @@ -192,6 +191,10 @@ export function grantManagedCodexHookTrust( diagnostics.ledgerHits += 1 return { lane: 'rpc', entries: ledgerEntries } } + if (isCodexStateDbBackfillPending(plan.runtimeHomePath)) { + // Why: a short trust RPC can refresh Codex's abandoned lease and strand every pane again. + return fallback(plan, 'retry-cached') + } const hostKey = getCodexAppServerHostKey(plan.host) if (!codexAppServerCapabilityCache.shouldTry(hostKey)) { diff --git a/src/main/codex/codex-state-db-backfill-recovery.test.ts b/src/main/codex/codex-state-db-backfill-recovery.test.ts new file mode 100644 index 000000000..72f25b25b --- /dev/null +++ b/src/main/codex/codex-state-db-backfill-recovery.test.ts @@ -0,0 +1,196 @@ +import { EventEmitter } from 'node:events' +import { link, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import * as ownerIdentity from '../agent-hooks/managed-hook-owner-identity' +import { + resolveCodexBackfillSupervisorLockRoot, + runCodexStateDbBackfillRecovery, + withCodexBackfillSupervisorLock +} from './codex-state-db-backfill-recovery' +import type { CodexStateDbBackfillStatus } from './codex-state-db' + +const temporaryRoots: string[] = [] +const originalPlatform = process.platform + +function createFakeChild(): EventEmitter & { + stdin: { end: ReturnType } + exitCode: number | null + signalCode: NodeJS.Signals | null + kill: ReturnType +} { + return Object.assign(new EventEmitter(), { + stdin: { end: vi.fn() }, + exitCode: null, + signalCode: null, + kill: vi.fn(() => true) + }) +} + +async function createTemporaryRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'orca-codex-backfill-recovery-')) + temporaryRoots.push(root) + return root +} + +afterEach(async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform }) + vi.restoreAllMocks() + vi.unstubAllEnvs() + await Promise.all( + temporaryRoots.splice(0).map((root) => rm(root, { recursive: true, force: true })) + ) +}) + +describe('Codex state DB backfill recovery', () => { + it('keeps the successful app-server claimant alive until Codex marks its DB complete', async () => { + const child = createFakeChild() + const terminate = vi.fn(async () => {}) + const readStatus = vi + .fn() + .mockReturnValueOnce({ kind: 'incomplete', stateDbPath: '/state.sqlite', status: 'running' }) + .mockReturnValue({ kind: 'complete', stateDbPath: '/state.sqlite' }) + + await expect( + runCodexStateDbBackfillRecovery('/managed-home', new AbortController().signal, { + spawnProcess: vi.fn(() => child) as never, + readStatus, + terminate, + sleep: vi.fn(async () => {}), + now: vi.fn(() => 1_000) + }) + ).resolves.toEqual({ outcome: 'completed', spawnCount: 1 }) + expect(terminate).toHaveBeenCalledWith(child) + }) + + it('retries a live foreign lease until one durable claimant can recover it', async () => { + const first = createFakeChild() + const second = createFakeChild() + const children = [first, second] + let now = 0 + let spawnCount = 0 + const spawnProcess = vi.fn(() => { + const child = children[spawnCount++] + if (child === first) { + queueMicrotask(() => child.emit('exit', 1, null)) + } + return child + }) + const readStatus = vi.fn( + (): CodexStateDbBackfillStatus => + spawnCount >= 2 + ? { kind: 'complete', stateDbPath: '/state.sqlite' } + : { kind: 'incomplete', stateDbPath: '/state.sqlite', status: 'running' } + ) + + await expect( + runCodexStateDbBackfillRecovery('/managed-home', new AbortController().signal, { + spawnProcess: spawnProcess as never, + readStatus, + terminate: vi.fn(async () => {}), + sleep: vi.fn(async (ms: number) => { + now += ms + await Promise.resolve() + }), + now: () => now + }) + ).resolves.toEqual({ outcome: 'completed', spawnCount: 2 }) + }) + + it('routes a WSL managed home through its distro and Linux CODEX_HOME', async () => { + Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' }) + const child = createFakeChild() + const spawnProcess = vi.fn(() => child) + const readStatus = vi + .fn() + .mockReturnValueOnce({ kind: 'incomplete', stateDbPath: 'state.sqlite', status: 'running' }) + .mockReturnValue({ kind: 'complete', stateDbPath: 'state.sqlite' }) + + await runCodexStateDbBackfillRecovery( + '\\\\wsl.localhost\\Ubuntu\\home\\alice\\.codex', + new AbortController().signal, + { + spawnProcess: spawnProcess as never, + readStatus, + terminate: vi.fn(async () => {}), + sleep: vi.fn(async () => {}), + now: vi.fn(() => 1_000) + } + ) + + expect(spawnProcess).toHaveBeenCalledWith( + 'wsl.exe', + expect.arrayContaining(['-d', 'Ubuntu']), + expect.objectContaining({ windowsHide: true }) + ) + const spawnCall = spawnProcess.mock.calls[0] as unknown as [string, string[]] + const command = spawnCall[1].join(' ') + expect(command).toContain('export CODEX_HOME=') + expect(command).toContain('/home/alice/.codex') + }) +}) + +describe.skipIf(process.platform === 'win32')('Codex backfill supervisor owner lock', () => { + it('recovers a dead owner whose PID was reused with a different start identity', async () => { + const userData = await createTemporaryRoot() + vi.stubEnv('ORCA_USER_DATA_PATH', userData) + const home = join(userData, 'managed-home') + const lockRoot = resolveCodexBackfillSupervisorLockRoot(home) + const lockParent = join(lockRoot, '.orca') + const token = '00000000-0000-4000-8000-000000000000' + const ownerPath = join(lockParent, `managed-hook-install.owner-${token}.json`) + const lockPath = join(lockParent, 'managed-hook-install.lock') + await mkdir(lockParent, { recursive: true }) + await writeFile( + ownerPath, + JSON.stringify({ + token, + pid: process.pid, + hostIdentity: await ownerIdentity.readManagedHookHostIdentity(), + processIdentity: 'stale-process-start-time' + }) + ) + await link(ownerPath, lockPath) + + await expect( + withCodexBackfillSupervisorLock(home, undefined, async () => 'recovered') + ).resolves.toBe('recovered') + }) + + it('does not interfere with a live supervisor from another Orca instance', async () => { + const userData = await createTemporaryRoot() + vi.stubEnv('ORCA_USER_DATA_PATH', userData) + const home = join(userData, 'managed-home') + let releaseFirst!: () => void + const first = withCodexBackfillSupervisorLock( + home, + undefined, + async () => await new Promise((resolve) => (releaseFirst = resolve)) + ) + await vi.waitFor(async () => { + await expect( + import('node:fs/promises').then(({ readFile }) => + readFile( + join( + resolveCodexBackfillSupervisorLockRoot(home), + '.orca', + 'managed-hook-install.lock' + ), + 'utf8' + ) + ) + ).resolves.toContain('processIdentity') + }) + const controller = new AbortController() + const secondRun = vi.fn(async () => {}) + setTimeout(() => controller.abort(), 40) + + await expect( + withCodexBackfillSupervisorLock(home, controller.signal, secondRun) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(secondRun).not.toHaveBeenCalled() + releaseFirst() + await first + }) +}) diff --git a/src/main/codex/codex-state-db-backfill-recovery.ts b/src/main/codex/codex-state-db-backfill-recovery.ts new file mode 100644 index 000000000..733a6b669 --- /dev/null +++ b/src/main/codex/codex-state-db-backfill-recovery.ts @@ -0,0 +1,288 @@ +import { spawn, type ChildProcess } from 'node:child_process' +import { createHash } from 'node:crypto' +import { join } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' +import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-path' +import { parseWslUncPath } from '../../shared/wsl-paths' +import { withManagedHookInstallLock } from '../agent-hooks/managed-hook-install-lock' +import { readManagedHookHostIdentity } from '../agent-hooks/managed-hook-owner-identity' +import { buildWslCodexAppServerArgs } from '../codex-accounts/wsl-codex-command' +import { resolveCodexCommand } from '../codex-cli/command' +import { terminateCodexProbeChild } from '../rate-limits/codex-probe-termination' +import { getSpawnArgsForWindows } from '../win32-utils' +import { getOrcaUserDataPath } from './codex-home-paths' +import { + BACKFILL_PENDING_MIN_SESSION_FILES, + countCodexSessionFilesUpTo, + isCodexStateDbBackfillPending, + readCodexStateDbBackfillStatus, + type CodexStateDbBackfillStatus +} from './codex-state-db' + +const RECOVERY_POLL_INTERVAL_MS = 5_000 +const RECOVERY_RETRY_DELAY_MS = 2_000 +const RECOVERY_FAST_EXIT_MS = 10_000 +const RECOVERY_MAX_FAST_FAILURES = 5 +const RECOVERY_MAX_TOTAL_MS = 60 * 60_000 +const RECOVERY_OWNER_CHECK_TIMEOUT_MS = 1_000 +const RECOVERY_CODEX_ARGS = ['-s', 'read-only', '-a', 'untrusted', 'app-server'] as const + +export type CodexStateDbBackfillRecoverySummary = { + outcome: + | 'completed' + | 'already-complete' + | 'not-needed' + | 'unreadable' + | 'stopped' + | 'gave-up' + | 'codex-unavailable' + spawnCount: number +} + +type RecoveryDependencies = { + spawnProcess: typeof spawn + resolveCommand: () => string + readStatus: (codexHomePath: string) => CodexStateDbBackfillStatus + countSessions: (sessionsRoot: string, limit: number) => number + now: () => number + sleep: (ms: number, signal: AbortSignal) => Promise + terminate: (child: ChildProcess) => Promise +} + +const defaultDependencies: RecoveryDependencies = { + spawnProcess: spawn, + resolveCommand: resolveCodexCommand, + readStatus: readCodexStateDbBackfillStatus, + countSessions: countCodexSessionFilesUpTo, + now: Date.now, + sleep: async (ms, signal) => await delay(ms, undefined, { signal }), + terminate: async (child) => await terminateCodexProbeChild(child) +} + +function finish( + outcome: CodexStateDbBackfillRecoverySummary['outcome'], + spawnCount: number +): CodexStateDbBackfillRecoverySummary { + return { outcome, spawnCount } +} + +function initialRecoveryDecision( + codexHomePath: string, + dependencies: RecoveryDependencies +): CodexStateDbBackfillRecoverySummary['outcome'] | null { + const status = dependencies.readStatus(codexHomePath) + if (status.kind === 'complete') { + return 'already-complete' + } + if (status.kind === 'unreadable') { + return 'unreadable' + } + if ( + (status.kind === 'missing' || status.kind === 'not-tracked') && + dependencies.countSessions( + join(codexHomePath, 'sessions'), + BACKFILL_PENDING_MIN_SESSION_FILES + ) < BACKFILL_PENDING_MIN_SESSION_FILES + ) { + return 'not-needed' + } + return null +} + +function spawnRecoveryProcess( + codexHomePath: string, + dependencies: RecoveryDependencies +): ChildProcess { + const wslHome = process.platform === 'win32' ? parseWslUncPath(codexHomePath) : null + if (wslHome) { + return dependencies.spawnProcess( + 'wsl.exe', + buildWslCodexAppServerArgs(wslHome.distro, wslHome.linuxPath), + { + stdio: ['pipe', 'ignore', 'ignore'], + windowsHide: true, + env: process.env + } + ) + } + const command = dependencies.resolveCommand() + const { spawnCmd, spawnArgs } = getSpawnArgsForWindows(command, [...RECOVERY_CODEX_ARGS]) + return dependencies.spawnProcess(spawnCmd, spawnArgs, { + cwd: codexHomePath, + stdio: ['pipe', 'ignore', 'ignore'], + windowsHide: true, + env: { ...process.env, CODEX_HOME: codexHomePath } + }) +} + +/** Keeps a sanctioned app-server claimant alive until Codex completes its own backfill. */ +export async function runCodexStateDbBackfillRecovery( + codexHomePath: string, + signal: AbortSignal, + dependenciesOverride: Partial = {} +): Promise { + const dependencies = { ...defaultDependencies, ...dependenciesOverride } + const initialOutcome = initialRecoveryDecision(codexHomePath, dependencies) + if (initialOutcome) { + return finish(initialOutcome, 0) + } + + const deadline = dependencies.now() + RECOVERY_MAX_TOTAL_MS + let spawnCount = 0 + let fastFailures = 0 + while (!signal.aborted && dependencies.now() < deadline) { + const spawnedAt = dependencies.now() + const child = spawnRecoveryProcess(codexHomePath, dependencies) + spawnCount += 1 + let childDown = false + let spawnFailed = false + let exitedAt = spawnedAt + child.once('error', () => { + childDown = true + spawnFailed = true + exitedAt = dependencies.now() + }) + child.once('exit', () => { + childDown = true + exitedAt = dependencies.now() + }) + + try { + while (!childDown && !signal.aborted && dependencies.now() < deadline) { + await dependencies.sleep(RECOVERY_POLL_INTERVAL_MS, signal) + const status = dependencies.readStatus(codexHomePath) + if (status.kind === 'complete') { + await dependencies.terminate(child) + return finish('completed', spawnCount) + } + if (status.kind === 'unreadable') { + await dependencies.terminate(child) + return finish('unreadable', spawnCount) + } + } + } catch (error) { + if (!signal.aborted) { + await dependencies.terminate(child) + throw error + } + } + + if (signal.aborted) { + await dependencies.terminate(child) + return finish('stopped', spawnCount) + } + if (!childDown) { + await dependencies.terminate(child) + return finish('gave-up', spawnCount) + } + if (spawnFailed && spawnCount === 1) { + return finish('codex-unavailable', spawnCount) + } + if (exitedAt - spawnedAt < RECOVERY_FAST_EXIT_MS) { + fastFailures += 1 + if (fastFailures >= RECOVERY_MAX_FAST_FAILURES) { + return finish('gave-up', spawnCount) + } + } + try { + await dependencies.sleep(RECOVERY_RETRY_DELAY_MS, signal) + } catch { + return finish('stopped', spawnCount) + } + } + return finish(signal.aborted ? 'stopped' : 'gave-up', spawnCount) +} + +export function resolveCodexBackfillSupervisorLockRoot(codexHomePath: string): string { + const homeKey = normalizeRuntimePathForComparison(codexHomePath) + const digest = createHash('sha256').update(homeKey).digest('hex') + return join(getOrcaUserDataPath(), 'codex-state-db-backfill-locks', digest) +} + +function scopeRecoveryHostIdentity(hostIdentity: string, codexHomePath: string): string { + const wslHome = process.platform === 'win32' ? parseWslUncPath(codexHomePath) : null + return wslHome ? `${hostIdentity}:wsl:${wslHome.distro.toLowerCase()}` : hostIdentity +} + +export async function withCodexBackfillSupervisorLock( + codexHomePath: string, + signal: AbortSignal | undefined, + run: () => Promise +): Promise { + const hostIdentity = scopeRecoveryHostIdentity(await readManagedHookHostIdentity(), codexHomePath) + // Reuse the crash-safe hard-link claim protocol; its storage root is Codex-specific. + return await withManagedHookInstallLock( + resolveCodexBackfillSupervisorLockRoot(codexHomePath), + signal, + run, + hostIdentity, + { waitTimeoutMs: RECOVERY_OWNER_CHECK_TIMEOUT_MS } + ) +} + +type ActiveRecovery = { + controller: AbortController + ready: Promise + task: Promise +} + +const activeRecoveries = new Map() +let stopping = false + +export function startCodexStateDbBackfillRecoveryInBackground( + codexHomePath: string +): Promise { + const key = normalizeRuntimePathForComparison(codexHomePath) + const existing = activeRecoveries.get(key) + if (existing) { + return existing.task + } + if (stopping || !isCodexStateDbBackfillPending(codexHomePath)) { + return Promise.resolve(null) + } + const controller = new AbortController() + let markReady!: () => void + const ready = new Promise((resolve) => (markReady = resolve)) + const task = withCodexBackfillSupervisorLock(codexHomePath, controller.signal, async () => { + console.info(`[codex-state-db-backfill] supervising Codex index at ${codexHomePath}`) + markReady() + return await runCodexStateDbBackfillRecovery(codexHomePath, controller.signal) + }).catch((error: unknown) => { + if (!controller.signal.aborted) { + console.warn('[codex-state-db-backfill] recovery supervisor stopped:', error) + } + return null + }) + void task.finally(markReady) + activeRecoveries.set(key, { controller, ready, task }) + void task.finally(() => { + if (activeRecoveries.get(key)?.task === task) { + activeRecoveries.delete(key) + } + }) + return task +} + +/** Waits only for exact-owner arbitration, never for the potentially long Codex index. */ +export async function ensureCodexStateDbBackfillRecoveryStarted( + codexHomePath: string +): Promise { + void startCodexStateDbBackfillRecoveryInBackground(codexHomePath) + await activeRecoveries.get(normalizeRuntimePathForComparison(codexHomePath))?.ready +} + +export async function stopCodexStateDbBackfillRecoveries(): Promise { + stopping = true + const recoveries = [...activeRecoveries.values()] + for (const recovery of recoveries) { + recovery.controller.abort() + } + await Promise.allSettled(recoveries.map(({ task }) => task)) +} + +export const _internals = { + resetForTests(): void { + stopping = false + activeRecoveries.clear() + } +} diff --git a/src/main/codex/codex-state-db.test.ts b/src/main/codex/codex-state-db.test.ts new file mode 100644 index 000000000..7cf0ac88a --- /dev/null +++ b/src/main/codex/codex-state-db.test.ts @@ -0,0 +1,76 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import SyncDatabase from '../sqlite/sync-database' +import { + findNewestCodexStateDbPath, + isCodexStateDbBackfillPending, + readCodexStateDbBackfillStatus +} from './codex-state-db' + +const temporaryHomes: string[] = [] + +async function createHome(): Promise { + const home = await mkdtemp(join(tmpdir(), 'orca-codex-state-db-')) + temporaryHomes.push(home) + return home +} + +function createBackfillDb(home: string, version: number, status: string): string { + const path = join(home, `state_${version}.sqlite`) + const db = new SyncDatabase(path) + db.exec( + 'CREATE TABLE backfill_state (id INTEGER PRIMARY KEY, status TEXT NOT NULL); ' + + `INSERT INTO backfill_state (id, status) VALUES (1, '${status}')` + ) + db.close() + return path +} + +afterEach(async () => { + await Promise.all( + temporaryHomes.splice(0).map((home) => rm(home, { recursive: true, force: true })) + ) +}) + +describe('Codex state DB backfill status', () => { + it('reads the newest schema DB without mutating an incomplete row', async () => { + const home = await createHome() + createBackfillDb(home, 4, 'complete') + const newest = createBackfillDb(home, 5, 'running') + + expect(findNewestCodexStateDbPath(home)).toBe(newest) + expect(readCodexStateDbBackfillStatus(home)).toEqual({ + kind: 'incomplete', + stateDbPath: newest, + status: 'running' + }) + + const db = new SyncDatabase(newest, { readonly: true, fileMustExist: true }) + expect(db.prepare('SELECT status FROM backfill_state WHERE id = 1').get()).toEqual({ + status: 'running' + }) + db.close() + }) + + it('treats a large unindexed rollout history as pending', async () => { + const home = await createHome() + const sessions = join(home, 'sessions', '2026', '08', '04') + await mkdir(sessions, { recursive: true }) + await Promise.all( + Array.from({ length: 100 }, (_, index) => + writeFile(join(sessions, `rollout-${index}.jsonl`), '{}\n') + ) + ) + + expect(isCodexStateDbBackfillPending(home)).toBe(true) + }) + + it('does not call a complete backfill pending', async () => { + const home = await createHome() + createBackfillDb(home, 5, 'complete') + + expect(isCodexStateDbBackfillPending(home)).toBe(false) + }) +}) diff --git a/src/main/codex/codex-state-db.ts b/src/main/codex/codex-state-db.ts new file mode 100644 index 000000000..e2b437aea --- /dev/null +++ b/src/main/codex/codex-state-db.ts @@ -0,0 +1,115 @@ +import { readdirSync } from 'node:fs' +import { join } from 'node:path' +import SyncDatabase from '../sqlite/sync-database' + +const STATE_DB_FILE_PATTERN = /^state_(\d+)\.sqlite$/ + +export type CodexStateDbBackfillStatus = + | { kind: 'complete'; stateDbPath: string } + | { kind: 'incomplete'; stateDbPath: string; status: string } + | { kind: 'missing' } + | { kind: 'not-tracked'; stateDbPath: string } + | { kind: 'unreadable'; stateDbPath: string; error: string } + +export function findNewestCodexStateDbPath(codexHomePath: string): string | null { + let entries: string[] + try { + entries = readdirSync(codexHomePath) + } catch { + return null + } + let newest: { version: number; name: string } | null = null + for (const name of entries) { + const match = STATE_DB_FILE_PATTERN.exec(name) + if (!match) { + continue + } + const version = Number(match[1]) + if (!newest || version > newest.version) { + newest = { version, name } + } + } + return newest ? join(codexHomePath, newest.name) : null +} + +/** Reads Codex-owned backfill metadata without creating or mutating its database. */ +export function readCodexStateDbBackfillStatus(codexHomePath: string): CodexStateDbBackfillStatus { + const stateDbPath = findNewestCodexStateDbPath(codexHomePath) + if (!stateDbPath) { + return { kind: 'missing' } + } + let db: SyncDatabase | null = null + try { + db = new SyncDatabase(stateDbPath, { readonly: true, fileMustExist: true }) + const table = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'backfill_state'") + .get() + if (!table) { + return { kind: 'not-tracked', stateDbPath } + } + const row = db.prepare('SELECT status FROM backfill_state WHERE id = 1').get() as + | { status?: unknown } + | undefined + if (!row || typeof row.status !== 'string') { + return { kind: 'not-tracked', stateDbPath } + } + return row.status === 'complete' + ? { kind: 'complete', stateDbPath } + : { kind: 'incomplete', stateDbPath, status: row.status } + } catch (error) { + return { + kind: 'unreadable', + stateDbPath, + error: error instanceof Error ? error.message : String(error) + } + } finally { + try { + db?.close() + } catch { + // A close failure cannot change the read-only result already collected. + } + } +} + +export function countCodexSessionFilesUpTo(sessionsRoot: string, limit: number): number { + let count = 0 + const pendingDirectories = [sessionsRoot] + while (pendingDirectories.length > 0 && count < limit) { + const directory = pendingDirectories.pop() as string + let entries + try { + entries = readdirSync(directory, { withFileTypes: true }) + } catch { + continue + } + for (const entry of entries) { + if (entry.isDirectory()) { + pendingDirectories.push(join(directory, entry.name)) + } else if (entry.isFile() && entry.name.endsWith('.jsonl')) { + count += 1 + if (count >= limit) { + break + } + } + } + } + return count +} + +export const BACKFILL_PENDING_MIN_SESSION_FILES = 100 + +export function isCodexStateDbBackfillPending(codexHomePath: string): boolean { + const status = readCodexStateDbBackfillStatus(codexHomePath) + if (status.kind === 'incomplete') { + return true + } + if (status.kind !== 'missing' && status.kind !== 'not-tracked') { + return false + } + return ( + countCodexSessionFilesUpTo( + join(codexHomePath, 'sessions'), + BACKFILL_PENDING_MIN_SESSION_FILES + ) >= BACKFILL_PENDING_MIN_SESSION_FILES + ) +} diff --git a/src/main/index.ts b/src/main/index.ts index e51db18a9..912941275 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -219,6 +219,10 @@ import { import { setCodexTrustGrantTelemetry } from './codex/codex-trust-grant-telemetry' import { startCodexSessionBackfillInBackground } from './codex/codex-session-backfill' import { startCodexSessionIndexHealInBackground } from './codex/codex-session-index-heal' +import { + startCodexStateDbBackfillRecoveryInBackground, + stopCodexStateDbBackfillRecoveries +} from './codex/codex-state-db-backfill-recovery' import { createCodexSessionMigrationScheduler } from './codex/codex-session-migration-scheduler' import { prepareLegacySharedCodexSessionResume } from './codex/codex-legacy-session-resume' import { resolveHostCodexSessionSourceHome } from './codex/codex-session-source-home' @@ -2232,6 +2236,7 @@ void app.whenReady().then(async () => { openCodeUsage = new OpenCodeUsageStore(store) rateLimits = new RateLimitService() codexRuntimeHome = new CodexRuntimeHomeService(store) + void startCodexStateDbBackfillRecoveryInBackground(getOrcaManagedCodexHomePath()) // Why: an incapable trust-grant host must fall back to the managed home for // every consumer (PTY env, rate limits, commit messages) in one place. codexRuntimeHome.setRealHomeLaneGate(() => isRealHomeCodexHookLaneUsable()) @@ -3059,6 +3064,7 @@ app.on('will-quit', (e) => { pluginMarketplaceService = null pluginMarketplaceInstaller = null const pluginHostShutdown = pluginService?.dispose() ?? Promise.resolve() + const codexBackfillRecoveryShutdown = stopCodexStateDbBackfillRecoveries() pluginService = null setUnreadDockBadgeCount(0) agentHookServer.stop() @@ -3118,6 +3124,7 @@ app.on('will-quit', (e) => { { name: 'emulator', promise: emulatorShutdown }, { name: 'ssh', promise: sshShutdown }, { name: 'plugin-hosts', promise: pluginHostShutdown }, + { name: 'codex-backfill-recovery', promise: codexBackfillRecoveryShutdown }, { name: 'usage-cache', promise: usageCacheFlush }, { name: 'stats', promise: statsFlush }, { name: 'state', promise: storeFlush } diff --git a/src/main/ipc/pty.test.ts b/src/main/ipc/pty.test.ts index 678cc8774..c3d3a7795 100644 --- a/src/main/ipc/pty.test.ts +++ b/src/main/ipc/pty.test.ts @@ -65,7 +65,8 @@ const { clearMigrationUnsupportedPtysForPaneKeyMock, clearPaneKeyAliasesForPtyMock, recordCodexPaneAccountMock, - forgetCodexPaneAccountMock + forgetCodexPaneAccountMock, + ensureCodexBackfillRecoveryMock } = vi.hoisted(() => ({ handleMock: vi.fn(), onMock: vi.fn(), @@ -99,7 +100,8 @@ const { clearMigrationUnsupportedPtysForPaneKeyMock: vi.fn(), clearPaneKeyAliasesForPtyMock: vi.fn(), recordCodexPaneAccountMock: vi.fn(), - forgetCodexPaneAccountMock: vi.fn() + forgetCodexPaneAccountMock: vi.fn(), + ensureCodexBackfillRecoveryMock: vi.fn(() => Promise.resolve()) })) vi.mock('electron', () => ({ @@ -209,6 +211,10 @@ vi.mock('../codex/codex-pane-account-registry', () => ({ recordCodexPaneAccount: recordCodexPaneAccountMock, forgetCodexPaneAccount: forgetCodexPaneAccountMock })) + +vi.mock('../codex/codex-state-db-backfill-recovery', () => ({ + ensureCodexStateDbBackfillRecoveryStarted: ensureCodexBackfillRecoveryMock +})) import { LocalPtyProvider, _resetLocalPtyProviderStateForTest @@ -383,6 +389,8 @@ describe('registerPtyHandlers', () => { clearPaneKeyAliasesForPtyMock.mockReset() recordCodexPaneAccountMock.mockReset() forgetCodexPaneAccountMock.mockReset() + ensureCodexBackfillRecoveryMock.mockReset() + ensureCodexBackfillRecoveryMock.mockResolvedValue(undefined) mainWindow.webContents.on.mockReset() mainWindow.webContents.send.mockReset() mainWindow.webContents.removeListener.mockReset() @@ -3060,6 +3068,38 @@ describe('registerPtyHandlers', () => { }) }) + it('arbitrates the exact backfill owner before spawning Codex', async () => { + let releaseRecovery!: () => void + ensureCodexBackfillRecoveryMock.mockReturnValue( + new Promise((resolve) => (releaseRecovery = resolve)) + ) + readFileSyncMock.mockReturnValue(TEST_CODEX_AUTH_JSON) + handlers.clear() + registerPtyHandlers(mainWindow as never, undefined, () => TEST_CODEX_HOME, (() => ({ + codexManagedAccounts: [ + { + id: 'account-1', + managedHomePath: TEST_CODEX_HOME, + managedHomeRuntime: 'host' + } + ] + })) as never) + + const spawnPromise = handlers.get('pty:spawn')!(null, { + cols: 80, + rows: 24, + launchAgent: 'codex' + }) + await vi.waitFor(() => + expect(ensureCodexBackfillRecoveryMock).toHaveBeenCalledWith(TEST_CODEX_HOME) + ) + expect(spawnMock).not.toHaveBeenCalled() + + releaseRecovery() + await spawnPromise + expect(spawnMock).toHaveBeenCalledTimes(1) + }) + it('does not gate a bare local shell on managed Codex auth', async () => { readFileSyncMock.mockImplementation((filePath: string) => { if (filePath.endsWith('auth.json')) { diff --git a/src/main/ipc/pty.ts b/src/main/ipc/pty.ts index 46498fdf0..54ce28b68 100644 --- a/src/main/ipc/pty.ts +++ b/src/main/ipc/pty.ts @@ -199,6 +199,7 @@ import { } from '../codex/codex-pane-account-registry' import { resolveCodexPaneLaunchAccount } from '../codex/codex-pane-launch-account' import { getSystemCodexHomePath } from '../codex/codex-home-paths' +import { ensureCodexStateDbBackfillRecoveryStarted } from '../codex/codex-state-db-backfill-recovery' import { environmentCodexHomeOverrideContextsEqual, getCustomCodexHomeOverrideForLaunch, @@ -4495,6 +4496,9 @@ export function registerPtyHandlers( }) selectedCodexHomePath = resolution instanceof Promise ? await resolution : resolution } + if (args.launchAgent === 'codex' && selectedCodexHomePath) { + await ensureCodexStateDbBackfillRecoveryStarted(selectedCodexHomePath) + } const codexResumeHomeSelected = Boolean( codexResumeHome && codexHomePathsEqual(selectedCodexHomePath, codexResumeHome.codexHomePath) ) @@ -5956,6 +5960,9 @@ export function registerPtyHandlers( }) selectedCodexHomePath = resolution instanceof Promise ? await resolution : resolution } + if (args.launchAgent === 'codex' && selectedCodexHomePath) { + await ensureCodexStateDbBackfillRecoveryStarted(selectedCodexHomePath) + } const codexResumeHomeSelected = Boolean( codexResumeHome && codexHomePathsEqual(selectedCodexHomePath, codexResumeHome.codexHomePath) diff --git a/src/main/rate-limits/codex-fetcher.test.ts b/src/main/rate-limits/codex-fetcher.test.ts index aedd38656..2f7ee834d 100644 --- a/src/main/rate-limits/codex-fetcher.test.ts +++ b/src/main/rate-limits/codex-fetcher.test.ts @@ -2,11 +2,20 @@ import { EventEmitter } from 'node:events' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -const { childSpawnMock, readFileMock, resolveCodexCommandMock, ptySpawnMock } = vi.hoisted(() => ({ +const { + childSpawnMock, + readFileMock, + resolveCodexCommandMock, + ptySpawnMock, + isBackfillPendingMock, + startBackfillRecoveryMock +} = vi.hoisted(() => ({ childSpawnMock: vi.fn(), readFileMock: vi.fn(), resolveCodexCommandMock: vi.fn(), - ptySpawnMock: vi.fn() + ptySpawnMock: vi.fn(), + isBackfillPendingMock: vi.fn(() => false), + startBackfillRecoveryMock: vi.fn(() => Promise.resolve(null)) })) vi.mock('node:child_process', () => ({ @@ -25,6 +34,14 @@ vi.mock('node-pty', () => ({ spawn: ptySpawnMock })) +vi.mock('../codex/codex-state-db', () => ({ + isCodexStateDbBackfillPending: isBackfillPendingMock +})) + +vi.mock('../codex/codex-state-db-backfill-recovery', () => ({ + startCodexStateDbBackfillRecoveryInBackground: startBackfillRecoveryMock +})) + // Default to signed-in so the spawn paths under test still run; the auth gate // itself is covered by codex-auth-presence.test.ts and the no-auth case below. vi.mock('./codex-auth-presence', () => ({ @@ -116,6 +133,7 @@ describe('fetchCodexRateLimits', () => { resolveCodexCommandMock.mockReturnValue('codex') vi.mocked(probeCodexAuthPresence).mockResolvedValue('present') readFileMock.mockRejectedValue(new Error('no auth fixture')) + isBackfillPendingMock.mockReturnValue(false) vi.stubGlobal('fetch', vi.fn()) }) @@ -138,6 +156,20 @@ describe('fetchCodexRateLimits', () => { expect(ptySpawnMock).not.toHaveBeenCalled() }) + it('does not let a quota probe steal an incomplete state-DB backfill lease', async () => { + isBackfillPendingMock.mockReturnValue(true) + + await expect( + fetchCodexRateLimits({ codexHomePath: '/managed-home', allowPtyFallback: false }) + ).resolves.toMatchObject({ + status: 'error', + error: expect.stringContaining('session index') + }) + expect(startBackfillRecoveryMock).toHaveBeenCalledWith('/managed-home') + expect(childSpawnMock).not.toHaveBeenCalled() + expect(ptySpawnMock).not.toHaveBeenCalled() + }) + it('preserves the aborted result when cancellation lands during the auth check', async () => { let resolveAuth!: (presence: 'absent') => void vi.mocked(probeCodexAuthPresence).mockReturnValueOnce( diff --git a/src/main/rate-limits/codex-fetcher.ts b/src/main/rate-limits/codex-fetcher.ts index 50c97ff5d..381875741 100644 --- a/src/main/rate-limits/codex-fetcher.ts +++ b/src/main/rate-limits/codex-fetcher.ts @@ -41,6 +41,8 @@ import { resolveCodexHomeProcessLockKey, withCodexHomeProcessLock } from '../codex-cli/codex-home-process-lock' +import { isCodexStateDbBackfillPending } from '../codex/codex-state-db' +import { startCodexStateDbBackfillRecoveryInBackground } from '../codex/codex-state-db-backfill-recovery' const RPC_TIMEOUT_MS = 10_000 const WSL_RPC_TIMEOUT_MS = 25_000 @@ -1249,6 +1251,19 @@ export async function fetchCodexRateLimits( } } + if (options?.codexHomePath && isCodexStateDbBackfillPending(options.codexHomePath)) { + // Why: a bounded quota probe can steal an expired backfill lease, then die before indexing finishes. + void startCodexStateDbBackfillRecoveryInBackground(options.codexHomePath) + return { + provider: 'codex', + session: null, + weekly: null, + updatedAt: Date.now(), + error: 'Codex is rebuilding its session index; usage will refresh when recovery finishes', + status: 'error' + } + } + // Why: probes spawn a real codex process inside the live credential home; // the per-home lock keeps Orca's own spawns (probe vs probe, probe vs // commit-message run) from refreshing one auth.json concurrently. diff --git a/src/renderer/src/components/terminal-pane/codex-backfill-error-detector.test.ts b/src/renderer/src/components/terminal-pane/codex-backfill-error-detector.test.ts new file mode 100644 index 000000000..6226d178e --- /dev/null +++ b/src/renderer/src/components/terminal-pane/codex-backfill-error-detector.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { + CODEX_BACKFILL_RECOVERY_NOTICE, + createCodexBackfillErrorDetector +} from './codex-backfill-error-detector' + +describe('Codex backfill error detector', () => { + it('recognizes the timeout across ANSI-decorated chunks once', () => { + const detector = createCodexBackfillErrorDetector() + + expect(detector.observe('\u001b[31mError: timed out waiting for state DB back')).toBeNull() + expect(detector.observe('fill\u001b[0m\r\n')).toBe(CODEX_BACKFILL_RECOVERY_NOTICE) + expect(detector.observe('timed out waiting for state db backfill')).toBeNull() + }) + + it('does not classify the generic damaged-database message', () => { + const detector = createCodexBackfillErrorDetector() + + expect(detector.observe('local database appears to be damaged')).toBeNull() + }) +}) diff --git a/src/renderer/src/components/terminal-pane/codex-backfill-error-detector.ts b/src/renderer/src/components/terminal-pane/codex-backfill-error-detector.ts new file mode 100644 index 000000000..7c95bd7e4 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/codex-backfill-error-detector.ts @@ -0,0 +1,33 @@ +export const CODEX_BACKFILL_TIMEOUT_SIGNATURE = 'timed out waiting for state db backfill' + +export const CODEX_BACKFILL_RECOVERY_NOTICE = [ + 'Codex could not start because its session-history index is still incomplete.', + 'Keep Orca open for a few minutes, then retry this pane. Orca attempts background recovery for managed local and WSL homes.' +].join('\n') + +const ANSI_ESCAPE_PATTERN = + // eslint-disable-next-line no-control-regex -- terminal escape sequences contain control bytes + /\u001b(?:\[[0-9;?]*[ -/]*[@-~]|\][^\u0007\u001b]*(?:\u0007|\u001b\\)?)/g +const DETECTOR_BUFFER_MAX_CHARS = 4096 + +export type CodexBackfillErrorDetector = { observe(chunk: string): string | null } + +/** Scans one Codex pane's output once for the unambiguous backfill timeout. */ +export function createCodexBackfillErrorDetector(): CodexBackfillErrorDetector { + let tail = '' + let armed = true + return { + observe(chunk: string): string | null { + if (!armed) { + return null + } + const normalized = (tail + chunk).replace(ANSI_ESCAPE_PATTERN, '').replace(/\r/g, '') + tail = normalized.slice(-DETECTOR_BUFFER_MAX_CHARS) + if (!tail.toLowerCase().includes(CODEX_BACKFILL_TIMEOUT_SIGNATURE)) { + return null + } + armed = false + return CODEX_BACKFILL_RECOVERY_NOTICE + } + } +} diff --git a/src/renderer/src/components/terminal-pane/pty-connection.test.ts b/src/renderer/src/components/terminal-pane/pty-connection.test.ts index 29f693640..58408b8ba 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.test.ts @@ -2324,6 +2324,31 @@ describe('connectPanePty', () => { ) }) + it('surfaces an actionable error when a Codex backfill timeout drops to the shell', async () => { + const { connectPanePty } = await import('./pty-connection') + const transport = createMockTransport() + const capturedDataCallback: { current: ((data: string) => void) | null } = { current: null } + transport.connect.mockImplementation(async ({ callbacks }: { callbacks: ConnectCallbacks }) => { + capturedDataCallback.current = callbacks.onData ?? null + return 'pty-codex-backfill-timeout' + }) + transportFactoryQueue.push(transport) + const deps = createDeps({ + tabId: 'tab-codex-backfill-timeout', + startup: { command: 'codex', launchAgent: 'codex' } + }) + + connectPanePty(createPane(1) as never, createManager(1) as never, deps as never) + await flushAsyncTicks() + capturedDataCallback.current?.('timed out waiting for state db back') + capturedDataCallback.current?.('fill\r\n') + + expect(deps.onPtyErrorRef.current).toHaveBeenCalledWith( + 1, + expect.stringContaining('Orca attempts background recovery for managed local and WSL homes') + ) + }) + it('drops keystrokes while the replay guard is engaged, then forwards once it releases', async () => { // Regression (cold-restore reattach lockout): a stuck replay guard dropped every keystroke ("can't type after reconnecting"); engaged guard suppresses input, released forwards it. const { connectPanePty } = await import('./pty-connection') diff --git a/src/renderer/src/components/terminal-pane/pty-connection.ts b/src/renderer/src/components/terminal-pane/pty-connection.ts index 9b843723a..9d14131a4 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.ts @@ -235,6 +235,7 @@ import { getTerminalPasteSshRemotePlatform } from './terminal-paste-ssh-platform import { resolveTerminalPasteRuntime } from './terminal-paste-runtime' import { isKnownTuiAgentTerminalStartupCommand } from './terminal-startup-command-classifier' import { createCommandCodeOutputStatusDetector } from '../../../../shared/command-code-output-status' +import { createCodexBackfillErrorDetector } from './codex-backfill-error-detector' import type { PtyDataMeta } from './pty-dispatcher' import { getEagerPtyBufferHandle } from './pty-dispatcher' import { createTerminalGitHubPRLinkDetector } from '../../../../shared/terminal-github-pr-link-detector' @@ -4630,6 +4631,10 @@ export function connectPanePty( } deps.onPtyErrorRef?.current?.(pane.id, message) } + const codexBackfillErrorDetector = + paneStartup?.launchAgent === 'codex' || tab?.launchAgent === 'codex' + ? createCodexBackfillErrorDetector() + : null // Why: shared registration so both fresh-spawn and reattach paths install // the same SerializeAddon-backed serializer plus the onTitleChange wrapper @@ -7747,6 +7752,10 @@ export function connectPanePty( commandLifecycle.handlePtyData(data) } commandCodeOutputStatusDetector?.observe(data) + const codexBackfillNotice = codexBackfillErrorDetector?.observe(data) + if (codexBackfillNotice) { + reportError(codexBackfillNotice) + } // Why: split panes have visible-but-inactive panes the user watches; throttle only when the pane or whole document is hidden. const foreground = shouldWritePtyOutputForeground(deps.isVisibleRef.current) && meta?.background !== true