fix(codex): recover interrupted state DB backfills (#12617)
* fix(codex): recover interrupted state DB backfills * fix(codex): detect mixed-case backfill timeout * fix(codex): harden backfill recovery review findings * fix(codex): keep process identity retries safe
This commit is contained in:
Brennan Benson
GitHub
parent
8c3e9535c7
commit
f0443c326a
|
|
@ -35,6 +35,7 @@
|
|||
"../src/main/codex/codex-hook-trust-grant.ts",
|
||||
"../src/main/codex/codex-managed-trust-reconciliation.ts",
|
||||
"../src/main/codex/codex-process-exit-deadline.ts",
|
||||
"../src/main/codex/codex-state-db.ts",
|
||||
"../src/main/codex/codex-trust-config-rollback.ts",
|
||||
"../src/main/codex/codex-trust-grant-telemetry.ts",
|
||||
"../src/main/codex/codex-trust-grant-host.ts",
|
||||
|
|
@ -77,6 +78,7 @@
|
|||
// project's serve spec; the module has no imports, so listing it pulls in nothing else.
|
||||
"../src/main/startup/serve-mode-argv.ts",
|
||||
"../src/main/runtime/runtime-metadata.ts",
|
||||
"../src/main/sqlite/sync-database.ts",
|
||||
"../src/main/win32-utils.ts"
|
||||
],
|
||||
"compilerOptions": {
|
||||
|
|
|
|||
|
|
@ -46,7 +46,8 @@ async function releaseInstallLock(
|
|||
async function acquireInstallLock(
|
||||
home: string,
|
||||
signal?: AbortSignal,
|
||||
suppliedHostIdentity?: string
|
||||
suppliedHostIdentity?: string,
|
||||
waitTimeoutMs = LOCK_WAIT_TIMEOUT_MS
|
||||
): Promise<() => Promise<void>> {
|
||||
const lockParent = join(home, '.orca')
|
||||
const lockPath = join(lockParent, 'managed-hook-install.lock')
|
||||
|
|
@ -57,7 +58,7 @@ async function acquireInstallLock(
|
|||
if (typeof processIdentity !== 'string') {
|
||||
throw new Error('Could not identify the managed-hook installer process')
|
||||
}
|
||||
const deadline = Date.now() + LOCK_WAIT_TIMEOUT_MS
|
||||
const deadline = Date.now() + waitTimeoutMs
|
||||
|
||||
while (true) {
|
||||
signal?.throwIfAborted()
|
||||
|
|
@ -130,9 +131,15 @@ export async function withManagedHookInstallLock<T>(
|
|||
home: string,
|
||||
signal: AbortSignal | undefined,
|
||||
run: () => Promise<T>,
|
||||
hostIdentity?: string
|
||||
hostIdentity?: string,
|
||||
options?: { waitTimeoutMs?: number }
|
||||
): Promise<T> {
|
||||
const release = await acquireInstallLock(home, signal, hostIdentity)
|
||||
const release = await acquireInstallLock(
|
||||
home,
|
||||
signal,
|
||||
hostIdentity,
|
||||
options?.waitTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS
|
||||
)
|
||||
try {
|
||||
signal?.throwIfAborted()
|
||||
return await run()
|
||||
|
|
|
|||
|
|
@ -67,6 +67,33 @@ async function loadLinuxIdentity(fixture: LinuxIdentityFixture) {
|
|||
return { identity: await import('./managed-hook-owner-identity'), readFile }
|
||||
}
|
||||
|
||||
async function loadWindowsIdentity() {
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' })
|
||||
const execFileAsync = vi.fn(async (_file: string, args: string[]) => ({
|
||||
stdout: args.join(' ').includes('MachineGuid')
|
||||
? '\r\n MachineGuid REG_SZ AAAAAAAA-BBBB-4CCC-8DDD-EEEEEEEEEEEE\r\n'
|
||||
: '1777777777000\r\n'
|
||||
}))
|
||||
vi.doMock('node:util', () => ({ promisify: () => execFileAsync }))
|
||||
return { identity: await import('./managed-hook-owner-identity'), execFileAsync }
|
||||
}
|
||||
|
||||
async function loadDarwinIdentity() {
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: 'darwin' })
|
||||
let psCalls = 0
|
||||
const execFileAsync = vi.fn(async (file: string) => {
|
||||
if (file === 'sysctl') {
|
||||
return { stdout: 'boot-session\n' }
|
||||
}
|
||||
if (file === 'ps' && ++psCalls === 1) {
|
||||
throw new Error('transient ps failure')
|
||||
}
|
||||
return { stdout: 'Wed Aug 5 12:00:00 2026 node app\n' }
|
||||
})
|
||||
vi.doMock('node:util', () => ({ promisify: () => execFileAsync }))
|
||||
return await import('./managed-hook-owner-identity')
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform })
|
||||
if (originalGetuidDescriptor) {
|
||||
|
|
@ -76,6 +103,8 @@ afterEach(() => {
|
|||
}
|
||||
vi.unstubAllEnvs()
|
||||
vi.doUnmock('node:fs/promises')
|
||||
vi.doUnmock('node:child_process')
|
||||
vi.doUnmock('node:util')
|
||||
vi.resetModules()
|
||||
})
|
||||
|
||||
|
|
@ -178,4 +207,28 @@ describe('managed hook owner identity', () => {
|
|||
'linux:pid:[4026533001]:current-boot-id:4242'
|
||||
)
|
||||
})
|
||||
|
||||
it('uses machine and process creation identities on Windows', async () => {
|
||||
const { identity, execFileAsync } = await loadWindowsIdentity()
|
||||
|
||||
await expect(identity.readManagedHookHostIdentity()).resolves.toBe(
|
||||
'win32:aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee'
|
||||
)
|
||||
await expect(identity.readManagedHookProcessIdentity(process.pid)).resolves.toBe(
|
||||
`win32:${process.pid}:1777777777000`
|
||||
)
|
||||
await expect(identity.readManagedHookProcessIdentity(process.pid)).resolves.toBe(
|
||||
`win32:${process.pid}:1777777777000`
|
||||
)
|
||||
expect(execFileAsync).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('retries an unverified macOS process identity', async () => {
|
||||
const identity = await loadDarwinIdentity()
|
||||
|
||||
await expect(identity.readManagedHookProcessIdentity(process.pid)).resolves.toBeUndefined()
|
||||
await expect(identity.readManagedHookProcessIdentity(process.pid)).resolves.toBe(
|
||||
'darwin:boot-session:Wed Aug 5 12:00:00 2026 node app'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,15 +1,30 @@
|
|||
import { execFile } from 'node:child_process'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { link, lstat, mkdir, readFile, readlink, unlink, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { join, win32 } from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const runtimeHostIdentity = `runtime:${randomUUID()}`
|
||||
const runtimeProcessIdentity = `runtime:${randomUUID()}`
|
||||
let hostIdentityPromise: Promise<string> | undefined
|
||||
let bootIdentityPromise: Promise<string | undefined> | undefined
|
||||
let selfProcessIdentityPromise: Promise<string | null | undefined> | undefined
|
||||
const HOST_TOKEN_PATTERN = /^[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/i
|
||||
const WINDOWS_PROCESS_IDENTITY_TIMEOUT_MS = 5_000
|
||||
|
||||
function getWindowsPowerShellPath(): string {
|
||||
return win32.join(
|
||||
process.env.SystemRoot ?? 'C:\\Windows',
|
||||
'System32',
|
||||
'WindowsPowerShell',
|
||||
'v1.0',
|
||||
'powershell.exe'
|
||||
)
|
||||
}
|
||||
|
||||
function getWindowsRegistryPath(): string {
|
||||
return win32.join(process.env.SystemRoot ?? 'C:\\Windows', 'System32', 'reg.exe')
|
||||
}
|
||||
|
||||
function hasCode(error: unknown, code: string): boolean {
|
||||
return error instanceof Error && 'code' in error && error.code === code
|
||||
|
|
@ -100,12 +115,29 @@ async function readHostIdentity(): Promise<string> {
|
|||
// but a later process gets a different identity and cannot steal its residue.
|
||||
return runtimeHostIdentity
|
||||
}
|
||||
if (process.platform === 'win32') {
|
||||
try {
|
||||
const { stdout } = await promisify(execFile)(
|
||||
getWindowsRegistryPath(),
|
||||
['query', 'HKLM\\SOFTWARE\\Microsoft\\Cryptography', '/v', 'MachineGuid'],
|
||||
{ encoding: 'utf8', timeout: 1_000, windowsHide: true }
|
||||
)
|
||||
const machineGuid = /^\s*MachineGuid\s+REG_\w+\s+(.+?)\s*$/im.exec(stdout)?.[1]
|
||||
return machineGuid ? `win32:${machineGuid.toLowerCase()}` : runtimeHostIdentity
|
||||
} catch {
|
||||
return runtimeHostIdentity
|
||||
}
|
||||
}
|
||||
if (process.platform === 'darwin') {
|
||||
try {
|
||||
const { stdout } = await execFileAsync('ioreg', ['-rd1', '-c', 'IOPlatformExpertDevice'], {
|
||||
encoding: 'utf8',
|
||||
timeout: 1_000
|
||||
})
|
||||
const { stdout } = await promisify(execFile)(
|
||||
'ioreg',
|
||||
['-rd1', '-c', 'IOPlatformExpertDevice'],
|
||||
{
|
||||
encoding: 'utf8',
|
||||
timeout: 1_000
|
||||
}
|
||||
)
|
||||
const platformId = /"IOPlatformUUID"\s*=\s*"([^"]+)"/.exec(stdout)?.[1]
|
||||
return platformId ? `darwin:${platformId}` : runtimeHostIdentity
|
||||
} catch {
|
||||
|
|
@ -127,7 +159,7 @@ export async function readBootIdentity(): Promise<string | undefined> {
|
|||
|
||||
if (process.platform === 'darwin') {
|
||||
try {
|
||||
const { stdout } = await execFileAsync('sysctl', ['-n', 'kern.bootsessionuuid'], {
|
||||
const { stdout } = await promisify(execFile)('sysctl', ['-n', 'kern.bootsessionuuid'], {
|
||||
encoding: 'utf8',
|
||||
timeout: 1_000
|
||||
})
|
||||
|
|
@ -160,6 +192,14 @@ export function scopeManagedHookHostIdentity(
|
|||
export async function readManagedHookProcessIdentity(
|
||||
pid: number
|
||||
): Promise<string | null | undefined> {
|
||||
if (process.platform === 'win32' && pid === process.pid) {
|
||||
selfProcessIdentityPromise ??= readProcessIdentity(pid)
|
||||
return await selfProcessIdentityPromise
|
||||
}
|
||||
return await readProcessIdentity(pid)
|
||||
}
|
||||
|
||||
async function readProcessIdentity(pid: number): Promise<string | null | undefined> {
|
||||
if (process.platform === 'linux') {
|
||||
try {
|
||||
const [statLine, pidNamespace, bootIdentity] = await Promise.all([
|
||||
|
|
@ -179,10 +219,37 @@ export async function readManagedHookProcessIdentity(
|
|||
}
|
||||
}
|
||||
|
||||
if (process.platform === 'win32') {
|
||||
try {
|
||||
const { stdout } = await promisify(execFile)(
|
||||
getWindowsPowerShellPath(),
|
||||
[
|
||||
'-NoProfile',
|
||||
'-NonInteractive',
|
||||
'-Command',
|
||||
`$ErrorActionPreference = 'Stop'; ` +
|
||||
`$p = Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}"; ` +
|
||||
`if (!$p) { 'missing'; exit 0 }; ` +
|
||||
`[string]([DateTimeOffset]$p.CreationDate).ToUnixTimeMilliseconds()`
|
||||
],
|
||||
{ encoding: 'utf8', timeout: WINDOWS_PROCESS_IDENTITY_TIMEOUT_MS, windowsHide: true }
|
||||
)
|
||||
const startedAt = stdout.trim()
|
||||
return startedAt === 'missing' ? null : startedAt ? `win32:${pid}:${startedAt}` : undefined
|
||||
} catch {
|
||||
try {
|
||||
process.kill(pid, 0)
|
||||
return pid === process.pid ? runtimeProcessIdentity : undefined
|
||||
} catch (error) {
|
||||
return hasCode(error, 'ESRCH') ? null : undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bootIdentityPromise ??= readBootIdentity()
|
||||
const bootIdentity = await bootIdentityPromise
|
||||
try {
|
||||
const { stdout } = await execFileAsync(
|
||||
const { stdout } = await promisify(execFile)(
|
||||
'ps',
|
||||
['-o', 'lstart=', '-o', 'command=', '-p', String(pid)],
|
||||
{
|
||||
|
|
|
|||
|
|
@ -97,6 +97,22 @@ function grantedSessionResult(entries: CodexTrustEntry[], hashPrefix = 'sha256:c
|
|||
}
|
||||
|
||||
describe('grantManagedCodexHookTrust', () => {
|
||||
it('does not let a short trust RPC claim an incomplete session index', () => {
|
||||
const sessions = join(runtimeHomeDir, 'sessions')
|
||||
mkdirSync(sessions, { recursive: true })
|
||||
for (let index = 0; index < 100; index += 1) {
|
||||
writeFileSync(join(sessions, `${index}.jsonl`), '{}\n')
|
||||
}
|
||||
const runner = vi.fn()
|
||||
_internals.setGrantSessionRunnerSync(runner)
|
||||
|
||||
expect(grantManagedCodexHookTrust(buildPlan([managedEntry('stop')]))).toMatchObject({
|
||||
lane: 'fallback',
|
||||
reason: 'retry-cached'
|
||||
})
|
||||
expect(runner).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns granted entries with codex-verbatim hashes and records the ledger', () => {
|
||||
const entries = [managedEntry('session_start'), managedEntry('stop')]
|
||||
const runner = vi.fn((_request: CodexHookTrustGrantRequest) => grantedSessionResult(entries))
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import {
|
|||
resolveCodexTrustGrantHost,
|
||||
type CodexTrustGrantHost
|
||||
} from './codex-trust-grant-host'
|
||||
import { isCodexStateDbBackfillPending } from './codex-state-db'
|
||||
|
||||
// Why: a transiently hung app-server must not block launch prep on every pane.
|
||||
// The legacy lane remains available while a short, host-scoped cooldown runs.
|
||||
|
|
@ -74,9 +75,7 @@ const diagnostics = {
|
|||
export type CodexTrustGrantDiagnostics = typeof diagnostics
|
||||
const transientRetryAfterByHost = new Map<string, number>()
|
||||
|
||||
export function getCodexTrustGrantDiagnostics(): CodexTrustGrantDiagnostics {
|
||||
return { ...diagnostics }
|
||||
}
|
||||
export const getCodexTrustGrantDiagnostics = (): CodexTrustGrantDiagnostics => ({ ...diagnostics })
|
||||
|
||||
type GrantSessionRunnerSync = (
|
||||
request: CodexHookTrustGrantRequest
|
||||
|
|
@ -192,6 +191,10 @@ export function grantManagedCodexHookTrust(
|
|||
diagnostics.ledgerHits += 1
|
||||
return { lane: 'rpc', entries: ledgerEntries }
|
||||
}
|
||||
if (isCodexStateDbBackfillPending(plan.runtimeHomePath)) {
|
||||
// Why: a short trust RPC can refresh Codex's abandoned lease and strand every pane again.
|
||||
return fallback(plan, 'retry-cached')
|
||||
}
|
||||
|
||||
const hostKey = getCodexAppServerHostKey(plan.host)
|
||||
if (!codexAppServerCapabilityCache.shouldTry(hostKey)) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,196 @@
|
|||
import { EventEmitter } from 'node:events'
|
||||
import { link, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import * as ownerIdentity from '../agent-hooks/managed-hook-owner-identity'
|
||||
import {
|
||||
resolveCodexBackfillSupervisorLockRoot,
|
||||
runCodexStateDbBackfillRecovery,
|
||||
withCodexBackfillSupervisorLock
|
||||
} from './codex-state-db-backfill-recovery'
|
||||
import type { CodexStateDbBackfillStatus } from './codex-state-db'
|
||||
|
||||
const temporaryRoots: string[] = []
|
||||
const originalPlatform = process.platform
|
||||
|
||||
function createFakeChild(): EventEmitter & {
|
||||
stdin: { end: ReturnType<typeof vi.fn> }
|
||||
exitCode: number | null
|
||||
signalCode: NodeJS.Signals | null
|
||||
kill: ReturnType<typeof vi.fn>
|
||||
} {
|
||||
return Object.assign(new EventEmitter(), {
|
||||
stdin: { end: vi.fn() },
|
||||
exitCode: null,
|
||||
signalCode: null,
|
||||
kill: vi.fn(() => true)
|
||||
})
|
||||
}
|
||||
|
||||
async function createTemporaryRoot(): Promise<string> {
|
||||
const root = await mkdtemp(join(tmpdir(), 'orca-codex-backfill-recovery-'))
|
||||
temporaryRoots.push(root)
|
||||
return root
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: originalPlatform })
|
||||
vi.restoreAllMocks()
|
||||
vi.unstubAllEnvs()
|
||||
await Promise.all(
|
||||
temporaryRoots.splice(0).map((root) => rm(root, { recursive: true, force: true }))
|
||||
)
|
||||
})
|
||||
|
||||
describe('Codex state DB backfill recovery', () => {
|
||||
it('keeps the successful app-server claimant alive until Codex marks its DB complete', async () => {
|
||||
const child = createFakeChild()
|
||||
const terminate = vi.fn(async () => {})
|
||||
const readStatus = vi
|
||||
.fn()
|
||||
.mockReturnValueOnce({ kind: 'incomplete', stateDbPath: '/state.sqlite', status: 'running' })
|
||||
.mockReturnValue({ kind: 'complete', stateDbPath: '/state.sqlite' })
|
||||
|
||||
await expect(
|
||||
runCodexStateDbBackfillRecovery('/managed-home', new AbortController().signal, {
|
||||
spawnProcess: vi.fn(() => child) as never,
|
||||
readStatus,
|
||||
terminate,
|
||||
sleep: vi.fn(async () => {}),
|
||||
now: vi.fn(() => 1_000)
|
||||
})
|
||||
).resolves.toEqual({ outcome: 'completed', spawnCount: 1 })
|
||||
expect(terminate).toHaveBeenCalledWith(child)
|
||||
})
|
||||
|
||||
it('retries a live foreign lease until one durable claimant can recover it', async () => {
|
||||
const first = createFakeChild()
|
||||
const second = createFakeChild()
|
||||
const children = [first, second]
|
||||
let now = 0
|
||||
let spawnCount = 0
|
||||
const spawnProcess = vi.fn(() => {
|
||||
const child = children[spawnCount++]
|
||||
if (child === first) {
|
||||
queueMicrotask(() => child.emit('exit', 1, null))
|
||||
}
|
||||
return child
|
||||
})
|
||||
const readStatus = vi.fn(
|
||||
(): CodexStateDbBackfillStatus =>
|
||||
spawnCount >= 2
|
||||
? { kind: 'complete', stateDbPath: '/state.sqlite' }
|
||||
: { kind: 'incomplete', stateDbPath: '/state.sqlite', status: 'running' }
|
||||
)
|
||||
|
||||
await expect(
|
||||
runCodexStateDbBackfillRecovery('/managed-home', new AbortController().signal, {
|
||||
spawnProcess: spawnProcess as never,
|
||||
readStatus,
|
||||
terminate: vi.fn(async () => {}),
|
||||
sleep: vi.fn(async (ms: number) => {
|
||||
now += ms
|
||||
await Promise.resolve()
|
||||
}),
|
||||
now: () => now
|
||||
})
|
||||
).resolves.toEqual({ outcome: 'completed', spawnCount: 2 })
|
||||
})
|
||||
|
||||
it('routes a WSL managed home through its distro and Linux CODEX_HOME', async () => {
|
||||
Object.defineProperty(process, 'platform', { configurable: true, value: 'win32' })
|
||||
const child = createFakeChild()
|
||||
const spawnProcess = vi.fn(() => child)
|
||||
const readStatus = vi
|
||||
.fn()
|
||||
.mockReturnValueOnce({ kind: 'incomplete', stateDbPath: 'state.sqlite', status: 'running' })
|
||||
.mockReturnValue({ kind: 'complete', stateDbPath: 'state.sqlite' })
|
||||
|
||||
await runCodexStateDbBackfillRecovery(
|
||||
'\\\\wsl.localhost\\Ubuntu\\home\\alice\\.codex',
|
||||
new AbortController().signal,
|
||||
{
|
||||
spawnProcess: spawnProcess as never,
|
||||
readStatus,
|
||||
terminate: vi.fn(async () => {}),
|
||||
sleep: vi.fn(async () => {}),
|
||||
now: vi.fn(() => 1_000)
|
||||
}
|
||||
)
|
||||
|
||||
expect(spawnProcess).toHaveBeenCalledWith(
|
||||
'wsl.exe',
|
||||
expect.arrayContaining(['-d', 'Ubuntu']),
|
||||
expect.objectContaining({ windowsHide: true })
|
||||
)
|
||||
const spawnCall = spawnProcess.mock.calls[0] as unknown as [string, string[]]
|
||||
const command = spawnCall[1].join(' ')
|
||||
expect(command).toContain('export CODEX_HOME=')
|
||||
expect(command).toContain('/home/alice/.codex')
|
||||
})
|
||||
})
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('Codex backfill supervisor owner lock', () => {
|
||||
it('recovers a dead owner whose PID was reused with a different start identity', async () => {
|
||||
const userData = await createTemporaryRoot()
|
||||
vi.stubEnv('ORCA_USER_DATA_PATH', userData)
|
||||
const home = join(userData, 'managed-home')
|
||||
const lockRoot = resolveCodexBackfillSupervisorLockRoot(home)
|
||||
const lockParent = join(lockRoot, '.orca')
|
||||
const token = '00000000-0000-4000-8000-000000000000'
|
||||
const ownerPath = join(lockParent, `managed-hook-install.owner-${token}.json`)
|
||||
const lockPath = join(lockParent, 'managed-hook-install.lock')
|
||||
await mkdir(lockParent, { recursive: true })
|
||||
await writeFile(
|
||||
ownerPath,
|
||||
JSON.stringify({
|
||||
token,
|
||||
pid: process.pid,
|
||||
hostIdentity: await ownerIdentity.readManagedHookHostIdentity(),
|
||||
processIdentity: 'stale-process-start-time'
|
||||
})
|
||||
)
|
||||
await link(ownerPath, lockPath)
|
||||
|
||||
await expect(
|
||||
withCodexBackfillSupervisorLock(home, undefined, async () => 'recovered')
|
||||
).resolves.toBe('recovered')
|
||||
})
|
||||
|
||||
it('does not interfere with a live supervisor from another Orca instance', async () => {
|
||||
const userData = await createTemporaryRoot()
|
||||
vi.stubEnv('ORCA_USER_DATA_PATH', userData)
|
||||
const home = join(userData, 'managed-home')
|
||||
let releaseFirst!: () => void
|
||||
const first = withCodexBackfillSupervisorLock(
|
||||
home,
|
||||
undefined,
|
||||
async () => await new Promise<void>((resolve) => (releaseFirst = resolve))
|
||||
)
|
||||
await vi.waitFor(async () => {
|
||||
await expect(
|
||||
import('node:fs/promises').then(({ readFile }) =>
|
||||
readFile(
|
||||
join(
|
||||
resolveCodexBackfillSupervisorLockRoot(home),
|
||||
'.orca',
|
||||
'managed-hook-install.lock'
|
||||
),
|
||||
'utf8'
|
||||
)
|
||||
)
|
||||
).resolves.toContain('processIdentity')
|
||||
})
|
||||
const controller = new AbortController()
|
||||
const secondRun = vi.fn(async () => {})
|
||||
setTimeout(() => controller.abort(), 40)
|
||||
|
||||
await expect(
|
||||
withCodexBackfillSupervisorLock(home, controller.signal, secondRun)
|
||||
).rejects.toMatchObject({ name: 'AbortError' })
|
||||
expect(secondRun).not.toHaveBeenCalled()
|
||||
releaseFirst()
|
||||
await first
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,288 @@
|
|||
import { spawn, type ChildProcess } from 'node:child_process'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { join } from 'node:path'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { normalizeRuntimePathForComparison } from '../../shared/cross-platform-path'
|
||||
import { parseWslUncPath } from '../../shared/wsl-paths'
|
||||
import { withManagedHookInstallLock } from '../agent-hooks/managed-hook-install-lock'
|
||||
import { readManagedHookHostIdentity } from '../agent-hooks/managed-hook-owner-identity'
|
||||
import { buildWslCodexAppServerArgs } from '../codex-accounts/wsl-codex-command'
|
||||
import { resolveCodexCommand } from '../codex-cli/command'
|
||||
import { terminateCodexProbeChild } from '../rate-limits/codex-probe-termination'
|
||||
import { getSpawnArgsForWindows } from '../win32-utils'
|
||||
import { getOrcaUserDataPath } from './codex-home-paths'
|
||||
import {
|
||||
BACKFILL_PENDING_MIN_SESSION_FILES,
|
||||
countCodexSessionFilesUpTo,
|
||||
isCodexStateDbBackfillPending,
|
||||
readCodexStateDbBackfillStatus,
|
||||
type CodexStateDbBackfillStatus
|
||||
} from './codex-state-db'
|
||||
|
||||
const RECOVERY_POLL_INTERVAL_MS = 5_000
|
||||
const RECOVERY_RETRY_DELAY_MS = 2_000
|
||||
const RECOVERY_FAST_EXIT_MS = 10_000
|
||||
const RECOVERY_MAX_FAST_FAILURES = 5
|
||||
const RECOVERY_MAX_TOTAL_MS = 60 * 60_000
|
||||
const RECOVERY_OWNER_CHECK_TIMEOUT_MS = 1_000
|
||||
const RECOVERY_CODEX_ARGS = ['-s', 'read-only', '-a', 'untrusted', 'app-server'] as const
|
||||
|
||||
export type CodexStateDbBackfillRecoverySummary = {
|
||||
outcome:
|
||||
| 'completed'
|
||||
| 'already-complete'
|
||||
| 'not-needed'
|
||||
| 'unreadable'
|
||||
| 'stopped'
|
||||
| 'gave-up'
|
||||
| 'codex-unavailable'
|
||||
spawnCount: number
|
||||
}
|
||||
|
||||
type RecoveryDependencies = {
|
||||
spawnProcess: typeof spawn
|
||||
resolveCommand: () => string
|
||||
readStatus: (codexHomePath: string) => CodexStateDbBackfillStatus
|
||||
countSessions: (sessionsRoot: string, limit: number) => number
|
||||
now: () => number
|
||||
sleep: (ms: number, signal: AbortSignal) => Promise<void>
|
||||
terminate: (child: ChildProcess) => Promise<void>
|
||||
}
|
||||
|
||||
const defaultDependencies: RecoveryDependencies = {
|
||||
spawnProcess: spawn,
|
||||
resolveCommand: resolveCodexCommand,
|
||||
readStatus: readCodexStateDbBackfillStatus,
|
||||
countSessions: countCodexSessionFilesUpTo,
|
||||
now: Date.now,
|
||||
sleep: async (ms, signal) => await delay(ms, undefined, { signal }),
|
||||
terminate: async (child) => await terminateCodexProbeChild(child)
|
||||
}
|
||||
|
||||
function finish(
|
||||
outcome: CodexStateDbBackfillRecoverySummary['outcome'],
|
||||
spawnCount: number
|
||||
): CodexStateDbBackfillRecoverySummary {
|
||||
return { outcome, spawnCount }
|
||||
}
|
||||
|
||||
function initialRecoveryDecision(
|
||||
codexHomePath: string,
|
||||
dependencies: RecoveryDependencies
|
||||
): CodexStateDbBackfillRecoverySummary['outcome'] | null {
|
||||
const status = dependencies.readStatus(codexHomePath)
|
||||
if (status.kind === 'complete') {
|
||||
return 'already-complete'
|
||||
}
|
||||
if (status.kind === 'unreadable') {
|
||||
return 'unreadable'
|
||||
}
|
||||
if (
|
||||
(status.kind === 'missing' || status.kind === 'not-tracked') &&
|
||||
dependencies.countSessions(
|
||||
join(codexHomePath, 'sessions'),
|
||||
BACKFILL_PENDING_MIN_SESSION_FILES
|
||||
) < BACKFILL_PENDING_MIN_SESSION_FILES
|
||||
) {
|
||||
return 'not-needed'
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function spawnRecoveryProcess(
|
||||
codexHomePath: string,
|
||||
dependencies: RecoveryDependencies
|
||||
): ChildProcess {
|
||||
const wslHome = process.platform === 'win32' ? parseWslUncPath(codexHomePath) : null
|
||||
if (wslHome) {
|
||||
return dependencies.spawnProcess(
|
||||
'wsl.exe',
|
||||
buildWslCodexAppServerArgs(wslHome.distro, wslHome.linuxPath),
|
||||
{
|
||||
stdio: ['pipe', 'ignore', 'ignore'],
|
||||
windowsHide: true,
|
||||
env: process.env
|
||||
}
|
||||
)
|
||||
}
|
||||
const command = dependencies.resolveCommand()
|
||||
const { spawnCmd, spawnArgs } = getSpawnArgsForWindows(command, [...RECOVERY_CODEX_ARGS])
|
||||
return dependencies.spawnProcess(spawnCmd, spawnArgs, {
|
||||
cwd: codexHomePath,
|
||||
stdio: ['pipe', 'ignore', 'ignore'],
|
||||
windowsHide: true,
|
||||
env: { ...process.env, CODEX_HOME: codexHomePath }
|
||||
})
|
||||
}
|
||||
|
||||
/** Keeps a sanctioned app-server claimant alive until Codex completes its own backfill. */
|
||||
export async function runCodexStateDbBackfillRecovery(
|
||||
codexHomePath: string,
|
||||
signal: AbortSignal,
|
||||
dependenciesOverride: Partial<RecoveryDependencies> = {}
|
||||
): Promise<CodexStateDbBackfillRecoverySummary> {
|
||||
const dependencies = { ...defaultDependencies, ...dependenciesOverride }
|
||||
const initialOutcome = initialRecoveryDecision(codexHomePath, dependencies)
|
||||
if (initialOutcome) {
|
||||
return finish(initialOutcome, 0)
|
||||
}
|
||||
|
||||
const deadline = dependencies.now() + RECOVERY_MAX_TOTAL_MS
|
||||
let spawnCount = 0
|
||||
let fastFailures = 0
|
||||
while (!signal.aborted && dependencies.now() < deadline) {
|
||||
const spawnedAt = dependencies.now()
|
||||
const child = spawnRecoveryProcess(codexHomePath, dependencies)
|
||||
spawnCount += 1
|
||||
let childDown = false
|
||||
let spawnFailed = false
|
||||
let exitedAt = spawnedAt
|
||||
child.once('error', () => {
|
||||
childDown = true
|
||||
spawnFailed = true
|
||||
exitedAt = dependencies.now()
|
||||
})
|
||||
child.once('exit', () => {
|
||||
childDown = true
|
||||
exitedAt = dependencies.now()
|
||||
})
|
||||
|
||||
try {
|
||||
while (!childDown && !signal.aborted && dependencies.now() < deadline) {
|
||||
await dependencies.sleep(RECOVERY_POLL_INTERVAL_MS, signal)
|
||||
const status = dependencies.readStatus(codexHomePath)
|
||||
if (status.kind === 'complete') {
|
||||
await dependencies.terminate(child)
|
||||
return finish('completed', spawnCount)
|
||||
}
|
||||
if (status.kind === 'unreadable') {
|
||||
await dependencies.terminate(child)
|
||||
return finish('unreadable', spawnCount)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (!signal.aborted) {
|
||||
await dependencies.terminate(child)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
if (signal.aborted) {
|
||||
await dependencies.terminate(child)
|
||||
return finish('stopped', spawnCount)
|
||||
}
|
||||
if (!childDown) {
|
||||
await dependencies.terminate(child)
|
||||
return finish('gave-up', spawnCount)
|
||||
}
|
||||
if (spawnFailed && spawnCount === 1) {
|
||||
return finish('codex-unavailable', spawnCount)
|
||||
}
|
||||
if (exitedAt - spawnedAt < RECOVERY_FAST_EXIT_MS) {
|
||||
fastFailures += 1
|
||||
if (fastFailures >= RECOVERY_MAX_FAST_FAILURES) {
|
||||
return finish('gave-up', spawnCount)
|
||||
}
|
||||
}
|
||||
try {
|
||||
await dependencies.sleep(RECOVERY_RETRY_DELAY_MS, signal)
|
||||
} catch {
|
||||
return finish('stopped', spawnCount)
|
||||
}
|
||||
}
|
||||
return finish(signal.aborted ? 'stopped' : 'gave-up', spawnCount)
|
||||
}
|
||||
|
||||
export function resolveCodexBackfillSupervisorLockRoot(codexHomePath: string): string {
|
||||
const homeKey = normalizeRuntimePathForComparison(codexHomePath)
|
||||
const digest = createHash('sha256').update(homeKey).digest('hex')
|
||||
return join(getOrcaUserDataPath(), 'codex-state-db-backfill-locks', digest)
|
||||
}
|
||||
|
||||
function scopeRecoveryHostIdentity(hostIdentity: string, codexHomePath: string): string {
|
||||
const wslHome = process.platform === 'win32' ? parseWslUncPath(codexHomePath) : null
|
||||
return wslHome ? `${hostIdentity}:wsl:${wslHome.distro.toLowerCase()}` : hostIdentity
|
||||
}
|
||||
|
||||
export async function withCodexBackfillSupervisorLock<T>(
|
||||
codexHomePath: string,
|
||||
signal: AbortSignal | undefined,
|
||||
run: () => Promise<T>
|
||||
): Promise<T> {
|
||||
const hostIdentity = scopeRecoveryHostIdentity(await readManagedHookHostIdentity(), codexHomePath)
|
||||
// Reuse the crash-safe hard-link claim protocol; its storage root is Codex-specific.
|
||||
return await withManagedHookInstallLock(
|
||||
resolveCodexBackfillSupervisorLockRoot(codexHomePath),
|
||||
signal,
|
||||
run,
|
||||
hostIdentity,
|
||||
{ waitTimeoutMs: RECOVERY_OWNER_CHECK_TIMEOUT_MS }
|
||||
)
|
||||
}
|
||||
|
||||
type ActiveRecovery = {
|
||||
controller: AbortController
|
||||
ready: Promise<void>
|
||||
task: Promise<CodexStateDbBackfillRecoverySummary | null>
|
||||
}
|
||||
|
||||
const activeRecoveries = new Map<string, ActiveRecovery>()
|
||||
let stopping = false
|
||||
|
||||
export function startCodexStateDbBackfillRecoveryInBackground(
|
||||
codexHomePath: string
|
||||
): Promise<CodexStateDbBackfillRecoverySummary | null> {
|
||||
const key = normalizeRuntimePathForComparison(codexHomePath)
|
||||
const existing = activeRecoveries.get(key)
|
||||
if (existing) {
|
||||
return existing.task
|
||||
}
|
||||
if (stopping || !isCodexStateDbBackfillPending(codexHomePath)) {
|
||||
return Promise.resolve(null)
|
||||
}
|
||||
const controller = new AbortController()
|
||||
let markReady!: () => void
|
||||
const ready = new Promise<void>((resolve) => (markReady = resolve))
|
||||
const task = withCodexBackfillSupervisorLock(codexHomePath, controller.signal, async () => {
|
||||
console.info(`[codex-state-db-backfill] supervising Codex index at ${codexHomePath}`)
|
||||
markReady()
|
||||
return await runCodexStateDbBackfillRecovery(codexHomePath, controller.signal)
|
||||
}).catch((error: unknown) => {
|
||||
if (!controller.signal.aborted) {
|
||||
console.warn('[codex-state-db-backfill] recovery supervisor stopped:', error)
|
||||
}
|
||||
return null
|
||||
})
|
||||
void task.finally(markReady)
|
||||
activeRecoveries.set(key, { controller, ready, task })
|
||||
void task.finally(() => {
|
||||
if (activeRecoveries.get(key)?.task === task) {
|
||||
activeRecoveries.delete(key)
|
||||
}
|
||||
})
|
||||
return task
|
||||
}
|
||||
|
||||
/** Waits only for exact-owner arbitration, never for the potentially long Codex index. */
|
||||
export async function ensureCodexStateDbBackfillRecoveryStarted(
|
||||
codexHomePath: string
|
||||
): Promise<void> {
|
||||
void startCodexStateDbBackfillRecoveryInBackground(codexHomePath)
|
||||
await activeRecoveries.get(normalizeRuntimePathForComparison(codexHomePath))?.ready
|
||||
}
|
||||
|
||||
export async function stopCodexStateDbBackfillRecoveries(): Promise<void> {
|
||||
stopping = true
|
||||
const recoveries = [...activeRecoveries.values()]
|
||||
for (const recovery of recoveries) {
|
||||
recovery.controller.abort()
|
||||
}
|
||||
await Promise.allSettled(recoveries.map(({ task }) => task))
|
||||
}
|
||||
|
||||
export const _internals = {
|
||||
resetForTests(): void {
|
||||
stopping = false
|
||||
activeRecoveries.clear()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import SyncDatabase from '../sqlite/sync-database'
|
||||
import {
|
||||
findNewestCodexStateDbPath,
|
||||
isCodexStateDbBackfillPending,
|
||||
readCodexStateDbBackfillStatus
|
||||
} from './codex-state-db'
|
||||
|
||||
const temporaryHomes: string[] = []
|
||||
|
||||
async function createHome(): Promise<string> {
|
||||
const home = await mkdtemp(join(tmpdir(), 'orca-codex-state-db-'))
|
||||
temporaryHomes.push(home)
|
||||
return home
|
||||
}
|
||||
|
||||
function createBackfillDb(home: string, version: number, status: string): string {
|
||||
const path = join(home, `state_${version}.sqlite`)
|
||||
const db = new SyncDatabase(path)
|
||||
db.exec(
|
||||
'CREATE TABLE backfill_state (id INTEGER PRIMARY KEY, status TEXT NOT NULL); ' +
|
||||
`INSERT INTO backfill_state (id, status) VALUES (1, '${status}')`
|
||||
)
|
||||
db.close()
|
||||
return path
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporaryHomes.splice(0).map((home) => rm(home, { recursive: true, force: true }))
|
||||
)
|
||||
})
|
||||
|
||||
describe('Codex state DB backfill status', () => {
|
||||
it('reads the newest schema DB without mutating an incomplete row', async () => {
|
||||
const home = await createHome()
|
||||
createBackfillDb(home, 4, 'complete')
|
||||
const newest = createBackfillDb(home, 5, 'running')
|
||||
|
||||
expect(findNewestCodexStateDbPath(home)).toBe(newest)
|
||||
expect(readCodexStateDbBackfillStatus(home)).toEqual({
|
||||
kind: 'incomplete',
|
||||
stateDbPath: newest,
|
||||
status: 'running'
|
||||
})
|
||||
|
||||
const db = new SyncDatabase(newest, { readonly: true, fileMustExist: true })
|
||||
expect(db.prepare('SELECT status FROM backfill_state WHERE id = 1').get()).toEqual({
|
||||
status: 'running'
|
||||
})
|
||||
db.close()
|
||||
})
|
||||
|
||||
it('treats a large unindexed rollout history as pending', async () => {
|
||||
const home = await createHome()
|
||||
const sessions = join(home, 'sessions', '2026', '08', '04')
|
||||
await mkdir(sessions, { recursive: true })
|
||||
await Promise.all(
|
||||
Array.from({ length: 100 }, (_, index) =>
|
||||
writeFile(join(sessions, `rollout-${index}.jsonl`), '{}\n')
|
||||
)
|
||||
)
|
||||
|
||||
expect(isCodexStateDbBackfillPending(home)).toBe(true)
|
||||
})
|
||||
|
||||
it('does not call a complete backfill pending', async () => {
|
||||
const home = await createHome()
|
||||
createBackfillDb(home, 5, 'complete')
|
||||
|
||||
expect(isCodexStateDbBackfillPending(home)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,115 @@
|
|||
import { readdirSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import SyncDatabase from '../sqlite/sync-database'
|
||||
|
||||
const STATE_DB_FILE_PATTERN = /^state_(\d+)\.sqlite$/
|
||||
|
||||
export type CodexStateDbBackfillStatus =
|
||||
| { kind: 'complete'; stateDbPath: string }
|
||||
| { kind: 'incomplete'; stateDbPath: string; status: string }
|
||||
| { kind: 'missing' }
|
||||
| { kind: 'not-tracked'; stateDbPath: string }
|
||||
| { kind: 'unreadable'; stateDbPath: string; error: string }
|
||||
|
||||
export function findNewestCodexStateDbPath(codexHomePath: string): string | null {
|
||||
let entries: string[]
|
||||
try {
|
||||
entries = readdirSync(codexHomePath)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
let newest: { version: number; name: string } | null = null
|
||||
for (const name of entries) {
|
||||
const match = STATE_DB_FILE_PATTERN.exec(name)
|
||||
if (!match) {
|
||||
continue
|
||||
}
|
||||
const version = Number(match[1])
|
||||
if (!newest || version > newest.version) {
|
||||
newest = { version, name }
|
||||
}
|
||||
}
|
||||
return newest ? join(codexHomePath, newest.name) : null
|
||||
}
|
||||
|
||||
/** Reads Codex-owned backfill metadata without creating or mutating its database. */
|
||||
export function readCodexStateDbBackfillStatus(codexHomePath: string): CodexStateDbBackfillStatus {
|
||||
const stateDbPath = findNewestCodexStateDbPath(codexHomePath)
|
||||
if (!stateDbPath) {
|
||||
return { kind: 'missing' }
|
||||
}
|
||||
let db: SyncDatabase | null = null
|
||||
try {
|
||||
db = new SyncDatabase(stateDbPath, { readonly: true, fileMustExist: true })
|
||||
const table = db
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'backfill_state'")
|
||||
.get()
|
||||
if (!table) {
|
||||
return { kind: 'not-tracked', stateDbPath }
|
||||
}
|
||||
const row = db.prepare('SELECT status FROM backfill_state WHERE id = 1').get() as
|
||||
| { status?: unknown }
|
||||
| undefined
|
||||
if (!row || typeof row.status !== 'string') {
|
||||
return { kind: 'not-tracked', stateDbPath }
|
||||
}
|
||||
return row.status === 'complete'
|
||||
? { kind: 'complete', stateDbPath }
|
||||
: { kind: 'incomplete', stateDbPath, status: row.status }
|
||||
} catch (error) {
|
||||
return {
|
||||
kind: 'unreadable',
|
||||
stateDbPath,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
db?.close()
|
||||
} catch {
|
||||
// A close failure cannot change the read-only result already collected.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function countCodexSessionFilesUpTo(sessionsRoot: string, limit: number): number {
|
||||
let count = 0
|
||||
const pendingDirectories = [sessionsRoot]
|
||||
while (pendingDirectories.length > 0 && count < limit) {
|
||||
const directory = pendingDirectories.pop() as string
|
||||
let entries
|
||||
try {
|
||||
entries = readdirSync(directory, { withFileTypes: true })
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
pendingDirectories.push(join(directory, entry.name))
|
||||
} else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
|
||||
count += 1
|
||||
if (count >= limit) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
export const BACKFILL_PENDING_MIN_SESSION_FILES = 100
|
||||
|
||||
export function isCodexStateDbBackfillPending(codexHomePath: string): boolean {
|
||||
const status = readCodexStateDbBackfillStatus(codexHomePath)
|
||||
if (status.kind === 'incomplete') {
|
||||
return true
|
||||
}
|
||||
if (status.kind !== 'missing' && status.kind !== 'not-tracked') {
|
||||
return false
|
||||
}
|
||||
return (
|
||||
countCodexSessionFilesUpTo(
|
||||
join(codexHomePath, 'sessions'),
|
||||
BACKFILL_PENDING_MIN_SESSION_FILES
|
||||
) >= BACKFILL_PENDING_MIN_SESSION_FILES
|
||||
)
|
||||
}
|
||||
|
|
@ -219,6 +219,10 @@ import {
|
|||
import { setCodexTrustGrantTelemetry } from './codex/codex-trust-grant-telemetry'
|
||||
import { startCodexSessionBackfillInBackground } from './codex/codex-session-backfill'
|
||||
import { startCodexSessionIndexHealInBackground } from './codex/codex-session-index-heal'
|
||||
import {
|
||||
startCodexStateDbBackfillRecoveryInBackground,
|
||||
stopCodexStateDbBackfillRecoveries
|
||||
} from './codex/codex-state-db-backfill-recovery'
|
||||
import { createCodexSessionMigrationScheduler } from './codex/codex-session-migration-scheduler'
|
||||
import { prepareLegacySharedCodexSessionResume } from './codex/codex-legacy-session-resume'
|
||||
import { resolveHostCodexSessionSourceHome } from './codex/codex-session-source-home'
|
||||
|
|
@ -2232,6 +2236,7 @@ void app.whenReady().then(async () => {
|
|||
openCodeUsage = new OpenCodeUsageStore(store)
|
||||
rateLimits = new RateLimitService()
|
||||
codexRuntimeHome = new CodexRuntimeHomeService(store)
|
||||
void startCodexStateDbBackfillRecoveryInBackground(getOrcaManagedCodexHomePath())
|
||||
// Why: an incapable trust-grant host must fall back to the managed home for
|
||||
// every consumer (PTY env, rate limits, commit messages) in one place.
|
||||
codexRuntimeHome.setRealHomeLaneGate(() => isRealHomeCodexHookLaneUsable())
|
||||
|
|
@ -3059,6 +3064,7 @@ app.on('will-quit', (e) => {
|
|||
pluginMarketplaceService = null
|
||||
pluginMarketplaceInstaller = null
|
||||
const pluginHostShutdown = pluginService?.dispose() ?? Promise.resolve()
|
||||
const codexBackfillRecoveryShutdown = stopCodexStateDbBackfillRecoveries()
|
||||
pluginService = null
|
||||
setUnreadDockBadgeCount(0)
|
||||
agentHookServer.stop()
|
||||
|
|
@ -3118,6 +3124,7 @@ app.on('will-quit', (e) => {
|
|||
{ name: 'emulator', promise: emulatorShutdown },
|
||||
{ name: 'ssh', promise: sshShutdown },
|
||||
{ name: 'plugin-hosts', promise: pluginHostShutdown },
|
||||
{ name: 'codex-backfill-recovery', promise: codexBackfillRecoveryShutdown },
|
||||
{ name: 'usage-cache', promise: usageCacheFlush },
|
||||
{ name: 'stats', promise: statsFlush },
|
||||
{ name: 'state', promise: storeFlush }
|
||||
|
|
|
|||
|
|
@ -65,7 +65,8 @@ const {
|
|||
clearMigrationUnsupportedPtysForPaneKeyMock,
|
||||
clearPaneKeyAliasesForPtyMock,
|
||||
recordCodexPaneAccountMock,
|
||||
forgetCodexPaneAccountMock
|
||||
forgetCodexPaneAccountMock,
|
||||
ensureCodexBackfillRecoveryMock
|
||||
} = vi.hoisted(() => ({
|
||||
handleMock: vi.fn(),
|
||||
onMock: vi.fn(),
|
||||
|
|
@ -99,7 +100,8 @@ const {
|
|||
clearMigrationUnsupportedPtysForPaneKeyMock: vi.fn(),
|
||||
clearPaneKeyAliasesForPtyMock: vi.fn(),
|
||||
recordCodexPaneAccountMock: vi.fn(),
|
||||
forgetCodexPaneAccountMock: vi.fn()
|
||||
forgetCodexPaneAccountMock: vi.fn(),
|
||||
ensureCodexBackfillRecoveryMock: vi.fn(() => Promise.resolve())
|
||||
}))
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
|
|
@ -209,6 +211,10 @@ vi.mock('../codex/codex-pane-account-registry', () => ({
|
|||
recordCodexPaneAccount: recordCodexPaneAccountMock,
|
||||
forgetCodexPaneAccount: forgetCodexPaneAccountMock
|
||||
}))
|
||||
|
||||
vi.mock('../codex/codex-state-db-backfill-recovery', () => ({
|
||||
ensureCodexStateDbBackfillRecoveryStarted: ensureCodexBackfillRecoveryMock
|
||||
}))
|
||||
import {
|
||||
LocalPtyProvider,
|
||||
_resetLocalPtyProviderStateForTest
|
||||
|
|
@ -383,6 +389,8 @@ describe('registerPtyHandlers', () => {
|
|||
clearPaneKeyAliasesForPtyMock.mockReset()
|
||||
recordCodexPaneAccountMock.mockReset()
|
||||
forgetCodexPaneAccountMock.mockReset()
|
||||
ensureCodexBackfillRecoveryMock.mockReset()
|
||||
ensureCodexBackfillRecoveryMock.mockResolvedValue(undefined)
|
||||
mainWindow.webContents.on.mockReset()
|
||||
mainWindow.webContents.send.mockReset()
|
||||
mainWindow.webContents.removeListener.mockReset()
|
||||
|
|
@ -3060,6 +3068,38 @@ describe('registerPtyHandlers', () => {
|
|||
})
|
||||
})
|
||||
|
||||
it('arbitrates the exact backfill owner before spawning Codex', async () => {
|
||||
let releaseRecovery!: () => void
|
||||
ensureCodexBackfillRecoveryMock.mockReturnValue(
|
||||
new Promise<void>((resolve) => (releaseRecovery = resolve))
|
||||
)
|
||||
readFileSyncMock.mockReturnValue(TEST_CODEX_AUTH_JSON)
|
||||
handlers.clear()
|
||||
registerPtyHandlers(mainWindow as never, undefined, () => TEST_CODEX_HOME, (() => ({
|
||||
codexManagedAccounts: [
|
||||
{
|
||||
id: 'account-1',
|
||||
managedHomePath: TEST_CODEX_HOME,
|
||||
managedHomeRuntime: 'host'
|
||||
}
|
||||
]
|
||||
})) as never)
|
||||
|
||||
const spawnPromise = handlers.get('pty:spawn')!(null, {
|
||||
cols: 80,
|
||||
rows: 24,
|
||||
launchAgent: 'codex'
|
||||
})
|
||||
await vi.waitFor(() =>
|
||||
expect(ensureCodexBackfillRecoveryMock).toHaveBeenCalledWith(TEST_CODEX_HOME)
|
||||
)
|
||||
expect(spawnMock).not.toHaveBeenCalled()
|
||||
|
||||
releaseRecovery()
|
||||
await spawnPromise
|
||||
expect(spawnMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('does not gate a bare local shell on managed Codex auth', async () => {
|
||||
readFileSyncMock.mockImplementation((filePath: string) => {
|
||||
if (filePath.endsWith('auth.json')) {
|
||||
|
|
|
|||
|
|
@ -199,6 +199,7 @@ import {
|
|||
} from '../codex/codex-pane-account-registry'
|
||||
import { resolveCodexPaneLaunchAccount } from '../codex/codex-pane-launch-account'
|
||||
import { getSystemCodexHomePath } from '../codex/codex-home-paths'
|
||||
import { ensureCodexStateDbBackfillRecoveryStarted } from '../codex/codex-state-db-backfill-recovery'
|
||||
import {
|
||||
environmentCodexHomeOverrideContextsEqual,
|
||||
getCustomCodexHomeOverrideForLaunch,
|
||||
|
|
@ -4495,6 +4496,9 @@ export function registerPtyHandlers(
|
|||
})
|
||||
selectedCodexHomePath = resolution instanceof Promise ? await resolution : resolution
|
||||
}
|
||||
if (args.launchAgent === 'codex' && selectedCodexHomePath) {
|
||||
await ensureCodexStateDbBackfillRecoveryStarted(selectedCodexHomePath)
|
||||
}
|
||||
const codexResumeHomeSelected = Boolean(
|
||||
codexResumeHome && codexHomePathsEqual(selectedCodexHomePath, codexResumeHome.codexHomePath)
|
||||
)
|
||||
|
|
@ -5956,6 +5960,9 @@ export function registerPtyHandlers(
|
|||
})
|
||||
selectedCodexHomePath = resolution instanceof Promise ? await resolution : resolution
|
||||
}
|
||||
if (args.launchAgent === 'codex' && selectedCodexHomePath) {
|
||||
await ensureCodexStateDbBackfillRecoveryStarted(selectedCodexHomePath)
|
||||
}
|
||||
const codexResumeHomeSelected = Boolean(
|
||||
codexResumeHome &&
|
||||
codexHomePathsEqual(selectedCodexHomePath, codexResumeHome.codexHomePath)
|
||||
|
|
|
|||
|
|
@ -2,11 +2,20 @@ import { EventEmitter } from 'node:events'
|
|||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const { childSpawnMock, readFileMock, resolveCodexCommandMock, ptySpawnMock } = vi.hoisted(() => ({
|
||||
const {
|
||||
childSpawnMock,
|
||||
readFileMock,
|
||||
resolveCodexCommandMock,
|
||||
ptySpawnMock,
|
||||
isBackfillPendingMock,
|
||||
startBackfillRecoveryMock
|
||||
} = vi.hoisted(() => ({
|
||||
childSpawnMock: vi.fn(),
|
||||
readFileMock: vi.fn(),
|
||||
resolveCodexCommandMock: vi.fn(),
|
||||
ptySpawnMock: vi.fn()
|
||||
ptySpawnMock: vi.fn(),
|
||||
isBackfillPendingMock: vi.fn(() => false),
|
||||
startBackfillRecoveryMock: vi.fn(() => Promise.resolve(null))
|
||||
}))
|
||||
|
||||
vi.mock('node:child_process', () => ({
|
||||
|
|
@ -25,6 +34,14 @@ vi.mock('node-pty', () => ({
|
|||
spawn: ptySpawnMock
|
||||
}))
|
||||
|
||||
vi.mock('../codex/codex-state-db', () => ({
|
||||
isCodexStateDbBackfillPending: isBackfillPendingMock
|
||||
}))
|
||||
|
||||
vi.mock('../codex/codex-state-db-backfill-recovery', () => ({
|
||||
startCodexStateDbBackfillRecoveryInBackground: startBackfillRecoveryMock
|
||||
}))
|
||||
|
||||
// Default to signed-in so the spawn paths under test still run; the auth gate
|
||||
// itself is covered by codex-auth-presence.test.ts and the no-auth case below.
|
||||
vi.mock('./codex-auth-presence', () => ({
|
||||
|
|
@ -116,6 +133,7 @@ describe('fetchCodexRateLimits', () => {
|
|||
resolveCodexCommandMock.mockReturnValue('codex')
|
||||
vi.mocked(probeCodexAuthPresence).mockResolvedValue('present')
|
||||
readFileMock.mockRejectedValue(new Error('no auth fixture'))
|
||||
isBackfillPendingMock.mockReturnValue(false)
|
||||
vi.stubGlobal('fetch', vi.fn())
|
||||
})
|
||||
|
||||
|
|
@ -138,6 +156,20 @@ describe('fetchCodexRateLimits', () => {
|
|||
expect(ptySpawnMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not let a quota probe steal an incomplete state-DB backfill lease', async () => {
|
||||
isBackfillPendingMock.mockReturnValue(true)
|
||||
|
||||
await expect(
|
||||
fetchCodexRateLimits({ codexHomePath: '/managed-home', allowPtyFallback: false })
|
||||
).resolves.toMatchObject({
|
||||
status: 'error',
|
||||
error: expect.stringContaining('session index')
|
||||
})
|
||||
expect(startBackfillRecoveryMock).toHaveBeenCalledWith('/managed-home')
|
||||
expect(childSpawnMock).not.toHaveBeenCalled()
|
||||
expect(ptySpawnMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('preserves the aborted result when cancellation lands during the auth check', async () => {
|
||||
let resolveAuth!: (presence: 'absent') => void
|
||||
vi.mocked(probeCodexAuthPresence).mockReturnValueOnce(
|
||||
|
|
|
|||
|
|
@ -41,6 +41,8 @@ import {
|
|||
resolveCodexHomeProcessLockKey,
|
||||
withCodexHomeProcessLock
|
||||
} from '../codex-cli/codex-home-process-lock'
|
||||
import { isCodexStateDbBackfillPending } from '../codex/codex-state-db'
|
||||
import { startCodexStateDbBackfillRecoveryInBackground } from '../codex/codex-state-db-backfill-recovery'
|
||||
|
||||
const RPC_TIMEOUT_MS = 10_000
|
||||
const WSL_RPC_TIMEOUT_MS = 25_000
|
||||
|
|
@ -1249,6 +1251,19 @@ export async function fetchCodexRateLimits(
|
|||
}
|
||||
}
|
||||
|
||||
if (options?.codexHomePath && isCodexStateDbBackfillPending(options.codexHomePath)) {
|
||||
// Why: a bounded quota probe can steal an expired backfill lease, then die before indexing finishes.
|
||||
void startCodexStateDbBackfillRecoveryInBackground(options.codexHomePath)
|
||||
return {
|
||||
provider: 'codex',
|
||||
session: null,
|
||||
weekly: null,
|
||||
updatedAt: Date.now(),
|
||||
error: 'Codex is rebuilding its session index; usage will refresh when recovery finishes',
|
||||
status: 'error'
|
||||
}
|
||||
}
|
||||
|
||||
// Why: probes spawn a real codex process inside the live credential home;
|
||||
// the per-home lock keeps Orca's own spawns (probe vs probe, probe vs
|
||||
// commit-message run) from refreshing one auth.json concurrently.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
CODEX_BACKFILL_RECOVERY_NOTICE,
|
||||
createCodexBackfillErrorDetector
|
||||
} from './codex-backfill-error-detector'
|
||||
|
||||
describe('Codex backfill error detector', () => {
|
||||
it('recognizes the timeout across ANSI-decorated chunks once', () => {
|
||||
const detector = createCodexBackfillErrorDetector()
|
||||
|
||||
expect(detector.observe('\u001b[31mError: timed out waiting for state DB back')).toBeNull()
|
||||
expect(detector.observe('fill\u001b[0m\r\n')).toBe(CODEX_BACKFILL_RECOVERY_NOTICE)
|
||||
expect(detector.observe('timed out waiting for state db backfill')).toBeNull()
|
||||
})
|
||||
|
||||
it('does not classify the generic damaged-database message', () => {
|
||||
const detector = createCodexBackfillErrorDetector()
|
||||
|
||||
expect(detector.observe('local database appears to be damaged')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
export const CODEX_BACKFILL_TIMEOUT_SIGNATURE = 'timed out waiting for state db backfill'
|
||||
|
||||
export const CODEX_BACKFILL_RECOVERY_NOTICE = [
|
||||
'Codex could not start because its session-history index is still incomplete.',
|
||||
'Keep Orca open for a few minutes, then retry this pane. Orca attempts background recovery for managed local and WSL homes.'
|
||||
].join('\n')
|
||||
|
||||
const ANSI_ESCAPE_PATTERN =
|
||||
// eslint-disable-next-line no-control-regex -- terminal escape sequences contain control bytes
|
||||
/\u001b(?:\[[0-9;?]*[ -/]*[@-~]|\][^\u0007\u001b]*(?:\u0007|\u001b\\)?)/g
|
||||
const DETECTOR_BUFFER_MAX_CHARS = 4096
|
||||
|
||||
export type CodexBackfillErrorDetector = { observe(chunk: string): string | null }
|
||||
|
||||
/** Scans one Codex pane's output once for the unambiguous backfill timeout. */
|
||||
export function createCodexBackfillErrorDetector(): CodexBackfillErrorDetector {
|
||||
let tail = ''
|
||||
let armed = true
|
||||
return {
|
||||
observe(chunk: string): string | null {
|
||||
if (!armed) {
|
||||
return null
|
||||
}
|
||||
const normalized = (tail + chunk).replace(ANSI_ESCAPE_PATTERN, '').replace(/\r/g, '')
|
||||
tail = normalized.slice(-DETECTOR_BUFFER_MAX_CHARS)
|
||||
if (!tail.toLowerCase().includes(CODEX_BACKFILL_TIMEOUT_SIGNATURE)) {
|
||||
return null
|
||||
}
|
||||
armed = false
|
||||
return CODEX_BACKFILL_RECOVERY_NOTICE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2324,6 +2324,31 @@ describe('connectPanePty', () => {
|
|||
)
|
||||
})
|
||||
|
||||
it('surfaces an actionable error when a Codex backfill timeout drops to the shell', async () => {
|
||||
const { connectPanePty } = await import('./pty-connection')
|
||||
const transport = createMockTransport()
|
||||
const capturedDataCallback: { current: ((data: string) => void) | null } = { current: null }
|
||||
transport.connect.mockImplementation(async ({ callbacks }: { callbacks: ConnectCallbacks }) => {
|
||||
capturedDataCallback.current = callbacks.onData ?? null
|
||||
return 'pty-codex-backfill-timeout'
|
||||
})
|
||||
transportFactoryQueue.push(transport)
|
||||
const deps = createDeps({
|
||||
tabId: 'tab-codex-backfill-timeout',
|
||||
startup: { command: 'codex', launchAgent: 'codex' }
|
||||
})
|
||||
|
||||
connectPanePty(createPane(1) as never, createManager(1) as never, deps as never)
|
||||
await flushAsyncTicks()
|
||||
capturedDataCallback.current?.('timed out waiting for state db back')
|
||||
capturedDataCallback.current?.('fill\r\n')
|
||||
|
||||
expect(deps.onPtyErrorRef.current).toHaveBeenCalledWith(
|
||||
1,
|
||||
expect.stringContaining('Orca attempts background recovery for managed local and WSL homes')
|
||||
)
|
||||
})
|
||||
|
||||
it('drops keystrokes while the replay guard is engaged, then forwards once it releases', async () => {
|
||||
// Regression (cold-restore reattach lockout): a stuck replay guard dropped every keystroke ("can't type after reconnecting"); engaged guard suppresses input, released forwards it.
|
||||
const { connectPanePty } = await import('./pty-connection')
|
||||
|
|
|
|||
|
|
@ -235,6 +235,7 @@ import { getTerminalPasteSshRemotePlatform } from './terminal-paste-ssh-platform
|
|||
import { resolveTerminalPasteRuntime } from './terminal-paste-runtime'
|
||||
import { isKnownTuiAgentTerminalStartupCommand } from './terminal-startup-command-classifier'
|
||||
import { createCommandCodeOutputStatusDetector } from '../../../../shared/command-code-output-status'
|
||||
import { createCodexBackfillErrorDetector } from './codex-backfill-error-detector'
|
||||
import type { PtyDataMeta } from './pty-dispatcher'
|
||||
import { getEagerPtyBufferHandle } from './pty-dispatcher'
|
||||
import { createTerminalGitHubPRLinkDetector } from '../../../../shared/terminal-github-pr-link-detector'
|
||||
|
|
@ -4630,6 +4631,10 @@ export function connectPanePty(
|
|||
}
|
||||
deps.onPtyErrorRef?.current?.(pane.id, message)
|
||||
}
|
||||
const codexBackfillErrorDetector =
|
||||
paneStartup?.launchAgent === 'codex' || tab?.launchAgent === 'codex'
|
||||
? createCodexBackfillErrorDetector()
|
||||
: null
|
||||
|
||||
// Why: shared registration so both fresh-spawn and reattach paths install
|
||||
// the same SerializeAddon-backed serializer plus the onTitleChange wrapper
|
||||
|
|
@ -7747,6 +7752,10 @@ export function connectPanePty(
|
|||
commandLifecycle.handlePtyData(data)
|
||||
}
|
||||
commandCodeOutputStatusDetector?.observe(data)
|
||||
const codexBackfillNotice = codexBackfillErrorDetector?.observe(data)
|
||||
if (codexBackfillNotice) {
|
||||
reportError(codexBackfillNotice)
|
||||
}
|
||||
// Why: split panes have visible-but-inactive panes the user watches; throttle only when the pane or whole document is hidden.
|
||||
const foreground =
|
||||
shouldWritePtyOutputForeground(deps.isVisibleRef.current) && meta?.background !== true
|
||||
|
|
|
|||
Loading…
Reference in New Issue