diff --git a/src/main/ai-vault/session-scanner-agent-parser.ts b/src/main/ai-vault/session-scanner-agent-parser.ts index 385b29619..adfb2aa86 100644 --- a/src/main/ai-vault/session-scanner-agent-parser.ts +++ b/src/main/ai-vault/session-scanner-agent-parser.ts @@ -9,11 +9,8 @@ import { import { parseKimiSessionFile } from './session-scanner-kimi-parser' import { splitOpenCodeSqliteCandidate } from './session-scanner-opencode-sqlite-paths' import { parseOpenCodeSqliteSession } from './session-scanner-opencode-sqlite' -import { - parseClaudeSessionFile, - parseCodexSessionFile, - parseGeminiSessionFile -} from './session-scanner-primary-parsers' +import { parseClaudeSessionFile, parseGeminiSessionFile } from './session-scanner-primary-parsers' +import { parseCodexSessionFile } from './session-scanner-codex-parser' import { parseCopilotSessionFile, parseCursorSessionFile, diff --git a/src/main/ai-vault/session-scanner-codex-parser.test.ts b/src/main/ai-vault/session-scanner-codex-parser.test.ts new file mode 100644 index 000000000..19581fc87 --- /dev/null +++ b/src/main/ai-vault/session-scanner-codex-parser.test.ts @@ -0,0 +1,88 @@ +import { mkdir, mkdtemp, rm, stat, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { dirname, join } from 'path' +import { afterEach, describe, expect, it } from 'vitest' +import { parseCodexSessionFile } from './session-scanner-codex-parser' + +let tempRoots: string[] = [] + +afterEach(async () => { + await Promise.all(tempRoots.map((root) => rm(root, { recursive: true, force: true }))) + tempRoots = [] +}) + +function jsonLines(records: unknown[]): string { + return records.map((record) => JSON.stringify(record)).join('\n') +} + +describe('parseCodexSessionFile', () => { + it('does not double-count usage when token count formats switch', async () => { + const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-codex-token-switch-')) + tempRoots.push(root) + const sessionPath = join(root, 'sessions', '2026', '06', '18', 'rollout-token-switch.jsonl') + await mkdir(dirname(sessionPath), { recursive: true }) + + await writeFile( + sessionPath, + jsonLines([ + { + timestamp: '2026-06-18T10:00:00.000Z', + type: 'session_meta', + payload: { id: 'token-format-switch', cwd: '/repo/app' } + }, + { + timestamp: '2026-06-18T10:00:01.000Z', + type: 'response_item', + payload: { + type: 'message', + role: 'user', + content: [{ type: 'text', text: 'Measure Codex token totals' }] + } + }, + { + timestamp: '2026-06-18T10:00:02.000Z', + type: 'event_msg', + payload: { + type: 'token_count', + info: { + last_token_usage: { + input_tokens: 70, + cached_input_tokens: 20, + output_tokens: 30, + total_tokens: 100 + } + } + } + }, + { + timestamp: '2026-06-18T10:00:03.000Z', + type: 'event_msg', + payload: { + type: 'token_count', + info: { + total_token_usage: { + input_tokens: 90, + cached_input_tokens: 25, + output_tokens: 60, + total_tokens: 150 + } + } + } + } + ]) + ) + + const sessionStat = await stat(sessionPath) + const session = await parseCodexSessionFile( + { + path: sessionPath, + mtimeMs: sessionStat.mtimeMs, + modifiedAt: sessionStat.mtime.toISOString() + }, + 'darwin', + root + ) + + expect(session?.totalTokens).toBe(150) + }) +}) diff --git a/src/main/ai-vault/session-scanner-codex-parser.ts b/src/main/ai-vault/session-scanner-codex-parser.ts new file mode 100644 index 000000000..5435b2eaf --- /dev/null +++ b/src/main/ai-vault/session-scanner-codex-parser.ts @@ -0,0 +1,273 @@ +import { createReadStream } from 'fs' +import { stat } from 'fs/promises' +import { basename, dirname, join } from 'path' +import { createInterface } from 'readline' +import type { AiVaultSession } from '../../shared/ai-vault-types' +import { + addPreviewContent, + createAccumulator, + finalizeSession, + sessionIdFromFileName, + updateTimeline +} from './session-scanner-accumulator' +import type { CodexUsageSnapshot, FileWithMtime } from './session-scanner-types' +import { + asRecord, + extractContentText, + extractGitBranch, + extractModel, + extractString, + normalizeCodexUsage, + normalizeTitleText, + parseJsonObject, + subtractCodexUsage +} from './session-scanner-values' + +const CODEX_SESSION_INDEX_FILE = 'session_index.jsonl' + +type CodexSessionIndexTitleCacheEntry = { + signature: string + titles: Map +} + +const codexSessionIndexTitleCache = new Map>() + +export async function parseCodexSessionFile( + file: FileWithMtime, + platform: NodeJS.Platform = process.platform, + codexHome: string | null = null +): Promise { + const accumulator = createAccumulator({ + agent: 'codex', + file, + sessionId: sessionIdFromFileName(file.path) + }) + let previousTotals: CodexUsageSnapshot | null = null + + const lines = createInterface({ + input: createReadStream(file.path, { encoding: 'utf-8' }), + crlfDelay: Infinity + }) + + for await (const line of lines) { + const record = parseJsonObject(line) + if (!record) { + continue + } + + updateTimeline(accumulator, extractString(record.timestamp)) + + const payload = asRecord(record.payload) + if (record.type === 'session_meta' && payload) { + if (isCodexWorkerSession(payload)) { + // Why: Codex writes internal worker/sub-agent transcripts into the same + // history tree; AI Vault should show user-started sessions only. + return null + } + const sessionId = extractString(payload.id) + if (sessionId) { + accumulator.sessionId = sessionId + } + const indexedTitle = + extractCodexSessionMetadataTitle(payload) ?? + (await readCodexSessionIndexTitle(file.path, codexHome, accumulator.sessionId)) + if (indexedTitle) { + accumulator.title = indexedTitle + } + const cwd = extractString(payload.cwd) + if (cwd) { + accumulator.cwd = cwd + } + accumulator.branch = extractGitBranch(payload.git) ?? accumulator.branch + continue + } + + if (record.type === 'turn_context' && payload) { + const cwd = extractString(payload.cwd) + if (cwd) { + accumulator.cwd = cwd + } + const model = extractModel(payload) + if (model) { + accumulator.model = model + } + continue + } + + if (!payload) { + continue + } + + if (record.type === 'response_item' && payload.type === 'message') { + accumulator.messageCount++ + if (payload.role === 'user' && !accumulator.title) { + accumulator.title = extractContentText(payload.content) + } + addPreviewContent( + accumulator, + payload.role === 'assistant' ? 'assistant' : payload.role === 'user' ? 'user' : 'unknown', + payload.content, + record.timestamp + ) + continue + } + + if (record.type !== 'event_msg') { + continue + } + + if (payload.type === 'user_message') { + accumulator.messageCount++ + if (!accumulator.title) { + accumulator.title = extractContentText(payload.message) + } + addPreviewContent(accumulator, 'user', payload.message, record.timestamp) + continue + } + + if (payload.type === 'agent_message') { + accumulator.messageCount++ + addPreviewContent(accumulator, 'assistant', payload.message, record.timestamp) + continue + } + + if (payload.type !== 'token_count') { + continue + } + + const info = asRecord(payload.info) + if (!info) { + continue + } + const totalUsage = normalizeCodexUsage(info.total_token_usage) + const lastUsage = normalizeCodexUsage(info.last_token_usage) + let delta: CodexUsageSnapshot | null = null + if (totalUsage) { + delta = subtractCodexUsage(totalUsage, previousTotals) + previousTotals = totalUsage + } else if (lastUsage) { + delta = lastUsage + previousTotals = previousTotals ? addCodexUsage(previousTotals, lastUsage) : lastUsage + } + if (delta) { + accumulator.totalTokens += delta.totalTokens + } + const model = extractModel(payload) + if (model) { + accumulator.model = model + } + } + + return finalizeSession(accumulator, platform, { codexHome }) +} + +function addCodexUsage( + base: CodexUsageSnapshot, + increment: CodexUsageSnapshot +): CodexUsageSnapshot { + return { + inputTokens: base.inputTokens + increment.inputTokens, + cachedInputTokens: base.cachedInputTokens + increment.cachedInputTokens, + outputTokens: base.outputTokens + increment.outputTokens, + reasoningOutputTokens: base.reasoningOutputTokens + increment.reasoningOutputTokens, + totalTokens: base.totalTokens + increment.totalTokens + } +} + +function extractCodexThreadSource(payload: Record): string | null { + return extractString(payload.thread_source) ?? extractString(payload.threadSource) +} + +function isCodexWorkerSession(payload: Record): boolean { + const threadSource = extractCodexThreadSource(payload) + if (threadSource) { + return threadSource.toLowerCase() !== 'user' + } + + const source = asRecord(payload.source) + return Boolean(asRecord(source?.subagent)) +} + +function extractCodexSessionMetadataTitle(payload: Record): string | null { + return ( + normalizeTitleText(extractString(payload.title) ?? '') ?? + normalizeTitleText(extractString(payload.thread_name) ?? '') ?? + normalizeTitleText(extractString(payload.threadName) ?? '') + ) +} + +async function readCodexSessionIndexTitle( + sessionFilePath: string, + codexHome: string | null, + sessionId: string +): Promise { + const resolvedCodexHome = codexHome ?? codexHomeFromSessionFilePath(sessionFilePath) + if (!resolvedCodexHome) { + return null + } + const titleBySessionId = await readCodexSessionIndexTitles(resolvedCodexHome) + return titleBySessionId.get(sessionId) ?? null +} + +function codexHomeFromSessionFilePath(sessionFilePath: string): string | null { + let currentDir = dirname(sessionFilePath) + while (currentDir && dirname(currentDir) !== currentDir) { + if (basename(currentDir) === 'sessions') { + return dirname(currentDir) + } + currentDir = dirname(currentDir) + } + return null +} + +async function readCodexSessionIndexTitles(codexHome: string): Promise> { + const indexPath = join(codexHome, CODEX_SESSION_INDEX_FILE) + let signature: string + try { + const indexStat = await stat(indexPath) + signature = `${indexStat.size}:${indexStat.mtimeMs}` + } catch { + return new Map() + } + + const cached = codexSessionIndexTitleCache.get(codexHome) + if (cached) { + const entry = await cached + if (entry.signature === signature) { + return entry.titles + } + } + + const pending = readCodexSessionIndexTitlesFromDisk(indexPath).then((titles) => ({ + signature, + titles + })) + codexSessionIndexTitleCache.set(codexHome, pending) + return (await pending).titles +} + +async function readCodexSessionIndexTitlesFromDisk( + indexPath: string +): Promise> { + const titleBySessionId = new Map() + try { + const lines = createInterface({ + input: createReadStream(indexPath, { encoding: 'utf-8' }), + crlfDelay: Infinity + }) + for await (const line of lines) { + const record = parseJsonObject(line) + if (!record) { + continue + } + const sessionId = extractString(record.id) + const title = normalizeTitleText(extractString(record.thread_name) ?? '') + if (sessionId && title) { + titleBySessionId.set(sessionId, title) + } + } + } catch { + // Codex creates the index opportunistically; older homes may only have raw transcripts. + } + return titleBySessionId +} diff --git a/src/main/ai-vault/session-scanner-primary-parsers.ts b/src/main/ai-vault/session-scanner-primary-parsers.ts index fced9ab1f..dc062f5ab 100644 --- a/src/main/ai-vault/session-scanner-primary-parsers.ts +++ b/src/main/ai-vault/session-scanner-primary-parsers.ts @@ -2,7 +2,7 @@ import { createReadStream } from 'fs' import { readFile } from 'fs/promises' import { createInterface } from 'readline' import type { AiVaultSession } from '../../shared/ai-vault-types' -import type { CodexUsageSnapshot, FileWithMtime, SessionAccumulator } from './session-scanner-types' +import type { FileWithMtime, SessionAccumulator } from './session-scanner-types' import { addPreviewContent, createAccumulator, @@ -16,14 +16,10 @@ import { asRecord, claudeUsageTotal, extractContentText, - extractGitBranch, extractMessageText, - extractModel, extractString, - normalizeCodexUsage, normalizeTitleText, parseJsonObject, - subtractCodexUsage, tokenTotal } from './session-scanner-values' @@ -99,139 +95,6 @@ export async function parseClaudeSessionFile( return finalizeSession(accumulator, platform) } -export async function parseCodexSessionFile( - file: FileWithMtime, - platform: NodeJS.Platform = process.platform, - codexHome: string | null = null -): Promise { - const accumulator = createAccumulator({ - agent: 'codex', - file, - sessionId: sessionIdFromFileName(file.path) - }) - let previousTotals: CodexUsageSnapshot | null = null - - const lines = createInterface({ - input: createReadStream(file.path, { encoding: 'utf-8' }), - crlfDelay: Infinity - }) - - for await (const line of lines) { - const record = parseJsonObject(line) - if (!record) { - continue - } - - updateTimeline(accumulator, extractString(record.timestamp)) - - const payload = asRecord(record.payload) - if (record.type === 'session_meta' && payload) { - if (isCodexWorkerSession(payload)) { - // Why: Codex writes internal worker/sub-agent transcripts into the same - // history tree; AI Vault should show user-started sessions only. - return null - } - const sessionId = extractString(payload.id) - if (sessionId) { - accumulator.sessionId = sessionId - } - const cwd = extractString(payload.cwd) - if (cwd) { - accumulator.cwd = cwd - } - accumulator.branch = extractGitBranch(payload.git) ?? accumulator.branch - continue - } - - if (record.type === 'turn_context' && payload) { - const cwd = extractString(payload.cwd) - if (cwd) { - accumulator.cwd = cwd - } - const model = extractModel(payload) - if (model) { - accumulator.model = model - } - continue - } - - if (!payload) { - continue - } - - if (record.type === 'response_item' && payload.type === 'message') { - accumulator.messageCount++ - if (payload.role === 'user' && !accumulator.title) { - accumulator.title = extractContentText(payload.content) - } - addPreviewContent( - accumulator, - payload.role === 'assistant' ? 'assistant' : payload.role === 'user' ? 'user' : 'unknown', - payload.content, - record.timestamp - ) - continue - } - - if (record.type !== 'event_msg') { - continue - } - - if (payload.type === 'user_message') { - accumulator.messageCount++ - if (!accumulator.title) { - accumulator.title = extractContentText(payload.message) - } - addPreviewContent(accumulator, 'user', payload.message, record.timestamp) - continue - } - - if (payload.type === 'agent_message') { - accumulator.messageCount++ - addPreviewContent(accumulator, 'assistant', payload.message, record.timestamp) - continue - } - - if (payload.type !== 'token_count') { - continue - } - - const info = asRecord(payload.info) - if (!info) { - continue - } - const totalUsage = normalizeCodexUsage(info.total_token_usage) - const lastUsage = normalizeCodexUsage(info.last_token_usage) - const delta = totalUsage ? subtractCodexUsage(totalUsage, previousTotals) : lastUsage - if (totalUsage) { - previousTotals = totalUsage - } - if (delta) { - accumulator.totalTokens += delta.totalTokens - } - const model = extractModel(payload) - if (model) { - accumulator.model = model - } - } - - return finalizeSession(accumulator, platform, { codexHome }) -} - -function extractCodexThreadSource(payload: Record): string | null { - return extractString(payload.thread_source) ?? extractString(payload.threadSource) -} - -function isCodexWorkerSession(payload: Record): boolean { - const threadSource = extractCodexThreadSource(payload) - if (threadSource) { - return threadSource.toLowerCase() !== 'user' - } - - const source = asRecord(payload.source) - return Boolean(asRecord(source?.subagent)) -} - export async function parseGeminiSessionFile( file: FileWithMtime, platform: NodeJS.Platform = process.platform diff --git a/src/main/ai-vault/session-scanner-text-normalization.ts b/src/main/ai-vault/session-scanner-text-normalization.ts index 8bd58ad64..88eae73eb 100644 --- a/src/main/ai-vault/session-scanner-text-normalization.ts +++ b/src/main/ai-vault/session-scanner-text-normalization.ts @@ -5,7 +5,7 @@ const HIDDEN_BLOCK_CLOSE_SCAN_LIMIT = 256 * 1024 const HIDDEN_BLOCK_OPEN_TAG_SCAN_LIMIT = 512 const FIELD_BLANK_SCAN_LIMIT = 1024 -const AGENTS_INSTRUCTIONS_PREFIX = '# AGENTS.md instructions for' +const AGENTS_INSTRUCTIONS_PREFIX = '# AGENTS.md instructions' const XML_INSTRUCTIONS_PREFIX = '' const HIDDEN_TEXT_BLOCKS = [ diff --git a/src/main/ai-vault/session-scanner.test.ts b/src/main/ai-vault/session-scanner.test.ts index d8afc36b7..a4c5cf654 100644 --- a/src/main/ai-vault/session-scanner.test.ts +++ b/src/main/ai-vault/session-scanner.test.ts @@ -113,7 +113,7 @@ describe('scanAiVaultSessions', () => { type: 'message', role: 'user', content: [ - { type: 'text', text: '# AGENTS.md instructions for /repo/app ' } + { type: 'text', text: '# AGENTS.md instructions\n\nrepo policy' } ] } }), @@ -165,6 +165,15 @@ describe('scanAiVaultSessions', () => { }) ].join('\n') ) + await writeFile( + join(root, 'session_index.jsonl'), + jsonLines([ + { + id: '019f0000-1111-7222-8333-444444444444', + thread_name: 'Indexed Codex resume picker title' + } + ]) + ) const result = await scanAiVaultSessions({ ...roots, @@ -174,7 +183,7 @@ describe('scanAiVaultSessions', () => { expect(result.issues).toEqual([]) expect(result.sessions).toHaveLength(2) expect(result.sessions.map((session) => session.title).sort()).toEqual([ - 'Fix the resume picker filters', + 'Indexed Codex resume picker title', 'Vault polish pass' ])