Fix Codex session history titles from injected context (#6151)
Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
parent
b8becaae39
commit
dd421a054b
|
|
@ -9,11 +9,8 @@ import {
|
|||
import { parseKimiSessionFile } from './session-scanner-kimi-parser'
|
||||
import { splitOpenCodeSqliteCandidate } from './session-scanner-opencode-sqlite-paths'
|
||||
import { parseOpenCodeSqliteSession } from './session-scanner-opencode-sqlite'
|
||||
import {
|
||||
parseClaudeSessionFile,
|
||||
parseCodexSessionFile,
|
||||
parseGeminiSessionFile
|
||||
} from './session-scanner-primary-parsers'
|
||||
import { parseClaudeSessionFile, parseGeminiSessionFile } from './session-scanner-primary-parsers'
|
||||
import { parseCodexSessionFile } from './session-scanner-codex-parser'
|
||||
import {
|
||||
parseCopilotSessionFile,
|
||||
parseCursorSessionFile,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,88 @@
|
|||
import { mkdir, mkdtemp, rm, stat, writeFile } from 'fs/promises'
|
||||
import { tmpdir } from 'os'
|
||||
import { dirname, join } from 'path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { parseCodexSessionFile } from './session-scanner-codex-parser'
|
||||
|
||||
let tempRoots: string[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempRoots.map((root) => rm(root, { recursive: true, force: true })))
|
||||
tempRoots = []
|
||||
})
|
||||
|
||||
function jsonLines(records: unknown[]): string {
|
||||
return records.map((record) => JSON.stringify(record)).join('\n')
|
||||
}
|
||||
|
||||
describe('parseCodexSessionFile', () => {
|
||||
it('does not double-count usage when token count formats switch', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'orca-ai-vault-codex-token-switch-'))
|
||||
tempRoots.push(root)
|
||||
const sessionPath = join(root, 'sessions', '2026', '06', '18', 'rollout-token-switch.jsonl')
|
||||
await mkdir(dirname(sessionPath), { recursive: true })
|
||||
|
||||
await writeFile(
|
||||
sessionPath,
|
||||
jsonLines([
|
||||
{
|
||||
timestamp: '2026-06-18T10:00:00.000Z',
|
||||
type: 'session_meta',
|
||||
payload: { id: 'token-format-switch', cwd: '/repo/app' }
|
||||
},
|
||||
{
|
||||
timestamp: '2026-06-18T10:00:01.000Z',
|
||||
type: 'response_item',
|
||||
payload: {
|
||||
type: 'message',
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: 'Measure Codex token totals' }]
|
||||
}
|
||||
},
|
||||
{
|
||||
timestamp: '2026-06-18T10:00:02.000Z',
|
||||
type: 'event_msg',
|
||||
payload: {
|
||||
type: 'token_count',
|
||||
info: {
|
||||
last_token_usage: {
|
||||
input_tokens: 70,
|
||||
cached_input_tokens: 20,
|
||||
output_tokens: 30,
|
||||
total_tokens: 100
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
timestamp: '2026-06-18T10:00:03.000Z',
|
||||
type: 'event_msg',
|
||||
payload: {
|
||||
type: 'token_count',
|
||||
info: {
|
||||
total_token_usage: {
|
||||
input_tokens: 90,
|
||||
cached_input_tokens: 25,
|
||||
output_tokens: 60,
|
||||
total_tokens: 150
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
])
|
||||
)
|
||||
|
||||
const sessionStat = await stat(sessionPath)
|
||||
const session = await parseCodexSessionFile(
|
||||
{
|
||||
path: sessionPath,
|
||||
mtimeMs: sessionStat.mtimeMs,
|
||||
modifiedAt: sessionStat.mtime.toISOString()
|
||||
},
|
||||
'darwin',
|
||||
root
|
||||
)
|
||||
|
||||
expect(session?.totalTokens).toBe(150)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,273 @@
|
|||
import { createReadStream } from 'fs'
|
||||
import { stat } from 'fs/promises'
|
||||
import { basename, dirname, join } from 'path'
|
||||
import { createInterface } from 'readline'
|
||||
import type { AiVaultSession } from '../../shared/ai-vault-types'
|
||||
import {
|
||||
addPreviewContent,
|
||||
createAccumulator,
|
||||
finalizeSession,
|
||||
sessionIdFromFileName,
|
||||
updateTimeline
|
||||
} from './session-scanner-accumulator'
|
||||
import type { CodexUsageSnapshot, FileWithMtime } from './session-scanner-types'
|
||||
import {
|
||||
asRecord,
|
||||
extractContentText,
|
||||
extractGitBranch,
|
||||
extractModel,
|
||||
extractString,
|
||||
normalizeCodexUsage,
|
||||
normalizeTitleText,
|
||||
parseJsonObject,
|
||||
subtractCodexUsage
|
||||
} from './session-scanner-values'
|
||||
|
||||
const CODEX_SESSION_INDEX_FILE = 'session_index.jsonl'
|
||||
|
||||
type CodexSessionIndexTitleCacheEntry = {
|
||||
signature: string
|
||||
titles: Map<string, string>
|
||||
}
|
||||
|
||||
const codexSessionIndexTitleCache = new Map<string, Promise<CodexSessionIndexTitleCacheEntry>>()
|
||||
|
||||
export async function parseCodexSessionFile(
|
||||
file: FileWithMtime,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
codexHome: string | null = null
|
||||
): Promise<AiVaultSession | null> {
|
||||
const accumulator = createAccumulator({
|
||||
agent: 'codex',
|
||||
file,
|
||||
sessionId: sessionIdFromFileName(file.path)
|
||||
})
|
||||
let previousTotals: CodexUsageSnapshot | null = null
|
||||
|
||||
const lines = createInterface({
|
||||
input: createReadStream(file.path, { encoding: 'utf-8' }),
|
||||
crlfDelay: Infinity
|
||||
})
|
||||
|
||||
for await (const line of lines) {
|
||||
const record = parseJsonObject(line)
|
||||
if (!record) {
|
||||
continue
|
||||
}
|
||||
|
||||
updateTimeline(accumulator, extractString(record.timestamp))
|
||||
|
||||
const payload = asRecord(record.payload)
|
||||
if (record.type === 'session_meta' && payload) {
|
||||
if (isCodexWorkerSession(payload)) {
|
||||
// Why: Codex writes internal worker/sub-agent transcripts into the same
|
||||
// history tree; AI Vault should show user-started sessions only.
|
||||
return null
|
||||
}
|
||||
const sessionId = extractString(payload.id)
|
||||
if (sessionId) {
|
||||
accumulator.sessionId = sessionId
|
||||
}
|
||||
const indexedTitle =
|
||||
extractCodexSessionMetadataTitle(payload) ??
|
||||
(await readCodexSessionIndexTitle(file.path, codexHome, accumulator.sessionId))
|
||||
if (indexedTitle) {
|
||||
accumulator.title = indexedTitle
|
||||
}
|
||||
const cwd = extractString(payload.cwd)
|
||||
if (cwd) {
|
||||
accumulator.cwd = cwd
|
||||
}
|
||||
accumulator.branch = extractGitBranch(payload.git) ?? accumulator.branch
|
||||
continue
|
||||
}
|
||||
|
||||
if (record.type === 'turn_context' && payload) {
|
||||
const cwd = extractString(payload.cwd)
|
||||
if (cwd) {
|
||||
accumulator.cwd = cwd
|
||||
}
|
||||
const model = extractModel(payload)
|
||||
if (model) {
|
||||
accumulator.model = model
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (!payload) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (record.type === 'response_item' && payload.type === 'message') {
|
||||
accumulator.messageCount++
|
||||
if (payload.role === 'user' && !accumulator.title) {
|
||||
accumulator.title = extractContentText(payload.content)
|
||||
}
|
||||
addPreviewContent(
|
||||
accumulator,
|
||||
payload.role === 'assistant' ? 'assistant' : payload.role === 'user' ? 'user' : 'unknown',
|
||||
payload.content,
|
||||
record.timestamp
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if (record.type !== 'event_msg') {
|
||||
continue
|
||||
}
|
||||
|
||||
if (payload.type === 'user_message') {
|
||||
accumulator.messageCount++
|
||||
if (!accumulator.title) {
|
||||
accumulator.title = extractContentText(payload.message)
|
||||
}
|
||||
addPreviewContent(accumulator, 'user', payload.message, record.timestamp)
|
||||
continue
|
||||
}
|
||||
|
||||
if (payload.type === 'agent_message') {
|
||||
accumulator.messageCount++
|
||||
addPreviewContent(accumulator, 'assistant', payload.message, record.timestamp)
|
||||
continue
|
||||
}
|
||||
|
||||
if (payload.type !== 'token_count') {
|
||||
continue
|
||||
}
|
||||
|
||||
const info = asRecord(payload.info)
|
||||
if (!info) {
|
||||
continue
|
||||
}
|
||||
const totalUsage = normalizeCodexUsage(info.total_token_usage)
|
||||
const lastUsage = normalizeCodexUsage(info.last_token_usage)
|
||||
let delta: CodexUsageSnapshot | null = null
|
||||
if (totalUsage) {
|
||||
delta = subtractCodexUsage(totalUsage, previousTotals)
|
||||
previousTotals = totalUsage
|
||||
} else if (lastUsage) {
|
||||
delta = lastUsage
|
||||
previousTotals = previousTotals ? addCodexUsage(previousTotals, lastUsage) : lastUsage
|
||||
}
|
||||
if (delta) {
|
||||
accumulator.totalTokens += delta.totalTokens
|
||||
}
|
||||
const model = extractModel(payload)
|
||||
if (model) {
|
||||
accumulator.model = model
|
||||
}
|
||||
}
|
||||
|
||||
return finalizeSession(accumulator, platform, { codexHome })
|
||||
}
|
||||
|
||||
function addCodexUsage(
|
||||
base: CodexUsageSnapshot,
|
||||
increment: CodexUsageSnapshot
|
||||
): CodexUsageSnapshot {
|
||||
return {
|
||||
inputTokens: base.inputTokens + increment.inputTokens,
|
||||
cachedInputTokens: base.cachedInputTokens + increment.cachedInputTokens,
|
||||
outputTokens: base.outputTokens + increment.outputTokens,
|
||||
reasoningOutputTokens: base.reasoningOutputTokens + increment.reasoningOutputTokens,
|
||||
totalTokens: base.totalTokens + increment.totalTokens
|
||||
}
|
||||
}
|
||||
|
||||
function extractCodexThreadSource(payload: Record<string, unknown>): string | null {
|
||||
return extractString(payload.thread_source) ?? extractString(payload.threadSource)
|
||||
}
|
||||
|
||||
function isCodexWorkerSession(payload: Record<string, unknown>): boolean {
|
||||
const threadSource = extractCodexThreadSource(payload)
|
||||
if (threadSource) {
|
||||
return threadSource.toLowerCase() !== 'user'
|
||||
}
|
||||
|
||||
const source = asRecord(payload.source)
|
||||
return Boolean(asRecord(source?.subagent))
|
||||
}
|
||||
|
||||
function extractCodexSessionMetadataTitle(payload: Record<string, unknown>): string | null {
|
||||
return (
|
||||
normalizeTitleText(extractString(payload.title) ?? '') ??
|
||||
normalizeTitleText(extractString(payload.thread_name) ?? '') ??
|
||||
normalizeTitleText(extractString(payload.threadName) ?? '')
|
||||
)
|
||||
}
|
||||
|
||||
async function readCodexSessionIndexTitle(
|
||||
sessionFilePath: string,
|
||||
codexHome: string | null,
|
||||
sessionId: string
|
||||
): Promise<string | null> {
|
||||
const resolvedCodexHome = codexHome ?? codexHomeFromSessionFilePath(sessionFilePath)
|
||||
if (!resolvedCodexHome) {
|
||||
return null
|
||||
}
|
||||
const titleBySessionId = await readCodexSessionIndexTitles(resolvedCodexHome)
|
||||
return titleBySessionId.get(sessionId) ?? null
|
||||
}
|
||||
|
||||
function codexHomeFromSessionFilePath(sessionFilePath: string): string | null {
|
||||
let currentDir = dirname(sessionFilePath)
|
||||
while (currentDir && dirname(currentDir) !== currentDir) {
|
||||
if (basename(currentDir) === 'sessions') {
|
||||
return dirname(currentDir)
|
||||
}
|
||||
currentDir = dirname(currentDir)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
async function readCodexSessionIndexTitles(codexHome: string): Promise<Map<string, string>> {
|
||||
const indexPath = join(codexHome, CODEX_SESSION_INDEX_FILE)
|
||||
let signature: string
|
||||
try {
|
||||
const indexStat = await stat(indexPath)
|
||||
signature = `${indexStat.size}:${indexStat.mtimeMs}`
|
||||
} catch {
|
||||
return new Map()
|
||||
}
|
||||
|
||||
const cached = codexSessionIndexTitleCache.get(codexHome)
|
||||
if (cached) {
|
||||
const entry = await cached
|
||||
if (entry.signature === signature) {
|
||||
return entry.titles
|
||||
}
|
||||
}
|
||||
|
||||
const pending = readCodexSessionIndexTitlesFromDisk(indexPath).then((titles) => ({
|
||||
signature,
|
||||
titles
|
||||
}))
|
||||
codexSessionIndexTitleCache.set(codexHome, pending)
|
||||
return (await pending).titles
|
||||
}
|
||||
|
||||
async function readCodexSessionIndexTitlesFromDisk(
|
||||
indexPath: string
|
||||
): Promise<Map<string, string>> {
|
||||
const titleBySessionId = new Map<string, string>()
|
||||
try {
|
||||
const lines = createInterface({
|
||||
input: createReadStream(indexPath, { encoding: 'utf-8' }),
|
||||
crlfDelay: Infinity
|
||||
})
|
||||
for await (const line of lines) {
|
||||
const record = parseJsonObject(line)
|
||||
if (!record) {
|
||||
continue
|
||||
}
|
||||
const sessionId = extractString(record.id)
|
||||
const title = normalizeTitleText(extractString(record.thread_name) ?? '')
|
||||
if (sessionId && title) {
|
||||
titleBySessionId.set(sessionId, title)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Codex creates the index opportunistically; older homes may only have raw transcripts.
|
||||
}
|
||||
return titleBySessionId
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ import { createReadStream } from 'fs'
|
|||
import { readFile } from 'fs/promises'
|
||||
import { createInterface } from 'readline'
|
||||
import type { AiVaultSession } from '../../shared/ai-vault-types'
|
||||
import type { CodexUsageSnapshot, FileWithMtime, SessionAccumulator } from './session-scanner-types'
|
||||
import type { FileWithMtime, SessionAccumulator } from './session-scanner-types'
|
||||
import {
|
||||
addPreviewContent,
|
||||
createAccumulator,
|
||||
|
|
@ -16,14 +16,10 @@ import {
|
|||
asRecord,
|
||||
claudeUsageTotal,
|
||||
extractContentText,
|
||||
extractGitBranch,
|
||||
extractMessageText,
|
||||
extractModel,
|
||||
extractString,
|
||||
normalizeCodexUsage,
|
||||
normalizeTitleText,
|
||||
parseJsonObject,
|
||||
subtractCodexUsage,
|
||||
tokenTotal
|
||||
} from './session-scanner-values'
|
||||
|
||||
|
|
@ -99,139 +95,6 @@ export async function parseClaudeSessionFile(
|
|||
return finalizeSession(accumulator, platform)
|
||||
}
|
||||
|
||||
export async function parseCodexSessionFile(
|
||||
file: FileWithMtime,
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
codexHome: string | null = null
|
||||
): Promise<AiVaultSession | null> {
|
||||
const accumulator = createAccumulator({
|
||||
agent: 'codex',
|
||||
file,
|
||||
sessionId: sessionIdFromFileName(file.path)
|
||||
})
|
||||
let previousTotals: CodexUsageSnapshot | null = null
|
||||
|
||||
const lines = createInterface({
|
||||
input: createReadStream(file.path, { encoding: 'utf-8' }),
|
||||
crlfDelay: Infinity
|
||||
})
|
||||
|
||||
for await (const line of lines) {
|
||||
const record = parseJsonObject(line)
|
||||
if (!record) {
|
||||
continue
|
||||
}
|
||||
|
||||
updateTimeline(accumulator, extractString(record.timestamp))
|
||||
|
||||
const payload = asRecord(record.payload)
|
||||
if (record.type === 'session_meta' && payload) {
|
||||
if (isCodexWorkerSession(payload)) {
|
||||
// Why: Codex writes internal worker/sub-agent transcripts into the same
|
||||
// history tree; AI Vault should show user-started sessions only.
|
||||
return null
|
||||
}
|
||||
const sessionId = extractString(payload.id)
|
||||
if (sessionId) {
|
||||
accumulator.sessionId = sessionId
|
||||
}
|
||||
const cwd = extractString(payload.cwd)
|
||||
if (cwd) {
|
||||
accumulator.cwd = cwd
|
||||
}
|
||||
accumulator.branch = extractGitBranch(payload.git) ?? accumulator.branch
|
||||
continue
|
||||
}
|
||||
|
||||
if (record.type === 'turn_context' && payload) {
|
||||
const cwd = extractString(payload.cwd)
|
||||
if (cwd) {
|
||||
accumulator.cwd = cwd
|
||||
}
|
||||
const model = extractModel(payload)
|
||||
if (model) {
|
||||
accumulator.model = model
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (!payload) {
|
||||
continue
|
||||
}
|
||||
|
||||
if (record.type === 'response_item' && payload.type === 'message') {
|
||||
accumulator.messageCount++
|
||||
if (payload.role === 'user' && !accumulator.title) {
|
||||
accumulator.title = extractContentText(payload.content)
|
||||
}
|
||||
addPreviewContent(
|
||||
accumulator,
|
||||
payload.role === 'assistant' ? 'assistant' : payload.role === 'user' ? 'user' : 'unknown',
|
||||
payload.content,
|
||||
record.timestamp
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if (record.type !== 'event_msg') {
|
||||
continue
|
||||
}
|
||||
|
||||
if (payload.type === 'user_message') {
|
||||
accumulator.messageCount++
|
||||
if (!accumulator.title) {
|
||||
accumulator.title = extractContentText(payload.message)
|
||||
}
|
||||
addPreviewContent(accumulator, 'user', payload.message, record.timestamp)
|
||||
continue
|
||||
}
|
||||
|
||||
if (payload.type === 'agent_message') {
|
||||
accumulator.messageCount++
|
||||
addPreviewContent(accumulator, 'assistant', payload.message, record.timestamp)
|
||||
continue
|
||||
}
|
||||
|
||||
if (payload.type !== 'token_count') {
|
||||
continue
|
||||
}
|
||||
|
||||
const info = asRecord(payload.info)
|
||||
if (!info) {
|
||||
continue
|
||||
}
|
||||
const totalUsage = normalizeCodexUsage(info.total_token_usage)
|
||||
const lastUsage = normalizeCodexUsage(info.last_token_usage)
|
||||
const delta = totalUsage ? subtractCodexUsage(totalUsage, previousTotals) : lastUsage
|
||||
if (totalUsage) {
|
||||
previousTotals = totalUsage
|
||||
}
|
||||
if (delta) {
|
||||
accumulator.totalTokens += delta.totalTokens
|
||||
}
|
||||
const model = extractModel(payload)
|
||||
if (model) {
|
||||
accumulator.model = model
|
||||
}
|
||||
}
|
||||
|
||||
return finalizeSession(accumulator, platform, { codexHome })
|
||||
}
|
||||
|
||||
function extractCodexThreadSource(payload: Record<string, unknown>): string | null {
|
||||
return extractString(payload.thread_source) ?? extractString(payload.threadSource)
|
||||
}
|
||||
|
||||
function isCodexWorkerSession(payload: Record<string, unknown>): boolean {
|
||||
const threadSource = extractCodexThreadSource(payload)
|
||||
if (threadSource) {
|
||||
return threadSource.toLowerCase() !== 'user'
|
||||
}
|
||||
|
||||
const source = asRecord(payload.source)
|
||||
return Boolean(asRecord(source?.subagent))
|
||||
}
|
||||
|
||||
export async function parseGeminiSessionFile(
|
||||
file: FileWithMtime,
|
||||
platform: NodeJS.Platform = process.platform
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ const HIDDEN_BLOCK_CLOSE_SCAN_LIMIT = 256 * 1024
|
|||
const HIDDEN_BLOCK_OPEN_TAG_SCAN_LIMIT = 512
|
||||
const FIELD_BLANK_SCAN_LIMIT = 1024
|
||||
|
||||
const AGENTS_INSTRUCTIONS_PREFIX = '# AGENTS.md instructions for'
|
||||
const AGENTS_INSTRUCTIONS_PREFIX = '# AGENTS.md instructions'
|
||||
const XML_INSTRUCTIONS_PREFIX = '<INSTRUCTIONS>'
|
||||
|
||||
const HIDDEN_TEXT_BLOCKS = [
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@ describe('scanAiVaultSessions', () => {
|
|||
type: 'message',
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: '# AGENTS.md instructions for /repo/app <INSTRUCTIONS>' }
|
||||
{ type: 'text', text: '# AGENTS.md instructions\n\n<INSTRUCTIONS>repo policy' }
|
||||
]
|
||||
}
|
||||
}),
|
||||
|
|
@ -165,6 +165,15 @@ describe('scanAiVaultSessions', () => {
|
|||
})
|
||||
].join('\n')
|
||||
)
|
||||
await writeFile(
|
||||
join(root, 'session_index.jsonl'),
|
||||
jsonLines([
|
||||
{
|
||||
id: '019f0000-1111-7222-8333-444444444444',
|
||||
thread_name: 'Indexed Codex resume picker title'
|
||||
}
|
||||
])
|
||||
)
|
||||
|
||||
const result = await scanAiVaultSessions({
|
||||
...roots,
|
||||
|
|
@ -174,7 +183,7 @@ describe('scanAiVaultSessions', () => {
|
|||
expect(result.issues).toEqual([])
|
||||
expect(result.sessions).toHaveLength(2)
|
||||
expect(result.sessions.map((session) => session.title).sort()).toEqual([
|
||||
'Fix the resume picker filters',
|
||||
'Indexed Codex resume picker title',
|
||||
'Vault polish pass'
|
||||
])
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue