diff --git a/docs/reference/telemetry-availability.md b/docs/reference/telemetry-availability.md index 9c3091b9a..4626c972e 100644 --- a/docs/reference/telemetry-availability.md +++ b/docs/reference/telemetry-availability.md @@ -39,13 +39,14 @@ D1+/D3+/D7+ retention means the user fired `app_opened` at least once after 24/7 - `source = 'unknown'` is not a real product surface. It means the caller omitted a source or the value failed schema validation. - `workspace_created` means create-worktree IPC succeeded. It is not a general "usable workspace exists" or "workspace revealed" marker. - `agent_started` means PTY spawn succeeded with agent telemetry attached. It is not first-repo activation and does not prove the user sent a prompt. +- `agent_prompt_sent` means a live agent hook observed an explicit non-empty user prompt. It excludes hydrated/replayed status, agent auto-start, bare shells, draft prefill, and hookless sessions; missing rows mean no hook-confirmed interaction was observed, not proof the user never typed. - Workspace-outcome joins are native Electron coverage unless the query explicitly proves remote/web instrumentation. Remote runtime and web paths can bypass native repo/worktree telemetry, so do not interpret missing workspace outcome rows as product drop-off for SSH, remote, or web users. ## Rollouts ### 2026-05-08 - Repo Cohort Property -Scope: `nth_repo_added` on repo/activation/retention events. Current schemas declare it on `app_opened`, `repo_added`, `add_repo_setup_step_action`, `add_repo_existing_workspaces_detected`, `workspace_created`, `workspace_create_failed`, `setup_script_prompt_shown`, `setup_script_prompt_action`, `agent_started`, and `agent_error`. The original rollout covered `app_opened`, `repo_added`, `add_repo_setup_step_action`, `workspace_created`, `workspace_create_failed`, `agent_started`, and `agent_error`; later events have their own first-seen timestamps below. +Scope: `nth_repo_added` on repo/activation/retention events. Current schemas declare it on `app_opened`, `repo_added`, `add_repo_setup_step_action`, `add_repo_existing_workspaces_detected`, `workspace_created`, `workspace_create_failed`, `setup_script_prompt_shown`, `setup_script_prompt_action`, `agent_started`, `agent_prompt_sent`, and `agent_error`. The original rollout covered `app_opened`, `repo_added`, `add_repo_setup_step_action`, `workspace_created`, `workspace_create_failed`, `agent_started`, and `agent_error`; later events have their own first-seen timestamps below. | Field | Value | | --- | --- | @@ -65,6 +66,7 @@ PostHog evidence checked at `2026-05-23T23:34:32Z`: | `app_opened` | `2026-05-08T18:40:00.354Z` (`1.3.42-rc.1`) | | `workspace_created` | `2026-05-08T19:15:06.913Z` | | `agent_started` | `2026-05-08T19:15:07.130Z` | +| `agent_prompt_sent` | `TBD` | | `repo_added` | `2026-05-08T20:01:06.364Z` (`1.3.42`) | | `add_repo_setup_step_action` | `2026-05-08T20:01:20.897Z` | | `workspace_create_failed` | `2026-05-08T23:04:32.315Z` | diff --git a/src/main/agent-hooks/server.test.ts b/src/main/agent-hooks/server.test.ts index 3f4c2d644..926f24d89 100644 --- a/src/main/agent-hooks/server.test.ts +++ b/src/main/agent-hooks/server.test.ts @@ -20,7 +20,8 @@ import { } from '../../shared/agent-status-types' import { makePaneKey } from '../../shared/stable-pane-id' -const { trackMock } = vi.hoisted(() => ({ +const { getCohortAtEmitMock, trackMock } = vi.hoisted(() => ({ + getCohortAtEmitMock: vi.fn(), trackMock: vi.fn() })) @@ -28,6 +29,10 @@ vi.mock('../telemetry/client', () => ({ track: trackMock })) +vi.mock('../telemetry/cohort-classifier', () => ({ + getCohortAtEmit: getCohortAtEmitMock +})) + const LEAF_1 = '11111111-1111-4111-8111-111111111111' const LEAF_2 = '22222222-2222-4222-8222-222222222222' const LEAF_3 = '33333333-3333-4333-8333-333333333333' @@ -62,6 +67,8 @@ function buildBody(payload: Record, overrides: Partial = beforeEach(() => { _internals.resetCachesForTests() trackMock.mockReset() + getCohortAtEmitMock.mockReset() + getCohortAtEmitMock.mockReturnValue({ nth_repo_added: 2 }) }) afterEach(() => { @@ -2171,6 +2178,537 @@ describe('AgentHookServer listener replay', () => { }) }) +describe('AgentHookServer prompt-sent telemetry', () => { + it('tracks a live local hook explicit prompt with conservative attribution', async () => { + const server = new AgentHookServer() + await server.start({ env: 'production' }) + try { + const env = server.buildPtyEnv() + const response = await fetch(`http://127.0.0.1:${env.ORCA_AGENT_HOOK_PORT}/hook/claude`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Orca-Agent-Hook-Token': env.ORCA_AGENT_HOOK_TOKEN + }, + body: JSON.stringify( + buildBody({ + hook_event_name: 'UserPromptSubmit', + prompt: ' fix the spinner ' + }) + ) + }) + + expect(response.status).toBe(204) + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'claude-code', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + } finally { + server.stop() + } + }) + + it('tracks a live SSH hook explicit prompt through ingestRemote', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'remote prompt', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'codex', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + }) + + it('dedupes adjacent same-turn reports without considering hook state', () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + try { + const server = new AgentHookServer() + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'same turn', agentType: 'gemini' } + }, + 'conn-1' + ) + vi.setSystemTime(1_500) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'done', prompt: 'same turn', agentType: 'gemini' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'gemini', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + } finally { + vi.useRealTimers() + } + }) + + it('tracks the same prompt again after a completed turn starts over', () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + try { + const server = new AgentHookServer() + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'continue', agentType: 'codex' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + payload: { state: 'done', prompt: 'continue', agentType: 'codex' } + }, + 'conn-1' + ) + vi.setSystemTime(1_500) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'continue', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + } + }) + + it('dedupes duplicate Command Code stop hooks but tracks same-prompt reruns', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-user-1', + payload: { state: 'done', prompt: 'rerun', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-user-1', + payload: { state: 'done', prompt: 'rerun', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-user-2', + payload: { state: 'done', prompt: 'rerun', agentType: 'command-code' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(2) + }) + + it('dedupes Command Code direct prompt hooks followed by transcript-backed stop hooks', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'same command', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-a-1', + payload: { state: 'done', prompt: 'same command', agentType: 'command-code' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + }) + + it('does not let a reused interaction key suppress different prompt text', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-reused', + payload: { state: 'done', prompt: 'first command', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-reused', + payload: { state: 'done', prompt: 'second command', agentType: 'command-code' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(2) + }) + + it('does not treat Command Code cached prompts as explicit prompt evidence', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'done', prompt: 'cached prompt', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: false, + payload: { state: 'done', prompt: 'cached prompt', agentType: 'command-code' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + }) + + it('preserves prompt dedupe when a live status row is dismissed', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'long turn', agentType: 'codex' } + }, + 'conn-1' + ) + server.dropStatusEntry(PANE) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'long turn', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + }) + + it('lets a dismissed completed row start the same prompt again', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'done', prompt: 'rerun after done', agentType: 'codex' } + }, + 'conn-1' + ) + server.dropStatusEntry(PANE) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'rerun after done', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(2) + }) + + it('dedupes the same prompt until a completed turn boundary is observed', () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + try { + const server = new AgentHookServer() + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'repeat later', agentType: 'codex' } + }, + 'conn-1' + ) + vi.setSystemTime(32_000) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'repeat later', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + } finally { + vi.useRealTimers() + } + }) + + it('does not track replays, empty prompts, or inherited prompt snapshots', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + isReplay: true, + payload: { state: 'working', prompt: 'replayed prompt', agentType: 'codex' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: GOOD_PANE, + tabId: 'tab-good', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: ' ', agentType: 'codex' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: FRESH_PANE, + tabId: 'tab-fresh', + worktreeId: 'wt-1', + payload: { state: 'working', prompt: 'inherited prompt', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('does not track hook status messages that preserve a cached prompt', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'real prompt', agentType: 'droid' } + }, + 'conn-1' + ) + trackMock.mockClear() + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: false, + payload: { state: 'waiting', prompt: 'real prompt', agentType: 'droid' } + }, + 'conn-1' + ) + + expect(trackMock).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('tracks OpenCode user MessagePart hooks once per message id', async () => { + const server = new AgentHookServer() + await server.start({ env: 'production' }) + try { + const env = server.buildPtyEnv() + const response = await fetch(`http://127.0.0.1:${env.ORCA_AGENT_HOOK_PORT}/hook/opencode`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Orca-Agent-Hook-Token': env.ORCA_AGENT_HOOK_TOKEN + }, + body: JSON.stringify( + buildBody({ + hook_event_name: 'MessagePart', + role: 'user', + text: 'fix', + messageID: 'msg-1' + }) + ) + }) + const updatedResponse = await fetch( + `http://127.0.0.1:${env.ORCA_AGENT_HOOK_PORT}/hook/opencode`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Orca-Agent-Hook-Token': env.ORCA_AGENT_HOOK_TOKEN + }, + body: JSON.stringify( + buildBody({ + hook_event_name: 'MessagePart', + role: 'user', + text: 'fix tests', + messageID: 'msg-1' + }) + ) + } + ) + + expect(response.status).toBe(204) + expect(updatedResponse.status).toBe(204) + expect(server.getStatusSnapshot()[0]).toMatchObject({ + state: 'working', + prompt: 'fix tests', + agentType: 'opencode' + }) + expect(trackMock).toHaveBeenCalledTimes(1) + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'opencode', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + } finally { + server.stop() + } + }) + + it('maps custom hook agent types to other', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'custom prompt', agentType: 'my-local-agent' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'other', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + }) + + it('does not block status cache mutation or listener fanout when telemetry throws', () => { + const server = new AgentHookServer() + const listener = vi.fn() + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + trackMock.mockImplementationOnce(() => { + throw new Error('telemetry unavailable') + }) + server.setListener(listener) + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'keep status moving', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(server.getStatusSnapshot()).toEqual([ + expect.objectContaining({ + paneKey: PANE, + state: 'working', + prompt: 'keep status moving', + agentType: 'codex' + }) + ]) + expect(listener).toHaveBeenCalledWith( + expect.objectContaining({ + paneKey: PANE, + payload: expect.objectContaining({ prompt: 'keep status moving' }) + }) + ) + errorSpy.mockRestore() + }) +}) + describe('Claude hook normalization', () => { it('PostToolUse for Edit surfaces toolName + file_path preview', () => { const result = _internals.normalizeHookPayload( @@ -2808,11 +3346,18 @@ describe('OpenCode hook normalization', () => { it('MessagePart with role=user surfaces text as the prompt and stays working', () => { const result = _internals.normalizeHookPayload( 'opencode', - buildBody({ hook_event_name: 'MessagePart', role: 'user', text: 'hi there' }), + buildBody({ + hook_event_name: 'MessagePart', + role: 'user', + text: 'hi there', + messageID: 'msg-1' + }), 'production' ) expect(result?.payload.state).toBe('working') expect(result?.payload.prompt).toBe('hi there') + expect(result?.hasExplicitPrompt).toBe(true) + expect(result?.promptInteractionKey).toBe('opencode-message-msg-1') }) it('MessagePart with role=assistant populates lastAssistantMessage', () => { @@ -3108,6 +3653,7 @@ describe('Droid hook normalization', () => { expect(done?.payload.state).toBe('done') expect(done?.payload.prompt).toBe('write tests') + expect(done?.hasExplicitPrompt).toBe(false) }) it('Notification ignores confirmation status text rather than treating it as permission', () => { @@ -3598,6 +4144,7 @@ describe('Copilot hook normalization', () => { expect(result?.payload.state).toBe('blocked') expect(result?.payload.prompt).toBe('deploy the app') expect(result?.payload.lastAssistantMessage).toBe('Which environment?') + expect(result?.hasExplicitPrompt).toBe(false) }) it('Notification(elicitation_dialog) accepts camelCase type and surfaces the question', () => { @@ -4215,6 +4762,32 @@ describe('Last-status persistence', () => { } }) + it('does not write prompt interaction keys to last-status.json', async () => { + const server = new AgentHookServer() + await server.start({ + env: 'production', + userDataPath + }) + try { + await postHookEvent( + server, + buildBody({ + hook_event_name: 'MessagePart', + role: 'user', + text: 'persist status only', + messageID: 'opencode-local-message-id' + }), + '/hook/opencode' + ) + server.flushStatusPersistSync() + const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + expect(file.entries[PANE].payload.prompt).toBe('persist status only') + expect(file.entries[PANE].promptInteractionKey).toBeUndefined() + } finally { + server.stop() + } + }) + it('hydrates last-status.json into the cache before listener registration', async () => { // Pre-populate the file directly to simulate a prior session. mkdirSync(join(userDataPath, 'agent-hooks'), { recursive: true }) diff --git a/src/main/agent-hooks/server.ts b/src/main/agent-hooks/server.ts index 0ff58601d..636b1e4af 100644 --- a/src/main/agent-hooks/server.ts +++ b/src/main/agent-hooks/server.ts @@ -11,11 +11,13 @@ // - the on-disk last-status cache (`last-status.json`) that survives // Orca restart so retained dashboard rows reappear on relaunch import { createServer, type IncomingMessage, type ServerResponse } from 'http' -import { randomUUID } from 'crypto' +import { createHash, randomBytes, randomUUID } from 'crypto' import { chmodSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'fs' import { join } from 'path' import { track } from '../telemetry/client' +import { getCohortAtEmit } from '../telemetry/cohort-classifier' +import { AGENT_KIND_VALUES, type AgentKind } from '../../shared/telemetry-events' import { ORCA_HOOK_PROTOCOL_VERSION } from '../../shared/agent-hook-types' import { clearAllListenerCaches, @@ -99,6 +101,7 @@ const LAST_STATUS_FILE_VERSION = 2 // hook-server batching; quit-time uses flushStatusPersistSync() for the // guaranteed final flush. const STATUS_PERSIST_DEBOUNCE_MS = 250 +const AGENT_PROMPT_SENT_AGENT_KINDS = new Set(AGENT_KIND_VALUES) // Why: bound the on-disk file's growth across many sessions. PTY-teardown // eviction handles closed panes, but daemon-restored PTYs that never re-attach @@ -113,6 +116,25 @@ type LastStatusFile = { entries: Record } +type AgentPromptSentDedupeEntry = { + agentKind: AgentKind + promptHash: string + promptInteractionKey?: string +} + +function agentTypeToPromptSentAgentKind(agentType: AgentType | undefined): AgentKind { + const normalized = agentType?.trim().toLowerCase() + if (!normalized || normalized === 'unknown') { + return 'other' + } + if (normalized === 'claude') { + return 'claude-code' + } + return AGENT_PROMPT_SENT_AGENT_KINDS.has(normalized as AgentKind) + ? (normalized as AgentKind) + : 'other' +} + function equivalentInterruptAgentType( actual: AgentType | undefined, baseline: AgentType | undefined @@ -383,6 +405,8 @@ export class AgentHookServer { // server instances in the same process (tests) don't share state. private statusPersistTimer: ReturnType | null = null private assistantMessageRetryTimers = new Map>() + private promptSentDedupeByPaneKey = new Map() + private promptSentHashSalt = randomBytes(16).toString('hex') // Why: identity check — skip writes when the JSON-stringified contents // exactly match the last successful disk write. Cheap protection against // re-firing trailing timers when nothing changed. @@ -529,6 +553,76 @@ export class AgentHookServer { } } + private hashPromptForTelemetryDedupe(prompt: string): string { + return createHash('sha256') + .update(this.promptSentHashSalt) + .update('\0') + .update(prompt) + .digest('hex') + } + + private maybeTrackAgentPromptSent( + payload: AgentHookEventPayload, + previousStatus: EnrichedAgentHookEventPayload | undefined + ): void { + if (payload.isReplay === true || payload.hasExplicitPrompt !== true) { + return + } + const prompt = payload.payload.prompt?.trim() ?? '' + if (prompt.length === 0) { + return + } + const agentKind = agentTypeToPromptSentAgentKind(payload.payload.agentType) + const promptHash = this.hashPromptForTelemetryDedupe(prompt) + const promptInteractionKey = + typeof payload.promptInteractionKey === 'string' && + payload.promptInteractionKey.trim().length > 0 + ? payload.promptInteractionKey.trim() + : undefined + const previousDedupe = this.promptSentDedupeByPaneKey.get(payload.paneKey) + const isCompletedTurnBoundary = + previousStatus?.payload.state === 'done' && payload.payload.state === 'working' + if ( + previousDedupe?.agentKind === agentKind && + previousDedupe.promptInteractionKey !== undefined && + previousDedupe.promptInteractionKey === promptInteractionKey && + (agentKind === 'opencode' || previousDedupe.promptHash === promptHash) + ) { + return + } + if ( + previousDedupe?.agentKind === agentKind && + previousDedupe.promptHash === promptHash && + !( + previousStatus?.payload.state === 'done' && + payload.payload.state === 'done' && + previousDedupe.promptInteractionKey !== undefined && + promptInteractionKey !== undefined && + previousDedupe.promptInteractionKey !== promptInteractionKey + ) && + !isCompletedTurnBoundary + ) { + return + } + this.promptSentDedupeByPaneKey.set(payload.paneKey, { + agentKind, + promptHash, + promptInteractionKey + }) + try { + // Why: hooks prove the user submitted a turn, but do not know which UI + // launched the terminal; keep attribution low-cardinality and conservative. + track('agent_prompt_sent', { + agent_kind: agentKind, + launch_source: 'unknown', + request_kind: 'followup', + ...getCohortAtEmit() + }) + } catch (err) { + console.error('[agent-hooks] prompt-sent telemetry failed', err) + } + } + private applyNormalizedStatus(payload: AgentHookEventPayload): EnrichedAgentHookEventPayload { const previous = this.state.lastStatusByPaneKey.get(payload.paneKey) as | EnrichedAgentHookEventPayload @@ -567,6 +661,7 @@ export class AgentHookServer { ) { this.clearAssistantMessageRetry(effectivePayload.paneKey) } + this.maybeTrackAgentPromptSent(effectivePayload, previous) const enriched = this.attachStatusTiming(effectivePayload) this.runtimeObservedStatusPaneKeys.add(enriched.paneKey) this.state.lastStatusByPaneKey.set(enriched.paneKey, enriched) @@ -702,6 +797,7 @@ export class AgentHookServer { if (entry.ptyId === ptyId) { this.legacyPaneKeyAliases.delete(legacyPaneKey) clearPaneCacheState(this.state, legacyPaneKey) + this.promptSentDedupeByPaneKey.delete(legacyPaneKey) const shouldClearStablePaneKey = options?.shouldClearStablePaneKey?.(entry.stablePaneKey) ?? true if (shouldClearStablePaneKey && this.state.lastStatusByPaneKey.has(entry.stablePaneKey)) { @@ -713,6 +809,7 @@ export class AgentHookServer { // cleanup is the only path that can evict that retained status. clearPaneCacheState(this.state, entry.stablePaneKey) this.runtimeObservedStatusPaneKeys.delete(entry.stablePaneKey) + this.promptSentDedupeByPaneKey.delete(entry.stablePaneKey) } aliasChanged = true } @@ -763,6 +860,7 @@ export class AgentHookServer { env?: string version?: string hasExplicitPrompt?: boolean + promptInteractionKey?: string hookEventName?: string toolUseId?: string toolAgentId?: string @@ -824,6 +922,11 @@ export class AgentHookServer { typeof envelope.hookEventName === 'string' && envelope.hookEventName.trim().length > 0 ? envelope.hookEventName.trim() : undefined + const promptInteractionKey = + typeof envelope.promptInteractionKey === 'string' && + envelope.promptInteractionKey.trim().length > 0 + ? envelope.promptInteractionKey.trim() + : undefined const toolUseId = typeof envelope.toolUseId === 'string' && envelope.toolUseId.trim().length > 0 ? envelope.toolUseId.trim() @@ -860,6 +963,7 @@ export class AgentHookServer { worktreeId, connectionId: trimmedConnectionId, hasExplicitPrompt: envelope.hasExplicitPrompt === true ? true : undefined, + promptInteractionKey, hookEventName, toolUseId, toolAgentId, @@ -1001,6 +1105,7 @@ export class AgentHookServer { this.lastStatusFilePath = null this.lastWrittenJson = null this.runtimeObservedStatusPaneKeys.clear() + this.promptSentDedupeByPaneKey.clear() this.legacyPaneKeyAliases.clear() clearAllListenerCaches(this.state) this.notifyStatusChangeListeners() @@ -1018,9 +1123,13 @@ export class AgentHookServer { if (!this.state.lastStatusByPaneKey.has(resolvedPaneKey)) { return } + const existing = this.state.lastStatusByPaneKey.get(resolvedPaneKey) this.state.lastStatusByPaneKey.delete(resolvedPaneKey) this.clearAssistantMessageRetry(resolvedPaneKey) this.runtimeObservedStatusPaneKeys.delete(resolvedPaneKey) + if (existing?.payload.state === 'done') { + this.promptSentDedupeByPaneKey.delete(resolvedPaneKey) + } this.scheduleStatusPersist() this.notifyStatusChangeListeners() } @@ -1034,11 +1143,13 @@ export class AgentHookServer { const hadStatus = this.state.lastStatusByPaneKey.has(resolvedPaneKey) this.clearAssistantMessageRetry(resolvedPaneKey) clearPaneCacheState(this.state, resolvedPaneKey) + this.promptSentDedupeByPaneKey.delete(resolvedPaneKey) let clearedAlias = false for (const [legacyPaneKey, stablePaneKey] of this.legacyPaneKeyAliases) { if (stablePaneKey.stablePaneKey === resolvedPaneKey) { this.legacyPaneKeyAliases.delete(legacyPaneKey) clearPaneCacheState(this.state, legacyPaneKey) + this.promptSentDedupeByPaneKey.delete(legacyPaneKey) clearedAlias = true } } @@ -1190,7 +1301,8 @@ export class AgentHookServer { if (!isValidPaneKey(paneKey)) { continue } - entries[paneKey] = payload as EnrichedAgentHookEventPayload + const { promptInteractionKey: _promptInteractionKey, ...persistedPayload } = payload + entries[paneKey] = persistedPayload as EnrichedAgentHookEventPayload } const file: LastStatusFile = { version: LAST_STATUS_FILE_VERSION, entries } return JSON.stringify(file) @@ -1270,6 +1382,10 @@ export class AgentHookServer { _getStateForTests(): HookListenerState { return this.state } + + _resetPromptSentDedupeForTests(): void { + this.promptSentDedupeByPaneKey.clear() + } } export const agentHookServer = new AgentHookServer() @@ -1287,5 +1403,6 @@ export const _internals = { parseFormEncodedBody, resetCachesForTests: (): void => { clearAllListenerCaches(agentHookServer._getStateForTests()) + agentHookServer._resetPromptSentDedupeForTests() } } diff --git a/src/main/opencode/hook-service.test.ts b/src/main/opencode/hook-service.test.ts index 53896f56e..ebbcf1752 100644 --- a/src/main/opencode/hook-service.test.ts +++ b/src/main/opencode/hook-service.test.ts @@ -210,7 +210,9 @@ describe('OpenCodeHookService buildPtyEnv / clearPty round-trip', () => { const pluginPath = join(env.OPENCODE_CONFIG_DIR!, 'plugins', 'orca-opencode-status.js') expect(existsSync(pluginPath)).toBe(true) // Sanity-check the file has plugin source, not a stray write. - expect(readFileSync(pluginPath, 'utf8')).toContain('OrcaOpenCodeStatusPlugin') + const pluginSource = readFileSync(pluginPath, 'utf8') + expect(pluginSource).toContain('OrcaOpenCodeStatusPlugin') + expect(pluginSource).toContain('messageID: part.messageID') }) it('clearPty removes the same directory buildPtyEnv created', () => { diff --git a/src/main/opencode/hook-service.ts b/src/main/opencode/hook-service.ts index 074ed6720..c96b30639 100644 --- a/src/main/opencode/hook-service.ts +++ b/src/main/opencode/hook-service.ts @@ -305,7 +305,7 @@ function getOpenCodePluginSource(): string { ' if (!part || part.type !== "text" || !part.text) return;', ' const role = messageRoleById.get(part.messageID);', ' if (!role) return;', - ' await post("MessagePart", { role, text: part.text });', + ' await post("MessagePart", { role, text: part.text, messageID: part.messageID });', ' return;', ' }', '', diff --git a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts index f4ea0b221..057e095de 100644 --- a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts +++ b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts @@ -19,6 +19,19 @@ import { agentHookServer, _internals as agentHookInternals } from '../agent-hook import { getSshPtyProvider } from '../ipc/pty' import { toAppSshPtyId } from '../providers/ssh-pty-id' +const { getCohortAtEmitMock, trackMock } = vi.hoisted(() => ({ + getCohortAtEmitMock: vi.fn(), + trackMock: vi.fn() +})) + +vi.mock('../telemetry/client', () => ({ + track: trackMock +})) + +vi.mock('../telemetry/cohort-classifier', () => ({ + getCohortAtEmit: getCohortAtEmitMock +})) + vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) @@ -184,6 +197,9 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { beforeEach(() => { vi.clearAllMocks() + trackMock.mockReset() + getCohortAtEmitMock.mockReset() + getCohortAtEmitMock.mockReturnValue({ nth_repo_added: 4 }) previousRemoteHooksFlag = process.env[ORCA_FEATURE_REMOTE_AGENT_HOOKS_ENV] process.env[ORCA_FEATURE_REMOTE_AGENT_HOOKS_ENV] = '1' warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) @@ -388,6 +404,97 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { ) }) + it('forwards remote hook transition metadata into main ingest', async () => { + relay = createFakeRelay() + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: relay.transport, + platform: 'linux-x64' + }) + const ingestSpy = vi.spyOn(agentHookServer, 'ingestRemote') + + session = createSession('conn-hook-metadata') + await session.establish({} as SshConnection) + + relay.notifyAgentHook( + makeEnvelope({ + source: 'claude', + hookEventName: 'PreToolUse', + promptInteractionKey: 'command-code-transcript-user-3', + toolUseId: 'toolu-1', + toolAgentId: 'agent-subagent-a', + toolAgentType: 'Review', + payload: { + state: 'working', + prompt: 'remote prompt', + agentType: 'claude', + toolName: 'Bash', + toolInput: 'pnpm test' + } + }) + ) + + await vi.waitFor(() => + expect(ingestSpy).toHaveBeenCalledWith( + expect.objectContaining({ + hookEventName: 'PreToolUse', + promptInteractionKey: 'command-code-transcript-user-3', + toolUseId: 'toolu-1', + toolAgentId: 'agent-subagent-a', + toolAgentType: 'Review' + }), + 'conn-hook-metadata' + ) + ) + ingestSpy.mockRestore() + }) + + it('tracks prompt sent from live SSH agent hooks but not replayed hooks', async () => { + relay = createFakeRelay() + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: relay.transport, + platform: 'linux-x64' + }) + + session = createSession('conn-live-telemetry') + await session.establish({} as SshConnection) + + relay.notifyAgentHook( + makeEnvelope({ + hasExplicitPrompt: true, + payload: { + state: 'working', + prompt: 'ssh live user prompt', + agentType: 'codex' + } + }) + ) + + await vi.waitFor(() => + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'codex', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 4 + }) + ) + + trackMock.mockClear() + relay.notifyAgentHook( + makeEnvelope({ + hasExplicitPrompt: true, + isReplay: true, + payload: { + state: 'working', + prompt: 'ssh replayed user prompt', + agentType: 'codex' + } + }) + ) + + await new Promise((resolve) => setImmediate(resolve)) + expect(trackMock).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + it('preserves replay metadata from remote hook notifications', async () => { relay = createFakeRelay() vi.mocked(deployAndLaunchRelay).mockResolvedValue({ diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index 73b72b864..76a9ac4ac 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -642,6 +642,11 @@ export class SshRelaySession { env?: unknown version?: unknown hasExplicitPrompt?: unknown + promptInteractionKey?: unknown + hookEventName?: unknown + toolUseId?: unknown + toolAgentId?: unknown + toolAgentType?: unknown isReplay?: unknown payload?: unknown } @@ -661,6 +666,16 @@ export class SshRelaySession { env: typeof envelope.env === 'string' ? envelope.env : undefined, version: typeof envelope.version === 'string' ? envelope.version : undefined, hasExplicitPrompt: envelope.hasExplicitPrompt === true ? true : undefined, + promptInteractionKey: + typeof envelope.promptInteractionKey === 'string' + ? envelope.promptInteractionKey + : undefined, + hookEventName: + typeof envelope.hookEventName === 'string' ? envelope.hookEventName : undefined, + toolUseId: typeof envelope.toolUseId === 'string' ? envelope.toolUseId : undefined, + toolAgentId: typeof envelope.toolAgentId === 'string' ? envelope.toolAgentId : undefined, + toolAgentType: + typeof envelope.toolAgentType === 'string' ? envelope.toolAgentType : undefined, isReplay: envelope.isReplay === true ? true : undefined, payload: envelope.payload }, diff --git a/src/main/telemetry/validator.test.ts b/src/main/telemetry/validator.test.ts index 934b2a090..d8b1d6d5b 100644 --- a/src/main/telemetry/validator.test.ts +++ b/src/main/telemetry/validator.test.ts @@ -31,6 +31,16 @@ describe('validate', () => { expect(result.ok).toBe(true) }) + it('accepts a well-formed agent_prompt_sent payload', () => { + const result = validate('agent_prompt_sent', { + agent_kind: 'claude-code', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 1 + }) + expect(result.ok).toBe(true) + }) + it('drops unknown event names', () => { const result = validate('not_a_real_event' as never, {}) expect(result.ok).toBe(false) diff --git a/src/relay/agent-hook-server.ts b/src/relay/agent-hook-server.ts index 2789fffde..2dd9c23b0 100644 --- a/src/relay/agent-hook-server.ts +++ b/src/relay/agent-hook-server.ts @@ -292,6 +292,7 @@ export class RelayAgentHookServer { worktreeId: event.worktreeId, connectionId: null, hasExplicitPrompt: event.hasExplicitPrompt, + promptInteractionKey: event.promptInteractionKey, hookEventName: event.hookEventName, toolUseId: event.toolUseId, toolAgentId: event.toolAgentId, diff --git a/src/renderer/src/lib/agent-paste-draft.test.ts b/src/renderer/src/lib/agent-paste-draft.test.ts index 36c0cd510..efd4960e5 100644 --- a/src/renderer/src/lib/agent-paste-draft.test.ts +++ b/src/renderer/src/lib/agent-paste-draft.test.ts @@ -10,7 +10,7 @@ const testState = vi.hoisted(() => ({ unsubscribe: vi.fn(), subscribeToPtyData: vi.fn(), isRemoteRuntimePtyId: vi.fn(), - sendRuntimePtyInput: vi.fn(), + sendRuntimePtyInputVerified: vi.fn(), subscribeToRuntimeTerminalData: vi.fn() })) @@ -26,7 +26,7 @@ vi.mock('@/components/terminal-pane/pty-dispatcher', () => ({ vi.mock('@/runtime/runtime-terminal-inspection', () => ({ isRemoteRuntimePtyId: testState.isRemoteRuntimePtyId, - sendRuntimePtyInput: testState.sendRuntimePtyInput + sendRuntimePtyInputVerified: testState.sendRuntimePtyInputVerified })) vi.mock('@/runtime/runtime-terminal-stream', () => ({ @@ -58,7 +58,8 @@ describe('pasteDraftWhenAgentReady', () => { ) testState.isRemoteRuntimePtyId.mockReset() testState.isRemoteRuntimePtyId.mockReturnValue(false) - testState.sendRuntimePtyInput.mockReset() + testState.sendRuntimePtyInputVerified.mockReset() + testState.sendRuntimePtyInputVerified.mockResolvedValue(true) testState.subscribeToRuntimeTerminalData.mockReset() }) @@ -77,16 +78,20 @@ describe('pasteDraftWhenAgentReady', () => { testState.ptyObserver?.(CODEX_COMPOSER_PROMPT_RENDER) await flushMicrotasks() - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() testState.ptyObserver?.(DECSET_BRACKETED_PASTE) await flushMicrotasks() - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() testState.ptyObserver?.(CODEX_COMPOSER_PROMPT_RENDER) await expect(promise).resolves.toBe(true) - expect(testState.sendRuntimePtyInput).toHaveBeenCalledWith({}, 'pty-1', PASTED_ISSUE_URL) + expect(testState.sendRuntimePtyInputVerified).toHaveBeenCalledWith( + {}, + 'pty-1', + PASTED_ISSUE_URL + ) expect(vi.getTimerCount()).toBe(0) }) @@ -103,7 +108,11 @@ describe('pasteDraftWhenAgentReady', () => { ) await expect(promise).resolves.toBe(true) - expect(testState.sendRuntimePtyInput).toHaveBeenCalledWith({}, 'pty-1', PASTED_ISSUE_URL) + expect(testState.sendRuntimePtyInputVerified).toHaveBeenCalledWith( + {}, + 'pty-1', + PASTED_ISSUE_URL + ) }) it('keeps the render-quiet wait for agents without the Codex ready signal', async () => { @@ -116,15 +125,19 @@ describe('pasteDraftWhenAgentReady', () => { testState.ptyObserver?.(DECSET_BRACKETED_PASTE) await flushMicrotasks() - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() await vi.advanceTimersByTimeAsync(1499) - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() await vi.advanceTimersByTimeAsync(1) await expect(promise).resolves.toBe(true) - expect(testState.sendRuntimePtyInput).toHaveBeenCalledWith({}, 'pty-1', PASTED_ISSUE_URL) + expect(testState.sendRuntimePtyInputVerified).toHaveBeenCalledWith( + {}, + 'pty-1', + PASTED_ISSUE_URL + ) }) it('does not paste for agents that already use native draft prefill', async () => { @@ -137,7 +150,7 @@ describe('pasteDraftWhenAgentReady', () => { ).resolves.toBe(false) expect(testState.subscribeToPtyData).not.toHaveBeenCalled() - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() }) it('can force paste and submit for native-prefill agents', async () => { @@ -154,7 +167,39 @@ describe('pasteDraftWhenAgentReady', () => { await vi.advanceTimersByTimeAsync(1500) await expect(promise).resolves.toBe(true) - expect(testState.sendRuntimePtyInput).toHaveBeenCalledWith({}, 'pty-1', `${PASTED_ISSUE_URL}\r`) + expect(testState.sendRuntimePtyInputVerified).toHaveBeenCalledWith( + {}, + 'pty-1', + `${PASTED_ISSUE_URL}\r` + ) + }) + + it('reports false when verified input delivery fails', async () => { + testState.sendRuntimePtyInputVerified.mockResolvedValue(false) + const promise = pasteDraftWhenAgentReady({ + tabId: 'tab-1', + content: ISSUE_URL, + agent: 'codex' + }) + await flushMicrotasks() + + testState.ptyObserver?.(`${DECSET_BRACKETED_PASTE}${CODEX_COMPOSER_PROMPT_RENDER}`) + + await expect(promise).resolves.toBe(false) + }) + + it('reports false when verified input delivery rejects', async () => { + testState.sendRuntimePtyInputVerified.mockRejectedValue(new Error('runtime timeout')) + const promise = pasteDraftWhenAgentReady({ + tabId: 'tab-1', + content: ISSUE_URL, + agent: 'codex' + }) + await flushMicrotasks() + + testState.ptyObserver?.(`${DECSET_BRACKETED_PASTE}${CODEX_COMPOSER_PROMPT_RENDER}`) + + await expect(promise).resolves.toBe(false) }) }) diff --git a/src/renderer/src/lib/agent-paste-draft.ts b/src/renderer/src/lib/agent-paste-draft.ts index b133c3591..00dcf7207 100644 --- a/src/renderer/src/lib/agent-paste-draft.ts +++ b/src/renderer/src/lib/agent-paste-draft.ts @@ -4,7 +4,6 @@ import { useAppStore } from '@/store' import { subscribeToPtyData } from '@/components/terminal-pane/pty-dispatcher' import { isRemoteRuntimePtyId, - sendRuntimePtyInput, sendRuntimePtyInputVerified } from '@/runtime/runtime-terminal-inspection' import { subscribeToRuntimeTerminalData } from '@/runtime/runtime-terminal-stream' @@ -95,12 +94,15 @@ export async function pasteDraftWhenAgentReady(args: { return false } - sendRuntimePtyInput( - useAppStore.getState().settings, - ptyId, - `${BRACKETED_PASTE_BEGIN}${content}${BRACKETED_PASTE_END}${submit ? '\r' : ''}` - ) - return true + try { + return await sendRuntimePtyInputVerified( + useAppStore.getState().settings, + ptyId, + `${BRACKETED_PASTE_BEGIN}${content}${BRACKETED_PASTE_END}${submit ? '\r' : ''}` + ) + } catch { + return false + } } export async function submitPromptToAgentTab(args: { diff --git a/src/renderer/src/lib/launch-agent-in-new-tab.test.ts b/src/renderer/src/lib/launch-agent-in-new-tab.test.ts index 8ae121335..f89c9375f 100644 --- a/src/renderer/src/lib/launch-agent-in-new-tab.test.ts +++ b/src/renderer/src/lib/launch-agent-in-new-tab.test.ts @@ -6,6 +6,7 @@ const mockSetActiveTabType = vi.fn() const mockSetTabBarOrder = vi.fn() const mockSetAgentStatus = vi.fn() const mockPasteDraftWhenAgentReady = vi.fn() +const mockTrack = vi.fn() const LEAF_ID = '11111111-1111-4111-8111-111111111111' @@ -50,7 +51,7 @@ vi.mock('@/lib/agent-paste-draft', () => ({ })) vi.mock('@/lib/telemetry', () => ({ - track: vi.fn(), + track: mockTrack, tuiAgentToAgentKind: (agent: string) => agent })) @@ -88,6 +89,32 @@ describe('launchAgentInNewTab', () => { ) }) + it('does not track prompt-sent for argv prompt launches', async () => { + const { launchAgentInNewTab } = await import('./launch-agent-in-new-tab') + + launchAgentInNewTab({ + agent: 'codex', + worktreeId: 'wt-1', + prompt: 'fix the spinner', + launchSource: 'onboarding' + }) + + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('does not track prompt-sent for draft launches', async () => { + const { launchAgentInNewTab } = await import('./launch-agent-in-new-tab') + + launchAgentInNewTab({ + agent: 'claude', + worktreeId: 'wt-1', + prompt: 'review this before sending', + promptDelivery: 'draft' + }) + + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + it('seeds working after Command Code submit-after-ready prompt delivery', async () => { const { launchAgentInNewTab } = await import('./launch-agent-in-new-tab') @@ -120,5 +147,21 @@ describe('launchAgentInNewTab', () => { prompt: 'large generated prompt', agentType: 'command-code' }) + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('does not track prompt-sent when submit-after-ready delivery fails', async () => { + mockPasteDraftWhenAgentReady.mockResolvedValue(false) + const { launchAgentInNewTab } = await import('./launch-agent-in-new-tab') + + launchAgentInNewTab({ + agent: 'command-code', + worktreeId: 'wt-1', + prompt: 'large generated prompt', + promptDelivery: 'submit-after-ready' + }) + await Promise.resolve() + + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) }) }) diff --git a/src/renderer/src/lib/launch-agent-in-new-tab.ts b/src/renderer/src/lib/launch-agent-in-new-tab.ts index dd305a189..0b4f084c2 100644 --- a/src/renderer/src/lib/launch-agent-in-new-tab.ts +++ b/src/renderer/src/lib/launch-agent-in-new-tab.ts @@ -93,7 +93,6 @@ export function launchAgentInNewTab(args: LaunchAgentInNewTabArgs): LaunchAgentI const trimmedPrompt = prompt?.trim() ?? '' const hasPrompt = trimmedPrompt.length > 0 const isFollowupPath = TUI_AGENT_CONFIG[agent].promptInjectionMode === 'stdin-after-start' - // Why: argv/flag agents fold the prompt into the launch command and // auto-submit — keeping behavior consistent with the composer/tab-bar `+` // mental model, where the prompt is "the first turn the user sent". @@ -189,7 +188,6 @@ export function launchAgentInNewTab(args: LaunchAgentInNewTabArgs): LaunchAgentI request_kind: 'new' } }) - // Why: schedule the bracketed-paste-after-ready follow-up immediately after // the startup command is queued. Fire-and-forget so callers keep their // synchronous `{ tabId, startupPlan }` signature. The helper short-circuits diff --git a/src/renderer/src/lib/new-workspace.test.ts b/src/renderer/src/lib/new-workspace.test.ts index 17d7ae21b..59812d340 100644 --- a/src/renderer/src/lib/new-workspace.test.ts +++ b/src/renderer/src/lib/new-workspace.test.ts @@ -1,5 +1,48 @@ -import { describe, expect, it } from 'vitest' -import { getWorkspaceSeedName, isGitLabIssueUrl } from './new-workspace' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockInspectRuntimeTerminalProcess, + mockSendRuntimePtyInputVerified, + mockPasteDraftWhenAgentReady, + mockTrack, + store +} = vi.hoisted(() => ({ + mockInspectRuntimeTerminalProcess: vi.fn(), + mockSendRuntimePtyInputVerified: vi.fn(), + mockPasteDraftWhenAgentReady: vi.fn(), + mockTrack: vi.fn(), + store: { + settings: {}, + activeTabIdByWorktree: { 'wt-1': 'tab-1' } as Record, + tabsByWorktree: { 'wt-1': [{ id: 'tab-1' }] } as Record, + ptyIdsByTabId: { 'tab-1': ['pty-1'] } as Record + } +})) + +vi.mock('@/store', () => ({ + useAppStore: { + getState: () => store + } +})) + +vi.mock('@/runtime/runtime-terminal-inspection', () => ({ + inspectRuntimeTerminalProcess: mockInspectRuntimeTerminalProcess, + sendRuntimePtyInputVerified: mockSendRuntimePtyInputVerified +})) + +vi.mock('@/lib/agent-paste-draft', () => ({ + pasteDraftWhenAgentReady: mockPasteDraftWhenAgentReady +})) + +vi.mock('@/lib/telemetry', () => ({ + track: mockTrack +})) + +import { + ensureAgentStartupInTerminal, + getWorkspaceSeedName, + isGitLabIssueUrl +} from './new-workspace' describe('getWorkspaceSeedName', () => { it('prefers an explicit name', () => { @@ -124,3 +167,88 @@ describe('isGitLabIssueUrl', () => { expect(isGitLabIssueUrl('https://github.com/group/project/issues/123')).toBe(false) }) }) + +describe('ensureAgentStartupInTerminal prompt delivery', () => { + beforeEach(() => { + vi.clearAllMocks() + store.settings = {} + store.activeTabIdByWorktree = { 'wt-1': 'tab-1' } + store.tabsByWorktree = { 'wt-1': [{ id: 'tab-1' }] } + store.ptyIdsByTabId = { 'tab-1': ['pty-1'] } + mockInspectRuntimeTerminalProcess.mockResolvedValue({ + foregroundProcess: 'aider', + hasChildProcesses: true + }) + mockSendRuntimePtyInputVerified.mockResolvedValue(true) + mockPasteDraftWhenAgentReady.mockResolvedValue(true) + }) + + it('sends a follow-up prompt through the terminal runtime without renderer telemetry', async () => { + await ensureAgentStartupInTerminal({ + worktreeId: 'wt-1', + startup: { + agent: 'aider', + launchCommand: 'aider', + expectedProcess: 'aider', + followupPrompt: 'fix the spinner' + } + }) + + expect(mockSendRuntimePtyInputVerified).toHaveBeenCalledWith({}, 'pty-1', 'fix the spinner\r') + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('does not track when follow-up prompt delivery is rejected by the terminal runtime', async () => { + mockSendRuntimePtyInputVerified.mockResolvedValue(false) + + await ensureAgentStartupInTerminal({ + worktreeId: 'wt-1', + startup: { + agent: 'aider', + launchCommand: 'aider', + expectedProcess: 'aider', + followupPrompt: 'fix the spinner' + } + }) + + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('does not track when follow-up prompt delivery rejects', async () => { + mockSendRuntimePtyInputVerified.mockRejectedValue(new Error('runtime timeout')) + + await expect( + ensureAgentStartupInTerminal({ + worktreeId: 'wt-1', + startup: { + agent: 'aider', + launchCommand: 'aider', + expectedProcess: 'aider', + followupPrompt: 'fix the spinner' + } + }) + ).resolves.toBeUndefined() + + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('does not track draft prompt delivery as a sent prompt', async () => { + await ensureAgentStartupInTerminal({ + worktreeId: 'wt-1', + startup: { + agent: 'claude', + launchCommand: 'claude', + expectedProcess: 'claude', + followupPrompt: null, + draftPrompt: 'review this before sending' + } + }) + + expect(mockPasteDraftWhenAgentReady).toHaveBeenCalledWith({ + tabId: 'tab-1', + content: 'review this before sending', + agent: 'claude' + }) + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) +}) diff --git a/src/renderer/src/lib/new-workspace.ts b/src/renderer/src/lib/new-workspace.ts index 4c32f0b39..6cc91987c 100644 --- a/src/renderer/src/lib/new-workspace.ts +++ b/src/renderer/src/lib/new-workspace.ts @@ -2,7 +2,7 @@ import { useAppStore } from '@/store' import { pasteDraftWhenAgentReady } from '@/lib/agent-paste-draft' import { inspectRuntimeTerminalProcess, - sendRuntimePtyInput + sendRuntimePtyInputVerified } from '@/runtime/runtime-terminal-inspection' import type { AgentStartupPlan } from '@/lib/tui-agent-startup' import { isShellProcess } from '@/lib/tui-agent-startup' @@ -244,7 +244,7 @@ export async function ensureAgentStartupInTerminal(args: { // session and submitted. Wait until the agent owns the PTY before writing. if (startup.followupPrompt) { await waitForAgentForeground(ptyId, startup.expectedProcess) - sendRuntimePtyInput(useAppStore.getState().settings, ptyId, `${startup.followupPrompt}\r`) + await sendFollowupPrompt(ptyId, startup.followupPrompt) } // Why: draftPrompt uses bracketed-paste so the URL lands atomically in the @@ -259,6 +259,14 @@ export async function ensureAgentStartupInTerminal(args: { } } +async function sendFollowupPrompt(ptyId: string, prompt: string): Promise { + try { + return await sendRuntimePtyInputVerified(useAppStore.getState().settings, ptyId, `${prompt}\r`) + } catch { + return false + } +} + // Why: legacy followupPrompt path used `agentOwnsForeground` exclusively (with // a hasChildProcesses fallback after several polls). Preserve that behavior so // stdin-after-start agents still receive their prompt under the same diff --git a/src/renderer/src/runtime/runtime-terminal-inspection.test.ts b/src/renderer/src/runtime/runtime-terminal-inspection.test.ts index eab9d599c..e3dd83c46 100644 --- a/src/renderer/src/runtime/runtime-terminal-inspection.test.ts +++ b/src/renderer/src/runtime/runtime-terminal-inspection.test.ts @@ -1,5 +1,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import { inspectRuntimeTerminalProcess, sendRuntimePtyInput } from './runtime-terminal-inspection' +import { + inspectRuntimeTerminalProcess, + sendRuntimePtyInput, + sendRuntimePtyInputVerified +} from './runtime-terminal-inspection' import { createCompatibleRuntimeStatusResponseIfNeeded, type RuntimeEnvironmentCallRequest @@ -10,6 +14,7 @@ describe('runtime terminal owner routing', () => { const runtimeCall = vi.fn() const runtimeTransportCall = vi.fn() const localWrite = vi.fn() + const localWriteAccepted = vi.fn() const localForeground = vi.fn() const localHasChildren = vi.fn() @@ -29,6 +34,7 @@ describe('runtime terminal owner routing', () => { runtimeEnvironments: { call: runtimeTransportCall }, pty: { write: localWrite, + writeAccepted: localWriteAccepted, getForegroundProcess: localForeground, hasChildProcesses: localHasChildren } @@ -83,4 +89,68 @@ describe('runtime terminal owner routing', () => { ) ).resolves.toEqual({ foregroundProcess: null, hasChildProcesses: false }) }) + + it('reports stale remote terminal handles as rejected during verified send', async () => { + runtimeCall.mockResolvedValue({ + ok: false, + error: { code: 'terminal_handle_stale', message: 'terminal_handle_stale' } + }) + + await expect( + sendRuntimePtyInputVerified( + { activeRuntimeEnvironmentId: 'env-2' }, + 'remote:env-1@@terminal-stale', + 'x' + ) + ).resolves.toBe(false) + }) + + it('reports declined remote terminal sends as rejected during verified send', async () => { + runtimeCall.mockResolvedValue({ + ok: true, + result: { send: { handle: 'terminal-1', accepted: false, bytesWritten: 0 } }, + _meta: { runtimeId: 'runtime-1' } + }) + + await expect( + sendRuntimePtyInputVerified( + { activeRuntimeEnvironmentId: 'env-2' }, + 'remote:env-1@@terminal-1', + 'x' + ) + ).resolves.toBe(false) + + expect(runtimeCall).toHaveBeenCalledWith({ + selector: 'env-1', + method: 'terminal.send', + params: { + terminal: 'terminal-1', + text: 'x', + client: { id: 'orca-desktop', type: 'desktop' } + }, + timeoutMs: 15_000 + }) + }) + + it('uses accepted local writes for verified input', async () => { + localWriteAccepted.mockResolvedValue(true) + + await expect( + sendRuntimePtyInputVerified({ activeRuntimeEnvironmentId: null }, 'local-pty', 'x') + ).resolves.toBe(true) + + expect(localWriteAccepted).toHaveBeenCalledWith('local-pty', 'x') + expect(localWrite).not.toHaveBeenCalled() + }) + + it('reports success after fallback fire-and-forget writes when local acceptance cannot be verified', async () => { + localWriteAccepted.mockResolvedValue(false) + + await expect( + sendRuntimePtyInputVerified({ activeRuntimeEnvironmentId: null }, 'local-pty', 'x') + ).resolves.toBe(true) + + expect(localWriteAccepted).toHaveBeenCalledWith('local-pty', 'x') + expect(localWrite).toHaveBeenCalledWith('local-pty', 'x') + }) }) diff --git a/src/renderer/src/runtime/runtime-terminal-inspection.ts b/src/renderer/src/runtime/runtime-terminal-inspection.ts index 18d87ec4a..b292e5dad 100644 --- a/src/renderer/src/runtime/runtime-terminal-inspection.ts +++ b/src/renderer/src/runtime/runtime-terminal-inspection.ts @@ -1,4 +1,5 @@ import type { GlobalSettings } from '../../../shared/types' +import type { RuntimeTerminalSend } from '../../../shared/runtime-types' import { RuntimeRpcCallError, callRuntimeRpc, getActiveRuntimeTarget } from './runtime-rpc-client' import { getRemoteRuntimePtyEnvironmentId, @@ -106,13 +107,24 @@ export async function sendRuntimePtyInputVerified( : getActiveRuntimeTarget(settings) const terminal = getRemoteRuntimeTerminalHandle(ptyId) if (target.kind !== 'environment' || !terminal) { - window.api.pty.write(ptyId, data) - return true + const accepted = await window.api.pty.writeAccepted(ptyId, data) + if (!accepted) { + window.api.pty.write(ptyId, data) + // Why: SSH/local fallback writes are fire-and-forget. Callers use this + // boolean to continue UX flow, while hook telemetry confirms real turns. + return true + } + return accepted } try { - await callRuntimeRpc(target, 'terminal.send', { terminal, text: data }, { timeoutMs: 15_000 }) - return true + const result = await callRuntimeRpc<{ send: RuntimeTerminalSend }>( + target, + 'terminal.send', + { terminal, text: data, client: { id: 'orca-desktop', type: 'desktop' } }, + { timeoutMs: 15_000 } + ) + return result.send.accepted === true } catch (error) { if (isTerminalGoneError(error)) { return false diff --git a/src/shared/agent-hook-listener.test.ts b/src/shared/agent-hook-listener.test.ts index 669da8480..03ce703c3 100644 --- a/src/shared/agent-hook-listener.test.ts +++ b/src/shared/agent-hook-listener.test.ts @@ -196,6 +196,52 @@ describe('shared agent-hook-listener', () => { toolName: 'shell_command', toolInput: 'pwd' }) + expect(tool?.hasExplicitPrompt).toBe(true) + expect(tool?.promptInteractionKey).toMatch(/^command-code-transcript-[a-f0-9]{12}-/) + + const directPrompt = normalizeHookPayload( + createHookListenerState(), + 'command-code', + { + paneKey: PANE_KEY, + payload: { + hook_event_name: 'PreToolUse', + prompt: 'Direct command prompt' + } + }, + 'production' + ) + expect(directPrompt?.hasExplicitPrompt).toBe(true) + + const directPromptWithTranscript = normalizeHookPayload( + createHookListenerState(), + 'command-code', + { + paneKey: PANE_KEY, + payload: { + hook_event_name: 'PreToolUse', + prompt: 'Run pwd and report it', + transcript_path: transcriptPath + } + }, + 'production' + ) + expect(directPromptWithTranscript?.hasExplicitPrompt).toBe(true) + expect(directPromptWithTranscript?.promptInteractionKey).toBe(tool?.promptInteractionKey) + + const statusMessage = normalizeHookPayload( + createHookListenerState(), + 'command-code', + { + paneKey: PANE_KEY, + payload: { + hook_event_name: 'PreToolUse', + message: 'Preparing tool call' + } + }, + 'production' + ) + expect(statusMessage?.hasExplicitPrompt).toBe(false) const done = normalizeHookPayload( state, @@ -219,6 +265,25 @@ describe('shared agent-hook-listener', () => { agentType: 'command-code', lastAssistantMessage: 'The output is /tmp/project.' }) + expect(done?.promptInteractionKey).toBe(tool?.promptInteractionKey) + + const cachedOnly = normalizeHookPayload( + state, + 'command-code', + { + paneKey: PANE_KEY, + tabId: 'tab-1', + worktreeId: 'wt', + env: 'production', + version: '1', + payload: { + hook_event_name: 'Stop' + } + }, + 'production' + ) + expect(cachedOnly?.payload.prompt).toBe('Run pwd and report it') + expect(cachedOnly?.hasExplicitPrompt).toBe(false) } finally { rmSync(tmpDir, { recursive: true, force: true }) } @@ -376,6 +441,7 @@ describe('shared agent-hook-listener', () => { prompt: 'Fix the failing test', agentType: 'antigravity' }) + expect(started?.hasExplicitPrompt).toBe(true) } finally { rmSync(tmpDir, { recursive: true, force: true }) } diff --git a/src/shared/agent-hook-listener.ts b/src/shared/agent-hook-listener.ts index e29f9d61e..619f50c1f 100644 --- a/src/shared/agent-hook-listener.ts +++ b/src/shared/agent-hook-listener.ts @@ -11,7 +11,7 @@ // which pull `electron` — so it is safe to import from `src/relay/`. See // docs/design/agent-status-over-ssh.md §3 ("relay normalizes; Orca routes"). import type { IncomingMessage } from 'http' -import { randomUUID } from 'crypto' +import { createHash, randomUUID } from 'crypto' import { homedir } from 'os' import { chmodSync, @@ -141,6 +141,9 @@ export type AgentHookEventPayload = { /** True when this hook event carried prompt text directly, instead of using * the listener's cached prompt from an earlier event in the same pane. */ hasExplicitPrompt?: boolean + /** Stable per-turn key when a source exposes enough local hook context to + * distinguish duplicate hook delivery from a same-text prompt rerun. */ + promptInteractionKey?: string /** Raw agent hook event name, used by main-process transition guards. */ hookEventName?: string /** Claude tool-use identifier when the hook source exposes one. */ @@ -235,7 +238,21 @@ export function readRequestBody(req: IncomingMessage): Promise { // ─── Per-pane field caches + extractors ───────────────────────────── -function extractPromptText(hookPayload: Record): string { +type ExtractedPromptText = { + text: string + source: + | 'prompt' + | 'user_prompt' + | 'userPrompt' + | 'initial_prompt' + | 'initialPrompt' + | 'user_message' + | 'message' + | 'role_user_text' + | null +} + +function extractPromptText(hookPayload: Record): ExtractedPromptText { const candidateKeys = [ 'prompt', 'user_prompt', @@ -250,7 +267,7 @@ function extractPromptText(hookPayload: Record): string { if (typeof value === 'string' && value.trim().length > 0) { // Why: trim so prompts match what readStringField produces elsewhere — // surrounding whitespace would otherwise leak into UI and caches. - return value.trim() + return { text: value.trim(), source: key as Exclude } } } // Why: OpenCode's plugin sends MessagePart events with { role, text }. When @@ -259,10 +276,10 @@ function extractPromptText(hookPayload: Record): string { if (hookPayload.role === 'user' && typeof hookPayload.text === 'string') { const trimmed = hookPayload.text.trim() if (trimmed.length > 0) { - return trimmed + return { text: trimmed, source: 'role_user_text' } } } - return '' + return { text: '', source: null } } function stripGrokUserQueryWrapper(promptText: string): string { @@ -664,6 +681,72 @@ function extractCommandCodeUserPromptFromLine(line: string): string | undefined return record.role === 'user' ? extractAssistantContentText(record.content) : undefined } +function hashInteractionKeyPart(value: string): string { + return createHash('sha256').update(value).digest('hex').slice(0, 12) +} + +function readLastCommandCodeUserPromptEntryFromTranscript( + transcriptPath: unknown +): { text: string; interactionKey: string } | undefined { + if (typeof transcriptPath !== 'string' || transcriptPath.length === 0) { + return undefined + } + try { + const stats = statSync(transcriptPath) + const size = stats.size + if (size <= 0) { + return undefined + } + const bytesToRead = Math.min(size, TRANSCRIPT_MAX_SCAN_BYTES) + const position = size - bytesToRead + const fd = openSync(transcriptPath, 'r') + try { + const buffer = Buffer.alloc(bytesToRead) + let filled = 0 + while (filled < bytesToRead) { + const n = readSync(fd, buffer, filled, bytesToRead - filled, position + filled) + if (n === 0) { + break + } + filled += n + } + let text = buffer.subarray(0, filled).toString('utf8') + let textBasePosition = position + if (position > 0) { + const firstNewline = text.indexOf('\n') + textBasePosition += firstNewline + 1 + text = firstNewline === -1 ? '' : text.slice(firstNewline + 1) + } + let lastPrompt: string | undefined + let lastPromptOffset = 0 + let lineStart = 0 + for (const line of text.split('\n')) { + const prompt = extractCommandCodeUserPromptFromLine(line.trim()) + if (prompt !== undefined) { + lastPrompt = prompt + lastPromptOffset = textBasePosition + Buffer.byteLength(text.slice(0, lineStart), 'utf8') + } + lineStart += line.length + 1 + } + return lastPrompt + ? { + text: lastPrompt, + interactionKey: [ + 'command-code-transcript', + hashInteractionKeyPart(transcriptPath), + String(lastPromptOffset), + hashInteractionKeyPart(lastPrompt) + ].join('-') + } + : undefined + } finally { + closeSync(fd) + } + } catch { + return undefined + } +} + function extractCommandCodeAssistantTextFromLine(line: string): string | undefined { let entry: unknown try { @@ -698,13 +781,6 @@ function extractCommandCodeAssistantTextFromLine(line: string): string | undefin return extractAssistantContentText(content) } -function readLastCommandCodeUserPromptFromTranscript(transcriptPath: unknown): string | undefined { - if (typeof transcriptPath !== 'string' || transcriptPath.length === 0) { - return undefined - } - return readLastTextFromTranscriptOnce(transcriptPath, extractCommandCodeUserPromptFromLine) -} - function readLastCommandCodeAssistantFromTranscript(transcriptPath: unknown): string | undefined { if (typeof transcriptPath !== 'string' || transcriptPath.length === 0) { return undefined @@ -1665,6 +1741,52 @@ function isNewTurnEvent(source: AgentHookSource, eventName: unknown): boolean { } } +function hasExplicitUserPrompt( + source: AgentHookSource, + eventName: unknown, + extractedPrompt: ExtractedPromptText, + resolvedPromptText: string, + hasTranscriptPromptEvidence = false +): boolean { + if ( + source === 'command-code' && + (eventName === 'PreToolUse' || eventName === 'Stop') && + (extractedPrompt.source !== 'message' || hasTranscriptPromptEvidence) && + resolvedPromptText.trim().length > 0 + ) { + // Why: Command Code exposes the submitted prompt through its transcript + // rather than direct hook fields. Treat the transcript-backed prompt as + // explicit so hook telemetry covers real Command Code turns. + return true + } + if ( + source === 'antigravity' && + isNewTurnEvent(source, eventName) && + resolvedPromptText.trim().length > 0 + ) { + return true + } + if (extractedPrompt.source === 'role_user_text') { + return source === 'opencode' && eventName === 'MessagePart' + } + if (extractedPrompt.text.length === 0) { + return false + } + // Why: bare `message` fields often contain permission or status copy. They + // may update visible status prompts, but they are not proof of user submit. + if (extractedPrompt.source === 'message') { + return false + } + if ( + extractedPrompt.source === 'user_prompt' || + extractedPrompt.source === 'userPrompt' || + extractedPrompt.source === 'user_message' + ) { + return isNewTurnEvent(source, eventName) + } + return isNewTurnEvent(source, eventName) +} + function extractToolFields( source: AgentHookSource, eventName: unknown, @@ -2222,12 +2344,6 @@ function normalizeCommandCodeEvent( return null } - const effectivePrompt = - promptText || - readLastCommandCodeUserPromptFromTranscript( - hookPayload.transcript_path ?? hookPayload.transcriptPath - ) || - '' const snapshot = resolveToolState( state, paneKey, @@ -2238,7 +2354,7 @@ function normalizeCommandCodeEvent( return parseAgentStatusPayload( JSON.stringify({ state: stateName, - prompt: resolvePrompt(state, paneKey, effectivePrompt, { + prompt: resolvePrompt(state, paneKey, promptText, { resetOnNewTurn: isNewTurnEvent('command-code', eventName) }), agentType: 'command-code', @@ -2424,11 +2540,15 @@ export function normalizeHookPayload( const worktreeId = readStringField(record, 'worktreeId') const hookPayloadRecord = hookPayload as Record + let promptInteractionKey: string | undefined const eventName = readFirstString(record, ['hook_event_name', 'hookEventName', 'hook_type', 'hookType']) ?? hookPayloadRecord.hook_event_name ?? hookPayloadRecord.hookEventName - const promptText = extractPromptText(hookPayload as Record) + const extractedPrompt = extractPromptText(hookPayload as Record) + const promptText = extractedPrompt.text + let resolvedPromptText = promptText + let hasTranscriptPromptEvidence = false // Why: exhaustive switch so adding a source to AgentHookSource fails // typecheck here instead of silently routing through OpenCode's normalizer. let payload: ParsedAgentStatusPayload | null @@ -2443,9 +2563,25 @@ export function normalizeHookPayload( payload = normalizeGeminiEvent(state, eventName, promptText, paneKey, hookPayloadRecord) break case 'antigravity': + if (isNewTurnEvent('antigravity', eventName)) { + resolvedPromptText = + promptText || + readLastUserPromptFromTranscript( + readFirstString(hookPayloadRecord, ['transcriptPath', 'transcript_path']) + ) || + '' + } payload = normalizeAntigravityEvent(state, eventName, promptText, paneKey, hookPayloadRecord) break case 'opencode': + if (extractedPrompt.source === 'role_user_text') { + const messageId = readFirstString(hookPayloadRecord, [ + 'messageID', + 'messageId', + 'message_id' + ]) + promptInteractionKey = messageId ? `opencode-message-${messageId}` : undefined + } payload = normalizeOpenCodeEvent(state, eventName, promptText, paneKey, hookPayloadRecord) break case 'cursor': @@ -2475,7 +2611,24 @@ export function normalizeHookPayload( payload = normalizeDroidEvent(state, eventName, promptText, paneKey, hookPayloadRecord) break case 'command-code': - payload = normalizeCommandCodeEvent(state, eventName, promptText, paneKey, hookPayloadRecord) + { + const transcriptPrompt = readLastCommandCodeUserPromptEntryFromTranscript( + hookPayloadRecord.transcript_path ?? hookPayloadRecord.transcriptPath + ) + hasTranscriptPromptEvidence = transcriptPrompt !== undefined + promptInteractionKey = transcriptPrompt?.interactionKey + resolvedPromptText = transcriptPrompt?.text ?? '' + if (promptText && extractedPrompt.source !== 'message') { + resolvedPromptText = promptText + } + } + payload = normalizeCommandCodeEvent( + state, + eventName, + resolvedPromptText, + paneKey, + hookPayloadRecord + ) break case 'grok': payload = normalizeGrokEvent(state, eventName, promptText, paneKey, hookPayloadRecord) @@ -2503,7 +2656,14 @@ export function normalizeHookPayload( tabId, worktreeId, connectionId: null, - hasExplicitPrompt: promptText.length > 0, + hasExplicitPrompt: hasExplicitUserPrompt( + source, + eventName, + extractedPrompt, + resolvedPromptText, + hasTranscriptPromptEvidence + ), + promptInteractionKey, hookEventName: typeof eventName === 'string' ? eventName : undefined, toolUseId: readFirstString(hookPayloadRecord, ['tool_use_id', 'toolUseId']), toolAgentId: readFirstString(hookPayloadRecord, ['agent_id', 'agentId']), diff --git a/src/shared/agent-hook-relay.ts b/src/shared/agent-hook-relay.ts index fa2d696dd..12cc21440 100644 --- a/src/shared/agent-hook-relay.ts +++ b/src/shared/agent-hook-relay.ts @@ -60,6 +60,9 @@ export type AgentHookRelayEnvelope = { /** Preserved from the relay-side normalized hook event so Orca can * distinguish a true same-prompt retry from a cached-prompt tool ping. */ hasExplicitPrompt?: boolean + /** Optional stable per-turn key from the relay-side listener. Used only for + * in-memory dedupe; never included in product telemetry payloads. */ + promptInteractionKey?: string /** Hook discriminator preserved for main-process transition rules. */ hookEventName?: string /** Claude tool execution id, when the source hook provides one. */ diff --git a/src/shared/telemetry-events.test.ts b/src/shared/telemetry-events.test.ts index 1f8f10dd2..37972dd91 100644 --- a/src/shared/telemetry-events.test.ts +++ b/src/shared/telemetry-events.test.ts @@ -111,6 +111,28 @@ describe('agent_started schema', () => { }) }) +describe('agent_prompt_sent schema', () => { + it('accepts a hook-confirmed prompt-send payload with cohort context', () => { + const parsed = eventSchemas.agent_prompt_sent.safeParse({ + agent_kind: 'codex', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 1 + }) + expect(parsed.success).toBe(true) + }) + + it('rejects prompt text via .strict()', () => { + const parsed = eventSchemas.agent_prompt_sent.safeParse({ + agent_kind: 'claude-code', + launch_source: 'unknown', + request_kind: 'followup', + prompt: 'please inspect /Users/alice/private-repo' + }) + expect(parsed.success).toBe(false) + }) +}) + describe('agent_hook_unattributed schema', () => { it('accepts the two bounded attribution failure reasons', () => { for (const reason of ['empty_pane_key', 'unknown_tab_id'] as const) { diff --git a/src/shared/telemetry-events.ts b/src/shared/telemetry-events.ts index 3aa012ea4..d84f51311 100644 --- a/src/shared/telemetry-events.ts +++ b/src/shared/telemetry-events.ts @@ -281,6 +281,14 @@ const agentStartedSchema = z nth_repo_added: nthRepoAddedSchema }) .strict() +const agentPromptSentSchema = z + .object({ + agent_kind: agentKindSchema, + launch_source: launchSourceSchema, + request_kind: requestKindSchema, + nth_repo_added: nthRepoAddedSchema + }) + .strict() // Enum-only by design for both fields. `error_message` and `error_stack` are // deliberately absent — `.strict()` rejects either key if a call site ever @@ -1017,6 +1025,7 @@ export const eventSchemas = { setup_script_prompt_action: setupScriptPromptActionSchema, agent_started: agentStartedSchema, + agent_prompt_sent: agentPromptSentSchema, agent_error: agentErrorSchema, agent_hook_install_failed: agentHookInstallFailedSchema, agent_hook_unattributed: agentHookUnattributedSchema, @@ -1107,6 +1116,7 @@ type _CohortExtendedRoster = | 'setup_script_prompt_shown' | 'setup_script_prompt_action' | 'agent_started' + | 'agent_prompt_sent' | 'agent_error' // Why: `z.object({}).strict()` infers a string index signature, which would // make every key appear present. Ignore index-signature-only keys here so