diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index c84075e6c..c4f70fb7a 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -6646,15 +6646,16 @@ "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], "coveredPlatforms": ["macos"], "coveredProviders": ["local", "daemon", "ssh"], - "coverageNotes": "A deterministic service-state-machine oracle now models a current-contract worker and coordinator whose renderer graph identities disappear across an app/runtime update. It exercises the production verifier with restored PTY and hydrated hook commitments, proves authenticated completion replay across a fresh runtime, explicit takeover, ordinary mail routing, remote-attachment process fencing, retained Task/Dispatch/terminal identity, and unchanged fixture marker bytes, and rejects foreign pane evidence. Other deterministic units cover authority-aware legacy formatting, exact legacy worker identity planning, local worker presentation, retained-output reads after adoption, reveal-failure warnings, stable-pane Run/Dispatch routing, the SSH in-process CLI fallback, and federated non-reveal. Two isolated macOS Electron journeys launch fake Codex workers through the real RPC path and record append-only spawn/interruption ledgers. They assert immediate inactive presentation, one live agent PID, stable PTY/incarnation/tab/leaf/worktree/Task/Dispatch identity, and no interruption after workspace re-entry; the restart journey additionally removes renderer ownership, marks the Dispatch legacy, retains the daemon process across an app restart, and proves exact background adoption with readable ACK output and no resume replay. Distinct A/B artifacts plus live SSH, WSL, folder, remote-runtime, Linux, and Windows cutover journeys remain explicit gaps.", + "coverageNotes": "A deterministic service-state-machine oracle now models a current-contract worker and coordinator whose renderer graph identities disappear across an app/runtime update. It exercises the production verifier with restored PTY and hydrated hook commitments, proves authenticated completion replay across a fresh runtime, explicit takeover, ordinary mail routing, remote-attachment process fencing, retained Task/Dispatch/terminal identity, and unchanged fixture marker bytes, and rejects foreign pane evidence. Other deterministic units cover authority-aware legacy formatting, exact legacy worker identity planning, local worker presentation, retained-output reads after adoption, reveal-failure warnings, stable-pane Run/Dispatch routing, creator pane/process/Run-generation fencing, indexed retained-Run lookup scaling, the SSH in-process CLI fallback, and federated non-reveal. Two isolated macOS Electron journeys launch fake Codex workers through the real RPC path and record append-only spawn/interruption ledgers. They assert immediate inactive presentation, one live agent PID, stable PTY/incarnation/tab/leaf/worktree/Task/Dispatch identity, and no interruption after workspace re-entry; the restart journey additionally removes renderer ownership, marks the Dispatch legacy, retains the daemon process across an app restart, and proves exact background adoption with readable ACK output and no resume replay. Distinct A/B artifacts plus live SSH, WSL, folder, remote-runtime, Linux, and Windows cutover journeys remain explicit gaps.", "motivatingLinks": ["https://github.com/stablyai/orca/pull/11107#discussion_r3663321387"], - "invariant": "Starting a worker in the coordinator's current workspace must materialize one inactive terminal tab before worker-start returns, preserve coordinator focus, and remain exactly once after workspace re-entry. After an app update or restart, an exact live legacy worker must fence automatic provider resume, adopt its original PTY into its original background pane, retain readable output, and clear the resume record without spawning, writing, signalling, interrupting, replacing, or focusing the worker. A current-contract worker whose renderer graph identity is temporarily absent must retain its Dispatch capability and settle exactly once from exact hook-attested handle, pane, and process evidence; otherwise only an exact attested coordinator may take over. An exact existing target workspace must receive a discoverable tab without stealing coordinator focus; if renderer reveal fails, worker-start must expose that the live worker remains background-only. Run and Dispatch checks must resolve through the caller's stable pane identity when a terminal handle is reminted, while a live handle outranks mismatched pane metadata, explicit legacy terminal inspection remains handle-scoped, and remote or headless worker presentation remains background-only.", - "oracle": "Drive Run create, Task create, and worker-start through production Electron runtimes with a deterministic Codex fixture. Require append-only ledgers with one still-live PID and no interruption, a visible inactive worker tab while the coordinator stays active, Run delivery through stable pane identity, and stable PTY/incarnation, tab, leaf, worktree, Task, and Dispatch across workspace re-entry. In a restart journey, retain the original daemon PTY and PID, remove renderer ownership, retain sleeping-session evidence, mark the Dispatch legacy, relaunch, and require exact inactive tab adoption, readable ACK output, cleared resume state, one spawn, and no resume argv or Conversation interrupted text after another workspace round trip. The service oracle removes renderer lookup identity from current-contract callers while retaining real restored-PTY and hook commitments, replays authenticated completion and takeover across fresh runtimes, and requires one Task, Dispatch, terminal authority, message, mutation, ordinary-mail delivery, remote process fencing, and unchanged fixture marker bytes while foreign pane evidence remains rejected. Unit tests separately assert authority-specific legacy affordances, exact identity and owner matching, retained-output fallback, pane-stable routing, federated non-activation, and SSH fallback parity.", + "invariant": "Starting a worker in the coordinator's current workspace must materialize one inactive terminal tab before worker-start returns, preserve coordinator focus, and remain exactly once after workspace re-entry. After an app update or restart, an exact live legacy worker must fence automatic provider resume, adopt its original PTY into its original background pane, retain readable output, and clear the resume record without spawning, writing, signalling, interrupting, replacing, or focusing the worker. A current-contract worker whose renderer graph identity is temporarily absent must retain its Dispatch capability and settle exactly once from exact hook-attested handle, pane, and process evidence; otherwise only an exact attested coordinator may take over. An exact existing target workspace must receive a discoverable tab without stealing coordinator focus; if renderer reveal fails, worker-start must expose that the live worker remains background-only. Run and Dispatch checks must resolve through the caller's stable pane identity when a terminal handle is reminted, while a live handle outranks mismatched pane metadata. A nested worker's creator edge requires the current creator pane, process incarnation, and owning Run generation; reminting and rebinding that pane to another Run must remove the stale edge. Explicit legacy terminal inspection remains handle-scoped, and remote or headless worker presentation remains background-only.", + "oracle": "Drive Run create, Task create, and worker-start through production Electron runtimes with a deterministic Codex fixture. Require append-only ledgers with one still-live PID and no interruption, a visible inactive worker tab while the coordinator stays active, Run delivery through stable pane identity, and stable PTY/incarnation, tab, leaf, worktree, Task, and Dispatch across workspace re-entry. In a restart journey, retain the original daemon PTY and PID, remove renderer ownership, retain sleeping-session evidence, mark the Dispatch legacy, relaunch, and require exact inactive tab adoption, readable ACK output, cleared resume state, one spawn, and no resume argv or Conversation interrupted text after another workspace round trip. The service oracle removes renderer lookup identity from current-contract callers while retaining real restored-PTY and hook commitments, replays authenticated completion and takeover across fresh runtimes, and requires one Task, Dispatch, terminal authority, message, mutation, ordinary-mail delivery, remote process fencing, and unchanged fixture marker bytes while foreign pane evidence remains rejected. Unit tests separately remint a creator pane and process from Run A into Run B, require the nested Run A worker to fall back to its current coordinator, require indexed query plans, and bound 300 Task reads with 50,000 retained Runs. They also assert authority-specific legacy affordances, exact identity and owner matching, retained-output fallback, pane-stable routing, federated non-activation, and SSH fallback parity.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/orchestration-runtime-update-settlement.test.ts --reporter=dot", "pnpm exec vitest run --config config/vitest.config.ts src/cli/handlers/orchestration.test.ts src/cli/handlers/orchestration-check-identity.test.ts src/cli/handlers/orchestration-worker-cli.test.ts src/main/runtime/rpc/methods/orchestration.test.ts src/main/ssh/ssh-remote-orca-cli.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/formatter.test.ts src/main/runtime/rpc/methods/orchestration-federation.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts", "pnpm run test:e2e -- tests/e2e/orchestration-worker-terminal-visibility.spec.ts --workers=1", "pnpm run test:e2e -- tests/e2e/orchestration-legacy-worker-restart-recovery.spec.ts --workers=1" @@ -6663,6 +6664,7 @@ "src/main/runtime/rpc/orchestration-runtime-update-settlement.test.ts", "src/main/runtime/orchestration/formatter.test.ts", "src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts", + "src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts", "src/main/runtime/orca-runtime.test.ts", "src/cli/handlers/orchestration.test.ts", "src/cli/handlers/orchestration-check-identity.test.ts", @@ -6684,6 +6686,13 @@ "ordinary mail and remote attachments use the same attested pane and process authority" ] }, + { + "file": "src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts", + "assertions": [ + "creator lookup uses the assignee-handle and pane-leaf indexes without a retained-Run scan", + "300 Task reads remain bounded with 50,000 unrelated retained Runs" + ] + }, { "file": "src/main/runtime/orchestration/orchestration-legacy-worker-terminal-recovery.test.ts", "assertions": [ diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index d8c943670..3aa249a0d 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -37942,6 +37942,7 @@ describe('OrcaRuntimeService', () => { handle === workerHandle ? { id: 'ctx-1', + run_id: 'run-1', task_id: 'task-1', assignee_handle: workerHandle, status: 'dispatched' @@ -37952,6 +37953,7 @@ describe('OrcaRuntimeService', () => { handle === workerHandle ? { id: 'ctx-done', + run_id: 'run-1', task_id: 'task-done', assignee_handle: workerHandle, status: 'completed', @@ -37961,14 +37963,16 @@ describe('OrcaRuntimeService', () => { ), getTask: vi.fn(() => ({ id: 'task-1', + run_id: 'run-1', task_title: 'Dispatch prompt work', display_name: 'Review dispatch prompts and make worker labels distinct', spec: 'Review dispatch prompts\n\nand make worker labels distinct', created_by_terminal_handle: coordinatorHandle })), - getActiveCoordinatorRun: vi.fn(() => ({ + getRun: vi.fn(() => ({ id: 'run-1', - coordinator_handle: coordinatorHandle + coordinator_handle: coordinatorHandle, + legacy: 0 })) } as never) runtime.attachWindow(1) @@ -38023,6 +38027,513 @@ describe('OrcaRuntimeService', () => { }) }) + it.each([ + ['fails closed when a modern dispatch owning Run is missing', 'run-missing', 'run-missing'], + ['fails closed when Task and Dispatch Runs disagree', 'run-dispatch', 'run-task'] + ])('%s', (_name, dispatchRunId, taskRunId) => { + const runtime = new OrcaRuntimeService(store) + const workerLeafId = '77777777-7777-4777-8777-777777777777' + const coordinatorLeafId = '88888888-8888-4888-8888-888888888888' + const workerPaneKey = makePaneKey('tab-worker', workerLeafId) + const workerHandle = runtime.preAllocateHandleForPty('pty-worker') + const coordinatorHandle = runtime.preAllocateHandleForPty('pty-coordinator') + const getActiveCoordinatorRun = vi.fn(() => ({ + id: 'run-legacy-unrelated', + coordinator_handle: coordinatorHandle + })) + runtime.setOrchestrationDb({ + getActiveDispatchForTerminal: vi.fn((handle: string) => + handle === workerHandle + ? { + id: 'ctx-missing-run', + run_id: dispatchRunId, + task_id: 'task-missing-run', + assignee_handle: workerHandle, + status: 'dispatched' + } + : undefined + ), + getLatestDispatchForTerminal: vi.fn(() => undefined), + getTask: vi.fn(() => ({ + id: 'task-missing-run', + run_id: taskRunId, + spec: 'modern task without proven Run', + created_by_terminal_handle: coordinatorHandle + })), + getRun: vi.fn(() => undefined), + getActiveCoordinatorRun + } as never) + runtime.attachWindow(1) + + const result = runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-worker', + worktreeId: TEST_WORKTREE_ID, + title: 'Worker', + activeLeafId: workerLeafId, + layout: null + }, + { + tabId: 'tab-coordinator', + worktreeId: TEST_WORKTREE_ID, + title: 'Coordinator', + activeLeafId: coordinatorLeafId, + layout: null + } + ], + leaves: [ + { + tabId: 'tab-worker', + worktreeId: TEST_WORKTREE_ID, + leafId: workerLeafId, + paneRuntimeId: 1, + ptyId: 'pty-worker', + paneTitle: null + }, + { + tabId: 'tab-coordinator', + worktreeId: TEST_WORKTREE_ID, + leafId: coordinatorLeafId, + paneRuntimeId: 2, + ptyId: 'pty-coordinator', + paneTitle: null + } + ] + }) + + expect(result.agentOrchestrationByPaneKey?.[workerPaneKey]).toEqual({ + taskId: 'task-missing-run', + dispatchId: 'ctx-missing-run', + dispatchStatus: 'dispatched', + taskTitle: 'modern task without proven Run', + displayName: 'modern task without proven Run' + }) + expect(getActiveCoordinatorRun).not.toHaveBeenCalled() + }) + + it('uses durable Run ownership before worktree-scoped legacy attribution', () => { + const childWorktreeId = `${TEST_REPO_ID}::${join(tmpdir(), 'workspaces', 'run-a-worker')}` + const folderWorktreeId = `${TEST_REPO_ID}::${join(tmpdir(), 'folder')}${FOLDER_WORKSPACE_INSTANCE_SEPARATOR}11111111-1111-4111-8111-111111111111` + const meta = store.getAllWorktreeMeta()[TEST_WORKTREE_ID] + const metaById = { + ...store.getAllWorktreeMeta(), + [childWorktreeId]: meta, + [folderWorktreeId]: meta + } + const runtime = new OrcaRuntimeService({ + ...store, + getAllWorktreeMeta: () => metaById, + getWorktreeMeta: (worktreeId: string) => metaById[worktreeId] + } as never) + const terminals = [ + { + name: 'coordinator-a', + worktreeId: TEST_WORKTREE_ID, + leafId: '11111111-1111-4111-8111-111111111111' + }, + { + name: 'coordinator-b', + worktreeId: TEST_WORKTREE_ID, + leafId: '22222222-2222-4222-8222-222222222222' + }, + { + name: 'worker-cross-worktree', + worktreeId: childWorktreeId, + leafId: '33333333-3333-4333-8333-333333333333' + }, + { + name: 'worker-same-worktree', + worktreeId: TEST_WORKTREE_ID, + leafId: '44444444-4444-4444-8444-444444444444' + }, + { + name: 'worker-folder', + worktreeId: folderWorktreeId, + leafId: '55555555-5555-4555-8555-555555555555' + }, + { + name: 'legacy-worker', + worktreeId: childWorktreeId, + leafId: '66666666-6666-4666-8666-666666666666' + } + ].map((terminal, index) => ({ + ...terminal, + tabId: `tab-${terminal.name}`, + ptyId: `pty-${terminal.name}`, + paneRuntimeId: index + 1 + })) + const terminalByName = Object.fromEntries( + terminals.map((terminal) => [terminal.name, terminal]) + ) + const handles = Object.fromEntries( + terminals.map((terminal) => [terminal.name, runtime.preAllocateHandleForPty(terminal.ptyId)]) + ) + const paneKey = (name: string): string => { + const terminal = terminalByName[name] + return makePaneKey(terminal.tabId, terminal.leafId) + } + const db = new OrchestrationDb(':memory:') + try { + const runA = db.createRun({ + objective: 'coordinate run A', + coordinatorHandle: handles['coordinator-a'], + coordinatorPaneKey: paneKey('coordinator-a') + }) + const runB = db.createRun({ + objective: 'coordinate run B', + coordinatorHandle: handles['coordinator-b'], + coordinatorPaneKey: paneKey('coordinator-b') + }) + const dispatches = Object.fromEntries( + [ + ['worker-cross-worktree', runA.id], + ['worker-same-worktree', runA.id], + ['worker-folder', runB.id] + ].map(([name, runId]) => { + const task = db.createTask({ spec: name, runId }) + return [name, db.createDispatchContext(task.id, handles[name], paneKey(name))] + }) + ) + const legacyTask = db.createTask({ spec: 'legacy worker' }) + const legacyDispatch = db.createDispatchContext( + legacyTask.id, + handles['legacy-worker'], + paneKey('legacy-worker') + ) + db.createCoordinatorRun({ + spec: 'unrelated legacy coordinator', + coordinatorHandle: handles['coordinator-b'] + }) + const getActiveCoordinatorRun = vi.spyOn(db, 'getActiveCoordinatorRun') + runtime.setOrchestrationDb(db) + runtime.attachWindow(1) + + const result = runtime.syncWindowGraph(1, { + tabs: terminals.map((terminal) => ({ + tabId: terminal.tabId, + worktreeId: terminal.worktreeId, + title: terminal.name, + activeLeafId: terminal.leafId, + layout: null + })), + leaves: terminals.map((terminal) => ({ + tabId: terminal.tabId, + worktreeId: terminal.worktreeId, + leafId: terminal.leafId, + paneRuntimeId: terminal.paneRuntimeId, + ptyId: terminal.ptyId, + paneTitle: null + })) + }) + + for (const [name, run, coordinator] of [ + ['worker-cross-worktree', runA, 'coordinator-a'], + ['worker-same-worktree', runA, 'coordinator-a'], + ['worker-folder', runB, 'coordinator-b'] + ] as const) { + expect(result.agentOrchestrationByPaneKey?.[paneKey(name)]).toMatchObject({ + taskId: dispatches[name].task_id, + dispatchId: dispatches[name].id, + dispatchStatus: 'dispatched', + parentTerminalHandle: handles[coordinator], + parentPaneKey: paneKey(coordinator), + coordinatorHandle: handles[coordinator], + orchestrationRunId: run.id + }) + } + const legacyContext = result.agentOrchestrationByPaneKey?.[paneKey('legacy-worker')] + expect(legacyContext).toMatchObject({ + taskId: legacyTask.id, + dispatchId: legacyDispatch.id, + dispatchStatus: 'dispatched' + }) + expect(legacyContext).not.toHaveProperty('parentTerminalHandle') + expect(legacyContext).not.toHaveProperty('coordinatorHandle') + expect(legacyContext).not.toHaveProperty('orchestrationRunId') + expect(getActiveCoordinatorRun).toHaveBeenCalledOnce() + } finally { + db.close() + } + }) + + it('uses the still-bound owning Run coordinator after a creator pane rebinds', () => { + const runtime = new OrcaRuntimeService(store) + const terminals = [ + { + name: 'coordinator', + leafId: '11111111-1111-4111-8111-111111111111' + }, + { + name: 'creator', + leafId: '22222222-2222-4222-8222-222222222222' + }, + { + name: 'worker', + leafId: '33333333-3333-4333-8333-333333333333' + }, + { + name: 'coordinator-created-worker', + leafId: '44444444-4444-4444-8444-444444444444' + } + ].map((terminal, index) => ({ + ...terminal, + tabId: `tab-${terminal.name}`, + ptyId: `pty-${terminal.name}`, + paneRuntimeId: index + 1 + })) + const terminalByName = Object.fromEntries( + terminals.map((terminal) => [terminal.name, terminal]) + ) + const handles = Object.fromEntries( + terminals.map((terminal) => [terminal.name, runtime.preAllocateHandleForPty(terminal.ptyId)]) + ) + const paneKey = (name: string): string => { + const terminal = terminalByName[name] + return makePaneKey(terminal.tabId, terminal.leafId) + } + const graph = () => ({ + tabs: terminals.map((terminal) => ({ + tabId: terminal.tabId, + worktreeId: TEST_WORKTREE_ID, + title: terminal.name, + activeLeafId: terminal.leafId, + layout: null + })), + leaves: terminals.map((terminal) => ({ + tabId: terminal.tabId, + worktreeId: TEST_WORKTREE_ID, + leafId: terminal.leafId, + paneRuntimeId: terminal.paneRuntimeId, + ptyId: terminal.ptyId, + paneTitle: null + })) + }) + const db = new OrchestrationDb(':memory:') + try { + runtime.setOrchestrationDb(db) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, graph()) + const runA = db.createRun({ + objective: 'own the nested worker', + coordinatorHandle: handles.coordinator, + coordinatorPaneKey: paneKey('coordinator') + }) + const creatorAuthority = runtime.getOrchestrationDispatchAuthority(handles.creator) + const coordinatorAuthority = runtime.getOrchestrationDispatchAuthority(handles.coordinator) + expect(creatorAuthority?.processIncarnation).toBeTruthy() + expect(coordinatorAuthority?.processIncarnation).toBeTruthy() + const creatorTask = db.createTask({ spec: 'create nested work', runId: runA.id }) + db.createDispatchContext( + creatorTask.id, + handles.creator, + paneKey('creator'), + undefined, + creatorAuthority?.processIncarnation ?? undefined + ) + const workerTask = db.createTask({ + spec: 'nested work', + runId: runA.id, + createdByTerminalHandle: handles.creator, + createdByPaneKey: paneKey('creator'), + createdByProcessIncarnation: creatorAuthority?.processIncarnation ?? undefined, + createdByRunGeneration: runA.consumer_generation + }) + const workerDispatch = db.createDispatchContext( + workerTask.id, + handles.worker, + paneKey('worker') + ) + const coordinatorCreatedTask = db.createTask({ + spec: 'coordinator-created work', + runId: runA.id, + createdByTerminalHandle: handles.coordinator, + createdByPaneKey: paneKey('coordinator'), + createdByProcessIncarnation: coordinatorAuthority?.processIncarnation ?? undefined, + createdByRunGeneration: runA.consumer_generation + }) + const coordinatorCreatedDispatch = db.createDispatchContext( + coordinatorCreatedTask.id, + handles['coordinator-created-worker'], + paneKey('coordinator-created-worker') + ) + expect( + runtime.syncWindowGraph(1, graph()).agentOrchestrationByPaneKey?.[paneKey('worker')] + ).toMatchObject({ + parentTerminalHandle: handles.creator, + parentPaneKey: paneKey('creator'), + coordinatorHandle: handles.coordinator, + orchestrationRunId: runA.id + }) + + const oldCreatorPaneKey = paneKey('creator') + terminalByName.creator.tabId = 'tab-creator-reminted' + terminalByName.creator.ptyId = 'pty-creator-reminted' + const remintedCreatorHandle = runtime.preAllocateHandleForPty(terminalByName.creator.ptyId) + runtime.syncWindowGraph(1, graph()) + const runB = db.createRun({ + objective: 'rebind the creator pane', + coordinatorHandle: remintedCreatorHandle, + coordinatorPaneKey: paneKey('creator') + }) + const reboundContext = runtime.syncWindowGraph(1, graph()).agentOrchestrationByPaneKey?.[ + paneKey('worker') + ] + + expect(db.getRun(runA.id)).toMatchObject({ + coordinator_handle: handles.coordinator, + consumer_generation: 1 + }) + expect(oldCreatorPaneKey).not.toBe(paneKey('creator')) + expect(db.getRun(runB.id)).toMatchObject({ coordinator_handle: remintedCreatorHandle }) + expect(reboundContext).toMatchObject({ + taskId: workerTask.id, + dispatchId: workerDispatch.id, + dispatchStatus: 'dispatched', + parentTerminalHandle: handles.coordinator, + parentPaneKey: paneKey('coordinator'), + coordinatorHandle: handles.coordinator, + orchestrationRunId: runA.id + }) + + db.createRun({ + objective: 'rebind the original coordinator pane', + coordinatorHandle: handles.coordinator, + coordinatorPaneKey: paneKey('coordinator') + }) + const unboundContext = runtime.syncWindowGraph(1, graph()).agentOrchestrationByPaneKey?.[ + paneKey('coordinator-created-worker') + ] + + expect(db.getRun(runA.id)).toMatchObject({ + coordinator_handle: null, + coordinator_pane_key: null, + consumer_generation: 2 + }) + expect(unboundContext).toEqual({ + taskId: coordinatorCreatedTask.id, + dispatchId: coordinatorCreatedDispatch.id, + dispatchStatus: 'dispatched', + taskTitle: 'coordinator-created work', + displayName: 'coordinator-created work', + orchestrationRunId: runA.id + }) + } finally { + db.close() + } + }) + + it('queries each stable terminal handle once while publishing orchestration context', () => { + const runtime = new OrcaRuntimeService(store) + const terminals = Array.from({ length: 100 }, (_, index) => ({ + tabId: `tab-query-${index}`, + leafId: `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`, + ptyId: `pty-query-${index}`, + paneRuntimeId: index + 1 + })) + const handles = terminals.map((terminal) => runtime.preAllocateHandleForPty(terminal.ptyId)) + const db = new OrchestrationDb(':memory:') + try { + const run = db.createRun({ + objective: 'query count oracle', + coordinatorHandle: handles[99], + coordinatorPaneKey: makePaneKey(terminals[99].tabId, terminals[99].leafId) + }) + const task = db.createTask({ spec: 'one dispatched terminal', runId: run.id }) + const dispatch = db.createDispatchContext( + task.id, + handles[0], + makePaneKey(terminals[0].tabId, terminals[0].leafId) + ) + const getActiveDispatchForTerminal = vi.spyOn(db, 'getActiveDispatchForTerminal') + const getLatestDispatchForTerminal = vi.spyOn(db, 'getLatestDispatchForTerminal') + const getTask = vi.spyOn(db, 'getTask') + const getRun = vi.spyOn(db, 'getRun') + const getActiveCoordinatorRun = vi.spyOn(db, 'getActiveCoordinatorRun') + runtime.setOrchestrationDb(db) + runtime.attachWindow(1) + + const graph = { + tabs: terminals.map((terminal) => ({ + tabId: terminal.tabId, + worktreeId: TEST_WORKTREE_ID, + title: terminal.tabId, + activeLeafId: terminal.leafId, + layout: null + })), + leaves: terminals.map((terminal) => ({ + tabId: terminal.tabId, + worktreeId: TEST_WORKTREE_ID, + leafId: terminal.leafId, + paneRuntimeId: terminal.paneRuntimeId, + ptyId: terminal.ptyId, + paneTitle: null + })) + } + runtime.syncWindowGraph(1, graph) + + const queryCounts = { + activeDispatch: getActiveDispatchForTerminal.mock.calls.length, + latestDispatch: getLatestDispatchForTerminal.mock.calls.length, + task: getTask.mock.calls.length, + run: getRun.mock.calls.length, + legacyCoordinator: getActiveCoordinatorRun.mock.calls.length + } + + db.completeDispatch(dispatch.id) + vi.useFakeTimers() + vi.setSystemTime(Date.now() + AGENT_STATUS_STALE_AFTER_MS + 5_000) + for (const query of [ + getActiveDispatchForTerminal, + getLatestDispatchForTerminal, + getTask, + getRun, + getActiveCoordinatorRun + ]) { + query.mockClear() + } + runtime.syncWindowGraph(1, graph) + + const historicalQueryCounts = { + activeDispatch: getActiveDispatchForTerminal.mock.calls.length, + latestDispatch: getLatestDispatchForTerminal.mock.calls.length, + task: getTask.mock.calls.length, + run: getRun.mock.calls.length, + legacyCoordinator: getActiveCoordinatorRun.mock.calls.length + } + expect({ + active: { + ...queryCounts, + total: Object.values(queryCounts).reduce((sum, n) => sum + n) + }, + historical: { + ...historicalQueryCounts, + total: Object.values(historicalQueryCounts).reduce((sum, n) => sum + n) + } + }).toEqual({ + active: { + activeDispatch: 100, + latestDispatch: 99, + task: 1, + run: 1, + legacyCoordinator: 0, + total: 201 + }, + historical: { + activeDispatch: 100, + latestDispatch: 100, + task: 0, + run: 0, + legacyCoordinator: 0, + total: 200 + } + }) + } finally { + vi.useRealTimers() + db.close() + } + }) + it('returns completed orchestration context for renderer-synced terminal leaves', () => { const runtime = new OrcaRuntimeService(store) const workerLeafId = '33333333-3333-4333-8333-333333333333' @@ -38037,6 +38548,7 @@ describe('OrcaRuntimeService', () => { handle === workerHandle ? { id: 'ctx-done', + run_id: 'run-1', task_id: 'task-done', assignee_handle: workerHandle, status: 'completed', @@ -38046,7 +38558,13 @@ describe('OrcaRuntimeService', () => { ), getTask: vi.fn(() => ({ id: 'task-done', + run_id: 'run-1', created_by_terminal_handle: coordinatorHandle + })), + getRun: vi.fn(() => ({ + id: 'run-1', + coordinator_handle: coordinatorHandle, + legacy: 0 })) } as never) runtime.attachWindow(1) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 8cca22329..c94d222de 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -29977,11 +29977,13 @@ export class OrcaRuntimeService { return undefined } const contexts: Record = {} + const queriedHandles = new Set() for (const leaf of this.leaves.values()) { if (!leaf.ptyId) { continue } const handle = this.issueHandle(leaf) + queriedHandles.add(handle) const context = this.getAgentStatusOrchestrationContextForHandle(handle, db) if (context) { contexts[this.makeRuntimePaneKey(leaf)] = context @@ -29992,6 +29994,10 @@ export class OrcaRuntimeService { continue } const handle = this.issuePtyHandle(pty) + if (queriedHandles.has(handle)) { + continue + } + queriedHandles.add(handle) const context = this.getAgentStatusOrchestrationContextForHandle(handle, db) if (context) { contexts[pty.paneKey] = context @@ -30011,7 +30017,7 @@ export class OrcaRuntimeService { if (!dispatch) { return undefined } - const task = db?.getTask?.(dispatch.task_id) + const task = db?.getTask?.(dispatch.task_id, dispatch.run_id) const display = typeof task?.spec === 'string' ? buildOrchestrationTaskDisplayMetadata({ @@ -30020,15 +30026,68 @@ export class OrcaRuntimeService { displayName: task.display_name }) : { taskTitle: '', displayName: '' } - const activeRun = - dispatch.status === 'pending' || dispatch.status === 'dispatched' + const owningRun = + task?.run_id && task.run_id === dispatch.run_id ? db?.getRun?.(dispatch.run_id) : undefined + const runCoordinatorHandle = owningRun?.coordinator_handle ?? undefined + const legacyActiveRun = + owningRun?.legacy === 1 && (dispatch.status === 'pending' || dispatch.status === 'dispatched') ? db?.getActiveCoordinatorRun?.() : undefined + // Why: legacy coordinator runs have no durable task ownership, so fail closed across worktrees. + const handleWorktreeId = legacyActiveRun ? this.getWorktreeIdForTerminalHandle(handle) : null + const legacyCoordinatorWorktreeId = legacyActiveRun + ? this.getWorktreeIdForTerminalHandle(legacyActiveRun.coordinator_handle) + : null + const scopedLegacyActiveRun = + legacyActiveRun && + handleWorktreeId && + legacyCoordinatorWorktreeId && + runtimeWorktreeIdsEqual(legacyCoordinatorWorktreeId, handleWorktreeId) + ? legacyActiveRun + : undefined + const coordinatorHandle = runCoordinatorHandle ?? scopedLegacyActiveRun?.coordinator_handle + const orchestrationRunId = owningRun?.legacy === 0 ? owningRun.id : scopedLegacyActiveRun?.id + const creatorPaneKey = task?.created_by_pane_key + const creatorPaneHandle = creatorPaneKey + ? this.getTerminalHandleForPaneKey(creatorPaneKey) + : null + const creatorAuthority = creatorPaneHandle + ? this.getOrchestrationDispatchAuthority(creatorPaneHandle) + : null + const storedCreatorPane = creatorPaneKey ? parsePaneKey(creatorPaneKey) : null + const currentCreatorPane = creatorAuthority?.paneKey + ? parsePaneKey(creatorAuthority.paneKey) + : null + const sameCreatorPane = Boolean( + creatorPaneKey && + creatorAuthority?.paneKey && + (creatorPaneKey === creatorAuthority.paneKey || + (storedCreatorPane && + currentCreatorPane && + storedCreatorPane.leafId === currentCreatorPane.leafId)) + ) + const paneRun = creatorPaneKey ? db?.getCurrentRunForPane?.(creatorPaneKey) : undefined + const sameRunCreatorDispatch = Boolean( + task?.creator_dispatch_id && + task.creator_dispatch_run_id === owningRun?.id && + task.creator_dispatch_pane_key && + task.creator_dispatch_process_incarnation === task.created_by_process_incarnation && + parsePaneKey(task.creator_dispatch_pane_key)?.leafId === storedCreatorPane?.leafId + ) + const currentCreatorHandle = + owningRun?.legacy === 0 && + task?.created_by_run_generation === owningRun.consumer_generation && + task.created_by_process_incarnation === creatorAuthority?.processIncarnation && + sameCreatorPane && + (paneRun + ? paneRun.id === owningRun.id && + paneRun.consumer_generation === task.created_by_run_generation + : sameRunCreatorDispatch) + ? (creatorPaneHandle ?? undefined) + : undefined const parentTerminalHandle = - task?.created_by_terminal_handle ?? - (activeRun?.coordinator_handle && activeRun.coordinator_handle !== handle - ? activeRun.coordinator_handle - : undefined) + currentCreatorHandle ?? + (coordinatorHandle && coordinatorHandle !== handle ? coordinatorHandle : undefined) const parentPaneKey = parentTerminalHandle ? this.getPaneKeyForTerminalHandle(parentTerminalHandle) : undefined @@ -30041,8 +30100,8 @@ export class OrcaRuntimeService { ...(display.displayName ? { displayName: display.displayName } : {}), ...(parentTerminalHandle ? { parentTerminalHandle } : {}), ...(parentPaneKey ? { parentPaneKey } : {}), - ...(activeRun?.coordinator_handle ? { coordinatorHandle: activeRun.coordinator_handle } : {}), - ...(activeRun?.id ? { orchestrationRunId: activeRun.id } : {}) + ...(coordinatorHandle ? { coordinatorHandle } : {}), + ...(orchestrationRunId ? { orchestrationRunId } : {}) } } @@ -30095,10 +30154,21 @@ export class OrcaRuntimeService { return pty } leafPty = pty ?? null + for (const candidate of this.leaves.values()) { + if (candidate.leafId !== parsed.leafId || !candidate.ptyId) { + continue + } + const remintedPty = this.ptysById.get(candidate.ptyId) + if (remintedPty?.connected) { + return remintedPty + } + leafPty ??= remintedPty ?? null + } } let newestMatch: RuntimePtyWorktreeRecord | null = null for (const pty of this.ptysById.values()) { - if (pty.paneKey === paneKey) { + const ptyPane = parsePaneKey(pty.paneKey ?? '') + if (pty.paneKey === paneKey || (parsed && ptyPane && parsed.leafId === ptyPane.leafId)) { if (pty.connected) { return pty } @@ -30123,6 +30193,18 @@ export class OrcaRuntimeService { return makePaneKey(record.tabId, record.leafId) } + private getWorktreeIdForTerminalHandle(handle: string): string | null { + const livePty = this.getLivePtyForHandle(handle) + if (livePty?.pty.worktreeId) { + return livePty.pty.worktreeId + } + const record = this.handles.get(handle) + if (!record || record.runtimeId !== this.runtimeId) { + return null + } + return record.worktreeId + } + private setPtyManagementTitleFromObservedTitle( pty: RuntimePtyWorktreeRecord, title: string | null | undefined, diff --git a/src/main/runtime/orchestration/coordinator.test.ts b/src/main/runtime/orchestration/coordinator.test.ts index 31e9180c9..cbb3c3f80 100644 --- a/src/main/runtime/orchestration/coordinator.test.ts +++ b/src/main/runtime/orchestration/coordinator.test.ts @@ -177,6 +177,42 @@ describe('Coordinator', () => { }) expect(db.getDispatchContext(task.id)?.assignee_pane_key).toBe('tab_a:leaf_a') + expect(db.getDispatchContext(task.id)?.process_incarnation).toBeNull() + + insertWorkerDone(db, { taskId: task.id }) + await runPromise + }) + + it('records authenticated process authority for automatic dispatch', async () => { + db = new OrchestrationDb(':memory:') + const runtime = createMockRuntime() + runtime.terminals = [{ handle: 'term_a', worktreeId: 'wt1', connected: true, writable: true }] + const withAuthority = Object.assign(runtime, { + getOrchestrationDispatchAuthority: (handle: string) => + handle === 'term_a' + ? { + paneKey: 'tab_a:leaf_a', + processIncarnation: 'pty_a:incarnation-a', + launchTokenHash: 'launch-token-hash' + } + : null + }) + const task = db.createTask({ spec: 'implement feature' }) + const coordinator = new Coordinator(db, withAuthority, { + spec: 'build it', + coordinatorHandle: 'coord', + pollIntervalMs: 50 + }) + const runPromise = coordinator.run() + await new Promise((r) => { + setTimeout(r, 100) + }) + + expect(db.getDispatchContext(task.id)).toMatchObject({ + assignee_pane_key: 'tab_a:leaf_a', + process_incarnation: 'pty_a:incarnation-a', + launch_token_hash: 'launch-token-hash' + }) insertWorkerDone(db, { taskId: task.id }) await runPromise diff --git a/src/main/runtime/orchestration/coordinator.ts b/src/main/runtime/orchestration/coordinator.ts index 7fcd5288f..df69ed9ef 100644 --- a/src/main/runtime/orchestration/coordinator.ts +++ b/src/main/runtime/orchestration/coordinator.ts @@ -26,8 +26,14 @@ export type CoordinatorRuntime = { behind: number recentSubjects: string[] } | null> - // Why: optional so lightweight runtime fakes keep compiling; when present, dispatch records the assignee's remint-stable pane identity. + // Why: pane-only fallback preserves reservation identity for lightweight runtime fakes. getTerminalPaneKey?(handle: string): string | null + // Why: automatic dispatch persists the same authenticated pane/process tuple as manual dispatch. + getOrchestrationDispatchAuthority?(handle: string): { + paneKey: string | null + processIncarnation: string | null + launchTokenHash: string | null + } | null // Why: Windows can host native and WSL workers at once, so the worker pane (not the coordinator) picks the packaged CLI name. getTerminalOrchestrationCliCommand?(handle: string): 'orca' | 'orca-ide' } @@ -419,10 +425,19 @@ export class Coordinator { } } + const dispatchAuthority = this.runtime.getOrchestrationDispatchAuthority?.(targetHandle) + const assigneePaneKey = + dispatchAuthority?.paneKey ?? this.runtime.getTerminalPaneKey?.(targetHandle) ?? undefined + const processIncarnation = + dispatchAuthority?.paneKey && dispatchAuthority.processIncarnation + ? dispatchAuthority.processIncarnation + : undefined const dispatch = this.db.createDispatchContext( task.id, targetHandle, - this.runtime.getTerminalPaneKey?.(targetHandle) ?? undefined + assigneePaneKey, + dispatchAuthority?.launchTokenHash ?? undefined, + processIncarnation ) // Why: dispatched agents use orca-dev in dev mode to reach the dev runtime's socket, not production (Section 6.4). diff --git a/src/main/runtime/orchestration/db.ts b/src/main/runtime/orchestration/db.ts index 5cdfccc31..3608ee7d7 100644 --- a/src/main/runtime/orchestration/db.ts +++ b/src/main/runtime/orchestration/db.ts @@ -66,8 +66,10 @@ function isEquivalentPaneKey(a: string, b: string): boolean { // Why: indexable pre-filter for isEquivalentPaneKey — equal strings and equal leaves both share the // text after the first ':', so this narrows candidates without deciding equivalence itself. -const PANE_KEY_MATCH_SUFFIX_SQL = +const RUN_PANE_KEY_MATCH_SUFFIX_SQL = "substr(coordinator_pane_key, instr(coordinator_pane_key, ':') + 1)" +const DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL = + "substr(assignee_pane_key, instr(assignee_pane_key, ':') + 1)" function paneKeyMatchSuffix(paneKey: string): string { const colon = paneKey.indexOf(':') @@ -263,13 +265,20 @@ export type RunListPage = { nextCursor: string | null } +export type TaskRuntimeLineageRow = TaskRow & { + creator_dispatch_id: string | null + creator_dispatch_run_id: string | null + creator_dispatch_pane_key: string | null + creator_dispatch_process_incarnation: string | null +} + type RunListCursor = { createdAt: string id: string } -// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership. -const SCHEMA_VERSION = 23 +// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup. +const SCHEMA_VERSION = 25 function hardenOrchestrationDatabaseFiles(dbPath: string | ':memory:'): void { if (dbPath === ':memory:' || process.platform === 'win32') { @@ -517,6 +526,9 @@ export class OrchestrationDb { run_id TEXT NOT NULL DEFAULT '${LEGACY_RUN_ID}', parent_id TEXT, created_by_terminal_handle TEXT, + created_by_pane_key TEXT, + created_by_process_incarnation TEXT, + created_by_run_generation INTEGER, task_title TEXT, display_name TEXT, spec TEXT NOT NULL, @@ -576,7 +588,7 @@ export class OrchestrationDb { CREATE INDEX IF NOT EXISTS idx_gates_status ON decision_gates(status); CREATE INDEX IF NOT EXISTS idx_runs_coordinator_pane_leaf - ON runs(${PANE_KEY_MATCH_SUFFIX_SQL}) + ON runs(${RUN_PANE_KEY_MATCH_SUFFIX_SQL}) WHERE coordinator_pane_key IS NOT NULL; CREATE TABLE IF NOT EXISTS coordinator_runs ( @@ -949,6 +961,29 @@ export class OrchestrationDb { if (current < 23) { this.backfillWorkerTerminalResources() } + if (current < 24) { + if (!this.hasColumn('tasks', 'created_by_pane_key')) { + this.db.exec('ALTER TABLE tasks ADD COLUMN created_by_pane_key TEXT') + } + if (!this.hasColumn('tasks', 'created_by_process_incarnation')) { + this.db.exec('ALTER TABLE tasks ADD COLUMN created_by_process_incarnation TEXT') + } + if (!this.hasColumn('tasks', 'created_by_run_generation')) { + this.db.exec('ALTER TABLE tasks ADD COLUMN created_by_run_generation INTEGER') + } + } + if (current < 25) { + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_dispatch_active_assignee_handle + ON dispatch_contexts(assignee_handle) + WHERE assignee_handle IS NOT NULL AND status IN ('pending', 'dispatched'); + `) + } + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_dispatch_assignee_pane_leaf + ON dispatch_contexts(${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL}) + WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched'); + `) this.createUndeliveredInboxIndexIfPossible() this.db.pragma(`user_version = ${SCHEMA_VERSION}`) @@ -2423,7 +2458,7 @@ export class OrchestrationDb { .prepare( `SELECT * FROM runs WHERE coordinator_pane_key IS NOT NULL AND legacy = 0 - AND ${PANE_KEY_MATCH_SUFFIX_SQL} = ? + AND ${RUN_PANE_KEY_MATCH_SUFFIX_SQL} = ? ORDER BY rowid` ) .all(paneKeyMatchSuffix(paneKey)) as RunRow[] @@ -3743,6 +3778,9 @@ export class OrchestrationDb { deps?: string[] parentId?: string createdByTerminalHandle?: string + createdByPaneKey?: string + createdByProcessIncarnation?: string + createdByRunGeneration?: number runId?: string }): TaskRow { const runId = task.runId ?? LEGACY_RUN_ID @@ -3770,13 +3808,20 @@ export class OrchestrationDb { }) this.db .prepare( - 'INSERT INTO tasks (id, run_id, parent_id, created_by_terminal_handle, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)' + `INSERT INTO tasks ( + id, run_id, parent_id, created_by_terminal_handle, created_by_pane_key, + created_by_process_incarnation, created_by_run_generation, + task_title, display_name, spec, status, deps + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ) .run( id, runId, task.parentId ?? null, task.createdByTerminalHandle ?? null, + task.createdByPaneKey ?? null, + task.createdByProcessIncarnation ?? null, + task.createdByRunGeneration ?? null, display.taskTitle || null, display.displayName || null, task.spec, @@ -3786,8 +3831,33 @@ export class OrchestrationDb { return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow } - getTask(id: string): TaskRow | undefined { - return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow | undefined + // Why: return the active creator Dispatch proof with the Task read; runtime still owns pane/process currency. + getTask(id: string): TaskRow | undefined + getTask(id: string, dispatchRunId: string): TaskRuntimeLineageRow | undefined + getTask(id: string, dispatchRunId?: string): TaskRow | TaskRuntimeLineageRow | undefined { + if (dispatchRunId === undefined) { + return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow | undefined + } + return this.db + .prepare( + `SELECT t.*, + creator.id AS creator_dispatch_id, + creator.run_id AS creator_dispatch_run_id, + creator.assignee_pane_key AS creator_dispatch_pane_key, + creator.process_incarnation AS creator_dispatch_process_incarnation + FROM tasks t + LEFT JOIN dispatch_contexts creator ON creator.rowid = ( + SELECT candidate.rowid + FROM dispatch_contexts candidate + WHERE candidate.assignee_handle = t.created_by_terminal_handle + AND candidate.run_id = ? + AND candidate.status IN ('pending', 'dispatched') + ORDER BY candidate.rowid DESC + LIMIT 1 + ) + WHERE t.id = ?` + ) + .get(dispatchRunId, id) as TaskRuntimeLineageRow | undefined } listTasks(filter?: { status?: TaskStatus; ready?: boolean; runId?: string }): TaskRow[] { @@ -6088,7 +6158,8 @@ export class OrchestrationDb { assigneeHandle: string, // Why: pane key is the remint-stable identity behind the handle — lets worker_done ownership survive handle reissue. assigneePaneKey?: string, - launchTokenHash?: string + launchTokenHash?: string, + processIncarnation?: string ): DispatchContextRow { const task = this.getTask(taskId) if (!task) { @@ -6118,8 +6189,9 @@ export class OrchestrationDb { .prepare( `INSERT INTO dispatch_contexts ( id, run_id, task_id, contract_version, launch_token_hash, - assignee_handle, assignee_pane_key, status, failure_count, dispatched_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, 'dispatched', ?, datetime('now'))` + assignee_handle, assignee_pane_key, process_incarnation, + status, failure_count, dispatched_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'dispatched', ?, datetime('now'))` ) .run( id, @@ -6129,6 +6201,7 @@ export class OrchestrationDb { launchTokenHash ?? null, assigneeHandle, assigneePaneKey ?? null, + processIncarnation ?? null, priorFailures ) this.hasAnyDispatchContextsCache = true @@ -6300,9 +6373,12 @@ export class OrchestrationDb { const actives = this.db .prepare( - "SELECT * FROM dispatch_contexts WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched')" + `SELECT * FROM dispatch_contexts + WHERE assignee_pane_key IS NOT NULL + AND status IN ('pending', 'dispatched') + AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ?` ) - .all() as DispatchContextRow[] + .all(paneKeyMatchSuffix(assigneePaneKey)) as DispatchContextRow[] for (const row of actives) { if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, assigneePaneKey)) { diff --git a/src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts b/src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts new file mode 100644 index 000000000..d13344004 --- /dev/null +++ b/src/main/runtime/orchestration/orchestration-creator-authority-performance.test.ts @@ -0,0 +1,156 @@ +import { afterEach, describe, expect, it } from 'vitest' +import type Database from '../../sqlite/sync-database' +import { OrchestrationDb } from './db' + +const CREATOR_PANE = 'tab-creator:11111111-1111-4111-8111-111111111111' +const CREATOR_PROCESS = 'pty-creator:incarnation-a' + +function sqliteFor(db: OrchestrationDb): Database.Database { + return (db as unknown as { db: Database.Database }).db +} + +describe('creator authority lookup performance', () => { + let db: OrchestrationDb | undefined + + afterEach(() => db?.close()) + + it('uses bounded creator-handle and pane-leaf indexes', () => { + db = new OrchestrationDb(':memory:') + const sqlite = sqliteFor(db) + const taskPlan = sqlite + .prepare( + `EXPLAIN QUERY PLAN + SELECT t.*, creator.id + FROM tasks t + LEFT JOIN dispatch_contexts creator ON creator.rowid = ( + SELECT candidate.rowid + FROM dispatch_contexts candidate + WHERE candidate.assignee_handle = t.created_by_terminal_handle + AND candidate.run_id = ? + AND candidate.status IN ('pending', 'dispatched') + ORDER BY candidate.rowid DESC LIMIT 1 + ) + WHERE t.id = ?` + ) + .all('run-owner', 'task-worker') as { detail: string }[] + const panePlan = sqlite + .prepare( + `EXPLAIN QUERY PLAN + SELECT * FROM dispatch_contexts + WHERE assignee_pane_key IS NOT NULL + AND status IN ('pending', 'dispatched') + AND substr(assignee_pane_key, instr(assignee_pane_key, ':') + 1) = ?` + ) + .all('11111111-1111-4111-8111-111111111111') as { detail: string }[] + const taskDetails = taskPlan.map((row) => row.detail).join(' | ') + const paneDetails = panePlan.map((row) => row.detail).join(' | ') + + expect(taskDetails).toContain('idx_dispatch_active_assignee_handle') + expect(taskDetails).not.toMatch(/SCAN (?:runs|rebound)/) + expect(paneDetails).toContain('idx_dispatch_assignee_pane_leaf') + }) + + it('keeps 300 Task reads bounded with 50,000 retained Runs', () => { + db = new OrchestrationDb(':memory:') + const run = db.createRun({ + objective: 'owner', + coordinatorHandle: 'term-coordinator', + coordinatorPaneKey: 'tab-coordinator:22222222-2222-4222-8222-222222222222' + }) + const creatorTask = db.createTask({ spec: 'creator', runId: run.id }) + db.createDispatchContext( + creatorTask.id, + 'term-creator', + CREATOR_PANE, + undefined, + CREATOR_PROCESS + ) + const workerTask = db.createTask({ + spec: 'worker', + runId: run.id, + createdByTerminalHandle: 'term-creator', + createdByPaneKey: CREATOR_PANE, + createdByProcessIncarnation: CREATOR_PROCESS, + createdByRunGeneration: run.consumer_generation + }) + sqliteFor(db) + .prepare( + `WITH RECURSIVE run_numbers(value) AS ( + VALUES (1) UNION ALL SELECT value + 1 FROM run_numbers WHERE value < ? + ) + INSERT INTO runs (id, objective, coordinator_handle, consumer_generation, legacy) + SELECT printf('retained_%05d', value), 'retained', printf('term_%05d', value), 1, 0 + FROM run_numbers` + ) + .run(50_000) + + for (let index = 0; index < 10; index += 1) { + db.getTask(workerTask.id, run.id) + } + const startedAt = performance.now() + for (let index = 0; index < 300; index += 1) { + expect(db.getTask(workerTask.id, run.id)?.creator_dispatch_run_id).toBe(run.id) + } + const elapsedMs = performance.now() - startedAt + + expect(elapsedMs).toBeLessThan(200) + }) + + it.each([20_000, 50_000])( + 'keeps active creator lookup bounded with %i retained same-handle Dispatches', + (retainedDispatchCount) => { + db = new OrchestrationDb(':memory:') + const run = db.createRun({ + objective: 'owner', + coordinatorHandle: 'term-coordinator', + coordinatorPaneKey: 'tab-coordinator:22222222-2222-4222-8222-222222222222' + }) + sqliteFor(db) + .prepare( + `WITH RECURSIVE dispatch_numbers(value) AS ( + VALUES (1) UNION ALL SELECT value + 1 FROM dispatch_numbers WHERE value < ? + ) + INSERT INTO dispatch_contexts ( + id, run_id, task_id, assignee_handle, status, completed_at + ) + SELECT printf('retained_ctx_%05d', value), ?, printf('retained_task_%05d', value), + 'term-creator', 'completed', datetime('now') + FROM dispatch_numbers` + ) + .run(retainedDispatchCount, run.id) + const creatorTask = db.createTask({ spec: 'creator', runId: run.id }) + const creatorDispatch = db.createDispatchContext( + creatorTask.id, + 'term-creator', + CREATOR_PANE, + undefined, + CREATOR_PROCESS + ) + const workerTask = db.createTask({ + spec: 'worker', + runId: run.id, + createdByTerminalHandle: 'term-creator', + createdByPaneKey: CREATOR_PANE, + createdByProcessIncarnation: CREATOR_PROCESS, + createdByRunGeneration: run.consumer_generation + }) + + for (let index = 0; index < 10; index += 1) { + db.getTask(workerTask.id, run.id) + db.getActiveDispatchForTerminal('term-creator') + } + const startedAt = performance.now() + for (let index = 0; index < 300; index += 1) { + expect(db.getTask(workerTask.id, run.id)?.creator_dispatch_id).toBe(creatorDispatch.id) + expect(db.getActiveDispatchForTerminal('term-creator')?.id).toBe(creatorDispatch.id) + } + const elapsedMs = performance.now() - startedAt + + const competingTask = db.createTask({ spec: 'competing creator', runId: run.id }) + expect(() => db!.createDispatchContext(competingTask.id, 'term-creator')).toThrow( + `Terminal term-creator already has an active dispatch (${creatorDispatch.id}` + ) + expect(elapsedMs).toBeLessThan(200) + } + ) +}) diff --git a/src/main/runtime/orchestration/orchestration-db-retention-pagination.test.ts b/src/main/runtime/orchestration/orchestration-db-retention-pagination.test.ts index 0b78eaebd..8cb0547ee 100644 --- a/src/main/runtime/orchestration/orchestration-db-retention-pagination.test.ts +++ b/src/main/runtime/orchestration/orchestration-db-retention-pagination.test.ts @@ -218,7 +218,7 @@ describe('OrchestrationDb dispatch assignee index migration', () => { } }) - it('migrates a populated v21 database idempotently', () => { + it('migrates a populated upstream v23 database idempotently', () => { tempDir = mkdtempSync(join(tmpdir(), 'orca-dispatch-index-migration-')) const dbPath = join(tempDir, 'orchestration.db') db = new OrchestrationDb(dbPath) @@ -228,14 +228,25 @@ describe('OrchestrationDb dispatch assignee index migration', () => { db = undefined const oldDb = new Database(dbPath) - oldDb.exec('DROP INDEX IF EXISTS idx_dispatch_assignee_handle') - oldDb.pragma('user_version = 21') + oldDb.exec(` + DROP INDEX IF EXISTS idx_dispatch_active_assignee_handle; + DROP INDEX IF EXISTS idx_dispatch_assignee_pane_leaf; + ALTER TABLE tasks DROP COLUMN created_by_pane_key; + ALTER TABLE tasks DROP COLUMN created_by_process_incarnation; + ALTER TABLE tasks DROP COLUMN created_by_run_generation; + `) + oldDb.pragma('user_version = 23') oldDb.close() db = new OrchestrationDb(dbPath) const sqlite = sqliteFor(db) - expect(sqlite.pragma('user_version', { simple: true })).toBe(23) + expect(sqlite.pragma('user_version', { simple: true })).toBe(25) expect(db.getDispatchContextById(dispatch.id)).toMatchObject({ assignee_handle: 'term_worker' }) + expect(db.getTask(task.id)).toMatchObject({ + created_by_pane_key: null, + created_by_process_incarnation: null, + created_by_run_generation: null + }) expect( sqlite .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name = ?") @@ -249,12 +260,75 @@ describe('OrchestrationDb dispatch assignee index migration', () => { ) .all('term_worker') as { detail: string }[] expect(plan.map((row) => row.detail).join('\n')).toContain( - 'USING INDEX idx_dispatch_assignee_handle' + 'USING INDEX idx_dispatch_active_assignee_handle' ) + expect( + sqlite + .prepare("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?") + .get('idx_dispatch_active_assignee_handle') + ).toMatchObject({ + sql: expect.stringContaining("status IN ('pending', 'dispatched')") + }) + expect( + sqlite + .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name = ?") + .get('idx_dispatch_assignee_pane_leaf') + ).toBeDefined() db.close() db = new OrchestrationDb(dbPath) - expect(sqliteFor(db).pragma('user_version', { simple: true })).toBe(23) + expect(sqliteFor(db).pragma('user_version', { simple: true })).toBe(25) expect(db.getDispatchContextById(dispatch.id)).toBeDefined() }) + + it('adds the active-handle index to a populated v24 database idempotently', () => { + tempDir = mkdtempSync(join(tmpdir(), 'orca-active-dispatch-index-migration-')) + const dbPath = join(tempDir, 'orchestration.db') + db = new OrchestrationDb(dbPath) + const run = db.createRun({ + objective: 'retained v24 authority', + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:leaf_coord' + }) + const task = db.createTask({ + spec: 'indexed lookup', + runId: run.id, + createdByTerminalHandle: 'term_creator', + createdByPaneKey: 'tab_creator:leaf_creator', + createdByProcessIncarnation: 'pty_creator:incarnation-a', + createdByRunGeneration: run.consumer_generation + }) + const dispatch = db.createDispatchContext(task.id, 'term_worker') + db.close() + db = undefined + + const oldDb = new Database(dbPath) + oldDb.exec('DROP INDEX IF EXISTS idx_dispatch_active_assignee_handle') + oldDb.pragma('user_version = 24') + oldDb.close() + + db = new OrchestrationDb(dbPath) + const sqlite = sqliteFor(db) + expect(sqlite.pragma('user_version', { simple: true })).toBe(25) + expect(db.getTask(task.id)).toMatchObject({ + created_by_pane_key: 'tab_creator:leaf_creator', + created_by_process_incarnation: 'pty_creator:incarnation-a', + created_by_run_generation: 1 + }) + expect(db.getDispatchContextById(dispatch.id)).toMatchObject({ + assignee_handle: 'term_worker' + }) + expect( + sqlite + .prepare("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?") + .get('idx_dispatch_active_assignee_handle') + ).toMatchObject({ + sql: expect.stringContaining('assignee_handle IS NOT NULL') + }) + + db.close() + db = new OrchestrationDb(dbPath) + expect(sqliteFor(db).pragma('user_version', { simple: true })).toBe(25) + expect(db.getTask(task.id)?.created_by_process_incarnation).toBe('pty_creator:incarnation-a') + }) }) diff --git a/src/main/runtime/orchestration/types.ts b/src/main/runtime/orchestration/types.ts index 76b3d536f..e399b13ce 100644 --- a/src/main/runtime/orchestration/types.ts +++ b/src/main/runtime/orchestration/types.ts @@ -246,6 +246,9 @@ export type TaskRow = { run_id: string parent_id: string | null created_by_terminal_handle: string | null + created_by_pane_key: string | null + created_by_process_incarnation: string | null + created_by_run_generation: number | null task_title: string | null display_name: string | null spec: string diff --git a/src/main/runtime/rpc/methods/orchestration.test.ts b/src/main/runtime/rpc/methods/orchestration.test.ts index e2aaf31b6..8ba154c48 100644 --- a/src/main/runtime/rpc/methods/orchestration.test.ts +++ b/src/main/runtime/rpc/methods/orchestration.test.ts @@ -1763,17 +1763,27 @@ describe('orchestration RPC methods', () => { expect(result.task.status).toBe('pending') }) - it('records the caller terminal handle when creating a task', async () => { + it('records the caller pane, process, and Run generation when creating a task', async () => { setup() vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => handle === 'term_creator' ? coordinatorPaneKey : null ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({ + terminalHandle: 'term_creator', + paneKey: coordinatorPaneKey, + processIncarnation: 'pty-creator:incarnation-a' + } as never) const result = (await call('orchestration.taskCreate', { spec: 'spawn related workspace', callerTerminalHandle: 'term_creator' })) as { task: { id: string } } - expect(db.getTask(result.task.id)?.created_by_terminal_handle).toBe('term_creator') + expect(db.getTask(result.task.id)).toMatchObject({ + created_by_terminal_handle: 'term_creator', + created_by_pane_key: coordinatorPaneKey, + created_by_process_incarnation: 'pty-creator:incarnation-a', + created_by_run_generation: 1 + }) }) it('rejects invalid deps JSON', async () => { @@ -1898,6 +1908,16 @@ describe('orchestration RPC methods', () => { vi.mocked(runtime.getTerminalPaneKey).mockImplementation((candidate) => candidate === handle ? `tab_worker:${handle}` : coordinatorPaneKey ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((candidate) => + candidate === handle + ? ({ + terminalHandle: handle, + paneKey: `tab_worker:${handle}`, + processIncarnation: `runtime_test:${handle}:1`, + launchTokenHash: null + } as never) + : null + ) } it('dispatches a task to a terminal', async () => { @@ -1929,7 +1949,7 @@ describe('orchestration RPC methods', () => { expect(db.getDispatchContextById(result.dispatch.id)?.assignee_pane_key).toBe('tab_w:leaf_w') }) - it('commits the target process launch token on a manual dispatch', async () => { + it('commits authenticated process authority on a manual dispatch', async () => { setup() vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({ runtimeId: runtime.getRuntimeId(), @@ -1948,9 +1968,24 @@ describe('orchestration RPC methods', () => { to: 'term_a' })) as { dispatch: { id: string } } - expect(db.getDispatchContextById(result.dispatch.id)?.launch_token_hash).toBe( - 'launch-token-hash' - ) + expect(db.getDispatchContextById(result.dispatch.id)).toMatchObject({ + assignee_pane_key: 'tab_w:leaf_w', + process_incarnation: 'runtime_test:term_a:1', + launch_token_hash: 'launch-token-hash' + }) + }) + + it('does not infer manual process authority from an unauthenticated handle', async () => { + setup() + const task = db.createTask({ spec: 'work' }) + + const result = (await call('orchestration.dispatch', { + task: task.id, + to: 'term_a' + })) as { dispatch: { id: string } } + + expect(runtime.getTerminalProcessIncarnation('term_a')).toBe('runtime_test:term_a:1') + expect(db.getDispatchContextById(result.dispatch.id)?.process_incarnation).toBeNull() }) it('rejects dispatch for a pending task', async () => { diff --git a/src/main/runtime/rpc/methods/orchestration.ts b/src/main/runtime/rpc/methods/orchestration.ts index 34ea1f368..0bdb24543 100644 --- a/src/main/runtime/rpc/methods/orchestration.ts +++ b/src/main/runtime/rpc/methods/orchestration.ts @@ -1100,6 +1100,16 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ throw new Error('Invalid --deps: must be a JSON array of task IDs') } } + const run = resolveRunScope(runtime, { + runId: params.run, + callerTerminalHandle: params.callerTerminalHandle, + requireCurrentConsumer: true, + legacyCoordinatorRunId, + callerEvidence: orchestrationCompatibilityEvidence + }) + const creatorAuthority = params.callerTerminalHandle + ? runtime.getOrchestrationDispatchAuthority(params.callerTerminalHandle) + : null const task = db.createTask({ spec: params.spec, taskTitle: params.taskTitle, @@ -1107,13 +1117,14 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ deps, parentId: params.parent, createdByTerminalHandle: params.callerTerminalHandle, - runId: resolveRunScope(runtime, { - runId: params.run, - callerTerminalHandle: params.callerTerminalHandle, - requireCurrentConsumer: true, - legacyCoordinatorRunId, - callerEvidence: orchestrationCompatibilityEvidence - }).id + ...(creatorAuthority?.paneKey && creatorAuthority.processIncarnation + ? { + createdByPaneKey: creatorAuthority.paneKey, + createdByProcessIncarnation: creatorAuthority.processIncarnation, + createdByRunGeneration: run.consumer_generation + } + : {}), + runId: run.id }) return { task } } @@ -1256,9 +1267,9 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ const assigneePaneKey = dispatchAuthority?.paneKey ?? runtime.getTerminalPaneKey(to) ?? undefined const processIncarnation = - dispatchAuthority?.processIncarnation ?? - runtime.getTerminalProcessIncarnation(to) ?? - undefined + dispatchAuthority?.paneKey && dispatchAuthority.processIncarnation + ? dispatchAuthority.processIncarnation + : undefined if (params.inject && (!assigneePaneKey || !processIncarnation)) { throw new OrchestrationError( 'stable_pane_required', @@ -1271,7 +1282,8 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ params.task, to, assigneePaneKey, - dispatchAuthority?.launchTokenHash ?? undefined + dispatchAuthority?.launchTokenHash ?? undefined, + processIncarnation ) const dispatchCapability = params.inject ? db.mintDispatchCapability({ diff --git a/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts b/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts index be5628df4..468f261b7 100644 --- a/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts +++ b/src/main/runtime/rpc/orchestration-legacy-coordinator-race.test.ts @@ -628,6 +628,16 @@ describe('legacy coordinator takeover races', () => { handle === COORDINATOR_HANDLE ? COORDINATOR_PANE : handle === targetHandle ? targetPane : null ) vi.spyOn(harness.runtime, 'getTerminalProcessIncarnation').mockReturnValue('current-process') + vi.spyOn(harness.runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => + handle === targetHandle + ? ({ + terminalHandle: targetHandle, + paneKey: targetPane, + processIncarnation: 'current-process', + launchTokenHash: null + } as never) + : null + ) let resolveDetection: ((detected: boolean) => void) | undefined let signalDetectionStarted: (() => void) | undefined const detectionStarted = new Promise((resolve) => { diff --git a/src/renderer/src/store/slices/agent-status.test.ts b/src/renderer/src/store/slices/agent-status.test.ts index 70c334eae..b6b2976e2 100644 --- a/src/renderer/src/store/slices/agent-status.test.ts +++ b/src/renderer/src/store/slices/agent-status.test.ts @@ -265,6 +265,42 @@ describe('agent status runtime orchestration metadata', () => { }) }) + it('clears stale lineage when the authoritative runtime snapshot loses its Run binding', () => { + vi.useFakeTimers() + const store = createTestStore() + const childPaneKey = 'tab-child:11111111-1111-4111-8111-111111111111' + + store.getState().setAgentStatus(childPaneKey, { + state: 'working', + prompt: 'child agent', + agentType: 'codex', + orchestration: { + taskId: 'task-1', + dispatchId: 'ctx-1', + dispatchStatus: 'dispatched', + parentTerminalHandle: 'term-old-coordinator', + parentPaneKey: 'tab-parent:22222222-2222-4222-8222-222222222222', + coordinatorHandle: 'term-old-coordinator', + orchestrationRunId: 'run-1' + } + }) + store.getState().setRuntimeAgentOrchestrationByPaneKey({ + [childPaneKey]: { + taskId: 'task-1', + dispatchId: 'ctx-1', + dispatchStatus: 'dispatched', + orchestrationRunId: 'run-1' + } + }) + + expect(store.getState().agentStatusByPaneKey[childPaneKey].orchestration).toEqual({ + taskId: 'task-1', + dispatchId: 'ctx-1', + dispatchStatus: 'dispatched', + orchestrationRunId: 'run-1' + }) + }) + it('updates runtime status for the same dispatch', () => { vi.useFakeTimers() const store = createTestStore() diff --git a/src/renderer/src/store/slices/agent-status.ts b/src/renderer/src/store/slices/agent-status.ts index d8c87362e..921cc58d0 100644 --- a/src/renderer/src/store/slices/agent-status.ts +++ b/src/renderer/src/store/slices/agent-status.ts @@ -1125,6 +1125,9 @@ function mergeCurrentOrchestrationContext( if (!sameDispatch) { return current } + if (current.dispatchStatus !== undefined) { + return orchestrationContextsEqual(existing, current) ? existing : current + } const merged = { ...existing, ...current } return orchestrationContextsEqual(existing, merged) ? existing : merged }