fix(hibernation): reap restored subagent rows with no live agent process (#11219)
* fix(hibernation): reap restored subagent rows with no live agent process A pane whose Claude session had a subagent in flight can be locked out of agent hibernation for good. A PTY that dies while Orca is down never runs the teardown that clears pane state, so hydrate rebuilds a subagent roster that nothing can retire: the existing reap needs the parent to emit a complete `background_tasks` inventory, and a parent that went idle before the restart never emits one. The restored row keeps gating the pane 'working', and hibernation only accepts 'done'. Observed locally: six panes parked at SubagentStop in state 'working' for 17 to 145 hours, each still holding a working child row. Adds a second reap path. Hydrate seeds are marked `restoredFromSnapshot`, cleared by any live lifecycle event or an id-exact running inventory entry. One post-restore sweep drops the rows still unconfirmed when the pane's PTY is absent from the live local inventory, then re-derives the child-gated 'working' to 'done'. The scan is local-only by construction: panes with a relay connection id are skipped and SSH-scoped PTY ids resolve as live, since a remote agent runs on the far host and could never appear in a local listing. An unreadable inventory is not evidence that anything exited, so it is a no-op. Panes that have reported to this runtime are left alone. `stateStartedAt` and `stateHistory` are untouched, so a draft typed while the pane was working still blocks hibernation. * fix(hibernation): prove local ownership before restored reap * fix(hibernation): require authoritative restored PTY absence * fix(hibernation): probe restored PTY liveness authoritatively * fix(hibernation): restart idle window after restored reap * fix(hibernation): type restored reconciliation timing * fix(hibernation): respect worktree host ownership * fix(hibernation): preserve same-id restored PTY rebinds * fix(hibernation): fence batched restored PTY probes
This commit is contained in:
Brennan Benson
GitHub
parent
f56c37b470
commit
24a2accc3c
|
|
@ -0,0 +1,559 @@
|
|||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { AgentSubagentSnapshot } from '../../shared/agent-status-types'
|
||||
import { makePaneKey } from '../../shared/stable-pane-id'
|
||||
import { toAppSshPtyId } from '../../shared/ssh-pty-id'
|
||||
import { AgentHookServer } from './server'
|
||||
import {
|
||||
indexPersistedPaneKeyPtyIds,
|
||||
isLocalExecutionHost,
|
||||
resolveAgentWorkspaceExecutionHostId,
|
||||
sweepRestoredSubagentsWithoutLiveAgent
|
||||
} from './restored-subagent-liveness-sweep'
|
||||
|
||||
const LEAF = '11111111-1111-4111-8111-111111111111'
|
||||
const PANE = makePaneKey('tab-1', LEAF)
|
||||
const PTY = 'wt-1__pty-1'
|
||||
const WORKING_CHILD: AgentSubagentSnapshot = {
|
||||
id: 'areview-loop-c237a4c577493352',
|
||||
state: 'working',
|
||||
startedAt: 1_000
|
||||
}
|
||||
|
||||
let dir: string
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'orca-restored-subagent-'))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
/** Persist a pane whose lead has finished but whose roster still holds a working
|
||||
* child, then restart into a fresh server — the shape a machine sleep leaves
|
||||
* behind when the child's SubagentStop is lost while Orca is down. */
|
||||
async function restartWithInFlightSubagent(options?: {
|
||||
connectionId?: string
|
||||
state?: 'working' | 'waiting'
|
||||
subagents?: AgentSubagentSnapshot[]
|
||||
additionalPaneKey?: string
|
||||
}): Promise<AgentHookServer> {
|
||||
const first = new AgentHookServer()
|
||||
await first.start({ env: 'production', userDataPath: dir })
|
||||
first.ingestTerminalStatus({
|
||||
paneKey: PANE,
|
||||
tabId: 'tab-1',
|
||||
worktreeId: 'wt-1',
|
||||
connectionId: options?.connectionId ?? null,
|
||||
payload: {
|
||||
state: options?.state ?? 'working',
|
||||
prompt: 'review the PR',
|
||||
agentType: 'claude',
|
||||
subagents: options?.subagents ?? [WORKING_CHILD]
|
||||
}
|
||||
})
|
||||
if (options?.additionalPaneKey) {
|
||||
first.ingestTerminalStatus({
|
||||
paneKey: options.additionalPaneKey,
|
||||
tabId: 'tab-2',
|
||||
worktreeId: 'wt-1',
|
||||
connectionId: options.connectionId ?? null,
|
||||
payload: {
|
||||
state: options.state ?? 'working',
|
||||
prompt: 'review the PR',
|
||||
agentType: 'claude',
|
||||
subagents: options.subagents ?? [WORKING_CHILD]
|
||||
}
|
||||
})
|
||||
}
|
||||
first.flushStatusPersistSync()
|
||||
first.stop()
|
||||
|
||||
const restarted = new AgentHookServer()
|
||||
await restarted.start({ env: 'production', userDataPath: dir })
|
||||
return restarted
|
||||
}
|
||||
|
||||
function sweepWith(
|
||||
server: AgentHookServer,
|
||||
overrides: {
|
||||
probeLiveLocalPty?: (ptyId: string) => boolean | null | Promise<boolean | null>
|
||||
executionHostId?: string | null
|
||||
boundPtyIdByPaneKey?: Record<string, string>
|
||||
persistedPtyIdByPaneKey?: Record<string, string>
|
||||
} = {}
|
||||
): Promise<number> {
|
||||
return sweepRestoredSubagentsWithoutLiveAgent({
|
||||
probeLiveLocalPty: async (ptyId) =>
|
||||
overrides.probeLiveLocalPty ? await overrides.probeLiveLocalPty(ptyId) : false,
|
||||
isLocalExecutionHost: () =>
|
||||
isLocalExecutionHost(
|
||||
overrides.executionHostId === undefined ? 'local' : overrides.executionHostId
|
||||
),
|
||||
getBoundPtyIdForPaneKey: (paneKey) => overrides.boundPtyIdByPaneKey?.[paneKey],
|
||||
getPersistedPtyIdForPaneKey: (paneKey) => overrides.persistedPtyIdByPaneKey?.[paneKey],
|
||||
reap: (isLocalHost, isLocalPaneAgentLive, isLocalPaneLivenessEvidenceCurrent) =>
|
||||
server.reapRestoredClaudeSubagentsWithoutLiveAgent(
|
||||
isLocalHost,
|
||||
isLocalPaneAgentLive,
|
||||
isLocalPaneLivenessEvidenceCurrent
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
function paneStatus(
|
||||
server: AgentHookServer,
|
||||
paneKey = PANE
|
||||
): { state: string; subagents?: AgentSubagentSnapshot[] } {
|
||||
const entry = server.getStatusSnapshotForPane(paneKey)[0]
|
||||
return { state: entry?.state ?? 'missing', subagents: entry?.subagents }
|
||||
}
|
||||
|
||||
describe('restored subagent liveness sweep', () => {
|
||||
it('reaps the phantom seed so a slept-through pane reaches done', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
expect(paneStatus(server)).toEqual({ state: 'working', subagents: [WORKING_CHILD] })
|
||||
const previous = server.getStatusSnapshotForPane(PANE)[0]
|
||||
const previousReceivedAt = previous?.receivedAt ?? 0
|
||||
const reconciledAt = previousReceivedAt + 1
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValue(previousReceivedAt)
|
||||
|
||||
expect(await sweepWith(server, { persistedPtyIdByPaneKey: { [PANE]: PTY } })).toBe(1)
|
||||
|
||||
expect(paneStatus(server)).toEqual({ state: 'done', subagents: undefined })
|
||||
expect(server.getStatusSnapshotForPane(PANE)[0]).toMatchObject({
|
||||
receivedAt: reconciledAt,
|
||||
stateStartedAt: reconciledAt
|
||||
})
|
||||
now.mockRestore()
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a seed whose pane still has a live local PTY', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty: (ptyId) => ptyId === PTY,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(paneStatus(server)).toEqual({ state: 'working', subagents: [WORKING_CHILD] })
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('preserves timing when reaping rows does not change the lead state', async () => {
|
||||
const server = await restartWithInFlightSubagent({ state: 'waiting' })
|
||||
try {
|
||||
const previous = server.getStatusSnapshotForPane(PANE)[0]
|
||||
|
||||
expect(await sweepWith(server, { persistedPtyIdByPaneKey: { [PANE]: PTY } })).toBe(1)
|
||||
|
||||
expect(server.getStatusSnapshotForPane(PANE)[0]).toMatchObject({
|
||||
state: 'waiting',
|
||||
subagents: undefined,
|
||||
receivedAt: previous?.receivedAt,
|
||||
stateStartedAt: previous?.stateStartedAt
|
||||
})
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('uses targeted liveness for a PTY bound in this runtime', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
const probeLiveLocalPty = vi.fn((ptyId: string) => ptyId === PTY)
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
boundPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('probes only restored rosters and deduplicates a shared exact PTY', async () => {
|
||||
const otherPane = makePaneKey('tab-2', LEAF)
|
||||
const server = await restartWithInFlightSubagent({ additionalPaneKey: otherPane })
|
||||
const probeLiveLocalPty = vi.fn(() => false)
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY, [otherPane]: PTY }
|
||||
})
|
||||
).toBe(2)
|
||||
|
||||
expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY)
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not probe a hydrated Claude pane without restored rows', async () => {
|
||||
const server = await restartWithInFlightSubagent({ subagents: [] })
|
||||
const probeLiveLocalPty = vi.fn(() => false)
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(probeLiveLocalPty).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('never reaps an SSH-launched pane, whose agent cannot appear in a local scan', async () => {
|
||||
const sshPtyId = toAppSshPtyId('conn-1', PTY)
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
executionHostId: 'ssh:conn-1',
|
||||
persistedPtyIdByPaneKey: { [PANE]: sshPtyId }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(paneStatus(server)).toEqual({ state: 'working', subagents: [WORKING_CHILD] })
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('never reaps a relay-owned pane even with no local PTY at all', async () => {
|
||||
const server = await restartWithInFlightSubagent({ connectionId: 'conn-1' })
|
||||
try {
|
||||
expect(await sweepWith(server)).toBe(0)
|
||||
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('never reaps a runtime-hosted pane from the desktop local provider', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const probeLiveLocalPty = vi.fn(() => false)
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
executionHostId: 'runtime:ephemeral-vm-1',
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: 'remote:ephemeral-vm-1@@pty-1' }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(probeLiveLocalPty).not.toHaveBeenCalled()
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('does nothing when targeted PTY liveness is unknown', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty: () => null,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('does nothing without an exact pane PTY binding', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const probeLiveLocalPty = vi.fn(() => false)
|
||||
try {
|
||||
expect(await sweepWith(server, { probeLiveLocalPty })).toBe(0)
|
||||
|
||||
expect(probeLiveLocalPty).not.toHaveBeenCalled()
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('skips panes that have reported to this runtime', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
server.ingestTerminalStatus({
|
||||
paneKey: PANE,
|
||||
tabId: 'tab-1',
|
||||
worktreeId: 'wt-1',
|
||||
payload: {
|
||||
state: 'working',
|
||||
prompt: 'now reviewing round two',
|
||||
agentType: 'claude',
|
||||
subagents: [WORKING_CHILD]
|
||||
}
|
||||
})
|
||||
|
||||
expect(await sweepWith(server)).toBe(0)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a pane that reports while its liveness probe is pending', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
let resolveProbe!: (live: boolean | null) => void
|
||||
const probeLiveLocalPty = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean | null>((resolve) => {
|
||||
resolveProbe = resolve
|
||||
})
|
||||
)
|
||||
try {
|
||||
const sweep = sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledOnce())
|
||||
server.ingestTerminalStatus({
|
||||
paneKey: PANE,
|
||||
tabId: 'tab-1',
|
||||
worktreeId: 'wt-1',
|
||||
payload: {
|
||||
state: 'working',
|
||||
prompt: 'confirmed during probe',
|
||||
agentType: 'claude',
|
||||
subagents: [WORKING_CHILD]
|
||||
}
|
||||
})
|
||||
resolveProbe(false)
|
||||
|
||||
expect(await sweep).toBe(0)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a pane that binds a newer PTY while the old probe is pending', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const boundPtyIdByPaneKey = { [PANE]: PTY }
|
||||
let resolveProbe!: (live: boolean | null) => void
|
||||
const probeLiveLocalPty = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean | null>((resolve) => {
|
||||
resolveProbe = resolve
|
||||
})
|
||||
)
|
||||
try {
|
||||
const sweep = sweepWith(server, { probeLiveLocalPty, boundPtyIdByPaneKey })
|
||||
await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY))
|
||||
boundPtyIdByPaneKey[PANE] = 'wt-1__pty-new'
|
||||
resolveProbe(false)
|
||||
|
||||
expect(await sweep).toBe(0)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a pane that rebinds its persisted PTY while the probe is pending', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const boundPtyIdByPaneKey: Record<string, string> = {}
|
||||
let resolveProbe!: (live: boolean | null) => void
|
||||
const probeLiveLocalPty = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean | null>((resolve) => {
|
||||
resolveProbe = resolve
|
||||
})
|
||||
)
|
||||
try {
|
||||
const sweep = sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
boundPtyIdByPaneKey,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledExactlyOnceWith(PTY))
|
||||
boundPtyIdByPaneKey[PANE] = PTY
|
||||
resolveProbe(false)
|
||||
|
||||
expect(await sweep).toBe(0)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps a same-id rebind that lands after its probe but before the batch settles', async () => {
|
||||
const otherPane = makePaneKey('tab-2', LEAF)
|
||||
const otherPty = 'wt-1__pty-2'
|
||||
const server = await restartWithInFlightSubagent({ additionalPaneKey: otherPane })
|
||||
const boundPtyIdByPaneKey: Record<string, string> = {}
|
||||
let resolveOtherProbe!: (live: boolean | null) => void
|
||||
const probeLiveLocalPty = vi.fn((ptyId: string) =>
|
||||
ptyId === PTY
|
||||
? Promise.resolve(false)
|
||||
: new Promise<boolean | null>((resolve) => {
|
||||
resolveOtherProbe = resolve
|
||||
})
|
||||
)
|
||||
try {
|
||||
const sweep = sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
boundPtyIdByPaneKey,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY, [otherPane]: otherPty }
|
||||
})
|
||||
await vi.waitFor(() => expect(probeLiveLocalPty).toHaveBeenCalledTimes(2))
|
||||
boundPtyIdByPaneKey[PANE] = PTY
|
||||
resolveOtherProbe(false)
|
||||
|
||||
expect(await sweep).toBe(1)
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
expect(paneStatus(server, otherPane).state).toBe('done')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('leaves a pane whose lead is genuinely mid-turn at working', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
try {
|
||||
// Why: the lead's own tool event proves the process is alive; only the
|
||||
// child-gated 'working' may be re-derived, so the state must survive.
|
||||
server.ingestTerminalStatus({
|
||||
paneKey: makePaneKey('tab-2', LEAF),
|
||||
tabId: 'tab-2',
|
||||
worktreeId: 'wt-1',
|
||||
payload: { state: 'working', prompt: 'still going', agentType: 'claude' }
|
||||
})
|
||||
|
||||
await sweepWith(server)
|
||||
|
||||
expect(paneStatus(server, makePaneKey('tab-2', LEAF)).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('reports zero and leaves state alone when targeted liveness throws', async () => {
|
||||
const server = await restartWithInFlightSubagent()
|
||||
const probeLiveLocalPty = vi.fn(() => {
|
||||
throw new Error('daemon unreachable')
|
||||
})
|
||||
try {
|
||||
expect(
|
||||
await sweepWith(server, {
|
||||
probeLiveLocalPty,
|
||||
persistedPtyIdByPaneKey: { [PANE]: PTY }
|
||||
})
|
||||
).toBe(0)
|
||||
|
||||
expect(paneStatus(server).state).toBe('working')
|
||||
} finally {
|
||||
server.stop()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('indexPersistedPaneKeyPtyIds', () => {
|
||||
it('maps layout leaves to pane keys and ignores empty bindings', () => {
|
||||
expect(
|
||||
indexPersistedPaneKeyPtyIds({
|
||||
'tab-1': { ptyIdsByLeafId: { [LEAF]: PTY, 'leaf-empty': '' } },
|
||||
'tab-2': undefined,
|
||||
'tab-3': {}
|
||||
})
|
||||
).toEqual(new Map([[PANE, PTY]]))
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveAgentWorkspaceExecutionHostId', () => {
|
||||
const localRepo = {
|
||||
id: 'local-repo',
|
||||
connectionId: null,
|
||||
executionHostId: 'local' as const
|
||||
}
|
||||
const runtimeRepo = {
|
||||
id: 'runtime-repo',
|
||||
connectionId: null,
|
||||
executionHostId: 'runtime:ephemeral-vm-1' as const
|
||||
}
|
||||
const futureHostRepo = {
|
||||
id: 'future-repo',
|
||||
connectionId: null,
|
||||
executionHostId: 'container:future-host'
|
||||
}
|
||||
const deps = {
|
||||
getRepo: (repoId: string) =>
|
||||
[localRepo, runtimeRepo, futureHostRepo].find((candidate) => candidate.id === repoId),
|
||||
getWorktreeMeta: (worktreeId: string) => {
|
||||
if (worktreeId === 'local-repo::/runtime-worktree') {
|
||||
return { hostId: 'runtime:worktree-owner' }
|
||||
}
|
||||
if (worktreeId === 'runtime-repo::/local-worktree') {
|
||||
return { hostId: 'local' }
|
||||
}
|
||||
if (worktreeId === 'local-repo::/future-worktree') {
|
||||
return { hostId: 'container:future-host' }
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
getFolderWorkspace: (id: string) =>
|
||||
id === 'folder-runtime' ? { projectGroupId: 'group-runtime', connectionId: null } : undefined,
|
||||
getProjectGroups: () => [
|
||||
{
|
||||
id: 'group-runtime',
|
||||
connectionId: null,
|
||||
executionHostId: 'runtime:ephemeral-vm-1'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
it('positively identifies local ownership and rejects runtime hosts', () => {
|
||||
expect(resolveAgentWorkspaceExecutionHostId('local-repo::/repo', deps)).toBe('local')
|
||||
expect(resolveAgentWorkspaceExecutionHostId('runtime-repo::/repo', deps)).toBe(
|
||||
'runtime:ephemeral-vm-1'
|
||||
)
|
||||
expect(resolveAgentWorkspaceExecutionHostId('folder:folder-runtime', deps)).toBe(
|
||||
'runtime:ephemeral-vm-1'
|
||||
)
|
||||
expect(resolveAgentWorkspaceExecutionHostId('local-repo::/runtime-worktree', deps)).toBe(
|
||||
'runtime:worktree-owner'
|
||||
)
|
||||
expect(resolveAgentWorkspaceExecutionHostId('runtime-repo::/local-worktree', deps)).toBe(
|
||||
'local'
|
||||
)
|
||||
expect(resolveAgentWorkspaceExecutionHostId('future-repo::/repo', deps)).toBeNull()
|
||||
expect(resolveAgentWorkspaceExecutionHostId('local-repo::/future-worktree', deps)).toBeNull()
|
||||
expect(isLocalExecutionHost('local')).toBe(true)
|
||||
expect(isLocalExecutionHost('runtime:ephemeral-vm-1')).toBe(false)
|
||||
})
|
||||
|
||||
it('treats missing workspace provenance as unknown', () => {
|
||||
expect(resolveAgentWorkspaceExecutionHostId('missing::/repo', deps)).toBeNull()
|
||||
expect(resolveAgentWorkspaceExecutionHostId(undefined, deps)).toBeNull()
|
||||
expect(isLocalExecutionHost(null)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
import {
|
||||
LOCAL_EXECUTION_HOST_ID,
|
||||
parseExecutionHostId,
|
||||
toSshExecutionHostId,
|
||||
type ExecutionHostId
|
||||
} from '../../shared/execution-host'
|
||||
import type { FolderWorkspace, ProjectGroup } from '../../shared/types'
|
||||
import { getRepoIdFromWorktreeId } from '../../shared/worktree-id'
|
||||
import { parseWorkspaceKey } from '../../shared/workspace-scope'
|
||||
|
||||
export type RestoredSubagentLivenessSweepDeps = {
|
||||
/** Targeted provider liveness, or null when the provider cannot prove either state. */
|
||||
probeLiveLocalPty: (ptyId: string) => Promise<boolean | null>
|
||||
isLocalExecutionHost: (worktreeId: string | undefined) => boolean
|
||||
/** PTY bound to this pane in the current session, if it has one. */
|
||||
getBoundPtyIdForPaneKey: (paneKey: string) => string | undefined
|
||||
/** PTY this pane was bound to when the session was last persisted; covers panes
|
||||
* whose surviving daemon session has not been reattached yet. */
|
||||
getPersistedPtyIdForPaneKey: (paneKey: string) => string | undefined
|
||||
reap: (
|
||||
isLocalExecutionHost: (worktreeId: string | undefined) => boolean,
|
||||
isLocalPaneAgentLive: (paneKey: string) => Promise<boolean>,
|
||||
isLocalPaneLivenessEvidenceCurrent: (paneKey: string) => boolean
|
||||
) => Promise<number>
|
||||
}
|
||||
|
||||
/** Drop restored rows only when the owning host is local and its provider proves
|
||||
* the exact PTY absent. */
|
||||
export async function sweepRestoredSubagentsWithoutLiveAgent(
|
||||
deps: RestoredSubagentLivenessSweepDeps
|
||||
): Promise<number> {
|
||||
const probesByPtyId = new Map<string, Promise<boolean | null>>()
|
||||
const boundPtyIdAtProbeByPaneKey = new Map<string, string | undefined>()
|
||||
return await deps.reap(
|
||||
(worktreeId) => deps.isLocalExecutionHost(worktreeId),
|
||||
async (paneKey) => {
|
||||
const boundPtyId = deps.getBoundPtyIdForPaneKey(paneKey)
|
||||
boundPtyIdAtProbeByPaneKey.set(paneKey, boundPtyId)
|
||||
const ptyId = boundPtyId ?? deps.getPersistedPtyIdForPaneKey(paneKey)
|
||||
if (!ptyId) {
|
||||
return true
|
||||
}
|
||||
try {
|
||||
let probe = probesByPtyId.get(ptyId)
|
||||
if (!probe) {
|
||||
probe = deps.probeLiveLocalPty(ptyId)
|
||||
probesByPtyId.set(ptyId, probe)
|
||||
}
|
||||
const live = await probe
|
||||
const currentBoundPtyId = deps.getBoundPtyIdForPaneKey(paneKey)
|
||||
// Why: cold restore can rebind the persisted id while its absence probe is in flight.
|
||||
return currentBoundPtyId !== boundPtyId || live !== false
|
||||
} catch {
|
||||
return true
|
||||
}
|
||||
},
|
||||
(paneKey) =>
|
||||
!boundPtyIdAtProbeByPaneKey.has(paneKey) ||
|
||||
deps.getBoundPtyIdForPaneKey(paneKey) === boundPtyIdAtProbeByPaneKey.get(paneKey)
|
||||
)
|
||||
}
|
||||
|
||||
/** Index the persisted terminal layouts as `paneKey -> ptyId`. Layout leaves are
|
||||
* the only persisted binding that carries a stable pane key, so tab-level PTY ids
|
||||
* (legacy numeric panes) are deliberately skipped. */
|
||||
export function indexPersistedPaneKeyPtyIds(
|
||||
layoutsByTabId: Record<string, { ptyIdsByLeafId?: Record<string, string> } | undefined>
|
||||
): Map<string, string> {
|
||||
const byPaneKey = new Map<string, string>()
|
||||
for (const [tabId, layout] of Object.entries(layoutsByTabId)) {
|
||||
for (const [leafId, ptyId] of Object.entries(layout?.ptyIdsByLeafId ?? {})) {
|
||||
if (ptyId) {
|
||||
byPaneKey.set(`${tabId}:${leafId}`, ptyId)
|
||||
}
|
||||
}
|
||||
}
|
||||
return byPaneKey
|
||||
}
|
||||
|
||||
type AgentWorkspaceExecutionHostDeps = {
|
||||
getRepo: (repoId: string) => ExecutionHostOwner | null | undefined
|
||||
getWorktreeMeta: (worktreeId: string) => { hostId?: string | null } | null | undefined
|
||||
getFolderWorkspace: (
|
||||
folderWorkspaceId: string
|
||||
) => Pick<FolderWorkspace, 'projectGroupId' | 'connectionId'> | null | undefined
|
||||
getProjectGroups: () => readonly Pick<ProjectGroup, 'id' | 'connectionId' | 'executionHostId'>[]
|
||||
}
|
||||
|
||||
type ExecutionHostOwner = {
|
||||
connectionId?: string | null
|
||||
executionHostId?: string | null
|
||||
}
|
||||
|
||||
function resolveDeclaredExecutionHost(owner: ExecutionHostOwner): ExecutionHostId | null {
|
||||
if (owner.executionHostId?.trim()) {
|
||||
return parseExecutionHostId(owner.executionHostId)?.id ?? null
|
||||
}
|
||||
const connectionId = owner.connectionId?.trim()
|
||||
return connectionId ? toSshExecutionHostId(connectionId) : LOCAL_EXECUTION_HOST_ID
|
||||
}
|
||||
|
||||
/** Resolve persisted workspace ownership; unknown provenance is not local authority. */
|
||||
export function resolveAgentWorkspaceExecutionHostId(
|
||||
workspaceId: string | undefined,
|
||||
deps: AgentWorkspaceExecutionHostDeps
|
||||
): ExecutionHostId | null {
|
||||
if (!workspaceId) {
|
||||
return null
|
||||
}
|
||||
const scope = parseWorkspaceKey(workspaceId)
|
||||
if (scope?.type === 'folder') {
|
||||
const workspace = deps.getFolderWorkspace(scope.folderWorkspaceId)
|
||||
const group = workspace
|
||||
? deps.getProjectGroups().find((candidate) => candidate.id === workspace.projectGroupId)
|
||||
: undefined
|
||||
if (!workspace || !group) {
|
||||
return null
|
||||
}
|
||||
return resolveDeclaredExecutionHost({
|
||||
connectionId: workspace.connectionId ?? group.connectionId,
|
||||
executionHostId: group.executionHostId
|
||||
})
|
||||
}
|
||||
const worktreeId = scope?.type === 'worktree' ? scope.worktreeId : workspaceId
|
||||
const declaredWorktreeHost = deps.getWorktreeMeta(worktreeId)?.hostId?.trim()
|
||||
if (declaredWorktreeHost) {
|
||||
return parseExecutionHostId(declaredWorktreeHost)?.id ?? null
|
||||
}
|
||||
const repo = deps.getRepo(getRepoIdFromWorktreeId(worktreeId))
|
||||
return repo ? resolveDeclaredExecutionHost(repo) : null
|
||||
}
|
||||
|
||||
export function isLocalExecutionHost(hostId: string | null | undefined): boolean {
|
||||
return parseExecutionHostId(hostId)?.kind === 'local'
|
||||
}
|
||||
|
|
@ -25,6 +25,7 @@ import {
|
|||
normalizeHookPayload,
|
||||
parseFormEncodedBody,
|
||||
readRequestBody,
|
||||
reapRestoredClaudeSubagentsForDeadPane,
|
||||
reconcileRemoteCodexState,
|
||||
resolveHookSource,
|
||||
preparePendingGrokResultDiscovery,
|
||||
|
|
@ -35,6 +36,11 @@ import {
|
|||
type AgentHookEventPayload,
|
||||
type HookListenerState
|
||||
} from '../../shared/agent-hook-listener'
|
||||
import {
|
||||
claudeRosterHasRestoredSnapshotSubagent,
|
||||
claudeRosterHasWorkingSubagent,
|
||||
claudeRosterToSnapshots
|
||||
} from '../../shared/claude-subagent-roster'
|
||||
import type { AgentHookSource } from '../../shared/agent-hook-relay'
|
||||
import {
|
||||
CLAUDE_STATUSLINE_PATHNAME,
|
||||
|
|
@ -2171,6 +2177,91 @@ export class AgentHookServer {
|
|||
}
|
||||
}
|
||||
|
||||
/** Second reap path for restored Claude subagent rows: drop the ones whose pane
|
||||
* has no live local agent process behind it any more. A PTY that dies while Orca
|
||||
* is down never runs the teardown that clears pane state, so hydrate rebuilds a
|
||||
* roster nothing can ever retire — the inventory reap needs the parent to emit a
|
||||
* complete `background_tasks` list and an idle parent never does. The row then
|
||||
* gates the pane 'working' for the rest of its life and hibernation, which
|
||||
* requires 'done', can never reclaim the agent's heap.
|
||||
*
|
||||
* Both the execution host and relay binding must prove local ownership before
|
||||
* targeted PTY liveness is consulted. Panes that reported in this runtime are
|
||||
* also skipped. Returns the number of panes changed. */
|
||||
async reapRestoredClaudeSubagentsWithoutLiveAgent(
|
||||
isLocalExecutionHost: (worktreeId: string | undefined) => boolean,
|
||||
isLocalPaneAgentLive: (paneKey: string) => Promise<boolean>,
|
||||
isLocalPaneLivenessEvidenceCurrent: (paneKey: string) => boolean
|
||||
): Promise<number> {
|
||||
const candidates: { paneKey: string; entry: EnrichedAgentHookEventPayload }[] = []
|
||||
for (const [paneKey, entry] of this.state.lastStatusByPaneKey) {
|
||||
const enriched = entry as EnrichedAgentHookEventPayload
|
||||
if (
|
||||
enriched.payload.agentType === 'claude' &&
|
||||
enriched.connectionId === null &&
|
||||
isLocalExecutionHost(enriched.worktreeId) &&
|
||||
claudeRosterHasRestoredSnapshotSubagent(
|
||||
this.state.claudeSubagentRosterByPaneKey.get(paneKey)
|
||||
) &&
|
||||
!this.runtimeObservedStatusPaneKeys.has(paneKey)
|
||||
) {
|
||||
candidates.push({ paneKey, entry: enriched })
|
||||
}
|
||||
}
|
||||
const liveness = await Promise.all(
|
||||
candidates.map(async (candidate) => {
|
||||
try {
|
||||
return await isLocalPaneAgentLive(candidate.paneKey)
|
||||
} catch {
|
||||
return true
|
||||
}
|
||||
})
|
||||
)
|
||||
let changedPanes = 0
|
||||
for (const [index, candidate] of candidates.entries()) {
|
||||
const { paneKey, entry: enriched } = candidate
|
||||
if (
|
||||
liveness[index] ||
|
||||
!isLocalPaneLivenessEvidenceCurrent(paneKey) ||
|
||||
this.state.lastStatusByPaneKey.get(paneKey) !== enriched ||
|
||||
this.runtimeObservedStatusPaneKeys.has(paneKey) ||
|
||||
!isLocalExecutionHost(enriched.worktreeId)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
if (!reapRestoredClaudeSubagentsForDeadPane(this.state, paneKey)) {
|
||||
continue
|
||||
}
|
||||
changedPanes += 1
|
||||
const roster = this.state.claudeSubagentRosterByPaneKey.get(paneKey)
|
||||
const subagents = claudeRosterToSnapshots(roster)
|
||||
// Why: the pane's persisted 'working' was the child gate holding a finished
|
||||
// lead open (subagent events never set lead state). With the last working row
|
||||
// gone and no process left to report, 'done' is the only truthful state — and
|
||||
// the one hibernation needs once this pane's agent is restored.
|
||||
const state =
|
||||
enriched.payload.state === 'working' && !claudeRosterHasWorkingSubagent(roster)
|
||||
? 'done'
|
||||
: enriched.payload.state
|
||||
const stateChanged = state !== enriched.payload.state
|
||||
const reconciledAt = stateChanged
|
||||
? Math.max(Date.now(), enriched.receivedAt + 1)
|
||||
: enriched.receivedAt
|
||||
const reconciled: EnrichedAgentHookEventPayload = {
|
||||
...enriched,
|
||||
receivedAt: reconciledAt,
|
||||
stateStartedAt: stateChanged ? reconciledAt : enriched.stateStartedAt,
|
||||
payload: { ...enriched.payload, state, subagents }
|
||||
}
|
||||
this.state.lastStatusByPaneKey.set(paneKey, reconciled)
|
||||
}
|
||||
if (changedPanes > 0) {
|
||||
this.scheduleStatusPersist()
|
||||
this.notifyStatusChangeListeners()
|
||||
}
|
||||
return changedPanes
|
||||
}
|
||||
|
||||
buildPtyEnv(): Record<string, string> {
|
||||
if (this.port <= 0 || !this.token) {
|
||||
return {}
|
||||
|
|
|
|||
|
|
@ -1180,6 +1180,30 @@ describe('DaemonPtyAdapter (IPtyProvider)', () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe('probePtyLiveness', () => {
|
||||
it('reads daemon truth before a fresh adapter has attached the session', async () => {
|
||||
const { id } = await adapter.spawn({ cols: 80, rows: 24 })
|
||||
const activeSessionIds = (adapter as unknown as { activeSessionIds: Set<string> })
|
||||
.activeSessionIds
|
||||
activeSessionIds.clear()
|
||||
|
||||
expect(adapter.hasPty(id)).toBe(false)
|
||||
await expect(adapter.probePtyLiveness(id)).resolves.toBe(true)
|
||||
await expect(adapter.probePtyLiveness('missing-session')).resolves.toBe(false)
|
||||
})
|
||||
|
||||
it('returns unknown when the daemon cannot answer', async () => {
|
||||
const client = (
|
||||
adapter as unknown as {
|
||||
client: { request: (type: string, payload?: unknown) => Promise<unknown> }
|
||||
}
|
||||
).client
|
||||
vi.spyOn(client, 'request').mockRejectedValueOnce(new Error('unavailable'))
|
||||
|
||||
await expect(adapter.probePtyLiveness('session')).resolves.toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('getBufferSnapshot', () => {
|
||||
it('returns the daemon model with its absolute stream sequence', async () => {
|
||||
const { id } = await adapter.spawn({ cols: 80, rows: 24 })
|
||||
|
|
|
|||
|
|
@ -813,6 +813,18 @@ export class DaemonPtyAdapter implements IPtyProvider {
|
|||
return this.activeSessionIds.has(id)
|
||||
}
|
||||
|
||||
async probePtyLiveness(id: string): Promise<boolean | null> {
|
||||
try {
|
||||
const result = await this.client.request<{ size: { cols: number; rows: number } | null }>(
|
||||
'getSize',
|
||||
{ sessionId: id }
|
||||
)
|
||||
return result.size !== null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
write(id: string, data: string): void {
|
||||
this.markSessionDirty(id)
|
||||
// Why recoverable and not just active: rejecting a write asks the pane to remount,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
import type { IPtyProvider } from '../providers/types'
|
||||
import type { DaemonPtyAdapter } from './daemon-pty-adapter'
|
||||
|
||||
export async function probePtyOwners(
|
||||
id: string,
|
||||
routed: IPtyProvider | undefined,
|
||||
possibleOwners: readonly DaemonPtyAdapter[]
|
||||
): Promise<boolean | null> {
|
||||
if (routed) {
|
||||
return routed.probePtyLiveness
|
||||
? await routed.probePtyLiveness(id)
|
||||
: (routed.hasPty?.(id) ?? null)
|
||||
}
|
||||
const results = await Promise.all(possibleOwners.map((provider) => provider.probePtyLiveness(id)))
|
||||
return results.some((result) => result === true)
|
||||
? true
|
||||
: results.every((result) => result === false)
|
||||
? false
|
||||
: null
|
||||
}
|
||||
|
|
@ -62,6 +62,7 @@ function createAdapter(
|
|||
}))
|
||||
),
|
||||
hasPty: vi.fn((id: string) => sessions.includes(id)),
|
||||
probePtyLiveness: vi.fn(async (id: string) => sessions.includes(id)),
|
||||
write: vi.fn((id: string, data: string) => {
|
||||
writes.push({ id, data })
|
||||
}),
|
||||
|
|
@ -470,6 +471,25 @@ describe('DaemonPtyRouter', () => {
|
|||
expect(current.hasPty).not.toHaveBeenCalledWith('legacy-session')
|
||||
})
|
||||
|
||||
it('probes every possible daemon owner for an unmapped session', async () => {
|
||||
const current = createAdapter('current')
|
||||
const legacy = createAdapter('legacy', ['surviving-session'])
|
||||
const router = new DaemonPtyRouter({ current, legacy: [legacy] })
|
||||
|
||||
await expect(router.probePtyLiveness('surviving-session')).resolves.toBe(true)
|
||||
expect(current.probePtyLiveness).toHaveBeenCalledExactlyOnceWith('surviving-session')
|
||||
expect(legacy.probePtyLiveness).toHaveBeenCalledExactlyOnceWith('surviving-session')
|
||||
})
|
||||
|
||||
it('does not report absence while any possible daemon owner is unavailable', async () => {
|
||||
const current = createAdapter('current')
|
||||
const legacy = createAdapter('legacy')
|
||||
vi.mocked(legacy.probePtyLiveness).mockResolvedValue(null)
|
||||
const router = new DaemonPtyRouter({ current, legacy: [legacy] })
|
||||
|
||||
await expect(router.probePtyLiveness('unknown-session')).resolves.toBeNull()
|
||||
})
|
||||
|
||||
it('fails listProcesses closed when any routed adapter cannot list sessions', async () => {
|
||||
const current = createAdapter('current', ['current-session'])
|
||||
const legacy = createAdapter('legacy', ['legacy-session'])
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import type {
|
|||
} from '../providers/types'
|
||||
import type { PtyIncarnationId } from '../../shared/pty-incarnation'
|
||||
import type { PtyProcessInspection } from '../providers/pty-process-inspection'
|
||||
import { probePtyOwners } from './daemon-pty-liveness-probe'
|
||||
|
||||
export class DaemonPtyRouter implements IPtyProvider {
|
||||
private current: DaemonPtyAdapter
|
||||
|
|
@ -108,6 +109,10 @@ export class DaemonPtyRouter implements IPtyProvider {
|
|||
return this.current.hasPty(id) || this.legacy.some((adapter) => adapter.hasPty(id))
|
||||
}
|
||||
|
||||
async probePtyLiveness(id: string): Promise<boolean | null> {
|
||||
return await probePtyOwners(id, this.sessionAdapters.get(id), this.allAdapters())
|
||||
}
|
||||
|
||||
write(id: string, data: string): void {
|
||||
this.adapterFor(id).write(id, data)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import type { IPtyProvider, PtySpawnOptions, PtySpawnResult } from '../providers
|
|||
import type { PtyProcessInspection } from '../providers/pty-process-inspection'
|
||||
|
||||
type ProviderMock = IPtyProvider & {
|
||||
probePtyLiveness: (id: string) => Promise<boolean | null>
|
||||
inspectProcess: (id: string) => Promise<PtyProcessInspection>
|
||||
emitData: (id: string, data: string, sequenceChars?: number) => void
|
||||
emitReplay: (id: string, data: string) => void
|
||||
|
|
@ -31,6 +32,7 @@ function createProvider(
|
|||
}),
|
||||
attach: vi.fn(async () => {}),
|
||||
hasPty: vi.fn((id: string) => sessions.includes(id)),
|
||||
probePtyLiveness: vi.fn(async (id: string) => sessions.includes(id)),
|
||||
providesAgentSessionOwnerListings: vi.fn(() => authoritativeOwnerListings),
|
||||
write: vi.fn(),
|
||||
resize: vi.fn(),
|
||||
|
|
@ -254,6 +256,20 @@ describe('DegradedDaemonPtyProvider', () => {
|
|||
expect(fallback.write).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('probes daemon owners without borrowing fallback liveness', async () => {
|
||||
const current = createDaemonAdapter('current')
|
||||
const legacy = createDaemonAdapter('legacy')
|
||||
const fallback = createProvider('fallback', ['unknown-session'])
|
||||
const provider = new DegradedDaemonPtyProvider({ current, legacy: [legacy], fallback })
|
||||
vi.mocked(legacy.probePtyLiveness).mockResolvedValue(null)
|
||||
|
||||
await expect(provider.probePtyLiveness('unknown-session')).resolves.toBeNull()
|
||||
expect(fallback.probePtyLiveness).not.toHaveBeenCalled()
|
||||
|
||||
vi.mocked(current.probePtyLiveness).mockResolvedValue(true)
|
||||
await expect(provider.probePtyLiveness('unknown-session')).resolves.toBe(true)
|
||||
})
|
||||
|
||||
it('routes authoritative recovery snapshots to the owning daemon', async () => {
|
||||
const current = createDaemonAdapter('daemon', ['daemon-session'])
|
||||
const fallback = createProvider('fallback')
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ import type {
|
|||
PtySpawnOptions,
|
||||
PtySpawnResult
|
||||
} from '../providers/types'
|
||||
import { findDaemonAdapter, listProviderSessionIds } from './degraded-daemon-session-routing'
|
||||
import { probePtyOwners } from './daemon-pty-liveness-probe'
|
||||
|
||||
export class DegradedDaemonPtyProvider implements IPtyProvider {
|
||||
readonly routesFreshSpawnsToLocalProvider = true
|
||||
|
|
@ -81,6 +83,10 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
|||
return mapped ? (mapped.hasPty?.(id) ?? true) : this.findProviderForExistingSession(id) !== null
|
||||
}
|
||||
|
||||
async probePtyLiveness(id: string): Promise<boolean | null> {
|
||||
return await probePtyOwners(id, this.sessionProviders.get(id), this.allDaemonAdapters())
|
||||
}
|
||||
|
||||
// Why: an unknown id cannot borrow listing authority from the fresh-spawn provider.
|
||||
providesAgentSessionOwnerListings = (ptyId: string): boolean =>
|
||||
(
|
||||
|
|
@ -246,11 +252,15 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
|||
}
|
||||
|
||||
ackColdRestore(sessionId: string): void {
|
||||
this.daemonAdapterFor(sessionId)?.ackColdRestore(sessionId)
|
||||
findDaemonAdapter(this.sessionProviders, this.allDaemonAdapters(), sessionId)?.ackColdRestore(
|
||||
sessionId
|
||||
)
|
||||
}
|
||||
|
||||
clearTombstone(sessionId: string): void {
|
||||
this.daemonAdapterFor(sessionId)?.clearTombstone(sessionId)
|
||||
findDaemonAdapter(this.sessionProviders, this.allDaemonAdapters(), sessionId)?.clearTombstone(
|
||||
sessionId
|
||||
)
|
||||
}
|
||||
|
||||
async reconcileOnStartup(validWorktreeIds: Set<string>): Promise<{
|
||||
|
|
@ -289,7 +299,7 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
|||
}
|
||||
|
||||
getCurrentDaemonSessionIds(): string[] {
|
||||
return this.sessionIdsForProvider(this.current)
|
||||
return listProviderSessionIds(this.sessionProviders, this.current)
|
||||
}
|
||||
|
||||
fanoutCurrentDaemonSyntheticExits(code: number): void {
|
||||
|
|
@ -338,19 +348,6 @@ export class DegradedDaemonPtyProvider implements IPtyProvider {
|
|||
return null
|
||||
}
|
||||
|
||||
private sessionIdsForProvider(provider: IPtyProvider): string[] {
|
||||
return [...this.sessionProviders]
|
||||
.filter(([, mappedProvider]) => mappedProvider === provider)
|
||||
.map(([id]) => id)
|
||||
}
|
||||
|
||||
private daemonAdapterFor(sessionId: string): DaemonPtyAdapter | null {
|
||||
const provider = this.sessionProviders.get(sessionId)
|
||||
return provider && this.allDaemonAdapters().includes(provider as DaemonPtyAdapter)
|
||||
? (provider as DaemonPtyAdapter)
|
||||
: null
|
||||
}
|
||||
|
||||
private allProviders(): IPtyProvider[] {
|
||||
return [this.fallback, ...this.allDaemonAdapters()]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
import type { IPtyProvider } from '../providers/types'
|
||||
import type { DaemonPtyAdapter } from './daemon-pty-adapter'
|
||||
|
||||
export function listProviderSessionIds(
|
||||
sessionProviders: ReadonlyMap<string, IPtyProvider>,
|
||||
provider: IPtyProvider
|
||||
): string[] {
|
||||
return [...sessionProviders]
|
||||
.filter(([, mappedProvider]) => mappedProvider === provider)
|
||||
.map(([id]) => id)
|
||||
}
|
||||
|
||||
export function findDaemonAdapter(
|
||||
sessionProviders: ReadonlyMap<string, IPtyProvider>,
|
||||
daemonAdapters: readonly DaemonPtyAdapter[],
|
||||
sessionId: string
|
||||
): DaemonPtyAdapter | null {
|
||||
const provider = sessionProviders.get(sessionId)
|
||||
return provider && daemonAdapters.includes(provider as DaemonPtyAdapter)
|
||||
? (provider as DaemonPtyAdapter)
|
||||
: null
|
||||
}
|
||||
|
|
@ -44,6 +44,12 @@ import { registerMobileHandlers } from './ipc/mobile'
|
|||
import { initTelemetry, shutdownTelemetry, trackAppOpenedOnce, track } from './telemetry/client'
|
||||
import { classifyError } from './telemetry/classify-error'
|
||||
import { runManagedHookInstallers } from './agent-hooks/install-telemetry'
|
||||
import {
|
||||
indexPersistedPaneKeyPtyIds,
|
||||
isLocalExecutionHost,
|
||||
resolveAgentWorkspaceExecutionHostId,
|
||||
sweepRestoredSubagentsWithoutLiveAgent
|
||||
} from './agent-hooks/restored-subagent-liveness-sweep'
|
||||
import {
|
||||
isAgentStatusHooksEnabled,
|
||||
MANAGED_AGENT_HOOK_INSTALLERS,
|
||||
|
|
@ -763,6 +769,46 @@ ipcMain.handle(
|
|||
}
|
||||
)
|
||||
|
||||
/** A PTY that dies while Orca is down never runs the teardown that clears pane
|
||||
* state, so hydrate can rebuild a Claude subagent roster that no later hook can
|
||||
* retire — pinning the pane 'working' and locking its agent out of hibernation
|
||||
* for good. Once provider and hook hydration settle, targeted PTY liveness can
|
||||
* retire only rows whose local owner is proven gone. */
|
||||
async function reapRestoredSubagentsWithoutLiveAgent(): Promise<void> {
|
||||
const currentStore = store
|
||||
if (!currentStore) {
|
||||
return
|
||||
}
|
||||
const provider = getDaemonProvider()
|
||||
if (!provider) {
|
||||
return
|
||||
}
|
||||
const persistedPtyIdByPaneKey = indexPersistedPaneKeyPtyIds(
|
||||
currentStore.getWorkspaceSession().terminalLayoutsByTabId ?? {}
|
||||
)
|
||||
await sweepRestoredSubagentsWithoutLiveAgent({
|
||||
probeLiveLocalPty: (ptyId) => provider.probePtyLiveness(ptyId),
|
||||
isLocalExecutionHost: (worktreeId) =>
|
||||
isLocalExecutionHost(
|
||||
resolveAgentWorkspaceExecutionHostId(worktreeId, {
|
||||
getRepo: (repoId) => currentStore.getRepo(repoId),
|
||||
getWorktreeMeta: (resolvedWorktreeId) => currentStore.getWorktreeMeta(resolvedWorktreeId),
|
||||
getFolderWorkspace: (folderWorkspaceId) =>
|
||||
currentStore.getFolderWorkspace(folderWorkspaceId),
|
||||
getProjectGroups: () => currentStore.getProjectGroups()
|
||||
})
|
||||
),
|
||||
getBoundPtyIdForPaneKey: getPtyIdForPaneKey,
|
||||
getPersistedPtyIdForPaneKey: (paneKey) => persistedPtyIdByPaneKey.get(paneKey),
|
||||
reap: (isLocalHost, isLocalPaneAgentLive, isLocalPaneLivenessEvidenceCurrent) =>
|
||||
agentHookServer.reapRestoredClaudeSubagentsWithoutLiveAgent(
|
||||
isLocalHost,
|
||||
isLocalPaneAgentLive,
|
||||
isLocalPaneLivenessEvidenceCurrent
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
function startTerminalRuntimeStartupServices(): Promise<void> {
|
||||
logStartupMilestone('first-window-startup-services-start')
|
||||
const startupServices = startFirstWindowStartupServices({
|
||||
|
|
@ -811,6 +857,9 @@ function startTerminalRuntimeStartupServices(): Promise<void> {
|
|||
})
|
||||
void localPtyStartupReady.then(() => {
|
||||
logStartupMilestone('local-pty-startup-ready')
|
||||
void reapRestoredSubagentsWithoutLiveAgent().catch((error) => {
|
||||
console.warn('[agent-hooks] restored-subagent liveness probe failed:', error)
|
||||
})
|
||||
})
|
||||
return firstWindowStartupServicesReady
|
||||
}
|
||||
|
|
|
|||
|
|
@ -130,6 +130,8 @@ export type IPtyProvider = {
|
|||
supportsAgentSessionCreateOperations?: (options?: PtyProbeOptions) => boolean | Promise<boolean>
|
||||
attach(id: string): Promise<void>
|
||||
hasPty?: (id: string) => boolean
|
||||
/** Exact provider readback: false only when the provider answered that the PTY is absent. */
|
||||
probePtyLiveness?: (id: string) => Promise<boolean | null>
|
||||
write(id: string, data: string): void
|
||||
resize(id: string, cols: number, rows: number): void
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -44,6 +44,19 @@ describe('startup ordering', () => {
|
|||
)
|
||||
})
|
||||
|
||||
it('requires daemon authority before restored-subagent liveness runs', () => {
|
||||
const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8')
|
||||
const sweepStart = source.indexOf('function reapRestoredSubagentsWithoutLiveAgent()')
|
||||
const sweepEnd = source.indexOf('function startTerminalRuntimeStartupServices()', sweepStart)
|
||||
const sweep = source.slice(sweepStart, sweepEnd)
|
||||
|
||||
expect(sweepStart).toBeGreaterThanOrEqual(0)
|
||||
expect(sweepEnd).toBeGreaterThan(sweepStart)
|
||||
expect(sweep).toContain('const provider = getDaemonProvider()')
|
||||
expect(sweep).toContain('if (!provider) {')
|
||||
expect(sweep).toContain('provider.probePtyLiveness(ptyId)')
|
||||
})
|
||||
|
||||
it('bounds WSL reconciliation before serve RPC while leaving desktop startup independent', () => {
|
||||
const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8')
|
||||
const barrierStart = source.indexOf("ipcMain.handle('app:awaitFirstWindowStartupServices'")
|
||||
|
|
|
|||
|
|
@ -554,6 +554,41 @@ describe('agent sleep planner', () => {
|
|||
).toEqual([`tab-1:${LEAF}`, `tab-1:${OTHER_LEAF}`])
|
||||
})
|
||||
|
||||
it('restarts the idle window once a phantom subagent stops gating the pane working', () => {
|
||||
// Why: a restored subagent row holds a finished lead at 'working', which is
|
||||
// the one state hibernation never accepts — reaping it is what unlocks it.
|
||||
const gated = entry({
|
||||
state: 'working',
|
||||
subagents: [{ id: 'areview-loop-c237a4c577493352', state: 'working', startedAt: 1 }]
|
||||
})
|
||||
expect(plannedPaneKeys(snapshot({ agentStatusByPaneKey: { [gated.paneKey]: gated } }))).toEqual(
|
||||
[]
|
||||
)
|
||||
|
||||
const reaped = entry({ state: 'done', updatedAt: NOW, stateStartedAt: NOW })
|
||||
expect(
|
||||
plannedPaneKeys(snapshot({ agentStatusByPaneKey: { [reaped.paneKey]: reaped } }))
|
||||
).toEqual([])
|
||||
|
||||
const idleReaped = entry({ state: 'done' })
|
||||
expect(
|
||||
plannedPaneKeys(snapshot({ agentStatusByPaneKey: { [idleReaped.paneKey]: idleReaped } }))
|
||||
).toEqual([`tab-1:${LEAF}`])
|
||||
|
||||
// Why: reaping only clears the child gate — a draft typed into the composer
|
||||
// while that segment was open still dies with the PTY, so it keeps blocking.
|
||||
expect(
|
||||
plannedPaneKeys(
|
||||
snapshot({
|
||||
agentStatusByPaneKey: { [idleReaped.paneKey]: idleReaped },
|
||||
lastTerminalInputAtByPaneKey: {
|
||||
[idleReaped.paneKey]: idleReaped.stateStartedAt + 1
|
||||
}
|
||||
})
|
||||
)
|
||||
).toEqual([])
|
||||
})
|
||||
|
||||
it('clamps corrupt or out-of-range idle durations to the default', () => {
|
||||
expect(getEffectiveAgentHibernationIdleMs(0)).toBe(DEFAULT_AGENT_HIBERNATION_IDLE_MS)
|
||||
expect(getEffectiveAgentHibernationIdleMs(Number.NaN)).toBe(DEFAULT_AGENT_HIBERNATION_IDLE_MS)
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import {
|
|||
foldClaudeBackgroundTasksIntoRoster,
|
||||
idleClaudeTeammateByName,
|
||||
readClaudeBackgroundAgentTasks,
|
||||
reapRestoredClaudeSubagentsWithoutLiveAgent,
|
||||
stopClaudeSubagent,
|
||||
upsertWorkingClaudeSubagent,
|
||||
type ClaudeSubagentRoster
|
||||
|
|
@ -2483,11 +2484,32 @@ export function seedClaudeSubagentRosterFromSnapshots(
|
|||
agentType: snapshot.agentType,
|
||||
description: snapshot.description,
|
||||
// Why: the seed can be a phantom (child finished while Orca was down, SubagentStop lost); let a PRESENT background_tasks list omitting the id remove it, not gate the pane 'working' forever.
|
||||
backgroundTasksAuthoritative: true
|
||||
backgroundTasksAuthoritative: true,
|
||||
// Why: an idle parent never emits that list, so the inventory reap alone can strand the seed; mark it for the liveness reap below.
|
||||
restoredFromSnapshot: true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/** Reap this pane's unconfirmed restored seeds because no live agent process backs
|
||||
* the pane any more (its PTY died while Orca was down, so no finish hook could
|
||||
* arrive). Callers must have proven the pane is LOCAL-launched — a remote/SSH
|
||||
* agent runs on the far host and can never appear in a local process index.
|
||||
* Returns whether the roster changed. */
|
||||
export function reapRestoredClaudeSubagentsForDeadPane(
|
||||
state: HookListenerState,
|
||||
paneKey: string
|
||||
): boolean {
|
||||
const roster = state.claudeSubagentRosterByPaneKey.get(paneKey)
|
||||
if (!roster || !reapRestoredClaudeSubagentsWithoutLiveAgent(roster)) {
|
||||
return false
|
||||
}
|
||||
if (roster.size === 0) {
|
||||
state.claudeSubagentRosterByPaneKey.delete(paneKey)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/** Drop a child-owned waiting state when the child stops/idles, restoring the displaced lead state; without a stash, fall back to 'working' (a transient spinner beats a permanently stuck card). */
|
||||
function clearClaudePendingWaitForAgent(
|
||||
state: HookListenerState,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
foldClaudeBackgroundTasksIntoRoster,
|
||||
idleClaudeTeammateByName,
|
||||
readClaudeBackgroundAgentTasks,
|
||||
reapRestoredClaudeSubagentsWithoutLiveAgent,
|
||||
stopClaudeSubagent,
|
||||
upsertWorkingClaudeSubagent,
|
||||
type ClaudeSubagentRoster
|
||||
|
|
@ -477,3 +478,51 @@ describe('claude-subagent-roster', () => {
|
|||
expect(claudeRosterToSnapshots(new Map())).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('restored-row liveness reap', () => {
|
||||
const restored = (id: string): ClaudeSubagentRoster =>
|
||||
new Map([
|
||||
[
|
||||
id,
|
||||
{
|
||||
state: 'working' as const,
|
||||
startedAt: 100,
|
||||
backgroundTasksAuthoritative: true,
|
||||
restoredFromSnapshot: true
|
||||
}
|
||||
]
|
||||
])
|
||||
|
||||
it('drops a restored row when no agent process is left behind it', () => {
|
||||
const roster = restored('areview-loop-c237a4c577493352')
|
||||
expect(reapRestoredClaudeSubagentsWithoutLiveAgent(roster)).toBe(true)
|
||||
expect(claudeRosterHasWorkingSubagent(roster)).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps a row a live lifecycle event re-tracked', () => {
|
||||
const roster = restored('areview-loop-c237a4c577493352')
|
||||
upsertWorkingClaudeSubagent(roster, 'areview-loop-c237a4c577493352', {}, 150)
|
||||
expect(reapRestoredClaudeSubagentsWithoutLiveAgent(roster)).toBe(false)
|
||||
expect(claudeRosterHasWorkingSubagent(roster)).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps a row a live inventory confirmed as running', () => {
|
||||
const roster = restored('a9')
|
||||
foldClaudeBackgroundTasksIntoRoster(roster, [task({ id: 'a9' })], 150)
|
||||
expect(reapRestoredClaudeSubagentsWithoutLiveAgent(roster)).toBe(false)
|
||||
expect(claudeRosterHasWorkingSubagent(roster)).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves rows this listener tracked from live events alone', () => {
|
||||
const roster: ClaudeSubagentRoster = new Map()
|
||||
upsertWorkingClaudeSubagent(roster, 'a1', {}, 100)
|
||||
expect(reapRestoredClaudeSubagentsWithoutLiveAgent(roster)).toBe(false)
|
||||
expect(roster.has('a1')).toBe(true)
|
||||
})
|
||||
|
||||
it('leaves the inventory reap of a restored row working', () => {
|
||||
const roster = restored('aprobe1-6d3cb5b5')
|
||||
foldClaudeBackgroundTasksIntoRoster(roster, [task({ id: 'other', teammate: true })], 200)
|
||||
expect(roster.has('aprobe1-6d3cb5b5')).toBe(false)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -36,6 +36,13 @@ export type TrackedClaudeSubagent = {
|
|||
* removes it even when teammate-shaped, so it can't gate the pane
|
||||
* 'working' forever. Cleared once live activity re-tracks the id. */
|
||||
backgroundTasksAuthoritative?: boolean
|
||||
/** The row was rebuilt from a persisted snapshot at restore and no live event
|
||||
* has confirmed it since, so the only thing backing it is a claim written by
|
||||
* an agent process that may no longer exist. Cleared by any live activity on
|
||||
* the id. Lets a liveness check reap it when that process is gone — the
|
||||
* inventory reap alone needs the parent to speak, and an idle parent never
|
||||
* does. */
|
||||
restoredFromSnapshot?: boolean
|
||||
/** A subagent-typed background task listed this lifecycle id id-exact
|
||||
* (workflow/named lanes) — proof the task list tracks this id, so a later
|
||||
* complete list omitting it means finished/killed even though the id is
|
||||
|
|
@ -84,6 +91,9 @@ export function upsertWorkingClaudeSubagent(
|
|||
// background_tasks omission must stop reaping it (teammate-shaped ids
|
||||
// never appear there). The fold re-tags its own recreations after this.
|
||||
existing.backgroundTasksAuthoritative = undefined
|
||||
// Why: the live event proves the agent process behind the restored row is
|
||||
// still running it, so the liveness reap must stop treating it as a claim.
|
||||
existing.restoredFromSnapshot = undefined
|
||||
return
|
||||
}
|
||||
// Why: beyond the wire cap extra rows would be invisible anyway; idle
|
||||
|
|
@ -227,6 +237,9 @@ export function foldClaudeBackgroundTasksIntoRoster(
|
|||
existing.agentType = task.agentType ?? existing.agentType
|
||||
existing.description = task.description ?? existing.description
|
||||
existing.listedAsSubagentTask = true
|
||||
// Why: a live inventory listed the id as running — the restored claim is
|
||||
// now confirmed by the current process, so liveness can't reap it.
|
||||
existing.restoredFromSnapshot = undefined
|
||||
continue
|
||||
}
|
||||
if (!task.running) {
|
||||
|
|
@ -287,6 +300,37 @@ export function foldClaudeBackgroundTasksIntoRoster(
|
|||
}
|
||||
}
|
||||
|
||||
/** Second reap path for restored rows, used when the agent process that wrote
|
||||
* the snapshot is gone. The inventory reap needs the parent to emit a complete
|
||||
* `background_tasks` list; a parent that went idle before Orca restarted never
|
||||
* emits one, so an unconfirmed row would gate the pane 'working' forever and
|
||||
* keep it out of hibernation. Rows confirmed by live activity are untouched.
|
||||
* Returns whether anything was dropped. */
|
||||
export function reapRestoredClaudeSubagentsWithoutLiveAgent(roster: ClaudeSubagentRoster): boolean {
|
||||
let changed = false
|
||||
for (const [id, tracked] of roster) {
|
||||
if (tracked.restoredFromSnapshot === true) {
|
||||
roster.delete(id)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
export function claudeRosterHasRestoredSnapshotSubagent(
|
||||
roster: ClaudeSubagentRoster | undefined
|
||||
): boolean {
|
||||
if (!roster) {
|
||||
return false
|
||||
}
|
||||
for (const tracked of roster.values()) {
|
||||
if (tracked.restoredFromSnapshot === true) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** Whether a lifecycle agent id belongs to the named teammate. Teammate ids
|
||||
* embed the name as `a<name>-<hex>`; requiring a hyphen-free suffix keeps
|
||||
* teammate "rev" from matching "rev-two"'s ids (`arev-two-<hex>`), while a
|
||||
|
|
|
|||
Loading…
Reference in New Issue