[P2] fix(daemon): keep audit-only evidence off the main thread and out of false negatives (#11695)
* fix(daemon): keep audit-only evidence off the main thread and out of false negatives
Three audit-only daemon evidence defects:
- The Windows CIM probe emitted the "absent" marker whenever $p was empty, so a
non-terminating Get-CimInstance failure (Winmgmt down, corrupt WMI repository,
access denied) exited 0 with {"exists":false} and was recorded as proof the
daemon process was gone. The script now sets $ErrorActionPreference and reports
query failure explicitly; only a query that ran and found nothing yields
'missing', everything else stays 'unavailable'.
- probeMacosProcess read the process start time through execFileSync('ps') on the
Electron main thread. It now uses the async readMacosProcessStartedAtMs, matching
the async command-line sibling in the same probe.
- daemon_audit_eligibility fired on every successful inventory, so routine
listProcesses traffic could drain the shared 1,000-event per-session telemetry
ceiling. Repeated identical observations now collapse to one heartbeat per
5 minutes; a changed observation still emits immediately.
The evidence stays strictly audit-only — no lifecycle, routing or recovery path
consumes the observation.
Co-authored-by: Orca <help@stably.ai>
* fix(daemon): take the linux audit start-time read off the main thread
The linux branch of the audit-only evidence probe still bottomed out in
getProcessStartedAtMs, i.e. two readFileSync calls plus an
execFileSync('getconf', ['CLK_TCK']) spawn on the Electron main thread —
the same hazard the darwin half of this change removed. The `async`
wrapper hid it from the type signature and from the main-thread
regression test, which only covered darwin.
That path is reached whenever the pid record carries no start ticks or
boot id: legacy bare-integer pid files (the upgrade population this audit
exists to observe) and any host where /proc boot-id or start-tick reads
fail.
readLinuxProcessStartedAtMs now reads procfs asynchronously and caches
CLK_TCK for the process lifetime (it is fixed for the kernel's lifetime),
retrying only after a failed probe. The main-thread test now covers linux
with a ticks-less pid record.
Co-authored-by: Orca <help@stably.ai>
* fix(daemon): measure the audit heartbeat window on a monotonic clock
A backward wall-clock jump (NTP correction, VM resume) made elapsed time negative, which read as "still inside the window" and suppressed daemon_audit_eligibility heartbeats until wall time caught up. Rate-limit arithmetic now runs on performance.now() and treats any backward movement as eligible.
Co-authored-by: Orca <help@stably.ai>
* fix(daemon): guard the audit rate limiter and scope the CLK_TCK cache
The rate-limited tracker is the only production emitter, but its payload
serialization and clock read sat outside the guard that keeps audit
telemetry from affecting daemon availability — a throw there propagates
out of listProcesses' try and turns a successful inventory into a
reported inventory failure.
Also key the CLK_TCK cache by the runner that produced it: the value
belongs to the host executing getconf, not the module.
Co-authored-by: Orca <help@stably.ai>
* test(daemon): make audit CI synchronization deterministic
---------
Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
parent
60d2493bbb
commit
bc6a6e9ff0
|
|
@ -1,11 +1,20 @@
|
|||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { validate } from '../telemetry/validator'
|
||||
import { recordAuthenticatedInventory, type DaemonAuditContext } from './daemon-audit-classifier'
|
||||
import { DaemonPtyAdapter } from './daemon-pty-adapter'
|
||||
import { DaemonServer } from './daemon-server'
|
||||
import { getDaemonSocketPath } from './daemon-spawner'
|
||||
|
||||
const { trackMock } = vi.hoisted(() => ({ trackMock: vi.fn() }))
|
||||
vi.mock('../telemetry/client', () => ({ track: trackMock }))
|
||||
|
||||
import { trackDaemonAuditEligibility } from './daemon-audit-eligibility-event'
|
||||
import {
|
||||
createDaemonAuditEligibilityTracker,
|
||||
trackDaemonAuditEligibility
|
||||
} from './daemon-audit-eligibility-event'
|
||||
|
||||
const context: DaemonAuditContext = {
|
||||
protocolGeneration: 23,
|
||||
|
|
@ -47,5 +56,137 @@ describe('daemon audit eligibility telemetry', () => {
|
|||
expect(() =>
|
||||
trackDaemonAuditEligibility(recordAuthenticatedInventory(context, null))
|
||||
).not.toThrow()
|
||||
|
||||
// The rate-limited tracker is the production call site, so it carries the same guarantee.
|
||||
const trackEligibility = createDaemonAuditEligibilityTracker()
|
||||
expect(() => trackEligibility(recordAuthenticatedInventory(context, null))).not.toThrow()
|
||||
})
|
||||
|
||||
it('cannot affect callers when the rate-limit bookkeeping throws', () => {
|
||||
const trackEligibility = createDaemonAuditEligibilityTracker(() => {
|
||||
throw new Error('no clock')
|
||||
})
|
||||
|
||||
expect(() => trackEligibility(recordAuthenticatedInventory(context, null))).not.toThrow()
|
||||
|
||||
// A malformed observation must not escape the guard either.
|
||||
const malformed = recordAuthenticatedInventory(context, null)
|
||||
expect(() =>
|
||||
trackEligibility({ ...malformed, evidenceSources: undefined as never })
|
||||
).not.toThrow()
|
||||
expect(trackMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('collapses repeated identical observations into one heartbeat per window', () => {
|
||||
let nowMs = 1_700_000_000_000
|
||||
const trackEligibility = createDaemonAuditEligibilityTracker(() => nowMs)
|
||||
|
||||
for (let index = 0; index < 60; index += 1) {
|
||||
nowMs += 1_000
|
||||
trackEligibility(recordAuthenticatedInventory(context, null))
|
||||
}
|
||||
|
||||
expect(trackMock).toHaveBeenCalledOnce()
|
||||
|
||||
nowMs += 5 * 60_000
|
||||
trackEligibility(recordAuthenticatedInventory(context, null))
|
||||
expect(trackMock).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('keeps heartbeating after the clock jumps backward', () => {
|
||||
let nowMs = 1_700_000_000_000
|
||||
const trackEligibility = createDaemonAuditEligibilityTracker(() => nowMs)
|
||||
|
||||
trackEligibility(recordAuthenticatedInventory(context, null))
|
||||
expect(trackMock).toHaveBeenCalledOnce()
|
||||
|
||||
// An NTP correction / VM resume rewinds the clock by an hour.
|
||||
nowMs -= 60 * 60_000
|
||||
trackEligibility(recordAuthenticatedInventory(context, null))
|
||||
expect(trackMock).toHaveBeenCalledTimes(2)
|
||||
|
||||
// The window re-anchors on the rewound clock instead of emitting on every call.
|
||||
for (let index = 0; index < 60; index += 1) {
|
||||
nowMs += 1_000
|
||||
trackEligibility(recordAuthenticatedInventory(context, null))
|
||||
}
|
||||
expect(trackMock).toHaveBeenCalledTimes(2)
|
||||
|
||||
nowMs += 5 * 60_000
|
||||
trackEligibility(recordAuthenticatedInventory(context, null))
|
||||
expect(trackMock).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('measures the window on a monotonic clock rather than wall time', () => {
|
||||
const wallClock = vi.spyOn(Date, 'now').mockReturnValue(1_700_000_000_000)
|
||||
const trackEligibility = createDaemonAuditEligibilityTracker()
|
||||
|
||||
try {
|
||||
trackEligibility(recordAuthenticatedInventory(context, null))
|
||||
expect(trackMock).toHaveBeenCalledOnce()
|
||||
|
||||
// Wall time alone must not open the window: no real time has elapsed.
|
||||
wallClock.mockReturnValue(1_700_000_000_000 + 6 * 60_000)
|
||||
trackEligibility(recordAuthenticatedInventory(context, null))
|
||||
expect(trackMock).toHaveBeenCalledOnce()
|
||||
} finally {
|
||||
wallClock.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('emits immediately when the observation changes', () => {
|
||||
let nowMs = 1_700_000_000_000
|
||||
const trackEligibility = createDaemonAuditEligibilityTracker(() => nowMs)
|
||||
|
||||
trackEligibility(recordAuthenticatedInventory(context, null))
|
||||
nowMs += 1_000
|
||||
trackEligibility(
|
||||
recordAuthenticatedInventory(context, {
|
||||
identity: { pid: 42, startedAtMs: nowMs, launchNonce: 'launch-a' }
|
||||
})
|
||||
)
|
||||
|
||||
expect(trackMock).toHaveBeenCalledTimes(2)
|
||||
expect(trackMock.mock.calls[1][1]).toMatchObject({ exact_incarnation: 'endpoint-identity' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('daemon audit eligibility inventory volume', () => {
|
||||
let dir: string
|
||||
let socketPath: string
|
||||
let tokenPath: string
|
||||
let server: DaemonServer
|
||||
let adapter: DaemonPtyAdapter
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'daemon-audit-eligibility-'))
|
||||
socketPath = getDaemonSocketPath(dir)
|
||||
tokenPath = join(dir, 'daemon.token')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
adapter?.dispose()
|
||||
await server?.shutdown()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('does not emit one eligibility event per successful inventory', async () => {
|
||||
server = new DaemonServer({
|
||||
socketPath,
|
||||
tokenPath,
|
||||
spawnSubprocess: () => {
|
||||
throw new Error('Test must not create a PTY')
|
||||
}
|
||||
})
|
||||
await server.start()
|
||||
adapter = new DaemonPtyAdapter({ socketPath, tokenPath })
|
||||
|
||||
for (let index = 0; index < 40; index += 1) {
|
||||
await adapter.listProcesses()
|
||||
}
|
||||
|
||||
expect(
|
||||
trackMock.mock.calls.filter(([name]) => name === 'daemon_audit_eligibility')
|
||||
).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,29 +1,71 @@
|
|||
import { track } from '../telemetry/client'
|
||||
import type { EventProps } from '../../shared/telemetry-events'
|
||||
import type { DaemonAuditObservation } from './daemon-audit-classifier'
|
||||
|
||||
// Why: a steady daemon repeats a byte-identical observation on every listProcesses call, so
|
||||
// repeats are re-sent only as an occasional heartbeat — the shared per-session telemetry
|
||||
// ceiling is 1,000 events for the whole app and audit data must not crowd it out.
|
||||
const REPEATED_OBSERVATION_INTERVAL_MS = 5 * 60_000
|
||||
|
||||
export function trackDaemonAuditEligibility(observation: DaemonAuditObservation): void {
|
||||
try {
|
||||
track('daemon_audit_eligibility', {
|
||||
state: observation.state,
|
||||
reason: observation.reason,
|
||||
trigger: observation.trigger,
|
||||
evidence_sources: [...observation.evidenceSources],
|
||||
protocol_generation: observation.context.protocolGeneration,
|
||||
provider: observation.context.provider,
|
||||
endpoint_kind: observation.context.endpointKind,
|
||||
profile_scope: observation.context.profileScope ? 'configured' : 'unspecified',
|
||||
exact_incarnation: exactIncarnationKind(observation),
|
||||
reachability: observation.reachability,
|
||||
inventory_authority: observation.inventoryAuthority,
|
||||
process_liveness: observation.processLiveness,
|
||||
process_reason: observation.processReason,
|
||||
endpoint_state: observation.endpointState
|
||||
})
|
||||
track('daemon_audit_eligibility', auditEligibilityProperties(observation))
|
||||
} catch {
|
||||
// Audit telemetry cannot affect daemon availability.
|
||||
}
|
||||
}
|
||||
|
||||
export function createDaemonAuditEligibilityTracker(
|
||||
// Why: the window must be measured on a monotonic clock — a backward wall-clock jump (NTP
|
||||
// correction, VM resume) would otherwise park the next heartbeat in the future.
|
||||
monotonicNowMs: () => number = () => performance.now()
|
||||
): (observation: DaemonAuditObservation) => void {
|
||||
let lastProperties: string | null = null
|
||||
let lastTrackedAtMs = 0
|
||||
return (observation) => {
|
||||
// Why: the rate-limit bookkeeping runs inside the daemon's inventory path, so it is guarded
|
||||
// together with the emit — audit telemetry cannot affect daemon availability.
|
||||
try {
|
||||
const properties = JSON.stringify(auditEligibilityProperties(observation))
|
||||
const observedAtMs = monotonicNowMs()
|
||||
const elapsedMs = observedAtMs - lastTrackedAtMs
|
||||
if (
|
||||
properties === lastProperties &&
|
||||
elapsedMs >= 0 &&
|
||||
elapsedMs < REPEATED_OBSERVATION_INTERVAL_MS
|
||||
) {
|
||||
return
|
||||
}
|
||||
lastProperties = properties
|
||||
lastTrackedAtMs = observedAtMs
|
||||
trackDaemonAuditEligibility(observation)
|
||||
} catch {
|
||||
// Audit telemetry cannot affect daemon availability.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function auditEligibilityProperties(
|
||||
observation: DaemonAuditObservation
|
||||
): EventProps<'daemon_audit_eligibility'> {
|
||||
return {
|
||||
state: observation.state,
|
||||
reason: observation.reason,
|
||||
trigger: observation.trigger,
|
||||
evidence_sources: [...observation.evidenceSources],
|
||||
protocol_generation: observation.context.protocolGeneration,
|
||||
provider: observation.context.provider,
|
||||
endpoint_kind: observation.context.endpointKind,
|
||||
profile_scope: observation.context.profileScope ? 'configured' : 'unspecified',
|
||||
exact_incarnation: exactIncarnationKind(observation),
|
||||
reachability: observation.reachability,
|
||||
inventory_authority: observation.inventoryAuthority,
|
||||
process_liveness: observation.processLiveness,
|
||||
process_reason: observation.processReason,
|
||||
endpoint_state: observation.endpointState
|
||||
}
|
||||
}
|
||||
|
||||
function exactIncarnationKind(
|
||||
observation: DaemonAuditObservation
|
||||
): 'endpoint-identity' | 'endpoint-identity-linux-ticks' | 'unavailable' {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,94 @@
|
|||
import type * as ChildProcessModule from 'node:child_process'
|
||||
import type * as FsPromisesModule from 'node:fs/promises'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { ExactDaemonIncarnation } from './daemon-incarnation-evidence-types'
|
||||
|
||||
const LINUX_BOOT_TIME_SECONDS = 1_699_000_000
|
||||
const LINUX_START_TICKS = 1_234
|
||||
const LINUX_CLOCK_TICKS_PER_SECOND = 100
|
||||
|
||||
const { execFileMock, execFileSyncMock, readFileMock } = vi.hoisted(() => ({
|
||||
execFileMock: vi.fn(
|
||||
(
|
||||
file: string,
|
||||
_args: readonly string[],
|
||||
_options: unknown,
|
||||
callback: (error: Error | null, result: { stdout: string; stderr: string }) => void
|
||||
) => {
|
||||
callback(null, {
|
||||
stdout: file === 'getconf' ? '100\n' : `${new Date(1_700_000_000_000).toString()}\n`,
|
||||
stderr: ''
|
||||
})
|
||||
}
|
||||
),
|
||||
execFileSyncMock: vi.fn(() => ''),
|
||||
readFileMock: vi.fn(async (path: string) =>
|
||||
path === '/proc/stat' ? 'btime 1699000000\n' : `42 (orca-daemon) S${' 0'.repeat(18)} 1234 0 0\n`
|
||||
)
|
||||
}))
|
||||
|
||||
vi.mock('node:child_process', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof ChildProcessModule>()),
|
||||
execFile: execFileMock,
|
||||
execFileSync: execFileSyncMock
|
||||
}))
|
||||
|
||||
vi.mock('node:fs/promises', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof FsPromisesModule>()),
|
||||
readFile: readFileMock
|
||||
}))
|
||||
|
||||
const { probeDaemonProcessIdentity } = await import('./daemon-incarnation-evidence')
|
||||
|
||||
const endpoint = { socketPath: '/runtime/daemon.sock', tokenPath: '/runtime/daemon.token' }
|
||||
const exactIncarnation: ExactDaemonIncarnation = {
|
||||
identity: { pid: 42, startedAtMs: 1_700_000_000_000, launchNonce: 'launch-a' }
|
||||
}
|
||||
const daemonCommandLine = `node daemon-entry --socket ${endpoint.socketPath} --token ${endpoint.tokenPath}`
|
||||
|
||||
describe('daemon audit evidence main-thread cost', () => {
|
||||
it('reads the macOS process start time without a synchronous main-thread spawn', async () => {
|
||||
await expect(
|
||||
probeDaemonProcessIdentity(exactIncarnation, endpoint, {
|
||||
platform: 'darwin',
|
||||
signalProcess: () => 'occupied',
|
||||
readCommandLine: async () => daemonCommandLine
|
||||
})
|
||||
).resolves.toMatchObject({ state: 'present', reason: 'macos_identity_match' })
|
||||
|
||||
expect(execFileSyncMock).not.toHaveBeenCalled()
|
||||
expect(execFileMock).toHaveBeenCalledWith(
|
||||
'ps',
|
||||
['-p', '42', '-o', 'lstart='],
|
||||
expect.anything(),
|
||||
expect.any(Function)
|
||||
)
|
||||
})
|
||||
|
||||
// Legacy bare-integer pid files carry no start ticks, so linux falls back to the start time.
|
||||
it('reads the linux process start time without a synchronous main-thread spawn', async () => {
|
||||
const startedAtMs =
|
||||
LINUX_BOOT_TIME_SECONDS * 1000 + (LINUX_START_TICKS / LINUX_CLOCK_TICKS_PER_SECOND) * 1000
|
||||
|
||||
await expect(
|
||||
probeDaemonProcessIdentity(
|
||||
{ identity: { pid: 42, startedAtMs, launchNonce: 'launch-a' } },
|
||||
endpoint,
|
||||
{
|
||||
platform: 'linux',
|
||||
signalProcess: () => 'occupied',
|
||||
readLinuxStat: async () => ({ status: 'present', value: '42 (orca-daemon) S 1' }),
|
||||
readCommandLine: async () => daemonCommandLine
|
||||
}
|
||||
)
|
||||
).resolves.toMatchObject({ state: 'present', reason: 'linux_identity_match' })
|
||||
|
||||
expect(execFileSyncMock).not.toHaveBeenCalled()
|
||||
expect(execFileMock).toHaveBeenCalledWith(
|
||||
'getconf',
|
||||
['CLK_TCK'],
|
||||
expect.anything(),
|
||||
expect.any(Function)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
|
@ -70,6 +70,6 @@ export type DaemonProcessProbeDependencies = {
|
|||
readLinuxStat?: (pid: number) => Promise<LinuxStatEvidence>
|
||||
readBootIdentity?: () => Promise<string | undefined>
|
||||
readCommandLine?: (pid: number, platform: NodeJS.Platform) => Promise<string | undefined>
|
||||
readProcessStartedAtMs?: (pid: number) => number | null
|
||||
readProcessStartedAtMs?: (pid: number) => Promise<number | null>
|
||||
queryWindowsProcess?: (pid: number) => Promise<WindowsProcessEvidence>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ describe('daemon process identity evidence', () => {
|
|||
})
|
||||
|
||||
it('proves Linux pid reuse from native start ticks without derived milliseconds', async () => {
|
||||
const readProcessStartedAtMs = vi.fn(() => exactIncarnation.identity.startedAtMs)
|
||||
const readProcessStartedAtMs = vi.fn(async () => exactIncarnation.identity.startedAtMs)
|
||||
|
||||
await expect(
|
||||
probeDaemonProcessIdentity(
|
||||
|
|
@ -247,7 +247,7 @@ describe('daemon process identity evidence', () => {
|
|||
signalProcess: () => 'occupied',
|
||||
readCommandLine: async () =>
|
||||
`node daemon-entry --socket ${endpoint.socketPath} --token ${endpoint.tokenPath}`,
|
||||
readProcessStartedAtMs: () => exactIncarnation.identity.startedAtMs + 2_500
|
||||
readProcessStartedAtMs: async () => exactIncarnation.identity.startedAtMs + 2_500
|
||||
})
|
||||
).resolves.toMatchObject({ state: 'unknown', reason: 'macos_start_time_mismatch' })
|
||||
})
|
||||
|
|
@ -260,6 +260,21 @@ describe('daemon process identity evidence', () => {
|
|||
})
|
||||
).resolves.toMatchObject({ state: 'gone', reason: 'pid_missing' })
|
||||
})
|
||||
|
||||
it('keeps unsupported platforms indeterminate without running a Darwin probe', async () => {
|
||||
const signalProcess = vi.fn(() => 'occupied' as const)
|
||||
const readCommandLine = vi.fn(async () => 'node daemon-entry')
|
||||
|
||||
await expect(
|
||||
probeDaemonProcessIdentity(exactIncarnation, endpoint, {
|
||||
platform: 'freebsd',
|
||||
signalProcess,
|
||||
readCommandLine
|
||||
})
|
||||
).resolves.toMatchObject({ state: 'unknown', reason: 'inspection_failed' })
|
||||
expect(signalProcess).not.toHaveBeenCalled()
|
||||
expect(readCommandLine).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('daemon audit availability evidence', () => {
|
||||
|
|
|
|||
|
|
@ -1,9 +1,5 @@
|
|||
import { parseLinuxStartTicks, readBootIdentity } from '../agent-hooks/managed-hook-owner-identity'
|
||||
import {
|
||||
commandLineMatchesDaemon,
|
||||
getProcessStartedAtMs,
|
||||
startTimesWithinTolerance
|
||||
} from './daemon-health'
|
||||
import { commandLineMatchesDaemon, startTimesWithinTolerance } from './daemon-health'
|
||||
import {
|
||||
WINDOWS_CREATION_TIME_TOLERANCE_MS,
|
||||
type DaemonEvidenceSources,
|
||||
|
|
@ -15,7 +11,9 @@ import {
|
|||
import {
|
||||
inspectProcessSignal,
|
||||
queryWindowsProcess,
|
||||
readLinuxProcessStartedAtMs,
|
||||
readLinuxStat,
|
||||
readMacosProcessStartedAtMs,
|
||||
readProcessCommandLine
|
||||
} from './daemon-process-inspection'
|
||||
|
||||
|
|
@ -43,6 +41,9 @@ export async function probeDaemonProcessIdentity(
|
|||
return unknown('exact_identity_unavailable', ['pid_record'])
|
||||
}
|
||||
const platform = dependencies.platform ?? process.platform
|
||||
if (platform !== 'linux' && platform !== 'darwin' && platform !== 'win32') {
|
||||
return unknown('inspection_failed', ['process_signal'])
|
||||
}
|
||||
const signalProcess = dependencies.signalProcess ?? inspectProcessSignal
|
||||
const signal = signalProcess(exactIncarnation.identity.pid)
|
||||
if (platform !== 'win32' && signal === 'missing') {
|
||||
|
|
@ -134,7 +135,7 @@ async function probeLinuxProcess(
|
|||
])
|
||||
}
|
||||
|
||||
const startedAtMs = (dependencies.readProcessStartedAtMs ?? getProcessStartedAtMs)(
|
||||
const startedAtMs = await (dependencies.readProcessStartedAtMs ?? readLinuxProcessStartedAtMs)(
|
||||
exactIncarnation.identity.pid
|
||||
)
|
||||
if (startedAtMs === null) {
|
||||
|
|
@ -172,7 +173,7 @@ async function probeMacosProcess(
|
|||
if (!commandLineMatchesDaemon(commandLine, endpoint.socketPath, endpoint.tokenPath)) {
|
||||
return unknown('command_line_mismatch', ['process_command_line'])
|
||||
}
|
||||
const startedAtMs = (dependencies.readProcessStartedAtMs ?? getProcessStartedAtMs)(
|
||||
const startedAtMs = await (dependencies.readProcessStartedAtMs ?? readMacosProcessStartedAtMs)(
|
||||
exactIncarnation.identity.pid
|
||||
)
|
||||
if (startedAtMs === null) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,14 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { queryWindowsProcess, readProcessCommandLine } from './daemon-process-inspection'
|
||||
import {
|
||||
queryWindowsProcess,
|
||||
readLinuxProcessStartedAtMs,
|
||||
readMacosProcessStartedAtMs,
|
||||
readProcessCommandLine
|
||||
} from './daemon-process-inspection'
|
||||
|
||||
// btime 1699000000 with 1000 start ticks: 10s after boot at 100Hz, 1s at 1000Hz.
|
||||
const readProcStat = async (path: string): Promise<string> =>
|
||||
path === '/proc/stat' ? 'btime 1699000000\n' : `42 (orca-daemon) S${' 0'.repeat(18)} 1000 0 0\n`
|
||||
|
||||
describe('daemon process inspection', () => {
|
||||
it('falls back to ps when Linux procfs returns an empty command line', async () => {
|
||||
|
|
@ -23,6 +32,88 @@ describe('daemon process inspection', () => {
|
|||
expect(runCommand).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('asks PowerShell to report a failed CIM query instead of an absent process', async () => {
|
||||
const runCommand = vi.fn(
|
||||
async (_file: string, _args: string[], _timeoutMs: number) =>
|
||||
'{"status":"present","cmd":"daemon","start":1}'
|
||||
)
|
||||
|
||||
await queryWindowsProcess(42, { runCommand })
|
||||
|
||||
const script = runCommand.mock.calls[0]?.[1].at(-1) ?? ''
|
||||
expect(script).toContain("$ErrorActionPreference = 'Stop'")
|
||||
expect(script).toMatch(/catch \{[^}]*query_failed/)
|
||||
})
|
||||
|
||||
it('keeps a failed CIM query indeterminate instead of proving the process gone', async () => {
|
||||
const runCommand = vi.fn(async () => '{"status":"query_failed"}')
|
||||
|
||||
await expect(queryWindowsProcess(42, { runCommand })).resolves.toEqual({
|
||||
status: 'unavailable'
|
||||
})
|
||||
})
|
||||
|
||||
it('never reads a probe result without a success marker as proof of absence', async () => {
|
||||
const runCommand = vi.fn(async () => '{"exists":false}')
|
||||
|
||||
await expect(queryWindowsProcess(42, { runCommand })).resolves.toEqual({
|
||||
status: 'unavailable'
|
||||
})
|
||||
})
|
||||
|
||||
it('reports absence only from a CIM query that ran and found nothing', async () => {
|
||||
const runCommand = vi.fn(async () => '{"status":"missing"}')
|
||||
|
||||
await expect(queryWindowsProcess(42, { runCommand })).resolves.toEqual({ status: 'missing' })
|
||||
})
|
||||
|
||||
it('reads the macOS start time through an async spawn', async () => {
|
||||
const runCommand = vi.fn(async () => 'Sat Jan 1 00:00:00 2028\n')
|
||||
|
||||
await expect(readMacosProcessStartedAtMs(42, { runCommand })).resolves.toBe(
|
||||
Date.parse('Sat Jan 1 00:00:00 2028')
|
||||
)
|
||||
expect(runCommand).toHaveBeenCalledWith('ps', ['-p', '42', '-o', 'lstart='], 2_000)
|
||||
})
|
||||
|
||||
// CLK_TCK belongs to the host that runs getconf, so a second runner must not inherit the first's.
|
||||
it('scopes the CLK_TCK cache to the runner that produced it', async () => {
|
||||
const hundredHz = vi.fn(async () => '100')
|
||||
const thousandHz = vi.fn(async () => '1000')
|
||||
|
||||
await expect(
|
||||
readLinuxProcessStartedAtMs(42, { readTextFile: readProcStat, runCommand: hundredHz })
|
||||
).resolves.toBe(1_699_000_010_000)
|
||||
await expect(
|
||||
readLinuxProcessStartedAtMs(42, { readTextFile: readProcStat, runCommand: thousandHz })
|
||||
).resolves.toBe(1_699_000_001_000)
|
||||
expect(thousandHz).toHaveBeenCalledWith('getconf', ['CLK_TCK'], 1_000)
|
||||
|
||||
// The first runner stays cached: one spawn per runner, not per call.
|
||||
await expect(
|
||||
readLinuxProcessStartedAtMs(42, { readTextFile: readProcStat, runCommand: hundredHz })
|
||||
).resolves.toBe(1_699_000_010_000)
|
||||
expect(hundredHz).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('retries getconf for a runner whose first CLK_TCK read failed', async () => {
|
||||
let attempt = 0
|
||||
const runCommand = vi.fn(async () => {
|
||||
attempt += 1
|
||||
if (attempt === 1) {
|
||||
throw new Error('getconf missing')
|
||||
}
|
||||
return '100'
|
||||
})
|
||||
|
||||
await expect(
|
||||
readLinuxProcessStartedAtMs(42, { readTextFile: readProcStat, runCommand })
|
||||
).resolves.toBeNull()
|
||||
await expect(
|
||||
readLinuxProcessStartedAtMs(42, { readTextFile: readProcStat, runCommand })
|
||||
).resolves.toBe(1_699_000_010_000)
|
||||
})
|
||||
|
||||
it.each([0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1, Number.NaN])(
|
||||
'rejects unsafe Windows pid %s before command interpolation',
|
||||
async (pid) => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { execFile } from 'node:child_process'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { promisify } from 'node:util'
|
||||
import { parseLinuxBootTimeSeconds, parseLinuxProcStartTicks } from './daemon-health'
|
||||
import type {
|
||||
LinuxStatEvidence,
|
||||
ProcessSignalEvidence,
|
||||
|
|
@ -9,9 +10,11 @@ import type {
|
|||
|
||||
const execFileAsync = promisify(execFile)
|
||||
|
||||
type InspectionCommandRunner = (file: string, args: string[], timeoutMs: number) => Promise<string>
|
||||
|
||||
export type DaemonProcessInspectionDependencies = {
|
||||
readTextFile?: (path: string) => Promise<string>
|
||||
runCommand?: (file: string, args: string[], timeoutMs: number) => Promise<string>
|
||||
runCommand?: InspectionCommandRunner
|
||||
}
|
||||
|
||||
export function inspectProcessSignal(pid: number): ProcessSignalEvidence {
|
||||
|
|
@ -63,6 +66,9 @@ export async function readProcessCommandLine(
|
|||
}
|
||||
}
|
||||
|
||||
// Why: Get-CimInstance errors (Winmgmt down, corrupt WMI repository, access denied) are
|
||||
// non-terminating and exit 0 with an empty $p, which is indistinguishable from "no such
|
||||
// process" — so the script reports query failure explicitly instead of asserting absence.
|
||||
export async function queryWindowsProcess(
|
||||
pid: number,
|
||||
dependencies: DaemonProcessInspectionDependencies = {}
|
||||
|
|
@ -78,23 +84,26 @@ export async function queryWindowsProcess(
|
|||
'-NoProfile',
|
||||
'-NonInteractive',
|
||||
'-Command',
|
||||
`$p = Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}"; ` +
|
||||
`if (!$p) { @{ exists = $false } | ConvertTo-Json -Compress } else { ` +
|
||||
`$ErrorActionPreference = 'Stop'; ` +
|
||||
`try { $p = Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}" } ` +
|
||||
`catch { @{ status = 'query_failed' } | ConvertTo-Json -Compress; exit 0 }; ` +
|
||||
`if (!$p) { @{ status = 'missing' } | ConvertTo-Json -Compress; exit 0 }; ` +
|
||||
`$start = $null; if ($p.CreationDate) { ` +
|
||||
`$start = [long]([DateTimeOffset]$p.CreationDate).ToUnixTimeMilliseconds() }; ` +
|
||||
`@{ exists = $true; cmd = $p.CommandLine; start = $start } | ConvertTo-Json -Compress }`
|
||||
`@{ status = 'present'; cmd = $p.CommandLine; start = $start } | ConvertTo-Json -Compress`
|
||||
],
|
||||
3_000
|
||||
)
|
||||
const parsed = JSON.parse(stdout.trim()) as {
|
||||
exists?: unknown
|
||||
status?: unknown
|
||||
cmd?: unknown
|
||||
start?: unknown
|
||||
}
|
||||
if (parsed.exists === false) {
|
||||
// Only a query that ran and found nothing proves absence; anything else stays indeterminate.
|
||||
if (parsed.status === 'missing') {
|
||||
return { status: 'missing' }
|
||||
}
|
||||
if (parsed.exists !== true) {
|
||||
if (parsed.status !== 'present') {
|
||||
return { status: 'unavailable' }
|
||||
}
|
||||
return {
|
||||
|
|
@ -108,6 +117,75 @@ export async function queryWindowsProcess(
|
|||
}
|
||||
}
|
||||
|
||||
// Why: the sync procfs helper in daemon-health spawns getconf per call; CLK_TCK is fixed for
|
||||
// the kernel's lifetime, so cache one async spawn and only retry after a failure. The cache is
|
||||
// keyed by runner because CLK_TCK belongs to the host that executes the command, not the module.
|
||||
const clockTicksPerSecondByRunner = new WeakMap<InspectionCommandRunner, Promise<number | null>>()
|
||||
|
||||
async function readClockTicksPerSecond(
|
||||
runCommand: InspectionCommandRunner
|
||||
): Promise<number | null> {
|
||||
let pending = clockTicksPerSecondByRunner.get(runCommand)
|
||||
if (!pending) {
|
||||
pending = runCommand('getconf', ['CLK_TCK'], 1_000).then(
|
||||
(stdout) => {
|
||||
const ticks = Number(stdout.trim())
|
||||
return Number.isFinite(ticks) && ticks > 0 ? ticks : null
|
||||
},
|
||||
() => null
|
||||
)
|
||||
clockTicksPerSecondByRunner.set(runCommand, pending)
|
||||
}
|
||||
const ticksPerSecond = await pending
|
||||
if (ticksPerSecond === null && clockTicksPerSecondByRunner.get(runCommand) === pending) {
|
||||
clockTicksPerSecondByRunner.delete(runCommand)
|
||||
}
|
||||
return ticksPerSecond
|
||||
}
|
||||
|
||||
// Why: same main-thread hazard as darwin — the sync linux helper does two readFileSync calls
|
||||
// plus a getconf spawn, and this audit-only probe runs in the Electron main process.
|
||||
export async function readLinuxProcessStartedAtMs(
|
||||
pid: number,
|
||||
dependencies: DaemonProcessInspectionDependencies = {}
|
||||
): Promise<number | null> {
|
||||
const readTextFile =
|
||||
dependencies.readTextFile ?? (async (path: string) => await readFile(path, 'utf8'))
|
||||
try {
|
||||
const startTicks = parseLinuxProcStartTicks(await readTextFile(`/proc/${pid}/stat`))
|
||||
const bootTimeSeconds = parseLinuxBootTimeSeconds(await readTextFile('/proc/stat'))
|
||||
const ticksPerSecond = await readClockTicksPerSecond(
|
||||
dependencies.runCommand ?? runInspectionCommand
|
||||
)
|
||||
if (
|
||||
ticksPerSecond === null ||
|
||||
!Number.isFinite(startTicks) ||
|
||||
!Number.isFinite(bootTimeSeconds)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return bootTimeSeconds * 1000 + (startTicks / ticksPerSecond) * 1000
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// Why: `ps` is darwin's only start-time source and this audit-only probe runs in the
|
||||
// Electron main process, where the sync spawn blocks every IPC/UI turn for its duration.
|
||||
export async function readMacosProcessStartedAtMs(
|
||||
pid: number,
|
||||
dependencies: DaemonProcessInspectionDependencies = {}
|
||||
): Promise<number | null> {
|
||||
const runCommand = dependencies.runCommand ?? runInspectionCommand
|
||||
try {
|
||||
const stdout = await runCommand('ps', ['-p', String(pid), '-o', 'lstart='], 2_000)
|
||||
const startedAtMs = Date.parse(stdout.trim())
|
||||
return Number.isFinite(startedAtMs) ? startedAtMs : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function runInspectionCommand(
|
||||
file: string,
|
||||
args: string[],
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ import {
|
|||
type DaemonAuditTrigger
|
||||
} from './daemon-audit-classifier'
|
||||
import type { DaemonEvidenceSource, ExactDaemonIncarnation } from './daemon-incarnation-evidence'
|
||||
import { trackDaemonAuditEligibility } from './daemon-audit-eligibility-event'
|
||||
import { createDaemonAuditEligibilityTracker } from './daemon-audit-eligibility-event'
|
||||
|
||||
type PendingDaemonSpawnOperation = {
|
||||
exitsBySessionId: Map<string, { incarnationId?: string }[]>
|
||||
|
|
@ -159,6 +159,8 @@ export class DaemonPtyAdapter implements IPtyProvider {
|
|||
private lastAuthenticatedIdentity: DaemonEndpointIdentity | null = null
|
||||
private exactDaemonIncarnation: ExactDaemonIncarnation | null = null
|
||||
private lastAuditObservation: DaemonAuditObservation | null = null
|
||||
// Why: every listProcesses call republishes the same observation; unthrottled it drains the shared per-session telemetry ceiling.
|
||||
private readonly trackAuditEligibility = createDaemonAuditEligibilityTracker()
|
||||
private auditObservationListeners: ((observation: DaemonAuditObservation) => void)[] = []
|
||||
private identityChangeListeners: ((event: DaemonIdentityChangeEvent) => void)[] = []
|
||||
private historyManager: HistoryManager | null
|
||||
|
|
@ -1723,7 +1725,7 @@ export class DaemonPtyAdapter implements IPtyProvider {
|
|||
|
||||
private publishAuditObservation(observation: DaemonAuditObservation): void {
|
||||
this.lastAuditObservation = observation
|
||||
trackDaemonAuditEligibility(observation)
|
||||
this.trackAuditEligibility(observation)
|
||||
notifyAuditListeners(this.auditObservationListeners, observation)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -307,7 +307,6 @@ describe('Activity portal pane switching', () => {
|
|||
})
|
||||
|
||||
it('releases a latched readiness once the terminal attaches', async () => {
|
||||
// Drive rAF explicitly — wall-clock waits flake under CI load.
|
||||
const frames = installAnimationFrameController()
|
||||
const target = document.createElement('div')
|
||||
document.body.append(target)
|
||||
|
|
@ -334,20 +333,11 @@ describe('Activity portal pane switching', () => {
|
|||
}
|
||||
buildRoot('hidden')
|
||||
|
||||
let churning = true
|
||||
let churns = 0
|
||||
const statuses: ActivityPortalReadinessStatus[] = []
|
||||
|
||||
function ActivityTerminalSlot(): null {
|
||||
const status = useActivityTerminalPortalStatus(target, PANE_A.paneKey)
|
||||
statuses.push(status)
|
||||
useLayoutEffect(() => {
|
||||
if (!churning || churns > 30) {
|
||||
return
|
||||
}
|
||||
churns += 1
|
||||
buildRoot(status === 'unavailable' ? 'sibling' : 'hidden')
|
||||
})
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
@ -355,21 +345,24 @@ describe('Activity portal pane switching', () => {
|
|||
await act(async () => {
|
||||
root.render(<ActivityTerminalSlot />)
|
||||
})
|
||||
for (let frame = 0; frame < 40; frame += 1) {
|
||||
if (frames.pending() === 0 && statuses.at(-1) === 'unavailable') {
|
||||
break
|
||||
}
|
||||
await frames.flush()
|
||||
expect(statuses.at(-1)).toBe('unavailable')
|
||||
|
||||
// Feed each observed DOM state separately so MutationObserver cannot coalesce the flips.
|
||||
for (let flip = 0; flip < 9; flip += 1) {
|
||||
await act(async () => {
|
||||
buildRoot(flip % 2 === 0 ? 'sibling' : 'hidden')
|
||||
await Promise.resolve()
|
||||
})
|
||||
await frames.flush()
|
||||
}
|
||||
expect(statuses.at(-1)).toBe('unavailable')
|
||||
|
||||
churning = false
|
||||
await act(async () => {
|
||||
buildRoot('ready')
|
||||
await Promise.resolve()
|
||||
})
|
||||
for (let frame = 0; frame < 10 && statuses.at(-1) !== 'ready'; frame += 1) {
|
||||
await frames.flush()
|
||||
}
|
||||
await frames.flush()
|
||||
expect(statuses.at(-1)).toBe('ready')
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Reference in New Issue