perf(mobile-sync): memoize the agent-status projection per entry (#10787)
* perf(mobile-sync): memoize the agent-status projection per entry buildRuntimeMobileAgentStatusProjection re-serialized every live agent on every status ping. setAgentStatus replaces one entry and re-spreads the map, which defeats the reference-equality skip gate, so each ping paid for every other agent's prompt, 20-entry history, and 8 KB assistant message to discover they had not changed. Memoize each row's JSON by entry identity, the cachedTabsProjection pattern already used a few functions above. Per ping: 0.18 ms -> 0.014 ms at 8 agents, 0.88 ms -> 0.063 ms at 40. The output is byte-identical — joining pre-serialized rows matches whole-array stringify, which the new test pins against a verbatim copy of the old implementation. * test(perf): stop inflating the projection benchmark baseline The pre-fix arm stringified each row and parsed it back before stringifying the array, a per-row roundtrip the original never paid. That made the baseline artificially slow: the reported 5.9x-14.0x is really 2.1x-5.0x. Share one row builder between both arms, and check equivalence after a ping as well as on the cold call — a stale-row bug can only surface once the cache is actually exercised, which the cold-path check could never catch.
This commit is contained in:
Neil
GitHub
parent
5a30c5c2ed
commit
62158570e0
|
|
@ -0,0 +1,192 @@
|
|||
#!/usr/bin/env node
|
||||
// Benchmark: cost of the mobile agent-status projection per store mutation.
|
||||
//
|
||||
// buildRuntimeMobileAgentStatusProjection runs on the App.tsx global store
|
||||
// subscriber. setAgentStatus replaces one entry and re-spreads
|
||||
// agentStatusByPaneKey, which defeats the reference-equality skip gate, so before
|
||||
// the fix EVERY live agent was re-serialized on EVERY status ping — each carrying
|
||||
// a prompt, a 20-entry stateHistory, toolInput, and an 8 KB-capped
|
||||
// lastAssistantMessage.
|
||||
//
|
||||
// The fix memoizes each row's JSON by entry identity, mirroring the
|
||||
// cachedTabsProjection pattern already in the same file, so a ping re-serializes
|
||||
// only the agent that actually changed.
|
||||
//
|
||||
// The bucket width is re-read from the real module so a drifted constant fails
|
||||
// loudly here instead of quietly changing what this benchmark measures.
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const GRAPH_SOURCE = readFileSync(
|
||||
fileURLToPath(new URL('../../src/renderer/src/runtime/sync-runtime-graph.ts', import.meta.url)),
|
||||
'utf8'
|
||||
)
|
||||
|
||||
const bucketMatch = GRAPH_SOURCE.match(/AGENT_STATUS_SYNC_UPDATED_AT_BUCKET_MS = ([0-9_]+)/)
|
||||
if (!bucketMatch) {
|
||||
throw new Error(
|
||||
'sync-runtime-graph.ts no longer defines the updatedAt bucket; re-sync this benchmark.'
|
||||
)
|
||||
}
|
||||
const BUCKET_MS = Number(bucketMatch[1].replaceAll('_', ''))
|
||||
|
||||
const ITERATIONS = Number.parseInt(process.env.ORCA_AGENT_PROJECTION_BENCH_ITERATIONS ?? '400', 10)
|
||||
const WARMUP = Number.parseInt(process.env.ORCA_AGENT_PROJECTION_BENCH_WARMUP ?? '60', 10)
|
||||
|
||||
for (const [name, value] of [
|
||||
['ORCA_AGENT_PROJECTION_BENCH_ITERATIONS', ITERATIONS],
|
||||
['ORCA_AGENT_PROJECTION_BENCH_WARMUP', WARMUP]
|
||||
]) {
|
||||
if (!Number.isInteger(value) || value <= 0) {
|
||||
throw new Error(`${name} must be a positive integer, received ${value}`)
|
||||
}
|
||||
}
|
||||
|
||||
function toRow(paneKey, entry) {
|
||||
return {
|
||||
paneKey,
|
||||
entryPaneKey: entry.paneKey,
|
||||
state: entry.state,
|
||||
prompt: entry.prompt,
|
||||
updatedAtBucket: Math.floor(entry.updatedAt / BUCKET_MS),
|
||||
stateStartedAt: entry.stateStartedAt,
|
||||
agentType: entry.agentType ?? null,
|
||||
terminalTitle: entry.terminalTitle ?? null,
|
||||
stateHistory: entry.stateHistory.map((history) => ({
|
||||
state: history.state,
|
||||
prompt: history.prompt,
|
||||
startedAt: history.startedAt,
|
||||
interrupted: history.interrupted ?? null
|
||||
})),
|
||||
toolName: entry.toolName ?? null,
|
||||
toolInput: entry.toolInput ?? null,
|
||||
interactivePrompt: entry.interactivePrompt ?? null,
|
||||
lastAssistantMessage: entry.lastAssistantMessage ?? null,
|
||||
interrupted: entry.interrupted ?? null
|
||||
}
|
||||
}
|
||||
|
||||
function serializeEntry(paneKey, entry) {
|
||||
return JSON.stringify(toRow(paneKey, entry))
|
||||
}
|
||||
|
||||
// Pre-fix: build plain rows and stringify the array once — no per-row roundtrip,
|
||||
// which the original never paid and which would inflate the reported speedup.
|
||||
function buildFull(map) {
|
||||
return JSON.stringify(
|
||||
Object.entries(map)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([paneKey, entry]) => toRow(paneKey, entry))
|
||||
)
|
||||
}
|
||||
|
||||
// Post-fix: reuse each row's JSON while its entry object is unchanged.
|
||||
function makeCachedBuilder() {
|
||||
let cache = null
|
||||
return (map) => {
|
||||
if (cache?.source === map) {
|
||||
return cache.projection
|
||||
}
|
||||
const previous = cache?.entries
|
||||
const entries = new Map()
|
||||
const parts = []
|
||||
for (const [paneKey, entry] of Object.entries(map).sort(([a], [b]) => a.localeCompare(b))) {
|
||||
const prior = previous?.get(paneKey)
|
||||
const row =
|
||||
prior?.entry === entry ? prior : { entry, projection: serializeEntry(paneKey, entry) }
|
||||
entries.set(paneKey, row)
|
||||
parts.push(row.projection)
|
||||
}
|
||||
const projection = `[${parts.join(',')}]`
|
||||
cache = { source: map, entries, projection }
|
||||
return projection
|
||||
}
|
||||
}
|
||||
|
||||
// A live agent as the store actually holds it.
|
||||
function makeEntry(index, updatedAt) {
|
||||
return {
|
||||
paneKey: `tab-${index}:leaf-0`,
|
||||
state: 'working',
|
||||
prompt: 'implement the feature and run the tests '.repeat(4),
|
||||
updatedAt,
|
||||
stateStartedAt: 1740000000000,
|
||||
agentType: 'claude',
|
||||
terminalTitle: `agent ${index}`,
|
||||
stateHistory: Array.from({ length: 20 }, (_value, step) => ({
|
||||
state: 'working',
|
||||
prompt: `step ${step} of the current turn`,
|
||||
startedAt: 1740000000000 + step,
|
||||
interrupted: null
|
||||
})),
|
||||
toolName: 'shell_command',
|
||||
toolInput: 'rg --line-number "pattern" src/ '.repeat(8),
|
||||
interactivePrompt: null,
|
||||
// The cap the store applies to assistant text.
|
||||
lastAssistantMessage: 'x'.repeat(8000),
|
||||
interrupted: null
|
||||
}
|
||||
}
|
||||
|
||||
function makeMap(agents) {
|
||||
const map = {}
|
||||
for (let index = 0; index < agents; index += 1) {
|
||||
map[`tab-${index}:leaf-0`] = makeEntry(index, 1740000000000 + index * BUCKET_MS)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
// One status ping: one entry replaced, the map re-spread, every other entry
|
||||
// reference-identical — exactly what setAgentStatus produces.
|
||||
function ping(map, round) {
|
||||
return {
|
||||
...map,
|
||||
'tab-0:leaf-0': makeEntry(0, 1740000000000 + BUCKET_MS * (round + 1))
|
||||
}
|
||||
}
|
||||
|
||||
function measure(build, map) {
|
||||
let current = map
|
||||
for (let index = 0; index < WARMUP; index += 1) {
|
||||
current = ping(current, index)
|
||||
build(current)
|
||||
}
|
||||
const samples = []
|
||||
for (let round = 0; round < 5; round += 1) {
|
||||
const start = performance.now()
|
||||
for (let index = 0; index < ITERATIONS; index += 1) {
|
||||
current = ping(current, index)
|
||||
build(current)
|
||||
}
|
||||
samples.push((performance.now() - start) / ITERATIONS)
|
||||
}
|
||||
samples.sort((a, b) => a - b)
|
||||
return samples[2]
|
||||
}
|
||||
|
||||
const pad = (value, width) => String(value).padStart(width)
|
||||
console.log('Mobile agent-status projection, per status ping (one agent changed)')
|
||||
console.log(`bucket=${BUCKET_MS}ms iterations=${ITERATIONS} warmup=${WARMUP} (median of 5 rounds)`)
|
||||
console.log(`${pad('agents', 8)} ${pad('full', 11)} ${pad('cached', 11)} ${pad('speedup', 9)}`)
|
||||
for (const agents of [3, 8, 20, 40]) {
|
||||
const map = makeMap(agents)
|
||||
const cachedBuilder = makeCachedBuilder()
|
||||
if (buildFull(map) !== cachedBuilder(map)) {
|
||||
throw new Error(`projection mismatch at ${agents} agents`)
|
||||
}
|
||||
// Why also after a ping: the cold call reuses nothing, so a stale-row bug would
|
||||
// only surface once the cache is actually exercised.
|
||||
const pinged = ping(map, 0)
|
||||
if (buildFull(pinged) !== cachedBuilder(pinged)) {
|
||||
throw new Error(`projection mismatch after a ping at ${agents} agents`)
|
||||
}
|
||||
const full = measure(buildFull, map)
|
||||
const cached = measure(makeCachedBuilder(), map)
|
||||
console.log(
|
||||
`${pad(agents, 8)} ${pad(`${full.toFixed(4)} ms`, 11)} ${pad(`${cached.toFixed(4)} ms`, 11)} ${pad(`${(full / cached).toFixed(1)}x`, 9)}`
|
||||
)
|
||||
}
|
||||
console.log(
|
||||
'\nThis runs on the global store subscriber, so the cost is paid per status ping\nand scales with the number of agents running in parallel — the workload this\napp exists for.'
|
||||
)
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import type { AppState } from '@/store/types'
|
||||
import {
|
||||
AGENT_STATUS_SYNC_UPDATED_AT_BUCKET_MS_FOR_TESTS,
|
||||
buildRuntimeMobileAgentStatusProjectionForTests,
|
||||
resetRuntimeMobileAgentStatusProjectionCacheForTests
|
||||
} from './sync-runtime-graph'
|
||||
|
||||
// Reference: the pre-change whole-array serialization, kept verbatim. The bucket
|
||||
// width is read from the module under test so a drifted constant cannot make this
|
||||
// reference silently disagree for a reason unrelated to the change.
|
||||
const BUCKET_MS = AGENT_STATUS_SYNC_UPDATED_AT_BUCKET_MS_FOR_TESTS
|
||||
function referenceProjection(map: AppState['agentStatusByPaneKey']): string {
|
||||
return JSON.stringify(
|
||||
Object.entries(map)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([paneKey, entry]) => ({
|
||||
paneKey,
|
||||
entryPaneKey: entry.paneKey,
|
||||
state: entry.state,
|
||||
prompt: entry.prompt,
|
||||
updatedAtBucket: Math.floor(entry.updatedAt / BUCKET_MS),
|
||||
stateStartedAt: entry.stateStartedAt,
|
||||
agentType: entry.agentType ?? null,
|
||||
terminalTitle: entry.terminalTitle ?? null,
|
||||
stateHistory: entry.stateHistory.map((history) => ({
|
||||
state: history.state,
|
||||
prompt: history.prompt,
|
||||
startedAt: history.startedAt,
|
||||
interrupted: history.interrupted ?? null
|
||||
})),
|
||||
toolName: entry.toolName ?? null,
|
||||
toolInput: entry.toolInput ?? null,
|
||||
interactivePrompt: entry.interactivePrompt ?? null,
|
||||
lastAssistantMessage: entry.lastAssistantMessage ?? null,
|
||||
interrupted: entry.interrupted ?? null
|
||||
}))
|
||||
)
|
||||
}
|
||||
|
||||
function makeEntry(index: number, overrides: Record<string, unknown> = {}): never {
|
||||
return {
|
||||
paneKey: `tab-${index}:leaf-0`,
|
||||
state: 'working',
|
||||
prompt: `prompt ${index} with "quotes" and \\ backslash`,
|
||||
updatedAt: 1740000000000 + index * 17,
|
||||
stateStartedAt: 1740000000000,
|
||||
agentType: 'claude',
|
||||
terminalTitle: `agent ${index} 日本 \u{1f389}`,
|
||||
stateHistory: Array.from({ length: 3 }, (_value, h) => ({
|
||||
state: 'working',
|
||||
prompt: `step ${h}`,
|
||||
startedAt: 1740000000000 + h
|
||||
})),
|
||||
toolName: 'shell_command',
|
||||
toolInput: 'ls -la',
|
||||
lastAssistantMessage: 'answer',
|
||||
...overrides
|
||||
} as never
|
||||
}
|
||||
|
||||
describe('mobile agent-status projection equivalence', () => {
|
||||
it('matches the whole-array serialization across shapes and cache reuse', () => {
|
||||
resetRuntimeMobileAgentStatusProjectionCacheForTests()
|
||||
const shapes: AppState['agentStatusByPaneKey'][] = []
|
||||
shapes.push({})
|
||||
shapes.push({ 'tab-0:leaf-0': makeEntry(0) })
|
||||
const many: AppState['agentStatusByPaneKey'] = {}
|
||||
for (let index = 0; index < 12; index += 1) {
|
||||
many[`tab-${index}:leaf-0`] = makeEntry(index)
|
||||
}
|
||||
shapes.push(many)
|
||||
// Optional fields absent entirely, which the ?? null fallbacks must cover.
|
||||
shapes.push({
|
||||
'tab-9:leaf-1': makeEntry(9, {
|
||||
agentType: undefined,
|
||||
terminalTitle: undefined,
|
||||
toolName: undefined,
|
||||
toolInput: undefined,
|
||||
interactivePrompt: undefined,
|
||||
lastAssistantMessage: undefined,
|
||||
interrupted: undefined
|
||||
})
|
||||
})
|
||||
// Keys deliberately out of insertion order to pin the sort.
|
||||
shapes.push({
|
||||
'tab-z:leaf-0': makeEntry(2),
|
||||
'tab-a:leaf-0': makeEntry(1),
|
||||
'tab-m:leaf-0': makeEntry(3)
|
||||
})
|
||||
|
||||
for (const [index, shape] of shapes.entries()) {
|
||||
expect({
|
||||
index,
|
||||
projection: buildRuntimeMobileAgentStatusProjectionForTests(shape)
|
||||
}).toEqual({ index, projection: referenceProjection(shape) })
|
||||
}
|
||||
|
||||
// Now exercise the cache: replace one entry the way setAgentStatus does and
|
||||
// confirm the reused rows still produce the reference output. The changes must
|
||||
// be observable in the projection — a sub-bucket updatedAt nudge is not, so a
|
||||
// stale-entry reuse bug would slip through.
|
||||
let current = many
|
||||
for (let round = 0; round < 4; round += 1) {
|
||||
current = {
|
||||
...current,
|
||||
'tab-0:leaf-0': makeEntry(0, {
|
||||
updatedAt: 1740000000000 + BUCKET_MS * 3 * (round + 1),
|
||||
state: round % 2 === 0 ? 'done' : 'working',
|
||||
prompt: `changed prompt round ${round}`,
|
||||
lastAssistantMessage: `answer round ${round}`
|
||||
})
|
||||
}
|
||||
expect({
|
||||
round,
|
||||
projection: buildRuntimeMobileAgentStatusProjectionForTests(current)
|
||||
}).toEqual({ round, projection: referenceProjection(current) })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
@ -68,6 +68,15 @@ type TabsProjectionCache = {
|
|||
entries: Map<string, TabsProjectionCacheEntry>
|
||||
projection: string
|
||||
}
|
||||
type AgentStatusProjectionCacheEntry = {
|
||||
entry: AppState['agentStatusByPaneKey'][string]
|
||||
projection: string
|
||||
}
|
||||
type AgentStatusProjectionCache = {
|
||||
source: AppState['agentStatusByPaneKey']
|
||||
entries: Map<string, AgentStatusProjectionCacheEntry>
|
||||
projection: string
|
||||
}
|
||||
|
||||
const registeredTabs = new Map<string, RegisteredTerminalTab>()
|
||||
// Why: registration time suppresses the "no live transport" warning during the async PTY-connect window; after the grace period it's a real stuck state.
|
||||
|
|
@ -129,6 +138,7 @@ function jsonContentEquals(a: unknown, b: unknown): boolean {
|
|||
return true
|
||||
}
|
||||
let cachedTabsProjection: TabsProjectionCache | null = null
|
||||
let cachedAgentStatusProjection: AgentStatusProjectionCache | null = null
|
||||
let cachedOpenFileIndexesSource: AppState['openFiles'] | null = null
|
||||
let cachedOpenFileIndexes: OpenFileIndexes | null = null
|
||||
let cachedEditorDraftsSource: AppState['editorDrafts'] | null = null
|
||||
|
|
@ -491,35 +501,76 @@ function buildRuntimeMobileEditorDraftsProjection(editorDrafts: AppState['editor
|
|||
)
|
||||
}
|
||||
|
||||
function serializeRuntimeMobileAgentStatusEntry(
|
||||
paneKey: string,
|
||||
entry: AppState['agentStatusByPaneKey'][string]
|
||||
): string {
|
||||
return JSON.stringify({
|
||||
paneKey,
|
||||
entryPaneKey: entry.paneKey,
|
||||
state: entry.state,
|
||||
prompt: entry.prompt,
|
||||
updatedAtBucket: Math.floor(entry.updatedAt / AGENT_STATUS_SYNC_UPDATED_AT_BUCKET_MS),
|
||||
stateStartedAt: entry.stateStartedAt,
|
||||
agentType: entry.agentType ?? null,
|
||||
terminalTitle: entry.terminalTitle ?? null,
|
||||
stateHistory: entry.stateHistory.map((history) => ({
|
||||
state: history.state,
|
||||
prompt: history.prompt,
|
||||
startedAt: history.startedAt,
|
||||
interrupted: history.interrupted ?? null
|
||||
})),
|
||||
toolName: entry.toolName ?? null,
|
||||
toolInput: entry.toolInput ?? null,
|
||||
// Why: include so a newly-captured AskUserQuestion prompt re-fires the mobile republish even when no other field changed.
|
||||
interactivePrompt: entry.interactivePrompt ?? null,
|
||||
lastAssistantMessage: entry.lastAssistantMessage ?? null,
|
||||
interrupted: entry.interrupted ?? null
|
||||
})
|
||||
}
|
||||
|
||||
function buildRuntimeMobileAgentStatusProjection(
|
||||
agentStatusByPaneKey: AppState['agentStatusByPaneKey']
|
||||
): string {
|
||||
return JSON.stringify(
|
||||
Object.entries(agentStatusByPaneKey)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([paneKey, entry]) => ({
|
||||
paneKey,
|
||||
entryPaneKey: entry.paneKey,
|
||||
state: entry.state,
|
||||
prompt: entry.prompt,
|
||||
updatedAtBucket: Math.floor(entry.updatedAt / AGENT_STATUS_SYNC_UPDATED_AT_BUCKET_MS),
|
||||
stateStartedAt: entry.stateStartedAt,
|
||||
agentType: entry.agentType ?? null,
|
||||
terminalTitle: entry.terminalTitle ?? null,
|
||||
stateHistory: entry.stateHistory.map((history) => ({
|
||||
state: history.state,
|
||||
prompt: history.prompt,
|
||||
startedAt: history.startedAt,
|
||||
interrupted: history.interrupted ?? null
|
||||
})),
|
||||
toolName: entry.toolName ?? null,
|
||||
toolInput: entry.toolInput ?? null,
|
||||
// Why: include so a newly-captured AskUserQuestion prompt re-fires the mobile republish even when no other field changed.
|
||||
interactivePrompt: entry.interactivePrompt ?? null,
|
||||
lastAssistantMessage: entry.lastAssistantMessage ?? null,
|
||||
interrupted: entry.interrupted ?? null
|
||||
}))
|
||||
)
|
||||
if (cachedAgentStatusProjection?.source === agentStatusByPaneKey) {
|
||||
return cachedAgentStatusProjection.projection
|
||||
}
|
||||
|
||||
// Why per-entry: a status ping replaces one entry and re-spreads the map, so
|
||||
// without this every other live agent — each carrying a 20-entry history and an
|
||||
// 8 KB message — is re-serialized to discover it did not change.
|
||||
const previousEntries = cachedAgentStatusProjection?.entries
|
||||
const entries = new Map<string, AgentStatusProjectionCacheEntry>()
|
||||
const parts: string[] = []
|
||||
|
||||
for (const [paneKey, entry] of Object.entries(agentStatusByPaneKey).sort(([a], [b]) =>
|
||||
a.localeCompare(b)
|
||||
)) {
|
||||
const previous = previousEntries?.get(paneKey)
|
||||
const cached =
|
||||
previous?.entry === entry
|
||||
? previous
|
||||
: { entry, projection: serializeRuntimeMobileAgentStatusEntry(paneKey, entry) }
|
||||
entries.set(paneKey, cached)
|
||||
parts.push(cached.projection)
|
||||
}
|
||||
|
||||
const projection = `[${parts.join(',')}]`
|
||||
cachedAgentStatusProjection = { source: agentStatusByPaneKey, entries, projection }
|
||||
return projection
|
||||
}
|
||||
|
||||
export function buildRuntimeMobileAgentStatusProjectionForTests(
|
||||
agentStatusByPaneKey: AppState['agentStatusByPaneKey']
|
||||
): string {
|
||||
return buildRuntimeMobileAgentStatusProjection(agentStatusByPaneKey)
|
||||
}
|
||||
|
||||
export const AGENT_STATUS_SYNC_UPDATED_AT_BUCKET_MS_FOR_TESTS =
|
||||
AGENT_STATUS_SYNC_UPDATED_AT_BUCKET_MS
|
||||
|
||||
export function resetRuntimeMobileAgentStatusProjectionCacheForTests(): void {
|
||||
cachedAgentStatusProjection = null
|
||||
}
|
||||
|
||||
export function runtimeMobileSessionSyncKeysEqual(
|
||||
|
|
|
|||
Loading…
Reference in New Issue