orca/config/scripts/agent-hook-normalizer-roundtrip-benchmark.mjs

#!/usr/bin/env node
// Benchmark: cost of validating an agent-status payload on every hook event.
//
// 13 of the 15 per-source normalizers in agent-hook-listener.ts built a plain
// object, JSON.stringify'd it, and handed the string to parseAgentStatusPayload,
// which runs assertJsonTextStructureWithinLimits (a per-character scan of the
// WHOLE serialized string) plus JSON.parse — only to reach the same
// normalizeAgentStatusObject the object path calls directly. Claude and Codex
// were already converted, with a comment calling the round trip "pure overhead
// on this hot per-hook path"; the other 13 were not.
//
// Why the gap widens with payload size: the direct path's field normalizer stops
// at the field cap (`normalized.length < maxLength`), so it is O(cap). The round
// trip is O(input) — and the input is bounded only by the 1 MB hook request
// limit, since tool_response text is passed through uncapped for most sources.
//
// Amplifier this models in the second table: resolveToolState stores the raw
// value in lastToolByPaneKey and inherits it until a turn reset, so one large
// tool result is re-serialized and re-scanned on every later event of the turn.
import { readFileSync } from 'node:fs'
import { performance } from 'node:perf_hooks'
import { fileURLToPath } from 'node:url'

const TYPES_SOURCE = readFileSync(
  fileURLToPath(new URL('../../src/shared/agent-status-types.ts', import.meta.url)),
  'utf8'
)

function readMirroredConstant(name) {
  const match = TYPES_SOURCE.match(new RegExp(`${name}\\s*=\\s*([0-9_]+)`))
  if (!match) {
    throw new Error(`agent-status-types.ts no longer defines ${name}; re-sync this benchmark.`)
  }
  return Number(match[1].replaceAll('_', ''))
}

// Read the cap the direct path clamps at, so a drifted value fails loudly here
// instead of quietly changing what this benchmark claims.
const ASSISTANT_MESSAGE_CAP = readMirroredConstant('AGENT_STATUS_ASSISTANT_MESSAGE_MAX_LENGTH')

const ITERATIONS = Number.parseInt(process.env.ORCA_HOOK_NORM_BENCH_ITERATIONS ?? '400', 10)
const WARMUP = Number.parseInt(process.env.ORCA_HOOK_NORM_BENCH_WARMUP ?? '200', 10)

for (const [name, value] of [
  ['ORCA_HOOK_NORM_BENCH_ITERATIONS', ITERATIONS],
  ['ORCA_HOOK_NORM_BENCH_WARMUP', WARMUP]
]) {
  if (!Number.isInteger(value) || value <= 0) {
    throw new Error(`${name} must be a positive integer, received ${value}`)
  }
}

const STRUCTURAL_TOKENS = 4096
const NESTING_DEPTH = 16

// Mirror of assertJsonTextStructureWithinLimits — the per-character scan the
// round trip pays before JSON.parse even starts.
function scanJsonStructure(content) {
  let structuralTokens = 0
  let depth = 0
  let inString = false
  let escaped = false
  for (let index = 0; index < content.length; index += 1) {
    const character = content[index]
    if (inString) {
      if (escaped) {
        escaped = false
      } else if (character === '\\') {
        escaped = true
      } else if (character === '"') {
        inString = false
      }
      continue
    }
    if (character === '"') {
      inString = true
      continue
    }
    if (
      character !== '{' &&
      character !== '}' &&
      character !== '[' &&
      character !== ']' &&
      character !== ',' &&
      character !== ':'
    ) {
      continue
    }
    structuralTokens += 1
    if (structuralTokens > STRUCTURAL_TOKENS) {
      throw new Error('structuralTokens')
    }
    if (character === '{' || character === '[') {
      depth += 1
      if (depth > NESTING_DEPTH) {
        throw new Error('nestingDepth')
      }
    } else if (character === '}' || character === ']') {
      depth = Math.max(0, depth - 1)
    }
  }
}

// Mirror of the field normalizer's bounded walk: it stops consuming at the cap.
function normalizeField(value, maxLength) {
  if (typeof value !== 'string') {
    return undefined
  }
  let normalized = ''
  let newlineRun = 0
  for (let index = 0; index < value.length && normalized.length < maxLength; index += 1) {
    const code = value.charCodeAt(index)
    if (code === 13 || code === 10 || code === 0x2028 || code === 0x2029) {
      if (code === 13 && value.charCodeAt(index + 1) === 10) {
        index += 1
      }
      if (newlineRun < 2) {
        normalized += '\n'
      }
      newlineRun += 1
      continue
    }
    newlineRun = 0
    normalized += value[index]
  }
  return normalized
}

function normalizeObject(payload) {
  return {
    state: payload.state,
    prompt: normalizeField(payload.prompt, ASSISTANT_MESSAGE_CAP),
    agentType: payload.agentType,
    toolName: normalizeField(payload.toolName, ASSISTANT_MESSAGE_CAP),
    toolInput: normalizeField(payload.toolInput, ASSISTANT_MESSAGE_CAP),
    lastAssistantMessage: normalizeField(payload.lastAssistantMessage, ASSISTANT_MESSAGE_CAP)
  }
}

// Pre-fix: serialize, scan every character, parse, then normalize.
function validateViaRoundTrip(payload) {
  const json = JSON.stringify(payload)
  scanJsonStructure(json)
  return normalizeObject(JSON.parse(json))
}

// Post-fix: normalize the object that is already in hand.
function validateDirect(payload) {
  return normalizeObject(payload)
}

function makePayload(messageBytes) {
  return {
    state: 'working',
    prompt: 'do the thing',
    agentType: 'grok',
    toolName: 'shell_command',
    toolInput: 'ls -la',
    lastAssistantMessage: 'x'.repeat(messageBytes)
  }
}

function measure(fn, payload) {
  for (let index = 0; index < WARMUP; index += 1) {
    fn(payload)
  }
  const samples = []
  for (let round = 0; round < 5; round += 1) {
    const start = performance.now()
    for (let index = 0; index < ITERATIONS; index += 1) {
      fn(payload)
    }
    samples.push((performance.now() - start) / ITERATIONS)
  }
  samples.sort((a, b) => a - b)
  return samples[2]
}

const rows = []
for (const kb of [4, 16, 64, 256]) {
  const payload = makePayload(kb * 1024)
  const before = validateViaRoundTrip(payload)
  const after = validateDirect(payload)
  if (JSON.stringify(before) !== JSON.stringify(after)) {
    throw new Error(`normalizer mismatch at ${kb} KB`)
  }
  rows.push({
    label: `${kb} KB`,
    beforeUs: measure(validateViaRoundTrip, payload) * 1000,
    afterUs: measure(validateDirect, payload) * 1000
  })
}

const pad = (value, width) => String(value).padStart(width)
console.log('Agent-status payload validation, per hook event')
console.log(
  `field cap=${ASSISTANT_MESSAGE_CAP} iterations=${ITERATIONS} warmup=${WARMUP} (median of 5 rounds)`
)
console.log(
  `${pad('payload', 9)} ${pad('round trip', 12)} ${pad('direct', 10)} ${pad('speedup', 9)}`
)
for (const row of rows) {
  console.log(
    `${pad(row.label, 9)} ${pad(`${row.beforeUs.toFixed(1)} us`, 12)} ${pad(`${row.afterUs.toFixed(1)} us`, 10)} ${pad(`${(row.beforeUs / row.afterUs).toFixed(1)}x`, 9)}`
  )
}

// A single large tool result is inherited across the turn, so every later event
// re-pays the round trip on bytes that were already validated once.
const TURN_EVENTS = 20
const inherited = makePayload(200 * 1024)
const beforeTurnMs = (measure(validateViaRoundTrip, inherited) * TURN_EVENTS).toFixed(2)
const afterTurnMs = (measure(validateDirect, inherited) * TURN_EVENTS).toFixed(2)
console.log(
  `\nOne 200 KB tool result, inherited across ${TURN_EVENTS} later events in the same turn:` +
    `\n  round trip ${beforeTurnMs} ms total   direct ${afterTurnMs} ms total`
)
console.log(
  '\nThe direct path is flat because the field normalizer stops at the cap; the\nround trip is linear in the raw payload, which is bounded only by the 1 MB\nhook request limit.'
)