Gate terminal scale perf reports (#4818)

config/scripts/run-terminal-scale-perf-report-gate.mjs

@@ -0,0 +1,113 @@
+import { spawnSync } from 'node:child_process'
+import { closeSync, mkdirSync, openSync } from 'node:fs'
+import { dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const DEFAULT_REPORT_PATH = 'test-results/terminal-scale-perf-report.json'
+
+export function parseReportGateArgs(argv, env = process.env) {
+  const forwardedArgs = [...argv]
+  if (forwardedArgs[0] === '--') {
+    forwardedArgs.shift()
+  }
+
+  let reportPath = env.ORCA_E2E_TERMINAL_PERF_REPORT_PATH || DEFAULT_REPORT_PATH
+  const passthroughArgs = []
+  for (let index = 0; index < forwardedArgs.length; index += 1) {
+    const arg = forwardedArgs[index]
+    if (arg === '--report' || arg === '--report-path' || arg === '--output') {
+      const next = forwardedArgs[index + 1]
+      if (!next || next.startsWith('-')) {
+        throw new Error(`${arg} requires a path`)
+      }
+      reportPath = next
+      index += 1
+      continue
+    }
+    if (
+      arg.startsWith('--report=') ||
+      arg.startsWith('--report-path=') ||
+      arg.startsWith('--output=')
+    ) {
+      reportPath = arg.slice(arg.indexOf('=') + 1)
+      continue
+    }
+    if (arg === '--reporter' || arg.startsWith('--reporter=')) {
+      throw new Error('test:e2e:terminal-perf:scale:report always uses --reporter=json')
+    }
+    passthroughArgs.push(arg)
+  }
+
+  return { passthroughArgs, reportPath }
+}
+
+function runNodeScript(scriptPath, args, stdio, spawnSyncImpl, env) {
+  return spawnSyncImpl(process.execPath, [scriptPath, ...args], {
+    cwd: process.cwd(),
+    env,
+    stdio
+  })
+}
+
+function exitCode(result) {
+  if (result.signal) {
+    console.error(`Terminal scale perf command exited with signal ${result.signal}`)
+    return 1
+  }
+  return result.status ?? 1
+}
+
+export function runTerminalScalePerfReportGate({
+  argv = process.argv.slice(2),
+  env = process.env,
+  spawnSyncImpl = spawnSync
+} = {}) {
+  const { passthroughArgs, reportPath } = parseReportGateArgs(argv, env)
+  mkdirSync(dirname(reportPath), { recursive: true })
+
+  const reportFd = openSync(reportPath, 'w')
+  let scaleResult
+  try {
+    scaleResult = runNodeScript(
+      'config/scripts/run-terminal-scale-perf-e2e.mjs',
+      ['--', '--reporter=json', ...passthroughArgs],
+      ['inherit', reportFd, 'inherit'],
+      spawnSyncImpl,
+      env
+    )
+  } finally {
+    closeSync(reportFd)
+  }
+
+  const scaleExitCode = exitCode(scaleResult)
+  if (scaleExitCode !== 0) {
+    console.error(`Terminal scale perf report saved to ${reportPath}`)
+    return scaleExitCode
+  }
+
+  console.log(`Terminal scale perf report saved to ${reportPath}`)
+  const summaryResult = runNodeScript(
+    'config/scripts/summarize-terminal-perf-report.mjs',
+    [reportPath],
+    'inherit',
+    spawnSyncImpl,
+    env
+  )
+  const summaryExitCode = exitCode(summaryResult)
+  if (summaryExitCode !== 0) {
+    return summaryExitCode
+  }
+
+  const budgetResult = runNodeScript(
+    'config/scripts/check-terminal-perf-report-budgets.mjs',
+    [reportPath],
+    'inherit',
+    spawnSyncImpl,
+    env
+  )
+  return exitCode(budgetResult)
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  process.exit(runTerminalScalePerfReportGate())
+}

config/scripts/run-terminal-scale-perf-report-gate.test.mjs

@@ -0,0 +1,123 @@
+import { mkdtempSync, readFileSync, rmSync, writeSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import {
+  parseReportGateArgs,
+  runTerminalScalePerfReportGate
+} from './run-terminal-scale-perf-report-gate.mjs'
+
+const tempDirs = []
+
+function tempReportPath() {
+  const dir = mkdtempSync(join(tmpdir(), 'orca-terminal-perf-gate-'))
+  tempDirs.push(dir)
+  return join(dir, 'report.json')
+}
+
+function makeSpawnSync({ scaleStatus = 0 } = {}) {
+  const calls = []
+  const spawnSyncImpl = vi.fn((command, args, options) => {
+    calls.push({ args, command, options })
+    if (args[0] === 'config/scripts/run-terminal-scale-perf-e2e.mjs') {
+      writeSync(options.stdio[1], '{"suites":[]}')
+      return { signal: null, status: scaleStatus }
+    }
+    return { signal: null, status: 0 }
+  })
+  return { calls, spawnSyncImpl }
+}
+
+afterEach(() => {
+  while (tempDirs.length > 0) {
+    rmSync(tempDirs.pop(), { force: true, recursive: true })
+  }
+})
+
+describe('run-terminal-scale-perf-report-gate', () => {
+  it('parses report path flags while forwarding remaining Playwright args', () => {
+    expect(
+      parseReportGateArgs(['--', '--report', 'tmp/report.json', '--grep', 'ACK-backpressured'])
+    ).toEqual({
+      passthroughArgs: ['--grep', 'ACK-backpressured'],
+      reportPath: 'tmp/report.json'
+    })
+
+    expect(parseReportGateArgs(['--output=out.json'], {})).toEqual({
+      passthroughArgs: [],
+      reportPath: 'out.json'
+    })
+  })
+
+  it('rejects ambiguous report paths and reporter overrides', () => {
+    expect(() => parseReportGateArgs(['--report', '--grep', 'ACK'])).toThrow(
+      '--report requires a path'
+    )
+    expect(() => parseReportGateArgs(['--reporter=line'])).toThrow(
+      'test:e2e:terminal-perf:scale:report always uses --reporter=json'
+    )
+    expect(() => parseReportGateArgs(['--reporter', 'line'])).toThrow(
+      'test:e2e:terminal-perf:scale:report always uses --reporter=json'
+    )
+  })
+
+  it('runs the scale suite with JSON output, then summarizes and budget-checks the report', () => {
+    const reportPath = tempReportPath()
+    const { calls, spawnSyncImpl } = makeSpawnSync()
+
+    const status = runTerminalScalePerfReportGate({
+      argv: ['--report', reportPath, '--grep', '25 ACK-backpressured real PTYs'],
+      env: { ...process.env, ORCA_TEST_MARKER: '1' },
+      spawnSyncImpl
+    })
+
+    expect(status).toBe(0)
+    expect(readFileSync(reportPath, 'utf8')).toBe('{"suites":[]}')
+    expect(calls.map((call) => call.args[0])).toEqual([
+      'config/scripts/run-terminal-scale-perf-e2e.mjs',
+      'config/scripts/summarize-terminal-perf-report.mjs',
+      'config/scripts/check-terminal-perf-report-budgets.mjs'
+    ])
+    expect(calls[0].args).toEqual([
+      'config/scripts/run-terminal-scale-perf-e2e.mjs',
+      '--',
+      '--reporter=json',
+      '--grep',
+      '25 ACK-backpressured real PTYs'
+    ])
+    expect(calls[0].options.env.ORCA_TEST_MARKER).toBe('1')
+    expect(calls[1].args).toEqual(['config/scripts/summarize-terminal-perf-report.mjs', reportPath])
+    expect(calls[2].args).toEqual([
+      'config/scripts/check-terminal-perf-report-budgets.mjs',
+      reportPath
+    ])
+  })
+
+  it('uses the report path from env when no flag is provided', () => {
+    const reportPath = tempReportPath()
+    const { calls, spawnSyncImpl } = makeSpawnSync()
+
+    const status = runTerminalScalePerfReportGate({
+      env: { ...process.env, ORCA_E2E_TERMINAL_PERF_REPORT_PATH: reportPath },
+      spawnSyncImpl
+    })
+
+    expect(status).toBe(0)
+    expect(calls[1].args).toEqual(['config/scripts/summarize-terminal-perf-report.mjs', reportPath])
+  })
+
+  it('stops before summarize and budget checks when the scale run fails', () => {
+    const reportPath = tempReportPath()
+    const { calls, spawnSyncImpl } = makeSpawnSync({ scaleStatus: 7 })
+
+    const status = runTerminalScalePerfReportGate({
+      argv: ['--report', reportPath],
+      spawnSyncImpl
+    })
+
+    expect(status).toBe(7)
+    expect(calls.map((call) => call.args[0])).toEqual([
+      'config/scripts/run-terminal-scale-perf-e2e.mjs'
+    ])
+  })
+})

package.json

@@ -62,6 +62,7 @@
     "test:e2e": "pnpm run ensure:electron-runtime && npx playwright test --config tests/playwright.config.ts --project electron-headless",
     "test:e2e:terminal-perf": "pnpm run ensure:electron-runtime && npx playwright test tests/e2e/terminal-typing-latency.spec.ts tests/e2e/terminal-foreground-redraw-freeze.spec.ts tests/e2e/terminal-output-scheduler.spec.ts tests/e2e/terminal-hidden-tui-visual-restore.spec.ts tests/e2e/artificial-opencode-terminal-load.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=2",
     "test:e2e:terminal-perf:scale": "pnpm run ensure:electron-runtime && node config/scripts/run-terminal-scale-perf-e2e.mjs",
+    "test:e2e:terminal-perf:scale:report": "pnpm run ensure:electron-runtime && node config/scripts/run-terminal-scale-perf-report-gate.mjs",
     "test:e2e:terminal-perf:check-report": "node config/scripts/check-terminal-perf-report-budgets.mjs",
     "test:e2e:terminal-perf:summarize": "node config/scripts/summarize-terminal-perf-report.mjs",
     "test:e2e:ssh-docker-perf": "node config/scripts/run-ssh-docker-perf-e2e.mjs",