From 318e2b4c2c501b50280dc9e5b11ed34bd899bdd9 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Fri, 8 May 2026 19:57:18 -0700 Subject: [PATCH] feat(telemetry): onboarding cohort + extension events (#1608) Co-authored-by: Orca --- src/main/index.ts | 2 + src/main/ipc/telemetry.test.ts | 89 ++++++- src/main/ipc/telemetry.ts | 14 +- .../onboarding-cohort-classifier.test.ts | 142 +++++++++++ .../telemetry/onboarding-cohort-classifier.ts | 107 +++++++++ src/main/telemetry/validator.test.ts | 132 ++++++++++ .../src/components/onboarding/AgentStep.tsx | 16 +- .../components/onboarding/OnboardingFlow.tsx | 2 +- .../src/components/onboarding/ThemeStep.tsx | 86 ++++++- .../use-onboarding-flow-persistence.ts | 18 +- .../onboarding/use-onboarding-flow.ts | 158 +++++++++--- src/shared/telemetry-events.ts | 226 ++++++++++++++++-- src/shared/types.ts | 8 + 13 files changed, 921 insertions(+), 79 deletions(-) create mode 100644 src/main/telemetry/onboarding-cohort-classifier.test.ts create mode 100644 src/main/telemetry/onboarding-cohort-classifier.ts diff --git a/src/main/index.ts b/src/main/index.ts index 8c1f4c7e2..354b35fe2 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -17,6 +17,7 @@ import { registerCoreHandlers } from './ipc/register-core-handlers' import { registerMobileHandlers } from './ipc/mobile' import { initTelemetry, shutdownTelemetry, trackAppOpenedOnce } from './telemetry/client' import { initCohortClassifier } from './telemetry/cohort-classifier' +import { initOnboardingCohortClassifier } from './telemetry/onboarding-cohort-classifier' import { resolveConsent } from './telemetry/consent' import { triggerStartupNotificationRegistration } from './ipc/notifications' import { OrcaRuntimeService } from './runtime/orca-runtime' @@ -445,6 +446,7 @@ app.whenReady().then(async () => { // regardless of whether it originates from the renderer, an IPC handler, // or `trackAppOpenedOnce` / `did-finish-load`. initCohortClassifier(store) + initOnboardingCohortClassifier(store) stats = new StatsCollector() claudeUsage = new ClaudeUsageStore(store) codexUsage = new CodexUsageStore(store) diff --git a/src/main/ipc/telemetry.test.ts b/src/main/ipc/telemetry.test.ts index 2fcfbf8bf..5622502e8 100644 --- a/src/main/ipc/telemetry.test.ts +++ b/src/main/ipc/telemetry.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines -- Why: a single test file pins the IPC boundary behavior for all four telemetry handlers plus the cohort-injection invariants; splitting would fragment the threat-model coverage. */ // IPC boundary behavior for the telemetry surface. Strict type narrows must // drop obviously-malformed calls before they reach the validator (the // renderer is in the threat model). Pins the consent-mutation rate limit: @@ -17,14 +18,16 @@ const { setOptInMock, persistBannerAcknowledgeMock, consumeConsentMutationTokenMock, - getCohortAtEmitMock + getCohortAtEmitMock, + getOnboardingCohortAtEmitMock } = vi.hoisted(() => ({ handleMock: vi.fn(), trackMock: vi.fn(), setOptInMock: vi.fn(), persistBannerAcknowledgeMock: vi.fn(), consumeConsentMutationTokenMock: vi.fn(), - getCohortAtEmitMock: vi.fn() + getCohortAtEmitMock: vi.fn(), + getOnboardingCohortAtEmitMock: vi.fn() })) vi.mock('electron', () => ({ ipcMain: { handle: handleMock } })) @@ -39,6 +42,9 @@ vi.mock('../telemetry/burst-cap', () => ({ vi.mock('../telemetry/cohort-classifier', () => ({ getCohortAtEmit: getCohortAtEmitMock })) +vi.mock('../telemetry/onboarding-cohort-classifier', () => ({ + getOnboardingCohortAtEmit: getOnboardingCohortAtEmitMock +})) import { _resetStoreForTests, registerTelemetryHandlers } from './telemetry' @@ -89,6 +95,8 @@ describe('telemetry IPC handlers', () => { consumeConsentMutationTokenMock.mockReturnValue(true) getCohortAtEmitMock.mockReset() getCohortAtEmitMock.mockReturnValue({ nth_repo_added: 0 }) + getOnboardingCohortAtEmitMock.mockReset() + getOnboardingCohortAtEmitMock.mockReturnValue({ cohort: undefined }) _resetStoreForTests() }) afterEach(() => { @@ -158,6 +166,83 @@ describe('telemetry IPC handlers', () => { expect(trackMock).toHaveBeenCalledWith('app_opened', { nth_repo_added: undefined }) }) + // Threat-model parity with the cohort override test: a compromised + // renderer must NOT be able to forge `nth_repo_added` either. The same + // spread-order invariant applies — `{ ...baseProps, ...getCohortAtEmit() }` + // — and the same future-refactor regression risk exists. Pinning both + // fields keeps the threat model symmetric. + it('main-derived nth_repo_added overrides renderer-supplied value', () => { + registerWith({ installId: 'x', existedBeforeTelemetryRelease: false, optedIn: true }) + getCohortAtEmitMock.mockReturnValue({ nth_repo_added: 2 }) + const handler = handlers.get('telemetry:track')! + handler({}, 'app_opened', { nth_repo_added: 99 }) + expect(trackMock).toHaveBeenCalledWith('app_opened', { nth_repo_added: 2 }) + }) + + // ── Onboarding cohort injection (mirrors the nth_repo_added pattern) ── + + it('injects onboarding cohort on events whose schema declares cohort', () => { + registerWith({ installId: 'x', existedBeforeTelemetryRelease: false, optedIn: true }) + getOnboardingCohortAtEmitMock.mockReturnValue({ cohort: 'fresh_install' }) + const handler = handlers.get('telemetry:track')! + handler({}, 'onboarding_step_viewed', { step: 1 }) + expect(trackMock).toHaveBeenCalledWith('onboarding_step_viewed', { + step: 1, + cohort: 'fresh_install' + }) + }) + + it('does NOT inject onboarding cohort on non-onboarding events', () => { + registerWith({ installId: 'x', existedBeforeTelemetryRelease: false, optedIn: true }) + const handler = handlers.get('telemetry:track')! + handler({}, 'settings_changed', { setting_key: 'editorAutoSave', value_kind: 'bool' }) + expect(getOnboardingCohortAtEmitMock).not.toHaveBeenCalled() + }) + + it('forwards undefined onboarding cohort fail-soft', () => { + registerWith({ installId: 'x', existedBeforeTelemetryRelease: true, optedIn: null }) + getOnboardingCohortAtEmitMock.mockReturnValue({ cohort: undefined }) + const handler = handlers.get('telemetry:track')! + handler({}, 'onboarding_started', {}) + expect(trackMock).toHaveBeenCalledWith('onboarding_started', { cohort: undefined }) + }) + + // Threat-model invariant: a compromised renderer must NOT be able to forge + // `cohort` by including it in the props payload. The IPC handler spreads + // the main-derived cohort AFTER the caller-supplied props, so the main + // value wins. This test pins that invariant — flipping the spread order + // would silently let a compromised renderer fake any cohort value. + it('main-derived cohort overrides renderer-supplied cohort', () => { + registerWith({ installId: 'x', existedBeforeTelemetryRelease: false, optedIn: true }) + getOnboardingCohortAtEmitMock.mockReturnValue({ cohort: 'fresh_install' }) + const handler = handlers.get('telemetry:track')! + // Caller tries to forge cohort='upgrade_backfill'; main must overwrite. + handler({}, 'onboarding_started', { cohort: 'upgrade_backfill' }) + expect(trackMock).toHaveBeenCalledWith('onboarding_started', { + cohort: 'fresh_install' + }) + }) + + // Threat-model invariant under degraded classifier: a compromised + // renderer must NOT be able to forge `cohort` even when the classifier + // fails soft to `{ cohort: undefined }`. The IPC handler spreads the + // classifier output AFTER the caller-supplied props, so an explicit + // `undefined` from the classifier still overwrites a forged value. A + // future refactor that switches the spread to a conditional assign + // (`if (c.cohort !== undefined) baseProps.cohort = c.cohort`) would + // silently regress this — pinning it here. + it('main-derived undefined cohort overrides renderer-supplied cohort (degraded classifier)', () => { + registerWith({ installId: 'x', existedBeforeTelemetryRelease: true, optedIn: true }) + getOnboardingCohortAtEmitMock.mockReturnValue({ cohort: undefined }) + const handler = handlers.get('telemetry:track')! + // Compromised renderer attempts to forge cohort='upgrade_backfill'; + // main strips it via the explicit-undefined spread. + handler({}, 'onboarding_started', { cohort: 'upgrade_backfill' }) + expect(trackMock).toHaveBeenCalledWith('onboarding_started', { + cohort: undefined + }) + }) + it('drops track calls with a non-string name', () => { registerWith({ installId: 'x', existedBeforeTelemetryRelease: false, optedIn: true }) const handler = handlers.get('telemetry:track')! diff --git a/src/main/ipc/telemetry.ts b/src/main/ipc/telemetry.ts index c8e158150..29d57536d 100644 --- a/src/main/ipc/telemetry.ts +++ b/src/main/ipc/telemetry.ts @@ -39,9 +39,10 @@ import { ipcMain } from 'electron' import { consumeConsentMutationToken } from '../telemetry/burst-cap' import { persistBannerAcknowledgeWithoutEmitting, setOptIn, track } from '../telemetry/client' import { getCohortAtEmit } from '../telemetry/cohort-classifier' +import { getOnboardingCohortAtEmit } from '../telemetry/onboarding-cohort-classifier' import { resolveConsent, type ConsentState } from '../telemetry/consent' import type { Store } from '../persistence' -import { isCohortExtendedEvent } from '../../shared/telemetry-events' +import { isCohortExtendedEvent, isOnboardingEvent } from '../../shared/telemetry-events' import type { EventName, EventProps } from '../../shared/telemetry-events' import type { OptInVia } from '../../shared/telemetry-events' @@ -123,11 +124,20 @@ export function registerTelemetryHandlers(store: Store): void { // validation and silently drop the entire event. The renderer call sites // stay synchronous (matching the existing fire-and-forget shape) and // avoid an extra IPC round-trip to fetch cohort. + // + // Onboarding events get the same treatment for the `cohort` property, + // gated by `isOnboardingEvent` (events whose schema declares `cohort`). + // The two injection sets are disjoint by construction today — no schema + // declares both `nth_repo_added` and `cohort` — but combining them via + // spread keeps that an additive change rather than a structural one. const eventName = name as EventName const baseProps = (props ?? {}) as Record - const finalProps = isCohortExtendedEvent(eventName) + const withRepoCohort = isCohortExtendedEvent(eventName) ? { ...baseProps, ...getCohortAtEmit() } : baseProps + const finalProps = isOnboardingEvent(eventName) + ? { ...withRepoCohort, ...getOnboardingCohortAtEmit() } + : withRepoCohort // The casts to `EventName` / `EventProps` here are // pass-through only — this file does NOT pretend the renderer's // name/props are type-safe. The validator inside `track()` is the diff --git a/src/main/telemetry/onboarding-cohort-classifier.test.ts b/src/main/telemetry/onboarding-cohort-classifier.test.ts new file mode 100644 index 000000000..25aa5e2f6 --- /dev/null +++ b/src/main/telemetry/onboarding-cohort-classifier.test.ts @@ -0,0 +1,142 @@ +// Pins the onboarding-cohort-classifier contract: synchronous read of +// `existedBeforeTelemetryRelease` plus the upgrade-backfill onboarding +// shape, fail-soft to `undefined` on any failure mode, at most one warn per +// session. See docs/onboarding-telemetry-extensions.md §2. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ONBOARDING_FINAL_STEP } from '../../shared/constants' +import type { GlobalSettings, OnboardingState } from '../../shared/types' +import type { Store } from '../persistence' +import { + _resetSessionWarnFlagForTests, + _setStoreForTests, + getOnboardingCohortAtEmit, + initOnboardingCohortClassifier +} from './onboarding-cohort-classifier' + +type ExistedBefore = boolean | null + +function makeFakeStore(opts: { + existedBefore: ExistedBefore + onboarding?: Partial + throwOnSettings?: boolean + throwOnOnboarding?: boolean +}): Store { + return { + getSettings: vi.fn((): GlobalSettings => { + if (opts.throwOnSettings) { + throw new Error('disk fault') + } + return { + telemetry: { + installId: 'fake', + optedIn: true, + existedBeforeTelemetryRelease: + opts.existedBefore === null ? undefined : opts.existedBefore + } + } as unknown as GlobalSettings + }), + getOnboarding: vi.fn((): OnboardingState => { + if (opts.throwOnOnboarding) { + throw new Error('disk fault') + } + return { + outcome: null, + lastCompletedStep: -1, + closedAt: null, + checklist: {}, + ...opts.onboarding + } as OnboardingState + }) + } as unknown as Store +} + +describe('onboarding-cohort-classifier', () => { + beforeEach(() => { + vi.spyOn(console, 'warn').mockImplementation(() => {}) + _setStoreForTests(null) + _resetSessionWarnFlagForTests() + }) + afterEach(() => { + vi.restoreAllMocks() + _setStoreForTests(null) + }) + + it('classifies a fresh-install user as fresh_install', () => { + initOnboardingCohortClassifier(makeFakeStore({ existedBefore: false })) + expect(getOnboardingCohortAtEmit()).toEqual({ cohort: 'fresh_install' }) + }) + + it('classifies an existing user with backfilled completion as upgrade_backfill', () => { + initOnboardingCohortClassifier( + makeFakeStore({ + existedBefore: true, + onboarding: { outcome: 'completed', lastCompletedStep: ONBOARDING_FINAL_STEP } + }) + ) + expect(getOnboardingCohortAtEmit()).toEqual({ cohort: 'upgrade_backfill' }) + }) + + it('classifies a live-completed existing user as upgrade_backfill (known limitation)', () => { + // Pins current behavior: a real existing-user who completes the wizard + // live writes the same canonical (`outcome: 'completed'`, + // `lastCompletedStep === ONBOARDING_FINAL_STEP`) shape that the migration + // backfill writes, so the classifier cannot distinguish them. This test + // documents the limitation; dashboards should filter `cohort` on + // `_started` and forward-fill across the session. + initOnboardingCohortClassifier( + makeFakeStore({ + existedBefore: true, + onboarding: { + outcome: 'completed', + lastCompletedStep: ONBOARDING_FINAL_STEP, + closedAt: 1234567890 + } + }) + ) + expect(getOnboardingCohortAtEmit()).toEqual({ cohort: 'upgrade_backfill' }) + }) + + it('classifies an existing user mid-wizard as fresh_install (the funnel cohort)', () => { + // Existing user but the onboarding state isn't the migration's + // canonical force-completed shape. Funnel-wise this user is going + // through the wizard live, so they belong with `fresh_install`. + initOnboardingCohortClassifier( + makeFakeStore({ + existedBefore: true, + onboarding: { outcome: null, lastCompletedStep: 1 } + }) + ) + expect(getOnboardingCohortAtEmit()).toEqual({ cohort: 'fresh_install' }) + }) + + it('returns undefined when existedBeforeTelemetryRelease is unset', () => { + initOnboardingCohortClassifier(makeFakeStore({ existedBefore: null })) + expect(getOnboardingCohortAtEmit()).toEqual({ cohort: undefined }) + }) + + it('returns undefined when the store is not initialized', () => { + expect(getOnboardingCohortAtEmit()).toEqual({ cohort: undefined }) + }) + + it('never throws and returns undefined when getSettings throws', () => { + initOnboardingCohortClassifier(makeFakeStore({ existedBefore: false, throwOnSettings: true })) + expect(() => getOnboardingCohortAtEmit()).not.toThrow() + expect(getOnboardingCohortAtEmit()).toEqual({ cohort: undefined }) + }) + + it('never throws and returns undefined when getOnboarding throws', () => { + initOnboardingCohortClassifier(makeFakeStore({ existedBefore: true, throwOnOnboarding: true })) + expect(() => getOnboardingCohortAtEmit()).not.toThrow() + expect(getOnboardingCohortAtEmit()).toEqual({ cohort: undefined }) + }) + + it('warns at most once per session even across many degraded calls', () => { + initOnboardingCohortClassifier(makeFakeStore({ existedBefore: false, throwOnSettings: true })) + const warnSpy = console.warn as unknown as ReturnType + for (let i = 0; i < 50; i++) { + getOnboardingCohortAtEmit() + } + expect(warnSpy).toHaveBeenCalledTimes(1) + }) +}) diff --git a/src/main/telemetry/onboarding-cohort-classifier.ts b/src/main/telemetry/onboarding-cohort-classifier.ts new file mode 100644 index 000000000..d065eb659 --- /dev/null +++ b/src/main/telemetry/onboarding-cohort-classifier.ts @@ -0,0 +1,107 @@ +// Cohort discriminator for onboarding-wizard telemetry events. See +// docs/onboarding-telemetry-extensions.md §2. +// +// `'fresh_install'` ⇔ `existedBeforeTelemetryRelease === false`. New users +// land on the wizard on first launch; the cohort never moves. +// +// `'upgrade_backfill'` ⇔ `existedBeforeTelemetryRelease === true` AND the +// onboarding state was backfilled at load time as a completed wizard run +// (the persistence migration at `src/main/persistence.ts:362-369` writes +// `outcome: 'completed'` and `lastCompletedStep: ONBOARDING_FINAL_STEP` for +// existing users that lack an onboarding block). A pre-existing user who is +// dropped into the wizard via the upgrade-backfill surface emits this +// cohort on every wizard event. +// +// Known limitation: the discriminator infers the migration-backfilled state +// from its canonical shape (`outcome === 'completed'` AND +// `lastCompletedStep === ONBOARDING_FINAL_STEP`). That shape is *also* what +// a live wizard completion writes via `closeWith('completed', ...)`. As a +// result, a real existing-user (`existedBeforeTelemetryRelease === true`) +// who goes through the wizard live will be classified as `fresh_install` +// during the wizard, then *flip* to `upgrade_backfill` on the very next +// event after `closeWith` persists the completion. Dashboard-side +// workaround: filter `cohort` on the `_started` event and forward-fill +// across the session, rather than re-reading the cohort on terminal +// events. Structural follow-up (out of scope here): add a sentinel +// `wasBackfilledByMigration: true` field at migration time so the +// classifier can disambiguate without dashboard-side gymnastics. +// +// Failure mode: this module never throws. On any read error or +// store-not-yet-initialized condition, `getOnboardingCohortAtEmit` returns +// `{ cohort: undefined }`. The schemas declare the field `.optional()`, so +// an event with an undefined cohort still validates and emits — it just +// lands without the cohort property. Mirrors `getCohortAtEmit`. + +import { ONBOARDING_FINAL_STEP } from '../../shared/constants' +import type { OnboardingCohort } from '../../shared/telemetry-events' +import type { Store } from '../persistence' + +let storeRef: Store | null = null + +let warnedThisSession = false + +export function initOnboardingCohortClassifier(store: Store): void { + storeRef = store + warnedThisSession = false +} + +export function getOnboardingCohortAtEmit(): { cohort: OnboardingCohort | undefined } { + if (!storeRef) { + warnOnce('store not initialized') + return { cohort: undefined } + } + try { + // Why: fresh_install classification depends only on the settings flag, + // so we read settings first and skip getOnboarding() entirely on that + // branch — a failing onboarding read must not demote a fresh-install + // user to `{ cohort: undefined }`. + const settings = storeRef.getSettings() + const existedBefore = settings.telemetry?.existedBeforeTelemetryRelease + if (existedBefore === false) { + return { cohort: 'fresh_install' } + } + if (existedBefore === true) { + // Why: an existing-user cohort marker can coexist with a fresh + // wizard run (the migration only backfills when there's no + // onboarding block on disk). The `upgrade_backfill` cohort is + // specifically the user who was force-completed by the migration — + // detected by the canonical `outcome === 'completed'` AND + // `lastCompletedStep === ONBOARDING_FINAL_STEP` shape that + // persistence.ts:362-369 writes. Caveat: the same canonical shape is + // produced by `closeWith('completed', ...)` after a live wizard run, + // so an existing user who completes the wizard live will be + // classified as `fresh_install` during the wizard and then flip to + // `upgrade_backfill` on the next event after completion is + // persisted. See the top-of-file "Known limitation" block for the + // dashboard-side workaround and the proposed sentinel-field fix. + const onboarding = storeRef.getOnboarding() + if ( + onboarding.outcome === 'completed' && + onboarding.lastCompletedStep === ONBOARDING_FINAL_STEP + ) { + return { cohort: 'upgrade_backfill' } + } + return { cohort: 'fresh_install' } + } + return { cohort: undefined } + } catch (err) { + warnOnce(err instanceof Error ? err.message : String(err)) + return { cohort: undefined } + } +} + +function warnOnce(reason: string): void { + if (warnedThisSession) { + return + } + warnedThisSession = true + console.warn('[telemetry-onboarding-cohort] classifier returned undefined', { reason }) +} + +export function _setStoreForTests(store: Store | null): void { + storeRef = store +} + +export function _resetSessionWarnFlagForTests(): void { + warnedThisSession = false +} diff --git a/src/main/telemetry/validator.test.ts b/src/main/telemetry/validator.test.ts index bd0bcf349..c22ce9078 100644 --- a/src/main/telemetry/validator.test.ts +++ b/src/main/telemetry/validator.test.ts @@ -150,6 +150,138 @@ describe('validate', () => { expect(result.ok).toBe(false) }) + // ── Onboarding extensions (docs/onboarding-telemetry-extensions.md) ─ + + it('accepts onboarding_agent_picked with all required fields', () => { + const result = validate('onboarding_agent_picked', { + agent_kind: 'claude-code', + on_path: true, + detected_count: 2, + detection_state: 'complete', + from_collapsed_section: false + }) + expect(result.ok).toBe(true) + }) + + it('accepts onboarding_agent_picked with cohort injected', () => { + // Mirrors the IPC handler injection: schemas declare `cohort` as + // `.optional()`, so the classifier-injected value must not trip + // `.strict()`. + const result = validate('onboarding_agent_picked', { + agent_kind: 'codex', + on_path: false, + detected_count: 0, + detection_state: 'pending', + from_collapsed_section: true, + cohort: 'fresh_install' + }) + expect(result.ok).toBe(true) + }) + + it('rejects onboarding_agent_picked with unknown detection_state', () => { + const result = validate('onboarding_agent_picked', { + agent_kind: 'claude-code', + on_path: true, + detected_count: 1, + detection_state: 'detecting', + from_collapsed_section: false + } as never) + expect(result.ok).toBe(false) + }) + + it('accepts onboarding_ghostty_discovered with field_group_count_bucket', () => { + const result = validate('onboarding_ghostty_discovered', { + state: 'found', + field_group_count_bucket: '4-7' + }) + expect(result.ok).toBe(true) + }) + + it('rejects onboarding_ghostty_discovered with raw count instead of bucket', () => { + // Pin the privacy-doctrine contract: raw counts are an environment + // fingerprint and must not ship. + const result = validate('onboarding_ghostty_discovered', { + state: 'found', + field_group_count_bucket: 5 + } as never) + expect(result.ok).toBe(false) + }) + + it('accepts onboarding_ghostty_import_clicked with no payload', () => { + const result = validate('onboarding_ghostty_import_clicked', {}) + expect(result.ok).toBe(true) + }) + + it('accepts onboarding_ghostty_import_failed with each enum reason', () => { + for (const reason of ['no_config', 'empty_diff', 'unknown'] as const) { + const result = validate('onboarding_ghostty_import_failed', { reason }) + expect(result.ok).toBe(true) + } + }) + + it('accepts onboarding_step_completed with duration_ms and advanced_via', () => { + const result = validate('onboarding_step_completed', { + step: 1, + value_kind: 'agent', + duration_ms: 1234, + advanced_via: 'keyboard' + }) + expect(result.ok).toBe(true) + }) + + it('accepts onboarding_step_completed without the new optional fields', () => { + // Pre-deploy events (no `duration_ms`, no `advanced_via`) must still + // validate cleanly — that's the point of `.optional()`. + const result = validate('onboarding_step_completed', { + step: 2, + value_kind: 'theme' + }) + expect(result.ok).toBe(true) + }) + + it('rejects onboarding_step_completed with negative duration_ms', () => { + const result = validate('onboarding_step_completed', { + step: 1, + value_kind: 'agent', + duration_ms: -5 + } as never) + expect(result.ok).toBe(false) + }) + + it('rejects onboarding_step_completed with unknown advanced_via', () => { + const result = validate('onboarding_step_completed', { + step: 1, + value_kind: 'agent', + advanced_via: 'voice' + } as never) + expect(result.ok).toBe(false) + }) + + it('accepts onboarding_started with cohort upgrade_backfill', () => { + const result = validate('onboarding_started', { cohort: 'upgrade_backfill' }) + expect(result.ok).toBe(true) + }) + + it('rejects cohort on a non-onboarding event', () => { + // The IPC injection set is derived from `'cohort' in schema.shape`; + // strict() rejection here is what makes that selectivity safe. + const result = validate('app_opened', { + cohort: 'fresh_install' + } as never) + expect(result.ok).toBe(false) + }) + + it('accepts onboarding_started with cohort: undefined (classifier fail-soft)', () => { + // The IPC handler injects `getOnboardingCohortAtEmit()` even when it + // returns `{ cohort: undefined }` — the spread `{ ...withRepoCohort, + // ...{ cohort: undefined } }` produces an explicit-undefined key, not a + // missing key. Zod's `.optional()` treats those as the same; this test + // pins the behavior so the load-bearing fail-soft path is not silently + // broken by a future zod or schema change. + const result = validate('onboarding_started', { cohort: undefined }) + expect(result.ok).toBe(true) + }) + // Rate-limit: at most one warn per event name per 60s. We cannot easily // control Date.now() without mocking time, so the coarse assertion is // that repeat-dropping the same event name does not emit a warn on every diff --git a/src/renderer/src/components/onboarding/AgentStep.tsx b/src/renderer/src/components/onboarding/AgentStep.tsx index 116b6f548..3cf79fc29 100644 --- a/src/renderer/src/components/onboarding/AgentStep.tsx +++ b/src/renderer/src/components/onboarding/AgentStep.tsx @@ -6,17 +6,15 @@ import type { TuiAgent } from '../../../../shared/types' type AgentStepProps = { selectedAgent: TuiAgent | null - onSelect: (agent: TuiAgent) => void + // `fromCollapsedSection` tells the controller whether the click happened + // under the `
` disclosure so `onboarding_agent_picked` can carry + // it without re-deriving from props at the emit site. + onSelect: (agent: TuiAgent, fromCollapsedSection: boolean) => void detectedSet: Set isDetecting: boolean } -export function AgentStep({ - selectedAgent, - onSelect, - detectedSet, - isDetecting -}: AgentStepProps) { +export function AgentStep({ selectedAgent, onSelect, detectedSet, isDetecting }: AgentStepProps) { const detected = AGENT_CATALOG.filter((agent) => detectedSet.has(agent.id)) const rest = AGENT_CATALOG.filter((agent) => !detectedSet.has(agent.id)) const hasDetected = detected.length > 0 @@ -75,7 +73,7 @@ export function AgentStep({ key={agent.id} agent={agent} selected={selectedAgent === agent.id} - onClick={() => onSelect(agent.id)} + onClick={() => onSelect(agent.id, false)} /> ))} @@ -95,7 +93,7 @@ export function AgentStep({ key={agent.id} agent={agent} selected={selectedAgent === agent.id} - onClick={() => onSelect(agent.id)} + onClick={() => onSelect(agent.id, true)} /> ))} diff --git a/src/renderer/src/components/onboarding/OnboardingFlow.tsx b/src/renderer/src/components/onboarding/OnboardingFlow.tsx index 0b9370f1b..424945689 100644 --- a/src/renderer/src/components/onboarding/OnboardingFlow.tsx +++ b/src/renderer/src/components/onboarding/OnboardingFlow.tsx @@ -65,7 +65,7 @@ export default function OnboardingFlow({ if (currentStep.id === 'repo') { void flowOpenFolder() } else { - void flowNext() + void flowNext('keyboard') } } window.addEventListener('keydown', onKeyDown, { capture: true }) diff --git a/src/renderer/src/components/onboarding/ThemeStep.tsx b/src/renderer/src/components/onboarding/ThemeStep.tsx index c4c940594..096232f1f 100644 --- a/src/renderer/src/components/onboarding/ThemeStep.tsx +++ b/src/renderer/src/components/onboarding/ThemeStep.tsx @@ -2,7 +2,12 @@ import { useEffect, useState } from 'react' import { Check, Monitor, Moon, Settings2, Sun } from 'lucide-react' import { toast } from 'sonner' import { cn } from '@/lib/utils' -import type { GhosttyImportPreview, GlobalSettings } from '../../../../shared/types' +import { track } from '@/lib/telemetry' +import type { + DiscoveryStatusEmitted, + GhosttyImportPreview, + GlobalSettings +} from '../../../../shared/types' import ghosttyIcon from '../../../../../resources/ghostty.svg' type ThemeStepProps = { @@ -12,12 +17,37 @@ type ThemeStepProps = { updateSettings: (updates: Partial) => Promise } +// The two UI-only states (`'idle'`, `'detecting'`) never fire telemetry. The +// remaining states are exactly `DiscoveryStatusEmitted`, which is the +// schema-side enum the compile-time guard in +// `src/shared/telemetry-events.ts` locks against. type DiscoveryState = | { status: 'idle' } | { status: 'detecting' } | { status: 'found'; preview: GhosttyImportPreview; fields: string[] } | { status: 'imported'; fields: string[] } | { status: 'absent' } +type _DiscoveryStatusEmittedSync = + Exclude extends DiscoveryStatusEmitted + ? DiscoveryStatusEmitted extends Exclude + ? true + : never + : never +const _discoveryStatusEmittedSyncCheck: _DiscoveryStatusEmittedSync = true +void _discoveryStatusEmittedSyncCheck + +function fieldGroupCountBucket(count: number): '0' | '1-3' | '4-7' | '8+' { + if (count <= 0) { + return '0' + } + if (count <= 3) { + return '1-3' + } + if (count <= 7) { + return '4-7' + } + return '8+' +} export function ThemeStep({ theme, onThemeChange, settings, updateSettings }: ThemeStepProps) { const [importing, setImporting] = useState(false) @@ -27,6 +57,13 @@ export function ThemeStep({ theme, onThemeChange, settings, updateSettings }: Th // "we found your Ghostty config" prompt instead of a buried Import button. // Settings are not applied until the user clicks Import (per design doc). useEffect(() => { + // Why: Ghostty config-import is darwin-only (see src/main/ghostty/discovery.ts). + // Skip the IPC + telemetry emission entirely on non-Mac so the + // `_discovered: absent` rate measured by the Mac-cohort dashboard isn't + // polluted by a population that cannot have a Ghostty config. + if (!navigator.userAgent.includes('Mac')) { + return + } let cancelled = false setDiscovery({ status: 'detecting' }) void window.api.settings @@ -41,15 +78,28 @@ export function ThemeStep({ theme, onThemeChange, settings, updateSettings }: Th // tell, so don't make a claim either way. if (!preview.found || Object.keys(preview.diff).length === 0) { setDiscovery({ status: 'absent' }) + track('onboarding_ghostty_discovered', { + state: 'absent', + field_group_count_bucket: '0' + }) return } - setDiscovery({ status: 'found', preview, fields: humanFields(preview.diff) }) + const fields = humanFields(preview.diff) + setDiscovery({ status: 'found', preview, fields }) + track('onboarding_ghostty_discovered', { + state: 'found', + field_group_count_bucket: fieldGroupCountBucket(fields.length) + }) }) .catch(() => { if (cancelled) { return } setDiscovery({ status: 'absent' }) + track('onboarding_ghostty_discovered', { + state: 'absent', + field_group_count_bucket: '0' + }) }) return () => { cancelled = true @@ -60,11 +110,16 @@ export function ThemeStep({ theme, onThemeChange, settings, updateSettings }: Th if (!settings || importing) { return } + // Why: track AFTER the busy guard so a double-click during an in-flight + // import doesn't inflate the click counter when no second import attempt + // actually proceeds. + track('onboarding_ghostty_import_clicked', {}) setImporting(true) try { const resolved = preview.found ? preview : await window.api.settings.previewGhosttyImport() if (!resolved.found || Object.keys(resolved.diff).length === 0) { toast.info('No Ghostty settings found to import') + track('onboarding_ghostty_import_failed', { reason: 'empty_diff' }) return } await updateSettings({ @@ -83,11 +138,17 @@ export function ThemeStep({ theme, onThemeChange, settings, updateSettings }: Th if (resolved.diff.theme) { onThemeChange(resolved.diff.theme) } - setDiscovery({ status: 'imported', fields: humanFields(resolved.diff) }) + const importedFields = humanFields(resolved.diff) + setDiscovery({ status: 'imported', fields: importedFields }) + track('onboarding_ghostty_discovered', { + state: 'imported', + field_group_count_bucket: fieldGroupCountBucket(importedFields.length) + }) } catch (err) { toast.error('Failed to import Ghostty settings', { description: err instanceof Error ? err.message : String(err) }) + track('onboarding_ghostty_import_failed', { reason: 'unknown' }) } finally { setImporting(false) } @@ -169,11 +230,14 @@ function GhosttyDiscoveryRow({ disabled: boolean onImport: (preview: GhosttyImportPreview) => void }) { - if (discovery.status === 'absent') { + // Why: 'idle' is the pre-effect state that persists on non-Mac (the + // discovery effect short-circuits there), so render nothing instead of + // showing the dashed-border "Looking for a Ghostty config…" placeholder. + if (discovery.status === 'absent' || discovery.status === 'idle') { return null } - if (discovery.status === 'detecting' || discovery.status === 'idle') { + if (discovery.status === 'detecting') { return (
@@ -204,8 +268,7 @@ function GhosttyDiscoveryRow({
Ghostty config detected.{' '} - Import {fields.length > 0 ? fields.map((f) => f.toLowerCase()).join(', ') : 'settings'} - ? + Import {fields.length > 0 ? fields.map((f) => f.toLowerCase()).join(', ') : 'settings'}?
{preview.configPath && ( @@ -232,7 +295,10 @@ function ChromePreview({ variant }: { variant: GlobalSettings['theme'] }) { if (variant === 'system') { return (
-
+
): string[] { { label: 'Mouse', keys: ['terminalMouseHideWhileTyping', 'terminalFocusFollowsMouse'] }, { label: 'macOS Option key', keys: ['terminalMacOptionAsAlt'] } ] - return groups - .filter(({ keys }) => keys.some((k) => k in diff)) - .map(({ label }) => label) + return groups.filter(({ keys }) => keys.some((k) => k in diff)).map(({ label }) => label) } diff --git a/src/renderer/src/components/onboarding/use-onboarding-flow-persistence.ts b/src/renderer/src/components/onboarding/use-onboarding-flow-persistence.ts index a244d3872..1c668f281 100644 --- a/src/renderer/src/components/onboarding/use-onboarding-flow-persistence.ts +++ b/src/renderer/src/components/onboarding/use-onboarding-flow-persistence.ts @@ -26,6 +26,11 @@ type CloseWithDeps = { setError: (msg: string | null) => void } +export type DismissedExtras = { + advancedVia: 'button' | 'keyboard' + durationMs: number +} + export function useCloseWith({ onOnboardingChange, onboardingChecklist, @@ -37,7 +42,8 @@ export function useCloseWith({ outcome: 'completed' | 'dismissed', checklist: Partial, lastStepReached: StepNumber, - completedPath?: 'open_folder' | 'clone_url' + completedPath?: 'open_folder' | 'clone_url', + dismissedExtras?: DismissedExtras ): Promise => { let nextState: OnboardingState try { @@ -82,7 +88,15 @@ export function useCloseWith({ }) } } else if (outcome === 'dismissed') { - track('onboarding_dismissed', { last_step: lastStepReached }) + track('onboarding_dismissed', { + last_step: lastStepReached, + ...(dismissedExtras + ? { + duration_ms: dismissedExtras.durationMs, + advanced_via: dismissedExtras.advancedVia + } + : {}) + }) } return true }, diff --git a/src/renderer/src/components/onboarding/use-onboarding-flow.ts b/src/renderer/src/components/onboarding/use-onboarding-flow.ts index 182512400..1ed581e5e 100644 --- a/src/renderer/src/components/onboarding/use-onboarding-flow.ts +++ b/src/renderer/src/components/onboarding/use-onboarding-flow.ts @@ -1,19 +1,16 @@ +/* eslint-disable max-lines -- Why: this hook is the single orchestrator for every onboarding-step transition (navigation, persistence, telemetry, ref-mirror, auto-select); splitting would force callers to coordinate ordering across multiple hooks and lose the controller-shape contract OnboardingFlow.tsx consumes. */ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { toast } from 'sonner' import { AGENT_CATALOG } from '@/lib/agent-catalog' import { useAppStore } from '@/store' import { activateAndRevealWorktree } from '@/lib/worktree-activation' import { applyDocumentTheme } from '@/lib/document-theme' -import { track } from '@/lib/telemetry' +import { track, tuiAgentToAgentKind } from '@/lib/telemetry' import { isGitRepoKind } from '../../../../shared/repo-kind' import type { GlobalSettings, OnboardingState, TuiAgent } from '../../../../shared/types' import type { NotificationDraft } from './NotificationStep' import { STEPS, type StepNumber } from './use-onboarding-flow-types' -import { - persistStep, - useCloseWith, - usePersistCurrentStep -} from './use-onboarding-flow-persistence' +import { persistStep, useCloseWith, usePersistCurrentStep } from './use-onboarding-flow-persistence' export { STEPS } from './use-onboarding-flow-types' export type { StepId, StepNumber } from './use-onboarding-flow-types' @@ -28,9 +25,7 @@ export function useOnboardingFlow( const updateSettings = useAppStore((s) => s.updateSettings) const refreshDetectedAgents = useAppStore((s) => s.refreshDetectedAgents) const detectedAgentIds = useAppStore((s) => s.detectedAgentIds) - const isDetectingAgents = useAppStore( - (s) => s.isDetectingAgents || s.isRefreshingAgents - ) + const isDetectingAgents = useAppStore((s) => s.isDetectingAgents || s.isRefreshingAgents) const fetchRepos = useAppStore((s) => s.fetchRepos) const fetchWorktrees = useAppStore((s) => s.fetchWorktrees) const openModal = useAppStore((s) => s.openModal) @@ -89,14 +84,55 @@ export function useOnboardingFlow( themeInteractedRef.current = true setTheme(value) }, []) - const setSelectedAgentInteractive = useCallback((value: TuiAgent | null) => { - agentInteractedRef.current = true - setSelectedAgent(value) - }, []) + // `fromCollapsedSection` is the click-site signal for whether the picked + // agent lived under the `
` disclosure in AgentStep. AgentStep is + // the only call site that has the real answer; main-side detected_count / + // detection_state are merged in here from the store. + const detectedAgentIdsRef = useRef(detectedAgentIds ?? []) + const isDetectingRef = useRef(isDetectingAgents) + const selectedAgentRef = useRef(selectedAgent) + useEffect(() => { + selectedAgentRef.current = selectedAgent + }, [selectedAgent]) + const setSelectedAgentInteractive = useCallback( + (value: TuiAgent | null, fromCollapsedSection = false) => { + agentInteractedRef.current = true + // Why: de-dup re-clicks on the current agent so dashboards count + // mind-changes only, not idle reselection of the same option. + const prev = selectedAgentRef.current + setSelectedAgent(value) + if (value === null || value === prev) { + return + } + // Why: emit at click time, not at step completion, so we capture + // mind-changes within the step. `tuiAgentToAgentKind` falls back to + // `'other'` for any string outside the union. + const detected = detectedAgentIdsRef.current + track('onboarding_agent_picked', { + agent_kind: tuiAgentToAgentKind(value), + on_path: detected.includes(value), + detected_count: detected.length, + detection_state: isDetectingRef.current ? 'pending' : 'complete', + from_collapsed_section: fromCollapsedSection + }) + }, + [] + ) const detectedSet = useMemo(() => new Set(detectedAgentIds ?? []), [detectedAgentIds]) const currentStep = STEPS[stepIndex] + // Why: refs let `setSelectedAgentInteractive` (a stable useCallback) read + // the freshest detection snapshot at click time without re-rebinding the + // handler whenever the store flips a flag. Mirrors the + // `selectedAgentRef` pattern above. + useEffect(() => { + detectedAgentIdsRef.current = detectedAgentIds ?? [] + }, [detectedAgentIds]) + useEffect(() => { + isDetectingRef.current = isDetectingAgents + }, [isDetectingAgents]) + // Why: pin start time once so onboarding_completed reports a real funnel duration. const startTimeRef = useRef(Date.now()) @@ -142,17 +178,24 @@ export function useOnboardingFlow( // eslint-disable-next-line react-hooks/exhaustive-deps }, []) + // Session-local step duration: re-pinned on every step view so a resumed + // user emits `duration_ms` for the visible step measuring only the + // post-resume time. Optional on the schema so a missing baseline (e.g. the + // _viewed effect was skipped or StrictMode double-mounted) fail-soft drops + // the field rather than the event. See docs/onboarding-telemetry-extensions.md. + const stepStartedAtRef = useRef(Date.now()) useEffect(() => { + stepStartedAtRef.current = Date.now() track('onboarding_step_viewed', { step: currentStep.stepNumber }) }, [currentStep.stepNumber]) + const consumeStepDurationMs = useCallback((): number => { + return Math.max(0, Date.now() - stepStartedAtRef.current) + }, []) + // Why: only auto-pick on first mount when detection completes; otherwise // selecting an agent would re-trigger this effect and clobber/race user clicks. const didAutoSelectRef = useRef(false) - const selectedAgentRef = useRef(selectedAgent) - useEffect(() => { - selectedAgentRef.current = selectedAgent - }, [selectedAgent]) useEffect(() => { if (didAutoSelectRef.current) { return @@ -197,7 +240,15 @@ export function useOnboardingFlow( if (!closed) { return } - track('onboarding_step_completed', { step: 4, value_kind: 'repo' }) + // Why: step 4 has no keyboard-vs-button advance — Cmd+Enter routes to + // `openFolder()` which collapses both into the path-clicked path. Emit + // `duration_ms` only; `advanced_via` is intentionally absent for step 4. + // See docs/onboarding-telemetry-extensions.md §3. + track('onboarding_step_completed', { + step: 4, + value_kind: 'repo', + duration_ms: consumeStepDurationMs() + }) if (isGit) { openModal('new-workspace-composer', { initialRepoId: repoId, @@ -206,7 +257,7 @@ export function useOnboardingFlow( }) } }, - [closeWith, fetchRepos, fetchWorktrees, openModal] + [closeWith, consumeStepDurationMs, fetchRepos, fetchWorktrees, openModal] ) const persistCurrentStep = usePersistCurrentStep({ @@ -221,19 +272,31 @@ export function useOnboardingFlow( setError }) - const next = useCallback(async () => { - if (busyLabel || currentStep.id === 'repo') { - return - } - const ok = await persistCurrentStep() - if (ok) { - track('onboarding_step_completed', { - step: currentStep.stepNumber, - value_kind: currentStep.valueKind - }) - setStepIndex((idx) => Math.min(idx + 1, STEPS.length - 1)) - } - }, [busyLabel, currentStep.id, currentStep.stepNumber, currentStep.valueKind, persistCurrentStep]) + const next = useCallback( + async (advancedVia: 'button' | 'keyboard' = 'button') => { + if (busyLabel || currentStep.id === 'repo') { + return + } + const ok = await persistCurrentStep() + if (ok) { + track('onboarding_step_completed', { + step: currentStep.stepNumber, + value_kind: currentStep.valueKind, + duration_ms: consumeStepDurationMs(), + advanced_via: advancedVia + }) + setStepIndex((idx) => Math.min(idx + 1, STEPS.length - 1)) + } + }, + [ + busyLabel, + consumeStepDurationMs, + currentStep.id, + currentStep.stepNumber, + currentStep.valueKind, + persistCurrentStep + ] + ) const openFolder = useCallback(async () => { // Why: re-entry guard — rapid Cmd+Enter must not launch duplicate pickers. @@ -278,7 +341,10 @@ export function useOnboardingFlow( track('onboarding_step4_path_clicked', { path: 'clone_url' }) setBusyLabel('Cloning repo…') try { - const repo = await window.api.repos.clone({ url: trimmed, destination: settings.workspaceDir }) + const repo = await window.api.repos.clone({ + url: trimmed, + destination: settings.workspaceDir + }) await completeRepo(repo.id, true, 'clone_url') } catch (err) { setError(err instanceof Error ? err.message : String(err)) @@ -295,7 +361,16 @@ export function useOnboardingFlow( if (busyLabel) { return } - track('onboarding_step_skipped', { step: currentStep.stepNumber }) + // Why: skip has no keyboard path today, so `advanced_via` is always + // `'button'`. Including the field keeps the shape uniform with the + // completed/dismissed events and lets a future keyboard-skip arrive + // without a schema migration. + const durationMs = consumeStepDurationMs() + track('onboarding_step_skipped', { + step: currentStep.stepNumber, + duration_ms: durationMs, + advanced_via: 'button' + }) // Why: theme step previews on the document without persisting. On skip, // revert to the saved theme before advancing so the preview doesn't leak. if (currentStep.id === 'theme' && settings) { @@ -303,7 +378,10 @@ export function useOnboardingFlow( applyDocumentTheme(settings.theme) } if (currentStep.id === 'repo') { - await closeWith('dismissed', {}, currentStep.stepNumber) + await closeWith('dismissed', {}, currentStep.stepNumber, undefined, { + advancedVia: 'button', + durationMs + }) return } // Why: persistence-only path — does NOT trigger requestPermission, so @@ -315,7 +393,15 @@ export function useOnboardingFlow( return } setStepIndex((idx) => Math.min(idx + 1, STEPS.length - 1)) - }, [busyLabel, closeWith, currentStep.id, currentStep.stepNumber, onOnboardingChange, settings]) + }, [ + busyLabel, + closeWith, + consumeStepDurationMs, + currentStep.id, + currentStep.stepNumber, + onOnboardingChange, + settings + ]) const back = useCallback(() => { setStepIndex((idx) => Math.max(idx - 1, 0)) diff --git a/src/shared/telemetry-events.ts b/src/shared/telemetry-events.ts index 83ce4af25..3de78556e 100644 --- a/src/shared/telemetry-events.ts +++ b/src/shared/telemetry-events.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines -- Why: this is the single source of truth for every telemetry event schema, enum, and the cohort-injection set predicates. Splitting it would scatter the .strict() / Zod-first doctrine across files and break the EventMap derivation that makes adding an event a one-line change. */ // Single source of truth for telemetry event names, schemas, and enums. // // Zod-first: every event schema is declared once and the compile-time @@ -15,7 +16,7 @@ import { z } from 'zod' import { ONBOARDING_FINAL_STEP } from './constants' -import type { GlobalSettings, OnboardingChecklistState } from './types' +import type { DiscoveryStatusEmitted, GlobalSettings, OnboardingChecklistState } from './types' // ── Shared property enums ─────────────────────────────────────────────── @@ -302,26 +303,72 @@ type _OnboardingChecklistItemSync = const _onboardingChecklistItemSyncCheck: _OnboardingChecklistItemSync = true void _onboardingChecklistItemSyncCheck +// Cohort discriminator threaded onto every onboarding-wizard event by the +// IPC `telemetry:track` handler (mirrors `nth_repo_added`). `.optional()` is +// load-bearing: the classifier returns `undefined` when settings can't be +// read, and `.strict()` would otherwise reject the event entirely. +// +// Adding a new onboarding event: include `cohort: cohortSchema` on its +// schema. The injection set in `telemetry:track` is derived from +// `'cohort' in schema.shape`, so there is no parallel hand-maintained list. +const cohortSchema = z.enum(['fresh_install', 'upgrade_backfill']).optional() + +// `'button' | 'keyboard'` records whether the user advanced via a footer +// button click or via Cmd/Ctrl+Enter. Skip and dismiss don't have a keyboard +// path today (the field will only ever be `'button'` for those events) but +// the uniform shape lets a future keyboard skip arrive without a schema +// migration. +const advancedViaSchema = z.enum(['button', 'keyboard']).optional() + const onboardingStartedSchema = z - .object({ resumed_from_step: onboardingStepSchema.optional() }) + .object({ resumed_from_step: onboardingStepSchema.optional(), cohort: cohortSchema }) + .strict() +const onboardingStepViewedSchema = z + .object({ step: onboardingStepSchema, cohort: cohortSchema }) .strict() -const onboardingStepViewedSchema = z.object({ step: onboardingStepSchema }).strict() const onboardingStepCompletedSchema = z - .object({ step: onboardingStepSchema, value_kind: onboardingValueKindSchema }) + .object({ + step: onboardingStepSchema, + value_kind: onboardingValueKindSchema, + duration_ms: z.number().int().nonnegative().optional(), + advanced_via: advancedViaSchema, + cohort: cohortSchema + }) + .strict() +const onboardingStepSkippedSchema = z + .object({ + step: onboardingStepSchema, + duration_ms: z.number().int().nonnegative().optional(), + advanced_via: advancedViaSchema, + cohort: cohortSchema + }) + .strict() +const onboardingStep4PathClickedSchema = z + .object({ path: onboardingPathSchema, cohort: cohortSchema }) .strict() -const onboardingStepSkippedSchema = z.object({ step: onboardingStepSchema }).strict() -const onboardingStep4PathClickedSchema = z.object({ path: onboardingPathSchema }).strict() const onboardingStep4PathFailedSchema = z - .object({ path: onboardingPathSchema, reason: onboardingFailureReasonSchema }) + .object({ + path: onboardingPathSchema, + reason: onboardingFailureReasonSchema, + cohort: cohortSchema + }) .strict() const onboardingCompletedSchema = z .object({ path: onboardingPathSchema, is_git_repo: z.boolean(), - total_duration_ms: z.number().int().nonnegative() + total_duration_ms: z.number().int().nonnegative(), + cohort: cohortSchema + }) + .strict() +const onboardingDismissedSchema = z + .object({ + last_step: onboardingStepSchema, + duration_ms: z.number().int().nonnegative().optional(), + advanced_via: advancedViaSchema, + cohort: cohortSchema }) .strict() -const onboardingDismissedSchema = z.object({ last_step: onboardingStepSchema }).strict() const activationChecklistItemCompletedSchema = z .object({ item: onboardingChecklistItemSchema, @@ -329,6 +376,68 @@ const activationChecklistItemCompletedSchema = z }) .strict() +// Fired at click time from `setSelectedAgentInteractive` so we capture +// mind-changes within the step rather than just the final pick. `agent_kind` +// uses `tuiAgentToAgentKind` so the wire enum stays closed even when stale +// persisted settings present a string outside `TuiAgent` (the fallback is +// `'other'`). +const onboardingAgentPickedSchema = z + .object({ + agent_kind: agentKindSchema, + on_path: z.boolean(), + detected_count: z.number().int().nonnegative(), + // `'pending'` when the merged isDetectingAgents/isRefreshingAgents flag + // is truthy at click time — distinguishes "picked the only detected + // agent" from "picked before detection finished." + detection_state: z.enum(['complete', 'pending']), + // `true` when the selected agent lived under the `
` disclosure + // ("Show N more"). Signals whether users go looking for less-popular + // agents — input for catalog ordering decisions. + from_collapsed_section: z.boolean(), + cohort: cohortSchema + }) + .strict() + +// Mirrors the renderer's DiscoveryState taxonomy in ThemeStep.tsx. `failed` +// is intentionally NOT a discovery state — it is the outcome of an Import +// attempt, reported by `onboarding_ghostty_import_failed`. +const ghosttyDiscoveryStateSchema = z.enum(['found', 'absent', 'imported']) + +// Compile-time guard: every member of ghosttyDiscoveryStateSchema must be a +// discovery `status` the renderer can actually emit. Adding a new +// DiscoveryState member in ThemeStep.tsx without updating the schema (or +// vice versa) breaks the build here rather than silently dropping telemetry. +type _GhosttyDiscoveryStateSync = + z.infer extends DiscoveryStatusEmitted + ? DiscoveryStatusEmitted extends z.infer + ? true + : never + : never +const _ghosttyDiscoveryStateSyncCheck: _GhosttyDiscoveryStateSync = true +void _ghosttyDiscoveryStateSyncCheck + +const onboardingGhosttyDiscoveredSchema = z + .object({ + state: ghosttyDiscoveryStateSchema, + // Bucketed, not raw, count: exact group counts are an environment + // fingerprint (heavy customizers are uniquely identifiable). Buckets + // cover the nine possible group labels in `humanFields()` without + // re-emitting the count itself. + field_group_count_bucket: z.enum(['0', '1-3', '4-7', '8+']), + cohort: cohortSchema + }) + .strict() +const onboardingGhosttyImportClickedSchema = z.object({ cohort: cohortSchema }).strict() +const onboardingGhosttyImportFailedSchema = z + .object({ + // `'no_config'` is reserved for a future explicit "preview returned + // found:false" branch. Today's call sites emit `'empty_diff'` (the + // import resolved to no changes) or `'unknown'` (caught throw). + reason: z.enum(['no_config', 'empty_diff', 'unknown']), + cohort: cohortSchema + }) + .strict() + // ── Event registry: the one record the validator consumes ─────────────── // // The validator does `eventSchemas[name].safeParse(props)`. `EventMap` is @@ -366,6 +475,10 @@ export const eventSchemas = { onboarding_step4_path_failed: onboardingStep4PathFailedSchema, onboarding_completed: onboardingCompletedSchema, onboarding_dismissed: onboardingDismissedSchema, + onboarding_agent_picked: onboardingAgentPickedSchema, + onboarding_ghostty_discovered: onboardingGhosttyDiscoveredSchema, + onboarding_ghostty_import_clicked: onboardingGhosttyImportClickedSchema, + onboarding_ghostty_import_failed: onboardingGhosttyImportFailedSchema, activation_checklist_item_completed: activationChecklistItemCompletedSchema } as const @@ -373,6 +486,20 @@ export type EventMap = { [N in keyof typeof eventSchemas]: z.infer<(typeof event export type EventName = keyof EventMap export type EventProps = EventMap[N] +// Why: events whose schemas declare a given property name. Extracted so the +// cast (Object.entries → [EventName, ZodTypeAny]) stays in one place; if the +// schema-registry shape ever changes, only one site needs to update. +// Safely skips non-`ZodObject` schemas (e.g. a future `z.discriminatedUnion` +// or `z.union`) — those have no `.shape`, and probing `key in undefined` +// would throw at module load and take the telemetry module down on import. +function eventsWithShapeKey(key: string): ReadonlySet { + return new Set( + (Object.entries(eventSchemas) as [EventName, z.ZodTypeAny][]) + .filter(([, schema]) => schema instanceof z.ZodObject && key in schema.shape) + .map(([name]) => name) + ) +} + // Events whose schemas declare `nth_repo_added`. Derived from `eventSchemas` // at module load by probing each schema's `.shape` — there is no parallel // hand-maintained list to drift out of sync. The IPC `telemetry:track` @@ -384,18 +511,85 @@ export type EventProps = EventMap[N] // Schema-additions checklist for adding a new cohort-extended event: // add `nth_repo_added: nthRepoAddedSchema` to the event's schema above. // That is the *only* step — this set updates automatically. -const COHORT_EXTENDED_SET: ReadonlySet = new Set( - (Object.entries(eventSchemas) as [EventName, z.ZodObject][]) - .filter(([, schema]) => 'nth_repo_added' in schema.shape) - .map(([name]) => name) -) +const COHORT_EXTENDED_SET = eventsWithShapeKey('nth_repo_added') export const COHORT_EXTENDED: readonly EventName[] = Array.from(COHORT_EXTENDED_SET) -export type CohortExtendedEvent = EventName -export function isCohortExtendedEvent(name: EventName): name is CohortExtendedEvent { +// Compile-time roster of events that must declare `nth_repo_added`. Same +// rationale as `_OnboardingCohortRosterSync` below — guards the runtime +// injection set against silent schema drift. +type _CohortExtendedRoster = + | 'app_opened' + | 'repo_added' + | 'add_repo_setup_step_action' + | 'workspace_created' + | 'workspace_create_failed' + | 'agent_started' + | 'agent_error' +type _DerivedCohortExtendedEvents = { + [N in EventName]: 'nth_repo_added' extends keyof EventMap[N] ? N : never +}[EventName] +type _CohortExtendedRosterSync = + _CohortExtendedRoster extends _DerivedCohortExtendedEvents + ? _DerivedCohortExtendedEvents extends _CohortExtendedRoster + ? true + : never + : never +const _cohortExtendedRosterSyncCheck: _CohortExtendedRosterSync = true +void _cohortExtendedRosterSyncCheck + +export function isCohortExtendedEvent(name: EventName): boolean { return COHORT_EXTENDED_SET.has(name) } +// Onboarding events — derived the same way as `COHORT_EXTENDED_SET`: probe +// each schema's `.shape` for the `cohort` key. The IPC `telemetry:track` +// handler injects the onboarding cohort property only when the incoming +// event name is in this set; schemas are `.strict()`, so injecting `cohort` +// on an event whose schema does not declare it would fail validation and +// silently drop the entire event. +// +// Adding a new onboarding event: include `cohort: cohortSchema` on its +// schema. This set updates automatically. +const ONBOARDING_COHORT_SET = eventsWithShapeKey('cohort') +// `NonNullable` strips `undefined` introduced by `cohortSchema`'s `.optional()`. +export type OnboardingCohort = NonNullable> + +// Compile-time roster of events that must declare `cohort`. If a schema +// refactor drops the field from one of these, this fails tsc rather than +// silently dropping the event from the runtime injection set above (which +// the `.optional()` schema would tolerate without any test failure). +// +// Adding a new onboarding event: add its name here AND declare +// `cohort: cohortSchema` on its schema. Both are required. +type _OnboardingCohortRoster = + | 'onboarding_started' + | 'onboarding_step_viewed' + | 'onboarding_step_completed' + | 'onboarding_step_skipped' + | 'onboarding_step4_path_clicked' + | 'onboarding_step4_path_failed' + | 'onboarding_completed' + | 'onboarding_dismissed' + | 'onboarding_agent_picked' + | 'onboarding_ghostty_discovered' + | 'onboarding_ghostty_import_clicked' + | 'onboarding_ghostty_import_failed' +type _DerivedOnboardingCohortEvents = { + [N in EventName]: 'cohort' extends keyof EventMap[N] ? N : never +}[EventName] +type _OnboardingCohortRosterSync = + _OnboardingCohortRoster extends _DerivedOnboardingCohortEvents + ? _DerivedOnboardingCohortEvents extends _OnboardingCohortRoster + ? true + : never + : never +const _onboardingCohortRosterSyncCheck: _OnboardingCohortRosterSync = true +void _onboardingCohortRosterSyncCheck + +export function isOnboardingEvent(name: EventName): boolean { + return ONBOARDING_COHORT_SET.has(name) +} + // Common props attached by the client — declared here so the validator knows // which keys to allow on every outgoing event. // diff --git a/src/shared/types.ts b/src/shared/types.ts index e5a59a7b9..715129a16 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -1297,6 +1297,14 @@ export type GhosttyImportPreview = { error?: string } +// Subset of the renderer's onboarding-step Ghostty `DiscoveryState['status']` +// values that ever ship a telemetry event. The UI-only states (`'idle'`, +// `'detecting'`) never fire `onboarding_ghostty_discovered`. Lives in +// `shared/` because the schema in `telemetry-events.ts` (node-tsconfig) and +// `ThemeStep.tsx` (web-tsconfig) both need it for the compile-time +// schema-vs-renderer enum sync guard. +export type DiscoveryStatusEmitted = 'found' | 'absent' | 'imported' + export type NotificationEventSource = 'agent-task-complete' | 'terminal-bell' | 'test' export type NotificationDispatchRequest = {