From 71a3bba90dcfa44949f59d2559dc3209b0c986da Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Mon, 18 May 2026 18:45:20 -0400 Subject: [PATCH] Improve voice dictation startup reliability (#2267) Co-authored-by: Orca --- src/main/ipc/speech.ts | 195 +++++---- src/main/speech/stt-service.test.ts | 110 +++++- src/main/speech/stt-service.ts | 30 +- src/preload/api-types.ts | 28 +- src/preload/index.ts | 48 ++- .../dictation/DictationController.tsx | 371 +++++++++--------- .../dictation/DictationIndicator.tsx | 17 +- .../dictation-final-segments.test.ts | 21 + .../dictation/dictation-final-segments.ts | 46 +++ .../dictation/dictation-insertion-target.ts | 92 +++++ .../dictation/dictation-stopped-sessions.ts | 24 ++ src/renderer/src/hooks/use-audio-capture.ts | 314 +++++++++++---- src/shared/speech-types.ts | 14 + 13 files changed, 925 insertions(+), 385 deletions(-) create mode 100644 src/renderer/src/components/dictation/dictation-final-segments.test.ts create mode 100644 src/renderer/src/components/dictation/dictation-final-segments.ts create mode 100644 src/renderer/src/components/dictation/dictation-insertion-target.ts create mode 100644 src/renderer/src/components/dictation/dictation-stopped-sessions.ts diff --git a/src/main/ipc/speech.ts b/src/main/ipc/speech.ts index 2454e686a..957ae1b02 100644 --- a/src/main/ipc/speech.ts +++ b/src/main/ipc/speech.ts @@ -46,96 +46,129 @@ export function registerSpeechHandlers(store: Store): void { return join(app.getPath('userData'), `speech-hotwords-${digest}.txt`) } - ipcMain.handle('speech:startDictation', async (event, modelId: string, hotwords?: string[]) => { - const window = BrowserWindow.fromWebContents(event.sender) - if (!window) { - return - } - let resolvedHotwordsPath: string | undefined - const cleanupOnWindowClosed = (): void => { - void getSpeechSttService(store) - .stopDictation('desktop') - .finally(() => { + const getDesktopOwner = (senderId: number, sessionId: string): string => + `desktop:${senderId}:${sessionId}` + + ipcMain.handle( + 'speech:startDictation', + async (event, modelId: string, hotwords?: string[], sessionId = 'desktop') => { + const window = BrowserWindow.fromWebContents(event.sender) + if (!window) { + return + } + let resolvedHotwordsPath: string | undefined + let windowClosed = false + const owner = getDesktopOwner(event.sender.id, sessionId) + const cleanupOnWindowClosed = (): void => { + windowClosed = true + void getSpeechSttService(store) + .stopDictation(owner) + .finally(() => { + if (resolvedHotwordsPath) { + unlink(resolvedHotwordsPath).catch(() => {}) + } + }) + .catch(() => {}) + } + const cleanupSessionListener = (): void => { + window.off('closed', cleanupOnWindowClosed) + } + window.once('closed', cleanupOnWindowClosed) + + try { + // Why: on macOS, the Electron binary needs explicit TCC permission for + // the microphone. Without it, getUserMedia succeeds but returns a silent + // stream (all zeros). Check status and attempt to trigger the system + // permission prompt if not yet granted. + if (process.platform === 'darwin') { + const micStatus = systemPreferences.getMediaAccessStatus('microphone') + if (micStatus !== 'granted') { + await systemPreferences.askForMediaAccess('microphone') + const newStatus = systemPreferences.getMediaAccessStatus('microphone') + if (newStatus !== 'granted') { + throw new Error( + 'Microphone access not granted. In System Settings > Privacy & Security > Microphone, ' + + 'click "+" and add the Electron app, then restart Orca.' + ) + } + } + } + + if (hotwords && hotwords.length > 0) { + const content = `${hotwords.map((w) => `${w} :2.0`).join('\n')}\n` + const hotwordsFilePath = getHotwordsFilePath(content) + await writeFile(hotwordsFilePath, content, 'utf-8') + resolvedHotwordsPath = hotwordsFilePath + } + + if (windowClosed || window.isDestroyed()) { + cleanupSessionListener() if (resolvedHotwordsPath) { unlink(resolvedHotwordsPath).catch(() => {}) } - }) - } - window.once('closed', cleanupOnWindowClosed) - - // Why: on macOS, the Electron binary needs explicit TCC permission for - // the microphone. Without it, getUserMedia succeeds but returns a silent - // stream (all zeros). Check status and attempt to trigger the system - // permission prompt if not yet granted. - if (process.platform === 'darwin') { - const micStatus = systemPreferences.getMediaAccessStatus('microphone') - if (micStatus !== 'granted') { - await systemPreferences.askForMediaAccess('microphone') - const newStatus = systemPreferences.getMediaAccessStatus('microphone') - if (newStatus !== 'granted') { - throw new Error( - 'Microphone access not granted. In System Settings > Privacy & Security > Microphone, ' + - 'click "+" and add the Electron app, then restart Orca.' - ) + return } + + await getSpeechSttService(store).startDictation( + modelId, + (msg) => { + if (window.isDestroyed()) { + return + } + switch (msg.type) { + case 'ready': + window.webContents.send('speech:ready', { sessionId }) + break + case 'partial': + window.webContents.send('speech:partial', { text: msg.text ?? '', sessionId }) + break + case 'final': + window.webContents.send('speech:final', { text: msg.text ?? '', sessionId }) + break + case 'stopped': + cleanupSessionListener() + window.webContents.send('speech:stopped', { sessionId }) + break + case 'error': + window.webContents.send('speech:error', { error: msg.error ?? '', sessionId }) + void getSpeechSttService(store) + .stopDictation(owner) + .catch(() => undefined) + .finally(cleanupSessionListener) + break + } + }, + resolvedHotwordsPath, + owner + ) + if (resolvedHotwordsPath) { + unlink(resolvedHotwordsPath).catch(() => {}) + } + } catch (err) { + cleanupSessionListener() + if (resolvedHotwordsPath) { + unlink(resolvedHotwordsPath).catch(() => {}) + } + throw err } } + ) - if (hotwords && hotwords.length > 0) { - const content = `${hotwords.map((w) => `${w} :2.0`).join('\n')}\n` - const hotwordsFilePath = getHotwordsFilePath(content) - await writeFile(hotwordsFilePath, content, 'utf-8') - resolvedHotwordsPath = hotwordsFilePath - } - - try { - await getSpeechSttService(store).startDictation( - modelId, - (msg) => { - if (window.isDestroyed()) { - return - } - switch (msg.type) { - case 'ready': - window.webContents.send('speech:ready') - break - case 'partial': - window.webContents.send('speech:partial', msg.text) - break - case 'final': - window.webContents.send('speech:final', msg.text) - break - case 'stopped': - window.webContents.send('speech:stopped') - break - case 'error': - window.webContents.send('speech:error', msg.error) - break - } - }, - resolvedHotwordsPath, - 'desktop' + ipcMain.handle( + 'speech:feedAudio', + async (_event, buffer: Buffer, sampleRate: number, sessionId = 'desktop') => { + // Why: the preload sends audio as a Buffer to avoid Float32Array data + // being zeroed out during contextBridge + IPC serialization. + const samples = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) + getSpeechSttService(store).feedAudio( + samples, + sampleRate, + getDesktopOwner(_event.sender.id, sessionId) ) - if (resolvedHotwordsPath) { - unlink(resolvedHotwordsPath).catch(() => {}) - } - } catch (err) { - window.off('closed', cleanupOnWindowClosed) - if (resolvedHotwordsPath) { - unlink(resolvedHotwordsPath).catch(() => {}) - } - throw err } - }) + ) - ipcMain.handle('speech:feedAudio', async (_event, buffer: Buffer, sampleRate: number) => { - // Why: the preload sends audio as a Buffer to avoid Float32Array data - // being zeroed out during contextBridge + IPC serialization. - const samples = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4) - getSpeechSttService(store).feedAudio(samples, sampleRate, 'desktop') - }) - - ipcMain.handle('speech:stopDictation', async () => { - await getSpeechSttService(store).stopDictation('desktop') + ipcMain.handle('speech:stopDictation', async (_event, sessionId = 'desktop') => { + await getSpeechSttService(store).stopDictation(getDesktopOwner(_event.sender.id, sessionId)) }) } diff --git a/src/main/speech/stt-service.test.ts b/src/main/speech/stt-service.test.ts index dee0d649b..6bd82d3c2 100644 --- a/src/main/speech/stt-service.test.ts +++ b/src/main/speech/stt-service.test.ts @@ -4,8 +4,11 @@ const { MockWorker, getCreatedWorkerCount, getLastWorker, resetWorkers } = vi.ho class HoistedMockWorker extends EventTarget { static created = 0 static instances: HoistedMockWorker[] = [] + static emitReadyOnInit = true terminated = false emitStoppedOnStop = true + emitReadyOnInit = HoistedMockWorker.emitReadyOnInit + messages: WorkerMessage[] = [] private listeners = new Map void>>() constructor(_path: string, _options: unknown) { @@ -38,7 +41,8 @@ const { MockWorker, getCreatedWorkerCount, getLastWorker, resetWorkers } = vi.ho } postMessage(message: WorkerMessage): void { - if (message.type === 'init') { + this.messages.push(message) + if (message.type === 'init' && this.emitReadyOnInit) { queueMicrotask(() => this.emit('message', { type: 'ready' })) } if (message.type === 'stop' && this.emitStoppedOnStop) { @@ -63,6 +67,7 @@ const { MockWorker, getCreatedWorkerCount, getLastWorker, resetWorkers } = vi.ho resetWorkers: () => { HoistedMockWorker.created = 0 HoistedMockWorker.instances = [] + HoistedMockWorker.emitReadyOnInit = true } } }) @@ -89,7 +94,7 @@ vi.mock('./model-catalog', () => ({ }) })) -import { SttService } from './stt-service' +import { IDLE_WORKER_TEARDOWN_MS, SttService } from './stt-service' describe('SttService', () => { beforeEach(() => { @@ -109,6 +114,107 @@ describe('SttService', () => { expect(getCreatedWorkerCount()).toBe(1) }) + it('keeps an idle worker warm for an hour', async () => { + vi.useFakeTimers() + try { + const service = new SttService({ + getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }), + getModelDir: vi.fn().mockReturnValue('/tmp/model-a') + } as never) + + await service.startDictation('model-a', vi.fn(), undefined, 'desktop') + const worker = getLastWorker() + expect(worker).toBeDefined() + + await service.stopDictation('desktop') + await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 1) + expect(worker!.terminated).toBe(false) + + await vi.advanceTimersByTimeAsync(IDLE_WORKER_TEARDOWN_MS - 5 * 60 * 1000 - 1) + expect(worker!.terminated).toBe(true) + } finally { + vi.useRealTimers() + } + }) + + it('resets the idle teardown timer after each stop', async () => { + vi.useFakeTimers() + try { + const service = new SttService({ + getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }), + getModelDir: vi.fn().mockReturnValue('/tmp/model-a') + } as never) + + await service.startDictation('model-a', vi.fn(), undefined, 'desktop') + const worker = getLastWorker() + expect(worker).toBeDefined() + + await service.stopDictation('desktop') + await vi.advanceTimersByTimeAsync(IDLE_WORKER_TEARDOWN_MS / 2) + await service.startDictation('model-a', vi.fn(), undefined, 'desktop') + await service.stopDictation('desktop') + + await vi.advanceTimersByTimeAsync(IDLE_WORKER_TEARDOWN_MS / 2 + 1) + expect(worker!.terminated).toBe(false) + + await vi.advanceTimersByTimeAsync(IDLE_WORKER_TEARDOWN_MS / 2 - 1) + expect(worker!.terminated).toBe(true) + } finally { + vi.useRealTimers() + } + }) + + it('drops stale audio while the worker is warm but no dictation owner is active', async () => { + const service = new SttService({ + getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }), + getModelDir: vi.fn().mockReturnValue('/tmp/model-a') + } as never) + + await service.startDictation('model-a', vi.fn(), undefined, 'desktop:1') + const worker = getLastWorker() + expect(worker).toBeDefined() + + await service.stopDictation('desktop:1') + service.feedAudio(new Float32Array([1]), 16000, 'desktop:1') + + expect(worker!.messages.filter((message) => message.type === 'feed')).toHaveLength(0) + }) + + it('keeps startup cancellation tombstoned after the worker has been created', async () => { + const service = new SttService({ + getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }), + getModelDir: vi.fn().mockReturnValue('/tmp/model-a') + } as never) + + MockWorker.emitReadyOnInit = false + const startPromise = service.startDictation('model-a', vi.fn(), undefined, 'desktop:1') + await Promise.resolve() + const worker = getLastWorker() + expect(worker).toBeDefined() + + await service.stopDictation('desktop:1') + worker!.emit('message', { type: 'ready' }) + + await expect(startPromise).rejects.toThrow('dictation_canceled') + await expect(service.startDictation('model-a', vi.fn(), undefined, 'desktop:2')).resolves.toBe( + undefined + ) + }) + + it('does not treat internal warm-worker replacement as startup cancellation', async () => { + const service = new SttService({ + getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }), + getModelDir: vi.fn().mockReturnValue('/tmp/model-a') + } as never) + + await service.startDictation('model-a', vi.fn(), '/tmp/hotwords-a.txt', 'desktop:1') + await service.stopDictation('desktop:1') + + await expect( + service.startDictation('model-a', vi.fn(), '/tmp/hotwords-b.txt', 'desktop:1') + ).resolves.toBe(undefined) + }) + it('allows slow offline stop decoding before terminating the worker', async () => { vi.useFakeTimers() try { diff --git a/src/main/speech/stt-service.ts b/src/main/speech/stt-service.ts index 59b908120..e31218645 100644 --- a/src/main/speech/stt-service.ts +++ b/src/main/speech/stt-service.ts @@ -5,6 +5,7 @@ import { getCatalogModel } from './model-catalog' import type { ModelManager } from './model-manager' const STOP_DICTATION_TIMEOUT_MS = 60_000 +export const IDLE_WORKER_TEARDOWN_MS = 60 * 60 * 1000 export type SttEvent = | { type: 'ready' } @@ -53,7 +54,7 @@ export class SttService { try { await this._startDictation(modelId, sink, hotwordsFilePath, owner) if (this.canceledOwners.delete(owner)) { - await this.stopDictation(owner) + await this.stopDictation(owner, { cancelStarting: false }) throw new Error('dictation_canceled') } this.activeOwner = owner @@ -81,7 +82,7 @@ export class SttService { } if (this.worker) { - await this.stopDictation(owner) + await this.stopDictation(owner, { cancelStarting: false }) await this.teardownIdleWorker() } @@ -204,17 +205,23 @@ export class SttService { feedAudio(samples: Float32Array, sampleRate: number, owner = 'desktop'): void { const currentOwner = this.activeOwner ?? this.startingOwner - if (currentOwner && currentOwner !== owner) { + if (!currentOwner) { + return + } + if (currentOwner !== owner) { throw new Error('dictation_owner_mismatch') } this.worker?.postMessage({ type: 'feed', samples, sampleRate }, [samples.buffer as ArrayBuffer]) } - async stopDictation(owner = 'desktop'): Promise { + async stopDictation( + owner = 'desktop', + options: { cancelStarting?: boolean } = { cancelStarting: true } + ): Promise { + if (options.cancelStarting !== false && this.startingOwner === owner) { + this.canceledOwners.add(owner) + } if (!this.worker) { - if (this.startingOwner === owner) { - this.canceledOwners.add(owner) - } return } const currentOwner = this.activeOwner ?? this.startingOwner @@ -275,12 +282,9 @@ export class SttService { // Why: keep the native recognizer warm for repeated dictations, but release // the ONNX model after a quiet period so long-running Orca sessions don't // pin speech memory forever. - this.idleTeardownTimer = setTimeout( - () => { - void this.teardownIdleWorker() - }, - 5 * 60 * 1000 - ) + this.idleTeardownTimer = setTimeout(() => { + void this.teardownIdleWorker() + }, IDLE_WORKER_TEARDOWN_MS) this.idleTeardownTimer.unref?.() } diff --git a/src/preload/api-types.ts b/src/preload/api-types.ts index 40ef93f6f..919cf1610 100644 --- a/src/preload/api-types.ts +++ b/src/preload/api-types.ts @@ -206,7 +206,13 @@ import type { ClaudeUsageSummary } from '../shared/claude-usage-types' import type { RateLimitState } from '../shared/rate-limit-types' -import type { SpeechModelManifest, SpeechModelState } from '../shared/speech-types' +import type { + SpeechErrorEvent, + SpeechLifecycleEvent, + SpeechModelManifest, + SpeechModelState, + SpeechTranscriptEvent +} from '../shared/speech-types' import type { WorkspaceSpaceAnalyzeResult, WorkspaceSpaceScanProgress @@ -1910,17 +1916,21 @@ export type PreloadApi = { downloadModel: (modelId: string) => Promise cancelDownload: (modelId: string) => Promise deleteModel: (modelId: string) => Promise - startDictation: (modelId: string, hotwords?: string[]) => Promise - feedAudio: (samples: Float32Array, sampleRate: number) => Promise - stopDictation: () => Promise - onPartialTranscript: (callback: (text: string) => void) => () => void - onFinalTranscript: (callback: (text: string) => void) => () => void + startDictation: ( + modelId: string, + hotwords: string[] | undefined, + sessionId: string + ) => Promise + feedAudio: (samples: Float32Array, sampleRate: number, sessionId?: string) => Promise + stopDictation: (sessionId?: string) => Promise + onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void + onFinalTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void onDownloadProgress: ( callback: (data: { modelId: string; progress: number }) => void ) => () => void - onReady: (callback: () => void) => () => void - onStopped: (callback: () => void) => () => void - onError: (callback: (error: string) => void) => () => void + onReady: (callback: (data: SpeechLifecycleEvent) => void) => () => void + onStopped: (callback: (data: SpeechLifecycleEvent) => void) => () => void + onError: (callback: (data: SpeechErrorEvent) => void) => () => void } } diff --git a/src/preload/index.ts b/src/preload/index.ts index 174247743..0cca2e304 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -97,7 +97,13 @@ import type { AgentStatusIpcPayload, MigrationUnsupportedPtyEntry } from '../shared/agent-status-types' -import type { SpeechModelManifest, SpeechModelState } from '../shared/speech-types' +import type { + SpeechErrorEvent, + SpeechLifecycleEvent, + SpeechModelManifest, + SpeechModelState, + SpeechTranscriptEvent +} from '../shared/speech-types' import type { TelemetryConsentState } from '../shared/telemetry-consent-types' import type { RefreshAgentsResult } from './api-types' import type { AgentKind, LaunchSource, RequestKind } from '../shared/telemetry-events' @@ -3030,25 +3036,32 @@ const api = { ipcRenderer.invoke('speech:cancelDownload', modelId), deleteModel: (modelId: string): Promise => ipcRenderer.invoke('speech:deleteModel', modelId), - startDictation: (modelId: string, hotwords?: string[]): Promise => - ipcRenderer.invoke('speech:startDictation', modelId, hotwords), - feedAudio: (samples: Float32Array, sampleRate: number): Promise => + startDictation: ( + modelId: string, + hotwords: string[] | undefined, + sessionId: string + ): Promise => ipcRenderer.invoke('speech:startDictation', modelId, hotwords, sessionId), + feedAudio: (samples: Float32Array, sampleRate: number, sessionId = 'desktop'): Promise => // Why: Float32Array data gets zeroed out when crossing the contextBridge // + IPC boundary. Wrapping in a Buffer preserves the raw bytes reliably. ipcRenderer.invoke( 'speech:feedAudio', Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength), - sampleRate + sampleRate, + sessionId ), - stopDictation: (): Promise => ipcRenderer.invoke('speech:stopDictation'), + stopDictation: (sessionId = 'desktop'): Promise => + ipcRenderer.invoke('speech:stopDictation', sessionId), - onPartialTranscript: (callback: (text: string) => void): (() => void) => { - const listener = (_event: Electron.IpcRendererEvent, text: string): void => callback(text) + onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void): (() => void) => { + const listener = (_event: Electron.IpcRendererEvent, data: SpeechTranscriptEvent): void => + callback(data) ipcRenderer.on('speech:partial', listener) return () => ipcRenderer.removeListener('speech:partial', listener) }, - onFinalTranscript: (callback: (text: string) => void): (() => void) => { - const listener = (_event: Electron.IpcRendererEvent, text: string): void => callback(text) + onFinalTranscript: (callback: (data: SpeechTranscriptEvent) => void): (() => void) => { + const listener = (_event: Electron.IpcRendererEvent, data: SpeechTranscriptEvent): void => + callback(data) ipcRenderer.on('speech:final', listener) return () => ipcRenderer.removeListener('speech:final', listener) }, @@ -3062,18 +3075,21 @@ const api = { ipcRenderer.on('speech:downloadProgress', listener) return () => ipcRenderer.removeListener('speech:downloadProgress', listener) }, - onReady: (callback: () => void): (() => void) => { - const listener = (): void => callback() + onReady: (callback: (data: SpeechLifecycleEvent) => void): (() => void) => { + const listener = (_event: Electron.IpcRendererEvent, data: SpeechLifecycleEvent): void => + callback(data) ipcRenderer.on('speech:ready', listener) return () => ipcRenderer.removeListener('speech:ready', listener) }, - onStopped: (callback: () => void): (() => void) => { - const listener = (): void => callback() + onStopped: (callback: (data: SpeechLifecycleEvent) => void): (() => void) => { + const listener = (_event: Electron.IpcRendererEvent, data: SpeechLifecycleEvent): void => + callback(data) ipcRenderer.on('speech:stopped', listener) return () => ipcRenderer.removeListener('speech:stopped', listener) }, - onError: (callback: (error: string) => void): (() => void) => { - const listener = (_event: Electron.IpcRendererEvent, error: string): void => callback(error) + onError: (callback: (data: SpeechErrorEvent) => void): (() => void) => { + const listener = (_event: Electron.IpcRendererEvent, data: SpeechErrorEvent): void => + callback(data) ipcRenderer.on('speech:error', listener) return () => ipcRenderer.removeListener('speech:error', listener) } diff --git a/src/renderer/src/components/dictation/DictationController.tsx b/src/renderer/src/components/dictation/DictationController.tsx index dccbff250..00a578999 100644 --- a/src/renderer/src/components/dictation/DictationController.tsx +++ b/src/renderer/src/components/dictation/DictationController.tsx @@ -3,84 +3,73 @@ import { useAppStore } from '@/store' import { useAudioCapture } from '@/hooks/use-audio-capture' import { toast } from 'sonner' import { DictationIndicator } from './DictationIndicator' +import { + captureInsertionTarget, + insertText, + type DictationInsertionTarget +} from './dictation-insertion-target' +import { formatFinalTranscriptSegment } from './dictation-final-segments' +import { waitForStoppedSession } from './dictation-stopped-sessions' const IS_MAC = navigator.userAgent.includes('Mac') -type DictationInsertionTarget = - | { kind: 'terminal'; tabId: string; paneId: number } - | { kind: 'text'; element: HTMLInputElement | HTMLTextAreaElement } - | { kind: 'contentEditable'; element: HTMLElement } - -// Why: splits compound identifiers into space-separated lowercase words -// so hotwords match natural speech (e.g., "DictationController" → "dictation controller"). -function splitIdentifier(name: string): string | undefined { - // PascalCase / camelCase → split on uppercase boundaries - // kebab-case / snake_case → split on - or _ - const words = name - .replace(/([a-z0-9])([A-Z])/g, '$1 $2') - .replace(/[-_]+/g, ' ') - .toLowerCase() - .trim() - return words.includes(' ') ? words : undefined -} - -function collectHotwords(): string[] { - const state = useAppStore.getState() - const seen = new Set() - const hotwords: string[] = [] - - const addWord = (word: string): void => { - if (word && !seen.has(word)) { - seen.add(word) - hotwords.push(word) - } - } - - const addFileHotwords = (relativePath: string): void => { - const basename = relativePath.split('/').pop() ?? '' - addWord(basename) - const dotIdx = basename.lastIndexOf('.') - const nameOnly = dotIdx > 0 ? basename.slice(0, dotIdx) : basename - if (dotIdx > 0) { - addWord(nameOnly) - } - const spoken = splitIdentifier(nameOnly) - if (spoken) { - addWord(spoken) - } - } - - for (const file of state.openFiles) { - if (file.relativePath) { - addFileHotwords(file.relativePath) - } - } - - const worktreeId = state.activeWorktreeId - if (worktreeId) { - const closed = state.recentlyClosedEditorTabsByWorktree[worktreeId] ?? [] - for (const snap of closed) { - if (snap.relativePath) { - addFileHotwords(snap.relativePath) - } - } - } - - return hotwords.slice(0, 50) -} - export function DictationController() { const dictationState = useAppStore((s) => s.dictationState) const setDictationState = useAppStore((s) => s.setDictationState) const setPartialTranscript = useAppStore((s) => s.setPartialTranscript) const settings = useAppStore((s) => s.settings) - const { start: startCapture, stop: stopCapture } = useAudioCapture() + const { + start: startCapture, + stop: stopCapture, + flushBufferedAudio, + discardBufferedAudio, + getCapturedChunkCount + } = useAudioCapture() const dictationStateRef = useRef(dictationState) dictationStateRef.current = dictationState const dictationRunRef = useRef(0) const holdGestureActiveRef = useRef(false) const insertionTargetRef = useRef(null) + const activeSessionIdRef = useRef(null) + const stoppedSessionIdsRef = useRef(new Set()) + const stoppedResolversRef = useRef(new Map void>()) + const stopRequestedDuringStartRef = useRef(false) + const finalTranscriptReceivedRef = useRef(false) + const intentionalTargetCancellationRef = useRef(false) + const insertedFinalTranscriptRef = useRef('') + + const finishDictationSession = useCallback( + async (sessionId: string) => { + dictationStateRef.current = 'stopping' + setDictationState('stopping') + stopCapture() + try { + await window.api.speech.stopDictation(sessionId) + } catch { + // Swallow stop errors — the worker may already be torn down. + } + // Why: stopDictation() resolves on main-process completion, while final + // transcript delivery is renderer IPC. Wait for this session's stopped + // event so old finals cannot be mistaken for the next dictation run. + await waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef) + if (!finalTranscriptReceivedRef.current && getCapturedChunkCount() > 0) { + toast.message('No speech detected.') + } + insertionTargetRef.current = null + finalTranscriptReceivedRef.current = false + insertedFinalTranscriptRef.current = '' + intentionalTargetCancellationRef.current = false + stopRequestedDuringStartRef.current = false + if (activeSessionIdRef.current === sessionId) { + activeSessionIdRef.current = null + } + dictationStateRef.current = 'idle' + setDictationState('idle') + setPartialTranscript('') + }, + [setDictationState, setPartialTranscript, stopCapture, getCapturedChunkCount] + ) const startDictation = useCallback(async () => { if (dictationStateRef.current !== 'idle') { @@ -107,41 +96,82 @@ export function DictationController() { } const runId = dictationRunRef.current + 1 + const sessionId = String(runId) dictationRunRef.current = runId + activeSessionIdRef.current = sessionId insertionTargetRef.current = captureInsertionTarget() + stopRequestedDuringStartRef.current = false + finalTranscriptReceivedRef.current = false + insertedFinalTranscriptRef.current = '' + intentionalTargetCancellationRef.current = false dictationStateRef.current = 'starting' setDictationState('starting') - const hotwords = collectHotwords() - let speechStarted = false + let captureStarted = false try { - await window.api.speech.startDictation(modelId, hotwords.length > 0 ? hotwords : undefined) - speechStarted = true + // Why: worker startup can take seconds after idle teardown. Capture first + // and buffer locally so speech during "Starting..." is not discarded. + await startCapture({ bufferAudio: true, sessionId }) + captureStarted = true + if (stopRequestedDuringStartRef.current) { + stopCapture({ preserveBufferedAudio: true }) + } if (dictationRunRef.current !== runId) { + discardBufferedAudio() + stopCapture() insertionTargetRef.current = null - await window.api.speech.stopDictation().catch(() => undefined) return } - await startCapture() + + await window.api.speech.startDictation(modelId, undefined, sessionId) if (dictationRunRef.current !== runId) { + discardBufferedAudio() insertionTargetRef.current = null stopCapture() - await window.api.speech.stopDictation().catch(() => undefined) + await window.api.speech.stopDictation(sessionId).catch(() => undefined) return } + + await flushBufferedAudio() + if (dictationRunRef.current !== runId) { + discardBufferedAudio() + insertionTargetRef.current = null + stopCapture() + await window.api.speech.stopDictation(sessionId).catch(() => undefined) + return + } + if (stopRequestedDuringStartRef.current) { + await finishDictationSession(sessionId) + return + } + dictationStateRef.current = 'listening' setDictationState('listening') } catch (err) { if (dictationRunRef.current !== runId) { return } - if (speechStarted) { - await window.api.speech.stopDictation().catch(() => undefined) + await window.api.speech.stopDictation(sessionId).catch(() => undefined) + if (captureStarted) { + stopCapture() + } + discardBufferedAudio() + const message = String(err) + insertionTargetRef.current = null + intentionalTargetCancellationRef.current = false + stopRequestedDuringStartRef.current = false + finalTranscriptReceivedRef.current = false + insertedFinalTranscriptRef.current = '' + activeSessionIdRef.current = null + setPartialTranscript('') + if (message.includes('dictation_canceled')) { + dictationStateRef.current = 'idle' + setDictationState('idle') + return } dictationStateRef.current = 'error' setDictationState('error') - const message = String(err) if (message.includes('Permission') || message.includes('NotAllowed')) { toast.error('Microphone access denied. Grant access in system settings, then restart Orca.') } else if (message.includes('not ready')) { @@ -159,29 +189,39 @@ export function DictationController() { } else { toast.error(`Dictation failed: ${message}`) } - stopCapture() dictationStateRef.current = 'idle' setDictationState('idle') } - }, [settings, setDictationState, startCapture, stopCapture]) + }, [ + settings, + setDictationState, + startCapture, + flushBufferedAudio, + discardBufferedAudio, + stopCapture, + finishDictationSession, + setPartialTranscript + ]) const stopDictation = useCallback(async () => { - if (dictationStateRef.current !== 'listening' && dictationStateRef.current !== 'starting') { + if (dictationStateRef.current === 'starting') { + stopRequestedDuringStartRef.current = true + dictationStateRef.current = 'stopping' + setDictationState('stopping') + stopCapture({ preserveBufferedAudio: true }) return } - dictationRunRef.current += 1 - dictationStateRef.current = 'stopping' - setDictationState('stopping') - stopCapture() - try { - await window.api.speech.stopDictation() - } catch { - // Swallow stop errors — the worker may already be torn down. + + if (dictationStateRef.current !== 'listening') { + return } - dictationStateRef.current = 'idle' - setDictationState('idle') - setPartialTranscript('') - }, [setDictationState, setPartialTranscript, stopCapture]) + + const sessionId = activeSessionIdRef.current + if (!sessionId) { + return + } + await finishDictationSession(sessionId) + }, [finishDictationSession, setDictationState, stopCapture]) // Toggle mode: use IPC from main process (before-input-event intercepts // the keyDown so Cmd+E doesn't reach xterm or trigger system shortcuts). @@ -270,6 +310,7 @@ export function DictationController() { holdGestureActiveRef.current = false if (dictationStateRef.current !== 'idle' && dictationStateRef.current !== 'stopping') { insertionTargetRef.current = null + intentionalTargetCancellationRef.current = true void stopDictation() } } @@ -300,121 +341,75 @@ export function DictationController() { ]) useEffect(() => { - const cleanupPartial = window.api.speech.onPartialTranscript((text) => { - setPartialTranscript(text) + const cleanupPartial = window.api.speech.onPartialTranscript((data) => { + if (data.sessionId !== activeSessionIdRef.current) { + return + } + setPartialTranscript(data.text) }) - const cleanupFinal = window.api.speech.onFinalTranscript((text) => { + const cleanupFinal = window.api.speech.onFinalTranscript((data) => { + if (data.sessionId !== activeSessionIdRef.current || !data.text) { + return + } setPartialTranscript('') + finalTranscriptReceivedRef.current = true const target = insertionTargetRef.current - insertionTargetRef.current = null if (target) { - insertText(text, target) + const textToInsert = formatFinalTranscriptSegment( + data.text, + insertedFinalTranscriptRef.current + ) + insertText(textToInsert, target) + insertedFinalTranscriptRef.current += textToInsert + } else if (!intentionalTargetCancellationRef.current) { + toast.message('Dictation finished, but no text field was focused.') } }) - const cleanupError = window.api.speech.onError((error) => { - toast.error(`Speech error: ${error}`) + const cleanupStopped = window.api.speech.onStopped((data) => { + const resolver = stoppedResolversRef.current.get(data.sessionId) + if (resolver) { + stoppedResolversRef.current.delete(data.sessionId) + resolver() + return + } + stoppedSessionIdsRef.current.add(data.sessionId) + }) + + const cleanupError = window.api.speech.onError((data) => { + if (data.sessionId !== activeSessionIdRef.current) { + return + } + const sessionId = data.sessionId + dictationRunRef.current += 1 + activeSessionIdRef.current = null + toast.error(`Speech error: ${data.error}`) + dictationStateRef.current = 'stopping' + setDictationState('stopping') stopCapture() - insertionTargetRef.current = null - dictationStateRef.current = 'idle' - setDictationState('idle') - setPartialTranscript('') + discardBufferedAudio() + void (async () => { + await window.api.speech.stopDictation(sessionId).catch(() => undefined) + await waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef) + insertionTargetRef.current = null + intentionalTargetCancellationRef.current = false + stopRequestedDuringStartRef.current = false + finalTranscriptReceivedRef.current = false + insertedFinalTranscriptRef.current = '' + dictationStateRef.current = 'idle' + setDictationState('idle') + setPartialTranscript('') + })() }) return () => { cleanupPartial() cleanupFinal() + cleanupStopped() cleanupError() } - }, [setPartialTranscript, setDictationState, stopCapture]) + }, [setPartialTranscript, setDictationState, stopCapture, discardBufferedAudio]) return } - -function captureInsertionTarget(): DictationInsertionTarget | null { - const activeElement = document.activeElement - - if (!activeElement) { - return null - } - - if (activeElement.classList.contains('xterm-helper-textarea')) { - const paneElement = activeElement.closest('.pane[data-pane-id]') as HTMLElement | null - const tabElement = activeElement.closest('[data-terminal-tab-id]') as HTMLElement | null - const paneId = Number(paneElement?.dataset.paneId) - const tabId = tabElement?.dataset.terminalTabId - if (tabId && Number.isFinite(paneId)) { - return { kind: 'terminal', tabId, paneId } - } - return null - } - - if (activeElement instanceof HTMLInputElement || activeElement instanceof HTMLTextAreaElement) { - return { kind: 'text', element: activeElement } - } - - if (activeElement instanceof HTMLElement && activeElement.isContentEditable) { - return { kind: 'contentEditable', element: activeElement } - } - - return null -} - -function insertText(text: string, target: DictationInsertionTarget): void { - if (target.kind === 'terminal') { - document.dispatchEvent( - new CustomEvent('dictation:insertText', { - detail: { text, tabId: target.tabId, paneId: target.paneId } - }) - ) - return - } - - if (target.kind === 'text') { - const element = target.element - if (!element.isConnected) { - return - } - const start = element.selectionStart ?? element.value.length - const end = element.selectionEnd ?? start - element.setRangeText(text, start, end, 'end') - element.dispatchEvent(new Event('input', { bubbles: true })) - return - } - - if (target.kind === 'contentEditable') { - const element = target.element - if (!element.isConnected || !element.contains(document.activeElement)) { - return - } - const editorElement = findClosestEditorElement(element) ?? element - editorElement.dispatchEvent( - new InputEvent('beforeinput', { - bubbles: true, - cancelable: true, - inputType: 'insertText', - data: text - }) - ) - if (!document.execCommand('insertText', false, text)) { - const selection = window.getSelection() - if (selection && selection.rangeCount > 0) { - const range = selection.getRangeAt(0) - range.deleteContents() - const textNode = document.createTextNode(text) - range.insertNode(textNode) - range.setStartAfter(textNode) - range.collapse(true) - selection.removeAllRanges() - selection.addRange(range) - } - editorElement.dispatchEvent( - new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }) - ) - } - } -} -function findClosestEditorElement(element: HTMLElement): HTMLElement | null { - return element.closest('.ProseMirror, [contenteditable="true"]') -} diff --git a/src/renderer/src/components/dictation/DictationIndicator.tsx b/src/renderer/src/components/dictation/DictationIndicator.tsx index 979a30f0b..f1d026eaf 100644 --- a/src/renderer/src/components/dictation/DictationIndicator.tsx +++ b/src/renderer/src/components/dictation/DictationIndicator.tsx @@ -5,16 +5,25 @@ export function DictationIndicator() { const dictationState = useAppStore((s) => s.dictationState) const partialTranscript = useAppStore((s) => s.partialTranscript) - if (dictationState !== 'listening' && dictationState !== 'starting') { + if ( + dictationState !== 'listening' && + dictationState !== 'starting' && + dictationState !== 'stopping' + ) { return null } + const label = + dictationState === 'starting' + ? 'Starting...' + : dictationState === 'stopping' + ? 'Processing...' + : partialTranscript || 'Listening...' + return (
- - {dictationState === 'starting' ? 'Starting...' : partialTranscript || 'Listening...'} - + {label}
) } diff --git a/src/renderer/src/components/dictation/dictation-final-segments.test.ts b/src/renderer/src/components/dictation/dictation-final-segments.test.ts new file mode 100644 index 000000000..f9ee432e0 --- /dev/null +++ b/src/renderer/src/components/dictation/dictation-final-segments.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { formatFinalTranscriptSegment } from './dictation-final-segments' + +describe('formatFinalTranscriptSegment', () => { + it('adds a boundary between word-like streaming final segments', () => { + expect(formatFinalTranscriptSegment('world', 'hello')).toBe(' world') + }) + + it('adds a boundary after sentence and phrase punctuation', () => { + expect(formatFinalTranscriptSegment('World', 'Hello.')).toBe(' World') + expect(formatFinalTranscriptSegment('world', 'hello,')).toBe(' world') + }) + + it('does not add a boundary before punctuation', () => { + expect(formatFinalTranscriptSegment('.', 'hello')).toBe('.') + }) + + it('does not add a boundary around CJK final segments', () => { + expect(formatFinalTranscriptSegment('世界', '你好')).toBe('世界') + }) +}) diff --git a/src/renderer/src/components/dictation/dictation-final-segments.ts b/src/renderer/src/components/dictation/dictation-final-segments.ts new file mode 100644 index 000000000..d9ad0a68b --- /dev/null +++ b/src/renderer/src/components/dictation/dictation-final-segments.ts @@ -0,0 +1,46 @@ +const WORD_BOUNDARY_CHAR_RE = /^[\p{L}\p{N}]$/u +const CJK_BOUNDARY_CHAR_RE = + /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]$/u +const NO_SPACE_BEFORE_CHAR_RE = /^[,.;:!?%。，、！？；：）)\]}]$/u +const NO_SPACE_AFTER_CHAR_RE = /^[([{（《「『]$/u +const SPACE_AFTER_CHAR_RE = /^[,.;:!?%]$/u + +function getFirstNonWhitespaceChar(text: string): string { + return Array.from(text.trimStart())[0] ?? '' +} + +function getLastNonWhitespaceChar(text: string): string { + return Array.from(text.trimEnd()).at(-1) ?? '' +} + +function shouldInsertSpaceBetweenFinalSegments(previousText: string, nextText: string): boolean { + if (!previousText || !nextText || /\s$/.test(previousText) || /^\s/.test(nextText)) { + return false + } + + const previousChar = getLastNonWhitespaceChar(previousText) + const nextChar = getFirstNonWhitespaceChar(nextText) + if (!previousChar || !nextChar) { + return false + } + if ( + CJK_BOUNDARY_CHAR_RE.test(previousChar) || + CJK_BOUNDARY_CHAR_RE.test(nextChar) || + NO_SPACE_BEFORE_CHAR_RE.test(nextChar) || + NO_SPACE_AFTER_CHAR_RE.test(previousChar) + ) { + return false + } + + return ( + (WORD_BOUNDARY_CHAR_RE.test(previousChar) || SPACE_AFTER_CHAR_RE.test(previousChar)) && + WORD_BOUNDARY_CHAR_RE.test(nextChar) + ) +} + +export function formatFinalTranscriptSegment(text: string, previousInsertedText: string): string { + if (shouldInsertSpaceBetweenFinalSegments(previousInsertedText, text)) { + return ` ${text}` + } + return text +} diff --git a/src/renderer/src/components/dictation/dictation-insertion-target.ts b/src/renderer/src/components/dictation/dictation-insertion-target.ts new file mode 100644 index 000000000..58c21f92d --- /dev/null +++ b/src/renderer/src/components/dictation/dictation-insertion-target.ts @@ -0,0 +1,92 @@ +export type DictationInsertionTarget = + | { kind: 'terminal'; tabId: string; paneId: number } + | { kind: 'text'; element: HTMLInputElement | HTMLTextAreaElement } + | { kind: 'contentEditable'; element: HTMLElement } + +export function captureInsertionTarget(): DictationInsertionTarget | null { + const activeElement = document.activeElement + + if (!activeElement) { + return null + } + + if (activeElement.classList.contains('xterm-helper-textarea')) { + const paneElement = activeElement.closest('.pane[data-pane-id]') as HTMLElement | null + const tabElement = activeElement.closest('[data-terminal-tab-id]') as HTMLElement | null + const paneId = Number(paneElement?.dataset.paneId) + const tabId = tabElement?.dataset.terminalTabId + if (tabId && Number.isFinite(paneId)) { + return { kind: 'terminal', tabId, paneId } + } + return null + } + + if (activeElement instanceof HTMLInputElement || activeElement instanceof HTMLTextAreaElement) { + return { kind: 'text', element: activeElement } + } + + if (activeElement instanceof HTMLElement && activeElement.isContentEditable) { + return { kind: 'contentEditable', element: activeElement } + } + + return null +} + +export function insertText(text: string, target: DictationInsertionTarget): void { + if (target.kind === 'terminal') { + document.dispatchEvent( + new CustomEvent('dictation:insertText', { + detail: { text, tabId: target.tabId, paneId: target.paneId } + }) + ) + return + } + + if (target.kind === 'text') { + const element = target.element + if (!element.isConnected) { + return + } + const start = element.selectionStart ?? element.value.length + const end = element.selectionEnd ?? start + element.setRangeText(text, start, end, 'end') + element.dispatchEvent(new Event('input', { bubbles: true })) + return + } + + if (target.kind === 'contentEditable') { + const element = target.element + if (!element.isConnected || !element.contains(document.activeElement)) { + return + } + const editorElement = findClosestEditorElement(element) ?? element + editorElement.dispatchEvent( + new InputEvent('beforeinput', { + bubbles: true, + cancelable: true, + inputType: 'insertText', + data: text + }) + ) + if (!document.execCommand('insertText', false, text)) { + const selection = window.getSelection() + if (selection && selection.rangeCount > 0) { + const range = selection.getRangeAt(0) + range.deleteContents() + const textNode = document.createTextNode(text) + range.insertNode(textNode) + range.setStartAfter(textNode) + range.collapse(true) + selection.removeAllRanges() + selection.addRange(range) + } + editorElement.dispatchEvent( + new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }) + ) + } + } +} + +function findClosestEditorElement(element: HTMLElement): HTMLElement | null { + return element.closest('.ProseMirror, [contenteditable="true"]') +} diff --git a/src/renderer/src/components/dictation/dictation-stopped-sessions.ts b/src/renderer/src/components/dictation/dictation-stopped-sessions.ts new file mode 100644 index 000000000..b7e301470 --- /dev/null +++ b/src/renderer/src/components/dictation/dictation-stopped-sessions.ts @@ -0,0 +1,24 @@ +type RefLike = { current: T } + +const STOPPED_SESSION_WAIT_MS = 1000 + +export function waitForStoppedSession( + sessionId: string, + stoppedSessionIdsRef: RefLike>, + stoppedResolversRef: RefLike void>> +): Promise { + if (stoppedSessionIdsRef.current.delete(sessionId)) { + return Promise.resolve() + } + + return new Promise((resolve) => { + const timeoutId = window.setTimeout(() => { + stoppedResolversRef.current.delete(sessionId) + resolve() + }, STOPPED_SESSION_WAIT_MS) + stoppedResolversRef.current.set(sessionId, () => { + window.clearTimeout(timeoutId) + resolve() + }) + }) +} diff --git a/src/renderer/src/hooks/use-audio-capture.ts b/src/renderer/src/hooks/use-audio-capture.ts index fa2b2cb98..7348bce24 100644 --- a/src/renderer/src/hooks/use-audio-capture.ts +++ b/src/renderer/src/hooks/use-audio-capture.ts @@ -1,82 +1,39 @@ import { useRef, useCallback } from 'react' +type BufferedAudioChunk = { + samples: Float32Array + sampleRate: number + sessionId: string +} + +type StartAudioCaptureOptions = { + bufferAudio?: boolean + sessionId?: string +} + +type StopAudioCaptureOptions = { + preserveBufferedAudio?: boolean +} + +const MAX_BUFFERED_AUDIO_SECONDS = 30 +const MAX_BUFFERED_AUDIO_BYTES = 8 * 1024 * 1024 + export function useAudioCapture() { const streamRef = useRef(null) const contextRef = useRef(null) const processorRef = useRef(null) const sourceRef = useRef(null) const isCapturingRef = useRef(false) + const startRequestRef = useRef(0) + const bufferAudioRef = useRef(false) + const bufferedAudioGenerationRef = useRef(0) + const bufferedAudioRef = useRef([]) + const bufferedAudioBytesRef = useRef(0) + const bufferedAudioSecondsRef = useRef(0) + const capturedChunkCountRef = useRef(0) + const sessionIdRef = useRef('desktop') - const start = useCallback(async () => { - if (isCapturingRef.current) { - return - } - - const stream = await navigator.mediaDevices.getUserMedia({ - audio: { - channelCount: 1, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true - } - }) - streamRef.current = stream - - try { - // Why: requesting a specific sampleRate (e.g. 16kHz) in the AudioContext - // can produce silence on macOS because the hardware mic runs at 44.1/48kHz. - // Use the system default rate and let sherpa-onnx resample internally. - const context = new AudioContext() - contextRef.current = context - - // Why: some Chromium builds suspend the AudioContext until a user gesture. - // Resume it explicitly to ensure audio processing starts. - if (context.state === 'suspended') { - await context.resume() - } - - const source = context.createMediaStreamSource(stream) - - // Why: ScriptProcessorNode is deprecated but AudioWorklet requires a - // separate module file which complicates the Vite build pipeline. For - // the initial implementation, ScriptProcessorNode is simpler and the - // performance difference is negligible for speech capture. - const processor = context.createScriptProcessor(4096, 1, 1) - - const actualRate = context.sampleRate - - processor.onaudioprocess = (e: AudioProcessingEvent) => { - if (!isCapturingRef.current) { - return - } - const samples = new Float32Array(e.inputBuffer.getChannelData(0)) - window.api.speech.feedAudio(samples, actualRate) - } - - source.connect(processor) - processor.connect(context.destination) - - processorRef.current = processor - sourceRef.current = source - isCapturingRef.current = true - } catch (err) { - processorRef.current?.disconnect() - sourceRef.current?.disconnect() - processorRef.current = null - sourceRef.current = null - if (contextRef.current?.state !== 'closed') { - void contextRef.current?.close() - } - contextRef.current = null - stream.getTracks().forEach((track) => track.stop()) - streamRef.current = null - throw err - } - }, []) - - const stop = useCallback(() => { - isCapturingRef.current = false - + const cleanupCaptureResources = useCallback(() => { processorRef.current?.disconnect() sourceRef.current?.disconnect() processorRef.current = null @@ -87,9 +44,222 @@ export function useAudioCapture() { } contextRef.current = null - streamRef.current?.getTracks().forEach((t) => t.stop()) + streamRef.current?.getTracks().forEach((track) => track.stop()) streamRef.current = null }, []) - return { start, stop, isCapturingRef } + const resetBufferedAudio = useCallback(() => { + bufferedAudioGenerationRef.current += 1 + bufferedAudioRef.current = [] + bufferedAudioBytesRef.current = 0 + bufferedAudioSecondsRef.current = 0 + }, []) + + const removeOldestBufferedAudioChunk = useCallback(() => { + const chunk = bufferedAudioRef.current.shift() + if (!chunk) { + return + } + bufferedAudioBytesRef.current -= chunk.samples.byteLength + bufferedAudioSecondsRef.current -= chunk.samples.length / chunk.sampleRate + }, []) + + const appendBufferedAudioChunk = useCallback( + (chunk: BufferedAudioChunk) => { + bufferedAudioRef.current.push(chunk) + bufferedAudioBytesRef.current += chunk.samples.byteLength + bufferedAudioSecondsRef.current += chunk.samples.length / chunk.sampleRate + + // Why: worker/model startup can hang; keep only a bounded recent window + // so renderer memory cannot grow forever while buffering is enabled. + while ( + bufferedAudioRef.current.length > 0 && + (bufferedAudioBytesRef.current > MAX_BUFFERED_AUDIO_BYTES || + bufferedAudioSecondsRef.current > MAX_BUFFERED_AUDIO_SECONDS) + ) { + removeOldestBufferedAudioChunk() + } + }, + [removeOldestBufferedAudioChunk] + ) + + const start = useCallback( + async (options: StartAudioCaptureOptions = {}) => { + if (isCapturingRef.current) { + return + } + const startRequest = startRequestRef.current + 1 + startRequestRef.current = startRequest + cleanupCaptureResources() + sessionIdRef.current = options.sessionId ?? 'desktop' + bufferAudioRef.current = options.bufferAudio ?? false + resetBufferedAudio() + capturedChunkCountRef.current = 0 + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true + } + }) + if (startRequestRef.current !== startRequest) { + stream.getTracks().forEach((track) => track.stop()) + return + } + streamRef.current = stream + + let context: AudioContext | null = null + let source: MediaStreamAudioSourceNode | null = null + let processor: ScriptProcessorNode | null = null + try { + // Why: requesting a specific sampleRate (e.g. 16kHz) in the AudioContext + // can produce silence on macOS because the hardware mic runs at 44.1/48kHz. + // Use the system default rate and let sherpa-onnx resample internally. + context = new AudioContext() + contextRef.current = context + + // Why: some Chromium builds suspend the AudioContext until a user gesture. + // Resume it explicitly to ensure audio processing starts. + if (context.state === 'suspended') { + await context.resume() + } + if (startRequestRef.current !== startRequest || streamRef.current !== stream) { + if (contextRef.current === context) { + contextRef.current = null + } + if (context.state !== 'closed') { + void context.close() + } + if (streamRef.current === stream) { + streamRef.current = null + } + stream.getTracks().forEach((track) => track.stop()) + return + } + + source = context.createMediaStreamSource(stream) + + // Why: ScriptProcessorNode is deprecated but AudioWorklet requires a + // separate module file which complicates the Vite build pipeline. For + // the initial implementation, ScriptProcessorNode is simpler and the + // performance difference is negligible for speech capture. + processor = context.createScriptProcessor(4096, 1, 1) + + const actualRate = context.sampleRate + + processor.onaudioprocess = (e: AudioProcessingEvent) => { + if ( + !isCapturingRef.current || + startRequestRef.current !== startRequest || + processorRef.current !== processor + ) { + return + } + const samples = new Float32Array(e.inputBuffer.getChannelData(0)) + capturedChunkCountRef.current += 1 + if (bufferAudioRef.current) { + appendBufferedAudioChunk({ + samples, + sampleRate: actualRate, + sessionId: sessionIdRef.current + }) + return + } + void window.api.speech + .feedAudio(samples, actualRate, sessionIdRef.current) + .catch(() => undefined) + } + + source.connect(processor) + processor.connect(context.destination) + + processorRef.current = processor + sourceRef.current = source + isCapturingRef.current = true + } catch (err) { + processor?.disconnect() + source?.disconnect() + if (processorRef.current === processor) { + processorRef.current = null + } + if (sourceRef.current === source) { + sourceRef.current = null + } + if (contextRef.current === context) { + contextRef.current = null + } + if (context && context.state !== 'closed') { + void context.close() + } + stream.getTracks().forEach((track) => track.stop()) + if (streamRef.current === stream) { + streamRef.current = null + } + if (startRequestRef.current === startRequest) { + bufferAudioRef.current = false + resetBufferedAudio() + } + if (startRequestRef.current !== startRequest) { + return + } + throw err + } + }, + [appendBufferedAudioChunk, cleanupCaptureResources, resetBufferedAudio] + ) + + const flushBufferedAudio = useCallback(async () => { + const flushGeneration = bufferedAudioGenerationRef.current + try { + // Why: keep buffering enabled while draining so live audio appends behind + // startup audio instead of overtaking it through direct IPC sends. + while ( + bufferedAudioGenerationRef.current === flushGeneration && + bufferedAudioRef.current.length > 0 + ) { + const chunk = bufferedAudioRef.current[0] + if (!chunk) { + break + } + removeOldestBufferedAudioChunk() + await window.api.speech.feedAudio(chunk.samples, chunk.sampleRate, chunk.sessionId) + } + } finally { + if (bufferedAudioGenerationRef.current === flushGeneration) { + bufferAudioRef.current = false + resetBufferedAudio() + } + } + }, [removeOldestBufferedAudioChunk, resetBufferedAudio]) + + const discardBufferedAudio = useCallback(() => { + bufferAudioRef.current = false + resetBufferedAudio() + }, [resetBufferedAudio]) + + const getCapturedChunkCount = useCallback(() => capturedChunkCountRef.current, []) + + const stop = useCallback( + (options: StopAudioCaptureOptions = {}) => { + startRequestRef.current += 1 + isCapturingRef.current = false + bufferAudioRef.current = false + if (!options.preserveBufferedAudio) { + resetBufferedAudio() + } + cleanupCaptureResources() + }, + [cleanupCaptureResources, resetBufferedAudio] + ) + + return { + start, + stop, + flushBufferedAudio, + discardBufferedAudio, + getCapturedChunkCount, + isCapturingRef + } } diff --git a/src/shared/speech-types.ts b/src/shared/speech-types.ts index ae165b642..008ab1b97 100644 --- a/src/shared/speech-types.ts +++ b/src/shared/speech-types.ts @@ -28,6 +28,20 @@ export type SpeechModelState = { error?: string } +export type SpeechTranscriptEvent = { + text: string + sessionId: string +} + +export type SpeechLifecycleEvent = { + sessionId: string +} + +export type SpeechErrorEvent = { + error: string + sessionId: string +} + export type DictationState = 'idle' | 'starting' | 'listening' | 'stopping' | 'error' export type UserModelConfig = {