Improve voice dictation startup reliability (#2267)
Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Jinwoo Hong
GitHub
parent
9ad70b767c
commit
71a3bba90d
|
|
@ -46,96 +46,129 @@ export function registerSpeechHandlers(store: Store): void {
|
|||
return join(app.getPath('userData'), `speech-hotwords-${digest}.txt`)
|
||||
}
|
||||
|
||||
ipcMain.handle('speech:startDictation', async (event, modelId: string, hotwords?: string[]) => {
|
||||
const window = BrowserWindow.fromWebContents(event.sender)
|
||||
if (!window) {
|
||||
return
|
||||
}
|
||||
let resolvedHotwordsPath: string | undefined
|
||||
const cleanupOnWindowClosed = (): void => {
|
||||
void getSpeechSttService(store)
|
||||
.stopDictation('desktop')
|
||||
.finally(() => {
|
||||
const getDesktopOwner = (senderId: number, sessionId: string): string =>
|
||||
`desktop:${senderId}:${sessionId}`
|
||||
|
||||
ipcMain.handle(
|
||||
'speech:startDictation',
|
||||
async (event, modelId: string, hotwords?: string[], sessionId = 'desktop') => {
|
||||
const window = BrowserWindow.fromWebContents(event.sender)
|
||||
if (!window) {
|
||||
return
|
||||
}
|
||||
let resolvedHotwordsPath: string | undefined
|
||||
let windowClosed = false
|
||||
const owner = getDesktopOwner(event.sender.id, sessionId)
|
||||
const cleanupOnWindowClosed = (): void => {
|
||||
windowClosed = true
|
||||
void getSpeechSttService(store)
|
||||
.stopDictation(owner)
|
||||
.finally(() => {
|
||||
if (resolvedHotwordsPath) {
|
||||
unlink(resolvedHotwordsPath).catch(() => {})
|
||||
}
|
||||
})
|
||||
.catch(() => {})
|
||||
}
|
||||
const cleanupSessionListener = (): void => {
|
||||
window.off('closed', cleanupOnWindowClosed)
|
||||
}
|
||||
window.once('closed', cleanupOnWindowClosed)
|
||||
|
||||
try {
|
||||
// Why: on macOS, the Electron binary needs explicit TCC permission for
|
||||
// the microphone. Without it, getUserMedia succeeds but returns a silent
|
||||
// stream (all zeros). Check status and attempt to trigger the system
|
||||
// permission prompt if not yet granted.
|
||||
if (process.platform === 'darwin') {
|
||||
const micStatus = systemPreferences.getMediaAccessStatus('microphone')
|
||||
if (micStatus !== 'granted') {
|
||||
await systemPreferences.askForMediaAccess('microphone')
|
||||
const newStatus = systemPreferences.getMediaAccessStatus('microphone')
|
||||
if (newStatus !== 'granted') {
|
||||
throw new Error(
|
||||
'Microphone access not granted. In System Settings > Privacy & Security > Microphone, ' +
|
||||
'click "+" and add the Electron app, then restart Orca.'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hotwords && hotwords.length > 0) {
|
||||
const content = `${hotwords.map((w) => `${w} :2.0`).join('\n')}\n`
|
||||
const hotwordsFilePath = getHotwordsFilePath(content)
|
||||
await writeFile(hotwordsFilePath, content, 'utf-8')
|
||||
resolvedHotwordsPath = hotwordsFilePath
|
||||
}
|
||||
|
||||
if (windowClosed || window.isDestroyed()) {
|
||||
cleanupSessionListener()
|
||||
if (resolvedHotwordsPath) {
|
||||
unlink(resolvedHotwordsPath).catch(() => {})
|
||||
}
|
||||
})
|
||||
}
|
||||
window.once('closed', cleanupOnWindowClosed)
|
||||
|
||||
// Why: on macOS, the Electron binary needs explicit TCC permission for
|
||||
// the microphone. Without it, getUserMedia succeeds but returns a silent
|
||||
// stream (all zeros). Check status and attempt to trigger the system
|
||||
// permission prompt if not yet granted.
|
||||
if (process.platform === 'darwin') {
|
||||
const micStatus = systemPreferences.getMediaAccessStatus('microphone')
|
||||
if (micStatus !== 'granted') {
|
||||
await systemPreferences.askForMediaAccess('microphone')
|
||||
const newStatus = systemPreferences.getMediaAccessStatus('microphone')
|
||||
if (newStatus !== 'granted') {
|
||||
throw new Error(
|
||||
'Microphone access not granted. In System Settings > Privacy & Security > Microphone, ' +
|
||||
'click "+" and add the Electron app, then restart Orca.'
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
await getSpeechSttService(store).startDictation(
|
||||
modelId,
|
||||
(msg) => {
|
||||
if (window.isDestroyed()) {
|
||||
return
|
||||
}
|
||||
switch (msg.type) {
|
||||
case 'ready':
|
||||
window.webContents.send('speech:ready', { sessionId })
|
||||
break
|
||||
case 'partial':
|
||||
window.webContents.send('speech:partial', { text: msg.text ?? '', sessionId })
|
||||
break
|
||||
case 'final':
|
||||
window.webContents.send('speech:final', { text: msg.text ?? '', sessionId })
|
||||
break
|
||||
case 'stopped':
|
||||
cleanupSessionListener()
|
||||
window.webContents.send('speech:stopped', { sessionId })
|
||||
break
|
||||
case 'error':
|
||||
window.webContents.send('speech:error', { error: msg.error ?? '', sessionId })
|
||||
void getSpeechSttService(store)
|
||||
.stopDictation(owner)
|
||||
.catch(() => undefined)
|
||||
.finally(cleanupSessionListener)
|
||||
break
|
||||
}
|
||||
},
|
||||
resolvedHotwordsPath,
|
||||
owner
|
||||
)
|
||||
if (resolvedHotwordsPath) {
|
||||
unlink(resolvedHotwordsPath).catch(() => {})
|
||||
}
|
||||
} catch (err) {
|
||||
cleanupSessionListener()
|
||||
if (resolvedHotwordsPath) {
|
||||
unlink(resolvedHotwordsPath).catch(() => {})
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (hotwords && hotwords.length > 0) {
|
||||
const content = `${hotwords.map((w) => `${w} :2.0`).join('\n')}\n`
|
||||
const hotwordsFilePath = getHotwordsFilePath(content)
|
||||
await writeFile(hotwordsFilePath, content, 'utf-8')
|
||||
resolvedHotwordsPath = hotwordsFilePath
|
||||
}
|
||||
|
||||
try {
|
||||
await getSpeechSttService(store).startDictation(
|
||||
modelId,
|
||||
(msg) => {
|
||||
if (window.isDestroyed()) {
|
||||
return
|
||||
}
|
||||
switch (msg.type) {
|
||||
case 'ready':
|
||||
window.webContents.send('speech:ready')
|
||||
break
|
||||
case 'partial':
|
||||
window.webContents.send('speech:partial', msg.text)
|
||||
break
|
||||
case 'final':
|
||||
window.webContents.send('speech:final', msg.text)
|
||||
break
|
||||
case 'stopped':
|
||||
window.webContents.send('speech:stopped')
|
||||
break
|
||||
case 'error':
|
||||
window.webContents.send('speech:error', msg.error)
|
||||
break
|
||||
}
|
||||
},
|
||||
resolvedHotwordsPath,
|
||||
'desktop'
|
||||
ipcMain.handle(
|
||||
'speech:feedAudio',
|
||||
async (_event, buffer: Buffer, sampleRate: number, sessionId = 'desktop') => {
|
||||
// Why: the preload sends audio as a Buffer to avoid Float32Array data
|
||||
// being zeroed out during contextBridge + IPC serialization.
|
||||
const samples = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
|
||||
getSpeechSttService(store).feedAudio(
|
||||
samples,
|
||||
sampleRate,
|
||||
getDesktopOwner(_event.sender.id, sessionId)
|
||||
)
|
||||
if (resolvedHotwordsPath) {
|
||||
unlink(resolvedHotwordsPath).catch(() => {})
|
||||
}
|
||||
} catch (err) {
|
||||
window.off('closed', cleanupOnWindowClosed)
|
||||
if (resolvedHotwordsPath) {
|
||||
unlink(resolvedHotwordsPath).catch(() => {})
|
||||
}
|
||||
throw err
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
ipcMain.handle('speech:feedAudio', async (_event, buffer: Buffer, sampleRate: number) => {
|
||||
// Why: the preload sends audio as a Buffer to avoid Float32Array data
|
||||
// being zeroed out during contextBridge + IPC serialization.
|
||||
const samples = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
|
||||
getSpeechSttService(store).feedAudio(samples, sampleRate, 'desktop')
|
||||
})
|
||||
|
||||
ipcMain.handle('speech:stopDictation', async () => {
|
||||
await getSpeechSttService(store).stopDictation('desktop')
|
||||
ipcMain.handle('speech:stopDictation', async (_event, sessionId = 'desktop') => {
|
||||
await getSpeechSttService(store).stopDictation(getDesktopOwner(_event.sender.id, sessionId))
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,8 +4,11 @@ const { MockWorker, getCreatedWorkerCount, getLastWorker, resetWorkers } = vi.ho
|
|||
class HoistedMockWorker extends EventTarget {
|
||||
static created = 0
|
||||
static instances: HoistedMockWorker[] = []
|
||||
static emitReadyOnInit = true
|
||||
terminated = false
|
||||
emitStoppedOnStop = true
|
||||
emitReadyOnInit = HoistedMockWorker.emitReadyOnInit
|
||||
messages: WorkerMessage[] = []
|
||||
private listeners = new Map<string, Set<(...args: unknown[]) => void>>()
|
||||
|
||||
constructor(_path: string, _options: unknown) {
|
||||
|
|
@ -38,7 +41,8 @@ const { MockWorker, getCreatedWorkerCount, getLastWorker, resetWorkers } = vi.ho
|
|||
}
|
||||
|
||||
postMessage(message: WorkerMessage): void {
|
||||
if (message.type === 'init') {
|
||||
this.messages.push(message)
|
||||
if (message.type === 'init' && this.emitReadyOnInit) {
|
||||
queueMicrotask(() => this.emit('message', { type: 'ready' }))
|
||||
}
|
||||
if (message.type === 'stop' && this.emitStoppedOnStop) {
|
||||
|
|
@ -63,6 +67,7 @@ const { MockWorker, getCreatedWorkerCount, getLastWorker, resetWorkers } = vi.ho
|
|||
resetWorkers: () => {
|
||||
HoistedMockWorker.created = 0
|
||||
HoistedMockWorker.instances = []
|
||||
HoistedMockWorker.emitReadyOnInit = true
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
@ -89,7 +94,7 @@ vi.mock('./model-catalog', () => ({
|
|||
})
|
||||
}))
|
||||
|
||||
import { SttService } from './stt-service'
|
||||
import { IDLE_WORKER_TEARDOWN_MS, SttService } from './stt-service'
|
||||
|
||||
describe('SttService', () => {
|
||||
beforeEach(() => {
|
||||
|
|
@ -109,6 +114,107 @@ describe('SttService', () => {
|
|||
expect(getCreatedWorkerCount()).toBe(1)
|
||||
})
|
||||
|
||||
it('keeps an idle worker warm for an hour', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const service = new SttService({
|
||||
getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }),
|
||||
getModelDir: vi.fn().mockReturnValue('/tmp/model-a')
|
||||
} as never)
|
||||
|
||||
await service.startDictation('model-a', vi.fn(), undefined, 'desktop')
|
||||
const worker = getLastWorker()
|
||||
expect(worker).toBeDefined()
|
||||
|
||||
await service.stopDictation('desktop')
|
||||
await vi.advanceTimersByTimeAsync(5 * 60 * 1000 + 1)
|
||||
expect(worker!.terminated).toBe(false)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(IDLE_WORKER_TEARDOWN_MS - 5 * 60 * 1000 - 1)
|
||||
expect(worker!.terminated).toBe(true)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('resets the idle teardown timer after each stop', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const service = new SttService({
|
||||
getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }),
|
||||
getModelDir: vi.fn().mockReturnValue('/tmp/model-a')
|
||||
} as never)
|
||||
|
||||
await service.startDictation('model-a', vi.fn(), undefined, 'desktop')
|
||||
const worker = getLastWorker()
|
||||
expect(worker).toBeDefined()
|
||||
|
||||
await service.stopDictation('desktop')
|
||||
await vi.advanceTimersByTimeAsync(IDLE_WORKER_TEARDOWN_MS / 2)
|
||||
await service.startDictation('model-a', vi.fn(), undefined, 'desktop')
|
||||
await service.stopDictation('desktop')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(IDLE_WORKER_TEARDOWN_MS / 2 + 1)
|
||||
expect(worker!.terminated).toBe(false)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(IDLE_WORKER_TEARDOWN_MS / 2 - 1)
|
||||
expect(worker!.terminated).toBe(true)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('drops stale audio while the worker is warm but no dictation owner is active', async () => {
|
||||
const service = new SttService({
|
||||
getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }),
|
||||
getModelDir: vi.fn().mockReturnValue('/tmp/model-a')
|
||||
} as never)
|
||||
|
||||
await service.startDictation('model-a', vi.fn(), undefined, 'desktop:1')
|
||||
const worker = getLastWorker()
|
||||
expect(worker).toBeDefined()
|
||||
|
||||
await service.stopDictation('desktop:1')
|
||||
service.feedAudio(new Float32Array([1]), 16000, 'desktop:1')
|
||||
|
||||
expect(worker!.messages.filter((message) => message.type === 'feed')).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('keeps startup cancellation tombstoned after the worker has been created', async () => {
|
||||
const service = new SttService({
|
||||
getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }),
|
||||
getModelDir: vi.fn().mockReturnValue('/tmp/model-a')
|
||||
} as never)
|
||||
|
||||
MockWorker.emitReadyOnInit = false
|
||||
const startPromise = service.startDictation('model-a', vi.fn(), undefined, 'desktop:1')
|
||||
await Promise.resolve()
|
||||
const worker = getLastWorker()
|
||||
expect(worker).toBeDefined()
|
||||
|
||||
await service.stopDictation('desktop:1')
|
||||
worker!.emit('message', { type: 'ready' })
|
||||
|
||||
await expect(startPromise).rejects.toThrow('dictation_canceled')
|
||||
await expect(service.startDictation('model-a', vi.fn(), undefined, 'desktop:2')).resolves.toBe(
|
||||
undefined
|
||||
)
|
||||
})
|
||||
|
||||
it('does not treat internal warm-worker replacement as startup cancellation', async () => {
|
||||
const service = new SttService({
|
||||
getModelState: vi.fn().mockResolvedValue({ id: 'model-a', status: 'ready' }),
|
||||
getModelDir: vi.fn().mockReturnValue('/tmp/model-a')
|
||||
} as never)
|
||||
|
||||
await service.startDictation('model-a', vi.fn(), '/tmp/hotwords-a.txt', 'desktop:1')
|
||||
await service.stopDictation('desktop:1')
|
||||
|
||||
await expect(
|
||||
service.startDictation('model-a', vi.fn(), '/tmp/hotwords-b.txt', 'desktop:1')
|
||||
).resolves.toBe(undefined)
|
||||
})
|
||||
|
||||
it('allows slow offline stop decoding before terminating the worker', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { getCatalogModel } from './model-catalog'
|
|||
import type { ModelManager } from './model-manager'
|
||||
|
||||
const STOP_DICTATION_TIMEOUT_MS = 60_000
|
||||
export const IDLE_WORKER_TEARDOWN_MS = 60 * 60 * 1000
|
||||
|
||||
export type SttEvent =
|
||||
| { type: 'ready' }
|
||||
|
|
@ -53,7 +54,7 @@ export class SttService {
|
|||
try {
|
||||
await this._startDictation(modelId, sink, hotwordsFilePath, owner)
|
||||
if (this.canceledOwners.delete(owner)) {
|
||||
await this.stopDictation(owner)
|
||||
await this.stopDictation(owner, { cancelStarting: false })
|
||||
throw new Error('dictation_canceled')
|
||||
}
|
||||
this.activeOwner = owner
|
||||
|
|
@ -81,7 +82,7 @@ export class SttService {
|
|||
}
|
||||
|
||||
if (this.worker) {
|
||||
await this.stopDictation(owner)
|
||||
await this.stopDictation(owner, { cancelStarting: false })
|
||||
await this.teardownIdleWorker()
|
||||
}
|
||||
|
||||
|
|
@ -204,17 +205,23 @@ export class SttService {
|
|||
|
||||
feedAudio(samples: Float32Array, sampleRate: number, owner = 'desktop'): void {
|
||||
const currentOwner = this.activeOwner ?? this.startingOwner
|
||||
if (currentOwner && currentOwner !== owner) {
|
||||
if (!currentOwner) {
|
||||
return
|
||||
}
|
||||
if (currentOwner !== owner) {
|
||||
throw new Error('dictation_owner_mismatch')
|
||||
}
|
||||
this.worker?.postMessage({ type: 'feed', samples, sampleRate }, [samples.buffer as ArrayBuffer])
|
||||
}
|
||||
|
||||
async stopDictation(owner = 'desktop'): Promise<void> {
|
||||
async stopDictation(
|
||||
owner = 'desktop',
|
||||
options: { cancelStarting?: boolean } = { cancelStarting: true }
|
||||
): Promise<void> {
|
||||
if (options.cancelStarting !== false && this.startingOwner === owner) {
|
||||
this.canceledOwners.add(owner)
|
||||
}
|
||||
if (!this.worker) {
|
||||
if (this.startingOwner === owner) {
|
||||
this.canceledOwners.add(owner)
|
||||
}
|
||||
return
|
||||
}
|
||||
const currentOwner = this.activeOwner ?? this.startingOwner
|
||||
|
|
@ -275,12 +282,9 @@ export class SttService {
|
|||
// Why: keep the native recognizer warm for repeated dictations, but release
|
||||
// the ONNX model after a quiet period so long-running Orca sessions don't
|
||||
// pin speech memory forever.
|
||||
this.idleTeardownTimer = setTimeout(
|
||||
() => {
|
||||
void this.teardownIdleWorker()
|
||||
},
|
||||
5 * 60 * 1000
|
||||
)
|
||||
this.idleTeardownTimer = setTimeout(() => {
|
||||
void this.teardownIdleWorker()
|
||||
}, IDLE_WORKER_TEARDOWN_MS)
|
||||
this.idleTeardownTimer.unref?.()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -206,7 +206,13 @@ import type {
|
|||
ClaudeUsageSummary
|
||||
} from '../shared/claude-usage-types'
|
||||
import type { RateLimitState } from '../shared/rate-limit-types'
|
||||
import type { SpeechModelManifest, SpeechModelState } from '../shared/speech-types'
|
||||
import type {
|
||||
SpeechErrorEvent,
|
||||
SpeechLifecycleEvent,
|
||||
SpeechModelManifest,
|
||||
SpeechModelState,
|
||||
SpeechTranscriptEvent
|
||||
} from '../shared/speech-types'
|
||||
import type {
|
||||
WorkspaceSpaceAnalyzeResult,
|
||||
WorkspaceSpaceScanProgress
|
||||
|
|
@ -1910,17 +1916,21 @@ export type PreloadApi = {
|
|||
downloadModel: (modelId: string) => Promise<void>
|
||||
cancelDownload: (modelId: string) => Promise<void>
|
||||
deleteModel: (modelId: string) => Promise<void>
|
||||
startDictation: (modelId: string, hotwords?: string[]) => Promise<void>
|
||||
feedAudio: (samples: Float32Array, sampleRate: number) => Promise<void>
|
||||
stopDictation: () => Promise<void>
|
||||
onPartialTranscript: (callback: (text: string) => void) => () => void
|
||||
onFinalTranscript: (callback: (text: string) => void) => () => void
|
||||
startDictation: (
|
||||
modelId: string,
|
||||
hotwords: string[] | undefined,
|
||||
sessionId: string
|
||||
) => Promise<void>
|
||||
feedAudio: (samples: Float32Array, sampleRate: number, sessionId?: string) => Promise<void>
|
||||
stopDictation: (sessionId?: string) => Promise<void>
|
||||
onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void
|
||||
onFinalTranscript: (callback: (data: SpeechTranscriptEvent) => void) => () => void
|
||||
onDownloadProgress: (
|
||||
callback: (data: { modelId: string; progress: number }) => void
|
||||
) => () => void
|
||||
onReady: (callback: () => void) => () => void
|
||||
onStopped: (callback: () => void) => () => void
|
||||
onError: (callback: (error: string) => void) => () => void
|
||||
onReady: (callback: (data: SpeechLifecycleEvent) => void) => () => void
|
||||
onStopped: (callback: (data: SpeechLifecycleEvent) => void) => () => void
|
||||
onError: (callback: (data: SpeechErrorEvent) => void) => () => void
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -97,7 +97,13 @@ import type {
|
|||
AgentStatusIpcPayload,
|
||||
MigrationUnsupportedPtyEntry
|
||||
} from '../shared/agent-status-types'
|
||||
import type { SpeechModelManifest, SpeechModelState } from '../shared/speech-types'
|
||||
import type {
|
||||
SpeechErrorEvent,
|
||||
SpeechLifecycleEvent,
|
||||
SpeechModelManifest,
|
||||
SpeechModelState,
|
||||
SpeechTranscriptEvent
|
||||
} from '../shared/speech-types'
|
||||
import type { TelemetryConsentState } from '../shared/telemetry-consent-types'
|
||||
import type { RefreshAgentsResult } from './api-types'
|
||||
import type { AgentKind, LaunchSource, RequestKind } from '../shared/telemetry-events'
|
||||
|
|
@ -3030,25 +3036,32 @@ const api = {
|
|||
ipcRenderer.invoke('speech:cancelDownload', modelId),
|
||||
deleteModel: (modelId: string): Promise<void> =>
|
||||
ipcRenderer.invoke('speech:deleteModel', modelId),
|
||||
startDictation: (modelId: string, hotwords?: string[]): Promise<void> =>
|
||||
ipcRenderer.invoke('speech:startDictation', modelId, hotwords),
|
||||
feedAudio: (samples: Float32Array, sampleRate: number): Promise<void> =>
|
||||
startDictation: (
|
||||
modelId: string,
|
||||
hotwords: string[] | undefined,
|
||||
sessionId: string
|
||||
): Promise<void> => ipcRenderer.invoke('speech:startDictation', modelId, hotwords, sessionId),
|
||||
feedAudio: (samples: Float32Array, sampleRate: number, sessionId = 'desktop'): Promise<void> =>
|
||||
// Why: Float32Array data gets zeroed out when crossing the contextBridge
|
||||
// + IPC boundary. Wrapping in a Buffer preserves the raw bytes reliably.
|
||||
ipcRenderer.invoke(
|
||||
'speech:feedAudio',
|
||||
Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength),
|
||||
sampleRate
|
||||
sampleRate,
|
||||
sessionId
|
||||
),
|
||||
stopDictation: (): Promise<void> => ipcRenderer.invoke('speech:stopDictation'),
|
||||
stopDictation: (sessionId = 'desktop'): Promise<void> =>
|
||||
ipcRenderer.invoke('speech:stopDictation', sessionId),
|
||||
|
||||
onPartialTranscript: (callback: (text: string) => void): (() => void) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, text: string): void => callback(text)
|
||||
onPartialTranscript: (callback: (data: SpeechTranscriptEvent) => void): (() => void) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, data: SpeechTranscriptEvent): void =>
|
||||
callback(data)
|
||||
ipcRenderer.on('speech:partial', listener)
|
||||
return () => ipcRenderer.removeListener('speech:partial', listener)
|
||||
},
|
||||
onFinalTranscript: (callback: (text: string) => void): (() => void) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, text: string): void => callback(text)
|
||||
onFinalTranscript: (callback: (data: SpeechTranscriptEvent) => void): (() => void) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, data: SpeechTranscriptEvent): void =>
|
||||
callback(data)
|
||||
ipcRenderer.on('speech:final', listener)
|
||||
return () => ipcRenderer.removeListener('speech:final', listener)
|
||||
},
|
||||
|
|
@ -3062,18 +3075,21 @@ const api = {
|
|||
ipcRenderer.on('speech:downloadProgress', listener)
|
||||
return () => ipcRenderer.removeListener('speech:downloadProgress', listener)
|
||||
},
|
||||
onReady: (callback: () => void): (() => void) => {
|
||||
const listener = (): void => callback()
|
||||
onReady: (callback: (data: SpeechLifecycleEvent) => void): (() => void) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, data: SpeechLifecycleEvent): void =>
|
||||
callback(data)
|
||||
ipcRenderer.on('speech:ready', listener)
|
||||
return () => ipcRenderer.removeListener('speech:ready', listener)
|
||||
},
|
||||
onStopped: (callback: () => void): (() => void) => {
|
||||
const listener = (): void => callback()
|
||||
onStopped: (callback: (data: SpeechLifecycleEvent) => void): (() => void) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, data: SpeechLifecycleEvent): void =>
|
||||
callback(data)
|
||||
ipcRenderer.on('speech:stopped', listener)
|
||||
return () => ipcRenderer.removeListener('speech:stopped', listener)
|
||||
},
|
||||
onError: (callback: (error: string) => void): (() => void) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, error: string): void => callback(error)
|
||||
onError: (callback: (data: SpeechErrorEvent) => void): (() => void) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, data: SpeechErrorEvent): void =>
|
||||
callback(data)
|
||||
ipcRenderer.on('speech:error', listener)
|
||||
return () => ipcRenderer.removeListener('speech:error', listener)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,84 +3,73 @@ import { useAppStore } from '@/store'
|
|||
import { useAudioCapture } from '@/hooks/use-audio-capture'
|
||||
import { toast } from 'sonner'
|
||||
import { DictationIndicator } from './DictationIndicator'
|
||||
import {
|
||||
captureInsertionTarget,
|
||||
insertText,
|
||||
type DictationInsertionTarget
|
||||
} from './dictation-insertion-target'
|
||||
import { formatFinalTranscriptSegment } from './dictation-final-segments'
|
||||
import { waitForStoppedSession } from './dictation-stopped-sessions'
|
||||
|
||||
const IS_MAC = navigator.userAgent.includes('Mac')
|
||||
|
||||
type DictationInsertionTarget =
|
||||
| { kind: 'terminal'; tabId: string; paneId: number }
|
||||
| { kind: 'text'; element: HTMLInputElement | HTMLTextAreaElement }
|
||||
| { kind: 'contentEditable'; element: HTMLElement }
|
||||
|
||||
// Why: splits compound identifiers into space-separated lowercase words
|
||||
// so hotwords match natural speech (e.g., "DictationController" → "dictation controller").
|
||||
function splitIdentifier(name: string): string | undefined {
|
||||
// PascalCase / camelCase → split on uppercase boundaries
|
||||
// kebab-case / snake_case → split on - or _
|
||||
const words = name
|
||||
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
||||
.replace(/[-_]+/g, ' ')
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
return words.includes(' ') ? words : undefined
|
||||
}
|
||||
|
||||
function collectHotwords(): string[] {
|
||||
const state = useAppStore.getState()
|
||||
const seen = new Set<string>()
|
||||
const hotwords: string[] = []
|
||||
|
||||
const addWord = (word: string): void => {
|
||||
if (word && !seen.has(word)) {
|
||||
seen.add(word)
|
||||
hotwords.push(word)
|
||||
}
|
||||
}
|
||||
|
||||
const addFileHotwords = (relativePath: string): void => {
|
||||
const basename = relativePath.split('/').pop() ?? ''
|
||||
addWord(basename)
|
||||
const dotIdx = basename.lastIndexOf('.')
|
||||
const nameOnly = dotIdx > 0 ? basename.slice(0, dotIdx) : basename
|
||||
if (dotIdx > 0) {
|
||||
addWord(nameOnly)
|
||||
}
|
||||
const spoken = splitIdentifier(nameOnly)
|
||||
if (spoken) {
|
||||
addWord(spoken)
|
||||
}
|
||||
}
|
||||
|
||||
for (const file of state.openFiles) {
|
||||
if (file.relativePath) {
|
||||
addFileHotwords(file.relativePath)
|
||||
}
|
||||
}
|
||||
|
||||
const worktreeId = state.activeWorktreeId
|
||||
if (worktreeId) {
|
||||
const closed = state.recentlyClosedEditorTabsByWorktree[worktreeId] ?? []
|
||||
for (const snap of closed) {
|
||||
if (snap.relativePath) {
|
||||
addFileHotwords(snap.relativePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return hotwords.slice(0, 50)
|
||||
}
|
||||
|
||||
export function DictationController() {
|
||||
const dictationState = useAppStore((s) => s.dictationState)
|
||||
const setDictationState = useAppStore((s) => s.setDictationState)
|
||||
const setPartialTranscript = useAppStore((s) => s.setPartialTranscript)
|
||||
const settings = useAppStore((s) => s.settings)
|
||||
const { start: startCapture, stop: stopCapture } = useAudioCapture()
|
||||
const {
|
||||
start: startCapture,
|
||||
stop: stopCapture,
|
||||
flushBufferedAudio,
|
||||
discardBufferedAudio,
|
||||
getCapturedChunkCount
|
||||
} = useAudioCapture()
|
||||
|
||||
const dictationStateRef = useRef(dictationState)
|
||||
dictationStateRef.current = dictationState
|
||||
const dictationRunRef = useRef(0)
|
||||
const holdGestureActiveRef = useRef(false)
|
||||
const insertionTargetRef = useRef<DictationInsertionTarget | null>(null)
|
||||
const activeSessionIdRef = useRef<string | null>(null)
|
||||
const stoppedSessionIdsRef = useRef(new Set<string>())
|
||||
const stoppedResolversRef = useRef(new Map<string, () => void>())
|
||||
const stopRequestedDuringStartRef = useRef(false)
|
||||
const finalTranscriptReceivedRef = useRef(false)
|
||||
const intentionalTargetCancellationRef = useRef(false)
|
||||
const insertedFinalTranscriptRef = useRef('')
|
||||
|
||||
const finishDictationSession = useCallback(
|
||||
async (sessionId: string) => {
|
||||
dictationStateRef.current = 'stopping'
|
||||
setDictationState('stopping')
|
||||
stopCapture()
|
||||
try {
|
||||
await window.api.speech.stopDictation(sessionId)
|
||||
} catch {
|
||||
// Swallow stop errors — the worker may already be torn down.
|
||||
}
|
||||
// Why: stopDictation() resolves on main-process completion, while final
|
||||
// transcript delivery is renderer IPC. Wait for this session's stopped
|
||||
// event so old finals cannot be mistaken for the next dictation run.
|
||||
await waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef)
|
||||
if (!finalTranscriptReceivedRef.current && getCapturedChunkCount() > 0) {
|
||||
toast.message('No speech detected.')
|
||||
}
|
||||
insertionTargetRef.current = null
|
||||
finalTranscriptReceivedRef.current = false
|
||||
insertedFinalTranscriptRef.current = ''
|
||||
intentionalTargetCancellationRef.current = false
|
||||
stopRequestedDuringStartRef.current = false
|
||||
if (activeSessionIdRef.current === sessionId) {
|
||||
activeSessionIdRef.current = null
|
||||
}
|
||||
dictationStateRef.current = 'idle'
|
||||
setDictationState('idle')
|
||||
setPartialTranscript('')
|
||||
},
|
||||
[setDictationState, setPartialTranscript, stopCapture, getCapturedChunkCount]
|
||||
)
|
||||
|
||||
const startDictation = useCallback(async () => {
|
||||
if (dictationStateRef.current !== 'idle') {
|
||||
|
|
@ -107,41 +96,82 @@ export function DictationController() {
|
|||
}
|
||||
|
||||
const runId = dictationRunRef.current + 1
|
||||
const sessionId = String(runId)
|
||||
dictationRunRef.current = runId
|
||||
activeSessionIdRef.current = sessionId
|
||||
insertionTargetRef.current = captureInsertionTarget()
|
||||
stopRequestedDuringStartRef.current = false
|
||||
finalTranscriptReceivedRef.current = false
|
||||
insertedFinalTranscriptRef.current = ''
|
||||
intentionalTargetCancellationRef.current = false
|
||||
dictationStateRef.current = 'starting'
|
||||
setDictationState('starting')
|
||||
|
||||
const hotwords = collectHotwords()
|
||||
let speechStarted = false
|
||||
let captureStarted = false
|
||||
|
||||
try {
|
||||
await window.api.speech.startDictation(modelId, hotwords.length > 0 ? hotwords : undefined)
|
||||
speechStarted = true
|
||||
// Why: worker startup can take seconds after idle teardown. Capture first
|
||||
// and buffer locally so speech during "Starting..." is not discarded.
|
||||
await startCapture({ bufferAudio: true, sessionId })
|
||||
captureStarted = true
|
||||
if (stopRequestedDuringStartRef.current) {
|
||||
stopCapture({ preserveBufferedAudio: true })
|
||||
}
|
||||
if (dictationRunRef.current !== runId) {
|
||||
discardBufferedAudio()
|
||||
stopCapture()
|
||||
insertionTargetRef.current = null
|
||||
await window.api.speech.stopDictation().catch(() => undefined)
|
||||
return
|
||||
}
|
||||
await startCapture()
|
||||
|
||||
await window.api.speech.startDictation(modelId, undefined, sessionId)
|
||||
if (dictationRunRef.current !== runId) {
|
||||
discardBufferedAudio()
|
||||
insertionTargetRef.current = null
|
||||
stopCapture()
|
||||
await window.api.speech.stopDictation().catch(() => undefined)
|
||||
await window.api.speech.stopDictation(sessionId).catch(() => undefined)
|
||||
return
|
||||
}
|
||||
|
||||
await flushBufferedAudio()
|
||||
if (dictationRunRef.current !== runId) {
|
||||
discardBufferedAudio()
|
||||
insertionTargetRef.current = null
|
||||
stopCapture()
|
||||
await window.api.speech.stopDictation(sessionId).catch(() => undefined)
|
||||
return
|
||||
}
|
||||
if (stopRequestedDuringStartRef.current) {
|
||||
await finishDictationSession(sessionId)
|
||||
return
|
||||
}
|
||||
|
||||
dictationStateRef.current = 'listening'
|
||||
setDictationState('listening')
|
||||
} catch (err) {
|
||||
if (dictationRunRef.current !== runId) {
|
||||
return
|
||||
}
|
||||
if (speechStarted) {
|
||||
await window.api.speech.stopDictation().catch(() => undefined)
|
||||
await window.api.speech.stopDictation(sessionId).catch(() => undefined)
|
||||
if (captureStarted) {
|
||||
stopCapture()
|
||||
}
|
||||
discardBufferedAudio()
|
||||
const message = String(err)
|
||||
insertionTargetRef.current = null
|
||||
intentionalTargetCancellationRef.current = false
|
||||
stopRequestedDuringStartRef.current = false
|
||||
finalTranscriptReceivedRef.current = false
|
||||
insertedFinalTranscriptRef.current = ''
|
||||
activeSessionIdRef.current = null
|
||||
setPartialTranscript('')
|
||||
if (message.includes('dictation_canceled')) {
|
||||
dictationStateRef.current = 'idle'
|
||||
setDictationState('idle')
|
||||
return
|
||||
}
|
||||
dictationStateRef.current = 'error'
|
||||
setDictationState('error')
|
||||
const message = String(err)
|
||||
if (message.includes('Permission') || message.includes('NotAllowed')) {
|
||||
toast.error('Microphone access denied. Grant access in system settings, then restart Orca.')
|
||||
} else if (message.includes('not ready')) {
|
||||
|
|
@ -159,29 +189,39 @@ export function DictationController() {
|
|||
} else {
|
||||
toast.error(`Dictation failed: ${message}`)
|
||||
}
|
||||
stopCapture()
|
||||
dictationStateRef.current = 'idle'
|
||||
setDictationState('idle')
|
||||
}
|
||||
}, [settings, setDictationState, startCapture, stopCapture])
|
||||
}, [
|
||||
settings,
|
||||
setDictationState,
|
||||
startCapture,
|
||||
flushBufferedAudio,
|
||||
discardBufferedAudio,
|
||||
stopCapture,
|
||||
finishDictationSession,
|
||||
setPartialTranscript
|
||||
])
|
||||
|
||||
const stopDictation = useCallback(async () => {
|
||||
if (dictationStateRef.current !== 'listening' && dictationStateRef.current !== 'starting') {
|
||||
if (dictationStateRef.current === 'starting') {
|
||||
stopRequestedDuringStartRef.current = true
|
||||
dictationStateRef.current = 'stopping'
|
||||
setDictationState('stopping')
|
||||
stopCapture({ preserveBufferedAudio: true })
|
||||
return
|
||||
}
|
||||
dictationRunRef.current += 1
|
||||
dictationStateRef.current = 'stopping'
|
||||
setDictationState('stopping')
|
||||
stopCapture()
|
||||
try {
|
||||
await window.api.speech.stopDictation()
|
||||
} catch {
|
||||
// Swallow stop errors — the worker may already be torn down.
|
||||
|
||||
if (dictationStateRef.current !== 'listening') {
|
||||
return
|
||||
}
|
||||
dictationStateRef.current = 'idle'
|
||||
setDictationState('idle')
|
||||
setPartialTranscript('')
|
||||
}, [setDictationState, setPartialTranscript, stopCapture])
|
||||
|
||||
const sessionId = activeSessionIdRef.current
|
||||
if (!sessionId) {
|
||||
return
|
||||
}
|
||||
await finishDictationSession(sessionId)
|
||||
}, [finishDictationSession, setDictationState, stopCapture])
|
||||
|
||||
// Toggle mode: use IPC from main process (before-input-event intercepts
|
||||
// the keyDown so Cmd+E doesn't reach xterm or trigger system shortcuts).
|
||||
|
|
@ -270,6 +310,7 @@ export function DictationController() {
|
|||
holdGestureActiveRef.current = false
|
||||
if (dictationStateRef.current !== 'idle' && dictationStateRef.current !== 'stopping') {
|
||||
insertionTargetRef.current = null
|
||||
intentionalTargetCancellationRef.current = true
|
||||
void stopDictation()
|
||||
}
|
||||
}
|
||||
|
|
@ -300,121 +341,75 @@ export function DictationController() {
|
|||
])
|
||||
|
||||
useEffect(() => {
|
||||
const cleanupPartial = window.api.speech.onPartialTranscript((text) => {
|
||||
setPartialTranscript(text)
|
||||
const cleanupPartial = window.api.speech.onPartialTranscript((data) => {
|
||||
if (data.sessionId !== activeSessionIdRef.current) {
|
||||
return
|
||||
}
|
||||
setPartialTranscript(data.text)
|
||||
})
|
||||
|
||||
const cleanupFinal = window.api.speech.onFinalTranscript((text) => {
|
||||
const cleanupFinal = window.api.speech.onFinalTranscript((data) => {
|
||||
if (data.sessionId !== activeSessionIdRef.current || !data.text) {
|
||||
return
|
||||
}
|
||||
setPartialTranscript('')
|
||||
finalTranscriptReceivedRef.current = true
|
||||
const target = insertionTargetRef.current
|
||||
insertionTargetRef.current = null
|
||||
if (target) {
|
||||
insertText(text, target)
|
||||
const textToInsert = formatFinalTranscriptSegment(
|
||||
data.text,
|
||||
insertedFinalTranscriptRef.current
|
||||
)
|
||||
insertText(textToInsert, target)
|
||||
insertedFinalTranscriptRef.current += textToInsert
|
||||
} else if (!intentionalTargetCancellationRef.current) {
|
||||
toast.message('Dictation finished, but no text field was focused.')
|
||||
}
|
||||
})
|
||||
|
||||
const cleanupError = window.api.speech.onError((error) => {
|
||||
toast.error(`Speech error: ${error}`)
|
||||
const cleanupStopped = window.api.speech.onStopped((data) => {
|
||||
const resolver = stoppedResolversRef.current.get(data.sessionId)
|
||||
if (resolver) {
|
||||
stoppedResolversRef.current.delete(data.sessionId)
|
||||
resolver()
|
||||
return
|
||||
}
|
||||
stoppedSessionIdsRef.current.add(data.sessionId)
|
||||
})
|
||||
|
||||
const cleanupError = window.api.speech.onError((data) => {
|
||||
if (data.sessionId !== activeSessionIdRef.current) {
|
||||
return
|
||||
}
|
||||
const sessionId = data.sessionId
|
||||
dictationRunRef.current += 1
|
||||
activeSessionIdRef.current = null
|
||||
toast.error(`Speech error: ${data.error}`)
|
||||
dictationStateRef.current = 'stopping'
|
||||
setDictationState('stopping')
|
||||
stopCapture()
|
||||
insertionTargetRef.current = null
|
||||
dictationStateRef.current = 'idle'
|
||||
setDictationState('idle')
|
||||
setPartialTranscript('')
|
||||
discardBufferedAudio()
|
||||
void (async () => {
|
||||
await window.api.speech.stopDictation(sessionId).catch(() => undefined)
|
||||
await waitForStoppedSession(sessionId, stoppedSessionIdsRef, stoppedResolversRef)
|
||||
insertionTargetRef.current = null
|
||||
intentionalTargetCancellationRef.current = false
|
||||
stopRequestedDuringStartRef.current = false
|
||||
finalTranscriptReceivedRef.current = false
|
||||
insertedFinalTranscriptRef.current = ''
|
||||
dictationStateRef.current = 'idle'
|
||||
setDictationState('idle')
|
||||
setPartialTranscript('')
|
||||
})()
|
||||
})
|
||||
|
||||
return () => {
|
||||
cleanupPartial()
|
||||
cleanupFinal()
|
||||
cleanupStopped()
|
||||
cleanupError()
|
||||
}
|
||||
}, [setPartialTranscript, setDictationState, stopCapture])
|
||||
}, [setPartialTranscript, setDictationState, stopCapture, discardBufferedAudio])
|
||||
|
||||
return <DictationIndicator />
|
||||
}
|
||||
|
||||
function captureInsertionTarget(): DictationInsertionTarget | null {
|
||||
const activeElement = document.activeElement
|
||||
|
||||
if (!activeElement) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (activeElement.classList.contains('xterm-helper-textarea')) {
|
||||
const paneElement = activeElement.closest('.pane[data-pane-id]') as HTMLElement | null
|
||||
const tabElement = activeElement.closest('[data-terminal-tab-id]') as HTMLElement | null
|
||||
const paneId = Number(paneElement?.dataset.paneId)
|
||||
const tabId = tabElement?.dataset.terminalTabId
|
||||
if (tabId && Number.isFinite(paneId)) {
|
||||
return { kind: 'terminal', tabId, paneId }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
if (activeElement instanceof HTMLInputElement || activeElement instanceof HTMLTextAreaElement) {
|
||||
return { kind: 'text', element: activeElement }
|
||||
}
|
||||
|
||||
if (activeElement instanceof HTMLElement && activeElement.isContentEditable) {
|
||||
return { kind: 'contentEditable', element: activeElement }
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
function insertText(text: string, target: DictationInsertionTarget): void {
|
||||
if (target.kind === 'terminal') {
|
||||
document.dispatchEvent(
|
||||
new CustomEvent('dictation:insertText', {
|
||||
detail: { text, tabId: target.tabId, paneId: target.paneId }
|
||||
})
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if (target.kind === 'text') {
|
||||
const element = target.element
|
||||
if (!element.isConnected) {
|
||||
return
|
||||
}
|
||||
const start = element.selectionStart ?? element.value.length
|
||||
const end = element.selectionEnd ?? start
|
||||
element.setRangeText(text, start, end, 'end')
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }))
|
||||
return
|
||||
}
|
||||
|
||||
if (target.kind === 'contentEditable') {
|
||||
const element = target.element
|
||||
if (!element.isConnected || !element.contains(document.activeElement)) {
|
||||
return
|
||||
}
|
||||
const editorElement = findClosestEditorElement(element) ?? element
|
||||
editorElement.dispatchEvent(
|
||||
new InputEvent('beforeinput', {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
inputType: 'insertText',
|
||||
data: text
|
||||
})
|
||||
)
|
||||
if (!document.execCommand('insertText', false, text)) {
|
||||
const selection = window.getSelection()
|
||||
if (selection && selection.rangeCount > 0) {
|
||||
const range = selection.getRangeAt(0)
|
||||
range.deleteContents()
|
||||
const textNode = document.createTextNode(text)
|
||||
range.insertNode(textNode)
|
||||
range.setStartAfter(textNode)
|
||||
range.collapse(true)
|
||||
selection.removeAllRanges()
|
||||
selection.addRange(range)
|
||||
}
|
||||
editorElement.dispatchEvent(
|
||||
new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text })
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
function findClosestEditorElement(element: HTMLElement): HTMLElement | null {
|
||||
return element.closest('.ProseMirror, [contenteditable="true"]')
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,16 +5,25 @@ export function DictationIndicator() {
|
|||
const dictationState = useAppStore((s) => s.dictationState)
|
||||
const partialTranscript = useAppStore((s) => s.partialTranscript)
|
||||
|
||||
if (dictationState !== 'listening' && dictationState !== 'starting') {
|
||||
if (
|
||||
dictationState !== 'listening' &&
|
||||
dictationState !== 'starting' &&
|
||||
dictationState !== 'stopping'
|
||||
) {
|
||||
return null
|
||||
}
|
||||
|
||||
const label =
|
||||
dictationState === 'starting'
|
||||
? 'Starting...'
|
||||
: dictationState === 'stopping'
|
||||
? 'Processing...'
|
||||
: partialTranscript || 'Listening...'
|
||||
|
||||
return (
|
||||
<div className="fixed bottom-12 left-1/2 -translate-x-1/2 z-50 flex items-center gap-2 rounded-lg bg-foreground/90 px-3 py-1.5 text-background text-sm shadow-lg">
|
||||
<Mic className={`h-4 w-4 ${dictationState === 'listening' ? 'animate-pulse' : ''}`} />
|
||||
<span>
|
||||
{dictationState === 'starting' ? 'Starting...' : partialTranscript || 'Listening...'}
|
||||
</span>
|
||||
<span>{label}</span>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import { formatFinalTranscriptSegment } from './dictation-final-segments'
|
||||
|
||||
describe('formatFinalTranscriptSegment', () => {
|
||||
it('adds a boundary between word-like streaming final segments', () => {
|
||||
expect(formatFinalTranscriptSegment('world', 'hello')).toBe(' world')
|
||||
})
|
||||
|
||||
it('adds a boundary after sentence and phrase punctuation', () => {
|
||||
expect(formatFinalTranscriptSegment('World', 'Hello.')).toBe(' World')
|
||||
expect(formatFinalTranscriptSegment('world', 'hello,')).toBe(' world')
|
||||
})
|
||||
|
||||
it('does not add a boundary before punctuation', () => {
|
||||
expect(formatFinalTranscriptSegment('.', 'hello')).toBe('.')
|
||||
})
|
||||
|
||||
it('does not add a boundary around CJK final segments', () => {
|
||||
expect(formatFinalTranscriptSegment('世界', '你好')).toBe('世界')
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
const WORD_BOUNDARY_CHAR_RE = /^[\p{L}\p{N}]$/u
|
||||
const CJK_BOUNDARY_CHAR_RE =
|
||||
/^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]$/u
|
||||
const NO_SPACE_BEFORE_CHAR_RE = /^[,.;:!?%。，、！？；：）)\]}]$/u
|
||||
const NO_SPACE_AFTER_CHAR_RE = /^[([{（《「『]$/u
|
||||
const SPACE_AFTER_CHAR_RE = /^[,.;:!?%]$/u
|
||||
|
||||
function getFirstNonWhitespaceChar(text: string): string {
|
||||
return Array.from(text.trimStart())[0] ?? ''
|
||||
}
|
||||
|
||||
function getLastNonWhitespaceChar(text: string): string {
|
||||
return Array.from(text.trimEnd()).at(-1) ?? ''
|
||||
}
|
||||
|
||||
function shouldInsertSpaceBetweenFinalSegments(previousText: string, nextText: string): boolean {
|
||||
if (!previousText || !nextText || /\s$/.test(previousText) || /^\s/.test(nextText)) {
|
||||
return false
|
||||
}
|
||||
|
||||
const previousChar = getLastNonWhitespaceChar(previousText)
|
||||
const nextChar = getFirstNonWhitespaceChar(nextText)
|
||||
if (!previousChar || !nextChar) {
|
||||
return false
|
||||
}
|
||||
if (
|
||||
CJK_BOUNDARY_CHAR_RE.test(previousChar) ||
|
||||
CJK_BOUNDARY_CHAR_RE.test(nextChar) ||
|
||||
NO_SPACE_BEFORE_CHAR_RE.test(nextChar) ||
|
||||
NO_SPACE_AFTER_CHAR_RE.test(previousChar)
|
||||
) {
|
||||
return false
|
||||
}
|
||||
|
||||
return (
|
||||
(WORD_BOUNDARY_CHAR_RE.test(previousChar) || SPACE_AFTER_CHAR_RE.test(previousChar)) &&
|
||||
WORD_BOUNDARY_CHAR_RE.test(nextChar)
|
||||
)
|
||||
}
|
||||
|
||||
export function formatFinalTranscriptSegment(text: string, previousInsertedText: string): string {
|
||||
if (shouldInsertSpaceBetweenFinalSegments(previousInsertedText, text)) {
|
||||
return ` ${text}`
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
export type DictationInsertionTarget =
|
||||
| { kind: 'terminal'; tabId: string; paneId: number }
|
||||
| { kind: 'text'; element: HTMLInputElement | HTMLTextAreaElement }
|
||||
| { kind: 'contentEditable'; element: HTMLElement }
|
||||
|
||||
export function captureInsertionTarget(): DictationInsertionTarget | null {
|
||||
const activeElement = document.activeElement
|
||||
|
||||
if (!activeElement) {
|
||||
return null
|
||||
}
|
||||
|
||||
if (activeElement.classList.contains('xterm-helper-textarea')) {
|
||||
const paneElement = activeElement.closest('.pane[data-pane-id]') as HTMLElement | null
|
||||
const tabElement = activeElement.closest('[data-terminal-tab-id]') as HTMLElement | null
|
||||
const paneId = Number(paneElement?.dataset.paneId)
|
||||
const tabId = tabElement?.dataset.terminalTabId
|
||||
if (tabId && Number.isFinite(paneId)) {
|
||||
return { kind: 'terminal', tabId, paneId }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
if (activeElement instanceof HTMLInputElement || activeElement instanceof HTMLTextAreaElement) {
|
||||
return { kind: 'text', element: activeElement }
|
||||
}
|
||||
|
||||
if (activeElement instanceof HTMLElement && activeElement.isContentEditable) {
|
||||
return { kind: 'contentEditable', element: activeElement }
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export function insertText(text: string, target: DictationInsertionTarget): void {
|
||||
if (target.kind === 'terminal') {
|
||||
document.dispatchEvent(
|
||||
new CustomEvent('dictation:insertText', {
|
||||
detail: { text, tabId: target.tabId, paneId: target.paneId }
|
||||
})
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if (target.kind === 'text') {
|
||||
const element = target.element
|
||||
if (!element.isConnected) {
|
||||
return
|
||||
}
|
||||
const start = element.selectionStart ?? element.value.length
|
||||
const end = element.selectionEnd ?? start
|
||||
element.setRangeText(text, start, end, 'end')
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }))
|
||||
return
|
||||
}
|
||||
|
||||
if (target.kind === 'contentEditable') {
|
||||
const element = target.element
|
||||
if (!element.isConnected || !element.contains(document.activeElement)) {
|
||||
return
|
||||
}
|
||||
const editorElement = findClosestEditorElement(element) ?? element
|
||||
editorElement.dispatchEvent(
|
||||
new InputEvent('beforeinput', {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
inputType: 'insertText',
|
||||
data: text
|
||||
})
|
||||
)
|
||||
if (!document.execCommand('insertText', false, text)) {
|
||||
const selection = window.getSelection()
|
||||
if (selection && selection.rangeCount > 0) {
|
||||
const range = selection.getRangeAt(0)
|
||||
range.deleteContents()
|
||||
const textNode = document.createTextNode(text)
|
||||
range.insertNode(textNode)
|
||||
range.setStartAfter(textNode)
|
||||
range.collapse(true)
|
||||
selection.removeAllRanges()
|
||||
selection.addRange(range)
|
||||
}
|
||||
editorElement.dispatchEvent(
|
||||
new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text })
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function findClosestEditorElement(element: HTMLElement): HTMLElement | null {
|
||||
return element.closest('.ProseMirror, [contenteditable="true"]')
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
type RefLike<T> = { current: T }
|
||||
|
||||
const STOPPED_SESSION_WAIT_MS = 1000
|
||||
|
||||
export function waitForStoppedSession(
|
||||
sessionId: string,
|
||||
stoppedSessionIdsRef: RefLike<Set<string>>,
|
||||
stoppedResolversRef: RefLike<Map<string, () => void>>
|
||||
): Promise<void> {
|
||||
if (stoppedSessionIdsRef.current.delete(sessionId)) {
|
||||
return Promise.resolve()
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const timeoutId = window.setTimeout(() => {
|
||||
stoppedResolversRef.current.delete(sessionId)
|
||||
resolve()
|
||||
}, STOPPED_SESSION_WAIT_MS)
|
||||
stoppedResolversRef.current.set(sessionId, () => {
|
||||
window.clearTimeout(timeoutId)
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
@ -1,82 +1,39 @@
|
|||
import { useRef, useCallback } from 'react'
|
||||
|
||||
type BufferedAudioChunk = {
|
||||
samples: Float32Array
|
||||
sampleRate: number
|
||||
sessionId: string
|
||||
}
|
||||
|
||||
type StartAudioCaptureOptions = {
|
||||
bufferAudio?: boolean
|
||||
sessionId?: string
|
||||
}
|
||||
|
||||
type StopAudioCaptureOptions = {
|
||||
preserveBufferedAudio?: boolean
|
||||
}
|
||||
|
||||
const MAX_BUFFERED_AUDIO_SECONDS = 30
|
||||
const MAX_BUFFERED_AUDIO_BYTES = 8 * 1024 * 1024
|
||||
|
||||
export function useAudioCapture() {
|
||||
const streamRef = useRef<MediaStream | null>(null)
|
||||
const contextRef = useRef<AudioContext | null>(null)
|
||||
const processorRef = useRef<ScriptProcessorNode | null>(null)
|
||||
const sourceRef = useRef<MediaStreamAudioSourceNode | null>(null)
|
||||
const isCapturingRef = useRef(false)
|
||||
const startRequestRef = useRef(0)
|
||||
const bufferAudioRef = useRef(false)
|
||||
const bufferedAudioGenerationRef = useRef(0)
|
||||
const bufferedAudioRef = useRef<BufferedAudioChunk[]>([])
|
||||
const bufferedAudioBytesRef = useRef(0)
|
||||
const bufferedAudioSecondsRef = useRef(0)
|
||||
const capturedChunkCountRef = useRef(0)
|
||||
const sessionIdRef = useRef('desktop')
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (isCapturingRef.current) {
|
||||
return
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true
|
||||
}
|
||||
})
|
||||
streamRef.current = stream
|
||||
|
||||
try {
|
||||
// Why: requesting a specific sampleRate (e.g. 16kHz) in the AudioContext
|
||||
// can produce silence on macOS because the hardware mic runs at 44.1/48kHz.
|
||||
// Use the system default rate and let sherpa-onnx resample internally.
|
||||
const context = new AudioContext()
|
||||
contextRef.current = context
|
||||
|
||||
// Why: some Chromium builds suspend the AudioContext until a user gesture.
|
||||
// Resume it explicitly to ensure audio processing starts.
|
||||
if (context.state === 'suspended') {
|
||||
await context.resume()
|
||||
}
|
||||
|
||||
const source = context.createMediaStreamSource(stream)
|
||||
|
||||
// Why: ScriptProcessorNode is deprecated but AudioWorklet requires a
|
||||
// separate module file which complicates the Vite build pipeline. For
|
||||
// the initial implementation, ScriptProcessorNode is simpler and the
|
||||
// performance difference is negligible for speech capture.
|
||||
const processor = context.createScriptProcessor(4096, 1, 1)
|
||||
|
||||
const actualRate = context.sampleRate
|
||||
|
||||
processor.onaudioprocess = (e: AudioProcessingEvent) => {
|
||||
if (!isCapturingRef.current) {
|
||||
return
|
||||
}
|
||||
const samples = new Float32Array(e.inputBuffer.getChannelData(0))
|
||||
window.api.speech.feedAudio(samples, actualRate)
|
||||
}
|
||||
|
||||
source.connect(processor)
|
||||
processor.connect(context.destination)
|
||||
|
||||
processorRef.current = processor
|
||||
sourceRef.current = source
|
||||
isCapturingRef.current = true
|
||||
} catch (err) {
|
||||
processorRef.current?.disconnect()
|
||||
sourceRef.current?.disconnect()
|
||||
processorRef.current = null
|
||||
sourceRef.current = null
|
||||
if (contextRef.current?.state !== 'closed') {
|
||||
void contextRef.current?.close()
|
||||
}
|
||||
contextRef.current = null
|
||||
stream.getTracks().forEach((track) => track.stop())
|
||||
streamRef.current = null
|
||||
throw err
|
||||
}
|
||||
}, [])
|
||||
|
||||
const stop = useCallback(() => {
|
||||
isCapturingRef.current = false
|
||||
|
||||
const cleanupCaptureResources = useCallback(() => {
|
||||
processorRef.current?.disconnect()
|
||||
sourceRef.current?.disconnect()
|
||||
processorRef.current = null
|
||||
|
|
@ -87,9 +44,222 @@ export function useAudioCapture() {
|
|||
}
|
||||
contextRef.current = null
|
||||
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop())
|
||||
streamRef.current?.getTracks().forEach((track) => track.stop())
|
||||
streamRef.current = null
|
||||
}, [])
|
||||
|
||||
return { start, stop, isCapturingRef }
|
||||
const resetBufferedAudio = useCallback(() => {
|
||||
bufferedAudioGenerationRef.current += 1
|
||||
bufferedAudioRef.current = []
|
||||
bufferedAudioBytesRef.current = 0
|
||||
bufferedAudioSecondsRef.current = 0
|
||||
}, [])
|
||||
|
||||
const removeOldestBufferedAudioChunk = useCallback(() => {
|
||||
const chunk = bufferedAudioRef.current.shift()
|
||||
if (!chunk) {
|
||||
return
|
||||
}
|
||||
bufferedAudioBytesRef.current -= chunk.samples.byteLength
|
||||
bufferedAudioSecondsRef.current -= chunk.samples.length / chunk.sampleRate
|
||||
}, [])
|
||||
|
||||
const appendBufferedAudioChunk = useCallback(
|
||||
(chunk: BufferedAudioChunk) => {
|
||||
bufferedAudioRef.current.push(chunk)
|
||||
bufferedAudioBytesRef.current += chunk.samples.byteLength
|
||||
bufferedAudioSecondsRef.current += chunk.samples.length / chunk.sampleRate
|
||||
|
||||
// Why: worker/model startup can hang; keep only a bounded recent window
|
||||
// so renderer memory cannot grow forever while buffering is enabled.
|
||||
while (
|
||||
bufferedAudioRef.current.length > 0 &&
|
||||
(bufferedAudioBytesRef.current > MAX_BUFFERED_AUDIO_BYTES ||
|
||||
bufferedAudioSecondsRef.current > MAX_BUFFERED_AUDIO_SECONDS)
|
||||
) {
|
||||
removeOldestBufferedAudioChunk()
|
||||
}
|
||||
},
|
||||
[removeOldestBufferedAudioChunk]
|
||||
)
|
||||
|
||||
const start = useCallback(
|
||||
async (options: StartAudioCaptureOptions = {}) => {
|
||||
if (isCapturingRef.current) {
|
||||
return
|
||||
}
|
||||
const startRequest = startRequestRef.current + 1
|
||||
startRequestRef.current = startRequest
|
||||
cleanupCaptureResources()
|
||||
sessionIdRef.current = options.sessionId ?? 'desktop'
|
||||
bufferAudioRef.current = options.bufferAudio ?? false
|
||||
resetBufferedAudio()
|
||||
capturedChunkCountRef.current = 0
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true
|
||||
}
|
||||
})
|
||||
if (startRequestRef.current !== startRequest) {
|
||||
stream.getTracks().forEach((track) => track.stop())
|
||||
return
|
||||
}
|
||||
streamRef.current = stream
|
||||
|
||||
let context: AudioContext | null = null
|
||||
let source: MediaStreamAudioSourceNode | null = null
|
||||
let processor: ScriptProcessorNode | null = null
|
||||
try {
|
||||
// Why: requesting a specific sampleRate (e.g. 16kHz) in the AudioContext
|
||||
// can produce silence on macOS because the hardware mic runs at 44.1/48kHz.
|
||||
// Use the system default rate and let sherpa-onnx resample internally.
|
||||
context = new AudioContext()
|
||||
contextRef.current = context
|
||||
|
||||
// Why: some Chromium builds suspend the AudioContext until a user gesture.
|
||||
// Resume it explicitly to ensure audio processing starts.
|
||||
if (context.state === 'suspended') {
|
||||
await context.resume()
|
||||
}
|
||||
if (startRequestRef.current !== startRequest || streamRef.current !== stream) {
|
||||
if (contextRef.current === context) {
|
||||
contextRef.current = null
|
||||
}
|
||||
if (context.state !== 'closed') {
|
||||
void context.close()
|
||||
}
|
||||
if (streamRef.current === stream) {
|
||||
streamRef.current = null
|
||||
}
|
||||
stream.getTracks().forEach((track) => track.stop())
|
||||
return
|
||||
}
|
||||
|
||||
source = context.createMediaStreamSource(stream)
|
||||
|
||||
// Why: ScriptProcessorNode is deprecated but AudioWorklet requires a
|
||||
// separate module file which complicates the Vite build pipeline. For
|
||||
// the initial implementation, ScriptProcessorNode is simpler and the
|
||||
// performance difference is negligible for speech capture.
|
||||
processor = context.createScriptProcessor(4096, 1, 1)
|
||||
|
||||
const actualRate = context.sampleRate
|
||||
|
||||
processor.onaudioprocess = (e: AudioProcessingEvent) => {
|
||||
if (
|
||||
!isCapturingRef.current ||
|
||||
startRequestRef.current !== startRequest ||
|
||||
processorRef.current !== processor
|
||||
) {
|
||||
return
|
||||
}
|
||||
const samples = new Float32Array(e.inputBuffer.getChannelData(0))
|
||||
capturedChunkCountRef.current += 1
|
||||
if (bufferAudioRef.current) {
|
||||
appendBufferedAudioChunk({
|
||||
samples,
|
||||
sampleRate: actualRate,
|
||||
sessionId: sessionIdRef.current
|
||||
})
|
||||
return
|
||||
}
|
||||
void window.api.speech
|
||||
.feedAudio(samples, actualRate, sessionIdRef.current)
|
||||
.catch(() => undefined)
|
||||
}
|
||||
|
||||
source.connect(processor)
|
||||
processor.connect(context.destination)
|
||||
|
||||
processorRef.current = processor
|
||||
sourceRef.current = source
|
||||
isCapturingRef.current = true
|
||||
} catch (err) {
|
||||
processor?.disconnect()
|
||||
source?.disconnect()
|
||||
if (processorRef.current === processor) {
|
||||
processorRef.current = null
|
||||
}
|
||||
if (sourceRef.current === source) {
|
||||
sourceRef.current = null
|
||||
}
|
||||
if (contextRef.current === context) {
|
||||
contextRef.current = null
|
||||
}
|
||||
if (context && context.state !== 'closed') {
|
||||
void context.close()
|
||||
}
|
||||
stream.getTracks().forEach((track) => track.stop())
|
||||
if (streamRef.current === stream) {
|
||||
streamRef.current = null
|
||||
}
|
||||
if (startRequestRef.current === startRequest) {
|
||||
bufferAudioRef.current = false
|
||||
resetBufferedAudio()
|
||||
}
|
||||
if (startRequestRef.current !== startRequest) {
|
||||
return
|
||||
}
|
||||
throw err
|
||||
}
|
||||
},
|
||||
[appendBufferedAudioChunk, cleanupCaptureResources, resetBufferedAudio]
|
||||
)
|
||||
|
||||
const flushBufferedAudio = useCallback(async () => {
|
||||
const flushGeneration = bufferedAudioGenerationRef.current
|
||||
try {
|
||||
// Why: keep buffering enabled while draining so live audio appends behind
|
||||
// startup audio instead of overtaking it through direct IPC sends.
|
||||
while (
|
||||
bufferedAudioGenerationRef.current === flushGeneration &&
|
||||
bufferedAudioRef.current.length > 0
|
||||
) {
|
||||
const chunk = bufferedAudioRef.current[0]
|
||||
if (!chunk) {
|
||||
break
|
||||
}
|
||||
removeOldestBufferedAudioChunk()
|
||||
await window.api.speech.feedAudio(chunk.samples, chunk.sampleRate, chunk.sessionId)
|
||||
}
|
||||
} finally {
|
||||
if (bufferedAudioGenerationRef.current === flushGeneration) {
|
||||
bufferAudioRef.current = false
|
||||
resetBufferedAudio()
|
||||
}
|
||||
}
|
||||
}, [removeOldestBufferedAudioChunk, resetBufferedAudio])
|
||||
|
||||
const discardBufferedAudio = useCallback(() => {
|
||||
bufferAudioRef.current = false
|
||||
resetBufferedAudio()
|
||||
}, [resetBufferedAudio])
|
||||
|
||||
const getCapturedChunkCount = useCallback(() => capturedChunkCountRef.current, [])
|
||||
|
||||
const stop = useCallback(
|
||||
(options: StopAudioCaptureOptions = {}) => {
|
||||
startRequestRef.current += 1
|
||||
isCapturingRef.current = false
|
||||
bufferAudioRef.current = false
|
||||
if (!options.preserveBufferedAudio) {
|
||||
resetBufferedAudio()
|
||||
}
|
||||
cleanupCaptureResources()
|
||||
},
|
||||
[cleanupCaptureResources, resetBufferedAudio]
|
||||
)
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
flushBufferedAudio,
|
||||
discardBufferedAudio,
|
||||
getCapturedChunkCount,
|
||||
isCapturingRef
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,20 @@ export type SpeechModelState = {
|
|||
error?: string
|
||||
}
|
||||
|
||||
export type SpeechTranscriptEvent = {
|
||||
text: string
|
||||
sessionId: string
|
||||
}
|
||||
|
||||
export type SpeechLifecycleEvent = {
|
||||
sessionId: string
|
||||
}
|
||||
|
||||
export type SpeechErrorEvent = {
|
||||
error: string
|
||||
sessionId: string
|
||||
}
|
||||
|
||||
export type DictationState = 'idle' | 'starting' | 'listening' | 'stopping' | 'error'
|
||||
|
||||
export type UserModelConfig = {
|
||||
|
|
|
|||
Loading…
Reference in New Issue