diff --git a/src/main/speech/stt-offline-audio-chunker.test.ts b/src/main/speech/stt-offline-audio-chunker.test.ts new file mode 100644 index 000000000..4b90976b2 --- /dev/null +++ b/src/main/speech/stt-offline-audio-chunker.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest' +import { OFFLINE_DECODE_CHUNK_SECONDS, OfflineAudioChunker } from './stt-offline-audio-chunker' + +// Why: a small rate keeps test arrays tiny while exercising the same +// seconds-based limits used with real 16 kHz audio. +const SAMPLE_RATE = 1000 +const CHUNK_LIMIT = OFFLINE_DECODE_CHUNK_SECONDS * SAMPLE_RATE + +function loudSignal(length: number): Float32Array { + const samples = new Float32Array(length) + for (let i = 0; i < length; i += 1) { + samples[i] = Math.sin(i * 0.3) * 0.8 + } + return samples +} + +describe('OfflineAudioChunker', () => { + it('buffers audio below the chunk limit without emitting chunks', () => { + const chunker = new OfflineAudioChunker(SAMPLE_RATE) + + expect(chunker.push(loudSignal(CHUNK_LIMIT - 1))).toEqual([]) + }) + + it('emits a bounded chunk once the limit is reached and keeps the remainder', () => { + const chunker = new OfflineAudioChunker(SAMPLE_RATE) + const total = CHUNK_LIMIT + 500 + + const ready = chunker.push(loudSignal(total)) + const remainder = chunker.flush() + + expect(ready).toHaveLength(1) + expect(ready[0].length).toBeLessThanOrEqual(CHUNK_LIMIT) + expect(ready[0].length).toBeGreaterThan(0) + expect(ready[0].length + (remainder?.length ?? 0)).toBe(total) + }) + + it('never emits a chunk larger than the limit across many small pushes', () => { + const chunker = new OfflineAudioChunker(SAMPLE_RATE) + const emitted: Float32Array[] = [] + const pushSize = 160 + const pushes = Math.ceil((CHUNK_LIMIT * 3.5) / pushSize) + for (let i = 0; i < pushes; i += 1) { + emitted.push(...chunker.push(loudSignal(pushSize))) + } + const remainder = chunker.flush() + + expect(emitted.length).toBeGreaterThanOrEqual(3) + for (const chunk of emitted) { + expect(chunk.length).toBeLessThanOrEqual(CHUNK_LIMIT) + } + const totalOut = emitted.reduce((sum, c) => sum + c.length, 0) + (remainder?.length ?? 0) + expect(totalOut).toBe(pushes * pushSize) + }) + + it('splits multiple chunks out of one oversized push', () => { + const chunker = new OfflineAudioChunker(SAMPLE_RATE) + + const ready = chunker.push(loudSignal(CHUNK_LIMIT * 2 + 100)) + + expect(ready.length).toBeGreaterThanOrEqual(2) + for (const chunk of ready) { + expect(chunk.length).toBeLessThanOrEqual(CHUNK_LIMIT) + } + }) + + it('splits at a silent pause near the chunk boundary instead of mid-speech', () => { + const chunker = new OfflineAudioChunker(SAMPLE_RATE) + const samples = loudSignal(CHUNK_LIMIT + 200) + // Quiet gap 2 s before the limit, inside the 5 s split-search window. + const gapStart = CHUNK_LIMIT - 2 * SAMPLE_RATE + const gapEnd = gapStart + Math.round(0.2 * SAMPLE_RATE) + samples.fill(0, gapStart, gapEnd) + + const [chunk] = chunker.push(samples) + + expect(chunk.length).toBeGreaterThanOrEqual(gapStart) + expect(chunk.length).toBeLessThanOrEqual(gapEnd) + }) + + it('conserves sample values across the split', () => { + const chunker = new OfflineAudioChunker(SAMPLE_RATE) + const samples = loudSignal(CHUNK_LIMIT + 50) + + const [chunk] = chunker.push(samples) + const remainder = chunker.flush() + + const rejoined = new Float32Array(samples.length) + rejoined.set(chunk, 0) + rejoined.set(remainder!, chunk.length) + expect(rejoined).toEqual(samples) + }) + + it('flush returns null when nothing is buffered', () => { + const chunker = new OfflineAudioChunker(SAMPLE_RATE) + + expect(chunker.flush()).toBeNull() + expect(chunker.push(new Float32Array(0))).toEqual([]) + expect(chunker.flush()).toBeNull() + }) +}) diff --git a/src/main/speech/stt-offline-audio-chunker.ts b/src/main/speech/stt-offline-audio-chunker.ts new file mode 100644 index 000000000..291454bf0 --- /dev/null +++ b/src/main/speech/stt-offline-audio-chunker.ts @@ -0,0 +1,92 @@ +// Why: offline recognizers decode a whole buffer per call, and ONNX Runtime's +// arena allocations scale with buffer length. Chromium's allocator shim kills +// the entire app on any single allocation >= 2 GiB (#7925), so audio must be +// decoded in bounded chunks regardless of how long dictation runs. +export const OFFLINE_DECODE_CHUNK_SECONDS = 30 + +// Why: cutting audio mid-word degrades transcription at chunk boundaries. +// Search the tail of each chunk for its quietest window and split at its +// center, so cuts land on real inter-word pauses whenever one exists. The +// window must be pause-sized (~100ms): shorter windows match momentary +// quiet inside a word (e.g. plosive closures) and cut mid-word. +const SPLIT_SEARCH_SECONDS = 5 +const SPLIT_ENERGY_WINDOW_SECONDS = 0.1 + +export class OfflineAudioChunker { + private buffered: Float32Array[] = [] + private bufferedSamples = 0 + private readonly chunkSampleLimit: number + private readonly splitSearchSamples: number + private readonly energyWindowSamples: number + + constructor(sampleRate: number) { + this.chunkSampleLimit = Math.max(1, Math.round(OFFLINE_DECODE_CHUNK_SECONDS * sampleRate)) + this.splitSearchSamples = Math.round(SPLIT_SEARCH_SECONDS * sampleRate) + this.energyWindowSamples = Math.max(1, Math.round(SPLIT_ENERGY_WINDOW_SECONDS * sampleRate)) + } + + /** Buffers samples and returns any full chunks now ready to decode. */ + push(samples: Float32Array): Float32Array[] { + if (samples.length === 0) { + return [] + } + this.buffered.push(samples) + this.bufferedSamples += samples.length + + const ready: Float32Array[] = [] + while (this.bufferedSamples >= this.chunkSampleLimit) { + const combined = this.combineBuffered() + const splitIndex = this.findQuietSplitIndex(combined) + ready.push(combined.slice(0, splitIndex)) + const tail = combined.slice(splitIndex) + this.buffered = tail.length > 0 ? [tail] : [] + this.bufferedSamples = tail.length + } + return ready + } + + /** Returns all remaining buffered audio (any length below the chunk limit). */ + flush(): Float32Array | null { + if (this.bufferedSamples === 0) { + return null + } + const combined = this.combineBuffered() + this.buffered = [] + this.bufferedSamples = 0 + return combined + } + + private combineBuffered(): Float32Array { + if (this.buffered.length === 1) { + return this.buffered[0] + } + const combined = new Float32Array(this.bufferedSamples) + let offset = 0 + for (const chunk of this.buffered) { + combined.set(chunk, offset) + offset += chunk.length + } + return combined + } + + private findQuietSplitIndex(samples: Float32Array): number { + const limit = Math.min(this.chunkSampleLimit, samples.length) + const window = this.energyWindowSamples + const searchStart = Math.max(0, limit - this.splitSearchSamples) + const hop = Math.max(1, Math.floor(window / 2)) + let bestIndex = limit + let bestEnergy = Infinity + for (let start = searchStart; start + window <= limit; start += hop) { + let energy = 0 + for (let i = start; i < start + window; i += 1) { + energy += samples[i] * samples[i] + } + if (energy < bestEnergy) { + bestEnergy = energy + bestIndex = start + Math.floor(window / 2) + } + } + // Why: the split must consume at least one sample or push() would loop forever. + return Math.max(1, bestIndex) + } +} diff --git a/src/main/speech/stt-worker.ts b/src/main/speech/stt-worker.ts index 9456f95c8..2f4f9b025 100644 --- a/src/main/speech/stt-worker.ts +++ b/src/main/speech/stt-worker.ts @@ -2,6 +2,7 @@ import { parentPort, workerData } from 'node:worker_threads' import { readdirSync } from 'node:fs' import { resampleToRate } from './stt-audio-resample' +import { OfflineAudioChunker } from './stt-offline-audio-chunker' type WorkerMessage = | { @@ -27,7 +28,7 @@ let sherpa: any = null let recognizer: any = null let stream: any = null let isStreaming = false -let offlineBuffer: Float32Array[] = [] +let offlineChunker: OfflineAudioChunker | null = null let offlineSampleRate = 16000 function loadSherpa(): any { @@ -111,7 +112,7 @@ function handleInit(msg: Extract): void { const { modelDir, modelType, streaming, sampleRate, files } = msg isStreaming = streaming - offlineBuffer = [] + offlineChunker = streaming ? null : new OfflineAudioChunker(sampleRate) offlineSampleRate = sampleRate const tokens = resolveTokens(files, modelDir) @@ -203,6 +204,17 @@ function handleInit(msg: Extract): void { } } +// Why: an offline stream is single-use — decode one bounded chunk, then +// recreate the stream so the recognizer is ready for the next chunk. +function decodeOfflineChunk(samples: Float32Array): string { + sherpa.acceptWaveformOffline(stream, { sampleRate: offlineSampleRate, samples }) + sherpa.decodeOfflineStream(recognizer, stream) + const resultJson = sherpa.getOfflineStreamResultAsJson(stream) + stream = sherpa.createOfflineStream(recognizer) + const result = JSON.parse(resultJson) + return result?.text?.trim() ?? '' +} + function handleFeed(msg: Extract): void { if (!recognizer || !stream) { return @@ -236,9 +248,17 @@ function handleFeed(msg: Extract): void { sherpa.reset(recognizer, stream) } } else { - // Why: offline recognizers cannot decode incrementally — they need all - // audio buffered first, then decoded in one shot when dictation stops. - offlineBuffer.push(new Float32Array(samples)) + // Why: decoding one unbounded capture in a single call makes ONNX tensor + // sizes scale with dictation length until a >=2 GiB allocation SIGTRAPs + // the whole app (#7925). Decode bounded chunks as they fill instead; + // each consumer already appends multiple 'final' segments per session. + const readyChunks = offlineChunker?.push(new Float32Array(samples)) ?? [] + for (const chunk of readyChunks) { + const text = decodeOfflineChunk(chunk) + if (text) { + parentPort?.postMessage({ type: 'final', text }) + } + } } } catch (err) { parentPort?.postMessage({ type: 'error', error: String(err) }) @@ -265,27 +285,15 @@ function handleStop(): void { } stream = sherpa.createOnlineStream(recognizer) } else { - // Why: offline recognizer decodes all audio at once — concatenate - // buffered chunks into a single Float32Array and feed it to the stream. - const totalLength = offlineBuffer.reduce((sum, chunk) => sum + chunk.length, 0) - if (totalLength > 0) { - const combined = new Float32Array(totalLength) - let offset = 0 - for (const chunk of offlineBuffer) { - combined.set(chunk, offset) - offset += chunk.length - } - sherpa.acceptWaveformOffline(stream, { sampleRate: offlineSampleRate, samples: combined }) - sherpa.decodeOfflineStream(recognizer, stream) - const resultJson = sherpa.getOfflineStreamResultAsJson(stream) - const result = JSON.parse(resultJson) - const text = result?.text?.trim() + // Why: the remainder is below the chunk limit by construction, so this + // last decode is bounded too. + const remaining = offlineChunker?.flush() + if (remaining && remaining.length > 0) { + const text = decodeOfflineChunk(remaining) if (text) { parentPort?.postMessage({ type: 'final', text }) } } - offlineBuffer = [] - stream = sherpa.createOfflineStream(recognizer) } } catch (err) { parentPort?.postMessage({ type: 'error', error: String(err) }) @@ -298,7 +306,7 @@ function handleTeardown(): void { stream = null recognizer = null sherpa = null - offlineBuffer = [] + offlineChunker = null process.exit(0) }