From 49153d933f1920955bcdba30cf2af8526acb80eb Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 31 May 2026 03:33:00 -0700 Subject: [PATCH] perf: cap mobile dictation audio backlog (#4078) --- ...ile-dictation-pending-audio-budget.test.ts | 32 +++++++++++++ .../mobile-dictation-pending-audio-budget.ts | 46 +++++++++++++++++++ .../hooks/use-mobile-dictation-source.test.ts | 22 +++++++++ mobile/src/hooks/use-mobile-dictation.ts | 22 ++++++++- 4 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts create mode 100644 mobile/src/hooks/mobile-dictation-pending-audio-budget.ts diff --git a/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts b/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts new file mode 100644 index 000000000..249f923c3 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest' +import { + MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES, + MobileDictationPendingAudioBudget +} from './mobile-dictation-pending-audio-budget' + +describe('MobileDictationPendingAudioBudget', () => { + it('caps pending raw PCM audio at five seconds', () => { + expect(MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES).toBe(160_000) + + const budget = new MobileDictationPendingAudioBudget() + expect(budget.tryReserve(159_999)).toBe(true) + expect(budget.tryReserve(1)).toBe(true) + expect(budget.pendingAudioBytes).toBe(160_000) + expect(budget.tryReserve(1)).toBe(false) + expect(budget.pendingAudioBytes).toBe(160_000) + }) + + it('releases completed chunks and clamps stale releases after reset', () => { + const budget = new MobileDictationPendingAudioBudget(10) + + expect(budget.tryReserve(6)).toBe(true) + budget.release(4) + expect(budget.pendingAudioBytes).toBe(2) + expect(budget.tryReserve(8)).toBe(true) + + budget.reset() + budget.release(6) + expect(budget.pendingAudioBytes).toBe(0) + expect(budget.tryReserve(10)).toBe(true) + }) +}) diff --git a/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts new file mode 100644 index 000000000..7f814e493 --- /dev/null +++ b/mobile/src/hooks/mobile-dictation-pending-audio-budget.ts @@ -0,0 +1,46 @@ +export const MOBILE_DICTATION_PCM_SAMPLE_RATE = 16000 + +const PCM_BYTES_PER_SAMPLE = 2 +const MAX_PENDING_AUDIO_SECONDS = 5 + +// Why: dictation chunk RPCs can wait through reconnect before timing out, so +// cap retained raw microphone audio before it is expanded into base64 payloads. +export const MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES = + MOBILE_DICTATION_PCM_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * MAX_PENDING_AUDIO_SECONDS + +export const MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE = + 'Connection is too slow for voice dictation. Try again when the connection improves.' + +export class MobileDictationPendingAudioBudget { + private pendingBytes = 0 + + constructor(private readonly maxPendingBytes = MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES) {} + + get pendingAudioBytes(): number { + return this.pendingBytes + } + + tryReserve(byteLength: number): boolean { + const normalizedByteLength = normalizeByteLength(byteLength) + if (this.pendingBytes + normalizedByteLength > this.maxPendingBytes) { + return false + } + this.pendingBytes += normalizedByteLength + return true + } + + release(byteLength: number): void { + this.pendingBytes = Math.max(0, this.pendingBytes - normalizeByteLength(byteLength)) + } + + reset(): void { + this.pendingBytes = 0 + } +} + +function normalizeByteLength(byteLength: number): number { + if (!Number.isFinite(byteLength) || byteLength <= 0) { + return 0 + } + return Math.floor(byteLength) +} diff --git a/mobile/src/hooks/use-mobile-dictation-source.test.ts b/mobile/src/hooks/use-mobile-dictation-source.test.ts index 7dc889341..6319e2d9b 100644 --- a/mobile/src/hooks/use-mobile-dictation-source.test.ts +++ b/mobile/src/hooks/use-mobile-dictation-source.test.ts @@ -26,4 +26,26 @@ describe('useMobileDictation source invariants', () => { expect(mirrorEffect).toContain('onErrorRef.current = onError') expect(mirrorEffect).toContain('}, [client, enabled, onTranscript, onError])') }) + + it('reserves pending audio bytes before encoding microphone chunks', () => { + const microphoneEffect = sliceBetween( + "addExpoTwoWayAudioEventListener('onMicrophoneData'", + 'return () => sub.remove()' + ) + + const reserveIndex = microphoneEffect.indexOf('tryReserve(byteLength)') + const encodeIndex = microphoneEffect.indexOf('bytesToBase64(bytes)') + expect(reserveIndex).toBeGreaterThanOrEqual(0) + expect(encodeIndex).toBeGreaterThanOrEqual(0) + expect(reserveIndex).toBeLessThan(encodeIndex) + expect(microphoneEffect).toContain('MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE') + expect(microphoneEffect).toContain('pendingAudioBudgetRef.current.release(byteLength)') + }) + + it('resets pending audio bytes whenever pending chunk tracking is cleared', () => { + const pendingChunkClears = source.match(/pendingChunksRef\.current\.clear\(\)/g) ?? [] + const pendingAudioResets = source.match(/pendingAudioBudgetRef\.current\.reset\(\)/g) ?? [] + + expect(pendingAudioResets).toHaveLength(pendingChunkClears.length) + }) }) diff --git a/mobile/src/hooks/use-mobile-dictation.ts b/mobile/src/hooks/use-mobile-dictation.ts index 55857b58e..d46a6147a 100644 --- a/mobile/src/hooks/use-mobile-dictation.ts +++ b/mobile/src/hooks/use-mobile-dictation.ts @@ -11,6 +11,11 @@ import { toggleRecording } from '@orca/expo-two-way-audio' import type { RpcClient } from '../transport/rpc-client' +import { + MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE, + MOBILE_DICTATION_PCM_SAMPLE_RATE, + MobileDictationPendingAudioBudget +} from './mobile-dictation-pending-audio-budget' type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | 'error' @@ -32,7 +37,6 @@ export type UseMobileDictationResult = { cancel: () => Promise } -const MOBILE_PCM_SAMPLE_RATE = 16000 const DICTATION_FINISH_TIMEOUT_MS = 75_000 function bytesToBase64(bytes: Uint8Array): string { @@ -53,6 +57,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil const onTranscriptRef = useRef(onTranscript) const onErrorRef = useRef(onError) const pendingChunksRef = useRef>>(new Set()) + const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget()) const acceptingChunksRef = useRef(false) const generationRef = useRef(0) const finishingIdRef = useRef(null) @@ -82,6 +87,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil activeIdRef.current = null acceptingChunksRef.current = false pendingChunksRef.current.clear() + pendingAudioBudgetRef.current.reset() toggleRecording(false) if (client && dictationId) { void client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) @@ -100,11 +106,16 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil } const raw = event.data const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw) + const byteLength = bytes.byteLength + if (!pendingAudioBudgetRef.current.tryReserve(byteLength)) { + failActiveDictation(dictationId, new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE)) + return + } const sendChunk = client .sendRequest('speech.dictation.chunk', { dictationId, audioBase64: bytesToBase64(bytes), - sampleRate: MOBILE_PCM_SAMPLE_RATE + sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE }) .then((response) => { if (!response.ok) { @@ -113,6 +124,9 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil }) .catch((err) => failActiveDictation(dictationId, err)) .finally(() => { + if (activeIdRef.current === dictationId || finishingIdRef.current === dictationId) { + pendingAudioBudgetRef.current.release(byteLength) + } pendingChunksRef.current.delete(sendChunk) }) pendingChunksRef.current.add(sendChunk) @@ -185,6 +199,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil acceptingChunksRef.current = true pendingChunksRef.current.clear() + pendingAudioBudgetRef.current.reset() toggleRecording(true) setStatus('recording') }, []) @@ -233,6 +248,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil activeIdRef.current = null finishingIdRef.current = null pendingChunksRef.current.clear() + pendingAudioBudgetRef.current.reset() setStatus('idle') if (text) { onTranscriptRef.current(text) @@ -256,6 +272,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil finishingIdRef.current = null acceptingChunksRef.current = false pendingChunksRef.current.clear() + pendingAudioBudgetRef.current.reset() toggleRecording(false) if (client && dictationId) { await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined) @@ -287,6 +304,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil finishingIdRef.current = null acceptingChunksRef.current = false pendingChunksRef.current.clear() + pendingAudioBudgetRef.current.reset() toggleRecording(false) void tearDown() if (clientRef.current && dictationId) {