perf: cap mobile dictation audio backlog (#4078)

mobile/src/hooks/mobile-dictation-pending-audio-budget.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest'
+import {
+  MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES,
+  MobileDictationPendingAudioBudget
+} from './mobile-dictation-pending-audio-budget'
+
+describe('MobileDictationPendingAudioBudget', () => {
+  it('caps pending raw PCM audio at five seconds', () => {
+    expect(MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES).toBe(160_000)
+
+    const budget = new MobileDictationPendingAudioBudget()
+    expect(budget.tryReserve(159_999)).toBe(true)
+    expect(budget.tryReserve(1)).toBe(true)
+    expect(budget.pendingAudioBytes).toBe(160_000)
+    expect(budget.tryReserve(1)).toBe(false)
+    expect(budget.pendingAudioBytes).toBe(160_000)
+  })
+
+  it('releases completed chunks and clamps stale releases after reset', () => {
+    const budget = new MobileDictationPendingAudioBudget(10)
+
+    expect(budget.tryReserve(6)).toBe(true)
+    budget.release(4)
+    expect(budget.pendingAudioBytes).toBe(2)
+    expect(budget.tryReserve(8)).toBe(true)
+
+    budget.reset()
+    budget.release(6)
+    expect(budget.pendingAudioBytes).toBe(0)
+    expect(budget.tryReserve(10)).toBe(true)
+  })
+})

mobile/src/hooks/mobile-dictation-pending-audio-budget.ts

@@ -0,0 +1,46 @@
+export const MOBILE_DICTATION_PCM_SAMPLE_RATE = 16000
+
+const PCM_BYTES_PER_SAMPLE = 2
+const MAX_PENDING_AUDIO_SECONDS = 5
+
+// Why: dictation chunk RPCs can wait through reconnect before timing out, so
+// cap retained raw microphone audio before it is expanded into base64 payloads.
+export const MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES =
+  MOBILE_DICTATION_PCM_SAMPLE_RATE * PCM_BYTES_PER_SAMPLE * MAX_PENDING_AUDIO_SECONDS
+
+export const MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE =
+  'Connection is too slow for voice dictation. Try again when the connection improves.'
+
+export class MobileDictationPendingAudioBudget {
+  private pendingBytes = 0
+
+  constructor(private readonly maxPendingBytes = MOBILE_DICTATION_MAX_PENDING_AUDIO_BYTES) {}
+
+  get pendingAudioBytes(): number {
+    return this.pendingBytes
+  }
+
+  tryReserve(byteLength: number): boolean {
+    const normalizedByteLength = normalizeByteLength(byteLength)
+    if (this.pendingBytes + normalizedByteLength > this.maxPendingBytes) {
+      return false
+    }
+    this.pendingBytes += normalizedByteLength
+    return true
+  }
+
+  release(byteLength: number): void {
+    this.pendingBytes = Math.max(0, this.pendingBytes - normalizeByteLength(byteLength))
+  }
+
+  reset(): void {
+    this.pendingBytes = 0
+  }
+}
+
+function normalizeByteLength(byteLength: number): number {
+  if (!Number.isFinite(byteLength) || byteLength <= 0) {
+    return 0
+  }
+  return Math.floor(byteLength)
+}

mobile/src/hooks/use-mobile-dictation-source.test.ts

@@ -26,4 +26,26 @@ describe('useMobileDictation source invariants', () => {
     expect(mirrorEffect).toContain('onErrorRef.current = onError')
     expect(mirrorEffect).toContain('}, [client, enabled, onTranscript, onError])')
   })
+
+  it('reserves pending audio bytes before encoding microphone chunks', () => {
+    const microphoneEffect = sliceBetween(
+      "addExpoTwoWayAudioEventListener('onMicrophoneData'",
+      'return () => sub.remove()'
+    )
+
+    const reserveIndex = microphoneEffect.indexOf('tryReserve(byteLength)')
+    const encodeIndex = microphoneEffect.indexOf('bytesToBase64(bytes)')
+    expect(reserveIndex).toBeGreaterThanOrEqual(0)
+    expect(encodeIndex).toBeGreaterThanOrEqual(0)
+    expect(reserveIndex).toBeLessThan(encodeIndex)
+    expect(microphoneEffect).toContain('MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE')
+    expect(microphoneEffect).toContain('pendingAudioBudgetRef.current.release(byteLength)')
+  })
+
+  it('resets pending audio bytes whenever pending chunk tracking is cleared', () => {
+    const pendingChunkClears = source.match(/pendingChunksRef\.current\.clear\(\)/g) ?? []
+    const pendingAudioResets = source.match(/pendingAudioBudgetRef\.current\.reset\(\)/g) ?? []
+
+    expect(pendingAudioResets).toHaveLength(pendingChunkClears.length)
+  })
 })

mobile/src/hooks/use-mobile-dictation.ts

@@ -11,6 +11,11 @@ import {
   toggleRecording
 } from '@orca/expo-two-way-audio'
 import type { RpcClient } from '../transport/rpc-client'
+import {
+  MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE,
+  MOBILE_DICTATION_PCM_SAMPLE_RATE,
+  MobileDictationPendingAudioBudget
+} from './mobile-dictation-pending-audio-budget'
 
 type DictationStatus = 'idle' | 'starting' | 'recording' | 'processing' | 'error'
 
@@ -32,7 +37,6 @@ export type UseMobileDictationResult = {
   cancel: () => Promise<void>
 }
 
-const MOBILE_PCM_SAMPLE_RATE = 16000
 const DICTATION_FINISH_TIMEOUT_MS = 75_000
 
 function bytesToBase64(bytes: Uint8Array): string {
@@ -53,6 +57,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
   const onTranscriptRef = useRef(onTranscript)
   const onErrorRef = useRef(onError)
   const pendingChunksRef = useRef<Set<Promise<void>>>(new Set())
+  const pendingAudioBudgetRef = useRef(new MobileDictationPendingAudioBudget())
   const acceptingChunksRef = useRef(false)
   const generationRef = useRef(0)
   const finishingIdRef = useRef<string | null>(null)
@@ -82,6 +87,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
       activeIdRef.current = null
       acceptingChunksRef.current = false
       pendingChunksRef.current.clear()
+      pendingAudioBudgetRef.current.reset()
       toggleRecording(false)
       if (client && dictationId) {
         void client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
@@ -100,11 +106,16 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
       }
       const raw = event.data
       const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw)
+      const byteLength = bytes.byteLength
+      if (!pendingAudioBudgetRef.current.tryReserve(byteLength)) {
+        failActiveDictation(dictationId, new Error(MOBILE_DICTATION_CONNECTION_SLOW_ERROR_MESSAGE))
+        return
+      }
       const sendChunk = client
         .sendRequest('speech.dictation.chunk', {
           dictationId,
           audioBase64: bytesToBase64(bytes),
-          sampleRate: MOBILE_PCM_SAMPLE_RATE
+          sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE
         })
         .then((response) => {
           if (!response.ok) {
@@ -113,6 +124,9 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
         })
         .catch((err) => failActiveDictation(dictationId, err))
         .finally(() => {
+          if (activeIdRef.current === dictationId || finishingIdRef.current === dictationId) {
+            pendingAudioBudgetRef.current.release(byteLength)
+          }
           pendingChunksRef.current.delete(sendChunk)
         })
       pendingChunksRef.current.add(sendChunk)
@@ -185,6 +199,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
 
     acceptingChunksRef.current = true
     pendingChunksRef.current.clear()
+    pendingAudioBudgetRef.current.reset()
     toggleRecording(true)
     setStatus('recording')
   }, [])
@@ -233,6 +248,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
       activeIdRef.current = null
       finishingIdRef.current = null
       pendingChunksRef.current.clear()
+      pendingAudioBudgetRef.current.reset()
       setStatus('idle')
       if (text) {
         onTranscriptRef.current(text)
@@ -256,6 +272,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
     finishingIdRef.current = null
     acceptingChunksRef.current = false
     pendingChunksRef.current.clear()
+    pendingAudioBudgetRef.current.reset()
     toggleRecording(false)
     if (client && dictationId) {
       await client.sendRequest('speech.dictation.cancel', { dictationId }).catch(() => undefined)
@@ -287,6 +304,7 @@ export function useMobileDictation(options: UseMobileDictationOptions): UseMobil
       finishingIdRef.current = null
       acceptingChunksRef.current = false
       pendingChunksRef.current.clear()
+      pendingAudioBudgetRef.current.reset()
       toggleRecording(false)
       void tearDown()
       if (clientRef.current && dictationId) {