diff --git a/mobile/app/h/[hostId]/session/[worktreeId].tsx b/mobile/app/h/[hostId]/session/[worktreeId].tsx index 5175ba90c..1f456dc90 100644 --- a/mobile/app/h/[hostId]/session/[worktreeId].tsx +++ b/mobile/app/h/[hostId]/session/[worktreeId].tsx @@ -36,11 +36,9 @@ import { FileText, GitBranch, Globe, - ImagePlus, Keyboard as KeyboardIcon, ListChecks, MessageSquare, - Mic, Monitor, Plus, RefreshCw, @@ -110,6 +108,10 @@ import { getTerminalLiveInputKeyboardType } from '../../../../src/terminal/terminal-keyboard-type' import { normalizeTerminalTextInput } from '../../../../src/terminal/terminal-text-input-normalization' +import { + appendBufferedDictation, + routeDictationTranscript +} from '../../../../src/terminal/terminal-live-dictation-routing' import { countTerminalGestureInputSequences } from '../../../../src/terminal/terminal-gesture-input' import { recoverActiveTerminalAfterForeground, @@ -163,6 +165,8 @@ import { type MobileClipboardImageResizer } from '../../../../src/session/mobile-clipboard-image' import { useMobileImageAttachment } from '../../../../src/session/use-mobile-image-attachment' +import { MobileTerminalLiveInputStatus } from '../../../../src/session/MobileTerminalLiveInputStatus' +import { MobileTerminalInputActions } from '../../../../src/session/MobileTerminalInputActions' import { classifyMobileArtifact } from '../../../../src/session/mobile-artifact-kind' import { useLiveWorktreeName } from '../../../../src/session/use-live-worktree-name' import { @@ -1044,6 +1048,10 @@ export default function SessionScreen() { const terminalRefs = useRef>(new Map()) const liveInputRef = useRef(null) const liveInputFocusTimerRef = useRef | null>(null) + const dictationRouteContextRef = useRef<{ + readonly handle: string | null + readonly liveInputEnabled: boolean + } | null>(null) const terminalUnsubsRef = useRef void>>(new Map()) const subscribingHandlesRef = useRef>(new Set()) const initializedHandlesRef = useRef>(new Set()) @@ -1232,15 +1240,29 @@ export default function SessionScreen() { client, enabled: canSend, onTranscript: (text) => { - setInput((current) => { - if (!current.trim()) { - return text + // Live mode inserts the transcript straight into its originating PTY as + // text (no Return — the user sends it themselves), matching live keystroke + // semantics; buffered mode keeps appending to the command field. + const routeContext = dictationRouteContextRef.current + dictationRouteContextRef.current = null + const route = routeDictationTranscript( + text, + routeContext?.liveInputEnabled ?? liveInputEnabled + ) + if (route.kind === 'live-insert') { + const insertHandle = routeContext?.handle ?? activeHandleRef.current + if (!insertHandle) { + return } - return `${current.trimEnd()} ${text}` - }) + sendLiveTerminalInput(insertHandle, route.text) + showToast('Dictation inserted') + return + } + setInput((current) => appendBufferedDictation(current, route.text)) showToast('Dictation inserted') }, onError: (err) => { + dictationRouteContextRef.current = null // Dictation isn't set up on the desktop yet → open the setup sheet so the // user can download a model + enable it from here, instead of a dead-end toast. if (isDictationSetupRequiredError(err.message)) { @@ -1253,16 +1275,28 @@ export default function SessionScreen() { }) const startDictation = useCallback(() => { + const routeContext = activeHandle + ? { handle: activeHandle, liveInputEnabled: liveInputTerminalHandles.has(activeHandle) } + : null + dictationRouteContextRef.current = routeContext void dictation.start().catch((err) => { + if (dictationRouteContextRef.current === routeContext) { + dictationRouteContextRef.current = null + } triggerError() showToast(err instanceof Error ? err.message : String(err)) }) - }, [dictation, triggerError, showToast]) + }, [activeHandle, dictation, liveInputTerminalHandles, triggerError, showToast]) + + const cancelDictation = useCallback(() => { + dictationRouteContextRef.current = null + void dictation.cancel() + }, [dictation]) // Toggle mode: one tap starts, the next stops; long-press cancels mid-record. const handleDictationToggle = useCallback(() => { if (dictation.isProcessing) { - void dictation.cancel() + cancelDictation() } else if (dictation.isStarting) { return } else if (dictation.isRecording) { @@ -1270,7 +1304,7 @@ export default function SessionScreen() { } else { startDictation() } - }, [dictation, startDictation]) + }, [cancelDictation, dictation, startDictation]) // Hold mode: press starts, release stops — like a walkie-talkie. const handleDictationPressIn = useCallback(() => { @@ -1284,9 +1318,9 @@ export default function SessionScreen() { void dictation.stop() } else if (dictation.isStarting) { // Released before recording began: cancel so we don't leave a live mic. - void dictation.cancel() + cancelDictation() } - }, [dictation]) + }, [cancelDictation, dictation]) const refreshDictationMode = useCallback(async () => { if (!client) { @@ -4166,10 +4200,11 @@ export default function SessionScreen() { }) if (response.ok) { if (tab.type === 'terminal' && typeof tab.terminal === 'string') { - unsubscribeTerminal(tab.terminal) - terminalRefs.current.delete(tab.terminal) - initializedHandlesRef.current.delete(tab.terminal) - clearTerminalLiveInputDefault(tab.terminal) + const terminalHandle = tab.terminal + unsubscribeTerminal(terminalHandle) + terminalRefs.current.delete(terminalHandle) + initializedHandlesRef.current.delete(terminalHandle) + clearTerminalLiveInputDefault(terminalHandle) } setSessionTabs((prev) => prev.filter((candidate) => candidate.id !== tab.id)) // Why: tombstone the closed tab and rely on the subscription/poll @@ -4980,16 +5015,34 @@ export default function SessionScreen() { {/* Input bar */} {liveInputEnabled ? ( - - - - Keyboard input directly goes to terminal - + + + + + + void attachImage('library')} + onAttachFile={() => void attachImage('files')} + onDictationToggle={handleDictationToggle} + onDictationPressIn={handleDictationPressIn} + onDictationPressOut={handleDictationPressOut} + onDictationCancel={cancelDictation} + /> - + ) : ( void handleSend()} /> - void attachImage('library')} - onLongPress={() => void attachImage('files')} - delayLongPress={350} - accessibilityLabel={isAttaching ? 'Sending image' : 'Attach a photo'} - accessibilityHint="Long press to attach a file instead" - > - {isAttaching ? ( - - ) : ( - - )} - - { - if (dictation.isRecording || dictation.isProcessing) { - void dictation.cancel() - } - } - : undefined - } - accessibilityLabel={ - dictation.isRecording - ? 'Stop voice dictation' - : dictation.isProcessing - ? 'Cancel voice dictation' - : dictation.isStarting - ? 'Starting voice dictation' - : 'Start voice dictation' - } - > - {dictation.isProcessing ? ( - - ) : dictation.isStarting || dictation.isRecording ? ( - - ) : ( - - )} - + void attachImage('library')} + onAttachFile={() => void attachImage('files')} + onDictationToggle={handleDictationToggle} + onDictationPressIn={handleDictationPressIn} + onDictationPressOut={handleDictationPressOut} + onDictationCancel={cancelDictation} + /> + readonly activeButtonStyle: StyleProp + readonly disabledButtonStyle: StyleProp + readonly onAttachImage: () => void + readonly onAttachFile: () => void + readonly onDictationToggle: () => void + readonly onDictationPressIn: () => void + readonly onDictationPressOut: () => void + readonly onDictationCancel: () => void +} + +// Image + mic peer actions shared by the live and buffered input bars so both +// surfaces offer identical multimodal entry points (and the JSX lives once). +export function MobileTerminalInputActions({ + canSend, + isAttaching, + dictation, + dictationMode, + buttonStyle, + activeButtonStyle, + disabledButtonStyle, + onAttachImage, + onAttachFile, + onDictationToggle, + onDictationPressIn, + onDictationPressOut, + onDictationCancel +}: MobileTerminalInputActionsProps) { + const dictationActive = dictation.isStarting || dictation.isRecording + return ( + <> + + {isAttaching ? ( + + ) : ( + + )} + + { + if (dictation.isRecording || dictation.isProcessing) { + onDictationCancel() + } + } + : undefined + } + accessibilityLabel={ + dictation.isRecording + ? 'Stop voice dictation' + : dictation.isProcessing + ? 'Cancel voice dictation' + : dictation.isStarting + ? 'Starting voice dictation' + : 'Start voice dictation' + } + > + {dictation.isProcessing ? ( + + ) : ( + + )} + + + ) +} diff --git a/mobile/src/session/MobileTerminalLiveInputStatus.tsx b/mobile/src/session/MobileTerminalLiveInputStatus.tsx new file mode 100644 index 000000000..bde542ab3 --- /dev/null +++ b/mobile/src/session/MobileTerminalLiveInputStatus.tsx @@ -0,0 +1,61 @@ +import { StyleSheet, Text, View } from 'react-native' +import { colors, typography } from '../theme/mobile-theme' + +type DictationStatus = { + readonly isStarting: boolean + readonly isRecording: boolean + readonly isProcessing: boolean +} + +type MobileTerminalLiveInputStatusProps = { + readonly dictation: DictationStatus + readonly isAttaching: boolean +} + +export function MobileTerminalLiveInputStatus({ + dictation, + isAttaching +}: MobileTerminalLiveInputStatusProps) { + const title = dictation.isRecording + ? 'Listening' + : dictation.isProcessing + ? 'Processing' + : dictation.isStarting + ? 'Starting mic' + : 'Live input' + const detail = dictation.isRecording + ? 'Tap mic to stop' + : dictation.isProcessing + ? 'Transcribing on desktop' + : isAttaching + ? 'Uploading image to host' + : 'Keyboard goes directly to terminal' + + return ( + + + {title} + + + {detail} + + + ) +} + +const styles = StyleSheet.create({ + status: { + flex: 1, + gap: 1 + }, + title: { + color: colors.textPrimary, + fontSize: typography.metaSize, + fontWeight: '600' + }, + detail: { + color: colors.textSecondary, + fontSize: typography.metaSize, + fontFamily: typography.monoFamily + } +}) diff --git a/mobile/src/terminal/terminal-live-dictation-routing.test.ts b/mobile/src/terminal/terminal-live-dictation-routing.test.ts new file mode 100644 index 000000000..81e38d2f3 --- /dev/null +++ b/mobile/src/terminal/terminal-live-dictation-routing.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import { + appendBufferedDictation, + routeDictationTranscript +} from './terminal-live-dictation-routing' + +describe('terminal live dictation routing', () => { + it('routes to a direct live insert when live input is active', () => { + expect(routeDictationTranscript('hello world', true)).toEqual({ + kind: 'live-insert', + text: 'hello world' + }) + }) + + it('routes to buffered append when live input is inactive', () => { + expect(routeDictationTranscript('hello world', false)).toEqual({ + kind: 'buffered-append', + text: 'hello world' + }) + }) + + it('replaces an empty or whitespace-only buffered field', () => { + expect(appendBufferedDictation('', 'spoken')).toBe('spoken') + expect(appendBufferedDictation(' ', 'spoken')).toBe('spoken') + }) + + it('appends after existing buffered text with one separating space', () => { + expect(appendBufferedDictation('ls -la', 'in src')).toBe('ls -la in src') + expect(appendBufferedDictation('ls -la ', 'in src')).toBe('ls -la in src') + }) +}) diff --git a/mobile/src/terminal/terminal-live-dictation-routing.ts b/mobile/src/terminal/terminal-live-dictation-routing.ts new file mode 100644 index 000000000..91776afdd --- /dev/null +++ b/mobile/src/terminal/terminal-live-dictation-routing.ts @@ -0,0 +1,25 @@ +// Routes a finished dictation transcript to the right surface: live mode inserts +// it straight into the originating PTY (matching live keystroke semantics, no +// auto-Return); buffered mode appends to the command field as before. + +export type LiveDictationRoute = + | { readonly kind: 'live-insert'; readonly text: string } + | { readonly kind: 'buffered-append'; readonly text: string } + +export function routeDictationTranscript( + transcript: string, + liveInputActive: boolean +): LiveDictationRoute { + return liveInputActive + ? { kind: 'live-insert', text: transcript } + : { kind: 'buffered-append', text: transcript } +} + +// Mirrors the prior buffered onTranscript behavior: append after existing text +// with a single separating space, or replace an empty/whitespace-only field. +export function appendBufferedDictation(current: string, transcript: string): string { + if (!current.trim()) { + return transcript + } + return `${current.trimEnd()} ${transcript}` +}