xiaoxue_admin
/
orca
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add live terminal input mode on mobile (#2982) 

Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Jinwoo Hong 2026-05-28 16:41:37 -04:00 committed by GitHub
parent f3b452fff5
commit 6df52607fc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 343 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

352
mobile/app/h/[hostId]/session/[worktreeId].tsx
View File

@ -31,6 +31,7 @@ import {
FileText,
GitBranch,
Globe,
Keyboard as KeyboardIcon,
Mic,
Monitor,
Plus,
@ -59,6 +60,10 @@ import {
type TerminalWebViewHandle
} from '../../../../src/terminal/TerminalWebView'
import { TERMINAL_ACCESSORY_KEYS } from '../../../../src/terminal/terminal-accessory-keys'
import {
getTerminalLiveSpecialKeyBytes,
isTerminalLiveInputWithinByteLimit
} from '../../../../src/terminal/terminal-live-input'
import { countTerminalGestureInputSequences } from '../../../../src/terminal/terminal-gesture-input'
import { MobileBrowserPane, type MobileBrowserTab } from '../../../../src/browser/MobileBrowserPane'
import { isBlankBrowserUrl, normalizeBrowserUrl } from '../../../../src/browser/browser-url'
@ -335,7 +340,8 @@ function TerminalPaneView({
onModesChanged,
onKeyboardAvoidanceMetrics,
onHaptic,
onTerminalInput
onTerminalInput,
onTerminalTap
}: {
handle: string
active: boolean
@ -350,6 +356,7 @@ function TerminalPaneView({
onKeyboardAvoidanceMetrics: (handle: string, metrics: TerminalKeyboardAvoidanceMetrics) => void
onHaptic: (kind: 'selection' | 'success' | 'error' | 'edge-bump') => void
onTerminalInput: (handle: string, bytes: string) => void
onTerminalTap: (handle: string) => void
}) {
const setRef = useCallback(
(ref: TerminalWebViewHandle | null) => {
@ -379,6 +386,7 @@ function TerminalPaneView({
onKeyboardAvoidanceMetrics={(m) => onKeyboardAvoidanceMetrics(handle, m)}
onHaptic={onHaptic}
onTerminalInput={(bytes) => onTerminalInput(handle, bytes)}
onTerminalTap={() => onTerminalTap(handle)}
/>
</View>
)
@ -679,6 +687,10 @@ export default function SessionScreen() {
const sessionTabsRef = useRef<MobileSessionTab[]>([])
const [terminalsLoaded, setTerminalsLoaded] = useState(false)
const [input, setInput] = useState('')
const [liveInputCapture, setLiveInputCapture] = useState('')
const [liveInputTerminalHandles, setLiveInputTerminalHandles] = useState<Set<string>>(
() => new Set()
)
const [activeHandle, setActiveHandle] = useState<string | null>(null)
const [activeSessionTabId, setActiveSessionTabId] = useState<string | null>(null)
const activeSessionTabIdRef = useRef<string | null>(null)
@ -745,6 +757,7 @@ export default function SessionScreen() {
const viewportRef = useRef<{ cols: number; rows: number } | null>(null)
const viewportMeasuredRef = useRef(false)
const terminalRefs = useRef<Map<string, TerminalWebViewHandle>>(new Map())
const liveInputRef = useRef<TextInput>(null)
const terminalUnsubsRef = useRef<Map<string, () => void>>(new Map())
const subscribingHandlesRef = useRef<Set<string>>(new Set())
const initializedHandlesRef = useRef<Set<string>>(new Set())
@ -781,6 +794,7 @@ export default function SessionScreen() {
activeSessionTab?.type !== 'markdown' &&
activeSessionTab?.type !== 'file' &&
activeSessionTab?.type !== 'browser'
const liveInputEnabled = activeHandle ? liveInputTerminalHandles.has(activeHandle) : false
const [browserScreencastSupported, setBrowserScreencastSupported] = useState<boolean | null>(null)
useEffect(() => {
@ -1818,6 +1832,8 @@ export default function SessionScreen() {
terminalsRef.current = []
setSessionTabs([])
setActiveSessionTabId(null)
setLiveInputCapture('')
setLiveInputTerminalHandles(new Set())
setMarkdownDocs(new Map())
setFileDocs(new Map())
}, [clearTerminalCache, worktreeId])
@ -2146,6 +2162,117 @@ export default function SessionScreen() {
}
}
const sendLiveTerminalInput = useCallback(
(handle: string, bytes: string) => {
if (bytes.length === 0) return
if (!isTerminalLiveInputWithinByteLimit(bytes)) {
triggerError()
showToast('Input too large (max 256 KiB)', 1500)
return
}
const rpc = clientRef.current
if (
!rpc ||
connStateRef.current !== 'connected' ||
handle !== activeHandleRef.current ||
activeSessionTabTypeRef.current !== 'terminal'
) {
return
}
void rpc
.sendRequest('terminal.send', {
terminal: handle,
text: bytes,
enter: false,
...(deviceTokenRef.current
? { client: { id: deviceTokenRef.current, type: 'mobile' as const } }
: {})
})
.catch(() => {
// Transient failure
})
},
[showToast]
)
const focusLiveInput = useCallback(() => {
if (!canSend || !liveInputEnabled) return
liveInputRef.current?.focus()
}, [canSend, liveInputEnabled])
const handleTerminalTap = useCallback(
(handle: string) => {
if (handle !== activeHandleRef.current) return
focusLiveInput()
},
[focusLiveInput]
)
const toggleLiveInput = useCallback(() => {
if (!activeHandle) return
const nextEnabled = !liveInputTerminalHandles.has(activeHandle)
setLiveInputTerminalHandles((prev) => {
const next = new Set(prev)
if (nextEnabled) {
next.add(activeHandle)
} else {
next.delete(activeHandle)
}
return next
})
setLiveInputCapture('')
if (nextEnabled) {
setTimeout(() => liveInputRef.current?.focus(), 50)
} else {
liveInputRef.current?.blur()
}
}, [activeHandle, liveInputTerminalHandles])
const handleLiveInputChange = useCallback(
(text: string) => {
if (!activeHandle) {
setLiveInputCapture('')
liveInputRef.current?.setNativeProps({ text: '' })
return
}
if (!liveInputTerminalHandles.has(activeHandle)) {
setLiveInputCapture('')
liveInputRef.current?.setNativeProps({ text: '' })
return
}
if (text.length > 0) {
sendLiveTerminalInput(activeHandle, text)
}
setLiveInputCapture('')
// Why: the field is only a keyboard capture surface. Clearing the
// native value prevents subsequent phone-keyboard events from replaying
// already-sent characters when React state remains the empty string.
liveInputRef.current?.setNativeProps({ text: '' })
},
[activeHandle, liveInputTerminalHandles, sendLiveTerminalInput]
)
const handleLiveInputKeyPress = useCallback(
(event: { nativeEvent: { key: string } }) => {
if (!activeHandle) return
if (!liveInputTerminalHandles.has(activeHandle)) return
const bytes = getTerminalLiveSpecialKeyBytes(event.nativeEvent.key)
if (!bytes) return
sendLiveTerminalInput(activeHandle, bytes)
setLiveInputCapture('')
liveInputRef.current?.setNativeProps({ text: '' })
},
[activeHandle, liveInputTerminalHandles, sendLiveTerminalInput]
)
const handleLiveInputSubmit = useCallback(() => {
if (!activeHandle) return
if (!liveInputTerminalHandles.has(activeHandle)) return
sendLiveTerminalInput(activeHandle, '\r')
setLiveInputCapture('')
liveInputRef.current?.setNativeProps({ text: '' })
}, [activeHandle, liveInputTerminalHandles, sendLiveTerminalInput])
const allowTerminalGestureInput = useCallback(
(handle: string, sequenceCount: number): boolean => {
const now = Date.now()
@ -3085,6 +3212,7 @@ export default function SessionScreen() {
onKeyboardAvoidanceMetrics={handleKeyboardAvoidanceMetrics}
onHaptic={handleHaptic}
onTerminalInput={handleTerminalInput}
onTerminalTap={handleTerminalTap}
/>
))}
{toastMessage && (
@ -3136,6 +3264,31 @@ export default function SessionScreen() {
/>
)}
</Pressable>
<Pressable
style={({ pressed }) => [
styles.accessoryKey,
liveInputEnabled && styles.accessoryKeyActive,
pressed && styles.accessoryKeyPressed,
!canSend && styles.accessoryKeyDisabled
]}
disabled={!canSend}
onPress={toggleLiveInput}
accessibilityLabel={
liveInputEnabled
? 'Switch to buffered command input'
: 'Switch to live terminal input'
}
>
<Text
style={[
styles.accessoryKeyText,
liveInputEnabled && styles.accessoryKeyTextActive,
!canSend && styles.accessoryKeyTextDisabled
]}
>
Live
</Text>
</Pressable>
{canPaste && (
<Pressable
style={({ pressed }) => [
@ -3223,72 +3376,107 @@ export default function SessionScreen() {
</View>
{/* Input bar */}
<View style={styles.inputBar}>
<TextInput
style={styles.textInput}
value={input}
onChangeText={setInput}
placeholder="Type a command…"
placeholderTextColor={colors.textMuted}
autoCapitalize="none"
autoCorrect={false}
returnKeyType="send"
editable={canSend}
onSubmitEditing={() => void handleSend()}
/>
{liveInputEnabled ? (
<Pressable
style={[
styles.dictationButton,
(dictation.isStarting || dictation.isRecording) && styles.dictationButtonActive,
!canSend && styles.sendButtonDisabled
]}
style={[styles.inputBar, styles.liveInputBar]}
disabled={!canSend}
onPress={() => {
if (dictation.isProcessing) {
void dictation.cancel()
} else if (dictation.isStarting) {
return
} else if (dictation.isRecording) {
void dictation.stop()
} else {
void dictation.start().catch((err) => {
triggerError()
showToast(err instanceof Error ? err.message : String(err))
})
}
}}
onLongPress={() => {
if (dictation.isRecording || dictation.isProcessing) {
void dictation.cancel()
}
}}
accessibilityLabel={
dictation.isRecording
? 'Stop voice dictation'
: dictation.isProcessing
? 'Cancel voice dictation'
: dictation.isStarting
? 'Starting voice dictation'
: 'Start voice dictation'
}
onPress={focusLiveInput}
accessibilityLabel="Focus live terminal input"
>
{dictation.isProcessing ? (
<ActivityIndicator size="small" color={colors.textSecondary} />
) : dictation.isStarting || dictation.isRecording ? (
<Mic size={17} color={colors.textPrimary} strokeWidth={2.4} />
) : (
<Mic size={17} color={colors.textSecondary} strokeWidth={2.4} />
)}
<View style={styles.liveInputBadge}>
<KeyboardIcon size={13} color={colors.textPrimary} strokeWidth={2.2} />
<Text style={styles.liveInputBadgeText}>Live</Text>
</View>
<Text style={styles.liveInputHint} numberOfLines={1}>
Keyboard input goes to terminal
</Text>
<TextInput
ref={liveInputRef}
style={styles.liveInputCapture}
value={liveInputCapture}
onChangeText={handleLiveInputChange}
onKeyPress={handleLiveInputKeyPress}
onSubmitEditing={handleLiveInputSubmit}
placeholder=""
autoCapitalize="none"
autoCorrect={false}
spellCheck={false}
keyboardType={Platform.OS === 'ios' ? 'ascii-capable' : 'visible-password'}
returnKeyType="default"
blurOnSubmit={false}
editable={canSend}
importantForAutofill="no"
textContentType="none"
/>
</Pressable>
<Pressable
style={[styles.sendButton, !canSend && styles.sendButtonDisabled]}
disabled={!canSend}
onPress={() => void handleSend()}
accessibilityLabel="Send command"
>
<ArrowUp size={18} color={colors.textSecondary} strokeWidth={2.5} />
</Pressable>
</View>
) : (
<View style={styles.inputBar}>
<TextInput
style={styles.textInput}
value={input}
onChangeText={setInput}
placeholder="Type a command…"
placeholderTextColor={colors.textMuted}
autoCapitalize="none"
autoCorrect={false}
returnKeyType="send"
editable={canSend}
onSubmitEditing={() => void handleSend()}
/>
<Pressable
style={[
styles.dictationButton,
(dictation.isStarting || dictation.isRecording) && styles.dictationButtonActive,
!canSend && styles.sendButtonDisabled
]}
disabled={!canSend}
onPress={() => {
if (dictation.isProcessing) {
void dictation.cancel()
} else if (dictation.isStarting) {
return
} else if (dictation.isRecording) {
void dictation.stop()
} else {
void dictation.start().catch((err) => {
triggerError()
showToast(err instanceof Error ? err.message : String(err))
})
}
}}
onLongPress={() => {
if (dictation.isRecording || dictation.isProcessing) {
void dictation.cancel()
}
}}
accessibilityLabel={
dictation.isRecording
? 'Stop voice dictation'
: dictation.isProcessing
? 'Cancel voice dictation'
: dictation.isStarting
? 'Starting voice dictation'
: 'Start voice dictation'
}
>
{dictation.isProcessing ? (
<ActivityIndicator size="small" color={colors.textSecondary} />
) : dictation.isStarting || dictation.isRecording ? (
<Mic size={17} color={colors.textPrimary} strokeWidth={2.4} />
) : (
<Mic size={17} color={colors.textSecondary} strokeWidth={2.4} />
)}
</Pressable>
<Pressable
style={[styles.sendButton, !canSend && styles.sendButtonDisabled]}
disabled={!canSend}
onPress={() => void handleSend()}
accessibilityLabel="Send command"
>
<ArrowUp size={18} color={colors.textSecondary} strokeWidth={2.5} />
</Pressable>
</View>
)}
</View>
)}
</View>
@ -4005,6 +4193,9 @@ const styles = StyleSheet.create({
accessoryKeyPressed: {
backgroundColor: colors.borderSubtle
},
accessoryKeyActive: {
backgroundColor: colors.accentBlue
},
customAccessoryKey: {
borderWidth: 1,
borderColor: colors.borderSubtle
@ -4017,6 +4208,10 @@ const styles = StyleSheet.create({
fontSize: 12,
fontFamily: typography.monoFamily
},
accessoryKeyTextActive: {
color: colors.textPrimary,
fontWeight: '700'
},
accessoryKeyTextDisabled: {
color: colors.textMuted
},
@ -4040,6 +4235,37 @@ const styles = StyleSheet.create({
fontFamily: typography.monoFamily,
marginRight: spacing.sm
},
liveInputBar: {
gap: spacing.sm
},
liveInputBadge: {
flexDirection: 'row',
alignItems: 'center',
gap: spacing.xs,
backgroundColor: colors.accentBlue,
paddingHorizontal: spacing.sm,
paddingVertical: spacing.xs,
borderRadius: radii.button
},
liveInputBadgeText: {
color: colors.textPrimary,
fontSize: 12,
fontWeight: '700',
fontFamily: typography.monoFamily
},
liveInputHint: {
flex: 1,
color: colors.textSecondary,
fontSize: typography.metaSize,
fontFamily: typography.monoFamily
},
liveInputCapture: {
position: 'absolute',
opacity: 0,
width: 1,
height: 1,
color: colors.textPrimary
},
sendButton: {
backgroundColor: colors.bgRaised,
width: 34,

12
mobile/src/terminal/TerminalWebView.tsx
View File

@ -31,6 +31,7 @@ export type TerminalSelectionEvents = {
onKeyboardAvoidanceMetrics?: (metrics: TerminalKeyboardAvoidanceMetrics) => void
onHaptic?: (kind: 'selection' | 'success' | 'error' | 'edge-bump') => void
onTerminalInput?: (bytes: string) => void
onTerminalTap?: () => void
}
export type TerminalWebViewHandle = {
@ -1538,6 +1539,9 @@ const XTERM_HTML = `<!DOCTYPE html>
return;
}
if (dispatch.mode === 'surface') {
if (e.touches.length === 0 && longPressOrigin && selMode !== 'select') {
notify({ type: 'terminal-tap' });
}
clearLongPress();
if (e.touches.length === 0) {
dispatch.mode = 'idle';
@ -1797,7 +1801,8 @@ export const TerminalWebView = forwardRef<TerminalWebViewHandle, Props>(function
onModesChanged,
onKeyboardAvoidanceMetrics,
onHaptic,
onTerminalInput
onTerminalInput,
onTerminalTap
},
ref
) {
@ -1900,6 +1905,8 @@ export const TerminalWebView = forwardRef<TerminalWebViewHandle, Props>(function
} else if (msg.type === 'terminal-input') {
const bytes = typeof msg.bytes === 'string' ? msg.bytes : ''
if (bytes.length > 0) onTerminalInput?.(bytes)
} else if (msg.type === 'terminal-tap') {
onTerminalTap?.()
} else if (msg.type === 'keyboard-avoidance-metrics') {
const cursorY = typeof msg.cursorY === 'number' ? msg.cursorY : 0
const rows = typeof msg.rows === 'number' ? msg.rows : 0
@ -1932,7 +1939,8 @@ export const TerminalWebView = forwardRef<TerminalWebViewHandle, Props>(function
onModesChanged,
onKeyboardAvoidanceMetrics,
onHaptic,
onTerminalInput
onTerminalInput,
onTerminalTap
]
)

25
mobile/src/terminal/terminal-live-input.test.ts Normal file
View File

@ -0,0 +1,25 @@
import { describe, expect, it } from 'vitest'
import {
TERMINAL_LIVE_INPUT_MAX_BYTES,
getTerminalLiveSpecialKeyBytes,
isTerminalLiveInputWithinByteLimit
} from './terminal-live-input'
describe('terminal live input', () => {
it('maps phone keyboard special keys to PTY bytes', () => {
expect(getTerminalLiveSpecialKeyBytes('Backspace')).toBe('\x7f')
expect(getTerminalLiveSpecialKeyBytes('Enter')).toBeNull()
expect(getTerminalLiveSpecialKeyBytes('a')).toBeNull()
})
it('enforces the paste-sized byte budget', () => {
expect(isTerminalLiveInputWithinByteLimit('hello')).toBe(true)
expect(isTerminalLiveInputWithinByteLimit('x'.repeat(TERMINAL_LIVE_INPUT_MAX_BYTES))).toBe(true)
expect(isTerminalLiveInputWithinByteLimit('x'.repeat(TERMINAL_LIVE_INPUT_MAX_BYTES + 1))).toBe(
false
)
expect(
isTerminalLiveInputWithinByteLimit('é'.repeat(TERMINAL_LIVE_INPUT_MAX_BYTES / 2 + 1))
).toBe(false)
})
})

19
mobile/src/terminal/terminal-live-input.ts Normal file
View File

@ -0,0 +1,19 @@
const TERMINAL_LIVE_INPUT_MAX_BYTES = 256 * 1024
const encoder = new TextEncoder()
export function getTerminalLiveSpecialKeyBytes(key: string): string | null {
if (key === 'Backspace') {
return '\x7f'
}
return null
}
export function isTerminalLiveInputWithinByteLimit(
text: string,
maxBytes = TERMINAL_LIVE_INPUT_MAX_BYTES
): boolean {
return encoder.encode(text).byteLength <= maxBytes
}
export { TERMINAL_LIVE_INPUT_MAX_BYTES }