diff --git a/mobile/src/session/MobileNativeChatOverlay.test.ts b/mobile/src/session/MobileNativeChatOverlay.test.ts index 2680064c9..c8e62c146 100644 --- a/mobile/src/session/MobileNativeChatOverlay.test.ts +++ b/mobile/src/session/MobileNativeChatOverlay.test.ts @@ -47,6 +47,7 @@ function overlayElement(tick: Tick): ReturnType { nativeChatStreamLive: tick.streamLive ?? false, nativeChatStreamScopeKey: tick.identity ?? 'tab-a', chatPending: [], + chatImagePreviewsByMessageId: {}, chatComposerText: '', setChatComposerText: vi.fn() } as unknown as MobileNativeChatController diff --git a/mobile/src/session/MobileNativeChatOverlay.tsx b/mobile/src/session/MobileNativeChatOverlay.tsx index 4361c185d..699f8a161 100644 --- a/mobile/src/session/MobileNativeChatOverlay.tsx +++ b/mobile/src/session/MobileNativeChatOverlay.tsx @@ -82,6 +82,7 @@ export function MobileNativeChatOverlay({ onLoadEarlier={session.loadEarlier} onSend={images.sendNativeChat} pending={controller.chatPending} + imagePreviewsByMessageId={controller.chatImagePreviewsByMessageId} composerText={controller.chatComposerText} onComposerTextChange={controller.setChatComposerText} onAttachImage={() => void images.attachImage('library')} diff --git a/mobile/src/session/MobileNativeChatView.tsx b/mobile/src/session/MobileNativeChatView.tsx index 48a1fcd31..478d36267 100644 --- a/mobile/src/session/MobileNativeChatView.tsx +++ b/mobile/src/session/MobileNativeChatView.tsx @@ -61,6 +61,9 @@ type Props = { /** Optimistic queued sends (owned by the route so they survive view switches). */ /** Optimistic user echoes, including any ridden-along image preview URIs. */ pending: MobileNativeChatPendingItem[] + /** Local photo URIs retained when the authoritative transcript replaces an + * optimistic image bubble. */ + imagePreviewsByMessageId?: Record /** Controlled composer text (owned by the route so dictation can write to it). */ composerText: string onComposerTextChange: (text: string) => void @@ -125,6 +128,7 @@ export function MobileNativeChatView({ onLoadEarlier, onSend, pending, + imagePreviewsByMessageId, composerText, onComposerTextChange, onAttachImage, @@ -177,8 +181,14 @@ export function MobileNativeChatView({ // route-owned optimistic queued messages. Memoize on the same deps so the // downstream autoscroll effects/`renderItem` keep referential stability. const { data } = useMemo( - () => buildMobileNativeChatTransientData({ folded, streaming, pending }), - [folded, streaming, pending] + () => + buildMobileNativeChatTransientData({ + folded, + streaming, + pending, + imagePreviewsByMessageId + }), + [folded, streaming, pending, imagePreviewsByMessageId] ) // Follow the tail as the conversation grows and keep the newest message above diff --git a/mobile/src/session/mobile-image-source-picker.test.ts b/mobile/src/session/mobile-image-source-picker.test.ts index fbdd9eb5d..35e3cff3b 100644 --- a/mobile/src/session/mobile-image-source-picker.test.ts +++ b/mobile/src/session/mobile-image-source-picker.test.ts @@ -12,13 +12,28 @@ vi.mock('expo-file-system', () => ({ File: vi.fn() })) -import { ImageLibraryPermissionError, pickMobileImage } from './mobile-image-source-picker' +import { + ImageLibraryPermissionError, + pickMobileImage, + pickMobileImages, + type PickedMobileImage +} from './mobile-image-source-picker' const granted = { granted: true } as Awaited< ReturnType > const denied = { granted: false } as typeof granted +async function collectImages( + images: AsyncIterable +): Promise { + const collected: PickedMobileImage[] = [] + for await (const image of images) { + collected.push(image) + } + return collected +} + function fileFactory( bytes: Uint8Array, options?: { fileSize?: number; handleSize?: number | null; readError?: Error } @@ -68,6 +83,57 @@ describe('pickMobileImage', () => { expect(file.close).toHaveBeenCalledTimes(1) }) + it('returns every selected library photo in order', async () => { + const bytesByUri = new Map([ + ['file:///a.jpg', new Uint8Array([1])], + ['file:///b.jpg', new Uint8Array([2])], + ['file:///c.jpg', new Uint8Array([3])] + ]) + const createFile = vi.fn((uri: string) => { + const bytes = bytesByUri.get(uri)! + let read = false + return { + size: bytes.length, + open: () => ({ + size: bytes.length, + readBytes: () => { + if (read) { + return new Uint8Array() + } + read = true + return bytes + }, + close: vi.fn() + }) + } + }) + const launchLibrary = vi.fn().mockResolvedValue({ + canceled: false, + assets: [...bytesByUri].map(([uri, bytes]) => ({ uri, fileSize: bytes.length })) + }) + + const result = await collectImages( + pickMobileImages('library', { + requestLibraryPermission: vi.fn().mockResolvedValue(granted), + launchLibrary, + createFile + }) + ) + + expect(result.map((image) => image.uri)).toEqual([ + 'file:///a.jpg', + 'file:///b.jpg', + 'file:///c.jpg' + ]) + expect(launchLibrary).toHaveBeenCalledWith( + expect.objectContaining({ + allowsMultipleSelection: true, + orderedSelection: true, + selectionLimit: 0 + }) + ) + }) + it('throws when photo library permission is denied', async () => { await expect( pickMobileImage('library', { diff --git a/mobile/src/session/mobile-image-source-picker.ts b/mobile/src/session/mobile-image-source-picker.ts index bf9c529cc..062085f72 100644 --- a/mobile/src/session/mobile-image-source-picker.ts +++ b/mobile/src/session/mobile-image-source-picker.ts @@ -83,11 +83,12 @@ async function readUriAsBase64( } } -async function pickFromLibrary( +async function* pickFromLibrary( + multiple: boolean, requestPermission: typeof ImagePicker.requestMediaLibraryPermissionsAsync = ImagePicker.requestMediaLibraryPermissionsAsync, launch: typeof ImagePicker.launchImageLibraryAsync = ImagePicker.launchImageLibraryAsync, createFile: MobileImageFileFactory = defaultMobileImageFileFactory -): Promise { +): AsyncGenerator { const permission = await requestPermission() // Why: `granted` covers full + limited iOS access; only a hard denial blocks us. if (!permission.granted) { @@ -96,51 +97,84 @@ async function pickFromLibrary( const result = await launch({ mediaTypes: ['images'], base64: false, - allowsMultipleSelection: false, + allowsMultipleSelection: multiple, + ...(multiple ? { selectionLimit: 0, orderedSelection: true } : {}), quality: 1 }) if (result.canceled) { - return null + return } - const asset = result.assets[0] - const base64 = asset?.uri ? await readUriAsBase64(asset.uri, asset.fileSize, createFile) : null - if (!base64) { - return null + for (const asset of result.assets) { + if (!asset.uri) { + continue + } + const base64 = await readUriAsBase64(asset.uri, asset.fileSize, createFile) + if (base64) { + yield { base64, uri: asset.uri } + } } - return { base64, ...(asset?.uri ? { uri: asset.uri } : {}) } } -async function pickFromFiles( +async function* pickFromFiles( + multiple: boolean, launch: typeof DocumentPicker.getDocumentAsync = DocumentPicker.getDocumentAsync, createFile: MobileImageFileFactory = defaultMobileImageFileFactory -): Promise { +): AsyncGenerator { const result = await launch({ type: 'image/*', - multiple: false, + multiple, copyToCacheDirectory: true }) if (result.canceled) { - return null + return } - const asset = result.assets[0] - if (!asset?.uri) { - return null + for (const asset of result.assets) { + if (!asset.uri) { + continue + } + const base64 = await readUriAsBase64(asset.uri, asset.size, createFile) + if (base64) { + yield { base64, uri: asset.uri } + } } - const base64 = await readUriAsBase64(asset.uri, asset.size, createFile) - return base64 ? { base64, uri: asset.uri } : null +} + +type MobileImagePickerDeps = { + readonly requestLibraryPermission?: typeof ImagePicker.requestMediaLibraryPermissionsAsync + readonly launchLibrary?: typeof ImagePicker.launchImageLibraryAsync + readonly launchFiles?: typeof DocumentPicker.getDocumentAsync + readonly createFile?: MobileImageFileFactory +} + +function pickMobileImagesWithMode( + source: MobileImageSource, + multiple: boolean, + deps?: MobileImagePickerDeps +): AsyncIterable { + if (source === 'library') { + return pickFromLibrary( + multiple, + deps?.requestLibraryPermission, + deps?.launchLibrary, + deps?.createFile + ) + } + return pickFromFiles(multiple, deps?.launchFiles, deps?.createFile) } export async function pickMobileImage( source: MobileImageSource, - deps?: { - readonly requestLibraryPermission?: typeof ImagePicker.requestMediaLibraryPermissionsAsync - readonly launchLibrary?: typeof ImagePicker.launchImageLibraryAsync - readonly launchFiles?: typeof DocumentPicker.getDocumentAsync - readonly createFile?: MobileImageFileFactory - } + deps?: MobileImagePickerDeps ): Promise { - if (source === 'library') { - return pickFromLibrary(deps?.requestLibraryPermission, deps?.launchLibrary, deps?.createFile) + for await (const image of pickMobileImagesWithMode(source, false, deps)) { + return image } - return pickFromFiles(deps?.launchFiles, deps?.createFile) + return null +} + +export function pickMobileImages( + source: MobileImageSource, + deps?: MobileImagePickerDeps +): AsyncIterable { + return pickMobileImagesWithMode(source, true, deps) } diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts new file mode 100644 index 000000000..4c0cee804 --- /dev/null +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest' +import type { NativeChatMessage } from '../../../src/shared/native-chat-types' +import { + findLandedImagePreviewEchoes, + migrateImagePreviewMessageIds, + type PendingImagePreviewEcho +} from './mobile-native-chat-draft-reconcile' + +function userText(id: string, text: string): NativeChatMessage { + return { + id, + role: 'user', + blocks: [{ type: 'text', text }], + timestamp: null, + source: 'transcript' + } +} + +function pending(id: string, images: string[], expectedOccurrence = 1): PendingImagePreviewEcho { + return { id, text: '', images, expectedOccurrence, baselineTailMessageId: null } +} + +describe('mobile native chat image preview reconciliation', () => { + it('keeps separate adjacent image-only sends independently reconcilable', () => { + const landed = findLandedImagePreviewEchoes( + [ + userText('source-a', '[Image: source: /tmp/a.png]'), + userText('source-b', '[Image: source: /tmp/b.png]') + ], + [pending('pending-a', ['file:///a.jpg']), pending('pending-b', ['file:///b.jpg'], 2)] + ) + + expect(landed).toEqual([ + { pendingId: 'pending-a', messageId: 'source-a', images: ['file:///a.jpg'] }, + { pendingId: 'pending-b', messageId: 'source-b', images: ['file:///b.jpg'] } + ]) + }) + + it('waits for a complete multi-image turn as transcript source frames stream in', () => { + const entry = pending('pending', ['file:///a.jpg', 'file:///b.jpg']) + const sourceA = userText('source-a', '[Image: source: /tmp/a.png]') + const sourceB = userText('source-b', '[Image: source: /tmp/b.png]') + + expect(findLandedImagePreviewEchoes([sourceA], [entry])).toEqual([]) + expect(findLandedImagePreviewEchoes([sourceA, sourceB], [entry])).toEqual([]) + expect( + findLandedImagePreviewEchoes( + [sourceA, sourceB, userText('prompt', '[Image #1] [Image #2]')], + [entry] + ) + ).toEqual([ + { + pendingId: 'pending', + messageId: 'prompt', + images: ['file:///a.jpg', 'file:///b.jpg'] + } + ]) + }) + + it('moves an early standalone preview to the later folded prompt id', () => { + const sessionKey = 'host\0worktree\0tab\0session' + const previous = { [sessionKey]: { source: ['file:///a.jpg'] } } + const messages = [ + userText('source', '[Image: source: /tmp/a.png]'), + userText('prompt', '[Image #1]') + ] + + expect(migrateImagePreviewMessageIds(previous, sessionKey, messages)).toEqual({ + [sessionKey]: { prompt: ['file:///a.jpg'] } + }) + }) +}) diff --git a/mobile/src/session/mobile-native-chat-draft-reconcile.ts b/mobile/src/session/mobile-native-chat-draft-reconcile.ts index 1af010ff5..360fd2258 100644 --- a/mobile/src/session/mobile-native-chat-draft-reconcile.ts +++ b/mobile/src/session/mobile-native-chat-draft-reconcile.ts @@ -1,6 +1,7 @@ -import type { NativeChatMessage } from '../../../src/shared/native-chat-types' +import { isImageRefBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types' import { isImageSourceUserTurn, + normalizeImageTranscriptMessages, stripImagePromptMarker } from './mobile-native-chat-image-transcript-markers' @@ -61,6 +62,149 @@ export function countImageSourceTurnsAfter( return count } +export type PendingImagePreviewEcho = { + id: string + text: string + images?: string[] + expectedOccurrence: number + baselineTailMessageId: string | null +} + +export type LandedImagePreviewEcho = { + pendingId: string + messageId: string + images: string[] +} + +const SENT_IMAGE_PREVIEW_LIMIT = 32 +const SENT_IMAGE_PREVIEW_SESSION_LIMIT = 8 + +export function mergeLandedImagePreviewEchoes( + previous: Record>, + sessionKey: string, + landed: readonly LandedImagePreviewEcho[] +): Record> { + const entries = Object.entries(previous[sessionKey] ?? {}) + for (const preview of landed) { + const existingIndex = entries.findIndex(([messageId]) => messageId === preview.messageId) + if (existingIndex >= 0) { + entries.splice(existingIndex, 1) + } + entries.push([preview.messageId, preview.images]) + } + const next = { ...previous } + delete next[sessionKey] + next[sessionKey] = Object.fromEntries(entries.slice(-SENT_IMAGE_PREVIEW_LIMIT)) + for (const key of Object.keys(next).slice(0, -SENT_IMAGE_PREVIEW_SESSION_LIMIT)) { + delete next[key] + } + return next +} + +function imagePreviewReplacementMessageId( + messages: readonly NativeChatMessage[], + sourceIndex: number +): string | null { + const source = messages[sourceIndex] + if (!source || !isImageSourceUserTurn(source)) { + return null + } + let nextIndex = sourceIndex + 1 + while ( + messages[nextIndex]?.source === source.source && + isImageSourceUserTurn(messages[nextIndex]!) + ) { + nextIndex++ + } + const prompt = messages[nextIndex] + const firstText = prompt?.blocks.find((block) => block.type === 'text') + return prompt?.role === 'user' && + prompt.source === source.source && + firstText?.type === 'text' && + stripImagePromptMarker(firstText.text) !== firstText.text + ? prompt.id + : null +} + +/** Moves previews forward when a progressive source-only transcript frame later + * folds into the marker-prefixed prompt with a different authoritative id. */ +export function migrateImagePreviewMessageIds( + previous: Record>, + sessionKey: string, + messages: readonly NativeChatMessage[] +): Record> { + const sessionPreviews = previous[sessionKey] + if (!sessionPreviews) { + return previous + } + const messageIndexById = new Map(messages.map((message, index) => [message.id, index])) + let nextSession: Record | null = null + for (const [messageId, images] of Object.entries(sessionPreviews)) { + const sourceIndex = messageIndexById.get(messageId) + if (sourceIndex === undefined) { + continue + } + const replacementId = imagePreviewReplacementMessageId(messages, sourceIndex) + if (!replacementId) { + continue + } + nextSession ??= { ...sessionPreviews } + delete nextSession[messageId] + nextSession[replacementId] = [...(nextSession[replacementId] ?? []), ...images] + } + return nextSession ? { ...previous, [sessionKey]: nextSession } : previous +} + +/** Binds local preview URIs to the authoritative transcript turn that replaced + * the optimistic bubble. Host paths and marker-only Codex turns cannot render + * the phone-local photo without this handoff. */ +export function findLandedImagePreviewEchoes( + messages: readonly NativeChatMessage[], + entries: readonly PendingImagePreviewEcho[] +): LandedImagePreviewEcho[] { + const normalized = normalizeImageTranscriptMessages(messages) + const messageIndexById = new Map(normalized.map((message, index) => [message.id, index])) + const claimedMessageIds = new Set() + const landed: LandedImagePreviewEcho[] = [] + + for (const entry of entries) { + if (!entry.images?.length) { + continue + } + const targetText = entry.text.trim() + const candidates = normalized.filter((message) => { + if (message.role !== 'user') { + return false + } + if (targetText) { + return normalizedUserText(message) === targetText + } + const imageCount = message.blocks.filter(isImageRefBlock).length + return message.blocks.length === 0 || imageCount >= entry.images!.length + }) + const tailIndex = entry.baselineTailMessageId + ? messageIndexById.get(entry.baselineTailMessageId) + : -1 + const occurrenceIndex = Math.max(0, entry.expectedOccurrence - 1) + const candidate = targetText + ? candidates[occurrenceIndex] + : candidates.filter( + (message) => + tailIndex === undefined || (messageIndexById.get(message.id) ?? -1) > tailIndex + )[occurrenceIndex] + if ( + !candidate || + claimedMessageIds.has(candidate.id) || + (tailIndex !== undefined && (messageIndexById.get(candidate.id) ?? -1) <= tailIndex) + ) { + continue + } + claimedMessageIds.add(candidate.id) + landed.push({ pendingId: entry.id, messageId: candidate.id, images: entry.images }) + } + return landed +} + export function findLandedUnconfirmedSends( messages: readonly NativeChatMessage[], entries: readonly UnconfirmedSend[] diff --git a/mobile/src/session/mobile-native-chat-image-attachment.test.ts b/mobile/src/session/mobile-native-chat-image-attachment.test.ts index 57c3f67bd..e61116f3c 100644 --- a/mobile/src/session/mobile-native-chat-image-attachment.test.ts +++ b/mobile/src/session/mobile-native-chat-image-attachment.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest' import type { RpcClient } from '../transport/rpc-client' import type { RpcResponse, RpcSuccess } from '../transport/types' -import { uploadMobileNativeChatImage } from './mobile-native-chat-image-attachment' +import { uploadMobileNativeChatImages } from './mobile-native-chat-image-attachment' function ok(id: string, result: unknown): RpcSuccess { return { id, ok: true, result, _meta: { runtimeId: 'runtime-1' } } @@ -16,6 +16,10 @@ function methodNotFound(id: string): RpcResponse { } } +function failed(id: string, message: string): RpcResponse { + return { id, ok: false, error: { code: 'failed', message }, _meta: { runtimeId: 'r' } } +} + function clientWithResponses(responses: RpcResponse[]): Pick & { calls: { method: string; params: unknown }[] } { @@ -33,67 +37,137 @@ function clientWithResponses(responses: RpcResponse[]): Pick { +describe('uploadMobileNativeChatImages', () => { it('uploads the picked image and returns its host path + local preview uri, without any terminal.send', async () => { const client = clientWithResponses([ methodNotFound('start'), ok('save', '/tmp/orca-attach.png') ]) - const result = await uploadMobileNativeChatImage('library', { + const result = await uploadMobileNativeChatImages('library', { client, getConnectionId: async () => 'conn-7', - pickImage: vi.fn().mockResolvedValue({ base64: 'AAAA', uri: 'file:///photo.jpg' }) + pickImages: vi.fn().mockResolvedValue([{ base64: 'AAAA', uri: 'file:///photo.jpg' }]) }) - expect(result).toEqual({ path: '/tmp/orca-attach.png', previewUri: 'file:///photo.jpg' }) + expect(result).toEqual([{ path: '/tmp/orca-attach.png', previewUri: 'file:///photo.jpg' }]) // Native chat defers the paste to submit — nothing is sent to the terminal here. expect(client.calls.some((call) => call.method === 'terminal.send')).toBe(false) const saveCall = client.calls.find((c) => c.method === 'clipboard.saveImageAsTempFile') expect(saveCall?.params).toMatchObject({ connectionId: 'conn-7' }) }) + it('uploads all three selected images in picker order', async () => { + const client = clientWithResponses([ + methodNotFound('start-a'), + ok('save-a', '/tmp/a.png'), + methodNotFound('start-b'), + ok('save-b', '/tmp/b.png'), + methodNotFound('start-c'), + ok('save-c', '/tmp/c.png') + ]) + + const order: string[] = [] + async function* pickImages() { + for (const image of [ + { base64: 'AAAA', uri: 'file:///a.jpg' }, + { base64: 'BBBB', uri: 'file:///b.jpg' }, + { base64: 'CCCC', uri: 'file:///c.jpg' } + ]) { + order.push(`read:${image.uri}`) + yield image + } + } + const result = await uploadMobileNativeChatImages('library', { + client, + getConnectionId: async () => 'conn-7', + pickImages, + onImageUploaded: (image) => order.push(`uploaded:${image.previewUri}`) + }) + + expect(result).toEqual([ + { path: '/tmp/a.png', previewUri: 'file:///a.jpg' }, + { path: '/tmp/b.png', previewUri: 'file:///b.jpg' }, + { path: '/tmp/c.png', previewUri: 'file:///c.jpg' } + ]) + expect(order).toEqual([ + 'read:file:///a.jpg', + 'uploaded:file:///a.jpg', + 'read:file:///b.jpg', + 'uploaded:file:///b.jpg', + 'read:file:///c.jpg', + 'uploaded:file:///c.jpg' + ]) + }) + it('returns null when the picker is cancelled and uploads nothing', async () => { const client = clientWithResponses([]) - const result = await uploadMobileNativeChatImage('library', { + const result = await uploadMobileNativeChatImages('library', { client, getConnectionId: async () => null, - pickImage: vi.fn().mockResolvedValue(null) + pickImages: vi.fn().mockResolvedValue([]) }) - expect(result).toBeNull() + expect(result).toEqual([]) expect(client.calls).toEqual([]) }) + it('reports completed uploads before a later image fails', async () => { + const client = clientWithResponses([ + methodNotFound('start-a'), + ok('save-a', '/tmp/a.png'), + methodNotFound('start-b'), + failed('save-b', 'upload failed') + ]) + const onImageUploaded = vi.fn() + + await expect( + uploadMobileNativeChatImages('library', { + client, + getConnectionId: async () => null, + pickImages: vi.fn().mockResolvedValue([ + { base64: 'AAAA', uri: 'file:///a.jpg' }, + { base64: 'BBBB', uri: 'file:///b.jpg' } + ]), + onImageUploaded + }) + ).rejects.toThrow('upload failed') + expect(onImageUploaded).toHaveBeenCalledOnce() + expect(onImageUploaded).toHaveBeenCalledWith({ + path: '/tmp/a.png', + previewUri: 'file:///a.jpg' + }) + }) + it('falls back to an inline data uri for the preview when the picker omits a uri', async () => { const client = clientWithResponses([methodNotFound('start'), ok('save', '/tmp/x.png')]) - const result = await uploadMobileNativeChatImage('files', { + const result = await uploadMobileNativeChatImages('files', { client, getConnectionId: async () => null, - pickImage: vi.fn().mockResolvedValue({ base64: 'BBBB' }) + pickImages: vi.fn().mockResolvedValue([{ base64: 'BBBB' }]) }) - expect(result).toEqual({ path: '/tmp/x.png', previewUri: 'data:image/png;base64,BBBB' }) + expect(result).toEqual([{ path: '/tmp/x.png', previewUri: 'data:image/png;base64,BBBB' }]) }) it('signals upload start only after a real image is picked', async () => { const onUploadStart = vi.fn() const cancelledClient = clientWithResponses([]) - await uploadMobileNativeChatImage('library', { + await uploadMobileNativeChatImages('library', { client: cancelledClient, getConnectionId: async () => null, - pickImage: vi.fn().mockResolvedValue(null), + pickImages: vi.fn().mockResolvedValue([]), onUploadStart }) expect(onUploadStart).not.toHaveBeenCalled() const client = clientWithResponses([methodNotFound('start'), ok('save', '/tmp/y.png')]) - await uploadMobileNativeChatImage('library', { + await uploadMobileNativeChatImages('library', { client, getConnectionId: async () => null, - pickImage: vi.fn().mockResolvedValue({ base64: 'CCCC', uri: 'file:///y.jpg' }), + pickImages: vi.fn().mockResolvedValue([{ base64: 'CCCC', uri: 'file:///y.jpg' }]), onUploadStart }) expect(onUploadStart).toHaveBeenCalledTimes(1) diff --git a/mobile/src/session/mobile-native-chat-image-attachment.ts b/mobile/src/session/mobile-native-chat-image-attachment.ts index f81b9d2ad..fc9a962ba 100644 --- a/mobile/src/session/mobile-native-chat-image-attachment.ts +++ b/mobile/src/session/mobile-native-chat-image-attachment.ts @@ -13,34 +13,67 @@ export type PendingNativeChatImage = { readonly previewUri: string } -export type UploadNativeChatImageDeps = { +export function appendPendingNativeChatImages( + current: readonly PendingNativeChatImage[], + uploaded: readonly Omit[], + idCounter: { current: number } +): PendingNativeChatImage[] { + return [ + ...current, + ...uploaded.map((image) => { + idCounter.current += 1 + return { id: `img-${idCounter.current}`, ...image } + }) + ] +} + +export type UploadNativeChatImagesDeps = { readonly client: Pick readonly getConnectionId: () => Promise // Injected so this module stays free of expo/react-native imports (unit-testable). - readonly pickImage: (source: MobileImageSource) => Promise + readonly pickImages: ( + source: MobileImageSource + ) => + | Iterable + | AsyncIterable + | Promise | AsyncIterable> // Fired once the user has picked an image and the host upload is about to start — // lets the UI show the attach spinner only for the transfer, not the picker. readonly onUploadStart?: () => void + /** Retains each completed upload if a later image in the same selection fails. */ + readonly onImageUploaded?: (image: Omit) => void } /** Picks an image and uploads it to the host, returning the host path + a local * preview URI — but does NOT paste it into the terminal. Unlike the terminal * attach flow, native chat holds the image as a composer chip and rides it along * on submit (desktop parity), so the chip and the agent input never diverge. - * Returns null when the user cancels the picker. */ -export async function uploadMobileNativeChatImage( + * Returns an empty array when the user cancels the picker. */ +export async function uploadMobileNativeChatImages( source: MobileImageSource, - { client, getConnectionId, pickImage, onUploadStart }: UploadNativeChatImageDeps -): Promise | null> { - const picked = await pickImage(source) - if (!picked) { - return null + { + client, + getConnectionId, + pickImages, + onUploadStart, + onImageUploaded + }: UploadNativeChatImagesDeps +): Promise[]> { + const picked = await pickImages(source) + const uploaded: Omit[] = [] + let connectionId: string | null = null + for await (const image of picked) { + if (uploaded.length === 0) { + onUploadStart?.() + connectionId = await getConnectionId() + } + const path = await saveMobileClipboardImageAsTempFile(client, image.base64, { connectionId }) + // Prefer the picker's local URI for the thumbnail; fall back to an inline data + // URI when the source omitted one (RN renders both). + const previewUri = image.uri ?? `data:image/png;base64,${image.base64}` + const result = { path, previewUri } + uploaded.push(result) + onImageUploaded?.(result) } - onUploadStart?.() - const connectionId = await getConnectionId() - const path = await saveMobileClipboardImageAsTempFile(client, picked.base64, { connectionId }) - // Prefer the picker's local URI for the thumbnail; fall back to an inline data - // URI when the source omitted one (RN renders both). - const previewUri = picked.uri ?? `data:image/png;base64,${picked.base64}` - return { path, previewUri } + return uploaded } diff --git a/mobile/src/session/mobile-native-chat-image-send.test.ts b/mobile/src/session/mobile-native-chat-image-send.test.ts index 46ac4730f..cf1c59adf 100644 --- a/mobile/src/session/mobile-native-chat-image-send.test.ts +++ b/mobile/src/session/mobile-native-chat-image-send.test.ts @@ -27,17 +27,22 @@ function clientWithResponses(responses: RpcResponse[]): Pick { it('clears the input line, then pastes each path as a bracketed, non-submitting terminal.send with the mobile client tag', async () => { - const client = clientWithResponses([sendResult(true), sendResult(true), sendResult(true)]) + const client = clientWithResponses([ + sendResult(true), + sendResult(true), + sendResult(true), + sendResult(true) + ]) const ok = await pasteMobileNativeChatImagePaths({ client, terminal: 'term-1', deviceToken: 'device-9', - imagePaths: ['/tmp/a.png', '/tmp/b.png'] + imagePaths: ['/tmp/a.png', '/tmp/b.png', '/tmp/c.png'] }) expect(ok).toBe(true) - expect(client.calls).toHaveLength(3) + expect(client.calls).toHaveLength(4) // Leading Ctrl+U clears any stale input so a retry can't duplicate the image. expect(client.calls[0]).toEqual({ method: 'terminal.send', @@ -50,6 +55,7 @@ describe('pasteMobileNativeChatImagePaths', () => { }) expect(client.calls[1]?.params.text).toBe('\x1b[200~/tmp/a.png\x1b[201~') expect(client.calls[2]?.params.text).toBe('\x1b[200~/tmp/b.png\x1b[201~') + expect(client.calls[3]?.params.text).toBe('\x1b[200~/tmp/c.png\x1b[201~') }) it('stops and reports failure as soon as a paste is rejected', async () => { diff --git a/mobile/src/session/mobile-native-chat-pending-echo.ts b/mobile/src/session/mobile-native-chat-pending-echo.ts new file mode 100644 index 000000000..b18136a9d --- /dev/null +++ b/mobile/src/session/mobile-native-chat-pending-echo.ts @@ -0,0 +1,91 @@ +export type MobileNativeChatPendingMessage = { + id: string + text: string + expectedOccurrence: number + /** Local preview URIs carried by the send for its optimistic echo. */ + images?: string[] + baselineTailMessageId: string | null +} + +export type MobileNativeChatSendOrigin = { + draftKey: string + pendingKey: string | null + normalizedText: string + baselineOccurrences: number + baselineTailMessageId: string | null +} + +type PendingByKey = Record + +export function combineMobileNativeChatPending( + session: MobileNativeChatPendingMessage[], + waiting: readonly MobileNativeChatPendingMessage[] +): MobileNativeChatPendingMessage[] { + if (waiting.length === 0) { + return session + } + const sessionIds = new Set(session.map((item) => item.id)) + return [...session, ...waiting.filter((item) => !sessionIds.has(item.id))] +} + +export function appendMobileNativeChatPending( + previous: PendingByKey, + key: string, + id: string, + origin: MobileNativeChatSendOrigin, + text: string, + images?: string[] +): PendingByKey { + const current = previous[key] ?? [] + const earlierOutstanding = current.filter( + (pending) => + pending.text.trim() === origin.normalizedText && + pending.expectedOccurrence > origin.baselineOccurrences + ).length + const expectedImageEchoOrdinal = + current.filter((pending) => pending.text.trim() === '' && pending.images?.length).length + 1 + return { + ...previous, + [key]: [ + ...current, + { + id, + text, + expectedOccurrence: + origin.normalizedText === '' + ? expectedImageEchoOrdinal + : origin.baselineOccurrences + earlierOutstanding + 1, + baselineTailMessageId: origin.baselineTailMessageId, + ...(images?.length ? { images } : {}) + } + ] + } +} + +export function mergeWaitingSessionPending( + previous: PendingByKey, + sessionKey: string, + waiting: readonly MobileNativeChatPendingMessage[] +): PendingByKey { + const current = previous[sessionKey] ?? [] + const currentIds = new Set(current.map((item) => item.id)) + const moved = waiting.filter((item) => !currentIds.has(item.id)) + return moved.length > 0 ? { ...previous, [sessionKey]: [...current, ...moved] } : previous +} + +export function removeWaitingSessionPending( + previous: PendingByKey, + draftKey: string, + movedIds: ReadonlySet +): PendingByKey { + const remaining = (previous[draftKey] ?? []).filter((item) => !movedIds.has(item.id)) + if (remaining.length > 0) { + return { ...previous, [draftKey]: remaining } + } + if (!(draftKey in previous)) { + return previous + } + const next = { ...previous } + delete next[draftKey] + return next +} diff --git a/mobile/src/session/mobile-native-chat-render-data.test.ts b/mobile/src/session/mobile-native-chat-render-data.test.ts index b4a273d31..c67257ec2 100644 --- a/mobile/src/session/mobile-native-chat-render-data.test.ts +++ b/mobile/src/session/mobile-native-chat-render-data.test.ts @@ -116,6 +116,36 @@ describe('buildMobileNativeChatTransientData', () => { expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }]) }) + it('keeps the phone-local image visible when the transcript replaces its optimistic echo', () => { + const folded = foldMobileNativeChatMessages([ + user('source', '[Image: source: /tmp/a.png]'), + user('prompt', '[Image #1] look at this') + ]) + const result = buildMobileNativeChatTransientData({ + folded, + streaming: null, + pending: [], + imagePreviewsByMessageId: { prompt: ['file:///phone-photo.jpg'] } + }) + + expect(result.data).toHaveLength(1) + expect(result.data[0]?.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png', url: 'file:///phone-photo.jpg' }, + { type: 'text', text: 'look at this' } + ]) + }) + + it('restores the local preview onto a marker-only transcript turn', () => { + const result = buildMobileNativeChatTransientData({ + folded: foldMobileNativeChatMessages([user('prompt', '[Image #1]')]), + streaming: null, + pending: [], + imagePreviewsByMessageId: { prompt: ['file:///phone-photo.jpg'] } + }) + + expect(result.data[0]?.blocks).toEqual([{ type: 'image-ref', url: 'file:///phone-photo.jpg' }]) + }) + it('appends a synthetic bubble for gated streaming text, between transcript and pending', () => { // Whether text streams at all is the gate's call // (`mobile-native-chat-streaming-gate.test.ts`); this only places it. diff --git a/mobile/src/session/mobile-native-chat-render-data.ts b/mobile/src/session/mobile-native-chat-render-data.ts index e11b9ac56..b3d4ba1c0 100644 --- a/mobile/src/session/mobile-native-chat-render-data.ts +++ b/mobile/src/session/mobile-native-chat-render-data.ts @@ -3,7 +3,7 @@ import { formatNativeChatEmptyStateCopy, type NativeChatEmptyStateCopy } from '../../../src/shared/native-chat-empty-state' -import type { NativeChatMessage } from '../../../src/shared/native-chat-types' +import { isImageRefBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types' import { foldToolMessages } from './mobile-native-chat-blocks' import { normalizeImageTranscriptMessages } from './mobile-native-chat-image-transcript-markers' import { stripNoiseMessages } from './mobile-native-chat-noise' @@ -55,15 +55,37 @@ export function foldMobileNativeChatMessages(messages: NativeChatMessage[]): Nat export function buildMobileNativeChatTransientData({ folded, streaming, - pending + pending, + imagePreviewsByMessageId }: { folded: NativeChatMessage[] /** Streaming bubble text, already gated by `deriveMobileNativeChatStreaming`. */ streaming: string | null pending: MobileNativeChatPendingItem[] + imagePreviewsByMessageId?: Record }): { folded: NativeChatMessage[]; streaming: string | null; data: NativeChatMessage[] } { + const renderedFolded = folded.map((message) => { + const previews = imagePreviewsByMessageId?.[message.id] + if (message.role !== 'user' || !previews?.length) { + return message + } + let previewIndex = 0 + const blocks = message.blocks.map((block) => { + if (!isImageRefBlock(block)) { + return block + } + const url = previews[previewIndex] + previewIndex += 1 + return url ? { ...block, url } : block + }) + while (previewIndex < previews.length) { + blocks.push({ type: 'image-ref', url: previews[previewIndex] }) + previewIndex += 1 + } + return { ...message, blocks } + }) const data: NativeChatMessage[] = [ - ...folded, + ...renderedFolded, ...(streaming ? [ { @@ -88,5 +110,5 @@ export function buildMobileNativeChatTransientData({ source: 'transcript' as const })) ] - return { folded, streaming, data } + return { folded: renderedFolded, streaming, data } } diff --git a/mobile/src/session/use-mobile-native-chat-controller.test.ts b/mobile/src/session/use-mobile-native-chat-controller.test.ts index a1ed94231..a9cd12016 100644 --- a/mobile/src/session/use-mobile-native-chat-controller.test.ts +++ b/mobile/src/session/use-mobile-native-chat-controller.test.ts @@ -40,6 +40,7 @@ vi.mock('./use-mobile-native-chat-drafts', () => ({ composerText: '', setComposerText: vi.fn(), pending: [], + imagePreviewsByMessageId: {}, captureSendOrigin, readSeededLaunchDraft: () => null, readSeededLaunchDraftSeed: () => null, diff --git a/mobile/src/session/use-mobile-native-chat-controller.ts b/mobile/src/session/use-mobile-native-chat-controller.ts index 49d75409c..5ed98925e 100644 --- a/mobile/src/session/use-mobile-native-chat-controller.ts +++ b/mobile/src/session/use-mobile-native-chat-controller.ts @@ -50,6 +50,7 @@ export type MobileNativeChatController = { chatComposerText: string setChatComposerText: Dispatch> chatPending: MobileNativeChatPendingMessage[] + chatImagePreviewsByMessageId: Record nativeChatSession: ReturnType nativeChatAgentWorking: boolean nativeChatStreamingText?: string @@ -160,6 +161,7 @@ export function useMobileNativeChatController(args: { composerText: chatComposerText, setComposerText: setChatComposerText, pending: chatPending, + imagePreviewsByMessageId: chatImagePreviewsByMessageId, captureSendOrigin, readSeededLaunchDraft, readSeededLaunchDraftSeed, @@ -326,6 +328,7 @@ export function useMobileNativeChatController(args: { chatComposerText, setChatComposerText, chatPending, + chatImagePreviewsByMessageId, nativeChatSession, nativeChatAgentWorking, nativeChatStreamingText, diff --git a/mobile/src/session/use-mobile-native-chat-drafts.test.ts b/mobile/src/session/use-mobile-native-chat-drafts.test.ts index bf3f94bd2..154ef3fb8 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts.test.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts.test.ts @@ -269,6 +269,7 @@ describe('useMobileNativeChatDrafts', () => { ) ) expect(state?.pending).toEqual([]) + expect(state?.imagePreviewsByMessageId).toEqual({ u1: ['file:///a.jpg'] }) }) it("keeps an image-only echo when an unrelated text send's echo lands", async () => { @@ -345,6 +346,29 @@ describe('useMobileNativeChatDrafts', () => { ) ) expect(state?.pending).toEqual([]) + expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] }) + }) + + it('hands a marker-only image preview to the authoritative user bubble', async () => { + await mount('a') + const origin = state?.captureSendOrigin('') + act(() => { + if (origin) { + state?.acceptSend(origin, '', ['file:///a.jpg']) + } + }) + + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + messages: [userTextMessage('u1', '[Image #1]')] + }) + ) + ) + + expect(state?.pending).toEqual([]) + expect(state?.imagePreviewsByMessageId).toEqual({ u1: ['file:///a.jpg'] }) }) it('does not reconcile a repeated send against an older identical turn', async () => { @@ -722,21 +746,59 @@ describe('useMobileNativeChatDrafts', () => { } }) - it('accepts and clears the first send before a provider session id exists', async () => { + it('preserves first-send images through session assignment and transcript replacement', async () => { await mount('a') await act(async () => renderer?.update(createElement(Harness, { tabId: 'a', sessionId: null }))) - act(() => state?.setComposerText('start the session')) + const images = ['file:///a.jpg', 'file:///b.jpg', 'file:///c.jpg'] + act(() => state?.setComposerText('look')) - const origin = state?.captureSendOrigin('start the session') + const origin = state?.captureSendOrigin('look') expect(origin).toMatchObject({ pendingKey: null }) act(() => { if (origin) { - state?.clearDraftForSend(origin, 'start the session') - state?.acceptSend(origin, 'start the session') + state?.clearDraftForSend(origin, 'look') + state?.acceptSend(origin, 'look', images) } }) expect(state?.composerText).toBe('') + expect(state?.pending.map((pending) => pending.images)).toEqual([images]) + + await act(async () => + renderer?.update(createElement(Harness, { tabId: 'a', sessionId: 'assigned' })) + ) + expect(state?.pending.map((pending) => pending.images)).toEqual([images]) + + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + sessionId: 'assigned', + messages: [ + userTextMessage('source-1', '[Image: source: /tmp/a.png]'), + userTextMessage('source-2', '[Image: source: /tmp/b.png]'), + userTextMessage('source-3', '[Image: source: /tmp/c.png]') + ] + }) + ) + ) + expect(state?.pending.map((pending) => pending.images)).toEqual([images]) + + await act(async () => + renderer?.update( + createElement(Harness, { + tabId: 'a', + sessionId: 'assigned', + messages: [ + userTextMessage('source-1', '[Image: source: /tmp/a.png]'), + userTextMessage('source-2', '[Image: source: /tmp/b.png]'), + userTextMessage('source-3', '[Image: source: /tmp/c.png]'), + userTextMessage('prompt', '[Image #1] [Image #2] [Image #3] look') + ] + }) + ) + ) expect(state?.pending).toEqual([]) + expect(state?.imagePreviewsByMessageId).toEqual({ prompt: images }) }) }) diff --git a/mobile/src/session/use-mobile-native-chat-drafts.ts b/mobile/src/session/use-mobile-native-chat-drafts.ts index a43bc8b28..9da4dea6a 100644 --- a/mobile/src/session/use-mobile-native-chat-drafts.ts +++ b/mobile/src/session/use-mobile-native-chat-drafts.ts @@ -3,35 +3,29 @@ import type { NativeChatMessage } from '../../../src/shared/native-chat-types' import { countImageSourceTurnsAfter, countUserTextOccurrences, + findLandedImagePreviewEchoes, findLandedUnconfirmedSends, + mergeLandedImagePreviewEchoes, + migrateImagePreviewMessageIds, normalizedUserText, type UnconfirmedSend } from './mobile-native-chat-draft-reconcile' +import { + appendMobileNativeChatPending, + combineMobileNativeChatPending, + mergeWaitingSessionPending, + removeWaitingSessionPending, + type MobileNativeChatPendingMessage, + type MobileNativeChatSendOrigin +} from './mobile-native-chat-pending-echo' import { mobileNativeChatScopeKey } from './mobile-native-chat-scope-key' import { useMobileNativeChatLaunchDraftSeed } from './use-mobile-native-chat-launch-draft-seed' import type { MobileNativeChatLaunchDraftSeed } from './use-mobile-native-chat-launch-draft-seed' -export type MobileNativeChatPendingMessage = { - id: string - text: string - expectedOccurrence: number - /** Local preview URIs of images ridden along on the send, rendered as thumbnails - * on the echo bubble so the sent photo shows before the transcript catches up. */ - images?: string[] - /** Transcript tail when sent — an image-only echo (no text to match) reconciles - * against new `[Image: source: …]` echo turns after this id, so pagination, - * agent replies, and unrelated text echoes can't clear it early. */ - baselineTailMessageId: string | null -} -export type MobileNativeChatSendOrigin = { - draftKey: string - pendingKey: string | null - normalizedText: string - baselineOccurrences: number - baselineTailMessageId: string | null -} +export type { MobileNativeChatPendingMessage, MobileNativeChatSendOrigin } const NO_PENDING_MESSAGES: MobileNativeChatPendingMessage[] = [] +const NO_IMAGE_PREVIEWS: Record = {} // How long an ack-lost send waits for its transcript echo before the UI surfaces // that delivery remains unconfirmed. @@ -57,6 +51,9 @@ export function useMobileNativeChatDrafts(args: { composerText: string setComposerText: Dispatch> pending: MobileNativeChatPendingMessage[] + /** Phone-local previews rebound to the transcript message that replaced the + * optimistic echo, keyed by authoritative message id. */ + imagePreviewsByMessageId: Record captureSendOrigin: (text: string) => MobileNativeChatSendOrigin | null /** Launch-context text still believed to be parked on the agent's TUI input * line, or null once it has been declined or retired. Send paths size their @@ -91,6 +88,12 @@ export function useMobileNativeChatDrafts(args: { const [pendingBySession, setPendingBySession] = useState< Record >({}) + const [pendingWaitingForSession, setPendingWaitingForSession] = useState< + Record + >({}) + const [imagePreviewsBySession, setImagePreviewsBySession] = useState< + Record> + >({}) const pendingCounterRef = useRef(0) const messagesRef = useRef(messages) messagesRef.current = messages @@ -162,41 +165,21 @@ export function useMobileNativeChatDrafts(args: { const acceptSend = useCallback( (origin: MobileNativeChatSendOrigin, text: string, images?: string[]) => { - // Why: the first prompt can be sent before the provider reports a session - // id; wait for an id before keying an optimistic echo. - if (!origin.pendingKey) { + if (!origin.pendingKey && !images?.length) { return } - const pendingKey = origin.pendingKey pendingCounterRef.current += 1 - setPendingBySession((previous) => { - const current = previous[pendingKey] ?? NO_PENDING_MESSAGES - const earlierOutstanding = current.filter( - (pending) => - pending.text.trim() === origin.normalizedText && - pending.expectedOccurrence > origin.baselineOccurrences - ).length - // An empty-text send reconciles by image-echo ordinal: every outstanding - // send's ridden-along images echo as `[Image: source: …]` turns after - // this send's baseline tail, ahead of this send's own echo. - const expectedImageEchoOrdinal = - current.reduce( - (sum, pending) => - sum + (pending.images?.length ?? (pending.text.trim() === '' ? 1 : 0)), - 0 - ) + 1 - const pending: MobileNativeChatPendingMessage = { - id: `pending-${pendingCounterRef.current}`, - text, - expectedOccurrence: - origin.normalizedText === '' - ? expectedImageEchoOrdinal - : origin.baselineOccurrences + earlierOutstanding + 1, - baselineTailMessageId: origin.baselineTailMessageId, - ...(images && images.length > 0 ? { images } : {}) - } - return { ...previous, [pendingKey]: [...current, pending] } - }) + const id = `pending-${pendingCounterRef.current}` + const key = origin.pendingKey + if (key) { + setPendingBySession((previous) => + appendMobileNativeChatPending(previous, key, id, origin, text, images) + ) + } else { + setPendingWaitingForSession((previous) => + appendMobileNativeChatPending(previous, origin.draftKey, id, origin, text, images) + ) + } }, [] ) @@ -255,9 +238,7 @@ export function useMobileNativeChatDrafts(args: { const landedSet = new Set(landed) unconfirmedRef.current = unconfirmedRef.current.filter((entry) => !landedSet.has(entry)) for (const entry of landed) { - if (entry.deadline !== null) { - clearTimeout(entry.deadline) - } + clearTimeout(entry.deadline ?? undefined) } }, [messages, draftKey, pendingKey]) @@ -266,21 +247,49 @@ export function useMobileNativeChatDrafts(args: { return () => { mountedRef.current = false for (const entry of unconfirmedRef.current) { - if (entry.deadline !== null) { - clearTimeout(entry.deadline) - } + clearTimeout(entry.deadline ?? undefined) } unconfirmedRef.current = [] } }, []) - const pending = pendingKey - ? (pendingBySession[pendingKey] ?? NO_PENDING_MESSAGES) + const waitingForSession = draftKey + ? (pendingWaitingForSession[draftKey] ?? NO_PENDING_MESSAGES) : NO_PENDING_MESSAGES useEffect(() => { - if (!pendingKey || pending.length === 0) { + if (!draftKey || !pendingKey || waitingForSession.length === 0) { return } + const movedIds = new Set(waitingForSession.map((item) => item.id)) + setPendingBySession((previous) => + mergeWaitingSessionPending(previous, pendingKey, waitingForSession) + ) + setPendingWaitingForSession((previous) => + removeWaitingSessionPending(previous, draftKey, movedIds) + ) + }, [draftKey, pendingKey, waitingForSession]) + + const sessionPending = pendingKey + ? (pendingBySession[pendingKey] ?? NO_PENDING_MESSAGES) + : NO_PENDING_MESSAGES + const pending = combineMobileNativeChatPending(sessionPending, waitingForSession) + useEffect(() => { + if (!pendingKey) { + return + } + setImagePreviewsBySession((previous) => + migrateImagePreviewMessageIds(previous, pendingKey, messages) + ) + if (pending.length === 0) { + return + } + const landedImagePreviews = findLandedImagePreviewEchoes(messages, pending) + const landedImagePendingIds = new Set(landedImagePreviews.map((preview) => preview.pendingId)) + if (landedImagePreviews.length > 0) { + setImagePreviewsBySession((previous) => + mergeLandedImagePreviewEchoes(previous, pendingKey, landedImagePreviews) + ) + } setPendingBySession((previous) => { const current = previous[pendingKey] ?? [] const landedCounts = new Map() @@ -297,12 +306,20 @@ export function useMobileNativeChatDrafts(args: { // baseline tail — text echoes are excluded so an unrelated outstanding // text send cannot clear it. Ordinal-vs-count stays stable when the effect // re-runs on the shrunken list, and ignores paginated-in history. - const next = current.filter((item) => - item.text.trim() === '' + const next = current.filter((item) => { + if (landedImagePendingIds.has(item.id)) { + return false + } + // Image echoes hand their local URIs to the authoritative message above; + // never drop them through the text-only fallback before that handoff. + if (item.images?.length) { + return true + } + return item.text.trim() === '' ? countImageSourceTurnsAfter(messages, item.baselineTailMessageId) < - item.expectedOccurrence + item.expectedOccurrence : (landedCounts.get(item.text.trim()) ?? 0) < item.expectedOccurrence - ) + }) if (next.length === current.length) { return previous } @@ -319,6 +336,9 @@ export function useMobileNativeChatDrafts(args: { composerText: draftKey ? (drafts[draftKey] ?? '') : '', setComposerText, pending, + imagePreviewsByMessageId: pendingKey + ? (imagePreviewsBySession[pendingKey] ?? NO_IMAGE_PREVIEWS) + : NO_IMAGE_PREVIEWS, captureSendOrigin, readSeededLaunchDraft, readSeededLaunchDraftSeed, diff --git a/mobile/src/session/use-mobile-native-chat-image-attachments.test.ts b/mobile/src/session/use-mobile-native-chat-image-attachments.test.ts index 13cb5d96c..a57e1c752 100644 --- a/mobile/src/session/use-mobile-native-chat-image-attachments.test.ts +++ b/mobile/src/session/use-mobile-native-chat-image-attachments.test.ts @@ -11,13 +11,13 @@ import { useMobileNativeChatImageAttachments } from './use-mobile-native-chat-im // Fully stub the picker so the real expo/react-native chain never loads under // the vitest transform (react-native ships Flow syntax rolldown can't parse). vi.mock('./mobile-image-source-picker', () => ({ - pickMobileImage: vi.fn(), + pickMobileImages: vi.fn(), ImageLibraryPermissionError: class ImageLibraryPermissionError extends Error {} })) -import { pickMobileImage } from './mobile-image-source-picker' +import { pickMobileImages } from './mobile-image-source-picker' -const pick = vi.mocked(pickMobileImage) +const pick = vi.mocked(pickMobileImages) function ok(id: string, result: unknown): RpcSuccess { return { id, ok: true, result, _meta: { runtimeId: 'r' } } @@ -124,7 +124,7 @@ describe('useMobileNativeChatImageAttachments', () => { } it('adds an uploaded image as a chip without pasting to the terminal', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')]) mount( baseArgs({ @@ -145,7 +145,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('rides pending images along on send: pastes the path, settles, then delegates the text', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -208,7 +208,7 @@ describe('useMobileNativeChatImageAttachments', () => { it('leads the image paste with a clear sized to a parked multi-line launch draft', async () => { // A single Ctrl+U kills only the last line, so the draft's earlier lines // would survive the clear and ride along with the image as prompt body. - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -242,7 +242,7 @@ describe('useMobileNativeChatImageAttachments', () => { it('spends one budget across the image paste and the text body that follows', async () => { vi.useFakeTimers() try { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -280,7 +280,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('routes an attachments-only send through baseSend with empty text so the echo still shows the photo', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -321,7 +321,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('keeps the chips and does not submit when the image paste is rejected', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -345,7 +345,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('keeps the chips and reports failure when the paste transport throws', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) // No terminal.send responses queued: the clear write throws (dropped transport). const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')]) const baseSend = vi.fn().mockResolvedValue('accepted') @@ -365,7 +365,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('surfaces an error instead of a silent no-op when the input lease gate is closed', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')]) const baseSend = vi.fn().mockResolvedValue('accepted') const onSendError = vi.fn() @@ -387,7 +387,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('scopes chips to the tab that attached them', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')]) const baseSend = vi.fn().mockResolvedValue('accepted') const args = baseArgs({ client: client as unknown as RpcClient, baseSend }) @@ -426,7 +426,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) mount(args) - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) let firstAttach: Promise | null = null await act(async () => { firstAttach = hook!.attachImage('library') @@ -438,7 +438,7 @@ describe('useMobileNativeChatImageAttachments', () => { expect(hook!.isAttaching).toBe(true) // A concurrent cancelled pick — its finally must leave the counter alone. - pick.mockResolvedValue(null) + pick.mockResolvedValue([]) await act(async () => { await hook!.attachImage('library') }) @@ -454,7 +454,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('clears only the chips that were sent, keeping one attached mid-send', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), // first attach @@ -488,7 +488,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) expect(releaseSettle).not.toBeNull() - pick.mockResolvedValue({ base64: 'BBBB', uri: 'file:///b.jpg' }) + pick.mockResolvedValue([{ base64: 'BBBB', uri: 'file:///b.jpg' }]) await act(async () => { await hook!.attachImage('library') }) @@ -511,7 +511,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('aborts the send when the active terminal changes during the settle window', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -560,7 +560,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('leads the next text-only send with Ctrl+U after a failed paste, even with the chip removed', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -596,7 +596,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('heals before the next text-only send when an image submit delivery is unknown (#10228)', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -634,7 +634,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('still heals after the session screen unmounts and remounts (#10228)', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -688,7 +688,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('retains the stale marker when a rejected healing clear blocks text-only send', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -730,7 +730,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('does not reroute text when the active terminal changes during a healing clear', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) let releaseClear: ((response: RpcResponse) => void) | null = null const deferredClear = new Promise((resolve) => { releaseClear = resolve @@ -774,7 +774,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) it('defers the heal instead of burning a rejected clear while the lease is closed', async () => { - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) const client = makeClient([ methodNotFound('start'), ok('save', '/tmp/a.png'), @@ -818,8 +818,8 @@ describe('useMobileNativeChatImageAttachments', () => { it('heals rejected image submits independently across terminals', async () => { pick - .mockResolvedValueOnce({ base64: 'AAAA', uri: 'file:///a.jpg' }) - .mockResolvedValueOnce({ base64: 'BBBB', uri: 'file:///b.jpg' }) + .mockResolvedValueOnce([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) + .mockResolvedValueOnce([{ base64: 'BBBB', uri: 'file:///b.jpg' }]) const client = makeClient([ methodNotFound('start-a'), ok('save-a', '/tmp/a.png'), @@ -896,7 +896,7 @@ describe('useMobileNativeChatImageAttachments', () => { }) }) mount(args) - pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' }) + pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }]) let attach: Promise | null = null await act(async () => { attach = hook!.attachImage('library') diff --git a/mobile/src/session/use-mobile-native-chat-image-attachments.ts b/mobile/src/session/use-mobile-native-chat-image-attachments.ts index 38f9e462e..dbcb97df5 100644 --- a/mobile/src/session/use-mobile-native-chat-image-attachments.ts +++ b/mobile/src/session/use-mobile-native-chat-image-attachments.ts @@ -5,11 +5,12 @@ import type { RpcClient } from '../transport/rpc-client' import type { ConnectionState } from '../transport/types' import { ImageLibraryPermissionError, - pickMobileImage, + pickMobileImages, type MobileImageSource } from './mobile-image-source-picker' import { - uploadMobileNativeChatImage, + appendPendingNativeChatImages, + uploadMobileNativeChatImages, type PendingNativeChatImage } from './mobile-native-chat-image-attachment' import { @@ -82,10 +83,6 @@ export type MobileNativeChatImageAttachments = { readonly sendNativeChat: (text: string) => Promise } -function getErrorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error) -} - const NO_ATTACHMENTS: PendingNativeChatImage[] = [] function withScopeAttachments( @@ -146,48 +143,53 @@ export function useMobileNativeChatImageAttachments({ // pick or pre-upload error never ran `onUploadStart`, so decrementing the // shared counter would clear a concurrent upload's in-flight flag early. let started = false + const uploadedImages: Omit[] = [] + let uploadError: unknown = null try { - const uploaded = await uploadMobileNativeChatImage(source, { + await uploadMobileNativeChatImages(source, { client, getConnectionId: getActiveWorktreeConnectionId, - pickImage: pickMobileImage, + pickImages: pickMobileImages, + onImageUploaded: (image) => uploadedImages.push(image), onUploadStart: () => { started = true attachingCount.current += 1 setIsAttaching(true) } }) - // Cancelled picker: no error, no toast. - if (!uploaded) { - return - } - idCounter.current += 1 - const chip = { id: `img-${idCounter.current}`, ...uploaded } - setAttachmentsByScope((prev) => ({ ...prev, [scope]: [...(prev[scope] ?? []), chip] })) - onAttachSuccess?.() } catch (error) { + uploadError = error + } finally { + if (started) { + attachingCount.current -= 1 + if (attachingCount.current === 0) { + setIsAttaching(false) + } + } + } + if (uploadedImages.length > 0) { + setAttachmentsByScope((prev) => ({ + ...prev, + [scope]: appendPendingNativeChatImages(prev[scope] ?? [], uploadedImages, idCounter) + })) + onAttachSuccess?.() + } + if (uploadError !== null) { + const message = uploadError instanceof Error ? uploadError.message : String(uploadError) onError?.() if (connStateRef.current !== 'connected') { showToast('Attach failed (disconnected)', 1500) return } - if (error instanceof ImageLibraryPermissionError) { + if (uploadError instanceof ImageLibraryPermissionError) { showToast('Photo permission denied', 1500) return } - if (getErrorMessage(error) === CLIPBOARD_IMAGE_TOO_LARGE_ERROR) { + if (message === CLIPBOARD_IMAGE_TOO_LARGE_ERROR) { showToast('Image too large to attach', 1500) return } showToast('Attach failed', 1500) - } finally { - if (started) { - attachingCount.current -= 1 - if (attachingCount.current <= 0) { - attachingCount.current = 0 - setIsAttaching(false) - } - } } }, [ diff --git a/src/shared/native-chat-image-transcript-markers.test.ts b/src/shared/native-chat-image-transcript-markers.test.ts index 58d6d0d16..e4856605a 100644 --- a/src/shared/native-chat-image-transcript-markers.test.ts +++ b/src/shared/native-chat-image-transcript-markers.test.ts @@ -35,6 +35,52 @@ describe('normalizeImageTranscriptMessages', () => { ]) }) + it('folds every source and strips every prompt marker for a multi-image send', () => { + const out = normalizeImageTranscriptMessages([ + userText('a', '[Image: source: /tmp/a.png]'), + userText('b', '[Image: source: /tmp/b.png]'), + userText('c', '[Image: source: /tmp/c.png]'), + userText('prompt', '[Image #1] [Image #2] [Image #3] compare these') + ]) + + expect(out).toHaveLength(1) + expect(out[0]).toMatchObject({ id: 'prompt' }) + expect(out[0]!.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'image-ref', path: '/tmp/b.png' }, + { type: 'image-ref', path: '/tmp/c.png' }, + { type: 'text', text: 'compare these' } + ]) + }) + + it('keeps all image refs when a multi-image send has no caption', () => { + const out = normalizeImageTranscriptMessages([ + userText('a', '[Image: source: /tmp/a.png]'), + userText('b', '[Image: source: /tmp/b.png]'), + userText('prompt', '[Image #1] [Image #2]') + ]) + + expect(out).toHaveLength(1) + expect(out[0]!.blocks).toEqual([ + { type: 'image-ref', path: '/tmp/a.png' }, + { type: 'image-ref', path: '/tmp/b.png' } + ]) + }) + + it('preserves adjacent standalone image turns without a prompt marker', () => { + const out = normalizeImageTranscriptMessages([ + userText('a', '[Image: source: /tmp/a.png]'), + userText('b', '[Image: source: /tmp/b.png]') + ]) + + expect(out).toHaveLength(2) + expect(out.map((message) => message.id)).toEqual(['a', 'b']) + expect(out.map((message) => message.blocks)).toEqual([ + [{ type: 'image-ref', path: '/tmp/a.png' }], + [{ type: 'image-ref', path: '/tmp/b.png' }] + ]) + }) + it('leaves ordinary user text untouched', () => { const out = normalizeImageTranscriptMessages([userText('a', 'how about this')]) expect(out[0]!.blocks).toEqual([{ type: 'text', text: 'how about this' }]) diff --git a/src/shared/native-chat-image-transcript-markers.ts b/src/shared/native-chat-image-transcript-markers.ts index 3f8f90f8c..9643cf518 100644 --- a/src/shared/native-chat-image-transcript-markers.ts +++ b/src/shared/native-chat-image-transcript-markers.ts @@ -1,7 +1,7 @@ import { isTextBlock, type NativeChatBlock, type NativeChatMessage } from './native-chat-types' const IMAGE_SOURCE_MARKER = /^\[Image:\s*source:\s*(.+?)\]\s*$/ -const IMAGE_PROMPT_MARKER = /^\[Image #\d+\]\s*/ +const IMAGE_PROMPT_MARKERS = /^(?:\[Image #\d+\]\s*)+/ function soleText(message: NativeChatMessage): string | null { return message.blocks.length === 1 && isTextBlock(message.blocks[0]) @@ -14,10 +14,12 @@ export function imageSourcePathFromText(text: string): string | null { } export function stripImagePromptMarker(text: string): string { - return text.replace(IMAGE_PROMPT_MARKER, '') + return text.replace(IMAGE_PROMPT_MARKERS, '') } -function stripFirstImagePromptMarker(blocks: readonly NativeChatBlock[]): NativeChatBlock[] { +function stripImagePromptMarkersFromFirstText( + blocks: readonly NativeChatBlock[] +): NativeChatBlock[] { let stripped = false const next: NativeChatBlock[] = [] for (const block of blocks) { @@ -36,13 +38,11 @@ function stripFirstImagePromptMarker(blocks: readonly NativeChatBlock[]): Native function imagePromptMarkerStartsMessage(message: NativeChatMessage): boolean { const firstText = message.blocks.find(isTextBlock) - return firstText ? IMAGE_PROMPT_MARKER.test(firstText.text) : false + return firstText ? IMAGE_PROMPT_MARKERS.test(firstText.text) : false } -/** Claude records an attached image as two user transcript turns: - * `[Image: source: /path]` and then `[Image #1] prompt`. Merge them back into - * one native turn so the UI keeps the same chip+text shape as the optimistic - * send and does not show raw TUI marker text after a view remount. */ +/** Claude records image paths as source turns followed by one marker-prefixed + * prompt. Merge the whole run back into one native user turn. */ export function normalizeImageTranscriptMessages( messages: readonly NativeChatMessage[] ): NativeChatMessage[] { @@ -54,27 +54,34 @@ export function normalizeImageTranscriptMessages( continue } const imagePath = imageSourcePathFromText(soleText(message) ?? '') - const next = messages[index + 1] - if ( - imagePath && - next?.role === 'user' && - next.source === message.source && - imagePromptMarkerStartsMessage(next) - ) { - normalized.push({ - ...next, - blocks: [ - { type: 'image-ref', path: imagePath }, - ...stripFirstImagePromptMarker(next.blocks) - ] - }) - index += 1 - continue - } - // A lone `[Image: source: /path]` turn (no following `[Image #1]` prompt — - // e.g. an image sent with no caption) still renders as an image chip rather - // than the raw marker text. if (imagePath) { + const imagePaths = [imagePath] + let nextIndex = index + 1 + while (nextIndex < messages.length) { + const candidate = messages[nextIndex]! + const candidatePath = imageSourcePathFromText(soleText(candidate) ?? '') + if (candidate.role !== 'user' || candidate.source !== message.source || !candidatePath) { + break + } + imagePaths.push(candidatePath) + nextIndex += 1 + } + const prompt = messages[nextIndex] + if ( + prompt?.role === 'user' && + prompt.source === message.source && + imagePromptMarkerStartsMessage(prompt) + ) { + normalized.push({ + ...prompt, + blocks: [ + ...imagePaths.map((path) => ({ type: 'image-ref' as const, path })), + ...stripImagePromptMarkersFromFirstText(prompt.blocks) + ] + }) + index = nextIndex + continue + } normalized.push({ ...message, blocks: [{ type: 'image-ref', path: imagePath }] @@ -83,7 +90,7 @@ export function normalizeImageTranscriptMessages( } normalized.push({ ...message, - blocks: stripFirstImagePromptMarker(message.blocks) + blocks: stripImagePromptMarkersFromFirstText(message.blocks) }) } return normalized