fix(mobile): preserve multi-image chat attachments (#12639)
* fix(mobile): preserve multi-image chat attachments * fix(mobile): use preferred array syntax * fix(mobile): harden multi-image attachment flow * fix(mobile): retain first-send image previews
This commit is contained in:
Brennan Benson
GitHub
parent
23238aee0b
commit
d4dfc35ac4
|
|
@ -47,6 +47,7 @@ function overlayElement(tick: Tick): ReturnType<typeof createElement> {
|
|||
nativeChatStreamLive: tick.streamLive ?? false,
|
||||
nativeChatStreamScopeKey: tick.identity ?? 'tab-a',
|
||||
chatPending: [],
|
||||
chatImagePreviewsByMessageId: {},
|
||||
chatComposerText: '',
|
||||
setChatComposerText: vi.fn()
|
||||
} as unknown as MobileNativeChatController
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ export function MobileNativeChatOverlay({
|
|||
onLoadEarlier={session.loadEarlier}
|
||||
onSend={images.sendNativeChat}
|
||||
pending={controller.chatPending}
|
||||
imagePreviewsByMessageId={controller.chatImagePreviewsByMessageId}
|
||||
composerText={controller.chatComposerText}
|
||||
onComposerTextChange={controller.setChatComposerText}
|
||||
onAttachImage={() => void images.attachImage('library')}
|
||||
|
|
|
|||
|
|
@ -61,6 +61,9 @@ type Props = {
|
|||
/** Optimistic queued sends (owned by the route so they survive view switches). */
|
||||
/** Optimistic user echoes, including any ridden-along image preview URIs. */
|
||||
pending: MobileNativeChatPendingItem[]
|
||||
/** Local photo URIs retained when the authoritative transcript replaces an
|
||||
* optimistic image bubble. */
|
||||
imagePreviewsByMessageId?: Record<string, string[]>
|
||||
/** Controlled composer text (owned by the route so dictation can write to it). */
|
||||
composerText: string
|
||||
onComposerTextChange: (text: string) => void
|
||||
|
|
@ -125,6 +128,7 @@ export function MobileNativeChatView({
|
|||
onLoadEarlier,
|
||||
onSend,
|
||||
pending,
|
||||
imagePreviewsByMessageId,
|
||||
composerText,
|
||||
onComposerTextChange,
|
||||
onAttachImage,
|
||||
|
|
@ -177,8 +181,14 @@ export function MobileNativeChatView({
|
|||
// route-owned optimistic queued messages. Memoize on the same deps so the
|
||||
// downstream autoscroll effects/`renderItem` keep referential stability.
|
||||
const { data } = useMemo(
|
||||
() => buildMobileNativeChatTransientData({ folded, streaming, pending }),
|
||||
[folded, streaming, pending]
|
||||
() =>
|
||||
buildMobileNativeChatTransientData({
|
||||
folded,
|
||||
streaming,
|
||||
pending,
|
||||
imagePreviewsByMessageId
|
||||
}),
|
||||
[folded, streaming, pending, imagePreviewsByMessageId]
|
||||
)
|
||||
|
||||
// Follow the tail as the conversation grows and keep the newest message above
|
||||
|
|
|
|||
|
|
@ -12,13 +12,28 @@ vi.mock('expo-file-system', () => ({
|
|||
File: vi.fn()
|
||||
}))
|
||||
|
||||
import { ImageLibraryPermissionError, pickMobileImage } from './mobile-image-source-picker'
|
||||
import {
|
||||
ImageLibraryPermissionError,
|
||||
pickMobileImage,
|
||||
pickMobileImages,
|
||||
type PickedMobileImage
|
||||
} from './mobile-image-source-picker'
|
||||
|
||||
const granted = { granted: true } as Awaited<
|
||||
ReturnType<typeof import('expo-image-picker').requestMediaLibraryPermissionsAsync>
|
||||
>
|
||||
const denied = { granted: false } as typeof granted
|
||||
|
||||
async function collectImages(
|
||||
images: AsyncIterable<PickedMobileImage>
|
||||
): Promise<PickedMobileImage[]> {
|
||||
const collected: PickedMobileImage[] = []
|
||||
for await (const image of images) {
|
||||
collected.push(image)
|
||||
}
|
||||
return collected
|
||||
}
|
||||
|
||||
function fileFactory(
|
||||
bytes: Uint8Array,
|
||||
options?: { fileSize?: number; handleSize?: number | null; readError?: Error }
|
||||
|
|
@ -68,6 +83,57 @@ describe('pickMobileImage', () => {
|
|||
expect(file.close).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('returns every selected library photo in order', async () => {
|
||||
const bytesByUri = new Map([
|
||||
['file:///a.jpg', new Uint8Array([1])],
|
||||
['file:///b.jpg', new Uint8Array([2])],
|
||||
['file:///c.jpg', new Uint8Array([3])]
|
||||
])
|
||||
const createFile = vi.fn((uri: string) => {
|
||||
const bytes = bytesByUri.get(uri)!
|
||||
let read = false
|
||||
return {
|
||||
size: bytes.length,
|
||||
open: () => ({
|
||||
size: bytes.length,
|
||||
readBytes: () => {
|
||||
if (read) {
|
||||
return new Uint8Array()
|
||||
}
|
||||
read = true
|
||||
return bytes
|
||||
},
|
||||
close: vi.fn()
|
||||
})
|
||||
}
|
||||
})
|
||||
const launchLibrary = vi.fn().mockResolvedValue({
|
||||
canceled: false,
|
||||
assets: [...bytesByUri].map(([uri, bytes]) => ({ uri, fileSize: bytes.length }))
|
||||
})
|
||||
|
||||
const result = await collectImages(
|
||||
pickMobileImages('library', {
|
||||
requestLibraryPermission: vi.fn().mockResolvedValue(granted),
|
||||
launchLibrary,
|
||||
createFile
|
||||
})
|
||||
)
|
||||
|
||||
expect(result.map((image) => image.uri)).toEqual([
|
||||
'file:///a.jpg',
|
||||
'file:///b.jpg',
|
||||
'file:///c.jpg'
|
||||
])
|
||||
expect(launchLibrary).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
allowsMultipleSelection: true,
|
||||
orderedSelection: true,
|
||||
selectionLimit: 0
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('throws when photo library permission is denied', async () => {
|
||||
await expect(
|
||||
pickMobileImage('library', {
|
||||
|
|
|
|||
|
|
@ -83,11 +83,12 @@ async function readUriAsBase64(
|
|||
}
|
||||
}
|
||||
|
||||
async function pickFromLibrary(
|
||||
async function* pickFromLibrary(
|
||||
multiple: boolean,
|
||||
requestPermission: typeof ImagePicker.requestMediaLibraryPermissionsAsync = ImagePicker.requestMediaLibraryPermissionsAsync,
|
||||
launch: typeof ImagePicker.launchImageLibraryAsync = ImagePicker.launchImageLibraryAsync,
|
||||
createFile: MobileImageFileFactory = defaultMobileImageFileFactory
|
||||
): Promise<PickedMobileImage | null> {
|
||||
): AsyncGenerator<PickedMobileImage> {
|
||||
const permission = await requestPermission()
|
||||
// Why: `granted` covers full + limited iOS access; only a hard denial blocks us.
|
||||
if (!permission.granted) {
|
||||
|
|
@ -96,51 +97,84 @@ async function pickFromLibrary(
|
|||
const result = await launch({
|
||||
mediaTypes: ['images'],
|
||||
base64: false,
|
||||
allowsMultipleSelection: false,
|
||||
allowsMultipleSelection: multiple,
|
||||
...(multiple ? { selectionLimit: 0, orderedSelection: true } : {}),
|
||||
quality: 1
|
||||
})
|
||||
if (result.canceled) {
|
||||
return null
|
||||
return
|
||||
}
|
||||
const asset = result.assets[0]
|
||||
const base64 = asset?.uri ? await readUriAsBase64(asset.uri, asset.fileSize, createFile) : null
|
||||
if (!base64) {
|
||||
return null
|
||||
for (const asset of result.assets) {
|
||||
if (!asset.uri) {
|
||||
continue
|
||||
}
|
||||
const base64 = await readUriAsBase64(asset.uri, asset.fileSize, createFile)
|
||||
if (base64) {
|
||||
yield { base64, uri: asset.uri }
|
||||
}
|
||||
}
|
||||
return { base64, ...(asset?.uri ? { uri: asset.uri } : {}) }
|
||||
}
|
||||
|
||||
async function pickFromFiles(
|
||||
async function* pickFromFiles(
|
||||
multiple: boolean,
|
||||
launch: typeof DocumentPicker.getDocumentAsync = DocumentPicker.getDocumentAsync,
|
||||
createFile: MobileImageFileFactory = defaultMobileImageFileFactory
|
||||
): Promise<PickedMobileImage | null> {
|
||||
): AsyncGenerator<PickedMobileImage> {
|
||||
const result = await launch({
|
||||
type: 'image/*',
|
||||
multiple: false,
|
||||
multiple,
|
||||
copyToCacheDirectory: true
|
||||
})
|
||||
if (result.canceled) {
|
||||
return null
|
||||
return
|
||||
}
|
||||
const asset = result.assets[0]
|
||||
if (!asset?.uri) {
|
||||
return null
|
||||
for (const asset of result.assets) {
|
||||
if (!asset.uri) {
|
||||
continue
|
||||
}
|
||||
const base64 = await readUriAsBase64(asset.uri, asset.size, createFile)
|
||||
if (base64) {
|
||||
yield { base64, uri: asset.uri }
|
||||
}
|
||||
}
|
||||
const base64 = await readUriAsBase64(asset.uri, asset.size, createFile)
|
||||
return base64 ? { base64, uri: asset.uri } : null
|
||||
}
|
||||
|
||||
type MobileImagePickerDeps = {
|
||||
readonly requestLibraryPermission?: typeof ImagePicker.requestMediaLibraryPermissionsAsync
|
||||
readonly launchLibrary?: typeof ImagePicker.launchImageLibraryAsync
|
||||
readonly launchFiles?: typeof DocumentPicker.getDocumentAsync
|
||||
readonly createFile?: MobileImageFileFactory
|
||||
}
|
||||
|
||||
function pickMobileImagesWithMode(
|
||||
source: MobileImageSource,
|
||||
multiple: boolean,
|
||||
deps?: MobileImagePickerDeps
|
||||
): AsyncIterable<PickedMobileImage> {
|
||||
if (source === 'library') {
|
||||
return pickFromLibrary(
|
||||
multiple,
|
||||
deps?.requestLibraryPermission,
|
||||
deps?.launchLibrary,
|
||||
deps?.createFile
|
||||
)
|
||||
}
|
||||
return pickFromFiles(multiple, deps?.launchFiles, deps?.createFile)
|
||||
}
|
||||
|
||||
export async function pickMobileImage(
|
||||
source: MobileImageSource,
|
||||
deps?: {
|
||||
readonly requestLibraryPermission?: typeof ImagePicker.requestMediaLibraryPermissionsAsync
|
||||
readonly launchLibrary?: typeof ImagePicker.launchImageLibraryAsync
|
||||
readonly launchFiles?: typeof DocumentPicker.getDocumentAsync
|
||||
readonly createFile?: MobileImageFileFactory
|
||||
}
|
||||
deps?: MobileImagePickerDeps
|
||||
): Promise<PickedMobileImage | null> {
|
||||
if (source === 'library') {
|
||||
return pickFromLibrary(deps?.requestLibraryPermission, deps?.launchLibrary, deps?.createFile)
|
||||
for await (const image of pickMobileImagesWithMode(source, false, deps)) {
|
||||
return image
|
||||
}
|
||||
return pickFromFiles(deps?.launchFiles, deps?.createFile)
|
||||
return null
|
||||
}
|
||||
|
||||
export function pickMobileImages(
|
||||
source: MobileImageSource,
|
||||
deps?: MobileImagePickerDeps
|
||||
): AsyncIterable<PickedMobileImage> {
|
||||
return pickMobileImagesWithMode(source, true, deps)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import type { NativeChatMessage } from '../../../src/shared/native-chat-types'
|
||||
import {
|
||||
findLandedImagePreviewEchoes,
|
||||
migrateImagePreviewMessageIds,
|
||||
type PendingImagePreviewEcho
|
||||
} from './mobile-native-chat-draft-reconcile'
|
||||
|
||||
function userText(id: string, text: string): NativeChatMessage {
|
||||
return {
|
||||
id,
|
||||
role: 'user',
|
||||
blocks: [{ type: 'text', text }],
|
||||
timestamp: null,
|
||||
source: 'transcript'
|
||||
}
|
||||
}
|
||||
|
||||
function pending(id: string, images: string[], expectedOccurrence = 1): PendingImagePreviewEcho {
|
||||
return { id, text: '', images, expectedOccurrence, baselineTailMessageId: null }
|
||||
}
|
||||
|
||||
describe('mobile native chat image preview reconciliation', () => {
|
||||
it('keeps separate adjacent image-only sends independently reconcilable', () => {
|
||||
const landed = findLandedImagePreviewEchoes(
|
||||
[
|
||||
userText('source-a', '[Image: source: /tmp/a.png]'),
|
||||
userText('source-b', '[Image: source: /tmp/b.png]')
|
||||
],
|
||||
[pending('pending-a', ['file:///a.jpg']), pending('pending-b', ['file:///b.jpg'], 2)]
|
||||
)
|
||||
|
||||
expect(landed).toEqual([
|
||||
{ pendingId: 'pending-a', messageId: 'source-a', images: ['file:///a.jpg'] },
|
||||
{ pendingId: 'pending-b', messageId: 'source-b', images: ['file:///b.jpg'] }
|
||||
])
|
||||
})
|
||||
|
||||
it('waits for a complete multi-image turn as transcript source frames stream in', () => {
|
||||
const entry = pending('pending', ['file:///a.jpg', 'file:///b.jpg'])
|
||||
const sourceA = userText('source-a', '[Image: source: /tmp/a.png]')
|
||||
const sourceB = userText('source-b', '[Image: source: /tmp/b.png]')
|
||||
|
||||
expect(findLandedImagePreviewEchoes([sourceA], [entry])).toEqual([])
|
||||
expect(findLandedImagePreviewEchoes([sourceA, sourceB], [entry])).toEqual([])
|
||||
expect(
|
||||
findLandedImagePreviewEchoes(
|
||||
[sourceA, sourceB, userText('prompt', '[Image #1] [Image #2]')],
|
||||
[entry]
|
||||
)
|
||||
).toEqual([
|
||||
{
|
||||
pendingId: 'pending',
|
||||
messageId: 'prompt',
|
||||
images: ['file:///a.jpg', 'file:///b.jpg']
|
||||
}
|
||||
])
|
||||
})
|
||||
|
||||
it('moves an early standalone preview to the later folded prompt id', () => {
|
||||
const sessionKey = 'host\0worktree\0tab\0session'
|
||||
const previous = { [sessionKey]: { source: ['file:///a.jpg'] } }
|
||||
const messages = [
|
||||
userText('source', '[Image: source: /tmp/a.png]'),
|
||||
userText('prompt', '[Image #1]')
|
||||
]
|
||||
|
||||
expect(migrateImagePreviewMessageIds(previous, sessionKey, messages)).toEqual({
|
||||
[sessionKey]: { prompt: ['file:///a.jpg'] }
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
import type { NativeChatMessage } from '../../../src/shared/native-chat-types'
|
||||
import { isImageRefBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types'
|
||||
import {
|
||||
isImageSourceUserTurn,
|
||||
normalizeImageTranscriptMessages,
|
||||
stripImagePromptMarker
|
||||
} from './mobile-native-chat-image-transcript-markers'
|
||||
|
||||
|
|
@ -61,6 +62,149 @@ export function countImageSourceTurnsAfter(
|
|||
return count
|
||||
}
|
||||
|
||||
export type PendingImagePreviewEcho = {
|
||||
id: string
|
||||
text: string
|
||||
images?: string[]
|
||||
expectedOccurrence: number
|
||||
baselineTailMessageId: string | null
|
||||
}
|
||||
|
||||
export type LandedImagePreviewEcho = {
|
||||
pendingId: string
|
||||
messageId: string
|
||||
images: string[]
|
||||
}
|
||||
|
||||
const SENT_IMAGE_PREVIEW_LIMIT = 32
|
||||
const SENT_IMAGE_PREVIEW_SESSION_LIMIT = 8
|
||||
|
||||
export function mergeLandedImagePreviewEchoes(
|
||||
previous: Record<string, Record<string, string[]>>,
|
||||
sessionKey: string,
|
||||
landed: readonly LandedImagePreviewEcho[]
|
||||
): Record<string, Record<string, string[]>> {
|
||||
const entries = Object.entries(previous[sessionKey] ?? {})
|
||||
for (const preview of landed) {
|
||||
const existingIndex = entries.findIndex(([messageId]) => messageId === preview.messageId)
|
||||
if (existingIndex >= 0) {
|
||||
entries.splice(existingIndex, 1)
|
||||
}
|
||||
entries.push([preview.messageId, preview.images])
|
||||
}
|
||||
const next = { ...previous }
|
||||
delete next[sessionKey]
|
||||
next[sessionKey] = Object.fromEntries(entries.slice(-SENT_IMAGE_PREVIEW_LIMIT))
|
||||
for (const key of Object.keys(next).slice(0, -SENT_IMAGE_PREVIEW_SESSION_LIMIT)) {
|
||||
delete next[key]
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
function imagePreviewReplacementMessageId(
|
||||
messages: readonly NativeChatMessage[],
|
||||
sourceIndex: number
|
||||
): string | null {
|
||||
const source = messages[sourceIndex]
|
||||
if (!source || !isImageSourceUserTurn(source)) {
|
||||
return null
|
||||
}
|
||||
let nextIndex = sourceIndex + 1
|
||||
while (
|
||||
messages[nextIndex]?.source === source.source &&
|
||||
isImageSourceUserTurn(messages[nextIndex]!)
|
||||
) {
|
||||
nextIndex++
|
||||
}
|
||||
const prompt = messages[nextIndex]
|
||||
const firstText = prompt?.blocks.find((block) => block.type === 'text')
|
||||
return prompt?.role === 'user' &&
|
||||
prompt.source === source.source &&
|
||||
firstText?.type === 'text' &&
|
||||
stripImagePromptMarker(firstText.text) !== firstText.text
|
||||
? prompt.id
|
||||
: null
|
||||
}
|
||||
|
||||
/** Moves previews forward when a progressive source-only transcript frame later
|
||||
* folds into the marker-prefixed prompt with a different authoritative id. */
|
||||
export function migrateImagePreviewMessageIds(
|
||||
previous: Record<string, Record<string, string[]>>,
|
||||
sessionKey: string,
|
||||
messages: readonly NativeChatMessage[]
|
||||
): Record<string, Record<string, string[]>> {
|
||||
const sessionPreviews = previous[sessionKey]
|
||||
if (!sessionPreviews) {
|
||||
return previous
|
||||
}
|
||||
const messageIndexById = new Map(messages.map((message, index) => [message.id, index]))
|
||||
let nextSession: Record<string, string[]> | null = null
|
||||
for (const [messageId, images] of Object.entries(sessionPreviews)) {
|
||||
const sourceIndex = messageIndexById.get(messageId)
|
||||
if (sourceIndex === undefined) {
|
||||
continue
|
||||
}
|
||||
const replacementId = imagePreviewReplacementMessageId(messages, sourceIndex)
|
||||
if (!replacementId) {
|
||||
continue
|
||||
}
|
||||
nextSession ??= { ...sessionPreviews }
|
||||
delete nextSession[messageId]
|
||||
nextSession[replacementId] = [...(nextSession[replacementId] ?? []), ...images]
|
||||
}
|
||||
return nextSession ? { ...previous, [sessionKey]: nextSession } : previous
|
||||
}
|
||||
|
||||
/** Binds local preview URIs to the authoritative transcript turn that replaced
|
||||
* the optimistic bubble. Host paths and marker-only Codex turns cannot render
|
||||
* the phone-local photo without this handoff. */
|
||||
export function findLandedImagePreviewEchoes(
|
||||
messages: readonly NativeChatMessage[],
|
||||
entries: readonly PendingImagePreviewEcho[]
|
||||
): LandedImagePreviewEcho[] {
|
||||
const normalized = normalizeImageTranscriptMessages(messages)
|
||||
const messageIndexById = new Map(normalized.map((message, index) => [message.id, index]))
|
||||
const claimedMessageIds = new Set<string>()
|
||||
const landed: LandedImagePreviewEcho[] = []
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.images?.length) {
|
||||
continue
|
||||
}
|
||||
const targetText = entry.text.trim()
|
||||
const candidates = normalized.filter((message) => {
|
||||
if (message.role !== 'user') {
|
||||
return false
|
||||
}
|
||||
if (targetText) {
|
||||
return normalizedUserText(message) === targetText
|
||||
}
|
||||
const imageCount = message.blocks.filter(isImageRefBlock).length
|
||||
return message.blocks.length === 0 || imageCount >= entry.images!.length
|
||||
})
|
||||
const tailIndex = entry.baselineTailMessageId
|
||||
? messageIndexById.get(entry.baselineTailMessageId)
|
||||
: -1
|
||||
const occurrenceIndex = Math.max(0, entry.expectedOccurrence - 1)
|
||||
const candidate = targetText
|
||||
? candidates[occurrenceIndex]
|
||||
: candidates.filter(
|
||||
(message) =>
|
||||
tailIndex === undefined || (messageIndexById.get(message.id) ?? -1) > tailIndex
|
||||
)[occurrenceIndex]
|
||||
if (
|
||||
!candidate ||
|
||||
claimedMessageIds.has(candidate.id) ||
|
||||
(tailIndex !== undefined && (messageIndexById.get(candidate.id) ?? -1) <= tailIndex)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
claimedMessageIds.add(candidate.id)
|
||||
landed.push({ pendingId: entry.id, messageId: candidate.id, images: entry.images })
|
||||
}
|
||||
return landed
|
||||
}
|
||||
|
||||
export function findLandedUnconfirmedSends(
|
||||
messages: readonly NativeChatMessage[],
|
||||
entries: readonly UnconfirmedSend[]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
import type { RpcResponse, RpcSuccess } from '../transport/types'
|
||||
import { uploadMobileNativeChatImage } from './mobile-native-chat-image-attachment'
|
||||
import { uploadMobileNativeChatImages } from './mobile-native-chat-image-attachment'
|
||||
|
||||
function ok(id: string, result: unknown): RpcSuccess {
|
||||
return { id, ok: true, result, _meta: { runtimeId: 'runtime-1' } }
|
||||
|
|
@ -16,6 +16,10 @@ function methodNotFound(id: string): RpcResponse {
|
|||
}
|
||||
}
|
||||
|
||||
function failed(id: string, message: string): RpcResponse {
|
||||
return { id, ok: false, error: { code: 'failed', message }, _meta: { runtimeId: 'r' } }
|
||||
}
|
||||
|
||||
function clientWithResponses(responses: RpcResponse[]): Pick<RpcClient, 'sendRequest'> & {
|
||||
calls: { method: string; params: unknown }[]
|
||||
} {
|
||||
|
|
@ -33,67 +37,137 @@ function clientWithResponses(responses: RpcResponse[]): Pick<RpcClient, 'sendReq
|
|||
}
|
||||
}
|
||||
|
||||
describe('uploadMobileNativeChatImage', () => {
|
||||
describe('uploadMobileNativeChatImages', () => {
|
||||
it('uploads the picked image and returns its host path + local preview uri, without any terminal.send', async () => {
|
||||
const client = clientWithResponses([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/orca-attach.png')
|
||||
])
|
||||
|
||||
const result = await uploadMobileNativeChatImage('library', {
|
||||
const result = await uploadMobileNativeChatImages('library', {
|
||||
client,
|
||||
getConnectionId: async () => 'conn-7',
|
||||
pickImage: vi.fn().mockResolvedValue({ base64: 'AAAA', uri: 'file:///photo.jpg' })
|
||||
pickImages: vi.fn().mockResolvedValue([{ base64: 'AAAA', uri: 'file:///photo.jpg' }])
|
||||
})
|
||||
|
||||
expect(result).toEqual({ path: '/tmp/orca-attach.png', previewUri: 'file:///photo.jpg' })
|
||||
expect(result).toEqual([{ path: '/tmp/orca-attach.png', previewUri: 'file:///photo.jpg' }])
|
||||
// Native chat defers the paste to submit — nothing is sent to the terminal here.
|
||||
expect(client.calls.some((call) => call.method === 'terminal.send')).toBe(false)
|
||||
const saveCall = client.calls.find((c) => c.method === 'clipboard.saveImageAsTempFile')
|
||||
expect(saveCall?.params).toMatchObject({ connectionId: 'conn-7' })
|
||||
})
|
||||
|
||||
it('uploads all three selected images in picker order', async () => {
|
||||
const client = clientWithResponses([
|
||||
methodNotFound('start-a'),
|
||||
ok('save-a', '/tmp/a.png'),
|
||||
methodNotFound('start-b'),
|
||||
ok('save-b', '/tmp/b.png'),
|
||||
methodNotFound('start-c'),
|
||||
ok('save-c', '/tmp/c.png')
|
||||
])
|
||||
|
||||
const order: string[] = []
|
||||
async function* pickImages() {
|
||||
for (const image of [
|
||||
{ base64: 'AAAA', uri: 'file:///a.jpg' },
|
||||
{ base64: 'BBBB', uri: 'file:///b.jpg' },
|
||||
{ base64: 'CCCC', uri: 'file:///c.jpg' }
|
||||
]) {
|
||||
order.push(`read:${image.uri}`)
|
||||
yield image
|
||||
}
|
||||
}
|
||||
const result = await uploadMobileNativeChatImages('library', {
|
||||
client,
|
||||
getConnectionId: async () => 'conn-7',
|
||||
pickImages,
|
||||
onImageUploaded: (image) => order.push(`uploaded:${image.previewUri}`)
|
||||
})
|
||||
|
||||
expect(result).toEqual([
|
||||
{ path: '/tmp/a.png', previewUri: 'file:///a.jpg' },
|
||||
{ path: '/tmp/b.png', previewUri: 'file:///b.jpg' },
|
||||
{ path: '/tmp/c.png', previewUri: 'file:///c.jpg' }
|
||||
])
|
||||
expect(order).toEqual([
|
||||
'read:file:///a.jpg',
|
||||
'uploaded:file:///a.jpg',
|
||||
'read:file:///b.jpg',
|
||||
'uploaded:file:///b.jpg',
|
||||
'read:file:///c.jpg',
|
||||
'uploaded:file:///c.jpg'
|
||||
])
|
||||
})
|
||||
|
||||
it('returns null when the picker is cancelled and uploads nothing', async () => {
|
||||
const client = clientWithResponses([])
|
||||
|
||||
const result = await uploadMobileNativeChatImage('library', {
|
||||
const result = await uploadMobileNativeChatImages('library', {
|
||||
client,
|
||||
getConnectionId: async () => null,
|
||||
pickImage: vi.fn().mockResolvedValue(null)
|
||||
pickImages: vi.fn().mockResolvedValue([])
|
||||
})
|
||||
|
||||
expect(result).toBeNull()
|
||||
expect(result).toEqual([])
|
||||
expect(client.calls).toEqual([])
|
||||
})
|
||||
|
||||
it('reports completed uploads before a later image fails', async () => {
|
||||
const client = clientWithResponses([
|
||||
methodNotFound('start-a'),
|
||||
ok('save-a', '/tmp/a.png'),
|
||||
methodNotFound('start-b'),
|
||||
failed('save-b', 'upload failed')
|
||||
])
|
||||
const onImageUploaded = vi.fn()
|
||||
|
||||
await expect(
|
||||
uploadMobileNativeChatImages('library', {
|
||||
client,
|
||||
getConnectionId: async () => null,
|
||||
pickImages: vi.fn().mockResolvedValue([
|
||||
{ base64: 'AAAA', uri: 'file:///a.jpg' },
|
||||
{ base64: 'BBBB', uri: 'file:///b.jpg' }
|
||||
]),
|
||||
onImageUploaded
|
||||
})
|
||||
).rejects.toThrow('upload failed')
|
||||
expect(onImageUploaded).toHaveBeenCalledOnce()
|
||||
expect(onImageUploaded).toHaveBeenCalledWith({
|
||||
path: '/tmp/a.png',
|
||||
previewUri: 'file:///a.jpg'
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to an inline data uri for the preview when the picker omits a uri', async () => {
|
||||
const client = clientWithResponses([methodNotFound('start'), ok('save', '/tmp/x.png')])
|
||||
|
||||
const result = await uploadMobileNativeChatImage('files', {
|
||||
const result = await uploadMobileNativeChatImages('files', {
|
||||
client,
|
||||
getConnectionId: async () => null,
|
||||
pickImage: vi.fn().mockResolvedValue({ base64: 'BBBB' })
|
||||
pickImages: vi.fn().mockResolvedValue([{ base64: 'BBBB' }])
|
||||
})
|
||||
|
||||
expect(result).toEqual({ path: '/tmp/x.png', previewUri: 'data:image/png;base64,BBBB' })
|
||||
expect(result).toEqual([{ path: '/tmp/x.png', previewUri: 'data:image/png;base64,BBBB' }])
|
||||
})
|
||||
|
||||
it('signals upload start only after a real image is picked', async () => {
|
||||
const onUploadStart = vi.fn()
|
||||
const cancelledClient = clientWithResponses([])
|
||||
await uploadMobileNativeChatImage('library', {
|
||||
await uploadMobileNativeChatImages('library', {
|
||||
client: cancelledClient,
|
||||
getConnectionId: async () => null,
|
||||
pickImage: vi.fn().mockResolvedValue(null),
|
||||
pickImages: vi.fn().mockResolvedValue([]),
|
||||
onUploadStart
|
||||
})
|
||||
expect(onUploadStart).not.toHaveBeenCalled()
|
||||
|
||||
const client = clientWithResponses([methodNotFound('start'), ok('save', '/tmp/y.png')])
|
||||
await uploadMobileNativeChatImage('library', {
|
||||
await uploadMobileNativeChatImages('library', {
|
||||
client,
|
||||
getConnectionId: async () => null,
|
||||
pickImage: vi.fn().mockResolvedValue({ base64: 'CCCC', uri: 'file:///y.jpg' }),
|
||||
pickImages: vi.fn().mockResolvedValue([{ base64: 'CCCC', uri: 'file:///y.jpg' }]),
|
||||
onUploadStart
|
||||
})
|
||||
expect(onUploadStart).toHaveBeenCalledTimes(1)
|
||||
|
|
|
|||
|
|
@ -13,34 +13,67 @@ export type PendingNativeChatImage = {
|
|||
readonly previewUri: string
|
||||
}
|
||||
|
||||
export type UploadNativeChatImageDeps = {
|
||||
export function appendPendingNativeChatImages(
|
||||
current: readonly PendingNativeChatImage[],
|
||||
uploaded: readonly Omit<PendingNativeChatImage, 'id'>[],
|
||||
idCounter: { current: number }
|
||||
): PendingNativeChatImage[] {
|
||||
return [
|
||||
...current,
|
||||
...uploaded.map((image) => {
|
||||
idCounter.current += 1
|
||||
return { id: `img-${idCounter.current}`, ...image }
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
export type UploadNativeChatImagesDeps = {
|
||||
readonly client: Pick<RpcClient, 'sendRequest'>
|
||||
readonly getConnectionId: () => Promise<string | null>
|
||||
// Injected so this module stays free of expo/react-native imports (unit-testable).
|
||||
readonly pickImage: (source: MobileImageSource) => Promise<PickedMobileImage | null>
|
||||
readonly pickImages: (
|
||||
source: MobileImageSource
|
||||
) =>
|
||||
| Iterable<PickedMobileImage>
|
||||
| AsyncIterable<PickedMobileImage>
|
||||
| Promise<Iterable<PickedMobileImage> | AsyncIterable<PickedMobileImage>>
|
||||
// Fired once the user has picked an image and the host upload is about to start —
|
||||
// lets the UI show the attach spinner only for the transfer, not the picker.
|
||||
readonly onUploadStart?: () => void
|
||||
/** Retains each completed upload if a later image in the same selection fails. */
|
||||
readonly onImageUploaded?: (image: Omit<PendingNativeChatImage, 'id'>) => void
|
||||
}
|
||||
|
||||
/** Picks an image and uploads it to the host, returning the host path + a local
|
||||
* preview URI — but does NOT paste it into the terminal. Unlike the terminal
|
||||
* attach flow, native chat holds the image as a composer chip and rides it along
|
||||
* on submit (desktop parity), so the chip and the agent input never diverge.
|
||||
* Returns null when the user cancels the picker. */
|
||||
export async function uploadMobileNativeChatImage(
|
||||
* Returns an empty array when the user cancels the picker. */
|
||||
export async function uploadMobileNativeChatImages(
|
||||
source: MobileImageSource,
|
||||
{ client, getConnectionId, pickImage, onUploadStart }: UploadNativeChatImageDeps
|
||||
): Promise<Omit<PendingNativeChatImage, 'id'> | null> {
|
||||
const picked = await pickImage(source)
|
||||
if (!picked) {
|
||||
return null
|
||||
{
|
||||
client,
|
||||
getConnectionId,
|
||||
pickImages,
|
||||
onUploadStart,
|
||||
onImageUploaded
|
||||
}: UploadNativeChatImagesDeps
|
||||
): Promise<Omit<PendingNativeChatImage, 'id'>[]> {
|
||||
const picked = await pickImages(source)
|
||||
const uploaded: Omit<PendingNativeChatImage, 'id'>[] = []
|
||||
let connectionId: string | null = null
|
||||
for await (const image of picked) {
|
||||
if (uploaded.length === 0) {
|
||||
onUploadStart?.()
|
||||
connectionId = await getConnectionId()
|
||||
}
|
||||
const path = await saveMobileClipboardImageAsTempFile(client, image.base64, { connectionId })
|
||||
// Prefer the picker's local URI for the thumbnail; fall back to an inline data
|
||||
// URI when the source omitted one (RN <Image> renders both).
|
||||
const previewUri = image.uri ?? `data:image/png;base64,${image.base64}`
|
||||
const result = { path, previewUri }
|
||||
uploaded.push(result)
|
||||
onImageUploaded?.(result)
|
||||
}
|
||||
onUploadStart?.()
|
||||
const connectionId = await getConnectionId()
|
||||
const path = await saveMobileClipboardImageAsTempFile(client, picked.base64, { connectionId })
|
||||
// Prefer the picker's local URI for the thumbnail; fall back to an inline data
|
||||
// URI when the source omitted one (RN <Image> renders both).
|
||||
const previewUri = picked.uri ?? `data:image/png;base64,${picked.base64}`
|
||||
return { path, previewUri }
|
||||
return uploaded
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,17 +27,22 @@ function clientWithResponses(responses: RpcResponse[]): Pick<RpcClient, 'sendReq
|
|||
|
||||
describe('pasteMobileNativeChatImagePaths', () => {
|
||||
it('clears the input line, then pastes each path as a bracketed, non-submitting terminal.send with the mobile client tag', async () => {
|
||||
const client = clientWithResponses([sendResult(true), sendResult(true), sendResult(true)])
|
||||
const client = clientWithResponses([
|
||||
sendResult(true),
|
||||
sendResult(true),
|
||||
sendResult(true),
|
||||
sendResult(true)
|
||||
])
|
||||
|
||||
const ok = await pasteMobileNativeChatImagePaths({
|
||||
client,
|
||||
terminal: 'term-1',
|
||||
deviceToken: 'device-9',
|
||||
imagePaths: ['/tmp/a.png', '/tmp/b.png']
|
||||
imagePaths: ['/tmp/a.png', '/tmp/b.png', '/tmp/c.png']
|
||||
})
|
||||
|
||||
expect(ok).toBe(true)
|
||||
expect(client.calls).toHaveLength(3)
|
||||
expect(client.calls).toHaveLength(4)
|
||||
// Leading Ctrl+U clears any stale input so a retry can't duplicate the image.
|
||||
expect(client.calls[0]).toEqual({
|
||||
method: 'terminal.send',
|
||||
|
|
@ -50,6 +55,7 @@ describe('pasteMobileNativeChatImagePaths', () => {
|
|||
})
|
||||
expect(client.calls[1]?.params.text).toBe('\x1b[200~/tmp/a.png\x1b[201~')
|
||||
expect(client.calls[2]?.params.text).toBe('\x1b[200~/tmp/b.png\x1b[201~')
|
||||
expect(client.calls[3]?.params.text).toBe('\x1b[200~/tmp/c.png\x1b[201~')
|
||||
})
|
||||
|
||||
it('stops and reports failure as soon as a paste is rejected', async () => {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
export type MobileNativeChatPendingMessage = {
|
||||
id: string
|
||||
text: string
|
||||
expectedOccurrence: number
|
||||
/** Local preview URIs carried by the send for its optimistic echo. */
|
||||
images?: string[]
|
||||
baselineTailMessageId: string | null
|
||||
}
|
||||
|
||||
export type MobileNativeChatSendOrigin = {
|
||||
draftKey: string
|
||||
pendingKey: string | null
|
||||
normalizedText: string
|
||||
baselineOccurrences: number
|
||||
baselineTailMessageId: string | null
|
||||
}
|
||||
|
||||
type PendingByKey = Record<string, MobileNativeChatPendingMessage[]>
|
||||
|
||||
export function combineMobileNativeChatPending(
|
||||
session: MobileNativeChatPendingMessage[],
|
||||
waiting: readonly MobileNativeChatPendingMessage[]
|
||||
): MobileNativeChatPendingMessage[] {
|
||||
if (waiting.length === 0) {
|
||||
return session
|
||||
}
|
||||
const sessionIds = new Set(session.map((item) => item.id))
|
||||
return [...session, ...waiting.filter((item) => !sessionIds.has(item.id))]
|
||||
}
|
||||
|
||||
export function appendMobileNativeChatPending(
|
||||
previous: PendingByKey,
|
||||
key: string,
|
||||
id: string,
|
||||
origin: MobileNativeChatSendOrigin,
|
||||
text: string,
|
||||
images?: string[]
|
||||
): PendingByKey {
|
||||
const current = previous[key] ?? []
|
||||
const earlierOutstanding = current.filter(
|
||||
(pending) =>
|
||||
pending.text.trim() === origin.normalizedText &&
|
||||
pending.expectedOccurrence > origin.baselineOccurrences
|
||||
).length
|
||||
const expectedImageEchoOrdinal =
|
||||
current.filter((pending) => pending.text.trim() === '' && pending.images?.length).length + 1
|
||||
return {
|
||||
...previous,
|
||||
[key]: [
|
||||
...current,
|
||||
{
|
||||
id,
|
||||
text,
|
||||
expectedOccurrence:
|
||||
origin.normalizedText === ''
|
||||
? expectedImageEchoOrdinal
|
||||
: origin.baselineOccurrences + earlierOutstanding + 1,
|
||||
baselineTailMessageId: origin.baselineTailMessageId,
|
||||
...(images?.length ? { images } : {})
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
export function mergeWaitingSessionPending(
|
||||
previous: PendingByKey,
|
||||
sessionKey: string,
|
||||
waiting: readonly MobileNativeChatPendingMessage[]
|
||||
): PendingByKey {
|
||||
const current = previous[sessionKey] ?? []
|
||||
const currentIds = new Set(current.map((item) => item.id))
|
||||
const moved = waiting.filter((item) => !currentIds.has(item.id))
|
||||
return moved.length > 0 ? { ...previous, [sessionKey]: [...current, ...moved] } : previous
|
||||
}
|
||||
|
||||
export function removeWaitingSessionPending(
|
||||
previous: PendingByKey,
|
||||
draftKey: string,
|
||||
movedIds: ReadonlySet<string>
|
||||
): PendingByKey {
|
||||
const remaining = (previous[draftKey] ?? []).filter((item) => !movedIds.has(item.id))
|
||||
if (remaining.length > 0) {
|
||||
return { ...previous, [draftKey]: remaining }
|
||||
}
|
||||
if (!(draftKey in previous)) {
|
||||
return previous
|
||||
}
|
||||
const next = { ...previous }
|
||||
delete next[draftKey]
|
||||
return next
|
||||
}
|
||||
|
|
@ -116,6 +116,36 @@ describe('buildMobileNativeChatTransientData', () => {
|
|||
expect(data[0]?.blocks).toEqual([{ type: 'image-ref', path: '/tmp/a.png' }])
|
||||
})
|
||||
|
||||
it('keeps the phone-local image visible when the transcript replaces its optimistic echo', () => {
|
||||
const folded = foldMobileNativeChatMessages([
|
||||
user('source', '[Image: source: /tmp/a.png]'),
|
||||
user('prompt', '[Image #1] look at this')
|
||||
])
|
||||
const result = buildMobileNativeChatTransientData({
|
||||
folded,
|
||||
streaming: null,
|
||||
pending: [],
|
||||
imagePreviewsByMessageId: { prompt: ['file:///phone-photo.jpg'] }
|
||||
})
|
||||
|
||||
expect(result.data).toHaveLength(1)
|
||||
expect(result.data[0]?.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png', url: 'file:///phone-photo.jpg' },
|
||||
{ type: 'text', text: 'look at this' }
|
||||
])
|
||||
})
|
||||
|
||||
it('restores the local preview onto a marker-only transcript turn', () => {
|
||||
const result = buildMobileNativeChatTransientData({
|
||||
folded: foldMobileNativeChatMessages([user('prompt', '[Image #1]')]),
|
||||
streaming: null,
|
||||
pending: [],
|
||||
imagePreviewsByMessageId: { prompt: ['file:///phone-photo.jpg'] }
|
||||
})
|
||||
|
||||
expect(result.data[0]?.blocks).toEqual([{ type: 'image-ref', url: 'file:///phone-photo.jpg' }])
|
||||
})
|
||||
|
||||
it('appends a synthetic bubble for gated streaming text, between transcript and pending', () => {
|
||||
// Whether text streams at all is the gate's call
|
||||
// (`mobile-native-chat-streaming-gate.test.ts`); this only places it.
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import {
|
|||
formatNativeChatEmptyStateCopy,
|
||||
type NativeChatEmptyStateCopy
|
||||
} from '../../../src/shared/native-chat-empty-state'
|
||||
import type { NativeChatMessage } from '../../../src/shared/native-chat-types'
|
||||
import { isImageRefBlock, type NativeChatMessage } from '../../../src/shared/native-chat-types'
|
||||
import { foldToolMessages } from './mobile-native-chat-blocks'
|
||||
import { normalizeImageTranscriptMessages } from './mobile-native-chat-image-transcript-markers'
|
||||
import { stripNoiseMessages } from './mobile-native-chat-noise'
|
||||
|
|
@ -55,15 +55,37 @@ export function foldMobileNativeChatMessages(messages: NativeChatMessage[]): Nat
|
|||
export function buildMobileNativeChatTransientData({
|
||||
folded,
|
||||
streaming,
|
||||
pending
|
||||
pending,
|
||||
imagePreviewsByMessageId
|
||||
}: {
|
||||
folded: NativeChatMessage[]
|
||||
/** Streaming bubble text, already gated by `deriveMobileNativeChatStreaming`. */
|
||||
streaming: string | null
|
||||
pending: MobileNativeChatPendingItem[]
|
||||
imagePreviewsByMessageId?: Record<string, string[]>
|
||||
}): { folded: NativeChatMessage[]; streaming: string | null; data: NativeChatMessage[] } {
|
||||
const renderedFolded = folded.map((message) => {
|
||||
const previews = imagePreviewsByMessageId?.[message.id]
|
||||
if (message.role !== 'user' || !previews?.length) {
|
||||
return message
|
||||
}
|
||||
let previewIndex = 0
|
||||
const blocks = message.blocks.map((block) => {
|
||||
if (!isImageRefBlock(block)) {
|
||||
return block
|
||||
}
|
||||
const url = previews[previewIndex]
|
||||
previewIndex += 1
|
||||
return url ? { ...block, url } : block
|
||||
})
|
||||
while (previewIndex < previews.length) {
|
||||
blocks.push({ type: 'image-ref', url: previews[previewIndex] })
|
||||
previewIndex += 1
|
||||
}
|
||||
return { ...message, blocks }
|
||||
})
|
||||
const data: NativeChatMessage[] = [
|
||||
...folded,
|
||||
...renderedFolded,
|
||||
...(streaming
|
||||
? [
|
||||
{
|
||||
|
|
@ -88,5 +110,5 @@ export function buildMobileNativeChatTransientData({
|
|||
source: 'transcript' as const
|
||||
}))
|
||||
]
|
||||
return { folded, streaming, data }
|
||||
return { folded: renderedFolded, streaming, data }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ vi.mock('./use-mobile-native-chat-drafts', () => ({
|
|||
composerText: '',
|
||||
setComposerText: vi.fn(),
|
||||
pending: [],
|
||||
imagePreviewsByMessageId: {},
|
||||
captureSendOrigin,
|
||||
readSeededLaunchDraft: () => null,
|
||||
readSeededLaunchDraftSeed: () => null,
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ export type MobileNativeChatController = {
|
|||
chatComposerText: string
|
||||
setChatComposerText: Dispatch<SetStateAction<string>>
|
||||
chatPending: MobileNativeChatPendingMessage[]
|
||||
chatImagePreviewsByMessageId: Record<string, string[]>
|
||||
nativeChatSession: ReturnType<typeof useMobileNativeChatSession>
|
||||
nativeChatAgentWorking: boolean
|
||||
nativeChatStreamingText?: string
|
||||
|
|
@ -160,6 +161,7 @@ export function useMobileNativeChatController(args: {
|
|||
composerText: chatComposerText,
|
||||
setComposerText: setChatComposerText,
|
||||
pending: chatPending,
|
||||
imagePreviewsByMessageId: chatImagePreviewsByMessageId,
|
||||
captureSendOrigin,
|
||||
readSeededLaunchDraft,
|
||||
readSeededLaunchDraftSeed,
|
||||
|
|
@ -326,6 +328,7 @@ export function useMobileNativeChatController(args: {
|
|||
chatComposerText,
|
||||
setChatComposerText,
|
||||
chatPending,
|
||||
chatImagePreviewsByMessageId,
|
||||
nativeChatSession,
|
||||
nativeChatAgentWorking,
|
||||
nativeChatStreamingText,
|
||||
|
|
|
|||
|
|
@ -269,6 +269,7 @@ describe('useMobileNativeChatDrafts', () => {
|
|||
)
|
||||
)
|
||||
expect(state?.pending).toEqual([])
|
||||
expect(state?.imagePreviewsByMessageId).toEqual({ u1: ['file:///a.jpg'] })
|
||||
})
|
||||
|
||||
it("keeps an image-only echo when an unrelated text send's echo lands", async () => {
|
||||
|
|
@ -345,6 +346,29 @@ describe('useMobileNativeChatDrafts', () => {
|
|||
)
|
||||
)
|
||||
expect(state?.pending).toEqual([])
|
||||
expect(state?.imagePreviewsByMessageId).toEqual({ u2: ['file:///a.jpg'] })
|
||||
})
|
||||
|
||||
it('hands a marker-only image preview to the authoritative user bubble', async () => {
|
||||
await mount('a')
|
||||
const origin = state?.captureSendOrigin('')
|
||||
act(() => {
|
||||
if (origin) {
|
||||
state?.acceptSend(origin, '', ['file:///a.jpg'])
|
||||
}
|
||||
})
|
||||
|
||||
await act(async () =>
|
||||
renderer?.update(
|
||||
createElement(Harness, {
|
||||
tabId: 'a',
|
||||
messages: [userTextMessage('u1', '[Image #1]')]
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
expect(state?.pending).toEqual([])
|
||||
expect(state?.imagePreviewsByMessageId).toEqual({ u1: ['file:///a.jpg'] })
|
||||
})
|
||||
|
||||
it('does not reconcile a repeated send against an older identical turn', async () => {
|
||||
|
|
@ -722,21 +746,59 @@ describe('useMobileNativeChatDrafts', () => {
|
|||
}
|
||||
})
|
||||
|
||||
it('accepts and clears the first send before a provider session id exists', async () => {
|
||||
it('preserves first-send images through session assignment and transcript replacement', async () => {
|
||||
await mount('a')
|
||||
await act(async () => renderer?.update(createElement(Harness, { tabId: 'a', sessionId: null })))
|
||||
act(() => state?.setComposerText('start the session'))
|
||||
const images = ['file:///a.jpg', 'file:///b.jpg', 'file:///c.jpg']
|
||||
act(() => state?.setComposerText('look'))
|
||||
|
||||
const origin = state?.captureSendOrigin('start the session')
|
||||
const origin = state?.captureSendOrigin('look')
|
||||
expect(origin).toMatchObject({ pendingKey: null })
|
||||
act(() => {
|
||||
if (origin) {
|
||||
state?.clearDraftForSend(origin, 'start the session')
|
||||
state?.acceptSend(origin, 'start the session')
|
||||
state?.clearDraftForSend(origin, 'look')
|
||||
state?.acceptSend(origin, 'look', images)
|
||||
}
|
||||
})
|
||||
|
||||
expect(state?.composerText).toBe('')
|
||||
expect(state?.pending.map((pending) => pending.images)).toEqual([images])
|
||||
|
||||
await act(async () =>
|
||||
renderer?.update(createElement(Harness, { tabId: 'a', sessionId: 'assigned' }))
|
||||
)
|
||||
expect(state?.pending.map((pending) => pending.images)).toEqual([images])
|
||||
|
||||
await act(async () =>
|
||||
renderer?.update(
|
||||
createElement(Harness, {
|
||||
tabId: 'a',
|
||||
sessionId: 'assigned',
|
||||
messages: [
|
||||
userTextMessage('source-1', '[Image: source: /tmp/a.png]'),
|
||||
userTextMessage('source-2', '[Image: source: /tmp/b.png]'),
|
||||
userTextMessage('source-3', '[Image: source: /tmp/c.png]')
|
||||
]
|
||||
})
|
||||
)
|
||||
)
|
||||
expect(state?.pending.map((pending) => pending.images)).toEqual([images])
|
||||
|
||||
await act(async () =>
|
||||
renderer?.update(
|
||||
createElement(Harness, {
|
||||
tabId: 'a',
|
||||
sessionId: 'assigned',
|
||||
messages: [
|
||||
userTextMessage('source-1', '[Image: source: /tmp/a.png]'),
|
||||
userTextMessage('source-2', '[Image: source: /tmp/b.png]'),
|
||||
userTextMessage('source-3', '[Image: source: /tmp/c.png]'),
|
||||
userTextMessage('prompt', '[Image #1] [Image #2] [Image #3] look')
|
||||
]
|
||||
})
|
||||
)
|
||||
)
|
||||
expect(state?.pending).toEqual([])
|
||||
expect(state?.imagePreviewsByMessageId).toEqual({ prompt: images })
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -3,35 +3,29 @@ import type { NativeChatMessage } from '../../../src/shared/native-chat-types'
|
|||
import {
|
||||
countImageSourceTurnsAfter,
|
||||
countUserTextOccurrences,
|
||||
findLandedImagePreviewEchoes,
|
||||
findLandedUnconfirmedSends,
|
||||
mergeLandedImagePreviewEchoes,
|
||||
migrateImagePreviewMessageIds,
|
||||
normalizedUserText,
|
||||
type UnconfirmedSend
|
||||
} from './mobile-native-chat-draft-reconcile'
|
||||
import {
|
||||
appendMobileNativeChatPending,
|
||||
combineMobileNativeChatPending,
|
||||
mergeWaitingSessionPending,
|
||||
removeWaitingSessionPending,
|
||||
type MobileNativeChatPendingMessage,
|
||||
type MobileNativeChatSendOrigin
|
||||
} from './mobile-native-chat-pending-echo'
|
||||
import { mobileNativeChatScopeKey } from './mobile-native-chat-scope-key'
|
||||
import { useMobileNativeChatLaunchDraftSeed } from './use-mobile-native-chat-launch-draft-seed'
|
||||
import type { MobileNativeChatLaunchDraftSeed } from './use-mobile-native-chat-launch-draft-seed'
|
||||
|
||||
export type MobileNativeChatPendingMessage = {
|
||||
id: string
|
||||
text: string
|
||||
expectedOccurrence: number
|
||||
/** Local preview URIs of images ridden along on the send, rendered as thumbnails
|
||||
* on the echo bubble so the sent photo shows before the transcript catches up. */
|
||||
images?: string[]
|
||||
/** Transcript tail when sent — an image-only echo (no text to match) reconciles
|
||||
* against new `[Image: source: …]` echo turns after this id, so pagination,
|
||||
* agent replies, and unrelated text echoes can't clear it early. */
|
||||
baselineTailMessageId: string | null
|
||||
}
|
||||
export type MobileNativeChatSendOrigin = {
|
||||
draftKey: string
|
||||
pendingKey: string | null
|
||||
normalizedText: string
|
||||
baselineOccurrences: number
|
||||
baselineTailMessageId: string | null
|
||||
}
|
||||
export type { MobileNativeChatPendingMessage, MobileNativeChatSendOrigin }
|
||||
|
||||
const NO_PENDING_MESSAGES: MobileNativeChatPendingMessage[] = []
|
||||
const NO_IMAGE_PREVIEWS: Record<string, string[]> = {}
|
||||
|
||||
// How long an ack-lost send waits for its transcript echo before the UI surfaces
|
||||
// that delivery remains unconfirmed.
|
||||
|
|
@ -57,6 +51,9 @@ export function useMobileNativeChatDrafts(args: {
|
|||
composerText: string
|
||||
setComposerText: Dispatch<SetStateAction<string>>
|
||||
pending: MobileNativeChatPendingMessage[]
|
||||
/** Phone-local previews rebound to the transcript message that replaced the
|
||||
* optimistic echo, keyed by authoritative message id. */
|
||||
imagePreviewsByMessageId: Record<string, string[]>
|
||||
captureSendOrigin: (text: string) => MobileNativeChatSendOrigin | null
|
||||
/** Launch-context text still believed to be parked on the agent's TUI input
|
||||
* line, or null once it has been declined or retired. Send paths size their
|
||||
|
|
@ -91,6 +88,12 @@ export function useMobileNativeChatDrafts(args: {
|
|||
const [pendingBySession, setPendingBySession] = useState<
|
||||
Record<string, MobileNativeChatPendingMessage[]>
|
||||
>({})
|
||||
const [pendingWaitingForSession, setPendingWaitingForSession] = useState<
|
||||
Record<string, MobileNativeChatPendingMessage[]>
|
||||
>({})
|
||||
const [imagePreviewsBySession, setImagePreviewsBySession] = useState<
|
||||
Record<string, Record<string, string[]>>
|
||||
>({})
|
||||
const pendingCounterRef = useRef(0)
|
||||
const messagesRef = useRef(messages)
|
||||
messagesRef.current = messages
|
||||
|
|
@ -162,41 +165,21 @@ export function useMobileNativeChatDrafts(args: {
|
|||
|
||||
const acceptSend = useCallback(
|
||||
(origin: MobileNativeChatSendOrigin, text: string, images?: string[]) => {
|
||||
// Why: the first prompt can be sent before the provider reports a session
|
||||
// id; wait for an id before keying an optimistic echo.
|
||||
if (!origin.pendingKey) {
|
||||
if (!origin.pendingKey && !images?.length) {
|
||||
return
|
||||
}
|
||||
const pendingKey = origin.pendingKey
|
||||
pendingCounterRef.current += 1
|
||||
setPendingBySession((previous) => {
|
||||
const current = previous[pendingKey] ?? NO_PENDING_MESSAGES
|
||||
const earlierOutstanding = current.filter(
|
||||
(pending) =>
|
||||
pending.text.trim() === origin.normalizedText &&
|
||||
pending.expectedOccurrence > origin.baselineOccurrences
|
||||
).length
|
||||
// An empty-text send reconciles by image-echo ordinal: every outstanding
|
||||
// send's ridden-along images echo as `[Image: source: …]` turns after
|
||||
// this send's baseline tail, ahead of this send's own echo.
|
||||
const expectedImageEchoOrdinal =
|
||||
current.reduce(
|
||||
(sum, pending) =>
|
||||
sum + (pending.images?.length ?? (pending.text.trim() === '' ? 1 : 0)),
|
||||
0
|
||||
) + 1
|
||||
const pending: MobileNativeChatPendingMessage = {
|
||||
id: `pending-${pendingCounterRef.current}`,
|
||||
text,
|
||||
expectedOccurrence:
|
||||
origin.normalizedText === ''
|
||||
? expectedImageEchoOrdinal
|
||||
: origin.baselineOccurrences + earlierOutstanding + 1,
|
||||
baselineTailMessageId: origin.baselineTailMessageId,
|
||||
...(images && images.length > 0 ? { images } : {})
|
||||
}
|
||||
return { ...previous, [pendingKey]: [...current, pending] }
|
||||
})
|
||||
const id = `pending-${pendingCounterRef.current}`
|
||||
const key = origin.pendingKey
|
||||
if (key) {
|
||||
setPendingBySession((previous) =>
|
||||
appendMobileNativeChatPending(previous, key, id, origin, text, images)
|
||||
)
|
||||
} else {
|
||||
setPendingWaitingForSession((previous) =>
|
||||
appendMobileNativeChatPending(previous, origin.draftKey, id, origin, text, images)
|
||||
)
|
||||
}
|
||||
},
|
||||
[]
|
||||
)
|
||||
|
|
@ -255,9 +238,7 @@ export function useMobileNativeChatDrafts(args: {
|
|||
const landedSet = new Set(landed)
|
||||
unconfirmedRef.current = unconfirmedRef.current.filter((entry) => !landedSet.has(entry))
|
||||
for (const entry of landed) {
|
||||
if (entry.deadline !== null) {
|
||||
clearTimeout(entry.deadline)
|
||||
}
|
||||
clearTimeout(entry.deadline ?? undefined)
|
||||
}
|
||||
}, [messages, draftKey, pendingKey])
|
||||
|
||||
|
|
@ -266,21 +247,49 @@ export function useMobileNativeChatDrafts(args: {
|
|||
return () => {
|
||||
mountedRef.current = false
|
||||
for (const entry of unconfirmedRef.current) {
|
||||
if (entry.deadline !== null) {
|
||||
clearTimeout(entry.deadline)
|
||||
}
|
||||
clearTimeout(entry.deadline ?? undefined)
|
||||
}
|
||||
unconfirmedRef.current = []
|
||||
}
|
||||
}, [])
|
||||
|
||||
const pending = pendingKey
|
||||
? (pendingBySession[pendingKey] ?? NO_PENDING_MESSAGES)
|
||||
const waitingForSession = draftKey
|
||||
? (pendingWaitingForSession[draftKey] ?? NO_PENDING_MESSAGES)
|
||||
: NO_PENDING_MESSAGES
|
||||
useEffect(() => {
|
||||
if (!pendingKey || pending.length === 0) {
|
||||
if (!draftKey || !pendingKey || waitingForSession.length === 0) {
|
||||
return
|
||||
}
|
||||
const movedIds = new Set(waitingForSession.map((item) => item.id))
|
||||
setPendingBySession((previous) =>
|
||||
mergeWaitingSessionPending(previous, pendingKey, waitingForSession)
|
||||
)
|
||||
setPendingWaitingForSession((previous) =>
|
||||
removeWaitingSessionPending(previous, draftKey, movedIds)
|
||||
)
|
||||
}, [draftKey, pendingKey, waitingForSession])
|
||||
|
||||
const sessionPending = pendingKey
|
||||
? (pendingBySession[pendingKey] ?? NO_PENDING_MESSAGES)
|
||||
: NO_PENDING_MESSAGES
|
||||
const pending = combineMobileNativeChatPending(sessionPending, waitingForSession)
|
||||
useEffect(() => {
|
||||
if (!pendingKey) {
|
||||
return
|
||||
}
|
||||
setImagePreviewsBySession((previous) =>
|
||||
migrateImagePreviewMessageIds(previous, pendingKey, messages)
|
||||
)
|
||||
if (pending.length === 0) {
|
||||
return
|
||||
}
|
||||
const landedImagePreviews = findLandedImagePreviewEchoes(messages, pending)
|
||||
const landedImagePendingIds = new Set(landedImagePreviews.map((preview) => preview.pendingId))
|
||||
if (landedImagePreviews.length > 0) {
|
||||
setImagePreviewsBySession((previous) =>
|
||||
mergeLandedImagePreviewEchoes(previous, pendingKey, landedImagePreviews)
|
||||
)
|
||||
}
|
||||
setPendingBySession((previous) => {
|
||||
const current = previous[pendingKey] ?? []
|
||||
const landedCounts = new Map<string, number>()
|
||||
|
|
@ -297,12 +306,20 @@ export function useMobileNativeChatDrafts(args: {
|
|||
// baseline tail — text echoes are excluded so an unrelated outstanding
|
||||
// text send cannot clear it. Ordinal-vs-count stays stable when the effect
|
||||
// re-runs on the shrunken list, and ignores paginated-in history.
|
||||
const next = current.filter((item) =>
|
||||
item.text.trim() === ''
|
||||
const next = current.filter((item) => {
|
||||
if (landedImagePendingIds.has(item.id)) {
|
||||
return false
|
||||
}
|
||||
// Image echoes hand their local URIs to the authoritative message above;
|
||||
// never drop them through the text-only fallback before that handoff.
|
||||
if (item.images?.length) {
|
||||
return true
|
||||
}
|
||||
return item.text.trim() === ''
|
||||
? countImageSourceTurnsAfter(messages, item.baselineTailMessageId) <
|
||||
item.expectedOccurrence
|
||||
item.expectedOccurrence
|
||||
: (landedCounts.get(item.text.trim()) ?? 0) < item.expectedOccurrence
|
||||
)
|
||||
})
|
||||
if (next.length === current.length) {
|
||||
return previous
|
||||
}
|
||||
|
|
@ -319,6 +336,9 @@ export function useMobileNativeChatDrafts(args: {
|
|||
composerText: draftKey ? (drafts[draftKey] ?? '') : '',
|
||||
setComposerText,
|
||||
pending,
|
||||
imagePreviewsByMessageId: pendingKey
|
||||
? (imagePreviewsBySession[pendingKey] ?? NO_IMAGE_PREVIEWS)
|
||||
: NO_IMAGE_PREVIEWS,
|
||||
captureSendOrigin,
|
||||
readSeededLaunchDraft,
|
||||
readSeededLaunchDraftSeed,
|
||||
|
|
|
|||
|
|
@ -11,13 +11,13 @@ import { useMobileNativeChatImageAttachments } from './use-mobile-native-chat-im
|
|||
// Fully stub the picker so the real expo/react-native chain never loads under
|
||||
// the vitest transform (react-native ships Flow syntax rolldown can't parse).
|
||||
vi.mock('./mobile-image-source-picker', () => ({
|
||||
pickMobileImage: vi.fn(),
|
||||
pickMobileImages: vi.fn(),
|
||||
ImageLibraryPermissionError: class ImageLibraryPermissionError extends Error {}
|
||||
}))
|
||||
|
||||
import { pickMobileImage } from './mobile-image-source-picker'
|
||||
import { pickMobileImages } from './mobile-image-source-picker'
|
||||
|
||||
const pick = vi.mocked(pickMobileImage)
|
||||
const pick = vi.mocked(pickMobileImages)
|
||||
|
||||
function ok(id: string, result: unknown): RpcSuccess {
|
||||
return { id, ok: true, result, _meta: { runtimeId: 'r' } }
|
||||
|
|
@ -124,7 +124,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
}
|
||||
|
||||
it('adds an uploaded image as a chip without pasting to the terminal', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')])
|
||||
mount(
|
||||
baseArgs({
|
||||
|
|
@ -145,7 +145,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('rides pending images along on send: pastes the path, settles, then delegates the text', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -208,7 +208,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
it('leads the image paste with a clear sized to a parked multi-line launch draft', async () => {
|
||||
// A single Ctrl+U kills only the last line, so the draft's earlier lines
|
||||
// would survive the clear and ride along with the image as prompt body.
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -242,7 +242,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
it('spends one budget across the image paste and the text body that follows', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -280,7 +280,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('routes an attachments-only send through baseSend with empty text so the echo still shows the photo', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -321,7 +321,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('keeps the chips and does not submit when the image paste is rejected', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -345,7 +345,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('keeps the chips and reports failure when the paste transport throws', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
// No terminal.send responses queued: the clear write throws (dropped transport).
|
||||
const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')])
|
||||
const baseSend = vi.fn().mockResolvedValue('accepted')
|
||||
|
|
@ -365,7 +365,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('surfaces an error instead of a silent no-op when the input lease gate is closed', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')])
|
||||
const baseSend = vi.fn().mockResolvedValue('accepted')
|
||||
const onSendError = vi.fn()
|
||||
|
|
@ -387,7 +387,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('scopes chips to the tab that attached them', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([methodNotFound('start'), ok('save', '/tmp/a.png')])
|
||||
const baseSend = vi.fn().mockResolvedValue('accepted')
|
||||
const args = baseArgs({ client: client as unknown as RpcClient, baseSend })
|
||||
|
|
@ -426,7 +426,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
mount(args)
|
||||
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
let firstAttach: Promise<void> | null = null
|
||||
await act(async () => {
|
||||
firstAttach = hook!.attachImage('library')
|
||||
|
|
@ -438,7 +438,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
expect(hook!.isAttaching).toBe(true)
|
||||
|
||||
// A concurrent cancelled pick — its finally must leave the counter alone.
|
||||
pick.mockResolvedValue(null)
|
||||
pick.mockResolvedValue([])
|
||||
await act(async () => {
|
||||
await hook!.attachImage('library')
|
||||
})
|
||||
|
|
@ -454,7 +454,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('clears only the chips that were sent, keeping one attached mid-send', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'), // first attach
|
||||
|
|
@ -488,7 +488,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
expect(releaseSettle).not.toBeNull()
|
||||
|
||||
pick.mockResolvedValue({ base64: 'BBBB', uri: 'file:///b.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'BBBB', uri: 'file:///b.jpg' }])
|
||||
await act(async () => {
|
||||
await hook!.attachImage('library')
|
||||
})
|
||||
|
|
@ -511,7 +511,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('aborts the send when the active terminal changes during the settle window', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -560,7 +560,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('leads the next text-only send with Ctrl+U after a failed paste, even with the chip removed', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -596,7 +596,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('heals before the next text-only send when an image submit delivery is unknown (#10228)', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -634,7 +634,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('still heals after the session screen unmounts and remounts (#10228)', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -688,7 +688,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('retains the stale marker when a rejected healing clear blocks text-only send', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -730,7 +730,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('does not reroute text when the active terminal changes during a healing clear', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
let releaseClear: ((response: RpcResponse) => void) | null = null
|
||||
const deferredClear = new Promise<RpcResponse>((resolve) => {
|
||||
releaseClear = resolve
|
||||
|
|
@ -774,7 +774,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
|
||||
it('defers the heal instead of burning a rejected clear while the lease is closed', async () => {
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start'),
|
||||
ok('save', '/tmp/a.png'),
|
||||
|
|
@ -818,8 +818,8 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
|
||||
it('heals rejected image submits independently across terminals', async () => {
|
||||
pick
|
||||
.mockResolvedValueOnce({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
.mockResolvedValueOnce({ base64: 'BBBB', uri: 'file:///b.jpg' })
|
||||
.mockResolvedValueOnce([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
.mockResolvedValueOnce([{ base64: 'BBBB', uri: 'file:///b.jpg' }])
|
||||
const client = makeClient([
|
||||
methodNotFound('start-a'),
|
||||
ok('save-a', '/tmp/a.png'),
|
||||
|
|
@ -896,7 +896,7 @@ describe('useMobileNativeChatImageAttachments', () => {
|
|||
})
|
||||
})
|
||||
mount(args)
|
||||
pick.mockResolvedValue({ base64: 'AAAA', uri: 'file:///a.jpg' })
|
||||
pick.mockResolvedValue([{ base64: 'AAAA', uri: 'file:///a.jpg' }])
|
||||
let attach: Promise<void> | null = null
|
||||
await act(async () => {
|
||||
attach = hook!.attachImage('library')
|
||||
|
|
|
|||
|
|
@ -5,11 +5,12 @@ import type { RpcClient } from '../transport/rpc-client'
|
|||
import type { ConnectionState } from '../transport/types'
|
||||
import {
|
||||
ImageLibraryPermissionError,
|
||||
pickMobileImage,
|
||||
pickMobileImages,
|
||||
type MobileImageSource
|
||||
} from './mobile-image-source-picker'
|
||||
import {
|
||||
uploadMobileNativeChatImage,
|
||||
appendPendingNativeChatImages,
|
||||
uploadMobileNativeChatImages,
|
||||
type PendingNativeChatImage
|
||||
} from './mobile-native-chat-image-attachment'
|
||||
import {
|
||||
|
|
@ -82,10 +83,6 @@ export type MobileNativeChatImageAttachments = {
|
|||
readonly sendNativeChat: (text: string) => Promise<boolean>
|
||||
}
|
||||
|
||||
function getErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
|
||||
const NO_ATTACHMENTS: PendingNativeChatImage[] = []
|
||||
|
||||
function withScopeAttachments(
|
||||
|
|
@ -146,48 +143,53 @@ export function useMobileNativeChatImageAttachments({
|
|||
// pick or pre-upload error never ran `onUploadStart`, so decrementing the
|
||||
// shared counter would clear a concurrent upload's in-flight flag early.
|
||||
let started = false
|
||||
const uploadedImages: Omit<PendingNativeChatImage, 'id'>[] = []
|
||||
let uploadError: unknown = null
|
||||
try {
|
||||
const uploaded = await uploadMobileNativeChatImage(source, {
|
||||
await uploadMobileNativeChatImages(source, {
|
||||
client,
|
||||
getConnectionId: getActiveWorktreeConnectionId,
|
||||
pickImage: pickMobileImage,
|
||||
pickImages: pickMobileImages,
|
||||
onImageUploaded: (image) => uploadedImages.push(image),
|
||||
onUploadStart: () => {
|
||||
started = true
|
||||
attachingCount.current += 1
|
||||
setIsAttaching(true)
|
||||
}
|
||||
})
|
||||
// Cancelled picker: no error, no toast.
|
||||
if (!uploaded) {
|
||||
return
|
||||
}
|
||||
idCounter.current += 1
|
||||
const chip = { id: `img-${idCounter.current}`, ...uploaded }
|
||||
setAttachmentsByScope((prev) => ({ ...prev, [scope]: [...(prev[scope] ?? []), chip] }))
|
||||
onAttachSuccess?.()
|
||||
} catch (error) {
|
||||
uploadError = error
|
||||
} finally {
|
||||
if (started) {
|
||||
attachingCount.current -= 1
|
||||
if (attachingCount.current === 0) {
|
||||
setIsAttaching(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (uploadedImages.length > 0) {
|
||||
setAttachmentsByScope((prev) => ({
|
||||
...prev,
|
||||
[scope]: appendPendingNativeChatImages(prev[scope] ?? [], uploadedImages, idCounter)
|
||||
}))
|
||||
onAttachSuccess?.()
|
||||
}
|
||||
if (uploadError !== null) {
|
||||
const message = uploadError instanceof Error ? uploadError.message : String(uploadError)
|
||||
onError?.()
|
||||
if (connStateRef.current !== 'connected') {
|
||||
showToast('Attach failed (disconnected)', 1500)
|
||||
return
|
||||
}
|
||||
if (error instanceof ImageLibraryPermissionError) {
|
||||
if (uploadError instanceof ImageLibraryPermissionError) {
|
||||
showToast('Photo permission denied', 1500)
|
||||
return
|
||||
}
|
||||
if (getErrorMessage(error) === CLIPBOARD_IMAGE_TOO_LARGE_ERROR) {
|
||||
if (message === CLIPBOARD_IMAGE_TOO_LARGE_ERROR) {
|
||||
showToast('Image too large to attach', 1500)
|
||||
return
|
||||
}
|
||||
showToast('Attach failed', 1500)
|
||||
} finally {
|
||||
if (started) {
|
||||
attachingCount.current -= 1
|
||||
if (attachingCount.current <= 0) {
|
||||
attachingCount.current = 0
|
||||
setIsAttaching(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
[
|
||||
|
|
|
|||
|
|
@ -35,6 +35,52 @@ describe('normalizeImageTranscriptMessages', () => {
|
|||
])
|
||||
})
|
||||
|
||||
it('folds every source and strips every prompt marker for a multi-image send', () => {
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('a', '[Image: source: /tmp/a.png]'),
|
||||
userText('b', '[Image: source: /tmp/b.png]'),
|
||||
userText('c', '[Image: source: /tmp/c.png]'),
|
||||
userText('prompt', '[Image #1] [Image #2] [Image #3] compare these')
|
||||
])
|
||||
|
||||
expect(out).toHaveLength(1)
|
||||
expect(out[0]).toMatchObject({ id: 'prompt' })
|
||||
expect(out[0]!.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'image-ref', path: '/tmp/b.png' },
|
||||
{ type: 'image-ref', path: '/tmp/c.png' },
|
||||
{ type: 'text', text: 'compare these' }
|
||||
])
|
||||
})
|
||||
|
||||
it('keeps all image refs when a multi-image send has no caption', () => {
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('a', '[Image: source: /tmp/a.png]'),
|
||||
userText('b', '[Image: source: /tmp/b.png]'),
|
||||
userText('prompt', '[Image #1] [Image #2]')
|
||||
])
|
||||
|
||||
expect(out).toHaveLength(1)
|
||||
expect(out[0]!.blocks).toEqual([
|
||||
{ type: 'image-ref', path: '/tmp/a.png' },
|
||||
{ type: 'image-ref', path: '/tmp/b.png' }
|
||||
])
|
||||
})
|
||||
|
||||
it('preserves adjacent standalone image turns without a prompt marker', () => {
|
||||
const out = normalizeImageTranscriptMessages([
|
||||
userText('a', '[Image: source: /tmp/a.png]'),
|
||||
userText('b', '[Image: source: /tmp/b.png]')
|
||||
])
|
||||
|
||||
expect(out).toHaveLength(2)
|
||||
expect(out.map((message) => message.id)).toEqual(['a', 'b'])
|
||||
expect(out.map((message) => message.blocks)).toEqual([
|
||||
[{ type: 'image-ref', path: '/tmp/a.png' }],
|
||||
[{ type: 'image-ref', path: '/tmp/b.png' }]
|
||||
])
|
||||
})
|
||||
|
||||
it('leaves ordinary user text untouched', () => {
|
||||
const out = normalizeImageTranscriptMessages([userText('a', 'how about this')])
|
||||
expect(out[0]!.blocks).toEqual([{ type: 'text', text: 'how about this' }])
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { isTextBlock, type NativeChatBlock, type NativeChatMessage } from './native-chat-types'
|
||||
|
||||
const IMAGE_SOURCE_MARKER = /^\[Image:\s*source:\s*(.+?)\]\s*$/
|
||||
const IMAGE_PROMPT_MARKER = /^\[Image #\d+\]\s*/
|
||||
const IMAGE_PROMPT_MARKERS = /^(?:\[Image #\d+\]\s*)+/
|
||||
|
||||
function soleText(message: NativeChatMessage): string | null {
|
||||
return message.blocks.length === 1 && isTextBlock(message.blocks[0])
|
||||
|
|
@ -14,10 +14,12 @@ export function imageSourcePathFromText(text: string): string | null {
|
|||
}
|
||||
|
||||
export function stripImagePromptMarker(text: string): string {
|
||||
return text.replace(IMAGE_PROMPT_MARKER, '')
|
||||
return text.replace(IMAGE_PROMPT_MARKERS, '')
|
||||
}
|
||||
|
||||
function stripFirstImagePromptMarker(blocks: readonly NativeChatBlock[]): NativeChatBlock[] {
|
||||
function stripImagePromptMarkersFromFirstText(
|
||||
blocks: readonly NativeChatBlock[]
|
||||
): NativeChatBlock[] {
|
||||
let stripped = false
|
||||
const next: NativeChatBlock[] = []
|
||||
for (const block of blocks) {
|
||||
|
|
@ -36,13 +38,11 @@ function stripFirstImagePromptMarker(blocks: readonly NativeChatBlock[]): Native
|
|||
|
||||
function imagePromptMarkerStartsMessage(message: NativeChatMessage): boolean {
|
||||
const firstText = message.blocks.find(isTextBlock)
|
||||
return firstText ? IMAGE_PROMPT_MARKER.test(firstText.text) : false
|
||||
return firstText ? IMAGE_PROMPT_MARKERS.test(firstText.text) : false
|
||||
}
|
||||
|
||||
/** Claude records an attached image as two user transcript turns:
|
||||
* `[Image: source: /path]` and then `[Image #1] prompt`. Merge them back into
|
||||
* one native turn so the UI keeps the same chip+text shape as the optimistic
|
||||
* send and does not show raw TUI marker text after a view remount. */
|
||||
/** Claude records image paths as source turns followed by one marker-prefixed
|
||||
* prompt. Merge the whole run back into one native user turn. */
|
||||
export function normalizeImageTranscriptMessages(
|
||||
messages: readonly NativeChatMessage[]
|
||||
): NativeChatMessage[] {
|
||||
|
|
@ -54,27 +54,34 @@ export function normalizeImageTranscriptMessages(
|
|||
continue
|
||||
}
|
||||
const imagePath = imageSourcePathFromText(soleText(message) ?? '')
|
||||
const next = messages[index + 1]
|
||||
if (
|
||||
imagePath &&
|
||||
next?.role === 'user' &&
|
||||
next.source === message.source &&
|
||||
imagePromptMarkerStartsMessage(next)
|
||||
) {
|
||||
normalized.push({
|
||||
...next,
|
||||
blocks: [
|
||||
{ type: 'image-ref', path: imagePath },
|
||||
...stripFirstImagePromptMarker(next.blocks)
|
||||
]
|
||||
})
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
// A lone `[Image: source: /path]` turn (no following `[Image #1]` prompt —
|
||||
// e.g. an image sent with no caption) still renders as an image chip rather
|
||||
// than the raw marker text.
|
||||
if (imagePath) {
|
||||
const imagePaths = [imagePath]
|
||||
let nextIndex = index + 1
|
||||
while (nextIndex < messages.length) {
|
||||
const candidate = messages[nextIndex]!
|
||||
const candidatePath = imageSourcePathFromText(soleText(candidate) ?? '')
|
||||
if (candidate.role !== 'user' || candidate.source !== message.source || !candidatePath) {
|
||||
break
|
||||
}
|
||||
imagePaths.push(candidatePath)
|
||||
nextIndex += 1
|
||||
}
|
||||
const prompt = messages[nextIndex]
|
||||
if (
|
||||
prompt?.role === 'user' &&
|
||||
prompt.source === message.source &&
|
||||
imagePromptMarkerStartsMessage(prompt)
|
||||
) {
|
||||
normalized.push({
|
||||
...prompt,
|
||||
blocks: [
|
||||
...imagePaths.map((path) => ({ type: 'image-ref' as const, path })),
|
||||
...stripImagePromptMarkersFromFirstText(prompt.blocks)
|
||||
]
|
||||
})
|
||||
index = nextIndex
|
||||
continue
|
||||
}
|
||||
normalized.push({
|
||||
...message,
|
||||
blocks: [{ type: 'image-ref', path: imagePath }]
|
||||
|
|
@ -83,7 +90,7 @@ export function normalizeImageTranscriptMessages(
|
|||
}
|
||||
normalized.push({
|
||||
...message,
|
||||
blocks: stripFirstImagePromptMarker(message.blocks)
|
||||
blocks: stripImagePromptMarkersFromFirstText(message.blocks)
|
||||
})
|
||||
}
|
||||
return normalized
|
||||
|
|
|
|||
Loading…
Reference in New Issue