fix(mobile): downscale oversized clipboard images so paste no longer fails (#5601)
* fix(mobile): downscale oversized clipboard images so paste no longer fails Pasting a large clipboard image (high-res screenshot or photo) failed with 'Image too large to paste' because the re-encoded PNG exceeded the 24 MiB base64 upload cap, with nothing shrinking it first. Now an oversized image is downscaled to fit the budget before upload: - computeMobileClipboardImageDownscale() picks target dimensions by area (~sqrt(budget/actual)), with a bounded retry loop since PNG size is nonlinear. - prepareMobileClipboardImageBase64() drives the loop with an injected resizer, so the byte/dimension logic is unit-tested without native modules. - The resizer stages the image to a temp file and hands ImageManipulator a file:// URI; the iOS native loader (Data(contentsOf:)) cannot decode large base64 data URIs, so a data URI made renderAsync throw. Adds expo-image-manipulator and expo-file-system. * fix(mobile): fail fast when resized clipboard image has no base64 Empty base64 from saveAsync would pass the downstream base64 check and upload a corrupt image; throw instead so the paste surfaces an error. Addresses CodeRabbit review on #5601. * Fix mobile clipboard image resize cleanup Co-authored-by: Orca <help@stably.ai> --------- Co-authored-by: Jinwoo-H <jinwoo0825@gmail.com> Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Carlos Baraza
GitHub
parent
18bdef9ed0
commit
f534cd5bc3
|
|
@ -1,6 +1,8 @@
|
|||
import { useState, useEffect, useRef, useCallback, useMemo } from 'react'
|
||||
import { Animated, AppState, Linking, type AppStateStatus } from 'react-native'
|
||||
import * as Clipboard from 'expo-clipboard'
|
||||
import { ImageManipulator, SaveFormat } from 'expo-image-manipulator'
|
||||
import { File as FsFile, Paths } from 'expo-file-system'
|
||||
import {
|
||||
BackHandler,
|
||||
FlatList,
|
||||
|
|
@ -146,7 +148,9 @@ import {
|
|||
} from '../../../../src/session/mobile-new-tab-agent-options'
|
||||
import {
|
||||
buildMobileImagePastePayload,
|
||||
saveMobileClipboardImageAsTempFile
|
||||
prepareMobileClipboardImageBase64,
|
||||
saveMobileClipboardImageAsTempFile,
|
||||
type MobileClipboardImageResizer
|
||||
} from '../../../../src/session/mobile-clipboard-image'
|
||||
import { useMobileImageAttachment } from '../../../../src/session/use-mobile-image-attachment'
|
||||
import { classifyMobileArtifact } from '../../../../src/session/mobile-artifact-kind'
|
||||
|
|
@ -204,6 +208,52 @@ import type {
|
|||
TerminalGestureInputQueue
|
||||
} from './mobile-session-route-types'
|
||||
|
||||
const CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i
|
||||
|
||||
// Why: clipboard images are re-encoded as lossless PNG, so high-res screenshots and
|
||||
// photos can exceed the upload byte budget; resize the raster down to fit before upload.
|
||||
// The image is staged to a temp file first because the iOS ImageManipulator loader
|
||||
// (Data(contentsOf:)) cannot decode large base64 data URIs — it needs a file:// URI.
|
||||
const resizeMobileClipboardImage: MobileClipboardImageResizer = async (source, target) => {
|
||||
const base64 = source.replace(CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE, '')
|
||||
const file = new FsFile(Paths.cache, `orca-clip-resize-${Date.now()}.png`)
|
||||
let context: ReturnType<typeof ImageManipulator.manipulate> | null = null
|
||||
let rendered: Awaited<
|
||||
ReturnType<ReturnType<typeof ImageManipulator.manipulate>['renderAsync']>
|
||||
> | null = null
|
||||
let resultUri: string | null = null
|
||||
try {
|
||||
file.create({ overwrite: true })
|
||||
file.write(base64, { encoding: 'base64' })
|
||||
context = ImageManipulator.manipulate(file.uri)
|
||||
context.resize({ width: target.width, height: target.height })
|
||||
rendered = await context.renderAsync()
|
||||
const result = await rendered.saveAsync({ format: SaveFormat.PNG, base64: true })
|
||||
resultUri = result.uri
|
||||
// Why: empty base64 would pass the downstream base64 check and upload a corrupt
|
||||
// image, so fail loudly here instead of silently sending an invalid payload.
|
||||
if (!result.base64) {
|
||||
throw new Error('Failed to encode resized clipboard image')
|
||||
}
|
||||
return { data: result.base64, width: result.width, height: result.height }
|
||||
} finally {
|
||||
rendered?.release()
|
||||
context?.release()
|
||||
if (resultUri) {
|
||||
try {
|
||||
new FsFile(resultUri).delete()
|
||||
} catch {
|
||||
// Best-effort cleanup; ImageManipulator saves into cache for every retry.
|
||||
}
|
||||
}
|
||||
try {
|
||||
file.delete()
|
||||
} catch {
|
||||
// Best-effort cleanup; the OS reclaims the cache directory regardless.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getActiveTabIdForHandle(
|
||||
tabs: MobileSessionTab[],
|
||||
terminalHandle: string | null
|
||||
|
|
@ -3474,7 +3524,8 @@ export default function SessionScreen() {
|
|||
return
|
||||
}
|
||||
const connectionId = await getActiveWorktreeConnectionId()
|
||||
const imagePath = await saveMobileClipboardImageAsTempFile(client, image.data, {
|
||||
const base64 = await prepareMobileClipboardImageBase64(image, resizeMobileClipboardImage)
|
||||
const imagePath = await saveMobileClipboardImageAsTempFile(client, base64, {
|
||||
connectionId
|
||||
})
|
||||
payload = buildMobileImagePastePayload(imagePath)
|
||||
|
|
|
|||
|
|
@ -26,7 +26,9 @@
|
|||
"expo-crypto": "^55.0.14",
|
||||
"expo-dev-client": "~55.0.35",
|
||||
"expo-document-picker": "^55.0.13",
|
||||
"expo-file-system": "55.0.19",
|
||||
"expo-haptics": "^55.0.14",
|
||||
"expo-image-manipulator": "^55.0.17",
|
||||
"expo-image-picker": "^55.0.20",
|
||||
"expo-linking": "^55.0.15",
|
||||
"expo-modules-core": "~55.0.25",
|
||||
|
|
|
|||
|
|
@ -41,9 +41,15 @@ importers:
|
|||
expo-document-picker:
|
||||
specifier: ^55.0.13
|
||||
version: 55.0.13(expo@55.0.23)
|
||||
expo-file-system:
|
||||
specifier: 55.0.19
|
||||
version: 55.0.19(expo@55.0.23)(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))
|
||||
expo-haptics:
|
||||
specifier: ^55.0.14
|
||||
version: 55.0.14(expo@55.0.23)
|
||||
expo-image-manipulator:
|
||||
specifier: ^55.0.17
|
||||
version: 55.0.17(expo@55.0.23)
|
||||
expo-image-picker:
|
||||
specifier: ^55.0.20
|
||||
version: 55.0.20(expo@55.0.23)
|
||||
|
|
@ -3378,6 +3384,11 @@ packages:
|
|||
peerDependencies:
|
||||
expo: '*'
|
||||
|
||||
expo-image-manipulator@55.0.17:
|
||||
resolution: {integrity: sha512-lTYhmXejHnMp+vjFID0Q/jD+Qic3U9OP4qw3tCVHCVp4TTmx9hbifTIN0hmw02ak7LsBkxuYnP+uglvUKgJAzA==}
|
||||
peerDependencies:
|
||||
expo: '*'
|
||||
|
||||
expo-image-picker@55.0.20:
|
||||
resolution: {integrity: sha512-lfWt/0rPWdKz8AdDEGmGHZIJSNlVc720Dlx5bfou10FU16ZV5wAbTU63nm2jkXd8hbXke4a/2Ha1dzxCVA+LQQ==}
|
||||
peerDependencies:
|
||||
|
|
@ -9960,6 +9971,11 @@ snapshots:
|
|||
dependencies:
|
||||
expo: 55.0.23(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.10)(expo-router@55.0.14)(react-dom@19.2.6(react@19.2.6))(react-native-webview@13.16.1(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native-worklets@0.8.3(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6)(typescript@5.9.3)
|
||||
|
||||
expo-image-manipulator@55.0.17(expo@55.0.23):
|
||||
dependencies:
|
||||
expo: 55.0.23(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.10)(expo-router@55.0.14)(react-dom@19.2.6(react@19.2.6))(react-native-webview@13.16.1(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native-worklets@0.8.3(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6)(typescript@5.9.3)
|
||||
expo-image-loader: 55.0.1(expo@55.0.23)
|
||||
|
||||
expo-image-picker@55.0.20(expo@55.0.23):
|
||||
dependencies:
|
||||
expo: 55.0.23(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.10)(expo-router@55.0.14)(react-dom@19.2.6(react@19.2.6))(react-native-webview@13.16.1(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native-worklets@0.8.3(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6)(typescript@5.9.3)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
buildMobileImagePastePayload,
|
||||
computeMobileClipboardImageDownscale,
|
||||
MOBILE_CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS,
|
||||
normalizeMobileClipboardImageBase64,
|
||||
prepareMobileClipboardImageBase64,
|
||||
saveMobileClipboardImageAsTempFile
|
||||
} from './mobile-clipboard-image'
|
||||
import type { RpcClient } from '../transport/rpc-client'
|
||||
|
|
@ -127,3 +129,62 @@ describe('mobile clipboard image paste helpers', () => {
|
|||
expect(buildMobileImagePastePayload('/tmp/\x1b.png')).toBe('\x1b[200~/tmp/\u241b.png\x1b[201~')
|
||||
})
|
||||
})
|
||||
|
||||
describe('mobile clipboard image downscaling', () => {
|
||||
it('does not downscale images already within the byte budget', () => {
|
||||
expect(computeMobileClipboardImageDownscale(50, 100, 100, 100)).toBeNull()
|
||||
})
|
||||
|
||||
it('shrinks both edges by ~sqrt(budget/actual) when over the budget', () => {
|
||||
// 400 base64 chars vs 100 budget -> scale sqrt(0.25) * 0.85 safety = 0.425
|
||||
// 40 * 0.425 = 17, 20 * 0.425 = 8.5 -> floor 8
|
||||
expect(computeMobileClipboardImageDownscale(400, 40, 20, 100)).toEqual({ width: 17, height: 8 })
|
||||
})
|
||||
|
||||
it('refuses to downscale when source dimensions are unusable', () => {
|
||||
expect(computeMobileClipboardImageDownscale(400, 0, 20, 100)).toBeNull()
|
||||
expect(computeMobileClipboardImageDownscale(400, 40, -1, 100)).toBeNull()
|
||||
})
|
||||
|
||||
it('returns the original base64 untouched when within budget', async () => {
|
||||
const resize = vi.fn()
|
||||
const data = `data:image/png;base64,${'a'.repeat(40)}`
|
||||
await expect(
|
||||
prepareMobileClipboardImageBase64({ data, size: { width: 10, height: 10 } }, resize, 100)
|
||||
).resolves.toBe(data)
|
||||
expect(resize).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('downscales oversized images in one pass when the result fits', async () => {
|
||||
const resize = vi.fn(async () => ({ data: 'b'.repeat(50), width: 17, height: 8 }))
|
||||
const data = `data:image/png;base64,${'a'.repeat(400)}`
|
||||
await expect(
|
||||
prepareMobileClipboardImageBase64({ data, size: { width: 40, height: 20 } }, resize, 100)
|
||||
).resolves.toBe('b'.repeat(50))
|
||||
expect(resize).toHaveBeenCalledTimes(1)
|
||||
expect(resize).toHaveBeenCalledWith(data, { width: 17, height: 8 })
|
||||
})
|
||||
|
||||
it('retries downscaling, feeding back result dimensions, until it fits', async () => {
|
||||
const resize = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ data: 'b'.repeat(150), width: 17, height: 8 })
|
||||
.mockResolvedValueOnce({ data: 'c'.repeat(40), width: 7, height: 3 })
|
||||
const data = `data:image/png;base64,${'a'.repeat(400)}`
|
||||
await expect(
|
||||
prepareMobileClipboardImageBase64({ data, size: { width: 40, height: 20 } }, resize, 100)
|
||||
).resolves.toBe('c'.repeat(40))
|
||||
expect(resize).toHaveBeenCalledTimes(2)
|
||||
expect(resize.mock.calls[1][0]).toBe('b'.repeat(150))
|
||||
expect(resize.mock.calls[1][1]).toEqual({ width: 11, height: 5 })
|
||||
})
|
||||
|
||||
it('gives up after bounded attempts and lets the downstream cap reject it', async () => {
|
||||
const resize = vi.fn(async () => ({ data: 'b'.repeat(150), width: 5, height: 5 }))
|
||||
const data = `data:image/png;base64,${'a'.repeat(400)}`
|
||||
await expect(
|
||||
prepareMobileClipboardImageBase64({ data, size: { width: 40, height: 20 } }, resize, 100)
|
||||
).resolves.toBe('b'.repeat(150))
|
||||
expect(resize).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -4,6 +4,10 @@ import type { RpcFailure, RpcSuccess } from '../transport/types'
|
|||
export const MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS = 24 * 1024 * 1024
|
||||
export const MOBILE_CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS = 512 * 1024
|
||||
export const MOBILE_CLIPBOARD_IMAGE_SINGLE_FRAME_FALLBACK_BASE64_CHARS = 256 * 1024
|
||||
// Why: PNG bytes don't scale exactly with pixel area, so undershoot the target on
|
||||
// each pass and let the bounded retry below converge instead of distorting in one shot.
|
||||
const MOBILE_CLIPBOARD_IMAGE_DOWNSCALE_SAFETY = 0.85
|
||||
const MOBILE_CLIPBOARD_IMAGE_MAX_DOWNSCALE_ATTEMPTS = 3
|
||||
|
||||
const DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i
|
||||
const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/
|
||||
|
|
@ -19,6 +23,75 @@ export function normalizeMobileClipboardImageBase64(data: string): string {
|
|||
return contentBase64
|
||||
}
|
||||
|
||||
export type MobileClipboardImage = {
|
||||
data: string
|
||||
size: { width: number; height: number }
|
||||
}
|
||||
|
||||
export type MobileClipboardImageResizer = (
|
||||
source: string,
|
||||
target: { width: number; height: number }
|
||||
) => Promise<{ data: string; width: number; height: number }>
|
||||
|
||||
/**
|
||||
* Returns the pixel dimensions to resize a clipboard image to so its base64 fits
|
||||
* the upload budget, or null when it already fits (or its dimensions are unusable).
|
||||
*/
|
||||
export function computeMobileClipboardImageDownscale(
|
||||
base64Length: number,
|
||||
width: number,
|
||||
height: number,
|
||||
maxBase64Length: number
|
||||
): { width: number; height: number } | null {
|
||||
if (base64Length <= maxBase64Length) {
|
||||
return null
|
||||
}
|
||||
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
|
||||
return null
|
||||
}
|
||||
const scale = Math.sqrt(maxBase64Length / base64Length) * MOBILE_CLIPBOARD_IMAGE_DOWNSCALE_SAFETY
|
||||
const nextWidth = Math.max(1, Math.floor(width * scale))
|
||||
const nextHeight = Math.max(1, Math.floor(height * scale))
|
||||
// Guard against a no-op shrink (already 1px) so the retry loop can't spin forever.
|
||||
if (nextWidth >= width && nextHeight >= height) {
|
||||
return null
|
||||
}
|
||||
return { width: nextWidth, height: nextHeight }
|
||||
}
|
||||
|
||||
/**
|
||||
* Downscales an oversized clipboard image until its base64 fits the upload budget,
|
||||
* delegating the actual raster resize to the injected `resize`. Returns the
|
||||
* upload-ready base64; if it still overflows after the bounded retries the
|
||||
* downstream size check rejects it with the same "too large" error as before.
|
||||
*/
|
||||
export async function prepareMobileClipboardImageBase64(
|
||||
image: MobileClipboardImage,
|
||||
resize: MobileClipboardImageResizer,
|
||||
maxBase64Length: number = MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS
|
||||
): Promise<string> {
|
||||
let data = image.data
|
||||
let width = image.size.width
|
||||
let height = image.size.height
|
||||
for (let attempt = 0; attempt < MOBILE_CLIPBOARD_IMAGE_MAX_DOWNSCALE_ATTEMPTS; attempt += 1) {
|
||||
const contentLength = data.replace(DATA_URL_PREFIX_RE, '').length
|
||||
const target = computeMobileClipboardImageDownscale(
|
||||
contentLength,
|
||||
width,
|
||||
height,
|
||||
maxBase64Length
|
||||
)
|
||||
if (!target) {
|
||||
return data
|
||||
}
|
||||
const resized = await resize(data, target)
|
||||
data = resized.data
|
||||
width = resized.width
|
||||
height = resized.height
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
function assertSuccess<T>(response: RpcSuccess | RpcFailure): T {
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
|
|
|
|||
Loading…
Reference in New Issue