From f534cd5bc3a9f35eb4cade2bc19a2f26c513e565 Mon Sep 17 00:00:00 2001 From: Carlos Baraza Date: Fri, 19 Jun 2026 03:25:06 +0200 Subject: [PATCH] fix(mobile): downscale oversized clipboard images so paste no longer fails (#5601) * fix(mobile): downscale oversized clipboard images so paste no longer fails Pasting a large clipboard image (high-res screenshot or photo) failed with 'Image too large to paste' because the re-encoded PNG exceeded the 24 MiB base64 upload cap, with nothing shrinking it first. Now an oversized image is downscaled to fit the budget before upload: - computeMobileClipboardImageDownscale() picks target dimensions by area (~sqrt(budget/actual)), with a bounded retry loop since PNG size is nonlinear. - prepareMobileClipboardImageBase64() drives the loop with an injected resizer, so the byte/dimension logic is unit-tested without native modules. - The resizer stages the image to a temp file and hands ImageManipulator a file:// URI; the iOS native loader (Data(contentsOf:)) cannot decode large base64 data URIs, so a data URI made renderAsync throw. Adds expo-image-manipulator and expo-file-system. * fix(mobile): fail fast when resized clipboard image has no base64 Empty base64 from saveAsync would pass the downstream base64 check and upload a corrupt image; throw instead so the paste surfaces an error. Addresses CodeRabbit review on #5601. * Fix mobile clipboard image resize cleanup Co-authored-by: Orca --------- Co-authored-by: Jinwoo-H Co-authored-by: Orca --- .../app/h/[hostId]/session/[worktreeId].tsx | 55 +++++++++++++- mobile/package.json | 2 + mobile/pnpm-lock.yaml | 16 ++++ .../session/mobile-clipboard-image.test.ts | 61 ++++++++++++++++ mobile/src/session/mobile-clipboard-image.ts | 73 +++++++++++++++++++ 5 files changed, 205 insertions(+), 2 deletions(-) diff --git a/mobile/app/h/[hostId]/session/[worktreeId].tsx b/mobile/app/h/[hostId]/session/[worktreeId].tsx index d2fe6201f..7add5207c 100644 --- a/mobile/app/h/[hostId]/session/[worktreeId].tsx +++ b/mobile/app/h/[hostId]/session/[worktreeId].tsx @@ -1,6 +1,8 @@ import { useState, useEffect, useRef, useCallback, useMemo } from 'react' import { Animated, AppState, Linking, type AppStateStatus } from 'react-native' import * as Clipboard from 'expo-clipboard' +import { ImageManipulator, SaveFormat } from 'expo-image-manipulator' +import { File as FsFile, Paths } from 'expo-file-system' import { BackHandler, FlatList, @@ -146,7 +148,9 @@ import { } from '../../../../src/session/mobile-new-tab-agent-options' import { buildMobileImagePastePayload, - saveMobileClipboardImageAsTempFile + prepareMobileClipboardImageBase64, + saveMobileClipboardImageAsTempFile, + type MobileClipboardImageResizer } from '../../../../src/session/mobile-clipboard-image' import { useMobileImageAttachment } from '../../../../src/session/use-mobile-image-attachment' import { classifyMobileArtifact } from '../../../../src/session/mobile-artifact-kind' @@ -204,6 +208,52 @@ import type { TerminalGestureInputQueue } from './mobile-session-route-types' +const CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i + +// Why: clipboard images are re-encoded as lossless PNG, so high-res screenshots and +// photos can exceed the upload byte budget; resize the raster down to fit before upload. +// The image is staged to a temp file first because the iOS ImageManipulator loader +// (Data(contentsOf:)) cannot decode large base64 data URIs — it needs a file:// URI. +const resizeMobileClipboardImage: MobileClipboardImageResizer = async (source, target) => { + const base64 = source.replace(CLIPBOARD_IMAGE_DATA_URL_PREFIX_RE, '') + const file = new FsFile(Paths.cache, `orca-clip-resize-${Date.now()}.png`) + let context: ReturnType | null = null + let rendered: Awaited< + ReturnType['renderAsync']> + > | null = null + let resultUri: string | null = null + try { + file.create({ overwrite: true }) + file.write(base64, { encoding: 'base64' }) + context = ImageManipulator.manipulate(file.uri) + context.resize({ width: target.width, height: target.height }) + rendered = await context.renderAsync() + const result = await rendered.saveAsync({ format: SaveFormat.PNG, base64: true }) + resultUri = result.uri + // Why: empty base64 would pass the downstream base64 check and upload a corrupt + // image, so fail loudly here instead of silently sending an invalid payload. + if (!result.base64) { + throw new Error('Failed to encode resized clipboard image') + } + return { data: result.base64, width: result.width, height: result.height } + } finally { + rendered?.release() + context?.release() + if (resultUri) { + try { + new FsFile(resultUri).delete() + } catch { + // Best-effort cleanup; ImageManipulator saves into cache for every retry. + } + } + try { + file.delete() + } catch { + // Best-effort cleanup; the OS reclaims the cache directory regardless. + } + } +} + function getActiveTabIdForHandle( tabs: MobileSessionTab[], terminalHandle: string | null @@ -3474,7 +3524,8 @@ export default function SessionScreen() { return } const connectionId = await getActiveWorktreeConnectionId() - const imagePath = await saveMobileClipboardImageAsTempFile(client, image.data, { + const base64 = await prepareMobileClipboardImageBase64(image, resizeMobileClipboardImage) + const imagePath = await saveMobileClipboardImageAsTempFile(client, base64, { connectionId }) payload = buildMobileImagePastePayload(imagePath) diff --git a/mobile/package.json b/mobile/package.json index 643749e94..81587560f 100644 --- a/mobile/package.json +++ b/mobile/package.json @@ -26,7 +26,9 @@ "expo-crypto": "^55.0.14", "expo-dev-client": "~55.0.35", "expo-document-picker": "^55.0.13", + "expo-file-system": "55.0.19", "expo-haptics": "^55.0.14", + "expo-image-manipulator": "^55.0.17", "expo-image-picker": "^55.0.20", "expo-linking": "^55.0.15", "expo-modules-core": "~55.0.25", diff --git a/mobile/pnpm-lock.yaml b/mobile/pnpm-lock.yaml index 68e5ea7c2..a3c258727 100644 --- a/mobile/pnpm-lock.yaml +++ b/mobile/pnpm-lock.yaml @@ -41,9 +41,15 @@ importers: expo-document-picker: specifier: ^55.0.13 version: 55.0.13(expo@55.0.23) + expo-file-system: + specifier: 55.0.19 + version: 55.0.19(expo@55.0.23)(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6)) expo-haptics: specifier: ^55.0.14 version: 55.0.14(expo@55.0.23) + expo-image-manipulator: + specifier: ^55.0.17 + version: 55.0.17(expo@55.0.23) expo-image-picker: specifier: ^55.0.20 version: 55.0.20(expo@55.0.23) @@ -3378,6 +3384,11 @@ packages: peerDependencies: expo: '*' + expo-image-manipulator@55.0.17: + resolution: {integrity: sha512-lTYhmXejHnMp+vjFID0Q/jD+Qic3U9OP4qw3tCVHCVp4TTmx9hbifTIN0hmw02ak7LsBkxuYnP+uglvUKgJAzA==} + peerDependencies: + expo: '*' + expo-image-picker@55.0.20: resolution: {integrity: sha512-lfWt/0rPWdKz8AdDEGmGHZIJSNlVc720Dlx5bfou10FU16ZV5wAbTU63nm2jkXd8hbXke4a/2Ha1dzxCVA+LQQ==} peerDependencies: @@ -9960,6 +9971,11 @@ snapshots: dependencies: expo: 55.0.23(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.10)(expo-router@55.0.14)(react-dom@19.2.6(react@19.2.6))(react-native-webview@13.16.1(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native-worklets@0.8.3(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6)(typescript@5.9.3) + expo-image-manipulator@55.0.17(expo@55.0.23): + dependencies: + expo: 55.0.23(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.10)(expo-router@55.0.14)(react-dom@19.2.6(react@19.2.6))(react-native-webview@13.16.1(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native-worklets@0.8.3(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6)(typescript@5.9.3) + expo-image-loader: 55.0.1(expo@55.0.23) + expo-image-picker@55.0.20(expo@55.0.23): dependencies: expo: 55.0.23(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.10)(expo-router@55.0.14)(react-dom@19.2.6(react@19.2.6))(react-native-webview@13.16.1(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native-worklets@0.8.3(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6))(react-native@0.83.9(@babel/core@7.29.0)(@react-native/metro-config@0.85.2(@babel/core@7.29.0))(@types/react@19.2.14)(react@19.2.6))(react@19.2.6)(typescript@5.9.3) diff --git a/mobile/src/session/mobile-clipboard-image.test.ts b/mobile/src/session/mobile-clipboard-image.test.ts index b7a7edb69..c0cc283fe 100644 --- a/mobile/src/session/mobile-clipboard-image.test.ts +++ b/mobile/src/session/mobile-clipboard-image.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it, vi } from 'vitest' import { buildMobileImagePastePayload, + computeMobileClipboardImageDownscale, MOBILE_CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS, normalizeMobileClipboardImageBase64, + prepareMobileClipboardImageBase64, saveMobileClipboardImageAsTempFile } from './mobile-clipboard-image' import type { RpcClient } from '../transport/rpc-client' @@ -127,3 +129,62 @@ describe('mobile clipboard image paste helpers', () => { expect(buildMobileImagePastePayload('/tmp/\x1b.png')).toBe('\x1b[200~/tmp/\u241b.png\x1b[201~') }) }) + +describe('mobile clipboard image downscaling', () => { + it('does not downscale images already within the byte budget', () => { + expect(computeMobileClipboardImageDownscale(50, 100, 100, 100)).toBeNull() + }) + + it('shrinks both edges by ~sqrt(budget/actual) when over the budget', () => { + // 400 base64 chars vs 100 budget -> scale sqrt(0.25) * 0.85 safety = 0.425 + // 40 * 0.425 = 17, 20 * 0.425 = 8.5 -> floor 8 + expect(computeMobileClipboardImageDownscale(400, 40, 20, 100)).toEqual({ width: 17, height: 8 }) + }) + + it('refuses to downscale when source dimensions are unusable', () => { + expect(computeMobileClipboardImageDownscale(400, 0, 20, 100)).toBeNull() + expect(computeMobileClipboardImageDownscale(400, 40, -1, 100)).toBeNull() + }) + + it('returns the original base64 untouched when within budget', async () => { + const resize = vi.fn() + const data = `data:image/png;base64,${'a'.repeat(40)}` + await expect( + prepareMobileClipboardImageBase64({ data, size: { width: 10, height: 10 } }, resize, 100) + ).resolves.toBe(data) + expect(resize).not.toHaveBeenCalled() + }) + + it('downscales oversized images in one pass when the result fits', async () => { + const resize = vi.fn(async () => ({ data: 'b'.repeat(50), width: 17, height: 8 })) + const data = `data:image/png;base64,${'a'.repeat(400)}` + await expect( + prepareMobileClipboardImageBase64({ data, size: { width: 40, height: 20 } }, resize, 100) + ).resolves.toBe('b'.repeat(50)) + expect(resize).toHaveBeenCalledTimes(1) + expect(resize).toHaveBeenCalledWith(data, { width: 17, height: 8 }) + }) + + it('retries downscaling, feeding back result dimensions, until it fits', async () => { + const resize = vi + .fn() + .mockResolvedValueOnce({ data: 'b'.repeat(150), width: 17, height: 8 }) + .mockResolvedValueOnce({ data: 'c'.repeat(40), width: 7, height: 3 }) + const data = `data:image/png;base64,${'a'.repeat(400)}` + await expect( + prepareMobileClipboardImageBase64({ data, size: { width: 40, height: 20 } }, resize, 100) + ).resolves.toBe('c'.repeat(40)) + expect(resize).toHaveBeenCalledTimes(2) + expect(resize.mock.calls[1][0]).toBe('b'.repeat(150)) + expect(resize.mock.calls[1][1]).toEqual({ width: 11, height: 5 }) + }) + + it('gives up after bounded attempts and lets the downstream cap reject it', async () => { + const resize = vi.fn(async () => ({ data: 'b'.repeat(150), width: 5, height: 5 })) + const data = `data:image/png;base64,${'a'.repeat(400)}` + await expect( + prepareMobileClipboardImageBase64({ data, size: { width: 40, height: 20 } }, resize, 100) + ).resolves.toBe('b'.repeat(150)) + expect(resize).toHaveBeenCalledTimes(3) + }) +}) diff --git a/mobile/src/session/mobile-clipboard-image.ts b/mobile/src/session/mobile-clipboard-image.ts index a6a8e9eee..3bc39749a 100644 --- a/mobile/src/session/mobile-clipboard-image.ts +++ b/mobile/src/session/mobile-clipboard-image.ts @@ -4,6 +4,10 @@ import type { RpcFailure, RpcSuccess } from '../transport/types' export const MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS = 24 * 1024 * 1024 export const MOBILE_CLIPBOARD_IMAGE_UPLOAD_CHUNK_BASE64_CHARS = 512 * 1024 export const MOBILE_CLIPBOARD_IMAGE_SINGLE_FRAME_FALLBACK_BASE64_CHARS = 256 * 1024 +// Why: PNG bytes don't scale exactly with pixel area, so undershoot the target on +// each pass and let the bounded retry below converge instead of distorting in one shot. +const MOBILE_CLIPBOARD_IMAGE_DOWNSCALE_SAFETY = 0.85 +const MOBILE_CLIPBOARD_IMAGE_MAX_DOWNSCALE_ATTEMPTS = 3 const DATA_URL_PREFIX_RE = /^data:image\/[a-z0-9.+-]+;base64,/i const BASE64_PATTERN = /^[A-Za-z0-9+/]*={0,2}$/ @@ -19,6 +23,75 @@ export function normalizeMobileClipboardImageBase64(data: string): string { return contentBase64 } +export type MobileClipboardImage = { + data: string + size: { width: number; height: number } +} + +export type MobileClipboardImageResizer = ( + source: string, + target: { width: number; height: number } +) => Promise<{ data: string; width: number; height: number }> + +/** + * Returns the pixel dimensions to resize a clipboard image to so its base64 fits + * the upload budget, or null when it already fits (or its dimensions are unusable). + */ +export function computeMobileClipboardImageDownscale( + base64Length: number, + width: number, + height: number, + maxBase64Length: number +): { width: number; height: number } | null { + if (base64Length <= maxBase64Length) { + return null + } + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return null + } + const scale = Math.sqrt(maxBase64Length / base64Length) * MOBILE_CLIPBOARD_IMAGE_DOWNSCALE_SAFETY + const nextWidth = Math.max(1, Math.floor(width * scale)) + const nextHeight = Math.max(1, Math.floor(height * scale)) + // Guard against a no-op shrink (already 1px) so the retry loop can't spin forever. + if (nextWidth >= width && nextHeight >= height) { + return null + } + return { width: nextWidth, height: nextHeight } +} + +/** + * Downscales an oversized clipboard image until its base64 fits the upload budget, + * delegating the actual raster resize to the injected `resize`. Returns the + * upload-ready base64; if it still overflows after the bounded retries the + * downstream size check rejects it with the same "too large" error as before. + */ +export async function prepareMobileClipboardImageBase64( + image: MobileClipboardImage, + resize: MobileClipboardImageResizer, + maxBase64Length: number = MOBILE_CLIPBOARD_IMAGE_MAX_BASE64_CHARS +): Promise { + let data = image.data + let width = image.size.width + let height = image.size.height + for (let attempt = 0; attempt < MOBILE_CLIPBOARD_IMAGE_MAX_DOWNSCALE_ATTEMPTS; attempt += 1) { + const contentLength = data.replace(DATA_URL_PREFIX_RE, '').length + const target = computeMobileClipboardImageDownscale( + contentLength, + width, + height, + maxBase64Length + ) + if (!target) { + return data + } + const resized = await resize(data, target) + data = resized.data + width = resized.width + height = resized.height + } + return data +} + function assertSuccess(response: RpcSuccess | RpcFailure): T { if (!response.ok) { throw new Error(response.error.message)