diff --git a/src/main/browser/browser-screencast-snapshot-scaling.test.ts b/src/main/browser/browser-screencast-snapshot-scaling.test.ts new file mode 100644 index 000000000..4358d6183 --- /dev/null +++ b/src/main/browser/browser-screencast-snapshot-scaling.test.ts @@ -0,0 +1,94 @@ +import { Buffer } from 'node:buffer' +import { EventEmitter } from 'node:events' +import { describe, expect, it, vi } from 'vitest' + +import { startBrowserScreencast } from './browser-screencast-stream' + +function createMockWebContents(capturePage: () => Promise) { + let attached = false + const dbg = new EventEmitter() as EventEmitter & { + isAttached: ReturnType + attach: ReturnType + detach: ReturnType + sendCommand: ReturnType + } + dbg.isAttached = vi.fn(() => attached) + dbg.attach = vi.fn(() => { + attached = true + }) + dbg.detach = vi.fn(() => { + attached = false + }) + dbg.sendCommand = vi.fn(async () => ({})) + return { isDestroyed: vi.fn(() => false), debugger: dbg, capturePage: vi.fn(capturePage) } +} + +function createCapturedImage(width: number, height: number) { + const resized = { + getSize: vi.fn(() => ({ width, height })), + resize: vi.fn(), + toJPEG: vi.fn(() => Buffer.from('scaled-frame')), + toPNG: vi.fn(() => Buffer.from('scaled-frame')) + } + const image = { + getSize: vi.fn(() => ({ width, height })), + resize: vi.fn(() => resized), + toJPEG: vi.fn(() => Buffer.from('captured-frame')), + toPNG: vi.fn(() => Buffer.from('captured-frame')) + } + return { image, resized } +} + +describe('browser screencast snapshot scaling', () => { + it('scales a hi-DPI capture down to the requested frame bounds', async () => { + const { image, resized } = createCapturedImage(4000, 3000) + const webContents = createMockWebContents(async () => image) + const onFrame = vi.fn() + + const session = await startBrowserScreencast(webContents as never, { + format: 'jpeg', + quality: 70, + maxWidth: 1440, + maxHeight: 1200, + viewportWidth: 2000, + viewportHeight: 1500, + deviceScaleFactor: 2, + everyNthFrame: 2, + minFrameIntervalMs: 0, + onFrame + }) + + await vi.waitFor(() => expect(onFrame).toHaveBeenCalledTimes(1)) + expect(image.resize).toHaveBeenCalledWith({ width: 1440, height: 1080 }) + expect(resized.toJPEG).toHaveBeenCalledWith(70) + expect(image.toJPEG).not.toHaveBeenCalled() + + session.stop() + await session.done + }) + + it('leaves a capture already within the frame bounds unscaled', async () => { + const { image } = createCapturedImage(1200, 800) + const webContents = createMockWebContents(async () => image) + const onFrame = vi.fn() + + const session = await startBrowserScreencast(webContents as never, { + format: 'jpeg', + quality: 70, + maxWidth: 1440, + maxHeight: 1200, + viewportWidth: 1200, + viewportHeight: 800, + everyNthFrame: 2, + minFrameIntervalMs: 0, + onFrame + }) + + await vi.waitFor(() => expect(onFrame).toHaveBeenCalledTimes(1)) + expect(image.resize).not.toHaveBeenCalled() + expect(image.toJPEG).toHaveBeenCalledWith(70) + + session.stop() + await session.done + }) +}) diff --git a/src/main/browser/browser-screencast-stream.ts b/src/main/browser/browser-screencast-stream.ts index ad2e1e27e..15a5395ce 100644 --- a/src/main/browser/browser-screencast-stream.ts +++ b/src/main/browser/browser-screencast-stream.ts @@ -1,6 +1,6 @@ /* eslint-disable max-lines -- Why: screencast setup, CDP lifecycle, metadata normalization, and stream teardown stay together so frame behavior cannot drift across files. */ import { Buffer } from 'node:buffer' -import type { WebContents } from 'electron' +import type { NativeImage, WebContents } from 'electron' import { BrowserScreencastOpcode, encodeBrowserScreencastFrame, @@ -86,6 +86,20 @@ function scaleToFit( } } +// Why: capturePage returns device pixels, so a hi-DPI viewport yields a bitmap several times +// larger than the live path is allowed to send. Apply the caller's cap here too. +function scaleSnapshotToFit(image: NativeImage, options: BrowserScreencastOptions): NativeImage { + const size = image.getSize() + if (!size.width || !size.height) { + return image + } + const fitted = scaleToFit(size.width, size.height, options.maxWidth, options.maxHeight) + if (fitted.width === size.width && fitted.height === size.height) { + return image + } + return image.resize(fitted) +} + function isNearSize( actual: { width: number; height: number }, expected: { width: number; height: number } @@ -518,8 +532,9 @@ export async function startBrowserScreencast( width: viewportWidth, height: viewportHeight }) + const capture = scaleSnapshotToFit(nativeImage, options) const buffer = - options.format === 'png' ? nativeImage.toPNG() : nativeImage.toJPEG(options.quality) + options.format === 'png' ? capture.toPNG() : capture.toJPEG(options.quality) if (buffer.byteLength > 0) { image = new Uint8Array(buffer) } diff --git a/src/main/runtime/orca-runtime-browser.test.ts b/src/main/runtime/orca-runtime-browser.test.ts index c45fc1403..5cc74aa4a 100644 --- a/src/main/runtime/orca-runtime-browser.test.ts +++ b/src/main/runtime/orca-runtime-browser.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { AgentBrowserBridge } from '../browser/agent-browser-bridge' +import { REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES } from '../../shared/remote-runtime-memory-limits' import type { RuntimeBrowserCommandHost } from './orca-runtime-browser' const { @@ -579,6 +580,32 @@ describe('RuntimeBrowserCommands browser screencast', () => { await second.session.done expect(secondStop).toHaveBeenCalledTimes(1) }, 10_000) + + it('admits screencast frames through the paired-runtime size guard', async () => { + const { RuntimeBrowserCommands } = await import('./orca-runtime-browser') + webContentsFromIdMock.mockReturnValue({ isDestroyed: () => false }) + const done = deferred() + startBrowserScreencastMock.mockResolvedValue({ + stop: vi.fn(() => done.resolve()), + done: done.promise + }) + const sendBinary = vi.fn(() => true) + + const commands = new RuntimeBrowserCommands(createHost()) + const started = await commands.browserScreencast( + { worktree: 'id:wt-1', page: 'page-1', format: 'jpeg' }, + { sendBinary } + ) + const { onFrame } = startBrowserScreencastMock.mock.calls[0][1] + + expect(onFrame(new Uint8Array(REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES + 1))).toBe(true) + expect(sendBinary).not.toHaveBeenCalled() + expect(onFrame(new Uint8Array(64))).toBe(true) + expect(sendBinary).toHaveBeenCalledTimes(1) + + started.session.stop() + await started.session.done + }) }) describe('RuntimeBrowserCommands headless offscreen routing', () => { diff --git a/src/main/runtime/orca-runtime-browser.ts b/src/main/runtime/orca-runtime-browser.ts index acf48dcd4..6070cefd3 100644 --- a/src/main/runtime/orca-runtime-browser.ts +++ b/src/main/runtime/orca-runtime-browser.ts @@ -68,6 +68,7 @@ import { selectBrowserProfile } from '../browser/browser-cookie-import' import { waitForTabRegistration, waitForWorktreeTabRegistration } from '../ipc/browser' +import { sendRemoteBrowserScreencastFrame } from './remote-browser-screencast-frame-admission' export type BrowserCommandTargetParams = { worktree?: string @@ -507,7 +508,7 @@ export class RuntimeBrowserCommands { mobile: params.mobile === true, everyNthFrame: clampInteger(params.everyNthFrame, 1, 10, 2), minFrameIntervalMs: clampInteger(params.minFrameIntervalMs, 0, 1000, 0), - onFrame: stream.sendBinary, + onFrame: (bytes) => sendRemoteBrowserScreencastFrame(stream.sendBinary, bytes), onEvent: stream.emit, onError: (message) => stream.emit?.({ type: 'error', message }) }) diff --git a/src/main/runtime/remote-browser-screencast-frame-admission.test.ts b/src/main/runtime/remote-browser-screencast-frame-admission.test.ts new file mode 100644 index 000000000..8f0f1e830 --- /dev/null +++ b/src/main/runtime/remote-browser-screencast-frame-admission.test.ts @@ -0,0 +1,149 @@ +import { Buffer } from 'node:buffer' +import { EventEmitter } from 'node:events' +import { describe, expect, it, vi } from 'vitest' + +import { deriveSharedKey, encrypt, generateKeyPair } from '../../shared/e2ee-crypto' +import { REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES } from '../../shared/remote-runtime-memory-limits' +import { startBrowserScreencast } from '../browser/browser-screencast-stream' +import { E2EEChannel } from './rpc/e2ee-channel' +import { sendRemoteBrowserScreencastFrame } from './remote-browser-screencast-frame-admission' + +function createMockWebContents() { + let attached = false + const dbg = new EventEmitter() as EventEmitter & { + isAttached: ReturnType + attach: ReturnType + detach: ReturnType + sendCommand: ReturnType + } + dbg.isAttached = vi.fn(() => attached) + dbg.attach = vi.fn(() => { + attached = true + }) + dbg.detach = vi.fn(() => { + attached = false + }) + dbg.sendCommand = vi.fn(async () => ({})) + return { isDestroyed: vi.fn(() => false), debugger: dbg } +} + +// Why: exercises the real E2EE channel so the oracle fails if an over-limit frame ever reaches +// the transport that answers it with close code 1013. +function createRuntimeBinarySender() { + const serverKeys = generateKeyPair() + const clientKeys = generateKeyPair() + const ws = { + OPEN: 1 as const, + readyState: 1, + send: vi.fn(), + close: vi.fn() + } + const onTransportError = vi.fn((code: number, reason: string) => ws.close(code, reason)) + const channel = new E2EEChannel(ws as never, { + serverSecretKey: serverKeys.secretKey, + resolveAuthenticatedDevice: (token) => + token === 'valid-token' + ? { deviceId: 'device-1', deviceToken: token, scope: 'runtime' } + : null, + onReady: vi.fn(), + onError: onTransportError + }) + const sharedKey = deriveSharedKey(clientKeys.secretKey, serverKeys.publicKey) + channel.handleRawMessage( + JSON.stringify({ + type: 'e2ee_hello', + publicKeyB64: Buffer.from(clientKeys.publicKey).toString('base64') + }) + ) + channel.handleRawMessage( + encrypt(JSON.stringify({ type: 'e2ee_auth', deviceToken: 'valid-token' }), sharedKey) + ) + let sendBinary: ((bytes: Uint8Array) => boolean | void) | undefined + channel.onMessage((_plaintext, _sendText, sendBinaryReply) => { + sendBinary = sendBinaryReply + }) + channel.handleRawMessage(encrypt('start-screencast', sharedKey)) + if (!sendBinary) { + throw new Error('Runtime binary sender was not established') + } + return { channel, ws, onTransportError, sendBinary } +} + +describe('sendRemoteBrowserScreencastFrame', () => { + it('reports an over-limit frame as handled so the producer does not retry it', () => { + const sendBinary = vi.fn(() => true) + const oversized = new Uint8Array(REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES + 1) + + expect(sendRemoteBrowserScreencastFrame(sendBinary, oversized)).toBe(true) + expect(sendBinary).not.toHaveBeenCalled() + }) + + it('still forwards backpressure for a frame the transport can accept', () => { + const withinLimit = new Uint8Array(1024) + + expect( + sendRemoteBrowserScreencastFrame( + vi.fn(() => false), + withinLimit + ) + ).toBe(false) + expect( + sendRemoteBrowserScreencastFrame( + vi.fn(() => true), + withinLimit + ) + ).toBe(true) + expect(sendRemoteBrowserScreencastFrame(vi.fn(), withinLimit)).toBe(true) + }) + + it('drops an oversized encoded frame without closing or stalling the paired stream', async () => { + vi.useFakeTimers() + const webContents = createMockWebContents() + const transport = createRuntimeBinarySender() + const session = await startBrowserScreencast(webContents as never, { + format: 'jpeg', + quality: 70, + maxWidth: 3840, + maxHeight: 2160, + everyNthFrame: 1, + minFrameIntervalMs: 0, + onFrame: (bytes) => sendRemoteBrowserScreencastFrame(transport.sendBinary, bytes) + }) + + try { + webContents.debugger.emit('message', {}, 'Page.screencastFrame', { + data: Buffer.alloc(REMOTE_RUNTIME_MAX_OUTBOUND_BINARY_FRAME_BYTES + 1).toString('base64'), + sessionId: 42, + metadata: {} + }) + await Promise.resolve() + + expect(transport.onTransportError).not.toHaveBeenCalled() + expect(transport.ws.close).not.toHaveBeenCalled() + expect(webContents.debugger.sendCommand).toHaveBeenCalledWith('Page.screencastFrameAck', { + sessionId: 42 + }) + const sendsAfterDrop = transport.ws.send.mock.calls.length + await vi.advanceTimersByTimeAsync(500) + expect(transport.ws.send).toHaveBeenCalledTimes(sendsAfterDrop) + + webContents.debugger.emit('message', {}, 'Page.screencastFrame', { + data: Buffer.from('next-frame').toString('base64'), + sessionId: 43, + metadata: {} + }) + await Promise.resolve() + + expect(transport.onTransportError).not.toHaveBeenCalled() + expect(webContents.debugger.sendCommand).toHaveBeenCalledWith('Page.screencastFrameAck', { + sessionId: 43 + }) + expect(transport.ws.send.mock.calls.length).toBeGreaterThan(sendsAfterDrop) + } finally { + session.stop() + await session.done + transport.channel.destroy() + vi.useRealTimers() + } + }) +}) diff --git a/src/main/runtime/remote-browser-screencast-frame-admission.ts b/src/main/runtime/remote-browser-screencast-frame-admission.ts new file mode 100644 index 000000000..6e4d1255b --- /dev/null +++ b/src/main/runtime/remote-browser-screencast-frame-admission.ts @@ -0,0 +1,14 @@ +import { isRemoteRuntimeBinaryFrameWithinLimit } from '../../shared/remote-runtime-memory-limits' + +// Why: the E2EE channel closes the socket (1013) on an over-limit binary frame, and the +// screencast producer reads `false` as backpressure and retries the identical frame. Reporting +// an over-limit frame as handled drops it so the stream advances instead of retrying forever. +export function sendRemoteBrowserScreencastFrame( + sendBinary: (bytes: Uint8Array) => boolean | void, + bytes: Uint8Array +): boolean { + if (!isRemoteRuntimeBinaryFrameWithinLimit(bytes)) { + return true + } + return sendBinary(bytes) !== false +}