perf: bound desktop screenshot payloads (#4083)

native/computer-use-linux/runtime.py

@@ -23,14 +23,20 @@ import gi
 gi.require_version("Atspi", "2.0")
 try:
     gi.require_version("Gdk", "3.0")
-    from gi.repository import Gdk
+    gi.require_version("GdkPixbuf", "2.0")
+    from gi.repository import Gdk, GdkPixbuf
 except (ImportError, ValueError):
     Gdk = None
+    GdkPixbuf = None
 from gi.repository import Atspi
 
 MAX_NODES = 1200
 MAX_DEPTH = 64
 TEXT_LIMIT = 500
+MAX_SCREENSHOT_PNG_BYTES = 900_000
+MAX_SCREENSHOT_EDGE = 1280
+MIN_SCREENSHOT_SCALE = 0.25
+SCREENSHOT_SCALE_STEP = 0.85
 BLOCKED_APP_FRAGMENTS = (
     "1password",
     "bitwarden",
@@ -400,7 +406,7 @@ def render_accessibility_tree(root, window_rect, root_path):
 
 
 def capture_png(rect):
-    if Gdk is None or rect is None or os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland":
+    if Gdk is None or GdkPixbuf is None or rect is None or os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland":
         return None
     screen = Gdk.Screen.get_default()
     root = screen.get_root_window() if screen else None
@@ -409,8 +415,59 @@ def capture_png(rect):
     pixbuf = Gdk.pixbuf_get_from_window(root, round(rect.x), round(rect.y), max(1, round(rect.width)), max(1, round(rect.height)))
     if pixbuf is None:
         return None
+    return bounded_png_payload(pixbuf)
+
+
+def png_bytes(pixbuf):
     ok, data = pixbuf.save_to_bufferv("png", [], [])
-    return base64.b64encode(bytes(data)).decode("ascii") if ok else None
+    return bytes(data) if ok else None
+
+
+def screenshot_payload(data, width, height, original_width):
+    return {
+        "base64": base64.b64encode(data).decode("ascii"),
+        "width": width,
+        "height": height,
+        "scale": width / max(1, original_width),
+    }
+
+
+def bounded_png_payload(pixbuf):
+    original_width = max(1, pixbuf.get_width())
+    original_height = max(1, pixbuf.get_height())
+    data = png_bytes(pixbuf)
+    if data is None:
+        return None
+    if len(data) <= MAX_SCREENSHOT_PNG_BYTES:
+        return screenshot_payload(data, original_width, original_height, original_width)
+
+    # Why: screenshots are sent through JSON/stdout; matching macOS bounds keeps
+    # large or high-DPI windows from multiplying native, base64, and Node memory.
+    best_data = data
+    best_width = original_width
+    best_height = original_height
+    scale = min(1.0, MAX_SCREENSHOT_EDGE / max(original_width, original_height))
+    while scale >= MIN_SCREENSHOT_SCALE:
+        width = max(1, round(original_width * scale))
+        height = max(1, round(original_height * scale))
+        if width == best_width and height == best_height:
+            scale *= SCREENSHOT_SCALE_STEP
+            continue
+        scaled = pixbuf.scale_simple(width, height, GdkPixbuf.InterpType.BILINEAR)
+        if scaled is None:
+            scale *= SCREENSHOT_SCALE_STEP
+            continue
+        candidate = png_bytes(scaled)
+        if candidate is not None:
+            if len(candidate) <= MAX_SCREENSHOT_PNG_BYTES:
+                return screenshot_payload(candidate, width, height, original_width)
+            if len(candidate) < len(best_data):
+                best_data = candidate
+                best_width = width
+                best_height = height
+        scale *= SCREENSHOT_SCALE_STEP
+
+    return screenshot_payload(best_data, best_width, best_height, original_width)
 
 
 def first_descendant(root, predicate):
@@ -470,13 +527,17 @@ def make_snapshot(query, include_screenshot, window_id=None, window_index=None,
     window_index, window = choose_window(app, window_id, window_index)
     bounds = screen_rect(window)
     records, lines, truncation = render_accessibility_tree(window, bounds, [window_index])
+    screenshot = capture_png(bounds) if include_screenshot else None
     return {
         "snapshotId": str(uuid.uuid4()),
         "app": app_json(app),
         "windowTitle": name_of(window),
         "windowId": window_index,
         "windowBounds": bounds.to_json() if bounds else None,
-        "screenshotPngBase64": capture_png(bounds) if include_screenshot else None,
+        "screenshotPngBase64": screenshot["base64"] if screenshot else None,
+        "screenshotWidth": screenshot["width"] if screenshot else None,
+        "screenshotHeight": screenshot["height"] if screenshot else None,
+        "screenshotScale": screenshot["scale"] if screenshot else None,
         "coordinateSpace": "window",
         "truncation": truncation,
         "treeLines": lines,
@@ -503,7 +564,7 @@ def handshake_response():
     is_wayland = os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland"
     has_hotkey = shutil.which("xdotool") is not None and not is_wayland
     has_clipboard = any(shutil.which(command) for command in ("wl-copy", "xclip", "xsel"))
-    has_screenshot = Gdk is not None and not is_wayland
+    has_screenshot = Gdk is not None and GdkPixbuf is not None and not is_wayland
     return {
         "platform": "linux",
         "provider": "orca-computer-use-linux",

native/computer-use-windows/runtime.ps1

@@ -62,6 +62,10 @@ public static class OrcaDesktopWin32 {
 $MaxNodes = 1200
 $MaxDepth = 64
 $TextLimit = 500
+$MaxScreenshotPngBytes = 900000
+$MaxScreenshotEdge = 1280
+$MinScreenshotScale = 0.25
+$ScreenshotScaleStep = 0.85
 $BlockedAppFragments = @(
     "1password",
     "bitwarden",
@@ -452,23 +456,103 @@ function Render-OrcaTree($RootElement, $WindowFrame) {
     [pscustomobject]@{ elements = @($records.ToArray()); lines = @($lines.ToArray()); truncation = $truncation }
 }
 
+function ConvertTo-OrcaPngBytes([System.Drawing.Image]$Image) {
+    $stream = $null
+    try {
+        $stream = New-Object System.IO.MemoryStream
+        $Image.Save($stream, [System.Drawing.Imaging.ImageFormat]::Png)
+        return ,$stream.ToArray()
+    } finally {
+        if ($null -ne $stream) { $stream.Dispose() }
+    }
+}
+
+function New-OrcaScreenshotPayload([byte[]]$Bytes, [int]$Width, [int]$Height, [double]$Scale) {
+    [pscustomobject]@{
+        base64 = [Convert]::ToBase64String($Bytes)
+        width = $Width
+        height = $Height
+        scale = $Scale
+    }
+}
+
+function Resize-OrcaBitmap([System.Drawing.Bitmap]$Source, [int]$Width, [int]$Height) {
+    $resized = $null
+    $graphics = $null
+    try {
+        $resized = New-Object System.Drawing.Bitmap $Width, $Height
+        $graphics = [System.Drawing.Graphics]::FromImage($resized)
+        $graphics.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::Bilinear
+        $graphics.DrawImage($Source, 0, 0, $Width, $Height)
+        $result = $resized
+        $resized = $null
+        return $result
+    } finally {
+        if ($null -ne $graphics) { $graphics.Dispose() }
+        if ($null -ne $resized) { $resized.Dispose() }
+    }
+}
+
+function Get-OrcaBoundedScreenshotPayload([System.Drawing.Bitmap]$Bitmap) {
+    $originalWidth = [int][Math]::Max(1, $Bitmap.Width)
+    $originalHeight = [int][Math]::Max(1, $Bitmap.Height)
+    $pngBytes = ConvertTo-OrcaPngBytes $Bitmap
+    if ($pngBytes.Length -le $MaxScreenshotPngBytes) {
+        return New-OrcaScreenshotPayload $pngBytes $originalWidth $originalHeight 1.0
+    }
+
+    # Why: screenshots cross process boundaries as PNG base64 in JSON; cap noisy
+    # large-window payloads to match the macOS provider's memory bounds.
+    $bestBytes = $pngBytes
+    $bestWidth = $originalWidth
+    $bestHeight = $originalHeight
+    $scale = [Math]::Min(1.0, $MaxScreenshotEdge / [double][Math]::Max($originalWidth, $originalHeight))
+    while ($scale -ge $MinScreenshotScale) {
+        $width = [int][Math]::Max(1, [Math]::Round($originalWidth * $scale))
+        $height = [int][Math]::Max(1, [Math]::Round($originalHeight * $scale))
+        if ($width -eq $bestWidth -and $height -eq $bestHeight) {
+            $scale *= $ScreenshotScaleStep
+            continue
+        }
+
+        $resized = $null
+        try {
+            $resized = Resize-OrcaBitmap $Bitmap $width $height
+            $candidateBytes = ConvertTo-OrcaPngBytes $resized
+            if ($candidateBytes.Length -le $MaxScreenshotPngBytes) {
+                return New-OrcaScreenshotPayload $candidateBytes $width $height ($width / [double]$originalWidth)
+            }
+            if ($candidateBytes.Length -lt $bestBytes.Length) {
+                $bestBytes = $candidateBytes
+                $bestWidth = $width
+                $bestHeight = $height
+            }
+        } finally {
+            if ($null -ne $resized) { $resized.Dispose() }
+        }
+
+        $scale *= $ScreenshotScaleStep
+    }
+
+    New-OrcaScreenshotPayload $bestBytes $bestWidth $bestHeight ($bestWidth / [double]$originalWidth)
+}
+
 function Get-OrcaScreenshot([bool]$IncludeScreenshot, $WindowFrame) {
     if (-not $IncludeScreenshot -or $null -eq $WindowFrame) { return $null }
+    $bitmap = $null
+    $graphics = $null
     try {
         $width = [int][Math]::Max(1, [Math]::Round($WindowFrame.width))
         $height = [int][Math]::Max(1, [Math]::Round($WindowFrame.height))
         $bitmap = New-Object System.Drawing.Bitmap $width, $height
         $graphics = [System.Drawing.Graphics]::FromImage($bitmap)
         $graphics.CopyFromScreen([int][Math]::Round($WindowFrame.x), [int][Math]::Round($WindowFrame.y), 0, 0, $bitmap.Size)
-        $stream = New-Object System.IO.MemoryStream
-        $bitmap.Save($stream, [System.Drawing.Imaging.ImageFormat]::Png)
-        $bytes = $stream.ToArray()
-        $graphics.Dispose()
-        $bitmap.Dispose()
-        $stream.Dispose()
-        [Convert]::ToBase64String($bytes)
+        Get-OrcaBoundedScreenshotPayload $bitmap
     } catch {
         $null
+    } finally {
+        if ($null -ne $graphics) { $graphics.Dispose() }
+        if ($null -ne $bitmap) { $bitmap.Dispose() }
     }
 }
 
@@ -479,6 +563,7 @@ function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId =
     $root = Get-OrcaRootElement $process
     $windowFrame = Get-OrcaWindowFrame $process $root
     $tree = Render-OrcaTree $root $windowFrame
+    $screenshot = Get-OrcaScreenshot $IncludeScreenshot $windowFrame
 
     [pscustomobject]@{
         snapshotId = [guid]::NewGuid().ToString()
@@ -486,7 +571,10 @@ function New-OrcaSnapshot([string]$Query, [bool]$IncludeScreenshot, $WindowId =
         windowTitle = $process.MainWindowTitle
         windowId = Get-OrcaWindowId $process
         windowBounds = $windowFrame
-        screenshotPngBase64 = Get-OrcaScreenshot $IncludeScreenshot $windowFrame
+        screenshotPngBase64 = if ($null -ne $screenshot) { $screenshot.base64 } else { $null }
+        screenshotWidth = if ($null -ne $screenshot) { $screenshot.width } else { $null }
+        screenshotHeight = if ($null -ne $screenshot) { $screenshot.height } else { $null }
+        screenshotScale = if ($null -ne $screenshot) { $screenshot.scale } else { $null }
         coordinateSpace = "window"
         truncation = $tree.truncation
         treeLines = @($tree.lines)

src/main/computer/desktop-script-provider-client.test.ts

@@ -107,6 +107,34 @@ describe('DesktopScriptProviderClient', () => {
     expect(typeof operationPath).toBe('string')
   })
 
+  it('uses bridge screenshot dimensions when the native payload is downscaled', async () => {
+    mockBridgeResponse({
+      ok: true,
+      snapshot: {
+        ...sampleBridgeSnapshot('Text Editor', 'initial'),
+        screenshotWidth: 150,
+        screenshotHeight: 100,
+        screenshotScale: 0.5
+      }
+    })
+
+    const client = new DesktopScriptProviderClient('linux', '/tmp/runtime.py')
+
+    const result = await client.snapshot({ app: 'Text Editor' })
+
+    expect(result.screenshot).toEqual({
+      data: 'iVBORw0KGgo=',
+      format: 'png',
+      width: 150,
+      height: 100,
+      scale: 0.5
+    })
+    expect(result.snapshot.window).toMatchObject({
+      width: 300,
+      height: 200
+    })
+  })
+
   it('targets cached elements by session and explicit window id', async () => {
     mockBridgeResponse({
       ok: true,

src/main/computer/desktop-script-provider-client.ts

@@ -71,6 +71,9 @@ type BridgeSnapshot = {
   windowId?: number | null
   windowBounds?: BridgeFrame | null
   screenshotPngBase64?: string | null
+  screenshotWidth?: number | null
+  screenshotHeight?: number | null
+  screenshotScale?: number | null
   coordinateSpace?: 'window'
   truncation?: {
     truncated?: boolean
@@ -488,13 +491,20 @@ function execBridge(
 function renderSnapshot(snapshot: BridgeSnapshot, noScreenshot: boolean): ComputerSnapshotResult {
   const bounds = snapshot.windowBounds
   const treeText = renderTreeText(snapshot)
+  // Why: Linux/Windows providers may downscale screenshots to cap IPC payloads,
+  // while window bounds remain the unscaled coordinate space for actions.
+  const screenshotWidth =
+    positiveRoundedNumber(snapshot.screenshotWidth) ?? Math.max(1, Math.round(bounds?.width ?? 1))
+  const screenshotHeight =
+    positiveRoundedNumber(snapshot.screenshotHeight) ?? Math.max(1, Math.round(bounds?.height ?? 1))
+  const screenshotScale = positiveNumber(snapshot.screenshotScale) ?? 1
   const screenshot = snapshot.screenshotPngBase64
     ? {
         data: snapshot.screenshotPngBase64,
         format: 'png' as const,
-        width: Math.max(1, Math.round(bounds?.width ?? 1)),
-        height: Math.max(1, Math.round(bounds?.height ?? 1)),
-        scale: 1
+        width: screenshotWidth,
+        height: screenshotHeight,
+        scale: screenshotScale
       }
     : null
   return {
@@ -546,6 +556,15 @@ function renderSnapshot(snapshot: BridgeSnapshot, noScreenshot: boolean): Comput
   }
 }
 
+function positiveNumber(value: number | null | undefined): number | null {
+  return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : null
+}
+
+function positiveRoundedNumber(value: number | null | undefined): number | null {
+  const numberValue = positiveNumber(value)
+  return numberValue === null ? null : Math.max(1, Math.round(numberValue))
+}
+
 function fallbackSnapshotId(snapshot: BridgeSnapshot): string {
   const appRef = snapshot.app.bundleId ?? snapshot.app.bundleIdentifier ?? snapshot.app.name
   return `${appRef}:${snapshot.app.pid}:${snapshot.windowId ?? 'window'}`