diff --git a/docs/next/CHANGELOG.md b/docs/next/CHANGELOG.md index 343086aa..5d123a8c 100644 --- a/docs/next/CHANGELOG.md +++ b/docs/next/CHANGELOG.md @@ -11,6 +11,7 @@ ### Changed - Bumped the client/server protocol version to 20 for pane terminal bell forwarding. +- Experimental pane graphics now support bounded named layers, acknowledged full-RGBA primary-layer direct file frames on audited local terminals, owned BGRA fallback, exact pixel mouse input, and placement-only resize replay. ### Fixed - `herdr config check` now reports unknown built-in theme names instead of silently accepting them. (#2452) diff --git a/docs/next/api/herdr-api.schema.json b/docs/next/api/herdr-api.schema.json index 1e8d7830..8a552da0 100644 --- a/docs/next/api/herdr-api.schema.json +++ b/docs/next/api/herdr-api.schema.json @@ -2452,6 +2452,12 @@ }, "PaneGraphicsClearParams": { "properties": { + "layer_id": { + "type": [ + "string", + "null" + ] + }, "pane_id": { "type": "string" } @@ -2465,7 +2471,8 @@ "enum": [ "png", "rgb", - "rgba" + "rgba", + "bgra" ], "type": "string" }, @@ -2515,6 +2522,12 @@ "minimum": 0, "type": "integer" }, + "layer_id": { + "type": [ + "string", + "null" + ] + }, "pane_id": { "type": "string" }, @@ -2526,6 +2539,11 @@ "viewport_col": 0, "viewport_row": 0 } + }, + "z_index": { + "default": 0, + "format": "int32", + "type": "integer" } }, "required": [ @@ -9234,6 +9252,30 @@ ], "type": "object" }, + { + "properties": { + "revision": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "sequence": { + "format": "uint64", + "minimum": 0, + "type": "integer" + }, + "type": { + "const": "pane_graphics_frame_ack", + "type": "string" + } + }, + "required": [ + "type", + "sequence", + "revision" + ], + "type": "object" + }, { "properties": { "cell_height_px": { @@ -9246,6 +9288,51 @@ "minimum": 0, "type": "integer" }, + "file_frame_damage": { + "default": false, + "description": "Accepts damage metadata while still consuming a complete canonical file.", + "type": "boolean" + }, + "file_frame_directory": { + "type": [ + "string", + "null" + ] + }, + "file_frame_formats": { + "items": { + "type": "string" + }, + "type": "array" + }, + "file_frame_max_bytes": { + "format": "uint", + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "file_frame_transport": { + "type": [ + "string", + "null" + ] + }, + "max_layers_per_pane": { + "default": 0, + "format": "uint", + "minimum": 0, + "type": "integer" + }, + "pane_visible": { + "description": "True only when this pane is on the currently rendered terminal surface.", + "type": "boolean" + }, + "pixel_mouse": { + "default": false, + "type": "boolean" + }, "type": { "const": "pane_graphics_info", "type": "string" @@ -9254,7 +9341,8 @@ "required": [ "type", "cell_width_px", - "cell_height_px" + "cell_height_px", + "pane_visible" ], "type": "object" }, diff --git a/docs/next/website/src/content/docs/socket-api.mdx b/docs/next/website/src/content/docs/socket-api.mdx index bcf0f14c..86551c03 100644 --- a/docs/next/website/src/content/docs/socket-api.mdx +++ b/docs/next/website/src/content/docs/socket-api.mdx @@ -177,10 +177,20 @@ Clients can treat `offset_from_bottom == 0` as at-bottom state. Pane graphics let a plugin place image data over a pane. They are available only when `[experimental].kitty_graphics = true`; otherwise every pane graphics -method returns `feature_disabled`. Calling `pane.graphics.info` returns the -attached client's cell width and height in pixels without creating a graphics -layer. `pane.graphics.set` accepts `png`, `rgb`, or `rgba` data in `data_base64`, -and `pane.graphics.clear` removes the layer. +method returns `feature_disabled`. Calling `pane.graphics.info` explicitly +activates capability discovery and returns the attached client's cell size, +file-frame options, pixel-mouse support, the 16-layer limit, and `pane_visible`. +`pane_visible` is true only when the target is in the active workspace and tab +and is not hidden by zoom. Short-lived UI modes do not change it. + +`pane.graphics.set`, `pane.graphics.clear`, and `pane.graphics.stream` accept an +optional `layer_id` (default `primary`). Set and stream also accept `z_index`; +layers are placed in stable `(z_index, layer_id)` order. Each stream exclusively owns its layer, and +closing it removes that layer. Inline frames accept `png`, `rgb`, `rgba`, or +`bgra`; BGRA is normalized once to owned RGBA. Herdr advances the host cache +one image transaction per render pass, so arbitrary layer sets progress without +an aggregate frame. Headless transport keeps each transaction within its 32 MiB +wire limit; local monolithic rendering does not apply that transport limit. ```json {"id":"graphics_info","method":"pane.graphics.info","params":{"pane_id":"w1:p1"}} @@ -190,14 +200,31 @@ and `pane.graphics.clear` removes the layer. For repeated frames, open a dedicated socket with `pane.graphics.stream`. After Herdr replies with `ok`, send one JSON header and then exactly `data_length` raw -bytes per frame. A stream owns that pane's graphics layer until the socket -closes; concurrent set, clear, or stream requests return `stream_conflict`. +bytes per inline frame. Concurrent operations on that layer return +`stream_conflict`. ```json -{"id":"graphics_stream","method":"pane.graphics.stream","params":{"pane_id":"w1:p1"}} +{"id":"graphics_stream","method":"pane.graphics.stream","params":{"pane_id":"w1:p1","z_index":0}} {"format":"png","image_width":800,"image_height":600,"data_length":12345,"placement":{"viewport_col":0,"viewport_row":0,"grid_cols":80,"grid_rows":30}} ``` +When `pane.graphics.info` advertises `file_frame_transport: "direct-kitty"`, an +eligible local Ghostty, kitty, or WezTerm client may submit an immutable private +`rgba` or `bgra` file with `file.path`, `sequence`, and `revision`. Direct Kitty +file transport is reserved for the default `primary` page layer; named secondary +layers use owned inline RGBA. BGRA is always copied, swizzled, and rendered +inline. Herdr replies with a `pane_graphics_frame_ack` only after the terminal +accepts the file, or after a safe owned inline fallback is installed. Confirmed +file-transport failure disables direct files for that client connection without +disabling exact pixel mouse. A timeout or client loss closes the stream without +acknowledging source reuse. Monolithic `--no-session` mode advertises neither +fast file transport nor exact pixel mouse and remains on owned inline fallback. + +Direct files are always complete canonical `width * height * 4` RGBA frames. +`file_frame_damage: true` means Herdr accepts optional damage metadata for +producer-side canonical-ring efficiency; it still copies or presents the full file. +Resize and full redraw replay placements without retransmitting pixels. + `pane.layout` returns the tab layout snapshot with `workspace_id`, `tab_id`, `zoomed`, outer `area`, `focused_pane_id`, pane rects, and split rects/ratios. `pane.neighbor` and `pane.edges` include that same layout snapshot so clients diff --git a/src/api/mod.rs b/src/api/mod.rs index 27e98d2b..a44899da 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,7 +7,6 @@ mod subscriptions; mod wait; pub use event_hub::EventHub; -pub(crate) use server::cancel_inactive_pane_graphics_streams; pub use server::{start_server, start_server_with_capabilities, ServerHandle}; pub use status::{read_runtime_status_at, RuntimeStatus}; @@ -62,6 +61,7 @@ pub(crate) fn request_changes_ui(request: &Request) -> bool { | Method::PaneGraphicsClear(_) | Method::PaneGraphicsStream(_) | Method::PaneGraphicsStreamSet(_) + | Method::PaneGraphicsStreamDirect(_) | Method::PaneGraphicsStreamOpen(_) | Method::PaneGraphicsStreamClose(_) | Method::PaneReportAgent(_) @@ -84,6 +84,7 @@ pub struct ApiRequestMessage { pub request: Request, pub respond_to: std::sync::mpsc::Sender, pub response_write_complete: Option>, + pub stream_active: Option>, } pub type ApiRequestSender = mpsc::UnboundedSender; diff --git a/src/api/schema.rs b/src/api/schema.rs index 447ea42c..6f76e3d3 100644 --- a/src/api/schema.rs +++ b/src/api/schema.rs @@ -187,6 +187,9 @@ pub enum Method { PaneGraphicsStreamSet(PaneGraphicsSetParams), #[serde(skip)] #[schemars(skip)] + PaneGraphicsStreamDirect(PaneGraphicsDirectParams), + #[serde(skip)] + #[schemars(skip)] PaneGraphicsStreamOpen(PaneGraphicsStreamParams), #[serde(skip)] #[schemars(skip)] diff --git a/src/api/schema/panes.rs b/src/api/schema/panes.rs index 2dffd7f9..c8d89c48 100644 --- a/src/api/schema/panes.rs +++ b/src/api/schema/panes.rs @@ -4,6 +4,10 @@ use serde::{Deserialize, Serialize}; pub(crate) const PANE_GRAPHICS_SET_MAX_BYTES: usize = 512 * 1024; pub(crate) const PANE_GRAPHICS_STREAM_MAX_BYTES: usize = 16 * 1024 * 1024; +pub(crate) const PANE_GRAPHICS_MAX_LAYERS_PER_PANE: usize = 16; +pub(crate) const PANE_GRAPHICS_MAX_LAYERS_TOTAL: usize = 64; +pub(crate) const PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL: usize = 64 * 1024 * 1024; +pub(crate) const PANE_GRAPHICS_PRIMARY_LAYER_ID: &str = "primary"; use super::agents::AgentSessionInfo; use super::common::{AgentStatus, PaneAgentState, ReadFormat, ReadSource, SplitDirection}; @@ -287,11 +291,16 @@ pub enum PaneGraphicsFormat { Png, Rgb, Rgba, + Bgra, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] pub struct PaneGraphicsSetParams { pub pane_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub layer_id: Option, + #[serde(default)] + pub z_index: i32, #[serde(skip)] #[schemars(skip)] pub owner: String, @@ -323,11 +332,32 @@ pub struct PaneGraphicsPlacementParams { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] pub struct PaneGraphicsClearParams { pub pane_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub layer_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PaneGraphicsDirectParams { + pub pane_id: String, + pub layer_id: Option, + pub z_index: i32, + pub owner: String, + pub image_width: u32, + pub image_height: u32, + pub format: PaneGraphicsFormat, + pub path: String, + pub sequence: u64, + pub revision: u64, + pub placement: PaneGraphicsPlacementParams, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] pub struct PaneGraphicsStreamParams { pub pane_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub layer_id: Option, + #[serde(default)] + pub z_index: i32, #[serde(skip)] #[schemars(skip)] pub owner: String, diff --git a/src/api/schema/response.rs b/src/api/schema/response.rs index db8cfc7b..ae5a32e0 100644 --- a/src/api/schema/response.rs +++ b/src/api/schema/response.rs @@ -162,9 +162,30 @@ pub enum ResponseResult { PaneRead { read: PaneReadResult, }, + PaneGraphicsFrameAck { + sequence: u64, + revision: u64, + }, PaneGraphicsInfo { cell_width_px: u32, cell_height_px: u32, + /// True only when this pane is on the currently rendered terminal surface. + pane_visible: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + file_frame_directory: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + file_frame_formats: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + file_frame_max_bytes: Option, + /// Accepts damage metadata while still consuming a complete canonical file. + #[serde(default)] + file_frame_damage: bool, + #[serde(default)] + max_layers_per_pane: usize, + #[serde(default)] + pixel_mouse: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + file_frame_transport: Option, }, AgentExplain { explain: serde_json::Value, diff --git a/src/api/server.rs b/src/api/server.rs index 3932b1ea..e8b73167 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -23,7 +23,6 @@ use crate::ipc::{ }; mod pane_graphics_stream; -pub(crate) use pane_graphics_stream::cancel_inactive_streams as cancel_inactive_pane_graphics_streams; const SOCKET_PERMISSION_MODE: u32 = 0o600; pub(super) const CONNECTION_POLL_INTERVAL: Duration = Duration::from_millis(100); @@ -332,12 +331,7 @@ fn handle_request( r#"{"id":"","error":{"code":"internal_error","message":"failed to encode response"}}"# .to_string() }), - _ => dispatch_to_app_with_timeout_and_write_completion( - request, - api_tx, - None, - response_write_complete, - ), + _ => dispatch_to_app(request, api_tx, None, response_write_complete, None), } } @@ -413,6 +407,7 @@ fn api_method_name(method: &Method) -> &'static str { Method::PaneGraphicsInfo(_) => "pane.graphics.info", Method::PaneGraphicsStream(_) => "pane.graphics.stream", Method::PaneGraphicsStreamSet(_) => "pane.graphics.stream.set", + Method::PaneGraphicsStreamDirect(_) => "pane.graphics.stream.direct", Method::PaneGraphicsStreamOpen(_) => "pane.graphics.stream.open", Method::PaneGraphicsStreamClose(_) => "pane.graphics.stream.close", Method::PaneReportAgent(_) => "pane.report_agent", @@ -749,22 +744,51 @@ pub(super) fn dispatch_to_app_with_timeout( api_tx: &ApiRequestSender, timeout: Option, ) -> String { - dispatch_to_app_with_timeout_and_write_completion(request, api_tx, timeout, None) + dispatch_to_app(request, api_tx, timeout, None, None) } -fn dispatch_to_app_with_timeout_and_write_completion( +pub(super) fn dispatch_stream_open( + request: Request, + api_tx: &ApiRequestSender, + timeout: Duration, + active: Arc, +) -> String { + dispatch_to_app(request, api_tx, Some(timeout), None, Some(active)) +} + +pub(super) fn dispatch_stream_frame( + request: Request, + api_tx: &ApiRequestSender, + active: Arc, +) -> String { + dispatch_to_app( + request, + api_tx, + Some(crate::app::pane_graphics::DIRECT_OUTER_TIMEOUT), + None, + Some(active), + ) +} + +fn dispatch_to_app( request: Request, api_tx: &ApiRequestSender, timeout: Option, response_write_complete: Option>, + stream_active: Option>, ) -> String { let request_id = request.id.clone(); + let request_active = stream_active.clone(); let (respond_to, response_rx) = std::sync::mpsc::channel(); if let Err(err) = api_tx.send(ApiRequestMessage { request, respond_to, response_write_complete, + stream_active, }) { + if let Some(active) = request_active { + active.store(false, Ordering::Release); + } return error_response_json( request_id, "server_unavailable", @@ -793,11 +817,16 @@ fn dispatch_to_app_with_timeout_and_write_completion( match response { Ok(response) => response, - Err(err) => error_response_json( - request_id, - "server_unavailable", - format!("request handling failed: {err}"), - ), + Err(err) => { + if let Some(active) = request_active { + active.store(false, Ordering::Release); + } + error_response_json( + request_id, + "server_unavailable", + format!("request handling failed: {err}"), + ) + } } } @@ -1335,6 +1364,8 @@ mod pane_graphics_request_tests { id: "graphics-max".into(), method: Method::PaneGraphicsSet(crate::api::schema::PaneGraphicsSetParams { pane_id: "pane_1".into(), + layer_id: None, + z_index: 0, owner: String::new(), format: crate::api::schema::PaneGraphicsFormat::Png, image_width: 1, diff --git a/src/api/server/pane_graphics_stream.rs b/src/api/server/pane_graphics_stream.rs index 747c0205..b6420f95 100644 --- a/src/api/server/pane_graphics_stream.rs +++ b/src/api/server/pane_graphics_stream.rs @@ -1,7 +1,6 @@ -use std::collections::HashMap; use std::io::{self, Read}; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; use std::time::{Duration, Instant}; use interprocess::local_socket::traits::Stream as _; @@ -14,9 +13,9 @@ use crate::api::ApiRequestSender; use crate::ipc::{is_connection_closed_error, LocalStream}; use super::{ - api_response_outcome, dispatch_to_app_with_timeout, write_json_line, - write_json_line_allow_disconnect, write_text_line_allow_disconnect, APP_RESPONSE_TIMEOUT, - CONNECTION_POLL_INTERVAL, + api_response_outcome, dispatch_stream_frame, dispatch_stream_open, + dispatch_to_app_with_timeout, write_json_line, write_json_line_allow_disconnect, + write_text_line_allow_disconnect, APP_RESPONSE_TIMEOUT, CONNECTION_POLL_INTERVAL, }; const MAX_STREAM_FRAME_HEADER_BYTES: usize = 64 * 1024; @@ -28,15 +27,25 @@ const STREAM_FRAME_BODY_TIMEOUT: Duration = Duration::from_secs(30); const STREAM_FALLBACK_POLL_INTERVAL: Duration = Duration::from_millis(1); const STREAM_FALLBACK_FAST_POLLS: u8 = 32; static NEXT_PANE_GRAPHICS_STREAM_OWNER: AtomicU64 = AtomicU64::new(1); -static REGISTERED_STREAM_COUNT: AtomicUsize = AtomicUsize::new(0); -static REGISTERED_STREAMS: OnceLock>>> = OnceLock::new(); + +#[derive(serde::Deserialize)] +struct FrameFile { + path: String, +} #[derive(serde::Deserialize)] struct FrameHeader { format: crate::api::schema::PaneGraphicsFormat, image_width: u32, image_height: u32, - data_length: usize, + #[serde(default)] + data_length: Option, + #[serde(default)] + file: Option, + #[serde(default)] + sequence: u64, + #[serde(default)] + revision: u64, #[serde(default)] placement: crate::api::schema::PaneGraphicsPlacementParams, } @@ -102,19 +111,24 @@ fn serve_with_timeouts( read_timeouts: ReadTimeouts, ) -> std::io::Result<()> { let pane_id = params.pane_id.clone(); + let layer_id = params.layer_id.clone(); + let z_index = params.z_index; let owner = next_owner(); params.owner = owner.clone(); - let open_response = dispatch_to_app_with_timeout( + let stream_active = Arc::new(AtomicBool::new(true)); + let open_response = dispatch_stream_open( Request { id: request_id.clone(), method: Method::PaneGraphicsStreamOpen(params), }, api_tx, - Some(open_timeout), + open_timeout, + Arc::clone(&stream_active), ); if api_response_outcome(&open_response) != "ok" { + stream_active.store(false, Ordering::Release); let write_result = write_text_line_allow_disconnect(&mut stream, &open_response); - clear_layer(&pane_id, &owner, api_tx); + clear_layer(&pane_id, layer_id.as_deref(), z_index, &owner, api_tx); write_result?; return Ok(()); } @@ -126,27 +140,28 @@ fn serve_with_timeouts( result: ResponseResult::Ok {}, }, ) { - clear_layer(&pane_id, &owner, api_tx); + stream_active.store(false, Ordering::Release); + clear_layer(&pane_id, layer_id.as_deref(), z_index, &owner, api_tx); if is_connection_closed_error(&err) { return Ok(()); } return Err(err); } - let stream_active = Arc::new(AtomicBool::new(true)); - register_stream(&owner, &stream_active); let result = serve_frames( &mut stream, &request_id, &owner, &pane_id, + layer_id.as_deref(), + z_index, api_tx, running, &stream_active, read_timeouts, ); - unregister_stream(&owner); - clear_layer(&pane_id, &owner, api_tx); + stream_active.store(false, Ordering::Release); + clear_layer(&pane_id, layer_id.as_deref(), z_index, &owner, api_tx); result } @@ -155,6 +170,8 @@ fn serve_frames( request_id: &str, owner: &str, pane_id: &str, + layer_id: Option<&str>, + z_index: i32, api_tx: &ApiRequestSender, running: &Arc, stream_active: &Arc, @@ -193,7 +210,67 @@ fn serve_frames( return Ok(()); } }; - if header.data_length == 0 { + if let Some(file) = header.file { + if !matches!( + header.format, + crate::api::schema::PaneGraphicsFormat::Rgba + | crate::api::schema::PaneGraphicsFormat::Bgra + ) { + write_json_line_allow_disconnect( + stream, + &ErrorResponse { + id: request_id.to_string(), + error: ErrorBody { + code: "invalid_frame".into(), + message: "file frames require rgba or bgra".into(), + }, + }, + )?; + return Ok(()); + } + let response = dispatch_stream_frame( + Request { + id: format!("{request_id}:file:{}", header.sequence), + method: Method::PaneGraphicsStreamDirect( + crate::api::schema::PaneGraphicsDirectParams { + pane_id: pane_id.to_owned(), + layer_id: layer_id.map(str::to_owned), + z_index, + owner: owner.to_owned(), + image_width: header.image_width, + image_height: header.image_height, + format: header.format, + path: file.path, + sequence: header.sequence, + revision: header.revision, + placement: header.placement, + }, + ), + }, + api_tx, + Arc::clone(stream_active), + ); + write_text_line_allow_disconnect(stream, &response)?; + if api_response_outcome(&response) != "ok" { + return Ok(()); + } + continue; + } + + let Some(data_length) = header.data_length else { + write_json_line_allow_disconnect( + stream, + &ErrorResponse { + id: request_id.to_string(), + error: ErrorBody { + code: "invalid_frame".into(), + message: "frame requires data_length or file".into(), + }, + }, + )?; + return Ok(()); + }; + if data_length == 0 { write_json_line_allow_disconnect( stream, &ErrorResponse { @@ -206,7 +283,7 @@ fn serve_frames( )?; return Ok(()); } - if header.data_length > crate::api::schema::PANE_GRAPHICS_STREAM_MAX_BYTES { + if data_length > crate::api::schema::PANE_GRAPHICS_STREAM_MAX_BYTES { write_json_line_allow_disconnect( stream, &ErrorResponse { @@ -222,7 +299,7 @@ fn serve_frames( let Some(data) = read_exact( stream, - header.data_length, + data_length, running, stream_active, timeouts.body_idle, @@ -239,6 +316,8 @@ fn serve_frames( id: frame_id, method: Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { pane_id: pane_id.to_string(), + layer_id: layer_id.map(str::to_owned), + z_index, owner: owner.to_string(), format: header.format, image_width: header.image_width, @@ -260,52 +339,6 @@ fn serve_frames( Ok(()) } -fn stream_registry() -> &'static Mutex>> { - REGISTERED_STREAMS.get_or_init(|| Mutex::new(HashMap::new())) -} - -fn register_stream(owner: &str, active: &Arc) { - let Ok(mut streams) = stream_registry().lock() else { - active.store(false, Ordering::Release); - return; - }; - if streams - .insert(owner.to_string(), Arc::downgrade(active)) - .is_none() - { - REGISTERED_STREAM_COUNT.fetch_add(1, Ordering::Release); - } -} - -fn unregister_stream(owner: &str) { - let Ok(mut streams) = stream_registry().lock() else { - return; - }; - if streams.remove(owner).is_some() { - REGISTERED_STREAM_COUNT.fetch_sub(1, Ordering::Release); - } -} - -pub(crate) fn cancel_inactive_streams(mut is_active: impl FnMut(&str) -> bool) { - if REGISTERED_STREAM_COUNT.load(Ordering::Acquire) == 0 { - return; - } - let Ok(mut streams) = stream_registry().lock() else { - return; - }; - let before = streams.len(); - streams.retain(|owner, active| { - let keep = is_active(owner); - if !keep { - if let Some(active) = active.upgrade() { - active.store(false, Ordering::Release); - } - } - keep - }); - REGISTERED_STREAM_COUNT.fetch_sub(before.saturating_sub(streams.len()), Ordering::Release); -} - fn stream_is_running(running: &AtomicBool, stream_active: &AtomicBool) -> bool { running.load(Ordering::Relaxed) && stream_active.load(Ordering::Acquire) } @@ -315,12 +348,20 @@ fn next_owner() -> String { format!("pane.graphics.stream:{}:{id}", std::process::id()) } -fn clear_layer(pane_id: &str, owner: &str, api_tx: &ApiRequestSender) { +fn clear_layer( + pane_id: &str, + layer_id: Option<&str>, + z_index: i32, + owner: &str, + api_tx: &ApiRequestSender, +) { let _response = dispatch_to_app_with_timeout( Request { id: format!("pane.graphics.stream.clear:{pane_id}"), method: Method::PaneGraphicsStreamClose(PaneGraphicsStreamParams { pane_id: pane_id.to_string(), + layer_id: layer_id.map(str::to_owned), + z_index, owner: owner.to_string(), }), }, @@ -574,7 +615,9 @@ fn read_should_retry(err: &io::Error) -> bool { mod tests { use super::*; use crate::api::schema::{ErrorResponse, Method, ResponseResult, SuccessResponse}; - use crate::api::{ApiRequestMessage, EventHub}; + use crate::api::ApiRequestMessage; + #[cfg(unix)] + use crate::api::EventHub; use crate::ipc::LocalStream; use interprocess::local_socket::traits::Listener as _; use std::io::{BufRead, BufReader, Write}; @@ -610,6 +653,42 @@ mod tests { assert!(owner.starts_with("pane.graphics.stream:")); } + fn respond_ok(message: ApiRequestMessage) { + let response = serde_json::to_string(&SuccessResponse { + id: message.request.id, + result: ResponseResult::Ok {}, + }) + .unwrap(); + message.respond_to.send(response).unwrap(); + } + + fn assert_close_and_respond(message: ApiRequestMessage, pane_id: &str, owner: &str) { + match &message.request.method { + Method::PaneGraphicsStreamClose(params) => { + assert_eq!(params.pane_id, pane_id); + assert_eq!(params.owner, owner); + } + other => panic!("unexpected close request: {other:?}"), + } + respond_ok(message); + } + + #[test] + fn browser_file_header_accepts_damage_but_keeps_full_canonical_frame() { + let header: FrameHeader = serde_json::from_str( + r#"{"format":"rgba","image_width":2,"image_height":3,"sequence":7,"revision":8,"file":{"path":"/private/frame"},"damage":{"x":1,"y":1,"width":1,"height":1},"transport":"direct-kitty","placement":{"grid_cols":2,"grid_rows":3}}"#, + ) + .unwrap(); + assert_eq!(header.data_length, None); + assert_eq!(header.file.unwrap().path, "/private/frame"); + assert_eq!((header.sequence, header.revision), (7, 8)); + assert_eq!((header.image_width, header.image_height), (2, 3)); + assert_eq!( + (header.placement.grid_cols, header.placement.grid_rows), + (2, 3) + ); + } + #[test] fn timed_read_skips_reset_after_stream_ends() { let mut reset_called = false; @@ -650,15 +729,7 @@ mod tests { } other => panic!("unexpected open request: {other:?}"), }; - open.respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: open.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); + respond_ok(open); let ack: SuccessResponse = serde_json::from_str(&read_response_line(&mut client)).unwrap(); assert_eq!(ack.id, "stream_1"); @@ -687,36 +758,11 @@ mod tests { } other => panic!("unexpected request: {other:?}"), } - msg.respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: msg.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); + respond_ok(msg); drop(client); running.store(false, Ordering::Relaxed); - let clear = api_rx.blocking_recv().unwrap(); - match &clear.request.method { - Method::PaneGraphicsStreamClose(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_eq!(params.owner, stream_owner); - } - other => panic!("unexpected clear request: {other:?}"), - } - clear - .respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: clear.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); + assert_close_and_respond(api_rx.blocking_recv().unwrap(), "pane_1", &stream_owner); assert!(server_thread.join().unwrap().is_ok()); } @@ -761,24 +807,7 @@ mod tests { assert_eq!(response.id, "stream_2"); assert_eq!(response.error.code, "feature_disabled"); - let close = api_rx.blocking_recv().unwrap(); - match &close.request.method { - Method::PaneGraphicsStreamClose(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_eq!(params.owner, stream_owner); - } - other => panic!("unexpected close request: {other:?}"), - } - close - .respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: close.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); + assert_close_and_respond(api_rx.blocking_recv().unwrap(), "pane_1", &stream_owner); drop(client); running.store(false, Ordering::Relaxed); @@ -798,6 +827,8 @@ mod tests { "stream_timeout".into(), PaneGraphicsStreamParams { pane_id: "pane_1".into(), + layer_id: None, + z_index: 0, owner: String::new(), }, &api_tx, @@ -823,24 +854,7 @@ mod tests { assert_eq!(response.error.code, "server_unavailable"); assert!(response.error.message.contains("timed out")); - let close = api_rx.blocking_recv().unwrap(); - match &close.request.method { - Method::PaneGraphicsStreamClose(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_eq!(params.owner, stream_owner); - } - other => panic!("unexpected close request: {other:?}"), - } - close - .respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: close.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); + assert_close_and_respond(api_rx.blocking_recv().unwrap(), "pane_1", &stream_owner); drop(open); drop(client); @@ -879,15 +893,7 @@ mod tests { }; drop(client); - open.respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: open.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); + respond_ok(open); let (close_tx, close_rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { @@ -897,23 +903,7 @@ mod tests { .recv_timeout(Duration::from_secs(2)) .unwrap() .unwrap(); - match &close.request.method { - Method::PaneGraphicsStreamClose(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_eq!(params.owner, stream_owner); - } - other => panic!("unexpected close request: {other:?}"), - } - close - .respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: close.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); + assert_close_and_respond(close, "pane_1", &stream_owner); running.store(false, Ordering::Relaxed); assert!(server_thread.join().unwrap().is_ok()); @@ -944,75 +934,6 @@ mod tests { stopper.join().unwrap(); } - #[test] - fn inactive_owner_cancels_idle_stream_and_dispatches_close() { - let (mut client, server, _path) = local_stream_pair("graphics-owner-cancel"); - let (api_tx, mut api_rx) = mpsc::unbounded_channel::(); - let running = Arc::new(AtomicBool::new(true)); - let server_running = Arc::clone(&running); - let server_thread = std::thread::spawn(move || { - serve_with_timeouts( - server, - "stream-cancel".into(), - PaneGraphicsStreamParams { - pane_id: "pane_1".into(), - owner: String::new(), - }, - &api_tx, - &server_running, - Duration::from_secs(1), - READ_TIMEOUTS, - ) - }); - - let open = api_rx.blocking_recv().unwrap(); - let owner = match &open.request.method { - Method::PaneGraphicsStreamOpen(params) => params.owner.clone(), - other => panic!("unexpected open request: {other:?}"), - }; - open.respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: open.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); - let ack: SuccessResponse = serde_json::from_str(&read_response_line(&mut client)).unwrap(); - assert_eq!(ack.id, "stream-cancel"); - - cancel_inactive_streams(|registered| registered != owner); - - let (close_tx, close_rx) = std::sync::mpsc::channel(); - std::thread::spawn(move || close_tx.send(api_rx.blocking_recv()).unwrap()); - let close = close_rx - .recv_timeout(Duration::from_secs(1)) - .expect("canceled idle stream should dispatch a close") - .expect("API request channel should remain open"); - match &close.request.method { - Method::PaneGraphicsStreamClose(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_eq!(params.owner, owner); - } - other => panic!("unexpected close request: {other:?}"), - } - close - .respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: close.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); - - drop(client); - running.store(false, Ordering::Relaxed); - assert!(server_thread.join().unwrap().is_ok()); - } - #[test] fn fallback_poll_backoff_preserves_fast_window_then_reaches_poll_ceiling() { let mut backoff = PollBackoff::new(); @@ -1102,6 +1023,8 @@ mod tests { "stream-timeout".into(), PaneGraphicsStreamParams { pane_id: "pane_1".into(), + layer_id: None, + z_index: 0, owner: String::new(), }, &api_tx, @@ -1121,15 +1044,7 @@ mod tests { Method::PaneGraphicsStreamOpen(params) => params.owner.clone(), other => panic!("unexpected open request: {other:?}"), }; - open.respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: open.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); + respond_ok(open); let ack: SuccessResponse = serde_json::from_str(&read_response_line(&mut client)).unwrap(); assert_eq!(ack.id, "stream-timeout"); client.write_all(b"{").unwrap(); @@ -1141,23 +1056,7 @@ mod tests { .recv_timeout(Duration::from_secs(1)) .unwrap() .unwrap(); - match &close.request.method { - Method::PaneGraphicsStreamClose(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_eq!(params.owner, owner); - } - other => panic!("unexpected close request: {other:?}"), - } - close - .respond_to - .send( - serde_json::to_string(&SuccessResponse { - id: close.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(), - ) - .unwrap(); + assert_close_and_respond(close, "pane_1", &owner); let error = server_thread.join().unwrap().unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::TimedOut); @@ -1176,6 +1075,8 @@ mod tests { "stream-oversized", "owner-1", "pane_1", + None, + 0, &api_tx, &server_running, &stream_active, diff --git a/src/app/actions.rs b/src/app/actions.rs index 40707cee..20deab2a 100644 --- a/src/app/actions.rs +++ b/src/app/actions.rs @@ -1662,8 +1662,6 @@ impl AppState { } for pane_id in pane_ids { self.plugin_panes.remove(&pane_id); - self.pane_graphics_layers.remove(&pane_id); - self.pane_graphics_streams.remove(&pane_id); } } @@ -3402,26 +3400,6 @@ mod tests { state } - fn insert_test_pane_graphics_layer(state: &mut AppState, pane_id: PaneId) { - state.pane_graphics_layers.insert( - pane_id, - crate::app::state::PaneGraphicsLayer::new( - crate::api::schema::PaneGraphicsFormat::Rgba, - 1, - 1, - vec![1, 2, 3, 4], - crate::api::schema::PaneGraphicsPlacementParams::default(), - ), - ); - } - - fn insert_test_pane_graphics_state(state: &mut AppState, pane_id: PaneId) { - insert_test_pane_graphics_layer(state, pane_id); - state - .pane_graphics_streams - .insert(pane_id, "test-stream".into()); - } - fn mark_linked_worktree(state: &mut AppState, ws_idx: usize) { state.workspaces[ws_idx].worktree_space = Some(crate::workspace::WorktreeSpaceMembership { key: "repo-key".into(), @@ -5834,13 +5812,9 @@ mod tests { entrypoint: "board".into(), }, ); - insert_test_pane_graphics_state(&mut state, closed); - state.close_pane(); assert_eq!(state.workspaces[0].panes.len(), 1); assert!(!state.plugin_panes.contains_key(&closed)); - assert!(!state.pane_graphics_layers.contains_key(&closed)); - assert!(!state.pane_graphics_streams.contains_key(&closed)); state.assert_invariants_for_test(); } @@ -5910,14 +5884,10 @@ mod tests { entrypoint: "board".into(), }, ); - insert_test_pane_graphics_state(&mut state, pane_id); - state.close_tab(); assert!(!state.terminals.contains_key(&terminal_id)); assert!(!state.plugin_panes.contains_key(&pane_id)); - assert!(!state.pane_graphics_layers.contains_key(&pane_id)); - assert!(!state.pane_graphics_streams.contains_key(&pane_id)); state.assert_invariants_for_test(); } @@ -5933,14 +5903,10 @@ mod tests { entrypoint: "board".into(), }, ); - insert_test_pane_graphics_state(&mut state, pane_id); - state.close_selected_workspace(); assert!(!state.terminals.contains_key(&terminal_id)); assert!(!state.plugin_panes.contains_key(&pane_id)); - assert!(!state.pane_graphics_layers.contains_key(&pane_id)); - assert!(!state.pane_graphics_streams.contains_key(&pane_id)); state.assert_invariants_for_test(); } diff --git a/src/app/api.rs b/src/app/api.rs index 6dc40c3f..0ccd9b21 100644 --- a/src/app/api.rs +++ b/src/app/api.rs @@ -1102,6 +1102,9 @@ impl App { Method::PaneGraphicsStreamSet(params) => { return self.handle_pane_graphics_stream_set(request.id, params); } + Method::PaneGraphicsStreamDirect(params) => { + return self.handle_pane_graphics_stream_direct(request.id, params); + } Method::PaneGraphicsStreamOpen(params) => { return self.handle_pane_graphics_stream_open(request.id, params); } diff --git a/src/app/api/pane_graphics.rs b/src/app/api/pane_graphics.rs index 7851f69a..f5f5f2b7 100644 --- a/src/app/api/pane_graphics.rs +++ b/src/app/api/pane_graphics.rs @@ -2,121 +2,126 @@ use base64::Engine; use crate::api::schema::{ PaneGraphicsClearParams, PaneGraphicsSetParams, PaneGraphicsStreamParams, ResponseResult, - PANE_GRAPHICS_SET_MAX_BYTES, PANE_GRAPHICS_STREAM_MAX_BYTES, + PANE_GRAPHICS_MAX_LAYERS_PER_PANE, PANE_GRAPHICS_PRIMARY_LAYER_ID, PANE_GRAPHICS_SET_MAX_BYTES, + PANE_GRAPHICS_STREAM_MAX_BYTES, }; -use crate::app::state::PaneGraphicsLayer; +use crate::app::pane_graphics::{Key as PaneGraphicsKey, Layer, Slot}; use crate::app::App; use crate::layout::PaneId; use super::responses::{encode_error, encode_success}; impl App { + /// Whether graphics for a pane can be placed on the surface rendered by this client. + /// + /// This deliberately follows the pane layout rather than pane existence: inactive + /// workspaces/tabs and panes hidden by zoom are not placeable. Short-lived UI modes do not + /// suspend the producer because the pane becomes visible again without a layout event. + fn pane_graphics_visible(&self, ws_idx: usize, pane_id: PaneId) -> bool { + if self.state.active != Some(ws_idx) { + return false; + } + let Some(tab) = self.state.workspaces[ws_idx].active_tab() else { + return false; + }; + if tab.zoomed { + tab.layout.focused() == pane_id + } else { + tab.layout.pane_ids().contains(&pane_id) + } + } + pub(super) fn handle_pane_graphics_info( &mut self, id: String, target: crate::api::schema::PaneTarget, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } - if self.parse_pane_id(&target.pane_id).is_none() { + let Some((ws_idx, pane_id)) = self.parse_pane_id(&target.pane_id) else { return pane_not_found(id, &target.pane_id); - } + }; + let pane_visible = self.pane_graphics_visible(ws_idx, pane_id); if !self.state.host_cell_size.is_known() { return encode_error(id, "cell_size_unavailable", "host cell size is unavailable"); } + let file_frame_directory = self + .direct_graphics_available + .then(|| self.pane_graphics_files.source_directory().ok()) + .flatten(); + let direct = file_frame_directory.is_some(); encode_success( id, ResponseResult::PaneGraphicsInfo { cell_width_px: self.state.host_cell_size.width_px, cell_height_px: self.state.host_cell_size.height_px, + pane_visible, + file_frame_directory: file_frame_directory + .map(|directory| directory.to_string_lossy().into_owned()), + file_frame_formats: if direct { + vec!["rgba".into(), "bgra".into()] + } else { + Vec::new() + }, + file_frame_max_bytes: direct.then_some(PANE_GRAPHICS_STREAM_MAX_BYTES), + // Damage metadata never changes the complete canonical frame contract. + file_frame_damage: true, + max_layers_per_pane: PANE_GRAPHICS_MAX_LAYERS_PER_PANE, + pixel_mouse: self.pixel_mouse_available, + file_frame_transport: direct.then(|| "direct-kitty".into()), }, ) } + pub(crate) fn attach_pane_graphics_stream_active( + &mut self, + params: &PaneGraphicsStreamParams, + active: std::sync::Arc, + response: &str, + ) { + if serde_json::from_str::(response).is_err() { + return; + } + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + return; + }; + let Ok(key) = graphics_key(pane_id, params.layer_id.as_deref()) else { + return; + }; + self.pane_graphics + .attach_stream_active(&key, ¶ms.owner, active); + } + pub(super) fn handle_pane_graphics_set( &mut self, id: String, params: PaneGraphicsSetParams, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - if self.state.pane_graphics_streams.contains_key(&pane_id) { + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + self.reclaim_inactive_stream(&key); + if self + .pane_graphics + .slots + .get(&key) + .is_some_and(|slot| slot.stream_owner.is_some()) + { return encode_error( id, "stream_conflict", - "pane already has an active graphics stream", + "pane graphics layer has an active stream", ); } - self.set_pane_graphics_layer(id, pane_id, params) - } - - fn set_pane_graphics_layer( - &mut self, - id: String, - pane_id: PaneId, - params: PaneGraphicsSetParams, - ) -> String { - if params.image_width == 0 || params.image_height == 0 { - return encode_error( - id, - "invalid_image", - "image_width and image_height must be greater than zero", - ); - } - let max_bytes = if params.data.is_some() { - PANE_GRAPHICS_STREAM_MAX_BYTES - } else { - PANE_GRAPHICS_SET_MAX_BYTES - }; - let data = match params.data { - Some(data) => data, - None => match base64::engine::general_purpose::STANDARD.decode(params.data_base64) { - Ok(data) => data, - Err(_) => { - return encode_error(id, "invalid_image", "data_base64 is not valid base64"); - } - }, - }; - if data.is_empty() { - return encode_error(id, "invalid_image", "image data must not be empty"); - } - if data.len() > max_bytes { - return encode_error(id, "image_too_large", "image data is too large"); - } - match pane_graphics_expected_data_len( - params.format, - params.image_width, - params.image_height, - ) { - Ok(Some(expected_len)) if data.len() != expected_len => { - return encode_error( - id, - "invalid_image", - "image data length does not match format and dimensions", - ); - } - Ok(_) => {} - Err(()) => { - return encode_error(id, "invalid_image", "image dimensions are too large"); - } - } - - let layer = PaneGraphicsLayer::new( - params.format, - params.image_width, - params.image_height, - data, - params.placement, - ); - self.state.pane_graphics_layers.insert(pane_id, layer); - self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); - - encode_success(id, ResponseResult::Ok {}) + self.set_layer(id, key, params) } pub(super) fn handle_pane_graphics_clear( @@ -124,23 +129,32 @@ impl App { id: String, params: PaneGraphicsClearParams, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - if self.state.pane_graphics_streams.contains_key(&pane_id) { + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + self.reclaim_inactive_stream(&key); + if self + .pane_graphics + .slots + .get(&key) + .is_some_and(|slot| slot.stream_owner.is_some()) + { return encode_error( id, "stream_conflict", - "pane already has an active graphics stream", + "pane graphics layer has an active stream", ); } - if self.state.pane_graphics_layers.remove(&pane_id).is_some() { - self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); + if self.pane_graphics.slots.remove(&key).is_some() { + self.pane_graphics.mark_changed(); } - encode_success(id, ResponseResult::Ok {}) } @@ -149,24 +163,32 @@ impl App { id: String, params: PaneGraphicsSetParams, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - match self.state.pane_graphics_streams.get(&pane_id) { - Some(owner) if owner == ¶ms.owner => {} - Some(_) => { - return encode_error( - id, - "stream_conflict", - "pane graphics stream owner does not match active stream", - ); + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + match self.pane_graphics.slots.get(&key) { + Some(slot) + if slot.stream_owner.as_ref() == Some(¶ms.owner) && slot.stream_is_active() => + { + self.set_layer(id, key, params) } - None => return encode_error(id, "stream_closed", "pane graphics stream is not active"), + Some(slot) if slot.stream_owner.as_ref() == Some(¶ms.owner) => { + encode_error(id, "stream_closed", "pane graphics stream is not active") + } + Some(_) => encode_error( + id, + "stream_conflict", + "pane graphics stream owner does not match active stream", + ), + None => encode_error(id, "stream_closed", "pane graphics stream is not active"), } - self.set_pane_graphics_layer(id, pane_id, params) } pub(super) fn handle_pane_graphics_stream_open( @@ -174,7 +196,7 @@ impl App { id: String, params: PaneGraphicsStreamParams, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } if params.owner.is_empty() { @@ -184,22 +206,57 @@ impl App { "pane graphics stream owner is required", ); } - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - if self.state.pane_graphics_streams.contains_key(&pane_id) { + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + let stale = self + .pane_graphics + .slots + .get(&key) + .is_some_and(|slot| slot.stream_owner.is_some() && !slot.stream_is_active()); + if stale { + self.pane_graphics.slots.remove(&key); + } + if self + .pane_graphics + .slots + .get(&key) + .and_then(|slot| slot.stream_owner.as_ref()) + .is_some() + { return encode_error( id, "stream_conflict", - "pane already has an active graphics stream", + "pane graphics layer has an active stream", ); } - self.state - .pane_graphics_streams - .insert(pane_id, params.owner); - self.state.pane_graphics_layers.remove(&pane_id); - self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); - + let exists = self.pane_graphics.slots.contains_key(&key); + if !self.pane_graphics.can_add_slot(&key) + || (!exists + && self.pane_graphics.layer_count(pane_id) >= PANE_GRAPHICS_MAX_LAYERS_PER_PANE) + { + return encode_error(id, "layer_limit", "pane graphics layer limit reached"); + } + let Some(host_image_id) = self.pane_graphics.reserve_image_id(&key) else { + return encode_error(id, "layer_limit", "pane graphics image ids are exhausted"); + }; + self.pane_graphics.slots.insert( + key, + Slot { + host_image_id, + layer: None, + stream_owner: Some(params.owner), + stream_active: Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new( + true, + ))), + direct_gate: None, + }, + ); + self.pane_graphics.mark_changed(); encode_success(id, ResponseResult::Ok {}) } @@ -208,55 +265,286 @@ impl App { id: String, params: PaneGraphicsStreamParams, ) -> String { - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - if self - .state - .pane_graphics_streams - .get(&pane_id) - .is_some_and(|owner| owner == ¶ms.owner) - { - self.state.pane_graphics_streams.remove(&pane_id); - self.state.pane_graphics_layers.remove(&pane_id); - self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); + if let Ok(key) = graphics_key(pane_id, params.layer_id.as_deref()) { + if self + .pane_graphics + .slots + .get(&key) + .and_then(|slot| slot.stream_owner.as_ref()) + .is_some_and(|owner| owner == ¶ms.owner) + { + self.pane_graphics.slots.remove(&key); + self.pane_graphics.mark_changed(); + } } - encode_success(id, ResponseResult::Ok {}) } -} -fn require_pane_graphics_enabled(app: &App, id: &str) -> Result<(), String> { - if app.state.kitty_graphics_enabled { - return Ok(()); + pub(super) fn handle_pane_graphics_stream_direct( + &mut self, + id: String, + params: crate::api::schema::PaneGraphicsDirectParams, + ) -> String { + if let Err(response) = require_enabled(self, &id) { + return response; + } + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + return pane_not_found(id, ¶ms.pane_id); + }; + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + let owner_matches = self.pane_graphics.slots.get(&key).is_some_and(|slot| { + slot.stream_owner.as_deref() == Some(params.owner.as_str()) && slot.stream_is_active() + }); + if !owner_matches { + return encode_error(id, "stream_closed", "pane graphics stream is not active"); + } + if params.image_width == 0 || params.image_height == 0 { + return encode_error( + id, + "invalid_image", + "image_width and image_height must be greater than zero", + ); + } + if !matches!( + params.format, + crate::api::schema::PaneGraphicsFormat::Rgba + | crate::api::schema::PaneGraphicsFormat::Bgra + ) { + return encode_error(id, "invalid_image", "direct frames require rgba or bgra"); + } + let expected_len = + match expected_len(params.format, params.image_width, params.image_height) { + Ok(Some(len)) if len <= PANE_GRAPHICS_STREAM_MAX_BYTES => len, + _ => return encode_error(id, "invalid_image", "invalid direct RGBA dimensions"), + }; + let lease = match self + .pane_graphics_files + .lease(std::path::Path::new(¶ms.path), expected_len) + { + Ok(lease) => lease, + Err(err) => return encode_error(id, "invalid_frame_file", err.to_string()), + }; + let primary = key.1 == PANE_GRAPHICS_PRIMARY_LAYER_ID; + let direct = self.direct_graphics_available + && primary + && params.format == crate::api::schema::PaneGraphicsFormat::Rgba; + if !direct && !self.pane_graphics.can_store_inline(&key, expected_len) { + return encode_error( + id, + "graphics_budget_exceeded", + "pane graphics inline memory limit reached", + ); + } + let layer = if direct { + Layer::direct( + params.image_width, + params.image_height, + lease, + params.placement, + params.z_index, + ) + } else { + let mut data = match lease.copy_rgba() { + Ok(data) => data, + Err(err) => return encode_error(id, "invalid_frame_file", err.to_string()), + }; + canonicalize_bgra(params.format, &mut data); + Layer::inline( + crate::api::schema::PaneGraphicsFormat::Rgba, + params.image_width, + params.image_height, + data, + params.placement, + params.z_index, + ) + }; + let Some(slot) = self.pane_graphics.slots.get_mut(&key) else { + return encode_error(id, "stream_closed", "pane graphics stream is not active"); + }; + if !slot.stream_is_active() { + return encode_error(id, "stream_closed", "pane graphics stream is not active"); + } + slot.layer = Some(layer); + slot.direct_gate = None; + self.pane_graphics.mark_changed(); + encode_success( + id, + ResponseResult::PaneGraphicsFrameAck { + sequence: params.sequence, + revision: params.revision, + }, + ) + } + + fn set_layer( + &mut self, + id: String, + key: PaneGraphicsKey, + params: PaneGraphicsSetParams, + ) -> String { + if !self.pane_graphics.can_add_slot(&key) + || (!self.pane_graphics.slots.contains_key(&key) + && self.pane_graphics.layer_count(key.0) >= PANE_GRAPHICS_MAX_LAYERS_PER_PANE) + { + return encode_error(id, "layer_limit", "pane graphics layer limit reached"); + } + if params.image_width == 0 || params.image_height == 0 { + return encode_error( + id, + "invalid_image", + "image_width and image_height must be greater than zero", + ); + } + let expected = match expected_len(params.format, params.image_width, params.image_height) { + Ok(expected) => expected, + Err(()) => return encode_error(id, "invalid_image", "image dimensions are too large"), + }; + let streamed = params.data.is_some(); + let mut data = match params.data { + Some(data) => data, + None => match base64::engine::general_purpose::STANDARD.decode(params.data_base64) { + Ok(data) => data, + Err(_) => { + return encode_error(id, "invalid_image", "data_base64 is not valid base64") + } + }, + }; + let limit = if streamed { + PANE_GRAPHICS_STREAM_MAX_BYTES + } else { + PANE_GRAPHICS_SET_MAX_BYTES + }; + if data.is_empty() { + return encode_error(id, "invalid_image", "image data must not be empty"); + } + if data.len() > limit { + return encode_error(id, "image_too_large", "image data is too large"); + } + if expected.is_some_and(|size| size != data.len()) { + return encode_error( + id, + "invalid_image", + "image data does not match the frame contract", + ); + } + let format = canonicalize_bgra(params.format, &mut data); + if !self.pane_graphics.can_store_inline(&key, data.len()) { + return encode_error( + id, + "graphics_budget_exceeded", + "pane graphics inline memory limit reached", + ); + } + let Some(host_image_id) = self.pane_graphics.reserve_image_id(&key) else { + return encode_error(id, "layer_limit", "pane graphics image ids are exhausted"); + }; + // Move the liveness handle out before replacing the slot so Slot::drop cannot + // deactivate the stream that the replacement still owns. + let (stream_owner, stream_active) = self + .pane_graphics + .slots + .get_mut(&key) + .map(|slot| (slot.stream_owner.take(), slot.stream_active.take())) + .unwrap_or_default(); + self.pane_graphics.slots.insert( + key, + Slot { + host_image_id, + layer: Some(Layer::inline( + format, + params.image_width, + params.image_height, + data, + params.placement, + params.z_index, + )), + stream_owner, + stream_active, + direct_gate: None, + }, + ); + self.pane_graphics.mark_changed(); + encode_success(id, ResponseResult::Ok {}) + } + + fn reclaim_inactive_stream(&mut self, key: &PaneGraphicsKey) { + let stale = self + .pane_graphics + .slots + .get(key) + .is_some_and(|slot| slot.stream_owner.is_some() && !slot.stream_is_active()); + if stale { + self.pane_graphics.slots.remove(key); + self.pane_graphics.mark_changed(); + } } - Err(encode_error( - id.to_owned(), - "feature_disabled", - "pane graphics require experimental.kitty_graphics", - )) } -fn pane_graphics_expected_data_len( +fn canonicalize_bgra( format: crate::api::schema::PaneGraphicsFormat, - image_width: u32, - image_height: u32, + data: &mut [u8], +) -> crate::api::schema::PaneGraphicsFormat { + if format != crate::api::schema::PaneGraphicsFormat::Bgra { + return format; + } + for pixel in data.chunks_exact_mut(4) { + pixel.swap(0, 2); + } + crate::api::schema::PaneGraphicsFormat::Rgba +} + +fn graphics_key(pane_id: PaneId, layer_id: Option<&str>) -> Result { + let layer_id = layer_id.unwrap_or(PANE_GRAPHICS_PRIMARY_LAYER_ID); + if layer_id.is_empty() || layer_id.len() > 64 { + return Err("layer_id must contain between 1 and 64 characters"); + } + if !layer_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) + { + return Err("layer_id contains unsupported characters"); + } + Ok((pane_id, layer_id.to_owned())) +} + +fn expected_len( + format: crate::api::schema::PaneGraphicsFormat, + width: u32, + height: u32, ) -> Result, ()> { - let bytes_per_pixel = match format { + let bytes = match format { crate::api::schema::PaneGraphicsFormat::Png => return Ok(None), crate::api::schema::PaneGraphicsFormat::Rgb => 3_u64, - crate::api::schema::PaneGraphicsFormat::Rgba => 4_u64, + crate::api::schema::PaneGraphicsFormat::Rgba + | crate::api::schema::PaneGraphicsFormat::Bgra => 4_u64, }; - let pixels = u64::from(image_width) - .checked_mul(u64::from(image_height)) - .ok_or(())?; - pixels - .checked_mul(bytes_per_pixel) - .and_then(|bytes| usize::try_from(bytes).ok()) + u64::from(width) + .checked_mul(u64::from(height)) + .and_then(|pixels| pixels.checked_mul(bytes)) + .and_then(|size| usize::try_from(size).ok()) .map(Some) .ok_or(()) } +fn require_enabled(app: &App, id: &str) -> Result<(), String> { + app.state + .kitty_graphics_enabled + .then_some(()) + .ok_or_else(|| { + encode_error( + id.to_owned(), + "feature_disabled", + "pane graphics require experimental.kitty_graphics", + ) + }) +} + fn pane_not_found(id: String, pane_id: &str) -> String { encode_error(id, "pane_not_found", format!("pane {pane_id} not found")) } @@ -268,435 +556,667 @@ mod tests { use crate::config::Config; use crate::workspace::Workspace; - fn app_with_test_workspace() -> (App, String) { - let (_api_tx, api_rx) = tokio::sync::mpsc::unbounded_channel(); + fn app() -> (App, String) { + let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); let mut app = App::new( &Config::default(), true, None, - api_rx, + rx, crate::api::EventHub::default(), ); - app.state.workspaces = vec![Workspace::test_new("pane-graphics")]; + app.state.workspaces = vec![Workspace::test_new("graphics")]; app.state.ensure_test_terminals(); - let pane_id = app.state.workspaces[0].tabs[0].root_pane; - let public_pane_id = app.public_pane_id(0, pane_id).unwrap(); - (app, public_pane_id) + let pane = app.state.workspaces[0].tabs[0].root_pane; + let public = app.public_pane_id(0, pane).unwrap(); + app.state.kitty_graphics_enabled = true; + (app, public) + } + + fn frame( + pane_id: String, + layer_id: Option<&str>, + owner: &str, + value: u8, + ) -> PaneGraphicsSetParams { + PaneGraphicsSetParams { + pane_id, + layer_id: layer_id.map(str::to_owned), + z_index: 2, + owner: owner.to_owned(), + format: crate::api::schema::PaneGraphicsFormat::Rgba, + image_width: 1, + image_height: 1, + data: Some(vec![value; 4]), + data_base64: String::new(), + placement: Default::default(), + } + } + + fn pane_visible(response: &str) -> bool { + serde_json::from_str::(response).unwrap()["result"]["pane_visible"] + .as_bool() + .unwrap() } #[test] - fn api_pane_graphics_info_is_explicit_and_side_effect_free() { - let (mut app, public_pane_id) = app_with_test_workspace(); - let request = |pane_id: String| crate::api::schema::Request { - id: "graphics-info".into(), - method: crate::api::schema::Method::PaneGraphicsInfo(crate::api::schema::PaneTarget { - pane_id, - }), - }; - - let response = app.handle_api_request(request(public_pane_id.clone())); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); - - app.state.kitty_graphics_enabled = true; + fn info_reports_visibility_for_terminal_surface_workspace_tab_and_zoom() { + let (mut app, pane_id) = app(); + app.state.mode = crate::app::Mode::Terminal; + app.state.active = Some(0); app.state.host_cell_size = crate::kitty_graphics::HostCellSize { - width_px: 11, - height_px: 22, + width_px: 10, + height_px: 20, }; - let response = app.handle_api_request(request(public_pane_id)); + let visible = app.handle_pane_graphics_info( + "visible".into(), + crate::api::schema::PaneTarget { + pane_id: pane_id.clone(), + }, + ); + assert!(pane_visible(&visible)); + + let hidden_workspace = Workspace::test_new("hidden"); + let hidden_workspace_pane = hidden_workspace.tabs[0].root_pane; + app.state.workspaces.push(hidden_workspace); + let hidden_workspace_id = app.public_pane_id(1, hidden_workspace_pane).unwrap(); + let hidden = app.handle_pane_graphics_info( + "hidden-workspace".into(), + crate::api::schema::PaneTarget { + pane_id: hidden_workspace_id, + }, + ); + assert!(!pane_visible(&hidden)); + + let inactive_tab = app.state.workspaces[0].test_add_tab(Some("inactive")); + let inactive_pane = app.state.workspaces[0].tabs[inactive_tab].root_pane; + let inactive_id = app.public_pane_id(0, inactive_pane).unwrap(); + let hidden_tab = app.handle_pane_graphics_info( + "hidden-tab".into(), + crate::api::schema::PaneTarget { + pane_id: inactive_id, + }, + ); + assert!(!pane_visible(&hidden_tab)); + + app.state.workspaces[0].test_split(ratatui::layout::Direction::Horizontal); + app.state.workspaces[0].tabs[0].zoomed = true; + let zoomed_away = app.handle_pane_graphics_info( + "zoomed-away".into(), + crate::api::schema::PaneTarget { + pane_id: pane_id.clone(), + }, + ); + assert!(!pane_visible(&zoomed_away)); + + app.state.workspaces[0].tabs[0].zoomed = false; + app.state.mode = crate::app::Mode::Navigate; + let short_lived_mode = app.handle_pane_graphics_info( + "navigate".into(), + crate::api::schema::PaneTarget { pane_id }, + ); + assert!(pane_visible(&short_lived_mode)); + } + + #[test] + fn monolithic_info_keeps_fast_transport_and_exact_pixels_disabled() { + let (mut app, pane_id) = app(); + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + assert!(!app.pane_graphics_files.is_initialized()); + let response = app + .handle_pane_graphics_info("info".into(), crate::api::schema::PaneTarget { pane_id }); let success: SuccessResponse = serde_json::from_str(&response).unwrap(); assert!(matches!( success.result, ResponseResult::PaneGraphicsInfo { - cell_width_px: 11, - cell_height_px: 22, + file_frame_damage: true, + max_layers_per_pane: 16, + pixel_mouse: false, + .. } )); - assert!(app.state.pane_graphics_layers.is_empty()); - assert!(app.state.pane_graphics_streams.is_empty()); + assert!(app.pane_graphics.slots.is_empty()); + assert!(!app.pane_graphics_files.is_initialized()); + } + + #[cfg(unix)] + #[test] + fn info_advertises_rgba_direct_and_bgra_fallback_file_formats_on_demand() { + let (mut app, pane_id) = app(); + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + app.direct_graphics_available = true; + let response = app + .handle_pane_graphics_info("info".into(), crate::api::schema::PaneTarget { pane_id }); + let value: serde_json::Value = serde_json::from_str(&response).unwrap(); + assert!(app.pane_graphics_files.is_initialized()); + assert_eq!( + value["result"]["file_frame_formats"], + serde_json::json!(["rgba", "bgra"]) + ); + assert_eq!(value["result"]["file_frame_damage"], true); + assert_eq!(value["result"]["file_frame_transport"], "direct-kitty"); } #[test] - fn api_pane_graphics_set_and_clear_updates_runtime_layers() { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]); + fn layers_are_isolated_and_stream_owned() { + let (mut app, pane_id) = app(); + for (layer, z, value) in [(None, 0, 1), (Some("chrome"), 10, 2)] { + let mut request = frame(pane_id.clone(), layer, "", value); + request.z_index = z; + let response = app.handle_pane_graphics_set(format!("set-{value}"), request); + assert!(serde_json::from_str::(&response).is_ok()); + } + let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); + assert_eq!( + app.pane_graphics + .slots + .get(&(pane, PANE_GRAPHICS_PRIMARY_LAYER_ID.into())) + .and_then(|slot| slot.layer.as_ref()) + .unwrap() + .z_index, + 0 + ); + assert_eq!( + app.pane_graphics + .slots + .get(&(pane, "chrome".into())) + .and_then(|slot| slot.layer.as_ref()) + .unwrap() + .z_index, + 10 + ); + let response = app.handle_pane_graphics_stream_open( + "open".into(), + PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some("chrome".into()), + z_index: 10, + owner: "stream".into(), + }, + ); + assert!(serde_json::from_str::(&response).is_ok()); + let response = + app.handle_pane_graphics_set("conflict".into(), frame(pane_id, Some("chrome"), "", 3)); + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "stream_conflict"); + } - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: String::new(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: data, - placement: crate::api::schema::PaneGraphicsPlacementParams { - grid_cols: 10, - grid_rows: 4, - ..Default::default() + fn error_code(response: &str) -> String { + serde_json::from_str::(response) + .unwrap() + .error + .code + } + + #[test] + fn validates_layer_and_frame_contracts() { + let cases = [ + (Some("bad space"), 1, 1, vec![1; 4], "invalid_layer_id"), + (None, 0, 1, vec![1; 4], "invalid_image"), + (None, 1, 1, Vec::new(), "invalid_image"), + (None, 1, 1, vec![1; 3], "invalid_image"), + ]; + for (layer, width, height, data, expected) in cases { + let (mut app, pane_id) = app(); + let mut params = frame(pane_id, layer, "", 1); + params.image_width = width; + params.image_height = height; + params.data = Some(data); + let response = app.handle_pane_graphics_set("invalid".into(), params); + assert_eq!(error_code(&response), expected); + assert!(app.pane_graphics.slots.is_empty()); + } + } + + #[test] + fn disabled_requests_have_no_graphics_side_effects() { + let (mut app, pane_id) = app(); + app.state.kitty_graphics_enabled = false; + let revision = app.pane_graphics.revision(); + + let response = app.handle_pane_graphics_set("disabled".into(), frame(pane_id, None, "", 1)); + + assert_eq!(error_code(&response), "feature_disabled"); + assert!(app.pane_graphics.slots.is_empty()); + assert_eq!(app.pane_graphics.revision(), revision); + } + + #[test] + fn stream_owner_controls_only_its_named_layer() { + let (mut app, pane_id) = app(); + let params = |owner: &str| PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some("chrome".into()), + z_index: 7, + owner: owner.into(), + }; + assert!(serde_json::from_str::( + &app.handle_pane_graphics_stream_open("open".into(), params("owner")) + ) + .is_ok()); + assert_eq!( + error_code(&app.handle_pane_graphics_stream_set( + "wrong".into(), + frame(pane_id.clone(), Some("chrome"), "other", 2), + )), + "stream_conflict" + ); + assert!( + serde_json::from_str::(&app.handle_pane_graphics_stream_set( + "frame".into(), + frame(pane_id.clone(), Some("chrome"), "owner", 3), + )) + .is_ok() + ); + app.handle_pane_graphics_stream_close("stale".into(), params("other")); + assert_eq!(app.pane_graphics.slots.len(), 1); + app.handle_pane_graphics_stream_close("close".into(), params("owner")); + assert!(app.pane_graphics.slots.is_empty()); + } + + #[test] + fn static_set_and_clear_reclaim_inactive_stream_layers() { + let (mut app, pane_id) = app(); + let open = |layer: &str| PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some(layer.into()), + z_index: 0, + owner: format!("owner-{layer}"), + }; + for layer in ["set", "clear"] { + assert!(serde_json::from_str::( + &app.handle_pane_graphics_stream_open(layer.into(), open(layer)) + ) + .is_ok()); + let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); + app.pane_graphics.slots[&(pane, layer.into())] + .stream_active + .as_ref() + .unwrap() + .store(false, std::sync::atomic::Ordering::Release); + } + + assert!( + serde_json::from_str::(&app.handle_pane_graphics_set( + "set-static".into(), + frame(pane_id.clone(), Some("set"), "", 4), + )) + .is_ok() + ); + assert!( + serde_json::from_str::(&app.handle_pane_graphics_clear( + "clear-static".into(), + PaneGraphicsClearParams { + pane_id, + layer_id: Some("clear".into()), }, - }), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); + )) + .is_ok() + ); + assert_eq!(app.pane_graphics.slots.len(), 1); + assert!(app + .pane_graphics + .slots + .values() + .all(|slot| slot.stream_owner.is_none())); + } + #[test] + fn layer_limit_counts_empty_and_populated_stream_slots() { + let (mut app, pane_id) = app(); + for index in 0..PANE_GRAPHICS_MAX_LAYERS_PER_PANE { + let layer_id = format!("layer-{index}"); + let response = app.handle_pane_graphics_stream_open( + layer_id.clone(), + PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some(layer_id), + z_index: index as i32, + owner: format!("owner-{index}"), + }, + ); + assert!(serde_json::from_str::(&response).is_ok()); + } + let response = app.handle_pane_graphics_set( + "set-overflow".into(), + frame(pane_id.clone(), Some("set-overflow"), "", 1), + ); + assert_eq!(error_code(&response), "layer_limit"); + let response = app.handle_pane_graphics_stream_open( + "overflow".into(), + PaneGraphicsStreamParams { + pane_id, + layer_id: Some("overflow".into()), + z_index: 0, + owner: "overflow".into(), + }, + ); + assert_eq!(error_code(&response), "layer_limit"); + assert_eq!(app.pane_graphics.slots.len(), 16); + } + + #[test] + fn set_and_clear_preserve_runtime_only_layer_details() { + let (mut app, pane_id) = app(); + let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); + let mut params = frame(pane_id.clone(), None, "", 9); + params.placement.grid_cols = 10; + params.placement.grid_rows = 4; + assert!(serde_json::from_str::( + &app.handle_pane_graphics_set("set".into(), params) + ) + .is_ok()); let layer = app - .state - .pane_graphics_layers - .get(&pane_id) - .expect("graphics layer stored"); - assert_eq!(layer.image_width, 1); - assert_eq!(layer.image_height, 1); - assert_eq!(layer.render.grid_cols, 10); - assert_eq!(layer.render.grid_rows, 4); + .pane_graphics + .slots + .get(&(pane, PANE_GRAPHICS_PRIMARY_LAYER_ID.into())) + .and_then(|slot| slot.layer.as_ref()) + .unwrap(); + assert_eq!(layer.inline_data(), Some([9; 4].as_slice())); + assert_eq!((layer.render.grid_cols, layer.render.grid_rows), (10, 4)); assert!(!app.state.session_dirty); - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-clear".into(), - method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams { - pane_id: public_pane_id, - }), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + assert!( + serde_json::from_str::(&app.handle_pane_graphics_clear( + "clear".into(), + PaneGraphicsClearParams { + pane_id, + layer_id: None, + }, + )) + .is_ok() + ); + assert!(app.pane_graphics.slots.is_empty()); } #[test] - fn api_pane_graphics_rejects_when_kitty_graphics_disabled() { - let (mut app, public_pane_id) = app_with_test_workspace(); - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: String::new(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: data, - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-clear".into(), - method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams { - pane_id: public_pane_id, - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); - } - - #[test] - fn api_pane_graphics_stream_claim_uses_resolved_pane_identity() { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let alias = format!("{public_pane_id}:alias"); - app.state - .public_pane_id_aliases - .insert(alias.clone(), pane_id); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-open".into(), - method: crate::api::schema::Method::PaneGraphicsStreamOpen( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: public_pane_id, - owner: "stream-1".into(), - }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert_eq!( - app.state.pane_graphics_streams.get(&pane_id), - Some(&"stream-1".to_string()) - ); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-open-alias".into(), - method: crate::api::schema::Method::PaneGraphicsStreamOpen( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: alias.clone(), - owner: "stream-2".into(), - }, - ), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "stream_conflict"); - assert_eq!( - app.state.pane_graphics_streams.get(&pane_id), - Some(&"stream-1".to_string()) - ); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-close-wrong-owner".into(), - method: crate::api::schema::Method::PaneGraphicsStreamClose( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: alias.clone(), - owner: "stream-2".into(), - }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert_eq!( - app.state.pane_graphics_streams.get(&pane_id), - Some(&"stream-1".to_string()) - ); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-close-alias".into(), - method: crate::api::schema::Method::PaneGraphicsStreamClose( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: alias, - owner: "stream-1".into(), - }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(!app.state.pane_graphics_streams.contains_key(&pane_id)); - } - - #[test] - fn api_pane_graphics_stream_owns_layer_while_active() { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let public_data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-open".into(), - method: crate::api::schema::Method::PaneGraphicsStreamOpen( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: public_pane_id.clone(), - owner: "stream-1".into(), - }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set-public".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: String::new(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: public_data.clone(), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "stream_conflict"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set-stream".into(), - method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: "stream-1".into(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: public_data, - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(app.state.pane_graphics_layers.contains_key(&pane_id)); - - let wrong_owner_data = base64::engine::general_purpose::STANDARD.encode([5_u8, 6, 7, 8]); - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set-stream-wrong-owner".into(), - method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: "stream-2".into(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: wrong_owner_data, - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "stream_conflict"); - assert_eq!( - app.state.pane_graphics_layers.get(&pane_id).unwrap().data, - vec![1_u8, 2, 3, 4] - ); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-clear-public".into(), - method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams { - pane_id: public_pane_id.clone(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "stream_conflict"); - assert!(app.state.pane_graphics_layers.contains_key(&pane_id)); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-close".into(), - method: crate::api::schema::Method::PaneGraphicsStreamClose( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: public_pane_id, - owner: "stream-1".into(), - }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); - } - - #[test] - fn api_pane_graphics_rejects_overflowing_raw_dimensions() { - for format in [ - crate::api::schema::PaneGraphicsFormat::Rgb, - crate::api::schema::PaneGraphicsFormat::Rgba, + fn raw_formats_require_exact_non_overflowing_lengths() { + for (format, exact) in [ + (crate::api::schema::PaneGraphicsFormat::Rgb, 6), + (crate::api::schema::PaneGraphicsFormat::Rgba, 8), ] { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-overflow".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id, - owner: String::new(), - format, - image_width: u32::MAX, - image_height: u32::MAX, - data: None, - data_base64: base64::engine::general_purpose::STANDARD.encode([1_u8]), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "invalid_image"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); - } - } - - #[test] - fn api_pane_graphics_validates_exact_raw_lengths() { - for (format, exact_len) in [ - (crate::api::schema::PaneGraphicsFormat::Rgb, 6_usize), - (crate::api::schema::PaneGraphicsFormat::Rgba, 8_usize), - ] { - for data_len in [exact_len, exact_len - 1] { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-raw-length".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id, - owner: String::new(), - format, - image_width: 2, - image_height: 1, - data: None, - data_base64: base64::engine::general_purpose::STANDARD - .encode(vec![1_u8; data_len]), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - - if data_len == exact_len { - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(app.state.pane_graphics_layers.contains_key(&pane_id)); + for (length, expected) in [(exact, "ok"), (exact - 1, "invalid_image")] { + let (mut app, pane_id) = app(); + let mut params = frame(pane_id, None, "", 1); + params.format = format; + params.image_width = 2; + params.data = Some(vec![1; length]); + let response = app.handle_pane_graphics_set("length".into(), params); + if expected == "ok" { + assert!(serde_json::from_str::(&response).is_ok()); } else { - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "invalid_image"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + assert_eq!(error_code(&response), expected); } } + let (mut app, pane_id) = app(); + let mut params = frame(pane_id, None, "", 1); + params.format = format; + params.image_width = u32::MAX; + params.image_height = u32::MAX; + assert_eq!( + error_code(&app.handle_pane_graphics_set("overflow".into(), params)), + "invalid_image" + ); } } #[test] - fn api_pane_graphics_uses_transport_specific_limits() { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-public-too-large".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id, - owner: String::new(), - format: crate::api::schema::PaneGraphicsFormat::Png, - image_width: 1, - image_height: 1, - data: None, - data_base64: base64::engine::general_purpose::STANDARD.encode(vec![ - 1_u8; - PANE_GRAPHICS_SET_MAX_BYTES - + 1 - ]), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); + fn public_frames_validate_base64_and_public_size_limit() { + let (mut app, pane_id) = app(); + let mut params = frame(pane_id.clone(), None, "", 1); + params.format = crate::api::schema::PaneGraphicsFormat::Png; + params.data = None; + params.data_base64 = "not base64".into(); + assert_eq!( + error_code(&app.handle_pane_graphics_set("base64".into(), params)), + "invalid_image" + ); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "image_too_large"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + let mut params = frame(pane_id, None, "", 1); + params.format = crate::api::schema::PaneGraphicsFormat::Png; + params.data = None; + params.data_base64 = + base64::engine::general_purpose::STANDARD + .encode(vec![1; PANE_GRAPHICS_SET_MAX_BYTES + 1]); + assert_eq!( + error_code(&app.handle_pane_graphics_set("large".into(), params)), + "image_too_large" + ); } #[test] - fn api_pane_graphics_stream_methods_require_kitty_graphics() { - let (mut app, public_pane_id) = app_with_test_workspace(); - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let open = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-open".into(), - method: crate::api::schema::Method::PaneGraphicsStreamOpen( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: public_pane_id.clone(), - owner: "stream-1".into(), - }, - ), - }); - let error: ErrorResponse = serde_json::from_str(&open).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); + fn aliases_resolve_to_the_same_layer_identity() { + let (mut app, pane_id) = app(); + let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); + let alias = format!("{pane_id}:alias"); + app.state.public_pane_id_aliases.insert(alias.clone(), pane); + let params = |target: String, owner: &str| PaneGraphicsStreamParams { + pane_id: target, + layer_id: None, + z_index: 0, + owner: owner.into(), + }; + assert!(serde_json::from_str::( + &app.handle_pane_graphics_stream_open("open".into(), params(pane_id.clone(), "owner"),) + ) + .is_ok()); + assert_eq!( + error_code(&app.handle_pane_graphics_stream_open( + "conflict".into(), + params(alias.clone(), "other"), + )), + "stream_conflict" + ); + app.handle_pane_graphics_stream_close("stale".into(), params(alias.clone(), "other")); + assert_eq!(app.pane_graphics.slots.len(), 1); + app.handle_pane_graphics_stream_close("close".into(), params(alias, "owner")); + assert!(app.pane_graphics.slots.is_empty()); + } - let set = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-set".into(), - method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { - pane_id: public_pane_id, - owner: "stream-1".into(), - format: crate::api::schema::PaneGraphicsFormat::Png, - image_width: 1, - image_height: 1, - data: Some(vec![1_u8]), - data_base64: String::new(), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&set).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); - assert!(!app.state.pane_graphics_streams.contains_key(&pane_id)); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + #[cfg(unix)] + fn direct_file(app: &App, name: &str, data: &[u8]) -> String { + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + let path = app + .pane_graphics_files + .source_directory() + .unwrap() + .join(name); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .unwrap(); + file.write_all(data).unwrap(); + path.to_string_lossy().into_owned() + } + + #[cfg(unix)] + fn direct_params( + pane_id: String, + owner: &str, + path: String, + ) -> crate::api::schema::PaneGraphicsDirectParams { + crate::api::schema::PaneGraphicsDirectParams { + pane_id, + layer_id: None, + z_index: 0, + owner: owner.into(), + image_width: 1, + image_height: 1, + format: crate::api::schema::PaneGraphicsFormat::Rgba, + path, + sequence: 1, + revision: 1, + placement: Default::default(), + } + } + + #[cfg(unix)] + #[test] + fn direct_file_is_leased_or_safely_copied_for_inline_fallback() { + let (mut app, pane_id) = app(); + let stream = PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: None, + z_index: 0, + owner: "owner".into(), + }; + app.handle_pane_graphics_stream_open("open".into(), stream.clone()); + for (width, height) in [(0, 1), (1, 0)] { + let mut params = direct_params(pane_id.clone(), "owner", "unused".into()); + (params.image_width, params.image_height) = (width, height); + assert_eq!( + error_code(&app.handle_pane_graphics_stream_direct("invalid".into(), params)), + "invalid_image" + ); + } + for format in [ + crate::api::schema::PaneGraphicsFormat::Rgb, + crate::api::schema::PaneGraphicsFormat::Png, + ] { + let mut params = direct_params(pane_id.clone(), "owner", "unused".into()); + params.format = format; + assert_eq!( + error_code(&app.handle_pane_graphics_stream_direct("invalid".into(), params)), + "invalid_image" + ); + } + app.direct_graphics_available = true; + let path = direct_file(&app, "direct", &[1, 2, 3, 4]); + let response = app.handle_pane_graphics_stream_direct( + "direct".into(), + crate::api::schema::PaneGraphicsDirectParams { + sequence: 7, + revision: 8, + ..direct_params(pane_id.clone(), "owner", path) + }, + ); + let ack: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!( + ack.result, + ResponseResult::PaneGraphicsFrameAck { + sequence: 7, + revision: 8 + } + )); + assert!(app + .pane_graphics + .slots + .values() + .next() + .unwrap() + .layer + .as_ref() + .unwrap() + .direct_lease() + .is_some()); + + app.handle_pane_graphics_stream_close("close".into(), stream.clone()); + app.handle_pane_graphics_stream_open("reopen".into(), stream); + app.direct_graphics_available = false; + let path = direct_file(&app, "fallback", &[5, 6, 7, 8]); + app.handle_pane_graphics_stream_direct( + "fallback".into(), + crate::api::schema::PaneGraphicsDirectParams { + sequence: 9, + revision: 10, + ..direct_params(pane_id, "owner", path) + }, + ); + assert_eq!( + app.pane_graphics + .slots + .values() + .next() + .unwrap() + .layer + .as_ref() + .unwrap() + .inline_data(), + Some([5, 6, 7, 8].as_slice()) + ); + } + + #[cfg(unix)] + #[test] + fn bgra_and_secondary_file_frames_are_canonical_owned_rgba() { + let (mut app, pane_id) = app(); + app.direct_graphics_available = true; + for (layer, format, source) in [ + ( + "browser-chrome", + crate::api::schema::PaneGraphicsFormat::Rgba, + [1, 2, 3, 4], + ), + ( + "primary", + crate::api::schema::PaneGraphicsFormat::Bgra, + [3, 2, 1, 4], + ), + ] { + app.handle_pane_graphics_stream_open( + format!("open-{layer}"), + PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some(layer.into()), + z_index: 0, + owner: layer.into(), + }, + ); + let path = direct_file(&app, layer, &source); + app.handle_pane_graphics_stream_direct( + format!("frame-{layer}"), + crate::api::schema::PaneGraphicsDirectParams { + layer_id: Some(layer.into()), + format, + ..direct_params(pane_id.clone(), layer, path) + }, + ); + let key = (app.parse_pane_id(&pane_id).unwrap().1, layer.into()); + assert_eq!( + app.pane_graphics.slots[&key] + .layer + .as_ref() + .unwrap() + .inline_data(), + Some([1, 2, 3, 4].as_slice()) + ); + } + } + + #[test] + fn disabled_info_and_stream_requests_are_rejected() { + let (mut app, pane_id) = app(); + app.state.kitty_graphics_enabled = false; + assert_eq!( + error_code(&app.handle_pane_graphics_info( + "info".into(), + crate::api::schema::PaneTarget { + pane_id: pane_id.clone(), + }, + )), + "feature_disabled" + ); + assert_eq!( + error_code(&app.handle_pane_graphics_stream_open( + "open".into(), + PaneGraphicsStreamParams { + pane_id, + layer_id: None, + z_index: 0, + owner: "owner".into(), + }, + )), + "feature_disabled" + ); + assert!(app.pane_graphics.slots.is_empty()); } } diff --git a/src/app/api/plugins/mod.rs b/src/app/api/plugins/mod.rs index a8cb4405..ddf3fba6 100644 --- a/src/app/api/plugins/mod.rs +++ b/src/app/api/plugins/mod.rs @@ -705,8 +705,10 @@ fn manifest_actions( #[cfg(test)] mod tests { use super::*; + #[cfg(unix)] + use crate::api::schema::PaneListParams; use crate::api::schema::{ - Method, PaneListParams, PluginSourceInfo, PluginSourceKind, Request, SuccessResponse, + Method, PluginSourceInfo, PluginSourceKind, Request, SuccessResponse, }; use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/src/app/ids.rs b/src/app/ids.rs index 4bebedc6..c2fc0a4d 100644 --- a/src/app/ids.rs +++ b/src/app/ids.rs @@ -103,7 +103,7 @@ impl App { None } - pub(super) fn parse_pane_id(&self, id: &str) -> Option<(usize, crate::layout::PaneId)> { + pub(crate) fn parse_pane_id(&self, id: &str) -> Option<(usize, crate::layout::PaneId)> { if let Some(alias) = self.state.public_pane_id_aliases.get(id).copied() { return self.find_pane(alias).map(|(ws_idx, _)| (ws_idx, alias)); } diff --git a/src/app/input/mod.rs b/src/app/input/mod.rs index f77821c9..3202ae26 100644 --- a/src/app/input/mod.rs +++ b/src/app/input/mod.rs @@ -493,15 +493,17 @@ impl App { self.close_popup_pane(); return; }; - let column = mouse.column.saturating_sub(inner.x); - let row = mouse.row.saturating_sub(inner.y); + let position = crate::input::mouse::Position::Cell { + column: mouse.column.saturating_sub(inner.x), + row: mouse.row.saturating_sub(inner.y), + }; let bytes = match mouse.kind { MouseEventKind::ScrollUp | MouseEventKind::ScrollDown | MouseEventKind::ScrollLeft | MouseEventKind::ScrollRight => match rt.wheel_routing() { Some(crate::pane::WheelRouting::MouseReport) => { - rt.encode_mouse_wheel(mouse.kind, column, row, mouse.modifiers) + rt.encode_mouse_wheel(mouse.kind, position, mouse.modifiers) } Some(crate::pane::WheelRouting::AlternateScroll) => { rt.encode_alternate_scroll(mouse.kind) @@ -517,11 +519,9 @@ impl App { } }, MouseEventKind::Down(_) | MouseEventKind::Up(_) | MouseEventKind::Drag(_) => { - rt.encode_mouse_button(mouse.kind, column, row, mouse.modifiers) - } - MouseEventKind::Moved => { - rt.encode_mouse_motion(mouse.kind, column, row, mouse.modifiers) + rt.encode_mouse_button(mouse.kind, position, mouse.modifiers) } + MouseEventKind::Moved => rt.encode_mouse_motion(mouse.kind, position, mouse.modifiers), }; let Some(bytes) = bytes else { return; diff --git a/src/app/input/mouse.rs b/src/app/input/mouse.rs index 2555d95d..8d8988f9 100644 --- a/src/app/input/mouse.rs +++ b/src/app/input/mouse.rs @@ -1667,6 +1667,33 @@ impl AppState { } } + fn pane_mouse_position( + &self, + runtime: &crate::terminal::TerminalRuntime, + inner: Rect, + mouse: MouseEvent, + ) -> Option { + let column = mouse.column.saturating_sub(inner.x); + let row = mouse.row.saturating_sub(inner.y); + let cell = crate::input::mouse::Position::Cell { column, row }; + let Some(host) = self.host_mouse_pixels else { + return Some(cell); + }; + let wants_pixels = runtime.input_state().is_some_and(|state| { + state.mouse_protocol_encoding == crate::input::MouseProtocolEncoding::SgrPixels + }); + if !wants_pixels { + return Some(cell); + } + let Some((width_px, height_px)) = runtime.pixel_size() else { + return Some(cell); + }; + Some( + host.pane_position(inner, width_px, height_px) + .unwrap_or(cell), + ) + } + pub(super) fn forward_pane_mouse_button( &self, terminal_runtimes: &TerminalRuntimeRegistry, @@ -1680,9 +1707,10 @@ impl AppState { else { return false; }; - let column = mouse.column.saturating_sub(info.inner_rect.x); - let row = mouse.row.saturating_sub(info.inner_rect.y); - let Some(bytes) = rt.encode_mouse_button(mouse.kind, column, row, mouse.modifiers) else { + let Some(position) = self.pane_mouse_position(rt, info.inner_rect, mouse) else { + return false; + }; + let Some(bytes) = rt.encode_mouse_button(mouse.kind, position, mouse.modifiers) else { return false; }; rt.scroll_reset(); @@ -1705,9 +1733,10 @@ impl AppState { else { return false; }; - let column = mouse.column.saturating_sub(info.inner_rect.x); - let row = mouse.row.saturating_sub(info.inner_rect.y); - let Some(bytes) = rt.encode_mouse_motion(mouse.kind, column, row, mouse.modifiers) else { + let Some(position) = self.pane_mouse_position(rt, info.inner_rect, mouse) else { + return false; + }; + let Some(bytes) = rt.encode_mouse_motion(mouse.kind, position, mouse.modifiers) else { return false; }; if let Err(err) = rt.try_send_bytes(Bytes::from(bytes)) { @@ -1733,9 +1762,10 @@ impl AppState { return false; } rt.scroll_reset(); - let column = mouse.column.saturating_sub(info.inner_rect.x); - let row = mouse.row.saturating_sub(info.inner_rect.y); - let Some(bytes) = rt.encode_mouse_wheel(mouse.kind, column, row, mouse.modifiers) else { + let Some(position) = self.pane_mouse_position(rt, info.inner_rect, mouse) else { + return false; + }; + let Some(bytes) = rt.encode_mouse_wheel(mouse.kind, position, mouse.modifiers) else { warn!(pane = info.id.raw(), kind = ?mouse.kind, "failed to encode mouse wheel event"); return true; }; @@ -1761,18 +1791,7 @@ impl AppState { match rt.wheel_routing() { Some(crate::pane::WheelRouting::HostScroll) | None => false, Some(crate::pane::WheelRouting::MouseReport) => { - rt.scroll_reset(); - let column = mouse.column.saturating_sub(info.inner_rect.x); - let row = mouse.row.saturating_sub(info.inner_rect.y); - let Some(bytes) = rt.encode_mouse_wheel(mouse.kind, column, row, mouse.modifiers) - else { - warn!(pane = info.id.raw(), kind = ?mouse.kind, "failed to encode mouse wheel event"); - return true; - }; - if let Err(err) = rt.try_send_bytes(Bytes::from(bytes)) { - warn!(pane = info.id.raw(), err = %err, "failed to forward mouse wheel event"); - } - true + self.forward_pane_reported_wheel(terminal_runtimes, info, mouse) } Some(crate::pane::WheelRouting::AlternateScroll) => { rt.scroll_reset(); @@ -2146,7 +2165,7 @@ mod tests { } #[tokio::test] - async fn configured_right_click_passthrough_forwards_full_gesture_to_pane() { + async fn configured_right_click_passthrough_forwards_gesture_outside_pane() { let mut app = app_for_mouse_test(); let mut ws = Workspace::test_new("test"); let pane_id = ws.tabs[0].root_pane; @@ -2177,11 +2196,11 @@ mod tests { }); app.handle_mouse(MouseEvent { modifiers: KeyModifiers::CONTROL, - ..mouse(MouseEventKind::Drag(MouseButton::Right), col + 1, row + 1) + ..mouse(MouseEventKind::Drag(MouseButton::Right), 0, 0) }); app.handle_mouse(MouseEvent { modifiers: KeyModifiers::CONTROL, - ..mouse(MouseEventKind::Up(MouseButton::Right), col + 1, row + 1) + ..mouse(MouseEventKind::Up(MouseButton::Right), 0, 0) }); assert_eq!(app.state.mode, Mode::Terminal); @@ -2193,11 +2212,11 @@ mod tests { ); assert_eq!( input_rx.try_recv().expect("forwarded right mouse drag"), - Bytes::from_static(b"\x1b[<34;4;5M") + Bytes::from_static(b"\x1b[<34;1;1M") ); assert_eq!( input_rx.try_recv().expect("forwarded right mouse up"), - Bytes::from_static(b"\x1b[<2;4;5m") + Bytes::from_static(b"\x1b[<2;1;1m") ); assert!(input_rx.try_recv().is_err()); } @@ -2321,7 +2340,7 @@ mod tests { } #[tokio::test] - async fn mouse_dispatcher_downgrades_sgr_pixel_motion_to_cell_coordinates() { + async fn ordinary_cell_mouse_downgrades_pixel_mode_to_cell_coordinates() { let mut app = app_for_mouse_test(); let mut ws = Workspace::test_new("test"); let pane_id = ws.tabs[0].root_pane; @@ -2339,8 +2358,16 @@ mod tests { app.state.active = Some(0); app.state.selected = 0; app.state.mode = Mode::Terminal; + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; crate::ui::compute_view(&mut app.state, Rect::new(0, 0, 106, 20)); let info = app.state.view.pane_infos[0].clone(); + app.state + .runtime_for_pane_in_workspace(&app.terminal_runtimes, 0, pane_id) + .unwrap() + .resize(info.inner_rect.height, info.inner_rect.width, 10, 20); assert!(info.inner_rect.x > 0, "sidebar offset should be present"); assert!(info.inner_rect.y > 0, "tab bar offset should be present"); @@ -2357,6 +2384,77 @@ mod tests { assert!(input_rx.try_recv().is_err()); } + #[tokio::test] + async fn dedicated_client_pixel_mouse_preserves_subcell_position() { + let mut app = app_for_mouse_test(); + let mut ws = Workspace::test_new("test"); + let pane_id = ws.tabs[0].root_pane; + let (runtime, mut input_rx) = + crate::terminal::TerminalRuntime::test_with_channel_and_scrollback_bytes( + 80, + 18, + 0, + b"\x1b[?1003h\x1b[?1006h\x1b[?1016h", + 4, + ); + ws.insert_test_runtime(pane_id, runtime); + app.state.workspaces = vec![ws]; + app.state.active = Some(0); + app.state.selected = 0; + app.state.mode = Mode::Terminal; + app.state.mouse_capture = false; + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + crate::ui::compute_view(&mut app.state, Rect::new(0, 0, 106, 20)); + let inner = app.state.view.pane_infos[0].inner_rect; + let geometry = crate::input::mouse::HostGeometry::new(106, 20, 1_060, 400).unwrap(); + let x = u32::from(inner.x + 2) * 10 + 8; + let y = u32::from(inner.y + 3) * 20 + 9; + let report = format!("\x1b[<35;{x};{y}M"); + app.state.host_mouse_pixels = Some(crate::input::mouse::HostPixels { x, y, geometry }); + let runtime = app + .state + .runtime_for_pane_in_workspace(&app.terminal_runtimes, 0, pane_id) + .unwrap(); + assert_eq!(runtime.pixel_size(), None); + assert_eq!( + app.state.pane_mouse_position( + runtime, + inner, + mouse(MouseEventKind::Moved, inner.x + 2, inner.y + 3), + ), + Some(crate::input::mouse::Position::Cell { column: 2, row: 3 }) + ); + runtime.resize(inner.height, inner.width, 10, 20); + let runtime = app + .state + .runtime_for_pane_in_workspace(&app.terminal_runtimes, 0, pane_id) + .unwrap(); + assert_eq!( + runtime.pixel_size(), + Some((u32::from(inner.width) * 10, u32::from(inner.height) * 20)) + ); + assert_eq!( + app.state.pane_mouse_position( + runtime, + inner, + mouse(MouseEventKind::Moved, inner.x + 2, inner.y + 3), + ), + Some(crate::input::mouse::Position::Pixels { x: 28, y: 69 }) + ); + app.state.host_mouse_pixels = None; + + assert!(app.route_client_pixel_mouse(7, report.as_bytes(), geometry)); + assert_eq!( + input_rx.try_recv().expect("forwarded exact mouse motion"), + Bytes::from_static(b"\x1b[<35;28;69M") + ); + assert!(input_rx.try_recv().is_err()); + assert!(app.state.host_mouse_pixels.is_none()); + } + #[tokio::test] async fn mouse_dispatcher_does_not_forward_motion_behind_herdr_modes() { let mut app = app_for_mouse_test(); diff --git a/src/app/mod.rs b/src/app/mod.rs index 70d5fe93..15f352af 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -16,6 +16,7 @@ mod creation; mod git_refresh; mod ids; mod input; +pub(crate) mod pane_graphics; mod popup; mod runtime; mod runtime_mutations; @@ -95,6 +96,10 @@ impl PaneClickState { pub struct App { pub state: AppState, + pub(crate) pane_graphics: pane_graphics::Runtime, + pub(crate) pane_graphics_files: Arc, + pub(crate) direct_graphics_available: bool, + pub(crate) pixel_mouse_available: bool, pub(crate) terminal_runtimes: crate::terminal::TerminalRuntimeRegistry, pub event_tx: mpsc::Sender, pub(crate) event_rx: mpsc::Receiver, @@ -672,9 +677,6 @@ impl App { integration_install_messages: Vec::new(), installed_plugins: load_plugin_registry(no_session), plugin_panes: std::collections::HashMap::new(), - pane_graphics_layers: std::collections::HashMap::new(), - pane_graphics_streams: std::collections::HashMap::new(), - pane_graphics_revision: 0, popup_pane: None, plugin_command_logs: Vec::new(), next_plugin_command_log_id: 1, @@ -682,6 +684,7 @@ impl App { global_menu: state::MenuListState::new(0), host_terminal_theme: crate::terminal_theme::TerminalTheme::default(), host_cell_size: crate::kitty_graphics::HostCellSize::default(), + host_mouse_pixels: None, session_dirty: false, terminal_runtime_shutdowns: Vec::new(), }; @@ -726,6 +729,10 @@ impl App { copy_feedback_deadline: None, last_api_notification_at: None, state, + pane_graphics: pane_graphics::Runtime::default(), + pane_graphics_files: Arc::new(crate::pane_graphics_files::FileStore::default()), + direct_graphics_available: false, + pixel_mouse_available: false, terminal_runtimes: restored_terminal_runtimes, event_tx, event_rx, @@ -1033,6 +1040,10 @@ impl App { needs_render = true; } + if self.pane_graphics.retain_live_panes(&self.state) { + needs_render = true; + } + let now = Instant::now(); self.sync_host_mouse_capture(&mut host_mouse_capture_active)?; self.sync_host_keyboard_report_all(&mut host_keyboard_report_all_active)?; @@ -1048,18 +1059,19 @@ impl App { terminal.swap_buffers(); self.full_redraw_pending = false; } - let mut cell_size = crate::kitty_graphics::HostCellSize::default(); + let mut cell_size = self.state.host_cell_size; terminal.draw(|frame| { let area = frame.area(); if kitty_graphics_enabled { - let observed_cell_size = - crate::kitty_graphics::HostCellSize::try_from_terminal(area); - if let Some(observed_cell_size) = observed_cell_size { + if let Some(observed_cell_size) = + crate::kitty_graphics::HostCellSize::try_from_terminal(area) + { self.state.host_cell_size = observed_cell_size; + cell_size = observed_cell_size; + } else if !cell_size.is_known() { + cell_size = + crate::kitty_graphics::HostCellSize::fallback_for_area(area); } - cell_size = observed_cell_size.unwrap_or_else(|| { - crate::kitty_graphics::HostCellSize::fallback_for_area(area) - }); crate::ui::compute_view_with_cell_size( &mut self.state, &self.terminal_runtimes, @@ -1082,6 +1094,7 @@ impl App { if kitty_graphics_enabled { crate::kitty_graphics::paint_local_pane_graphics( &self.state, + &self.pane_graphics, &self.terminal_runtimes, cell_size, )?; @@ -1468,8 +1481,7 @@ impl App { crate::kitty_graphics::set_enabled(config.experimental.kitty_graphics); if was_kitty_graphics_enabled && !config.experimental.kitty_graphics { let _ = crate::kitty_graphics::clear_all_host_graphics(); - self.state.pane_graphics_layers.clear(); - self.state.pane_graphics_streams.clear(); + self.pane_graphics.clear(); self.state.host_cell_size = crate::kitty_graphics::HostCellSize::default(); } self.state.reveal_hidden_cursor_for_cjk_ime = @@ -1663,6 +1675,31 @@ impl App { self.route_client_events(events, true); } + pub(crate) fn route_client_pixel_mouse( + &mut self, + source_id: InputSourceId, + data: &[u8], + geometry: crate::input::mouse::HostGeometry, + ) -> bool { + let Some((x, y)) = crate::input::mouse::parse_report(data) else { + return false; + }; + let Some((column, row)) = geometry.cell(x, y) else { + return false; + }; + let Some(cell_report) = crate::input::mouse::report_at_cell(data, column, row) else { + return false; + }; + let mut events = crate::raw_input::parse_raw_input_bytes_sync(&cell_report); + if events.len() != 1 || !matches!(events[0], crate::raw_input::RawInputEvent::Mouse(_)) { + return false; + } + self.state.host_mouse_pixels = Some(crate::input::mouse::HostPixels { x, y, geometry }); + self.route_client_events_from(source_id, std::mem::take(&mut events), false); + self.state.host_mouse_pixels = None; + true + } + pub(crate) fn route_client_events( &mut self, events: Vec, diff --git a/src/app/pane_graphics.rs b/src/app/pane_graphics.rs new file mode 100644 index 00000000..c1d4841a --- /dev/null +++ b/src/app/pane_graphics.rs @@ -0,0 +1,347 @@ +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; + +use crate::api::schema::{PaneGraphicsFormat, PaneGraphicsPlacementParams}; +use crate::layout::PaneId; + +pub(crate) type Key = (PaneId, String); + +pub(crate) const DIRECT_DELIVERY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +pub(crate) const DIRECT_RESPONSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3); +pub(crate) const DIRECT_OUTER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(9); + +#[derive(Debug)] +pub(crate) enum Backing { + Inline(Vec), + Direct(crate::pane_graphics_files::Lease), + Resident { len: usize, client_id: u64 }, +} + +#[derive(Debug)] +pub(crate) struct Layer { + pub(crate) format: PaneGraphicsFormat, + pub(crate) image_width: u32, + pub(crate) image_height: u32, + pub(crate) backing: Backing, + pub(crate) data_fingerprint: u64, + pub(crate) render: PaneGraphicsPlacementParams, + pub(crate) z_index: i32, +} + +#[derive(Debug)] +pub(crate) struct DirectGate { + pub(crate) transfer_id: u64, + pub(crate) client_id: u64, + pub(crate) deadline: std::time::Instant, + pub(crate) written: bool, + pub(crate) success_response: String, + pub(crate) respond_to: std::sync::mpsc::Sender, +} + +#[derive(Debug)] +pub(crate) struct Slot { + pub(crate) host_image_id: u32, + pub(crate) layer: Option, + pub(crate) stream_owner: Option, + pub(crate) stream_active: Option>, + pub(crate) direct_gate: Option, +} + +impl Slot { + #[cfg(test)] + pub(crate) fn test(host_image_id: u32, layer: Option) -> Self { + Self { + host_image_id, + layer, + stream_owner: None, + stream_active: None, + direct_gate: None, + } + } + + pub(crate) fn stream_is_active(&self) -> bool { + self.stream_active + .as_ref() + .is_some_and(|active| active.load(std::sync::atomic::Ordering::Acquire)) + } +} + +impl Drop for Slot { + fn drop(&mut self) { + if let Some(active) = &self.stream_active { + active.store(false, std::sync::atomic::Ordering::Release); + } + } +} + +#[derive(Debug)] +pub(crate) struct Runtime { + pub(crate) slots: HashMap, + revision: u64, + next_host_image_id: u32, +} + +impl Default for Runtime { + fn default() -> Self { + Self { + slots: HashMap::new(), + revision: 0, + next_host_image_id: 1 << 31, + } + } +} + +impl Runtime { + pub(crate) fn revision(&self) -> u64 { + self.revision + } + + pub(crate) fn mark_changed(&mut self) { + self.revision = self.revision.wrapping_add(1); + } + + pub(crate) fn clear(&mut self) { + self.slots.clear(); + self.mark_changed(); + } + + pub(crate) fn reserve_image_id(&mut self, key: &Key) -> Option { + if let Some(id) = self.slots.get(key).map(|slot| slot.host_image_id) { + return Some(id); + } + for _ in 0..=self.slots.len() { + let id = self.next_host_image_id | (1 << 31); + self.next_host_image_id = id.wrapping_add(1) | (1 << 31); + if self.slots.values().all(|slot| slot.host_image_id != id) { + return Some(id); + } + } + None + } + + pub(crate) fn layer_count(&self, pane_id: PaneId) -> usize { + self.slots.keys().filter(|(id, _)| *id == pane_id).count() + } + + pub(crate) fn can_add_slot(&self, key: &Key) -> bool { + self.slots.contains_key(key) + || self.slots.len() < crate::api::schema::panes::PANE_GRAPHICS_MAX_LAYERS_TOTAL + } + + pub(crate) fn can_store_inline(&self, key: &Key, len: usize) -> bool { + let replaced = self + .slots + .get(key) + .and_then(|slot| slot.layer.as_ref()) + .and_then(Layer::inline_data) + .map_or(0, <[u8]>::len); + self.inline_bytes() + .saturating_sub(replaced) + .saturating_add(len) + <= crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL + } + + fn inline_bytes(&self) -> usize { + self.slots + .values() + .filter_map(|slot| slot.layer.as_ref()) + .filter_map(Layer::inline_data) + .map(<[u8]>::len) + .sum() + } + + pub(crate) fn active_for_pane(&self, pane_id: PaneId) -> bool { + self.slots + .iter() + .any(|((id, _), slot)| *id == pane_id && slot.layer.is_some()) + } + + pub(crate) fn attach_stream_active( + &mut self, + key: &Key, + owner: &str, + active: std::sync::Arc, + ) { + if let Some(slot) = self + .slots + .get_mut(key) + .filter(|slot| slot.stream_owner.as_deref() == Some(owner)) + { + slot.stream_active = Some(active); + } + } + + pub(crate) fn retain_live_panes(&mut self, state: &crate::app::state::AppState) -> bool { + if self.slots.is_empty() { + return false; + } + let before = self.slots.len(); + self.slots.retain(|(pane_id, _), _| { + state + .workspaces + .iter() + .any(|workspace| workspace.pane_state(*pane_id).is_some()) + }); + if self.slots.len() == before { + return false; + } + self.mark_changed(); + true + } +} + +impl Layer { + pub(crate) fn inline( + format: PaneGraphicsFormat, + image_width: u32, + image_height: u32, + data: Vec, + render: PaneGraphicsPlacementParams, + z_index: i32, + ) -> Self { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + data.hash(&mut hasher); + Self { + format, + image_width, + image_height, + backing: Backing::Inline(data), + data_fingerprint: hasher.finish(), + render, + z_index, + } + } + + pub(crate) fn direct( + image_width: u32, + image_height: u32, + lease: crate::pane_graphics_files::Lease, + render: PaneGraphicsPlacementParams, + z_index: i32, + ) -> Self { + let data_fingerprint = lease.fingerprint(); + Self { + format: PaneGraphicsFormat::Rgba, + image_width, + image_height, + backing: Backing::Direct(lease), + data_fingerprint, + render, + z_index, + } + } + + pub(crate) fn data_len(&self) -> usize { + match &self.backing { + Backing::Inline(data) => data.len(), + Backing::Direct(lease) => lease.len(), + Backing::Resident { len, .. } => *len, + } + } + + pub(crate) fn inline_data(&self) -> Option<&[u8]> { + match &self.backing { + Backing::Inline(data) => Some(data), + Backing::Direct(_) | Backing::Resident { .. } => None, + } + } + + pub(crate) fn direct_lease(&self) -> Option<&crate::pane_graphics_files::Lease> { + match &self.backing { + Backing::Direct(lease) => Some(lease), + Backing::Inline(_) | Backing::Resident { .. } => None, + } + } + + pub(crate) fn mark_resident(&mut self, client_id: u64) -> bool { + let Backing::Direct(lease) = &self.backing else { + return false; + }; + self.backing = Backing::Resident { + len: lease.len(), + client_id, + }; + true + } + + pub(crate) fn resident_client(&self) -> Option { + match self.backing { + Backing::Resident { client_id, .. } => Some(client_id), + Backing::Inline(_) | Backing::Direct(_) => None, + } + } + + pub(crate) fn terminal_only(&self) -> bool { + matches!(self.backing, Backing::Direct(_) | Backing::Resident { .. }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stream_liveness_attaches_only_to_the_exact_owned_layer() { + let mut runtime = Runtime::default(); + let first = (PaneId::from_raw(1), "first".into()); + let second = (PaneId::from_raw(2), "second".into()); + for (key, id) in [(&first, 1), (&second, 2)] { + runtime.slots.insert( + key.clone(), + Slot { + host_image_id: (1 << 31) | id, + layer: None, + stream_owner: Some("shared-owner".into()), + stream_active: None, + direct_gate: None, + }, + ); + } + let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + + runtime.attach_stream_active(&second, "shared-owner", active.clone()); + + assert!(runtime.slots[&first].stream_active.is_none()); + assert!(std::sync::Arc::ptr_eq( + runtime.slots[&second].stream_active.as_ref().unwrap(), + &active + )); + } + + #[test] + fn runtime_bounds_total_slots_and_inline_bytes() { + let mut runtime = Runtime::default(); + for index in 0..crate::api::schema::panes::PANE_GRAPHICS_MAX_LAYERS_TOTAL { + let key = (PaneId::from_raw(1), format!("layer-{index}")); + runtime + .slots + .insert(key, Slot::test((1 << 31) | index as u32, None)); + } + assert!(!runtime.can_add_slot(&(PaneId::from_raw(2), "extra".into()))); + + let mut runtime = Runtime::default(); + let key = (PaneId::from_raw(1), "primary".into()); + runtime.slots.insert( + key.clone(), + Slot::test( + 1 << 31, + Some(Layer::inline( + PaneGraphicsFormat::Rgba, + 1, + 1, + vec![0; 4], + PaneGraphicsPlacementParams::default(), + 0, + )), + ), + ); + assert!(runtime.can_store_inline( + &key, + crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL, + )); + assert!(!runtime.can_store_inline( + &(PaneId::from_raw(2), "primary".into()), + crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL, + )); + } +} diff --git a/src/app/runtime.rs b/src/app/runtime.rs index 49420381..31426f87 100644 --- a/src/app/runtime.rs +++ b/src/app/runtime.rs @@ -61,6 +61,11 @@ impl App { msg: crate::api::ApiRequestMessage, ) -> bool { let previous_mode = self.state.mode; + let stream_open = match &msg.request.method { + crate::api::schema::Method::PaneGraphicsStreamOpen(params) => Some(params.clone()), + _ => None, + }; + let stream_active = msg.stream_active.clone(); let mut changed = self.expire_due_metadata(Instant::now()); changed |= crate::api::request_changes_ui(&msg.request); let skip_default_workspace = matches!( @@ -83,6 +88,9 @@ impl App { return changed | deferred_changed; } let response = self.handle_api_request(msg.request); + if let (Some(params), Some(active)) = (stream_open.as_ref(), stream_active) { + self.attach_pane_graphics_stream_active(params, active, &response); + } if !skip_default_workspace { changed |= self.ensure_default_workspace(); } diff --git a/src/app/state.rs b/src/app/state.rs index 04876290..e460f62b 100644 --- a/src/app/state.rs +++ b/src/app/state.rs @@ -4,7 +4,6 @@ use crate::config::{ use crossterm::event::{KeyCode, KeyModifiers}; use ratatui::layout::{Direction, Rect}; use ratatui::style::Color; -use std::hash::{Hash, Hasher}; use crate::detect::AgentState; use crate::layout::{PaneId, PaneInfo, SplitBorder}; @@ -12,49 +11,12 @@ use crate::selection::Selection; pub(crate) type InstalledPluginRegistry = std::collections::HashMap; - #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct PluginPaneRecord { pub plugin_id: String, pub entrypoint: String, } -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct PaneGraphicsLayer { - pub format: crate::api::schema::PaneGraphicsFormat, - pub image_width: u32, - pub image_height: u32, - pub data: Vec, - pub data_fingerprint: u64, - pub render: crate::api::schema::PaneGraphicsPlacementParams, -} - -impl PaneGraphicsLayer { - pub(crate) fn new( - format: crate::api::schema::PaneGraphicsFormat, - image_width: u32, - image_height: u32, - data: Vec, - render: crate::api::schema::PaneGraphicsPlacementParams, - ) -> Self { - let data_fingerprint = pane_graphics_data_fingerprint(&data); - Self { - format, - image_width, - image_height, - data, - data_fingerprint, - render, - } - } -} - -fn pane_graphics_data_fingerprint(data: &[u8]) -> u64 { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - data.hash(&mut hasher); - hasher.finish() -} - #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct PopupPaneState { pub pane_id: PaneId, @@ -1520,12 +1482,6 @@ pub struct AppState { pub(crate) installed_plugins: InstalledPluginRegistry, /// Pane ids opened through the plugin pane API. pub(crate) plugin_panes: std::collections::HashMap, - /// Runtime image layers owned by API clients and composited over panes. - pub(crate) pane_graphics_layers: std::collections::HashMap, - /// Active streaming graphics owner token by pane id. - pub(crate) pane_graphics_streams: std::collections::HashMap, - /// Monotonic marker for accepted pane graphics mutations. - pub(crate) pane_graphics_revision: u64, /// Session-modal terminal popup. This is intentionally outside workspace layouts. pub(crate) popup_pane: Option, /// Recent plugin action/event command executions. @@ -1538,6 +1494,8 @@ pub struct AppState { pub host_terminal_theme: TerminalTheme, /// Last known foreground host terminal cell size in pixels. pub(crate) host_cell_size: crate::kitty_graphics::HostCellSize, + /// Exact pixel provenance only while one confirmed SGR report is dispatched. + pub(crate) host_mouse_pixels: Option, /// Set when a persisted session snapshot would change. pub session_dirty: bool, /// Terminal runtimes that should be shut down by the app/runtime layer @@ -1884,9 +1842,6 @@ impl AppState { integration_install_messages: Vec::new(), installed_plugins: std::collections::HashMap::new(), plugin_panes: std::collections::HashMap::new(), - pane_graphics_layers: std::collections::HashMap::new(), - pane_graphics_streams: std::collections::HashMap::new(), - pane_graphics_revision: 0, popup_pane: None, plugin_command_logs: Vec::new(), next_plugin_command_log_id: 1, @@ -1894,6 +1849,7 @@ impl AppState { global_menu: MenuListState::new(0), host_terminal_theme: TerminalTheme::default(), host_cell_size: crate::kitty_graphics::HostCellSize::default(), + host_mouse_pixels: None, session_dirty: false, terminal_runtime_shutdowns: Vec::new(), } @@ -2000,6 +1956,10 @@ impl AppState { self.context_menu.is_none(), "empty app state must not keep context menu" ); + assert!( + self.host_mouse_pixels.is_none(), + "empty app state must not keep host mouse pixel provenance" + ); return; } diff --git a/src/client/direct_graphics.rs b/src/client/direct_graphics.rs new file mode 100644 index 00000000..1bf10041 --- /dev/null +++ b/src/client/direct_graphics.rs @@ -0,0 +1,393 @@ +const MAX_RESPONSE_BYTES: usize = 4096; +const KITTY_PREFIX: &[u8] = b"\x1b_G"; +const KITTY_SUFFIX: &[u8] = b"\x1b\\"; +const RESPONSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3); +const LATE_RESPONSE_DRAIN: std::time::Duration = std::time::Duration::from_secs(1); + +#[derive(Debug, PartialEq, Eq)] +pub(super) struct Response { + pub(super) transfer_id: u64, + pub(super) image_id: u32, + pub(super) success: bool, +} + +#[derive(Debug, Default)] +pub(super) struct ResponseMatcher { + expected: Option<(u64, u32, Option)>, + retired: Option<(u32, std::time::Instant)>, + active: std::sync::Arc, +} + +impl ResponseMatcher { + pub(super) fn active_handle(&self) -> std::sync::Arc { + self.active.clone() + } + + fn refresh_active(&self) { + self.active.store( + self.expected.is_some() || self.retired.is_some(), + std::sync::atomic::Ordering::Release, + ); + } + pub(super) fn interested(&mut self) -> bool { + self.expire(); + self.expected.is_some() || self.retired.is_some() + } + pub(super) fn arm(&mut self, transfer_id: u64, image_id: u32) -> bool { + self.expire(); + if self.expected.is_some() { + return false; + } + self.expected = Some((transfer_id, image_id, None)); + self.refresh_active(); + true + } + + pub(super) fn start(&mut self, transfer_id: u64) { + self.start_at(transfer_id, std::time::Instant::now()); + } + + fn start_at(&mut self, transfer_id: u64, now: std::time::Instant) { + if let Some((id, _, deadline)) = &mut self.expected { + if *id == transfer_id { + *deadline = Some(now + RESPONSE_TIMEOUT); + } + } + } + + pub(super) fn cancel(&mut self, transfer_id: u64) { + if self.expected.is_some_and(|(id, _, _)| id == transfer_id) { + self.expected = None; + self.refresh_active(); + } + } + + pub(super) fn retire(&mut self, transfer_id: u64) { + if self.expected.is_some_and(|(id, _, _)| id == transfer_id) { + if let Some((_, image_id, _)) = self.expected.take() { + self.retired = Some((image_id, std::time::Instant::now() + LATE_RESPONSE_DRAIN)); + self.refresh_active(); + } + } + } + + pub(super) fn expire(&mut self) { + self.expire_at(std::time::Instant::now()); + } + + fn expire_at(&mut self, now: std::time::Instant) { + if self + .expected + .is_some_and(|(_, _, deadline)| deadline.is_some_and(|deadline| deadline <= now)) + { + if let Some((_, image_id, _)) = self.expected.take() { + self.retired = Some((image_id, now + LATE_RESPONSE_DRAIN)); + } + } + if self.retired.is_some_and(|(_, deadline)| deadline <= now) { + self.retired = None; + } + self.refresh_active(); + } + + pub(super) fn consume(&mut self, bytes: &[u8]) -> Option> { + self.expire(); + if !bytes.starts_with(b"\x1b_G") + || bytes.len() > MAX_RESPONSE_BYTES + || !bytes.ends_with(b"\x1b\\") + { + return None; + } + let payload = &bytes[3..bytes.len() - 2]; + let separator = payload.iter().position(|byte| *byte == b';')?; + if let Some((retired_id, _)) = self.retired { + if matching_response_controls(&payload[..separator], retired_id) { + self.retired = None; + self.refresh_active(); + return Some(None); + } + } + let (transfer_id, image_id, _) = self.expected?; + if !matching_response_controls(&payload[..separator], image_id) { + return None; + } + self.expected = None; + self.refresh_active(); + Some(Some(Response { + transfer_id, + image_id, + success: &payload[separator + 1..] == b"OK", + })) + } +} + +#[derive(Default)] +pub(super) struct InputFilter { + pending: Vec, +} + +impl InputFilter { + pub(super) fn has_pending(&self) -> bool { + !self.pending.is_empty() + } + + pub(super) fn flush_if_inactive(&mut self, matcher: &mut ResponseMatcher) -> Option> { + (self.has_pending() && !matcher.interested()).then(|| std::mem::take(&mut self.pending)) + } + + pub(super) fn push( + &mut self, + bytes: &[u8], + matcher: &mut ResponseMatcher, + ) -> (Vec>, Vec) { + self.pending.extend_from_slice(bytes); + let mut output = Vec::new(); + let mut responses = Vec::new(); + if !matcher.interested() { + output.push(std::mem::take(&mut self.pending)); + return (output, responses); + } + loop { + let Some(start) = self + .pending + .windows(3) + .position(|window| window == KITTY_PREFIX) + else { + let keep = (1..KITTY_PREFIX.len()) + .rev() + .find(|len| self.pending.ends_with(&KITTY_PREFIX[..*len])) + .unwrap_or(0); + let emit = self.pending.len().saturating_sub(keep); + if emit > 0 { + output.push(self.pending.drain(..emit).collect()); + } + break; + }; + if start > 0 { + output.push(self.pending.drain(..start).collect()); + } + let end = self + .pending + .windows(2) + .position(|window| window == KITTY_SUFFIX); + let Some(end) = end else { + if self.pending.len() > MAX_RESPONSE_BYTES { + output.push(std::mem::take(&mut self.pending)); + } + break; + }; + let command: Vec = self.pending.drain(..end + 2).collect(); + match matcher.consume(&command) { + Some(Some(response)) => responses.push(response), + Some(None) => {} + None => output.push(command), + } + } + (output, responses) + } +} + +fn matching_response_controls(bytes: &[u8], expected: u32) -> bool { + let mut matched = false; + for field in bytes.split(|byte| *byte == b',') { + let Some(separator) = field.iter().position(|byte| *byte == b'=') else { + return false; + }; + let (key, value) = field.split_at(separator); + let value = &value[1..]; + if value.is_empty() || !value.iter().all(u8::is_ascii_digit) { + return false; + } + match key { + b"i" if !matched + && std::str::from_utf8(value).ok().and_then(|v| v.parse().ok()) + == Some(expected) => + { + matched = true + } + b"I" | b"p" => {} + _ => return false, + } + } + matched +} + +pub(super) fn valid_control(control: &str, image_id: u32) -> bool { + if control.len() > 1024 || control.contains([';', '\x1b']) { + return false; + } + let mut action = false; + let mut format = false; + let mut image = false; + let mut quiet = false; + let mut cursor = false; + for field in control.split(',') { + let Some((key, value)) = field.split_once('=') else { + return false; + }; + if key == "t" + || !matches!( + key, + "a" | "f" + | "s" + | "v" + | "i" + | "p" + | "c" + | "r" + | "z" + | "C" + | "q" + | "x" + | "y" + | "w" + | "h" + | "X" + | "Y" + ) + { + return false; + } + let numeric = value + .strip_prefix('-') + .unwrap_or(value) + .bytes() + .all(|byte| byte.is_ascii_digit()) + && !value.is_empty(); + if key != "a" && !numeric { + return false; + } + match key { + "a" => action = value == "T", + "f" => format = value == "32", + "i" => image = value.parse() == Ok(image_id), + "q" => quiet = value == "0", + "C" => cursor = value == "1", + _ => {} + } + } + action && format && image && quiet && cursor +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn matches_only_the_armed_image_and_preserves_unrelated_input() { + let mut matcher = ResponseMatcher::default(); + let active = matcher.active_handle(); + assert!(matcher.arm(7, 42)); + assert!(active.load(std::sync::atomic::Ordering::Acquire)); + assert!(!matcher.arm(8, 43)); + assert_eq!(matcher.consume(b"typed"), None); + assert_eq!(matcher.consume(b"\x1b_Gi=41;OK\x1b\\"), None); + assert_eq!( + matcher.consume(b"\x1b_Gi=42;OK\x1b\\"), + Some(Some(Response { + transfer_id: 7, + image_id: 42, + success: true, + })) + ); + assert!(!active.load(std::sync::atomic::Ordering::Acquire)); + } + + #[test] + fn explicit_error_is_reported_and_malformed_responses_are_ignored() { + let mut matcher = ResponseMatcher::default(); + matcher.arm(9, 44); + assert_eq!( + matcher.consume(b"\x1b_Gi=44;ENOENT\x1b\\"), + Some(Some(Response { + transfer_id: 9, + image_id: 44, + success: false, + })) + ); + matcher.arm(10, 45); + assert_eq!(matcher.consume(b"\x1b_Gi=45;broken"), None); + matcher.cancel(10); + matcher.arm(11, 46); + assert_eq!(matcher.consume(b"\x1b_Gi=46oops;OK\x1b\\"), None); + } + + #[test] + fn validated_control_is_one_owned_rgba_transmit_and_display() { + assert!(valid_control( + "a=T,f=32,s=10,v=20,i=42,p=7,c=5,r=6,z=-1,C=1,q=0,x=2", + 42 + )); + for invalid in [ + "a=T,f=24,i=42,C=1,q=0", + "a=T,f=32,i=41,C=1,q=0", + "a=T,t=f,f=32,i=42,C=1,q=0", + "a=p,f=32,i=42,C=1,q=0", + ] { + assert!(!valid_control(invalid, 42), "{invalid}"); + } + } + + #[test] + fn fragmented_response_is_held_before_generic_input_timeout() { + let mut matcher = ResponseMatcher::default(); + let mut filter = InputFilter::default(); + matcher.arm(19, 49); + for foreign in [ + b"\x1b_Ga=p,i=49;OK\x1b\\".as_slice(), + b"\x1b_Gi=49oops;OK\x1b\\", + ] { + let (output, responses) = filter.push(foreign, &mut matcher); + assert_eq!(output, [foreign.to_vec()]); + assert!(responses.is_empty()); + } + let (output, responses) = filter.push(b"typed\x1b_Gi=49;", &mut matcher); + assert_eq!(output, [b"typed".to_vec()]); + assert!(responses.is_empty()); + let (output, responses) = filter.push(b"OK\x1b\\tail", &mut matcher); + assert_eq!(output, [b"tail".to_vec()]); + assert_eq!(responses[0].transfer_id, 19); + assert!(responses[0].success); + } + + #[test] + fn attempted_output_failure_retires_response_without_forwarding_it() { + let mut matcher = ResponseMatcher::default(); + let mut filter = InputFilter::default(); + matcher.arm(20, 50); + matcher.retire(20); + for foreign in [ + b"\x1b_Ga=p,i=50;OK\x1b\\".as_slice(), + b"\x1b_Gi=50oops;OK\x1b\\", + ] { + let (output, responses) = filter.push(foreign, &mut matcher); + assert_eq!(output, [foreign.to_vec()]); + assert!(responses.is_empty()); + } + assert!(filter + .push(b"\x1b_Gi=50;EINVAL\x1b\\", &mut matcher) + .0 + .is_empty()); + } + + #[test] + fn timed_out_expectation_requires_a_new_transfer_before_matching() { + let mut matcher = ResponseMatcher::default(); + let started = std::time::Instant::now(); + assert!(matcher.arm(12, 47)); + matcher.start_at(12, started); + assert!(!matcher.arm(13, 48)); + matcher.expire_at(started + RESPONSE_TIMEOUT); + assert!(matcher.arm(13, 48)); + assert_eq!(matcher.consume(b"typed"), None); + assert_eq!(matcher.consume(b"\x1b_Gi=47;OK\x1b\\"), Some(None)); + assert_eq!( + matcher.consume(b"\x1b_Gi=48;OK\x1b\\"), + Some(Some(Response { + transfer_id: 13, + image_id: 48, + success: true, + })) + ); + } +} diff --git a/src/client/input.rs b/src/client/input.rs index 4dc26eae..456f59b5 100644 --- a/src/client/input.rs +++ b/src/client/input.rs @@ -41,6 +41,9 @@ pub fn stdin_reader_loop( host_color_query_sent: bool, host_cell_size_query_sent: bool, host_mouse_capture_active: Arc, + host_sgr_pixels_active: Arc, + #[cfg(unix)] direct_response: Arc>, + #[cfg(unix)] direct_response_active: Arc, ) { #[cfg(windows)] { @@ -48,6 +51,7 @@ pub fn stdin_reader_loop( host_color_query_sent, host_cell_size_query_sent, host_mouse_capture_active, + host_sgr_pixels_active, ); windows_stdin_reader_loop(event_tx, should_quit); } @@ -59,6 +63,9 @@ pub fn stdin_reader_loop( host_color_query_sent, host_cell_size_query_sent, host_mouse_capture_active, + host_sgr_pixels_active, + direct_response, + direct_response_active, ); } @@ -69,6 +76,9 @@ fn unix_stdin_reader_loop( host_color_query_sent: bool, host_cell_size_query_sent: bool, host_mouse_capture_active: Arc, + host_sgr_pixels_active: Arc, + direct_response: Arc>, + direct_response_active: Arc, ) { let stdin = io::stdin(); let mut reader = stdin.lock(); @@ -83,15 +93,71 @@ fn unix_stdin_reader_loop( framer.host_cell_size_query_sent(); } let mut pending_palette = Vec::new(); + let mut pending_mode = None; + let mut last_geometry = None; + let mut direct_filter = super::direct_graphics::InputFilter::default(); while !should_quit.load(Ordering::Acquire) { + if direct_filter.has_pending() + && stdin_read_ready(&reader, crate::raw_input::RAW_INPUT_IDLE_FLUSH_TIMEOUT_MS) + == Some(false) + { + let released = direct_response + .lock() + .ok() + .and_then(|mut matcher| direct_filter.flush_if_inactive(&mut matcher)); + if let Some(data) = released { + if event_tx + .blocking_send(ClientLoopEvent::StdinInput(data)) + .is_err() + { + return; + } + } + continue; + } match reader.read(&mut scratch) { Ok(0) => break, Ok(n) => { + let sgr_pixels = *pending_mode + .get_or_insert_with(|| host_sgr_pixels_active.load(Ordering::Acquire)); + if sgr_pixels { + last_geometry = retain_geometry( + last_geometry, + crate::input::mouse::HostGeometry::current(), + ); + } + let filtered = filter_direct_input( + &scratch[..n], + &mut direct_filter, + &direct_response, + &direct_response_active, + ); + let chunks = if let Some((raw_chunks, responses)) = filtered { + for response in responses { + if event_tx + .blocking_send(ClientLoopEvent::DirectGraphicsResponse(response)) + .is_err() + { + return; + } + } + raw_chunks + .into_iter() + .flat_map(|chunk| framer.push(&chunk)) + .collect() + } else { + framer.push(&scratch[..n]) + }; + if !framer.has_pending_input() { + pending_mode = None; + } if !send_unix_input_chunks( - framer.push(&scratch[..n]), + chunks, &event_tx, &mut pending_palette, + sgr_pixels, + last_geometry, ) { return; } @@ -104,8 +170,18 @@ fn unix_stdin_reader_loop( let had_pending = framer.has_pending_input(); let chunks = framer.flush_timeout(); let held_escape = had_pending && chunks.is_empty(); - if !send_unix_input_chunks(chunks, &event_tx, &mut pending_palette) - || !flush_unix_palette_input(&event_tx, &mut pending_palette) + let sgr_pixels = pending_mode + .unwrap_or_else(|| host_sgr_pixels_active.load(Ordering::Acquire)); + if !framer.has_pending_input() { + pending_mode = None; + } + if !send_unix_input_chunks( + chunks, + &event_tx, + &mut pending_palette, + sgr_pixels, + last_geometry, + ) || !flush_unix_palette_input(&event_tx, &mut pending_palette) { return; } @@ -114,13 +190,20 @@ fn unix_stdin_reader_loop( &reader, crate::raw_input::RAW_INPUT_IDLE_FLUSH_TIMEOUT_MS, ) == Some(false) - && !send_unix_input_chunks( - framer.flush_timeout(), + { + let chunks = framer.flush_timeout(); + if !framer.has_pending_input() { + pending_mode = None; + } + if !send_unix_input_chunks( + chunks, &event_tx, &mut pending_palette, - ) - { - return; + sgr_pixels, + last_geometry, + ) { + return; + } } } } @@ -134,11 +217,31 @@ fn unix_stdin_reader_loop( } } +#[cfg(unix)] +fn filter_direct_input( + bytes: &[u8], + filter: &mut super::direct_graphics::InputFilter, + response: &std::sync::Mutex, + active: &AtomicBool, +) -> Option<(Vec>, Vec)> { + if !active.load(Ordering::Acquire) && !filter.has_pending() { + return None; + } + Some( + response + .lock() + .map(|mut matcher| filter.push(bytes, &mut matcher)) + .unwrap_or_else(|_| (vec![bytes.to_vec()], Vec::new())), + ) +} + #[cfg(unix)] fn send_unix_input_chunks( chunks: Vec>, event_tx: &mpsc::Sender, pending_palette: &mut Vec>, + sgr_pixels: bool, + geometry: Option, ) -> bool { for data in chunks { let palette_response = std::str::from_utf8(&data) @@ -160,16 +263,36 @@ fn send_unix_input_chunks( if !default_color_response && !flush_unix_palette_input(event_tx, pending_palette) { return false; } - if event_tx - .blocking_send(ClientLoopEvent::StdinInput(data)) - .is_err() - { + let Some(event) = classify_unix_input(data, sgr_pixels, geometry) else { + continue; + }; + if event_tx.blocking_send(event).is_err() { return false; } } true } +#[cfg(unix)] +fn retain_geometry( + last: Option, + observed: Option, +) -> Option { + observed.or(last) +} + +#[cfg(unix)] +fn classify_unix_input( + data: Vec, + sgr_pixels: bool, + geometry: Option, +) -> Option { + if sgr_pixels && crate::input::mouse::parse_report(&data).is_some() { + return geometry.map(|geometry| ClientLoopEvent::PixelMouse(data, geometry)); + } + Some(ClientLoopEvent::StdinInput(data)) +} + #[cfg(unix)] fn flush_unix_palette_input( event_tx: &mpsc::Sender, @@ -495,6 +618,50 @@ mod tests { } } + #[test] + fn inactive_direct_input_bypasses_filter() { + let response = + std::sync::Mutex::new(super::super::direct_graphics::ResponseMatcher::default()); + let active = response.lock().unwrap().active_handle(); + let mut filter = super::super::direct_graphics::InputFilter::default(); + assert!(filter_direct_input(b"typed", &mut filter, &response, &active).is_none()); + assert!(!filter.has_pending()); + } + + #[test] + fn pixel_mouse_classification_is_narrow_and_uses_read_geometry() { + let geometry = crate::input::mouse::HostGeometry::new(80, 24, 800, 480).unwrap(); + let report = b"\x1b[<35;321;241M".to_vec(); + let Some(ClientLoopEvent::PixelMouse(data, captured)) = + classify_unix_input(report.clone(), true, Some(geometry)) + else { + panic!("expected dedicated pixel mouse event"); + }; + assert_eq!(data, report); + assert_eq!(captured, geometry); + assert!(classify_unix_input(report, true, None).is_none()); + + for raw in [ + b"key".as_slice(), + b"\x1b[200~paste\x1b[201~".as_slice(), + b"\x1b_Gi=7;unrelated\x1b\\".as_slice(), + b"\x1b[<35;2;3Mtail".as_slice(), + ] { + let Some(ClientLoopEvent::StdinInput(data)) = + classify_unix_input(raw.to_vec(), true, Some(geometry)) + else { + panic!("unrelated input must remain raw"); + }; + assert_eq!(data, raw); + } + } + + #[test] + fn transient_geometry_failure_keeps_last_real_value() { + let geometry = crate::input::mouse::HostGeometry::new(80, 24, 800, 480).unwrap(); + assert_eq!(retain_geometry(Some(geometry), None), Some(geometry)); + } + #[test] fn palette_replies_are_forwarded_as_one_input_batch() { let (tx, mut rx) = mpsc::channel(4); @@ -506,6 +673,8 @@ mod tests { ], &tx, &mut pending, + false, + None, )); assert!(rx.try_recv().is_err()); diff --git a/src/client/mod.rs b/src/client/mod.rs index 7932cb79..301377c9 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -12,9 +12,13 @@ //! - Forwards OSC 52 clipboard writes from server to its own stdout //! - Displays sound/toast notifications forwarded from server +#[cfg(unix)] +mod direct_graphics; mod input; use std::collections::HashSet; +#[cfg(unix)] +use std::io::IsTerminal as _; use std::io::{self, BufRead, Write as _}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; @@ -75,6 +79,12 @@ struct ClientState { sound_config: crate::config::SoundConfig, /// Whether this client may write Kitty graphics bytes to its host terminal. kitty_graphics_enabled: bool, + /// One bounded matcher, inactive unless a direct transmission is armed. + #[cfg(unix)] + direct_graphics_response: Arc>, + /// One server-retired direct transfer to suppress if it was still queued. + #[cfg(unix)] + retired_direct_graphics: Option<(u64, u32)>, /// Direct attach prefix escape state. None for full-app clients. attach_escape: Option, /// Rows scrolled for one direct-attach wheel notch. @@ -356,9 +366,9 @@ fn setup_terminal_with_capabilities( if enable_client_protocols { if mouse_capture { - set_mouse_capture(true)?; + set_mouse_capture(true, false)?; } else { - set_mouse_capture(false)?; + set_mouse_capture(false, false)?; } execute!(io::stdout(), EnableBracketedPaste, EnableFocusChange)?; if host_color_scheme_reports { @@ -370,9 +380,9 @@ fn setup_terminal_with_capabilities( write_host_color_scheme_report_mode(&mut io::stdout(), false)?; } if mouse_capture { - set_mouse_capture(true)?; + set_mouse_capture(true, false)?; } else { - set_mouse_capture(false)?; + set_mouse_capture(false, false)?; } } @@ -549,10 +559,15 @@ fn restore_windows_input_mode_value(mode: u32) { } } -fn set_mouse_capture(enabled: bool) -> io::Result<()> { +fn set_mouse_capture(enabled: bool, sgr_pixels: bool) -> io::Result<()> { crate::terminal_modes::clear_host_mouse_reporting(&mut io::stdout())?; if enabled { - execute!(io::stdout(), EnableMouseCapture) + execute!(io::stdout(), EnableMouseCapture)?; + if sgr_pixels { + io::stdout().write_all(b"\x1b[?1016h")?; + io::stdout().flush()?; + } + Ok(()) } else { match execute!(io::stdout(), DisableMouseCapture) { Ok(()) => Ok(()), @@ -685,6 +700,44 @@ fn handshake_read_timeout() -> Duration { LOCAL_HANDSHAKE_READ_TIMEOUT } +#[cfg(any(unix, test))] +fn direct_graphics_profile_values( + term_program: &str, + term: &str, + kitty_window: bool, + blocked_transport: bool, + terminals: bool, +) -> bool { + let supported = term_program.eq_ignore_ascii_case("ghostty") + || term_program.eq_ignore_ascii_case("wezterm") + || matches!(term, "xterm-ghostty" | "xterm-kitty" | "xterm-wezterm") + || kitty_window; + supported && !blocked_transport && terminals +} + +#[cfg(unix)] +fn direct_graphics_profile_allowed(direct_attach: bool) -> bool { + let term_program = std::env::var("TERM_PROGRAM").unwrap_or_default(); + let term = std::env::var("TERM").unwrap_or_default(); + direct_graphics_profile_values( + &term_program, + &term, + std::env::var_os("KITTY_WINDOW_ID").is_some(), + direct_attach + || is_remote_client_process() + || std::env::var_os("SSH_CONNECTION").is_some() + || std::env::var_os("SSH_TTY").is_some() + || std::env::var_os("TMUX").is_some() + || std::env::var_os("STY").is_some(), + io::stdin().is_terminal() && io::stdout().is_terminal(), + ) +} + +#[cfg(not(unix))] +fn direct_graphics_profile_allowed(_direct_attach: bool) -> bool { + false +} + fn requested_keybindings() -> ClientKeybindings { match std::env::var(crate::remote::REMOTE_KEYBINDINGS_ENV_VAR) .ok() @@ -726,6 +779,25 @@ fn set_handshake_recv_timeout( .map_err(ClientError::ConnectionFailed) } +fn client_launch_mode( + direct_attach_requested: bool, + exact_cell_size: bool, + cell_width_px: u32, + cell_height_px: u32, +) -> ClientLaunchMode { + if direct_attach_requested { + ClientLaunchMode::TerminalAttach + } else if exact_cell_size + && cell_width_px > 0 + && cell_height_px > 0 + && direct_graphics_profile_allowed(false) + { + ClientLaunchMode::AppDirectGraphics + } else { + ClientLaunchMode::App + } +} + /// Performs the client→server handshake. /// /// Sends Hello with the terminal size and protocol version, reads the Welcome @@ -736,6 +808,7 @@ fn do_handshake( rows: u16, cell_width_px: u32, cell_height_px: u32, + exact_cell_size: bool, requested_encoding: RenderEncoding, direct_attach_requested: bool, ) -> Result { @@ -752,11 +825,12 @@ fn do_handshake( cell_height_px, requested_encoding, keybindings: requested_keybindings(), - launch_mode: if direct_attach_requested { - ClientLaunchMode::TerminalAttach - } else { - ClientLaunchMode::App - }, + launch_mode: client_launch_mode( + direct_attach_requested, + exact_cell_size, + cell_width_px, + cell_height_px, + ), }; protocol::write_message(stream, &hello) .map_err(|e| ClientError::ConnectionFailed(io::Error::other(e.to_string())))?; @@ -801,6 +875,11 @@ enum ClientLoopEvent { /// Raw input bytes from stdin. #[cfg(unix)] StdinInput(Vec), + /// One confirmed SGR pixel report with geometry captured by the reader. + #[cfg(unix)] + PixelMouse(Vec, crate::input::mouse::HostGeometry), + #[cfg(unix)] + DirectGraphicsResponse(direct_graphics::Response), /// Structured input events from platforms without Unix-style stdin bytes. #[cfg(windows)] StdinEvents(Vec), @@ -929,6 +1008,7 @@ fn connect_terminal_session_stream( rows, 0, 0, + false, RenderEncoding::TerminalAnsi, true, ) { @@ -1149,7 +1229,7 @@ fn run_client_with_mode( }; // Get the terminal geometry before handshake (before raw mode). - let (cols, rows, cell_width_px, cell_height_px) = + let (cols, rows, cell_width_px, cell_height_px, exact_cell_size) = initial_terminal_geometry(kitty_graphics_enabled); // Perform handshake while the stream is still in blocking mode. @@ -1159,6 +1239,7 @@ fn run_client_with_mode( rows, cell_width_px, cell_height_px, + exact_cell_size, requested_encoding, direct_attach_requested, ) { @@ -1296,6 +1377,10 @@ async fn run_client_loop( reported_size: (cols, rows), sound_config: config.sound_config, kitty_graphics_enabled: config.kitty_graphics_enabled, + #[cfg(unix)] + direct_graphics_response: Arc::new(Mutex::new(direct_graphics::ResponseMatcher::default())), + #[cfg(unix)] + retired_direct_graphics: None, attach_escape, #[cfg(unix)] mouse_scroll_lines: config.mouse_scroll_lines, @@ -1309,6 +1394,7 @@ async fn run_client_loop( // Cell size reported by the host terminal, packed as width<<32 | height. // Zero means the host has not reported one. let reported_cell_size = Arc::new(AtomicU64::new(0)); + let host_sgr_pixels_active = Arc::new(AtomicBool::new(false)); // Channel for events from the stdin, resize, and server reader threads. let (event_tx, mut event_rx) = tokio::sync::mpsc::channel::(256); @@ -1323,6 +1409,14 @@ async fn run_client_loop( let stdin_quit = should_quit.clone(); let stdin_tx = event_tx.clone(); let stdin_mouse_capture_active = host_mouse_capture_active.clone(); + let stdin_sgr_pixels_active = host_sgr_pixels_active.clone(); + #[cfg(unix)] + let stdin_direct_response = state.direct_graphics_response.clone(); + #[cfg(unix)] + let stdin_direct_response_active = stdin_direct_response + .lock() + .map(|matcher| matcher.active_handle()) + .unwrap_or_default(); std::thread::spawn(move || { input::stdin_reader_loop( stdin_tx, @@ -1330,6 +1424,11 @@ async fn run_client_loop( will_query_host_terminal_theme, will_query_host_cell_size, stdin_mouse_capture_active, + stdin_sgr_pixels_active, + #[cfg(unix)] + stdin_direct_response, + #[cfg(unix)] + stdin_direct_response_active, ); }); @@ -1475,6 +1574,30 @@ async fn run_client_loop( return Err(ClientError::ConnectionLost(e)); } } + #[cfg(unix)] + ClientLoopEvent::DirectGraphicsResponse(response) => { + let message = ClientMessage::GraphicsTransmissionResult { + transfer_id: response.transfer_id, + image_id: response.image_id, + success: response.success, + }; + if let Err(err) = write_to_server(&mut write_stream, &message) { + return Err(ClientError::ConnectionLost(err)); + } + } + #[cfg(unix)] + ClientLoopEvent::PixelMouse(data, geometry) => { + let message = ClientMessage::InputPixels { + data, + cols: geometry.cols, + rows: geometry.rows, + width_px: geometry.width_px, + height_px: geometry.height_px, + }; + if let Err(err) = write_to_server(&mut write_stream, &message) { + return Err(ClientError::ConnectionLost(err)); + } + } #[cfg(windows)] ClientLoopEvent::StdinEvents(events) => { if state.attach_escape.is_some() { @@ -1578,6 +1701,98 @@ async fn run_client_loop( warn!(err = %err, "failed to emit terminal bell"); } } + ServerMessage::GraphicsFile { + path, + expected_len, + image_id, + transfer_id, + leading, + control, + } => { + #[cfg(unix)] + { + if state.retired_direct_graphics.take() == Some((transfer_id, image_id)) { + continue; + } + let valid = state.kitty_graphics_enabled + && usize::try_from(expected_len).ok().is_some_and(|len| { + crate::pane_graphics_files::validate_direct_source( + std::path::Path::new(&path), + len, + ) + .is_ok() + }) + && direct_graphics::valid_control(&control, image_id) + && state + .direct_graphics_response + .lock() + .is_ok_and(|mut matcher| matcher.arm(transfer_id, image_id)); + let sent = if valid { + let mut command = Vec::new(); + crate::kitty_graphics::encode_kitty_regular_file( + &mut command, + &leading, + &control, + &path, + ); + let mut stdout = io::stdout(); + let written = stdout + .write_all(&command) + .and_then(|()| stdout.flush()) + .is_ok(); + if written { + record_received_kitty_graphics(&command); + } + written + } else { + false + }; + if sent { + if let Ok(mut matcher) = state.direct_graphics_response.lock() { + matcher.start(transfer_id); + } + let started = ClientMessage::GraphicsTransmissionStarted { + transfer_id, + image_id, + }; + if let Err(err) = write_to_server(&mut write_stream, &started) { + return Err(ClientError::ConnectionLost(err)); + } + } else { + if let Ok(mut matcher) = state.direct_graphics_response.lock() { + if valid { + matcher.retire(transfer_id); + } else { + matcher.cancel(transfer_id); + } + } + let result = ClientMessage::GraphicsTransmissionResult { + transfer_id, + image_id, + success: false, + }; + if let Err(err) = write_to_server(&mut write_stream, &result) { + return Err(ClientError::ConnectionLost(err)); + } + } + } + #[cfg(not(unix))] + let _ = (path, expected_len, image_id, transfer_id, leading, control); + } + ServerMessage::GraphicsTransmissionRetired { + transfer_id, + image_id, + } => { + #[cfg(unix)] + { + state.retired_direct_graphics = Some((transfer_id, image_id)); + if let Ok(mut matcher) = state.direct_graphics_response.lock() { + matcher.retire(transfer_id); + } + } + #[cfg(not(unix))] + let _ = (transfer_id, image_id); + } ServerMessage::ServerShutdown { reason } => { return Err(ClientError::ServerShutdown { reason }); } @@ -1604,17 +1819,24 @@ async fn run_client_loop( &mut state.remote_image_paste_key, ); } - ServerMessage::MouseCapture { enabled } => { - let desired = enabled; - if desired != state.mouse_capture_active { - set_mouse_capture(desired).map_err(ClientError::ConnectionFailed)?; + ServerMessage::MouseCapture { + enabled, + sgr_pixels, + } => { + let next_sgr_pixels = enabled && sgr_pixels; + let mouse_mode_changed = enabled != state.mouse_capture_active + || next_sgr_pixels != host_sgr_pixels_active.load(Ordering::Acquire); + if mouse_mode_changed { + set_mouse_capture(enabled, next_sgr_pixels) + .map_err(ClientError::ConnectionFailed)?; #[cfg(windows)] - if windows_vti_input_backend_enabled() { + if enabled && windows_vti_input_backend_enabled() { let _ = enable_windows_virtual_terminal_input(); } - state.mouse_capture_active = desired; - host_mouse_capture_active.store(desired, Ordering::Release); } + state.mouse_capture_active = enabled; + host_mouse_capture_active.store(enabled, Ordering::Release); + host_sgr_pixels_active.store(next_sgr_pixels, Ordering::Release); } ServerMessage::KittyKeyboardReportAll { enabled } => { if enabled != state.keyboard_report_all_active { @@ -1643,7 +1865,12 @@ async fn run_client_loop( "server closed connection", ))); } - ClientLoopEvent::Timer => {} + ClientLoopEvent::Timer => { + #[cfg(unix)] + if let Ok(mut matcher) = state.direct_graphics_response.lock() { + matcher.expire(); + } + } } } @@ -2204,8 +2431,10 @@ fn ioctl_cell_size() -> Option<(u32, u32)> { } /// Cell size used when the ioctl reports no pixels. -fn cell_size_fallback(reported: u64) -> (u32, u32) { - unpack_cell_size(reported).unwrap_or((DEFAULT_CELL_WIDTH_PX, DEFAULT_CELL_HEIGHT_PX)) +fn cell_size_fallback(reported: u64, last: Option<(u32, u32)>) -> (u32, u32) { + unpack_cell_size(reported) + .or(last.filter(|(width, height)| *width > 0 && *height > 0)) + .unwrap_or((DEFAULT_CELL_WIDTH_PX, DEFAULT_CELL_HEIGHT_PX)) } #[cfg(any(unix, test))] @@ -2222,20 +2451,35 @@ fn unpack_cell_size(packed: u64) -> Option<(u32, u32)> { fn current_terminal_geometry( kitty_graphics_enabled: bool, reported_cell_size: &AtomicU64, + last_cell_size: Option<(u32, u32)>, ) -> (u16, u16, u32, u32) { let (cols, rows) = crossterm::terminal::size().unwrap_or((80, 24)); if !kitty_graphics_enabled { return (cols, rows, 0, 0); } - let (cell_width_px, cell_height_px) = ioctl_cell_size() - .unwrap_or_else(|| cell_size_fallback(reported_cell_size.load(Ordering::Acquire))); + let (cell_width_px, cell_height_px) = ioctl_cell_size().unwrap_or_else(|| { + cell_size_fallback(reported_cell_size.load(Ordering::Acquire), last_cell_size) + }); (cols, rows, cell_width_px, cell_height_px) } -/// Reads the terminal geometry before the handshake, before any host cell -/// size report can exist. -fn initial_terminal_geometry(kitty_graphics_enabled: bool) -> (u16, u16, u32, u32) { - current_terminal_geometry(kitty_graphics_enabled, &AtomicU64::new(0)) +/// Reads terminal geometry before the handshake. Direct graphics is eligible +/// only when the host supplied exact pixel dimensions through the ioctl. +fn initial_terminal_geometry(kitty_graphics_enabled: bool) -> (u16, u16, u32, u32, bool) { + let (cols, rows) = crossterm::terminal::size().unwrap_or((80, 24)); + if !kitty_graphics_enabled { + return (cols, rows, 0, 0, false); + } + match ioctl_cell_size() { + Some((width, height)) => (cols, rows, width, height, true), + None => ( + cols, + rows, + DEFAULT_CELL_WIDTH_PX, + DEFAULT_CELL_HEIGHT_PX, + false, + ), + } } /// Reports polled changes and signalled resizes that return to the same size. @@ -2272,7 +2516,11 @@ fn resize_poll_loop( while !should_quit.load(Ordering::Acquire) { std::thread::sleep(Duration::from_millis(100)); let signalled = crate::platform::take_terminal_resize_signal(); - let new_size = current_terminal_geometry(kitty_graphics_enabled, reported_cell_size); + let new_size = current_terminal_geometry( + kitty_graphics_enabled, + reported_cell_size, + Some((last_size.2, last_size.3)), + ); if resize_report_required(signalled, new_size, last_size) { last_size = new_size; if resize_tx @@ -2388,6 +2636,40 @@ mod tests { assert!(resize_report_required(false, (120, 40, 9, 18), size)); } + #[test] + fn approximate_cell_size_never_enables_direct_graphics() { + assert_eq!( + client_launch_mode(false, false, 8, 16), + ClientLaunchMode::App + ); + assert_eq!( + client_launch_mode(true, false, 8, 16), + ClientLaunchMode::TerminalAttach + ); + } + + #[test] + fn direct_graphics_profile_is_narrow_and_transport_safe() { + for (program, term, kitty, expected) in [ + ("ghostty", "", false, true), + ("WezTerm", "", false, true), + ("", "xterm-kitty", false, true), + ("", "xterm-256color", true, true), + ("", "xterm-256color", false, false), + ] { + assert_eq!( + direct_graphics_profile_values(program, term, kitty, false, true), + expected + ); + } + assert!(!direct_graphics_profile_values( + "ghostty", "", false, true, true + )); + assert!(!direct_graphics_profile_values( + "ghostty", "", false, false, false + )); + } + fn restore_env_var(key: &str, value: Option) { if let Some(value) = value { std::env::set_var(key, value); @@ -2758,11 +3040,15 @@ mod tests { } #[test] - fn cell_size_fallback_prefers_reported_size_over_default() { - assert_eq!(cell_size_fallback(0), (8, 16)); - assert_eq!(cell_size_fallback(pack_cell_size(10, 21)), (10, 21)); - assert_eq!(cell_size_fallback(pack_cell_size(10, 0)), (8, 16)); - assert_eq!(cell_size_fallback(pack_cell_size(0, 21)), (8, 16)); + fn cell_size_fallback_prefers_reported_then_previous_size() { + assert_eq!(cell_size_fallback(0, None), (8, 16)); + assert_eq!(cell_size_fallback(0, Some((11, 22))), (11, 22)); + assert_eq!( + cell_size_fallback(pack_cell_size(10, 21), Some((11, 22))), + (10, 21) + ); + assert_eq!(cell_size_fallback(pack_cell_size(10, 0), None), (8, 16)); + assert_eq!(cell_size_fallback(pack_cell_size(0, 21), None), (8, 16)); } #[test] diff --git a/src/ghostty/mod.rs b/src/ghostty/mod.rs index 93efbd76..4dead632 100644 --- a/src/ghostty/mod.rs +++ b/src/ghostty/mod.rs @@ -1416,11 +1416,11 @@ impl Terminal { self.get_optional_rgb_color(TERMINAL_DATA_COLOR_CURSOR) } - fn width_px(&self) -> Result { + pub(crate) fn width_px(&self) -> Result { self.get_u32(ffi::GhosttyTerminalData_GHOSTTY_TERMINAL_DATA_WIDTH_PX) } - fn height_px(&self) -> Result { + pub(crate) fn height_px(&self) -> Result { self.get_u32(ffi::GhosttyTerminalData_GHOSTTY_TERMINAL_DATA_HEIGHT_PX) } diff --git a/src/input/encode.rs b/src/input/encode.rs index c475e0da..8ec8a1c5 100644 --- a/src/input/encode.rs +++ b/src/input/encode.rs @@ -125,7 +125,7 @@ fn encode_mouse_cb( encoding: MouseProtocolEncoding, ) -> Option> { let mut cb = match (encoding, release) { - (MouseProtocolEncoding::Sgr, true) => base_button, + (MouseProtocolEncoding::Sgr | MouseProtocolEncoding::SgrPixels, true) => base_button, (_, true) => 3, (_, false) => base_button, }; @@ -143,7 +143,7 @@ fn encode_mouse_cb( let row = row as u32 + 1; match encoding { - MouseProtocolEncoding::Sgr => Some( + MouseProtocolEncoding::Sgr | MouseProtocolEncoding::SgrPixels => Some( format!( "\x1b[<{cb};{column};{row}{}", if release { 'm' } else { 'M' } diff --git a/src/input/mod.rs b/src/input/mod.rs index 0a716f9c..21893303 100644 --- a/src/input/mod.rs +++ b/src/input/mod.rs @@ -1,5 +1,6 @@ mod encode; mod model; +pub(crate) mod mouse; mod parse; #[allow(unused_imports)] diff --git a/src/input/model.rs b/src/input/model.rs index 9c3c14e7..21252d2d 100644 --- a/src/input/model.rs +++ b/src/input/model.rs @@ -317,6 +317,7 @@ pub enum MouseProtocolEncoding { Default, Utf8, Sgr, + SgrPixels, } #[cfg(test)] diff --git a/src/input/mouse.rs b/src/input/mouse.rs new file mode 100644 index 00000000..62b5f6c2 --- /dev/null +++ b/src/input/mouse.rs @@ -0,0 +1,194 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Position { + Cell { column: u16, row: u16 }, + Pixels { x: u32, y: u32 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct HostGeometry { + pub(crate) cols: u16, + pub(crate) rows: u16, + pub(crate) width_px: u32, + pub(crate) height_px: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct HostPixels { + pub(crate) x: u32, + pub(crate) y: u32, + pub(crate) geometry: HostGeometry, +} + +impl HostGeometry { + pub(crate) fn new(cols: u16, rows: u16, width_px: u32, height_px: u32) -> Option { + (cols > 0 && rows > 0 && width_px > 0 && height_px > 0).then_some(Self { + cols, + rows, + width_px, + height_px, + }) + } + + #[cfg(unix)] + pub(crate) fn current() -> Option { + let size = crossterm::terminal::window_size().ok()?; + Self::new( + size.columns, + size.rows, + u32::from(size.width), + u32::from(size.height), + ) + } + + pub(crate) fn cell(self, x: u32, y: u32) -> Option<(u16, u16)> { + Some(( + grid_cell(x.checked_sub(1)?, self.cols, self.width_px)?, + grid_cell(y.checked_sub(1)?, self.rows, self.height_px)?, + )) + } + + fn column_boundary(self, column: u16) -> Option { + boundary(column, self.cols, self.width_px) + } + + fn row_boundary(self, row: u16) -> Option { + boundary(row, self.rows, self.height_px) + } +} + +impl HostPixels { + pub(crate) fn pane_position( + self, + inner: ratatui::layout::Rect, + child_width_px: u32, + child_height_px: u32, + ) -> Option { + let start_x = self.geometry.column_boundary(inner.x)?; + let start_y = self.geometry.row_boundary(inner.y)?; + let end_x = self + .geometry + .column_boundary(inner.x.checked_add(inner.width)?)?; + let end_y = self + .geometry + .row_boundary(inner.y.checked_add(inner.height)?)?; + let x = self.x.checked_sub(1)?.checked_sub(start_x)?; + let y = self.y.checked_sub(1)?.checked_sub(start_y)?; + let source_width = end_x.checked_sub(start_x)?; + let source_height = end_y.checked_sub(start_y)?; + if x >= source_width || y >= source_height || child_width_px == 0 || child_height_px == 0 { + return None; + } + Some(Position::Pixels { + x: scale(x, source_width, child_width_px).checked_add(1)?, + y: scale(y, source_height, child_height_px).checked_add(1)?, + }) + } +} + +pub(crate) fn parse_report(data: &[u8]) -> Option<(u32, u32)> { + let body = data.strip_prefix(b"\x1b[<")?; + let body = body + .strip_suffix(b"M") + .or_else(|| body.strip_suffix(b"m"))?; + let mut fields = body.split(|byte| *byte == b';'); + parse_number(fields.next()?)?; + let x = parse_number(fields.next()?)?; + let y = parse_number(fields.next()?)?; + fields.next().is_none().then_some((x, y)) +} + +pub(crate) fn report_at_cell(data: &[u8], column: u16, row: u16) -> Option> { + let body = data.strip_prefix(b"\x1b[<")?; + let suffix = if body.ends_with(b"M") { 'M' } else { 'm' }; + let body = body.strip_suffix(&[suffix as u8])?; + let buttons = body.split(|byte| *byte == b';').next()?; + Some( + format!( + "\x1b[<{};{};{}{}", + std::str::from_utf8(buttons).ok()?, + u32::from(column) + 1, + u32::from(row) + 1, + suffix + ) + .into_bytes(), + ) +} + +fn parse_number(value: &[u8]) -> Option { + (!value.is_empty() && value.iter().all(u8::is_ascii_digit)) + .then(|| std::str::from_utf8(value).ok()?.parse().ok()) + .flatten() +} + +fn boundary(index: u16, count: u16, extent: u32) -> Option { + (count > 0 && index <= count && extent > 0) + .then(|| (u64::from(index) * u64::from(extent) / u64::from(count)) as u32) +} + +fn grid_cell(pixel: u32, count: u16, extent: u32) -> Option { + if count == 0 || extent == 0 || pixel >= extent { + return None; + } + let cell = ((u64::from(pixel) + 1) * u64::from(count) - 1) / u64::from(extent); + u16::try_from(cell).ok().filter(|cell| *cell < count) +} + +fn scale(pixel: u32, source: u32, target: u32) -> u32 { + ((u64::from(pixel) * u64::from(target)) / u64::from(source)) + .min(u64::from(target.saturating_sub(1))) as u32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parser_accepts_only_complete_sgr_mouse_reports() { + for (input, expected) in [ + (b"\x1b[<35;321;241M".as_slice(), Some((321, 241))), + (b"\x1b[<0;1;2m".as_slice(), Some((1, 2))), + (b"key".as_slice(), None), + (b"\x1b[<0;1;2Mkey".as_slice(), None), + (b"\x1b[<0;1M".as_slice(), None), + ] { + assert_eq!(parse_report(input), expected); + } + } + + #[test] + fn fractional_geometry_maps_exactly_to_pane_pixels() { + let geometry = HostGeometry::new(211, 57, 2_537, 1_429).unwrap(); + let inner = ratatui::layout::Rect::new(157, 7, 53, 49); + let start_x = geometry.column_boundary(inner.x).unwrap(); + let end_x = geometry.column_boundary(inner.x + inner.width).unwrap(); + let start_y = geometry.row_boundary(inner.y).unwrap(); + let end_y = geometry.row_boundary(inner.y + inner.height).unwrap(); + assert_eq!( + HostPixels { + x: start_x + 1, + y: start_y + 1, + geometry, + } + .pane_position(inner, 636, 1_225), + Some(Position::Pixels { x: 1, y: 1 }) + ); + assert_eq!( + HostPixels { + x: end_x, + y: end_y, + geometry, + } + .pane_position(inner, 636, 1_225), + Some(Position::Pixels { x: 636, y: 1_225 }) + ); + } + + #[test] + fn geometry_rejects_outside_pixels_and_maps_cells() { + let geometry = HostGeometry::new(80, 24, 800, 480).unwrap(); + assert_eq!(geometry.cell(1, 1), Some((0, 0))); + assert_eq!(geometry.cell(800, 480), Some((79, 23))); + assert_eq!(geometry.cell(801, 1), None); + assert_eq!(geometry.cell(0, 1), None); + } +} diff --git a/src/kitty_graphics.rs b/src/kitty_graphics.rs index 98577ef8..726e1d8b 100644 --- a/src/kitty_graphics.rs +++ b/src/kitty_graphics.rs @@ -18,7 +18,10 @@ use crate::layout::{PaneId, PaneInfo}; use crate::terminal::TerminalRuntimeRegistry; const KITTY_CHUNK_BYTES: usize = 3072; +pub(crate) const HEADLESS_GRAPHICS_TRANSACTION_BUDGET: usize = + crate::protocol::MAX_GRAPHICS_FRAME_SIZE - crate::protocol::MAX_FRAME_SIZE; const HOST_IMAGE_ID_BASE: u32 = 10_000; +#[cfg(test)] const PANE_GRAPHICS_IMAGE_ID_BIT: u32 = 1 << 31; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -73,6 +76,7 @@ struct HostViewKey { #[derive(Debug)] struct HostPlacement { pane_id: PaneId, + host_image_id: Option, area: Rect, cell_size: HostCellSize, source_key: HostSourceKey, @@ -80,10 +84,10 @@ struct HostPlacement { scrollback_offset: u32, } -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] enum HostSourceKey { Terminal { pane_id: PaneId, image_id: u32 }, - PaneLayer { pane_id: PaneId }, + PaneLayer { pane_id: PaneId, layer_id: String }, } #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] @@ -131,7 +135,11 @@ pub(crate) struct HostGraphicsCache { placements: HashMap<(u32, u32), PlacementSignature>, /// Host image currently backing each (pane, source image id) pair. sources: HashMap, + oversized: HashMap, + continuation: Option<(HostSourceKey, u32, usize)>, view: Option, + replay_placements: bool, + replayed_placements: HashSet<(u32, u32)>, } static KITTY_GRAPHICS_ENABLED: AtomicBool = AtomicBool::new(false); @@ -147,94 +155,130 @@ pub(crate) fn is_enabled() -> bool { pub(crate) fn paint_local_pane_graphics( app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, terminal_runtimes: &TerminalRuntimeRegistry, cell_size: HostCellSize, ) -> io::Result<()> { let cache = LOCAL_HOST_GRAPHICS.get_or_init(|| Mutex::new(HostGraphicsCache::default())); - let mut bytes = Vec::new(); - if let Ok(mut cache) = cache.lock() { - bytes = encode_local_pane_graphics( + let Ok(mut cache) = cache.lock() else { + return Ok(()); + }; + if graphics.slots.is_empty() && !cache.has_pane_sources() { + let encoded = encode_local_pane_graphics( app, + graphics, terminal_runtimes, app.view.tab_surface(), cell_size, + None, &mut cache, ); + drop(cache); + if encoded.bytes.is_empty() { + return Ok(()); + } + let mut stdout = io::stdout().lock(); + stdout.write_all(b"\x1b7")?; + stdout.write_all(&encoded.bytes)?; + stdout.write_all(b"\x1b8")?; + return stdout.flush(); } - if bytes.is_empty() { - return Ok(()); - } - - let mut framed = Vec::with_capacity(bytes.len() + 8); - framed.extend_from_slice(b"\x1b7"); - framed.extend_from_slice(&bytes); - framed.extend_from_slice(b"\x1b8"); let mut stdout = io::stdout().lock(); - stdout.write_all(&framed)?; + loop { + let encoded = encode_local_pane_graphics( + app, + graphics, + terminal_runtimes, + app.view.tab_surface(), + cell_size, + None, + &mut cache, + ); + if !encoded.bytes.is_empty() { + stdout.write_all(b"\x1b7")?; + stdout.write_all(&encoded.bytes)?; + stdout.write_all(b"\x1b8")?; + } + if !encoded.incomplete { + break; + } + } stdout.flush() } +pub(crate) struct EncodedGraphics { + pub(crate) bytes: Vec, + pub(crate) incomplete: bool, +} + pub(crate) fn encode_local_pane_graphics( app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, terminal_runtimes: &TerminalRuntimeRegistry, surface: crate::ui::TabSurfaceView<'_>, cell_size: HostCellSize, + transaction_budget: Option, cache: &mut HostGraphicsCache, -) -> Vec { - let mode_ok = app.mode == Mode::Terminal; - let cell_ok = cell_size.is_known(); - tracing::debug!( - mode_ok, - cell_ok, - cell_width_px = cell_size.width_px, - cell_height_px = cell_size.height_px, - active = ?app.active, - pane_infos_len = surface.pane_infos.len(), - "paint_local_pane_graphics entry" - ); - if !mode_ok || !cell_ok { - tracing::debug!( - reason = if !mode_ok { - "not terminal mode" - } else { - "cell size unknown" - }, - "paint_local_pane_graphics early return" +) -> EncodedGraphics { + let visible = app.mode == Mode::Terminal && cell_size.is_known(); + if graphics.slots.is_empty() { + let mut bytes = cache.clear_pane_sources(); + if !visible { + bytes.extend(cache.clear_bytes()); + return EncodedGraphics { + bytes, + incomplete: false, + }; + } + let placements = collect_visible_placements( + app, + graphics, + terminal_runtimes, + surface, + cell_size, + &cache.images, ); - return cache.clear_bytes(); + let view_changed = cache.update_view(active_view_key(app)); + cache.reset_incremental_state(); + encode_terminal_graphics_update_legacy(&mut bytes, &placements, view_changed, cache); + return EncodedGraphics { + bytes, + incomplete: false, + }; } - let view_key = active_view_key(app); - let placements = - collect_visible_placements(app, terminal_runtimes, surface, cell_size, &cache.images); - tracing::debug!( - placements_collected = placements.len(), - "collect_visible_placements result" - ); - - let mut bytes = Vec::new(); - let view_changed = cache.update_view(view_key); - encode_graphics_update( - &mut bytes, - &placements, - view_changed, - &mut cache.images, - &mut cache.placements, - &mut cache.sources, - ); - tracing::debug!( - placements = placements.len(), - bytes = bytes.len(), - cell_width_px = cell_size.width_px, - cell_height_px = cell_size.height_px, - "painting kitty graphics placements" - ); - bytes + let live_pane_sources = graphics + .slots + .iter() + .filter(|(_, slot)| slot.layer.is_some()) + .map(|((pane_id, layer_id), _)| HostSourceKey::PaneLayer { + pane_id: *pane_id, + layer_id: layer_id.clone(), + }) + .collect::>(); + let placements = if visible { + collect_visible_placements( + app, + graphics, + terminal_runtimes, + surface, + cell_size, + &cache.images, + ) + } else { + Vec::new() + }; + cache.update_view(visible.then(|| active_view_key(app)).flatten()); + // The host text blit overwrites Kitty placements, so every rendered frame must + // display cached images again even when their data and geometry are unchanged. + cache.request_placement_replay(); + encode_graphics_update_incremental(cache, &placements, &live_pane_sources, transaction_budget) } pub(crate) fn has_visible_pane_graphics( app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, terminal_runtimes: &TerminalRuntimeRegistry, surface: crate::ui::TabSurfaceView<'_>, cell_size: HostCellSize, @@ -257,10 +301,20 @@ pub(crate) fn has_visible_pane_graphics( for info in surface.pane_infos { let empty_uploaded = HashMap::new(); - if app.pane_graphics_layers.get(&info.id).is_some_and(|layer| { - let host_placement = - pane_graphics_host_placement(info, cell_size, layer, &empty_uploaded, false); - clipped_placement(&host_placement).is_some() + if graphics.slots.iter().any(|((pane_id, layer_id), slot)| { + *pane_id == info.id + && slot.layer.as_ref().is_some_and(|layer| { + clipped_placement(&pane_graphics_host_placement( + info, + layer_id, + slot.host_image_id, + cell_size, + layer, + &empty_uploaded, + false, + )) + .is_some() + }) }) { return true; } @@ -274,6 +328,7 @@ pub(crate) fn has_visible_pane_graphics( for placement in runtime.kitty_image_placements_with_data_filter(|_| false) { let host_placement = HostPlacement { pane_id: info.id, + host_image_id: None, area: info.inner_rect, cell_size, source_key: HostSourceKey::Terminal { @@ -292,73 +347,41 @@ pub(crate) fn has_visible_pane_graphics( false } -fn encode_graphics_update( +fn encode_terminal_graphics_update_legacy( bytes: &mut Vec, placements: &[HostPlacement], view_changed: bool, - host_images: &mut HashMap, - host_placements: &mut HashMap<(u32, u32), PlacementSignature>, - sources: &mut HashMap, + cache: &mut HostGraphicsCache, ) { - // Prune sources that are no longer visible: a stale entry would keep its - // old host image referenced and block the superseded-image delete. - let current_sources: HashSet = placements + let current_sources = placements .iter() - .filter(|placement| { - matches!(placement.source_key, HostSourceKey::Terminal { .. }) - || clipped_placement(placement).is_some() - }) - .map(|placement| placement.source_key) - .collect(); - let mut removed_pane_layer_images = HashSet::new(); - sources.retain(|source, host_id| { - let retain = current_sources.contains(source); - if !retain && matches!(source, HostSourceKey::PaneLayer { .. }) { - removed_pane_layer_images.insert(*host_id); - } - retain - }); - for host_id in removed_pane_layer_images { - if sources.values().any(|id| *id == host_id) { - continue; - } - encode_delete_image(bytes, host_id); - host_images.remove(&host_id); - host_placements.retain(|(image_id, _), _| *image_id != host_id); - } + .filter(|placement| matches!(placement.source_key, HostSourceKey::Terminal { .. })) + .map(|placement| placement.source_key.clone()) + .collect::>(); + cache + .sources + .retain(|source, _| current_sources.contains(source)); let mut current_placements = HashSet::new(); for placement in placements { - let clipped = clipped_placement(placement); - tracing::debug!( - pane_id = ?placement.pane_id, - has_clipped = clipped.is_some(), - grid_cols = placement.placement.render.grid_cols, - grid_rows = placement.placement.render.grid_rows, - viewport_col = placement.placement.render.viewport_col, - viewport_row = placement.placement.render.viewport_row, - area_w = placement.area.width, - area_h = placement.area.height, - "clipped_placement result" - ); - let Some((clipped, format_code)) = clipped else { + let Some((clipped, format_code)) = clipped_placement(placement) else { continue; }; let host_id = host_image_id(placement.pane_id, &placement.placement); - let host_placement_id = host_placement_id(placement.source_key, &placement.placement); + let placement_id = host_placement_id(&placement.source_key, &placement.placement); let image_signature = image_signature(placement, format_code); let placement_signature = placement_signature(clipped, placement.placement.z, placement.scrollback_offset); - let placement_key = (host_id, host_placement_id); + let placement_key = (host_id, placement_id); current_placements.insert(placement_key); - match host_images.get(&host_id).copied() { + match cache.images.get(&host_id).copied() { Some(existing) if existing == image_signature => {} Some(_) => { encode_delete_image(bytes, host_id); - host_placements.retain(|(image_id, placement_id), _| { + cache.placements.retain(|(image_id, id), _| { if *image_id == host_id { - current_placements.remove(&(*image_id, *placement_id)); + current_placements.remove(&(*image_id, *id)); false } else { true @@ -367,37 +390,32 @@ fn encode_graphics_update( if !encode_upload_image(bytes, placement, format_code, host_id) { continue; } - host_images.insert(host_id, image_signature); + cache.images.insert(host_id, image_signature); } None => { if !encode_upload_image(bytes, placement, format_code, host_id) { continue; } - host_images.insert(host_id, image_signature); + cache.images.insert(host_id, image_signature); } } - release_superseded_source_image( + release_superseded_terminal_image_legacy( bytes, - sources, - host_images, - host_placements, + cache, &mut current_placements, - placement.source_key, + placement.source_key.clone(), host_id, ); - // A different view can repaint the same cells with text or overlays and - // leave the host-side Kitty placement state out of sync with this cache. - // Re-emit the placement even when its geometry signature is unchanged. - match host_placements.get_mut(&placement_key) { + match cache.placements.get_mut(&placement_key) { Some(existing) if !view_changed && *existing == placement_signature => {} Some(existing) => { encode_display_placement( bytes, clipped, host_id, - host_placement_id, + placement_id, placement.placement.z, ); *existing = placement_signature; @@ -407,48 +425,42 @@ fn encode_graphics_update( bytes, clipped, host_id, - host_placement_id, + placement_id, placement.placement.z, ); - host_placements.insert(placement_key, placement_signature); + cache.placements.insert(placement_key, placement_signature); } } } - let mut stale_placements = Vec::new(); - for key in host_placements.keys() { - if current_placements.contains(key) { - continue; - } - stale_placements.push(*key); - } - for (host_id, host_placement_id) in stale_placements { - encode_delete_placement(bytes, host_id, host_placement_id); - host_placements.remove(&(host_id, host_placement_id)); + let stale = cache + .placements + .keys() + .filter(|key| !current_placements.contains(key)) + .copied() + .collect::>(); + for (host_id, placement_id) in stale { + encode_delete_placement(bytes, host_id, placement_id); + cache.placements.remove(&(host_id, placement_id)); } } -/// Records that `source` is now backed by `host_id` and deletes the host -/// image it previously pointed at once no other source references it. -fn release_superseded_source_image( +fn release_superseded_terminal_image_legacy( bytes: &mut Vec, - sources: &mut HashMap, - host_images: &mut HashMap, - host_placements: &mut HashMap<(u32, u32), PlacementSignature>, + cache: &mut HostGraphicsCache, current_placements: &mut HashSet<(u32, u32)>, source: HostSourceKey, host_id: u32, ) { - let Some(previous) = sources.insert(source, host_id) else { + let Some(previous) = cache.sources.insert(source, host_id) else { return; }; - if previous == host_id || sources.values().any(|id| *id == previous) { + if previous == host_id || cache.sources.values().any(|id| *id == previous) { return; } encode_delete_image(bytes, previous); - host_images.remove(&previous); - // The `d=I` delete also removes the image's placements host-side. - host_placements.retain(|(image_id, placement_id), _| { + cache.images.remove(&previous); + cache.placements.retain(|(image_id, placement_id), _| { if *image_id == previous { current_placements.remove(&(*image_id, *placement_id)); false @@ -458,6 +470,320 @@ fn release_superseded_source_image( }); } +fn encode_graphics_update_incremental( + cache: &mut HostGraphicsCache, + placements: &[HostPlacement], + live_pane_sources: &HashSet, + transaction_budget: Option, +) -> EncodedGraphics { + let desired_sources = placements + .iter() + .map(|placement| placement.source_key.clone()) + .collect::>(); + let desired_placements = placements + .iter() + .filter_map(|placement| { + clipped_placement(placement).map(|_| { + let host_id = placement + .host_image_id + .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement)); + ( + host_id, + host_placement_id(&placement.source_key, &placement.placement), + ) + }) + }) + .collect::>(); + let start = cache + .continuation + .as_ref() + .and_then(|(source, id, _)| { + placements + .iter() + .position(|placement| placement_identity(placement) == (source.clone(), *id)) + }) + .map(|index| index + 1) + .or_else(|| cache.continuation.as_ref().map(|cursor| cursor.2)) + .map_or(0, |index| index % placements.len().max(1)); + let mut bytes = Vec::new(); + let mut emitted = false; + + let mut dead_sources = cache + .sources + .keys() + .filter(|source| { + matches!(source, HostSourceKey::PaneLayer { .. }) + && !live_pane_sources.contains(*source) + }) + .cloned() + .collect::>(); + dead_sources.sort_by_key(source_order); + for source in dead_sources { + let host_id = cache.sources[&source]; + let last_reference = !cache + .sources + .iter() + .any(|(other, id)| *other != source && *id == host_id); + if emitted && last_reference { + return EncodedGraphics { + bytes, + incomplete: true, + }; + } + cache.sources.remove(&source); + if last_reference { + encode_delete_image(&mut bytes, host_id); + cache.images.remove(&host_id); + cache.placements.retain(|(id, _), _| *id != host_id); + cache.replayed_placements.retain(|(id, _)| *id != host_id); + emitted = true; + } + } + cache.sources.retain(|source, _| { + matches!(source, HostSourceKey::PaneLayer { .. }) || desired_sources.contains(source) + }); + cache + .oversized + .retain(|source, _| live_pane_sources.contains(source) || desired_sources.contains(source)); + + let mut stale = cache + .placements + .keys() + .filter(|key| !desired_placements.contains(key)) + .copied() + .collect::>(); + stale.sort_unstable(); + for key @ (host_id, placement_id) in stale { + if emitted { + return EncodedGraphics { + bytes, + incomplete: true, + }; + } + encode_delete_placement(&mut bytes, host_id, placement_id); + cache.placements.remove(&key); + cache.replayed_placements.remove(&key); + emitted = true; + } + + for offset in 0..placements.len() { + let index = (start + offset) % placements.len(); + let placement = &placements[index]; + let signature = image_signature(placement, kitty_format_code(placement.placement.format)); + if transaction_budget.is_some() + && cache.oversized.get(&placement.source_key) == Some(&signature) + { + continue; + } + cache.oversized.remove(&placement.source_key); + let host_id = placement + .host_image_id + .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement)); + if cache.images.get(&host_id) != Some(&signature) + && !image_transaction_fits(placement, transaction_budget) + { + cache + .oversized + .insert(placement.source_key.clone(), signature); + continue; + } + let mut candidate = cache.clone(); + let Some(transaction) = encode_placement_update(&mut candidate, placement) else { + continue; + }; + if transaction.is_empty() { + *cache = candidate; + continue; + } + if emitted { + return EncodedGraphics { + bytes, + incomplete: true, + }; + } + *cache = candidate; + let (source, id) = placement_identity(placement); + cache.continuation = Some((source, id, (index + 1) % placements.len())); + bytes = transaction; + emitted = true; + } + + cache.replay_placements = false; + cache.replayed_placements.clear(); + EncodedGraphics { + bytes, + incomplete: false, + } +} + +fn image_transaction_fits(placement: &HostPlacement, budget: Option) -> bool { + let Some(budget) = budget else { + return true; + }; + let data = placement.placement.data_len; + let encoded = data.div_ceil(3).saturating_mul(4); + let command_overhead = data.div_ceil(KITTY_CHUNK_BYTES).saturating_mul(16) + 1024; + encoded.saturating_add(command_overhead) <= budget +} + +fn placement_identity(placement: &HostPlacement) -> (HostSourceKey, u32) { + ( + placement.source_key.clone(), + host_placement_id(&placement.source_key, &placement.placement), + ) +} + +fn source_order(source: &HostSourceKey) -> (u32, String) { + match source { + HostSourceKey::Terminal { pane_id, .. } => (pane_id.raw(), String::new()), + HostSourceKey::PaneLayer { pane_id, layer_id } => (pane_id.raw(), layer_id.clone()), + } +} + +fn encode_placement_update( + cache: &mut HostGraphicsCache, + placement: &HostPlacement, +) -> Option> { + let (clipped, format_code) = clipped_placement(placement)?; + let host_id = placement + .host_image_id + .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement)); + let placement_id = host_placement_id(&placement.source_key, &placement.placement); + let key = (host_id, placement_id); + let image_signature = image_signature(placement, format_code); + let placement_signature = + placement_signature(clipped, placement.placement.z, placement.scrollback_offset); + let image_current = cache.images.get(&host_id) == Some(&image_signature); + let placement_current = cache.placements.get(&key) == Some(&placement_signature) + && (!cache.replay_placements || cache.replayed_placements.contains(&key)); + if image_current + && placement_current + && cache.sources.get(&placement.source_key) == Some(&host_id) + { + return None; + } + + let mut bytes = Vec::new(); + let mut displayed = false; + if !image_current { + if cache.images.contains_key(&host_id) + && matches!(placement.source_key, HostSourceKey::PaneLayer { .. }) + { + if !encode_transmit_and_display( + &mut bytes, + placement, + clipped, + format_code, + host_id, + placement_id, + ) { + return None; + } + displayed = true; + } else { + if cache.images.contains_key(&host_id) { + encode_delete_image(&mut bytes, host_id); + cache.placements.retain(|(id, _), _| *id != host_id); + cache.replayed_placements.retain(|(id, _)| *id != host_id); + } + if !encode_upload_image(&mut bytes, placement, format_code, host_id) { + return None; + } + } + cache.images.insert(host_id, image_signature); + } + + release_superseded_source_image(&mut bytes, cache, placement.source_key.clone(), host_id); + if !displayed && !placement_current { + encode_display_placement( + &mut bytes, + clipped, + host_id, + placement_id, + placement.placement.z, + ); + } + cache.placements.insert(key, placement_signature); + if cache.replay_placements { + cache.replayed_placements.insert(key); + } + Some(bytes) +} + +fn release_superseded_source_image( + bytes: &mut Vec, + cache: &mut HostGraphicsCache, + source: HostSourceKey, + host_id: u32, +) { + let Some(previous) = cache.sources.insert(source, host_id) else { + return; + }; + if previous == host_id || cache.sources.values().any(|id| *id == previous) { + return; + } + encode_delete_image(bytes, previous); + cache.images.remove(&previous); + cache.placements.retain(|(id, _), _| *id != previous); + cache.replayed_placements.retain(|(id, _)| *id != previous); +} + +#[cfg(test)] +fn drain_graphics_updates( + cache: &mut HostGraphicsCache, + placements: &[HostPlacement], + live: &HashSet, +) -> Vec { + let mut bytes = Vec::new(); + loop { + let encoded = encode_graphics_update_incremental(cache, placements, live, None); + bytes.extend(encoded.bytes); + if !encoded.incomplete { + return bytes; + } + } +} + +#[cfg(test)] +fn encode_graphics_update( + bytes: &mut Vec, + placements: &[HostPlacement], + replay: bool, + images: &mut HashMap, + host_placements: &mut HashMap<(u32, u32), PlacementSignature>, + sources: &mut HashMap, +) { + let mut cache = HostGraphicsCache { + images: std::mem::take(images), + placements: std::mem::take(host_placements), + sources: std::mem::take(sources), + ..HostGraphicsCache::default() + }; + let mut live = cache + .sources + .keys() + .filter(|source| matches!(source, HostSourceKey::PaneLayer { .. })) + .cloned() + .collect::>(); + live.extend( + placements + .iter() + .filter(|placement| matches!(placement.source_key, HostSourceKey::PaneLayer { .. })) + .map(|placement| placement.source_key.clone()), + ); + if live.is_empty() { + encode_terminal_graphics_update_legacy(bytes, placements, replay, &mut cache); + } else { + if replay { + cache.request_placement_replay(); + } + bytes.extend(drain_graphics_updates(&mut cache, placements, &live)); + } + *images = cache.images; + *host_placements = cache.placements; + *sources = cache.sources; +} + pub(crate) fn clear_all_host_graphics() -> io::Result<()> { let cache = LOCAL_HOST_GRAPHICS.get_or_init(|| Mutex::new(HostGraphicsCache::default())); let mut bytes = Vec::new(); @@ -473,10 +799,97 @@ pub(crate) fn clear_all_host_graphics() -> io::Result<()> { } impl HostGraphicsCache { + fn clear_pane_sources(&mut self) -> Vec { + let pane_sources = self + .sources + .keys() + .filter(|source| matches!(source, HostSourceKey::PaneLayer { .. })) + .cloned() + .collect::>(); + let mut removed_images = HashSet::new(); + for source in pane_sources { + if let Some(image_id) = self.sources.remove(&source) { + removed_images.insert(image_id); + } + self.oversized.remove(&source); + } + + let mut bytes = Vec::new(); + for image_id in removed_images { + if self.sources.values().any(|id| *id == image_id) { + continue; + } + encode_delete_image(&mut bytes, image_id); + self.images.remove(&image_id); + self.placements.retain(|(id, _), _| *id != image_id); + self.replayed_placements.retain(|(id, _)| *id != image_id); + } + self.reset_incremental_state(); + bytes + } + + fn has_pane_sources(&self) -> bool { + self.sources + .keys() + .any(|source| matches!(source, HostSourceKey::PaneLayer { .. })) + } + + fn reset_incremental_state(&mut self) { + self.oversized.clear(); + self.continuation = None; + self.replay_placements = false; + self.replayed_placements.clear(); + } + + pub(crate) fn trust_pane_layer( + &mut self, + key: &crate::app::pane_graphics::Key, + host_id: u32, + layer: &crate::app::pane_graphics::Layer, + ) { + let source = HostSourceKey::PaneLayer { + pane_id: key.0, + layer_id: key.1.clone(), + }; + self.oversized.remove(&source); + self.sources.insert(source, host_id); + self.images + .insert(host_id, pane_layer_image_signature(layer)); + } + + pub(crate) fn forget_pane_layer(&mut self, key: &crate::app::pane_graphics::Key, host_id: u32) { + let source = HostSourceKey::PaneLayer { + pane_id: key.0, + layer_id: key.1.clone(), + }; + self.sources.remove(&source); + self.oversized.remove(&source); + self.images.remove(&host_id); + self.placements.retain(|(id, _), _| *id != host_id); + self.replayed_placements.retain(|(id, _)| *id != host_id); + } + pub(crate) fn is_empty(&self) -> bool { self.images.is_empty() && self.placements.is_empty() } + pub(crate) fn request_placement_replay(&mut self) { + if !self.replay_placements { + self.replay_placements = true; + self.replayed_placements.clear(); + } + } + + #[cfg(test)] + fn hide_except_live_pane_layers(&mut self, live: &HashSet) -> Vec { + drain_graphics_updates(self, &[], live) + } + + #[cfg(test)] + pub(crate) fn test_image_count(&self) -> usize { + self.images.len() + } + #[cfg(test)] pub(crate) fn test_mark_non_empty(&mut self) { self.images.insert( @@ -499,15 +912,43 @@ impl HostGraphicsCache { self.images.clear(); self.placements.clear(); self.sources.clear(); + self.reset_incremental_state(); self.view = None; bytes } + pub(crate) fn clear_next(&mut self) -> EncodedGraphics { + self.continuation = None; + let mut bytes = Vec::new(); + if let Some(id) = self.images.keys().copied().min() { + encode_delete_image(&mut bytes, id); + self.images.remove(&id); + self.placements.retain(|(image, _), _| *image != id); + self.sources.retain(|_, image| *image != id); + self.replayed_placements.retain(|(image, _)| *image != id); + } else if let Some(key) = self.placements.keys().copied().min() { + encode_delete_placement(&mut bytes, key.0, key.1); + self.placements.remove(&key); + self.replayed_placements.remove(&key); + } else { + self.sources.clear(); + self.oversized.clear(); + self.view = None; + self.replay_placements = false; + self.replayed_placements.clear(); + } + EncodedGraphics { + bytes, + incomplete: !self.is_empty(), + } + } + fn update_view(&mut self, view_key: Option) -> bool { if self.view == view_key { return false; } self.view = view_key; + self.continuation = None; true } } @@ -523,6 +964,7 @@ fn active_view_key(app: &AppState) -> Option { fn collect_visible_placements( app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, terminal_runtimes: &TerminalRuntimeRegistry, surface: crate::ui::TabSurfaceView<'_>, cell_size: HostCellSize, @@ -553,9 +995,25 @@ fn collect_visible_placements( ); let mut placements = Vec::new(); for info in surface.pane_infos { - if let Some(layer) = app.pane_graphics_layers.get(&info.id) { + let mut pane_layers = graphics + .slots + .iter() + .filter_map(|((pane_id, layer_id), slot)| { + (*pane_id == info.id) + .then(|| { + slot.layer + .as_ref() + .map(|layer| (layer_id, slot.host_image_id, layer)) + }) + .flatten() + }) + .collect::>(); + pane_layers.sort_by_key(|(layer_id, _, layer)| (layer.z_index, layer_id.as_str())); + for (layer_id, host_image_id, layer) in pane_layers { placements.push(pane_graphics_host_placement( info, + layer_id, + host_image_id, cell_size, layer, uploaded_images, @@ -582,6 +1040,7 @@ fn collect_visible_placements( .unwrap_or(0); placements.push(HostPlacement { pane_id: info.id, + host_image_id: None, area: info.inner_rect, cell_size, source_key: HostSourceKey::Terminal { @@ -602,25 +1061,19 @@ fn collect_visible_placements( fn pane_graphics_host_placement( info: &PaneInfo, + layer_id: &str, + host_id: u32, cell_size: HostCellSize, - layer: &crate::app::state::PaneGraphicsLayer, + layer: &crate::app::pane_graphics::Layer, uploaded_images: &HashMap, include_data: bool, ) -> HostPlacement { let format = pane_graphics_kitty_format(layer.format); - let format_code = kitty_format_code(format); - let signature = ImageSignature { - image_width: layer.image_width, - image_height: layer.image_height, - format_code, - data_len: layer.data.len(), - data_fingerprint: layer.data_fingerprint, - }; - let host_id = pane_graphics_host_image_id(info.id, signature); + let signature = pane_layer_image_signature(layer); let data = if !include_data || uploaded_images.get(&host_id).copied() == Some(signature) { Vec::new() } else { - layer.data.clone() + layer.inline_data().map(<[u8]>::to_vec).unwrap_or_default() }; let render = layer.render; let grid_cols = if render.grid_cols == 0 { @@ -636,20 +1089,24 @@ fn pane_graphics_host_placement( HostPlacement { pane_id: info.id, + host_image_id: Some(host_id), area: info.inner_rect, cell_size, - source_key: HostSourceKey::PaneLayer { pane_id: info.id }, + source_key: HostSourceKey::PaneLayer { + pane_id: info.id, + layer_id: layer_id.to_owned(), + }, scrollback_offset: 0, placement: KittyImagePlacement { image_id: 1, placement_id: 1, - z: 0, + z: layer.z_index, x_offset: 0, y_offset: 0, image_width: layer.image_width, image_height: layer.image_height, format, - data_len: layer.data.len(), + data_len: layer.data_len(), data_fingerprint: layer.data_fingerprint, data, render: KittyPlacementRenderInfo { @@ -672,7 +1129,8 @@ fn pane_graphics_kitty_format(format: crate::api::schema::PaneGraphicsFormat) -> match format { crate::api::schema::PaneGraphicsFormat::Png => KittyImageFormat::Png, crate::api::schema::PaneGraphicsFormat::Rgb => KittyImageFormat::Rgb, - crate::api::schema::PaneGraphicsFormat::Rgba => KittyImageFormat::Rgba, + crate::api::schema::PaneGraphicsFormat::Rgba + | crate::api::schema::PaneGraphicsFormat::Bgra => KittyImageFormat::Rgba, } } @@ -697,20 +1155,14 @@ fn host_image_id_for_signature(pane_id: PaneId, signature: ImageSignature) -> u3 HOST_IMAGE_ID_BASE + ((hasher.finish() as u32) % 900_000) } -fn pane_graphics_host_image_id(pane_id: PaneId, signature: ImageSignature) -> u32 { - let mut hasher = DefaultHasher::new(); - pane_id.raw().hash(&mut hasher); - signature.hash(&mut hasher); - PANE_GRAPHICS_IMAGE_ID_BIT | ((hasher.finish() as u32) & !PANE_GRAPHICS_IMAGE_ID_BIT) -} - -fn host_placement_id(source_key: HostSourceKey, placement: &KittyImagePlacement) -> u32 { +fn host_placement_id(source_key: &HostSourceKey, placement: &KittyImagePlacement) -> u32 { let mut hasher = DefaultHasher::new(); match source_key { HostSourceKey::Terminal { pane_id, .. } => pane_id.raw().hash(&mut hasher), - HostSourceKey::PaneLayer { pane_id } => { + HostSourceKey::PaneLayer { pane_id, layer_id } => { "pane.graphics".hash(&mut hasher); pane_id.raw().hash(&mut hasher); + layer_id.hash(&mut hasher); } } placement.image_id.hash(&mut hasher); @@ -718,6 +1170,90 @@ fn host_placement_id(source_key: HostSourceKey, placement: &KittyImagePlacement) 1 + ((hasher.finish() as u32) % 900_000) } +pub(crate) struct DirectFileCommand { + pub(crate) leading: Vec, + pub(crate) control: String, +} + +pub(crate) fn prepare_direct_file( + app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, + surface: crate::ui::TabSurfaceView<'_>, + cell_size: HostCellSize, + cache: &mut HostGraphicsCache, + key: &crate::app::pane_graphics::Key, +) -> Option { + if app.mode != Mode::Terminal || !cell_size.is_known() || app.active.is_none() { + return None; + } + let info = surface.pane_infos.iter().find(|info| info.id == key.0)?; + let slot = graphics.slots.get(key)?; + let layer = slot.layer.as_ref()?; + layer.direct_lease()?; + let placement = pane_graphics_host_placement( + info, + &key.1, + slot.host_image_id, + cell_size, + layer, + &cache.images, + false, + ); + let (command, clipped, format_code, placement_id) = + direct_file_command(&placement, slot.host_image_id)?; + cache + .images + .insert(slot.host_image_id, image_signature(&placement, format_code)); + cache.placements.insert( + (slot.host_image_id, placement_id), + placement_signature(clipped, layer.z_index, 0), + ); + cache + .sources + .insert(placement.source_key, slot.host_image_id); + Some(command) +} + +fn direct_file_command( + placement: &HostPlacement, + host_image_id: u32, +) -> Option<(DirectFileCommand, ClippedPlacement, u32, u32)> { + let (clipped, format_code) = clipped_placement(placement)?; + let placement_id = host_placement_id(&placement.source_key, &placement.placement); + let mut control = format!( + "a=T,f={format_code},s={},v={},i={host_image_id},p={placement_id},c={},r={},z={},C=1,q=0", + placement.placement.image_width, + placement.placement.image_height, + clipped.cols, + clipped.rows, + placement.placement.z, + ); + append_placement_controls(&mut control, clipped); + Some(( + DirectFileCommand { + leading: format!("\x1b[{};{}H", clipped.y + 1, clipped.x + 1).into_bytes(), + control, + }, + clipped, + format_code, + placement_id, + )) +} + +#[cfg(unix)] +pub(crate) fn encode_kitty_regular_file( + out: &mut Vec, + leading: &[u8], + control: &str, + path: &str, +) { + let payload = base64::engine::general_purpose::STANDARD.encode(path.as_bytes()); + out.extend_from_slice(b"\x1b7"); + out.extend_from_slice(leading); + let _ = write!(out, "\x1b_G{control},t=f;{payload}\x1b\\"); + out.extend_from_slice(b"\x1b8"); +} + fn encode_delete_image(out: &mut Vec, id: u32) { let _ = write!(out, "\x1b_Ga=d,d=I,i={id},q=2;\x1b\\"); } @@ -747,6 +1283,31 @@ fn encode_upload_image( true } +fn encode_transmit_and_display( + out: &mut Vec, + placement: &HostPlacement, + clipped: ClippedPlacement, + format_code: u32, + host_id: u32, + host_placement_id: u32, +) -> bool { + if placement.placement.data.is_empty() { + return false; + } + let _ = write!(out, "\x1b[{};{}H", clipped.y + 1, clipped.x + 1); + let mut control = format!( + "a=T,t=d,f={format_code},s={},v={},i={host_id},p={host_placement_id},c={},r={},z={},C=1,q=2", + placement.placement.image_width, + placement.placement.image_height, + clipped.cols, + clipped.rows, + placement.placement.z, + ); + append_placement_controls(&mut control, clipped); + encode_kitty_data(out, &control, &placement.placement.data); + true +} + fn encode_display_placement( out: &mut Vec, clipped: ClippedPlacement, @@ -759,6 +1320,11 @@ fn encode_display_placement( "a=p,i={host_id},p={host_placement_id},c={},r={},z={z},C=1,q=2", clipped.cols, clipped.rows, ); + append_placement_controls(&mut control, clipped); + let _ = write!(out, "\x1b_G{control};\x1b\\"); +} + +fn append_placement_controls(control: &mut String, clipped: ClippedPlacement) { if clipped.source_x > 0 { let _ = write!(control, ",x={}", clipped.source_x); } @@ -777,8 +1343,6 @@ fn encode_display_placement( if clipped.y_offset > 0 { let _ = write!(control, ",Y={}", clipped.y_offset); } - - let _ = write!(out, "\x1b_G{control};\x1b\\"); } fn clipped_placement(placement: &HostPlacement) -> Option<(ClippedPlacement, u32)> { @@ -929,6 +1493,16 @@ fn scale_pixels(value: u32, source: u32, dest: u32) -> u32 { ((value as u64).saturating_mul(source as u64) / dest.max(1) as u64).min(u32::MAX as u64) as u32 } +fn pane_layer_image_signature(layer: &crate::app::pane_graphics::Layer) -> ImageSignature { + ImageSignature { + image_width: layer.image_width, + image_height: layer.image_height, + format_code: kitty_format_code(pane_graphics_kitty_format(layer.format)), + data_len: layer.data_len(), + data_fingerprint: layer.data_fingerprint, + } +} + fn image_signature(placement: &HostPlacement, format_code: u32) -> ImageSignature { ImageSignature { image_width: placement.placement.image_width, @@ -1001,9 +1575,22 @@ fn encode_kitty_data(out: &mut Vec, control: &str, data: &[u8]) { mod tests { use super::*; + #[test] + fn fallback_cell_size_is_usable_only_for_nonempty_areas() { + assert_eq!( + HostCellSize::fallback_for_area(Rect::new(0, 0, 80, 24)), + HostCellSize { + width_px: 8, + height_px: 16, + } + ); + assert!(!HostCellSize::fallback_for_area(Rect::default()).is_known()); + } + fn test_placement(viewport_col: i32, viewport_row: i32) -> HostPlacement { HostPlacement { pane_id: PaneId::from_raw(1), + host_image_id: None, area: Rect::new(0, 0, 20, 10), cell_size: HostCellSize { width_px: 10, @@ -1046,10 +1633,70 @@ mod tests { let mut placement = test_placement(viewport_col, viewport_row); placement.source_key = HostSourceKey::PaneLayer { pane_id: placement.pane_id, + layer_id: "primary".into(), }; placement } + fn update( + cache: &mut HostGraphicsCache, + placements: &[HostPlacement], + replay: bool, + ) -> Vec { + let mut bytes = Vec::new(); + encode_graphics_update( + &mut bytes, + placements, + replay, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, + ); + bytes + } + + #[test] + fn terminal_graphics_without_pane_layers_preserves_legacy_transcript() { + fn record(transcript: &mut Vec, bytes: &[u8]) { + transcript.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + transcript.extend_from_slice(bytes); + } + + fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf29ce484222325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x100000001b3) + }) + } + + let mut cache = HostGraphicsCache::default(); + let mut transcript = Vec::new(); + + record( + &mut transcript, + &update(&mut cache, &[test_placement(0, 0)], false), + ); + record( + &mut transcript, + &update(&mut cache, &[test_placement(0, 0)], false), + ); + record( + &mut transcript, + &update(&mut cache, &[test_placement(-1, 2)], false), + ); + record( + &mut transcript, + &update(&mut cache, &[test_placement(-1, 2)], true), + ); + + let mut changed = test_placement(-1, 2); + changed.placement.data_fingerprint = 43; + record(&mut transcript, &update(&mut cache, &[changed], false)); + record(&mut transcript, &update(&mut cache, &[], false)); + + assert_eq!(transcript.len(), 10_084); + assert_eq!(fnv1a(&transcript), 0xc5bd_83e4_b039_870e); + } + #[test] fn terminal_placement_id_preserves_legacy_identity() { let placement = test_placement(0, 0); @@ -1060,13 +1707,14 @@ mod tests { let expected = 1 + ((legacy.finish() as u32) % 900_000); assert_eq!( - host_placement_id(placement.source_key, &placement.placement), + host_placement_id(&placement.source_key, &placement.placement), expected ); assert_ne!( host_placement_id( - HostSourceKey::PaneLayer { + &HostSourceKey::PaneLayer { pane_id: placement.pane_id, + layer_id: "primary".into(), }, &placement.placement, ), @@ -1074,15 +1722,60 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn regular_file_command_is_rgba_quiet_zero_and_path_encoded() { + let mut bytes = Vec::new(); + encode_kitty_regular_file( + &mut bytes, + b"\x1b[2;3H", + "a=T,f=32,s=3,v=2,i=42,p=7,c=3,r=2,z=0,C=1,q=0", + "/private/frame", + ); + let text = String::from_utf8(bytes).unwrap(); + assert!(text.starts_with("\x1b7\x1b[2;3H\x1b_Ga=T,f=32")); + assert!(text.contains(",C=1,q=0,t=f;L3ByaXZhdGUvZnJhbWU=")); + assert!(text.ends_with("\x1b\\\x1b8")); + } + + #[test] + fn direct_file_uses_one_clipped_transmit_and_display_at_the_final_position() { + let mut placement = pane_layer_placement(-1, 2); + placement.area = Rect::new(10, 4, 8, 6); + let (command, _, _, _) = direct_file_command(&placement, (1 << 31) | 9).unwrap(); + let control = command.control; + + assert_eq!(command.leading, b"\x1b[7;11H"); + assert!(control.starts_with("a=T,f=32,s=30,v=30,i=2147483657,p=")); + assert!(control.contains(",c=2,r=3,z=0,C=1,q=0,x=10,w=20,h=30")); + assert!(direct_file_command(&pane_layer_placement(30, 0), 9).is_none()); + } + #[test] fn pane_graphics_image_ids_are_disjoint_from_terminal_image_ids() { let placement = test_placement(0, 0); let signature = image_signature(&placement, kitty_format_code(placement.placement.format)); let terminal_id = host_image_id_for_signature(placement.pane_id, signature); - let pane_graphics_id = pane_graphics_host_image_id(placement.pane_id, signature); + let mut graphics = crate::app::pane_graphics::Runtime::default(); + let primary = (placement.pane_id, "primary".into()); + let pane_graphics_id = graphics.reserve_image_id(&primary).unwrap(); + graphics.slots.insert( + primary.clone(), + crate::app::pane_graphics::Slot::test(pane_graphics_id, None), + ); assert_eq!(terminal_id & PANE_GRAPHICS_IMAGE_ID_BIT, 0); assert_ne!(pane_graphics_id & PANE_GRAPHICS_IMAGE_ID_BIT, 0); + assert_eq!( + pane_graphics_id, + graphics.reserve_image_id(&primary).unwrap() + ); + assert_ne!( + pane_graphics_id, + graphics + .reserve_image_id(&(placement.pane_id, "toolbar".into())) + .unwrap() + ); } #[test] @@ -1121,16 +1814,19 @@ mod tests { borders: ratatui::widgets::Borders::NONE, is_focused: true, }; - let layer = crate::app::state::PaneGraphicsLayer::new( + let layer = crate::app::pane_graphics::Layer::inline( crate::api::schema::PaneGraphicsFormat::Rgba, 80, 30, vec![255; 80 * 30 * 4], crate::api::schema::PaneGraphicsPlacementParams::default(), + 0, ); let placement = pane_graphics_host_placement( &info, + "primary", + PANE_GRAPHICS_IMAGE_ID_BIT | 1, HostCellSize { width_px: 10, height_px: 10, @@ -1151,166 +1847,55 @@ mod tests { #[test] fn graphics_update_uploads_once_then_repositions_only() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); + let mut cache = HostGraphicsCache::default(); + let first = update(&mut cache, &[test_placement(0, 0)], false); + assert!(String::from_utf8_lossy(&first).contains("a=t")); + assert!(String::from_utf8_lossy(&first).contains("a=p")); + assert!(update(&mut cache, &[test_placement(0, 0)], false).is_empty()); - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut images, - &mut placements, - &mut sources, - ); - let first = String::from_utf8_lossy(&bytes); - assert!(first.contains("a=t")); - assert!(first.contains("a=p")); - - bytes.clear(); - let same = test_placement(0, 0); - encode_graphics_update( - &mut bytes, - &[same], - false, - &mut images, - &mut placements, - &mut sources, - ); - assert!(bytes.is_empty()); - - let mut z_changed = test_placement(0, 0); - z_changed.placement.z = 1; - encode_graphics_update( - &mut bytes, - &[z_changed], - false, - &mut images, - &mut placements, - &mut sources, - ); - let z_changed_bytes = String::from_utf8_lossy(&bytes); - assert!(!z_changed_bytes.contains("a=t")); - assert!(z_changed_bytes.contains("a=p")); - - bytes.clear(); - let moved = test_placement(0, 1); - encode_graphics_update( - &mut bytes, - &[moved], - false, - &mut images, - &mut placements, - &mut sources, - ); - let moved_bytes = String::from_utf8_lossy(&bytes); - assert!(!moved_bytes.contains("a=t")); - assert!(moved_bytes.contains("a=p")); + let mut changed = test_placement(0, 0); + changed.placement.z = 1; + for placement in [changed, test_placement(0, 1)] { + let bytes = update(&mut cache, &[placement], false); + assert!(!String::from_utf8_lossy(&bytes).contains("a=t")); + assert!(String::from_utf8_lossy(&bytes).contains("a=p")); + } } #[test] fn view_change_redisplays_unchanged_visible_placement() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); - - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut images, - &mut placements, - &mut sources, - ); - assert_eq!(placements.len(), 1); - - bytes.clear(); - let same = test_placement(0, 0); - encode_graphics_update( - &mut bytes, - &[same], - true, - &mut images, - &mut placements, - &mut sources, - ); - let redisplay = String::from_utf8_lossy(&bytes); - assert!(!redisplay.contains("a=t")); - assert!(redisplay.contains("a=p")); - assert_eq!(placements.len(), 1); + let mut cache = HostGraphicsCache::default(); + update(&mut cache, &[test_placement(0, 0)], false); + assert_eq!(cache.placements.len(), 1); + let bytes = update(&mut cache, &[test_placement(0, 0)], true); + assert!(!String::from_utf8_lossy(&bytes).contains("a=t")); + assert!(String::from_utf8_lossy(&bytes).contains("a=p")); + assert_eq!(cache.placements.len(), 1); } #[test] fn surface_reset_deletes_then_reuploads_and_redisplays_placement() { let mut cache = HostGraphicsCache::default(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); - - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut cache.images, - &mut cache.placements, - &mut cache.sources, - ); - assert_eq!(cache.images.len(), 1); - assert_eq!(cache.placements.len(), 1); - - bytes = cache.clear_bytes(); - let same = test_placement(0, 0); - encode_graphics_update( - &mut bytes, - &[same], - false, - &mut cache.images, - &mut cache.placements, - &mut cache.sources, - ); - + update(&mut cache, &[test_placement(0, 0)], false); + assert_eq!((cache.images.len(), cache.placements.len()), (1, 1)); + let mut bytes = cache.clear_bytes(); + bytes.extend(update(&mut cache, &[test_placement(0, 0)], false)); let redisplay = String::from_utf8_lossy(&bytes); assert!(redisplay.contains("a=d,d=I")); assert!(redisplay.contains("a=t")); assert!(redisplay.contains("a=p")); - assert_eq!(cache.images.len(), 1); - assert_eq!(cache.placements.len(), 1); + assert_eq!((cache.images.len(), cache.placements.len()), (1, 1)); } #[test] fn scrollback_offset_change_redisplays_placement() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); - - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut images, - &mut placements, - &mut sources, - ); - - bytes.clear(); + let mut cache = HostGraphicsCache::default(); + update(&mut cache, &[test_placement(0, 0)], false); let mut scrolled = test_placement(0, 0); scrolled.scrollback_offset = 3; - encode_graphics_update( - &mut bytes, - &[scrolled], - false, - &mut images, - &mut placements, - &mut sources, - ); - let redisplay = String::from_utf8_lossy(&bytes); - assert!(!redisplay.contains("a=t")); - assert!(redisplay.contains("a=p")); + let bytes = update(&mut cache, &[scrolled], false); + assert!(!String::from_utf8_lossy(&bytes).contains("a=t")); + assert!(String::from_utf8_lossy(&bytes).contains("a=p")); } #[test] @@ -1376,6 +1961,42 @@ mod tests { assert_eq!(placements.len(), 1); } + #[test] + fn pane_layer_replacement_is_atomic_without_delete_to_blank() { + let mut images = HashMap::new(); + let mut placements = HashMap::new(); + let mut sources = HashMap::new(); + let mut bytes = Vec::new(); + let mut first = pane_layer_placement(0, 0); + first.host_image_id = Some((1 << 31) | 7); + encode_graphics_update( + &mut bytes, + &[first], + false, + &mut images, + &mut placements, + &mut sources, + ); + + bytes.clear(); + let mut changed = pane_layer_placement(0, 0); + changed.host_image_id = Some((1 << 31) | 7); + changed.placement.data_fingerprint += 1; + encode_graphics_update( + &mut bytes, + &[changed], + false, + &mut images, + &mut placements, + &mut sources, + ); + + let update = String::from_utf8_lossy(&bytes); + assert!(update.contains("a=T,t=d")); + assert!(update.contains(",p=") && update.contains(",C=1,q=2")); + assert!(!update.contains("a=d")); + } + #[test] fn replaced_image_content_deletes_superseded_host_image() { let mut images = HashMap::new(); @@ -1555,40 +2176,143 @@ mod tests { } #[test] - fn removed_pane_layer_deletes_unreferenced_host_image() { + fn trusted_direct_image_uses_reserved_id_for_placement_without_upload() { + let key = (PaneId::from_raw(1), "primary".to_owned()); + let layer = crate::app::pane_graphics::Layer::inline( + crate::api::schema::PaneGraphicsFormat::Rgba, + 30, + 30, + vec![255; 30 * 30 * 4], + Default::default(), + 0, + ); + let reserved_id = (1 << 31) | 77; + let mut cache = HostGraphicsCache::default(); + cache.trust_pane_layer(&key, reserved_id, &layer); + let mut placement = pane_layer_placement(0, 0); + placement.host_image_id = Some(reserved_id); + placement.placement.data.clear(); + placement.placement.data_len = layer.data_len(); + placement.placement.data_fingerprint = layer.data_fingerprint; + let mut bytes = Vec::new(); + + encode_graphics_update( + &mut bytes, + &[placement], + false, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, + ); + + let update = String::from_utf8_lossy(&bytes); + assert!(update.contains(&format!("a=p,i={reserved_id}"))); + assert!(!update.contains("a=t")); + + let live = HashSet::from([HostSourceKey::PaneLayer { + pane_id: key.0, + layer_id: key.1.clone(), + }]); + let hidden = String::from_utf8(cache.hide_except_live_pane_layers(&live)).unwrap(); + assert!(hidden.contains("a=d,d=i")); + assert!(!hidden.contains("a=d,d=I")); + assert!(cache.images.contains_key(&reserved_id)); + assert!(cache.placements.is_empty()); + + let mut returning = pane_layer_placement(0, 0); + returning.host_image_id = Some(reserved_id); + returning.placement.data.clear(); + returning.placement.data_len = layer.data_len(); + returning.placement.data_fingerprint = layer.data_fingerprint; + let mut replay = Vec::new(); + encode_graphics_update( + &mut replay, + &[returning], + false, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, + ); + let replay = String::from_utf8(replay).unwrap(); + assert!(replay.contains(&format!("a=p,i={reserved_id}"))); + assert!(!replay.contains("a=t")); + + cache.forget_pane_layer(&key, reserved_id); + let mut fallback = pane_layer_placement(0, 0); + fallback.host_image_id = Some(reserved_id); + let mut retransmit = Vec::new(); + encode_graphics_update( + &mut retransmit, + &[fallback], + false, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, + ); + assert!(String::from_utf8_lossy(&retransmit).contains("a=t")); + } + + #[test] + fn hidden_layer_and_full_redraw_replay_placement_without_pixels() { let mut images = HashMap::new(); let mut placements = HashMap::new(); let mut sources = HashMap::new(); let mut bytes = Vec::new(); + let placement = pane_layer_placement(0, 0); + encode_graphics_update( + &mut bytes, + &[placement], + false, + &mut images, + &mut placements, + &mut sources, + ); + + for (visible, replay) in [(false, false), (true, false), (true, true)] { + bytes.clear(); + let current = visible.then(|| pane_layer_placement(0, 0)); + encode_graphics_update( + &mut bytes, + current.as_slice(), + replay, + &mut images, + &mut placements, + &mut sources, + ); + let update = String::from_utf8_lossy(&bytes); + assert!(!update.contains("a=t")); + assert!(!update.contains("a=d,d=I")); + assert_eq!(update.contains("a=p"), visible); + } + assert_eq!(images.len(), 1); + assert_eq!(sources.len(), 1); + } + + #[test] + fn removed_pane_layer_deletes_unreferenced_host_image() { + let mut cache = HostGraphicsCache::default(); + let mut bytes = Vec::new(); encode_graphics_update( &mut bytes, &[pane_layer_placement(0, 0)], false, - &mut images, - &mut placements, - &mut sources, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, ); - let host_id = *images.keys().next().expect("uploaded pane layer"); + let host_id = *cache.images.keys().next().expect("uploaded pane layer"); - bytes.clear(); - encode_graphics_update( - &mut bytes, - &[], - false, - &mut images, - &mut placements, - &mut sources, - ); + bytes = drain_graphics_updates(&mut cache, &[], &HashSet::new()); let delete = String::from_utf8_lossy(&bytes); assert!(delete.contains(&format!("a=d,d=I,i={host_id}"))); - assert!(images.is_empty()); - assert!(placements.is_empty()); - assert!(sources.is_empty()); + assert!(cache.images.is_empty()); + assert!(cache.placements.is_empty()); + assert!(cache.sources.is_empty()); } #[test] - fn clipped_pane_layer_deletes_unreferenced_host_image() { + fn hidden_pane_layer_retains_image_and_removes_only_placement() { let mut images = HashMap::new(); let mut placements = HashMap::new(); let mut sources = HashMap::new(); @@ -1613,11 +2337,12 @@ mod tests { &mut sources, ); - let delete = String::from_utf8_lossy(&bytes); - assert!(delete.contains(&format!("a=d,d=I,i={host_id}"))); - assert!(images.is_empty()); + let update = String::from_utf8_lossy(&bytes); + assert!(update.contains("a=d,d=i")); + assert!(!update.contains(&format!("a=d,d=I,i={host_id}"))); + assert_eq!(images.len(), 1); assert!(placements.is_empty()); - assert!(sources.is_empty()); + assert_eq!(sources.len(), 1); } #[test] @@ -1668,35 +2393,118 @@ mod tests { #[test] fn removed_pane_layer_preserves_image_shared_with_terminal_source() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); + let mut cache = HostGraphicsCache::default(); let mut bytes = Vec::new(); encode_graphics_update( &mut bytes, &[pane_layer_placement(0, 0), test_placement(4, 0)], false, - &mut images, - &mut placements, - &mut sources, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, ); - assert_eq!(images.len(), 1); + assert_eq!(cache.images.len(), 1); - bytes.clear(); + bytes = drain_graphics_updates(&mut cache, &[], &HashSet::new()); encode_graphics_update( &mut bytes, &[test_placement(4, 0)], false, - &mut images, - &mut placements, - &mut sources, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, ); let update = String::from_utf8_lossy(&bytes); assert!(!update.contains("a=d,d=I")); - assert_eq!(images.len(), 1); - assert_eq!(placements.len(), 1); - assert_eq!(sources.len(), 1); + assert_eq!(cache.images.len(), 1); + assert_eq!(cache.placements.len(), 1); + assert_eq!(cache.sources.len(), 1); + } + + #[test] + fn changing_first_source_does_not_starve_second_source() { + let layers = |first| { + [(1, "a", first), (2, "b", 80)].map(|(id, name, fingerprint)| { + let mut placement = pane_layer_placement(0, 0); + placement.host_image_id = Some(PANE_GRAPHICS_IMAGE_ID_BIT | id); + placement.source_key = HostSourceKey::PaneLayer { + pane_id: placement.pane_id, + layer_id: name.into(), + }; + placement.placement.data_fingerprint = fingerprint; + placement + }) + }; + let initial = layers(42); + let live = initial.iter().map(|p| p.source_key.clone()).collect(); + let mut cache = HostGraphicsCache::default(); + assert!(encode_graphics_update_incremental(&mut cache, &initial, &live, None).incomplete); + assert!( + encode_graphics_update_incremental(&mut cache, &layers(43), &live, None).incomplete + ); + assert_eq!(cache.images.len(), 2, "second source uploaded next"); + + let terminal = |id| { + let mut placement = test_placement(0, 0); + placement.placement.image_id = id; + placement.placement.data_fingerprint = u64::from(id); + placement.source_key = HostSourceKey::Terminal { + pane_id: placement.pane_id, + image_id: id, + }; + placement + }; + let second = terminal(99).source_key; + let mut cache = HostGraphicsCache::default(); + for id in 1..=3 { + assert!( + encode_graphics_update_incremental( + &mut cache, + &[terminal(id), terminal(99)], + &HashSet::new(), + None, + ) + .incomplete + ); + } + assert!(cache.sources.contains_key(&second)); + } + + #[test] + fn large_terminal_image_is_local_but_quarantined_headless() { + let placements = || { + let mut large = test_placement(0, 0); + large.placement.data_len = 24 * 1024 * 1024; + let mut later = test_placement(4, 0); + later.placement.image_id = 8; + later.source_key = HostSourceKey::Terminal { + pane_id: later.pane_id, + image_id: 8, + }; + [large, later] + }; + for (budget, expected) in [ + (None, (true, 1, 0)), + (Some(HEADLESS_GRAPHICS_TRANSACTION_BUDGET), (false, 1, 1)), + ] { + let mut cache = HostGraphicsCache::default(); + let encoded = encode_graphics_update_incremental( + &mut cache, + &placements(), + &HashSet::new(), + budget, + ); + assert!(String::from_utf8_lossy(&encoded.bytes).contains("a=t")); + assert_eq!( + ( + encoded.incomplete, + cache.images.len(), + cache.oversized.len() + ), + expected + ); + } } #[test] @@ -1719,37 +2527,12 @@ mod tests { )); encode_display_placement(&mut encoded, clipped, host_id, 1, 0); - assert!(encoded.len() < crate::protocol::MAX_GRAPHICS_FRAME_SIZE); - } - - #[test] - fn view_change_deletes_stale_placement_immediately() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); - - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut images, - &mut placements, - &mut sources, - ); - bytes.clear(); - encode_graphics_update( - &mut bytes, - &[], - true, - &mut images, - &mut placements, - &mut sources, - ); - - let delete = String::from_utf8_lossy(&bytes); - assert!(delete.contains("a=d,d=i")); - assert!(placements.is_empty()); + let mut framed = Vec::new(); + crate::protocol::write_message( + &mut framed, + &crate::protocol::ServerMessage::Graphics { bytes: encoded }, + ) + .unwrap(); + assert!(framed.len() <= crate::protocol::MAX_GRAPHICS_FRAME_SIZE + 4); } } diff --git a/src/main.rs b/src/main.rs index c3269675..2db4f8aa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -75,6 +75,7 @@ mod logging; mod metadata_tokens; mod noninteractive_process; mod pane; +mod pane_graphics_files; mod persist; mod platform; mod plugin_command; diff --git a/src/pane.rs b/src/pane.rs index ab984946..da99f55e 100644 --- a/src/pane.rs +++ b/src/pane.rs @@ -2780,40 +2780,41 @@ impl PaneRuntime { pub fn encode_mouse_button( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { if !self.input_state()?.mouse_protocol_mode.reporting_enabled() { return None; } - self.terminal - .encode_mouse_button(kind, column, row, modifiers) + self.terminal.encode_mouse_button(kind, position, modifiers) } - pub fn encode_mouse_motion( + pub(crate) fn encode_mouse_motion( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { - self.terminal - .encode_mouse_motion(kind, column, row, modifiers) + self.terminal.encode_mouse_motion(kind, position, modifiers) } - pub fn encode_mouse_wheel( + pub(crate) fn encode_mouse_wheel( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { if self.wheel_routing()? != WheelRouting::MouseReport { return None; } - self.terminal - .encode_mouse_wheel(kind, column, row, modifiers) + self.terminal.encode_mouse_wheel(kind, position, modifiers) + } + + pub(crate) fn pixel_size(&self) -> Option<(u32, u32)> { + let (rows, cols, cell_width_px, cell_height_px) = self.current_size.get(); + let width = u32::from(cols).checked_mul(cell_width_px)?; + let height = u32::from(rows).checked_mul(cell_height_px)?; + (width > 0 && height > 0).then_some((width, height)) } pub fn encode_alternate_scroll( diff --git a/src/pane/input.rs b/src/pane/input.rs index 2e60bfc8..2ff299c9 100644 --- a/src/pane/input.rs +++ b/src/pane/input.rs @@ -60,23 +60,43 @@ pub(super) fn ghostty_mods_from_key_modifiers(modifiers: crossterm::event::KeyMo pub(super) fn ghostty_mouse_encoder_for_terminal( terminal: &crate::ghostty::Terminal, + position: crate::input::mouse::Position, ) -> Option { let mut encoder = crate::ghostty::MouseEncoder::new().ok()?; encoder.set_from_terminal(terminal); - if terminal - .mode_get(crate::ghostty::MODE_MOUSE_SGR_PIXELS) - .ok()? - { - // Herdr receives host mouse positions in terminal cells. Downgrade - // SGR-pixels to normal SGR so forwarded coordinates stay cell-local. - encoder.set_format(crate::ghostty::MOUSE_FORMAT_SGR); - } let cols = terminal.cols().ok()? as u32; let rows = terminal.rows().ok()? as u32; - encoder.set_size(cols, rows, 1, 1); + match position { + crate::input::mouse::Position::Cell { .. } => { + if terminal + .mode_get(crate::ghostty::MODE_MOUSE_SGR_PIXELS) + .ok()? + { + encoder.set_format(crate::ghostty::MOUSE_FORMAT_SGR); + } + encoder.set_size(cols, rows, 1, 1); + } + crate::input::mouse::Position::Pixels { .. } => { + let width_px = terminal.width_px().ok()?; + let height_px = terminal.height_px().ok()?; + if width_px == 0 || height_px == 0 || cols == 0 || rows == 0 { + return None; + } + encoder.set_size(width_px, height_px, width_px / cols, height_px / rows); + } + } Some(encoder) } +pub(super) fn ghostty_mouse_position_for_terminal( + position: crate::input::mouse::Position, +) -> Option<(f32, f32)> { + match position { + crate::input::mouse::Position::Pixels { x, y } => Some((x as f32, y as f32)), + crate::input::mouse::Position::Cell { column, row } => Some((column as f32, row as f32)), + } +} + pub(super) fn ghostty_mouse_event_from_button_kind( kind: crossterm::event::MouseEventKind, column: u16, diff --git a/src/pane/terminal.rs b/src/pane/terminal.rs index 8de84af3..91fcb4fe 100644 --- a/src/pane/terminal.rs +++ b/src/pane/terminal.rs @@ -23,7 +23,8 @@ use super::{ input::{ ghostty_key_event_from_terminal_key, ghostty_mouse_encoder_for_terminal, ghostty_mouse_event_from_button_kind, ghostty_mouse_event_from_motion_kind, - ghostty_mouse_event_from_wheel_kind, ghostty_prefers_herdr_text_encoding, + ghostty_mouse_event_from_wheel_kind, ghostty_mouse_position_for_terminal, + ghostty_prefers_herdr_text_encoding, }, kitty_keyboard::KittyKeyboardTracker, osc::{ @@ -537,37 +538,31 @@ impl PaneTerminal { self.ghostty.encode_terminal_key(key, protocol) } - pub fn encode_mouse_button( + pub(crate) fn encode_mouse_button( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { - self.ghostty - .encode_mouse_button(kind, column, row, modifiers) + self.ghostty.encode_mouse_button(kind, position, modifiers) } - pub fn encode_mouse_motion( + pub(crate) fn encode_mouse_motion( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { - self.ghostty - .encode_mouse_motion(kind, column, row, modifiers) + self.ghostty.encode_mouse_motion(kind, position, modifiers) } - pub fn encode_mouse_wheel( + pub(crate) fn encode_mouse_wheel( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { - self.ghostty - .encode_mouse_wheel(kind, column, row, modifiers) + self.ghostty.encode_mouse_wheel(kind, position, modifiers) } } @@ -1488,6 +1483,9 @@ impl GhosttyPaneTerminal { let _ = core .terminal .mode_set(crate::ghostty::MODE_MOUSE_SGR, false); + let _ = core + .terminal + .mode_set(crate::ghostty::MODE_MOUSE_SGR_PIXELS, false); match input_state.mouse_protocol_encoding { crate::input::MouseProtocolEncoding::Default => {} crate::input::MouseProtocolEncoding::Utf8 => { @@ -1498,6 +1496,11 @@ impl GhosttyPaneTerminal { crate::input::MouseProtocolEncoding::Sgr => { let _ = core.terminal.mode_set(crate::ghostty::MODE_MOUSE_SGR, true); } + crate::input::MouseProtocolEncoding::SgrPixels => { + let _ = core + .terminal + .mode_set(crate::ghostty::MODE_MOUSE_SGR_PIXELS, true); + } } if input_state.modify_other_keys { @@ -1688,6 +1691,10 @@ impl GhosttyPaneTerminal { .terminal .mode_get(crate::ghostty::MODE_MOUSE_UTF8) .ok()?; + let mouse_sgr_pixels = core + .terminal + .mode_get(crate::ghostty::MODE_MOUSE_SGR_PIXELS) + .ok()?; let mouse_alternate_scroll = core .terminal .mode_get(crate::ghostty::MODE_MOUSE_ALTERNATE_SCROLL) @@ -1703,7 +1710,9 @@ impl GhosttyPaneTerminal { } else { crate::input::MouseProtocolMode::None }; - let mouse_protocol_encoding = if mouse_sgr { + let mouse_protocol_encoding = if mouse_sgr_pixels { + crate::input::MouseProtocolEncoding::SgrPixels + } else if mouse_sgr { crate::input::MouseProtocolEncoding::Sgr } else if mouse_utf8 { crate::input::MouseProtocolEncoding::Utf8 @@ -1838,57 +1847,58 @@ impl GhosttyPaneTerminal { } } - pub fn encode_mouse_button( + pub(crate) fn encode_mouse_button( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { - let Ok(core) = self.core.lock() else { - return None; - }; - let mut encoder = ghostty_mouse_encoder_for_terminal(&core.terminal)?; - let event = ghostty_mouse_event_from_button_kind(kind, column, row, modifiers)?; - encoder - .encode(&event) - .ok() - .filter(|bytes| !bytes.is_empty()) + self.encode_mouse_event( + ghostty_mouse_event_from_button_kind(kind, 0, 0, modifiers)?, + position, + false, + ) } - pub fn encode_mouse_motion( + pub(crate) fn encode_mouse_motion( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { - let Ok(core) = self.core.lock() else { - return None; - }; - if !core.terminal.mode_get(MODE_MOUSE_ANY_MOTION).ok()? { + self.encode_mouse_event( + ghostty_mouse_event_from_motion_kind(kind, 0, 0, modifiers)?, + position, + true, + ) + } + + pub(crate) fn encode_mouse_wheel( + &self, + kind: crossterm::event::MouseEventKind, + position: crate::input::mouse::Position, + modifiers: crossterm::event::KeyModifiers, + ) -> Option> { + self.encode_mouse_event( + ghostty_mouse_event_from_wheel_kind(kind, 0, 0, modifiers)?, + position, + false, + ) + } + + fn encode_mouse_event( + &self, + mut event: crate::ghostty::MouseEvent, + position: crate::input::mouse::Position, + require_any_motion: bool, + ) -> Option> { + let core = self.core.lock().ok()?; + if require_any_motion && !core.terminal.mode_get(MODE_MOUSE_ANY_MOTION).ok()? { return None; } - let mut encoder = ghostty_mouse_encoder_for_terminal(&core.terminal)?; - let event = ghostty_mouse_event_from_motion_kind(kind, column, row, modifiers)?; - encoder - .encode(&event) - .ok() - .filter(|bytes| !bytes.is_empty()) - } - - pub fn encode_mouse_wheel( - &self, - kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, - modifiers: crossterm::event::KeyModifiers, - ) -> Option> { - let Ok(core) = self.core.lock() else { - return None; - }; - let mut encoder = ghostty_mouse_encoder_for_terminal(&core.terminal)?; - let event = ghostty_mouse_event_from_wheel_kind(kind, column, row, modifiers)?; + let mut encoder = ghostty_mouse_encoder_for_terminal(&core.terminal, position)?; + let (x, y) = ghostty_mouse_position_for_terminal(position)?; + event.set_position(x, y); encoder .encode(&event) .ok() @@ -4670,8 +4680,7 @@ mod tests { let encoded = pane.encode_mouse_button( crossterm::event::MouseEventKind::Up(crossterm::event::MouseButton::Left), - 11, - 9, + crate::input::mouse::Position::Cell { column: 11, row: 9 }, crossterm::event::KeyModifiers::empty(), ); @@ -4687,8 +4696,7 @@ mod tests { let encoded = pane.encode_mouse_button( crossterm::event::MouseEventKind::Drag(crossterm::event::MouseButton::Left), - 4, - 6, + crate::input::mouse::Position::Cell { column: 4, row: 6 }, crossterm::event::KeyModifiers::SHIFT, ); @@ -4704,8 +4712,7 @@ mod tests { let encoded = pane.encode_mouse_button( crossterm::event::MouseEventKind::Drag(crossterm::event::MouseButton::Left), - 4, - 6, + crate::input::mouse::Position::Cell { column: 4, row: 6 }, crossterm::event::KeyModifiers::empty(), ); @@ -4721,8 +4728,7 @@ mod tests { let encoded = pane.encode_mouse_motion( crossterm::event::MouseEventKind::Moved, - 4, - 6, + crate::input::mouse::Position::Cell { column: 4, row: 6 }, crossterm::event::KeyModifiers::empty(), ); @@ -4730,20 +4736,26 @@ mod tests { } #[test] - fn ghostty_mouse_sgr_pixels_downgrades_to_cell_coordinates() { + fn ghostty_mouse_sgr_pixels_preserves_exact_and_downgrades_cell_input() { let (tx, _rx) = mpsc::channel(4); let mut terminal = crate::ghostty::Terminal::new(80, 24, 0).unwrap(); + terminal.resize(80, 24, 10, 20).unwrap(); terminal.write(b"\x1b[?1003h\x1b[?1006h\x1b[?1016h"); let pane = GhosttyPaneTerminal::new(terminal, tx).unwrap(); - let encoded = pane.encode_mouse_motion( + let exact = pane.encode_mouse_motion( crossterm::event::MouseEventKind::Moved, - 4, - 6, + crate::input::mouse::Position::Pixels { x: 48, y: 139 }, + crossterm::event::KeyModifiers::empty(), + ); + let fallback = pane.encode_mouse_motion( + crossterm::event::MouseEventKind::Moved, + crate::input::mouse::Position::Cell { column: 4, row: 6 }, crossterm::event::KeyModifiers::empty(), ); - assert_eq!(encoded.as_deref(), Some(&b"\x1b[<35;5;7M"[..])); + assert_eq!(exact.as_deref(), Some(&b"\x1b[<35;48;139M"[..])); + assert_eq!(fallback.as_deref(), Some(&b"\x1b[<35;5;7M"[..])); } #[test] diff --git a/src/pane_graphics_files.rs b/src/pane_graphics_files.rs new file mode 100644 index 00000000..527be1d5 --- /dev/null +++ b/src/pane_graphics_files.rs @@ -0,0 +1,529 @@ +#[cfg(unix)] +use std::fs::OpenOptions; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock}; + +#[cfg(unix)] +use std::os::unix::fs::{FileExt, MetadataExt, OpenOptionsExt, PermissionsExt}; + +#[cfg(unix)] +const DIRECTORY_MODE: u32 = 0o700; +#[cfg(unix)] +const FILE_MODE: u32 = 0o600; + +#[derive(Debug)] +pub(crate) struct FileStore { + base: PathBuf, + generation: OnceLock>, + next_fingerprint: AtomicU64, +} + +#[derive(Debug)] +struct Generation { + root: PathBuf, + source: PathBuf, +} + +#[derive(Clone, Debug)] +pub(crate) struct Lease { + inner: Arc, +} + +#[derive(Debug)] +struct LeaseInner { + path: PathBuf, + file: File, + generation: Arc, + metadata: fs::Metadata, + len: usize, + fingerprint: u64, +} + +impl Default for FileStore { + fn default() -> Self { + Self::new(runtime_base()) + } +} + +impl FileStore { + fn new(base: PathBuf) -> Self { + Self { + base, + generation: OnceLock::new(), + next_fingerprint: AtomicU64::new(1), + } + } + + pub(crate) fn source_directory(&self) -> io::Result { + Ok(self.generation()?.source.clone()) + } + + pub(crate) fn lease(&self, path: &Path, expected_len: usize) -> io::Result { + let generation = self.generation()?; + validate_child(path, &generation.source)?; + let file = open_no_follow(path)?; + let metadata = file.metadata()?; + validate_metadata(&metadata, expected_len)?; + validate_path_identity(path, &metadata)?; + let fingerprint = self.next_fingerprint.fetch_add(1, Ordering::Relaxed); + Ok(Lease { + inner: Arc::new(LeaseInner { + path: path.to_owned(), + file, + generation, + metadata, + len: expected_len, + fingerprint, + }), + }) + } + + fn generation(&self) -> io::Result> { + if let Some(generation) = self.generation.get() { + return Ok(Arc::clone(generation)); + } + let generation = Arc::new(create_generation(&self.base)?); + match self.generation.set(Arc::clone(&generation)) { + Ok(()) => Ok(generation), + Err(_) => self + .generation + .get() + .cloned() + .ok_or_else(|| io::Error::other("pane graphics generation was lost")), + } + } + + #[cfg(test)] + pub(crate) fn is_initialized(&self) -> bool { + self.generation.get().is_some() + } +} + +impl Lease { + pub(crate) fn path(&self) -> &Path { + &self.inner.path + } + + pub(crate) fn len(&self) -> usize { + self.inner.len + } + + pub(crate) fn fingerprint(&self) -> u64 { + self.inner.fingerprint + } + + pub(crate) fn copy_rgba(&self) -> io::Result> { + let _keep_generation_alive = &self.inner.generation; + validate_path_identity(&self.inner.path, &self.inner.metadata)?; + let mut data = vec![0; self.inner.len]; + read_exact_at(&self.inner.file, &mut data)?; + if has_byte_at(&self.inner.file, self.inner.len as u64)? { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "frame length changed while leased", + )); + } + validate_metadata(&self.inner.file.metadata()?, self.inner.len)?; + validate_path_identity(&self.inner.path, &self.inner.metadata)?; + Ok(data) + } +} + +impl Drop for Generation { + fn drop(&mut self) { + if let Err(err) = fs::remove_dir_all(&self.root) { + if err.kind() != io::ErrorKind::NotFound { + tracing::warn!(path = %self.root.display(), err = %err, "failed to remove pane graphics directory"); + } + } + } +} + +#[cfg(unix)] +pub(crate) fn validate_direct_source(path: &Path, expected_len: usize) -> io::Result<()> { + let source = path.parent().ok_or_else(invalid_path)?; + let generation = source.parent().ok_or_else(invalid_path)?; + if !path.is_absolute() + || path.file_name().is_none() + || source.file_name().and_then(|name| name.to_str()) != Some("source") + || generation.parent() != Some(runtime_base().as_path()) + || !generation + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("server-")) + { + return Err(invalid_path()); + } + for directory in [runtime_base(), generation.to_owned(), source.to_owned()] { + validate_directory(&directory)?; + } + let file = open_no_follow(path)?; + let metadata = file.metadata()?; + validate_metadata(&metadata, expected_len)?; + validate_path_identity(path, &metadata) +} + +fn create_generation(base: &Path) -> io::Result { + #[cfg(not(unix))] + { + let _ = base; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "file-backed pane graphics require Unix", + )) + } + #[cfg(unix)] + { + fs::create_dir_all(base)?; + fs::set_permissions(base, fs::Permissions::from_mode(DIRECTORY_MODE))?; + validate_directory(base)?; + remove_stale_generations(base); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let root = base.join(format!("server-{}-{nonce}", std::process::id())); + fs::create_dir(&root)?; + fs::set_permissions(&root, fs::Permissions::from_mode(DIRECTORY_MODE))?; + let source = root.join("source"); + fs::create_dir(&source)?; + fs::set_permissions(&source, fs::Permissions::from_mode(DIRECTORY_MODE))?; + validate_directory(&root)?; + validate_directory(&source)?; + Ok(Generation { root, source }) + } +} + +fn runtime_base() -> PathBuf { + #[cfg(unix)] + { + let root = std::env::var_os("XDG_RUNTIME_DIR") + .map(PathBuf::from) + .filter(|path| path.is_absolute()) + .unwrap_or_else(|| PathBuf::from("/var/tmp")); + root.join(format!("herdr-pane-graphics-{}", effective_uid())) + } + #[cfg(not(unix))] + { + PathBuf::from("pane-graphics-unavailable") + } +} + +#[cfg(unix)] +fn read_exact_at(file: &File, data: &mut [u8]) -> io::Result<()> { + file.read_exact_at(data, 0) +} + +#[cfg(not(unix))] +fn read_exact_at(_file: &File, _data: &mut [u8]) -> io::Result<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only")) +} + +#[cfg(unix)] +fn has_byte_at(file: &File, offset: u64) -> io::Result { + Ok(file.read_at(&mut [0], offset)? != 0) +} + +#[cfg(not(unix))] +fn has_byte_at(_file: &File, _offset: u64) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only")) +} + +#[cfg(unix)] +fn remove_stale_generations(base: &Path) { + let Ok(entries) = fs::read_dir(base) else { + return; + }; + for entry in entries.flatten() { + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|name| name.strip_prefix("server-")) + .and_then(|name| name.split('-').next()) + .and_then(|pid| pid.parse::().ok()) + .filter(|pid| *pid > 0) + else { + continue; + }; + let Ok(file_type) = entry.file_type() else { + continue; + }; + if !file_type.is_dir() { + continue; + } + // SAFETY: kill with signal 0 probes process existence without sending a signal. + let alive = unsafe { libc::kill(pid, 0) } == 0; + if alive || io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) { + continue; + } + if let Err(err) = fs::remove_dir_all(entry.path()) { + tracing::warn!(path = %entry.path().display(), err = %err, "failed to remove stale pane graphics directory"); + } + } +} + +fn validate_child(path: &Path, directory: &Path) -> io::Result<()> { + if !path.is_absolute() || path.parent() != Some(directory) || path.file_name().is_none() { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "frame path must be a direct child of the advertised source directory", + )); + } + Ok(()) +} + +fn open_no_follow(path: &Path) -> io::Result { + #[cfg(unix)] + { + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path) + } + #[cfg(not(unix))] + { + let _ = path; + Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only")) + } +} + +fn validate_metadata(metadata: &fs::Metadata, expected_len: usize) -> io::Result<()> { + #[cfg(unix)] + { + if !metadata.file_type().is_file() + || metadata.uid() != effective_uid() + || metadata.mode() & 0o777 != FILE_MODE + || metadata.nlink() != 1 + || metadata.len() != expected_len as u64 + { + return Err(invalid( + "frame must be same-uid regular 0600 single-link exact-length file", + )); + } + Ok(()) + } + #[cfg(not(unix))] + { + let _ = (metadata, expected_len); + Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only")) + } +} + +#[cfg(unix)] +fn validate_directory(path: &Path) -> io::Result<()> { + let metadata = fs::symlink_metadata(path)?; + if !metadata.file_type().is_dir() + || metadata.uid() != effective_uid() + || metadata.mode() & 0o777 != DIRECTORY_MODE + { + return Err(invalid("pane graphics directory is not private")); + } + Ok(()) +} + +fn validate_path_identity(path: &Path, expected: &fs::Metadata) -> io::Result<()> { + #[cfg(unix)] + { + let actual = fs::symlink_metadata(path)?; + if actual.dev() != expected.dev() || actual.ino() != expected.ino() { + return Err(invalid("frame path changed while leased")); + } + Ok(()) + } + #[cfg(not(unix))] + { + let _ = (path, expected); + Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only")) + } +} + +#[cfg(unix)] +fn effective_uid() -> u32 { + // SAFETY: geteuid takes no arguments and has no preconditions. + unsafe { libc::geteuid() } +} + +#[cfg(unix)] +fn invalid(message: &str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} + +#[cfg(unix)] +fn invalid_path() -> io::Error { + io::Error::new( + io::ErrorKind::PermissionDenied, + "invalid pane graphics source path", + ) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::io::Write; + use std::os::unix::fs::{symlink, PermissionsExt}; + use std::sync::atomic::AtomicU64; + + static NEXT: AtomicU64 = AtomicU64::new(1); + + fn store() -> (FileStore, PathBuf) { + let base = PathBuf::from(format!( + "/var/tmp/herdr-graphics-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + (FileStore::new(base.clone()), base) + } + + fn frame(store: &FileStore, name: &str, data: &[u8]) -> PathBuf { + let path = store.source_directory().unwrap().join(name); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(FILE_MODE) + .open(&path) + .unwrap(); + file.write_all(data).unwrap(); + path + } + + #[test] + fn source_directory_is_lazy_private_and_removed_with_store() { + let (store, base) = store(); + assert!(!store.is_initialized()); + let source = store.source_directory().unwrap(); + assert!(store.is_initialized()); + validate_directory(&source).unwrap(); + drop(store); + assert!(!source.exists()); + let _ = fs::remove_dir(base); + } + + #[test] + fn lease_holds_exact_file_and_copies_from_validated_fd() { + let (store, base) = store(); + let path = frame(&store, "frame", &[1, 2, 3, 4]); + let lease = store.lease(&path, 4).unwrap(); + assert_eq!(lease.path(), path); + assert_eq!(lease.len(), 4); + assert!(lease.fingerprint() > 0); + assert_eq!(lease.copy_rgba().unwrap(), [1, 2, 3, 4]); + drop(lease); + drop(store); + let _ = fs::remove_dir(base); + } + + #[test] + fn validation_rejects_links_modes_lengths_and_outside_paths() { + let cases = ["mode", "length", "hard-link", "symlink", "outside"]; + for case in cases { + let (store, base) = store(); + let path = frame(&store, "frame", &[1, 2, 3, 4]); + let result = match case { + "mode" => { + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + store.lease(&path, 4) + } + "length" => store.lease(&path, 8), + "hard-link" => { + fs::hard_link(&path, store.source_directory().unwrap().join("other")).unwrap(); + store.lease(&path, 4) + } + "symlink" => { + let link = store.source_directory().unwrap().join("link"); + symlink(&path, &link).unwrap(); + store.lease(&link, 4) + } + "outside" => store.lease(&base.join("outside"), 4), + _ => unreachable!(), + }; + assert!(result.is_err(), "{case}"); + drop(store); + let _ = fs::remove_dir_all(base); + } + } + + #[test] + fn direct_source_validation_accepts_only_the_runtime_generation() { + let runtime_store = FileStore::default(); + let runtime_path = frame(&runtime_store, "direct", &[1, 2, 3, 4]); + validate_direct_source(&runtime_path, 4).unwrap(); + assert!(validate_direct_source(&runtime_path, 8).is_err()); + assert!(validate_direct_source(Path::new("relative"), 4).is_err()); + + let generation = runtime_path.parent().unwrap().parent().unwrap(); + let wrong_source = generation.join("frames"); + fs::create_dir(&wrong_source).unwrap(); + fs::set_permissions(&wrong_source, fs::Permissions::from_mode(DIRECTORY_MODE)).unwrap(); + let wrong_source_path = wrong_source.join("frame"); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(FILE_MODE) + .open(&wrong_source_path) + .unwrap(); + file.write_all(&[1, 2, 3, 4]).unwrap(); + assert!(validate_direct_source(&wrong_source_path, 4).is_err()); + + let (other_store, other_base) = store(); + let other_path = frame(&other_store, "other", &[1, 2, 3, 4]); + assert!(validate_direct_source(&other_path, 4).is_err()); + drop(other_store); + let _ = fs::remove_dir_all(other_base); + + let invalid_generation = runtime_base().join(format!( + "invalid-generation-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let invalid_source = invalid_generation.join("source"); + fs::create_dir_all(&invalid_source).unwrap(); + fs::set_permissions( + &invalid_generation, + fs::Permissions::from_mode(DIRECTORY_MODE), + ) + .unwrap(); + fs::set_permissions(&invalid_source, fs::Permissions::from_mode(DIRECTORY_MODE)).unwrap(); + let invalid_path = invalid_source.join("frame"); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(FILE_MODE) + .open(&invalid_path) + .unwrap(); + file.write_all(&[1, 2, 3, 4]).unwrap(); + assert!(validate_direct_source(&invalid_path, 4).is_err()); + let _ = fs::remove_dir_all(invalid_generation); + } + + #[test] + fn generation_creation_removes_dead_server_directories() { + let (store, base) = store(); + fs::create_dir_all(&base).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(DIRECTORY_MODE)).unwrap(); + let stale = base.join("server-2147483647-stale"); + fs::create_dir(&stale).unwrap(); + fs::set_permissions(&stale, fs::Permissions::from_mode(DIRECTORY_MODE)).unwrap(); + + store.source_directory().unwrap(); + + assert!(!stale.exists()); + drop(store); + let _ = fs::remove_dir_all(base); + } + + #[test] + fn replacement_after_lease_is_detected_before_fallback_copy() { + let (store, base) = store(); + let path = frame(&store, "frame", &[1, 2, 3, 4]); + let lease = store.lease(&path, 4).unwrap(); + fs::remove_file(&path).unwrap(); + let _replacement = frame(&store, "frame", &[5, 6, 7, 8]); + assert!(lease.copy_rgba().is_err()); + drop(lease); + drop(store); + let _ = fs::remove_dir_all(base); + } +} diff --git a/src/protocol/wire.rs b/src/protocol/wire.rs index 4af2c794..4a445361 100644 --- a/src/protocol/wire.rs +++ b/src/protocol/wire.rs @@ -57,6 +57,8 @@ pub enum ClientKeybindings { pub enum ClientLaunchMode { /// Full app client. App, + /// Full app client eligible for audited local direct graphics. + AppDirectGraphics, /// Direct terminal attach client. TerminalAttach, } @@ -428,6 +430,25 @@ pub enum ClientMessage { /// Replace an existing writable controller for this terminal. takeover: bool, }, + + /// Result of the one armed Herdr-owned direct Kitty transmission. + GraphicsTransmissionResult { + transfer_id: u64, + image_id: u32, + success: bool, + }, + + /// One confirmed SGR 1016 mouse report with read-time host geometry. + InputPixels { + data: Vec, + cols: u16, + rows: u16, + width_px: u32, + height_px: u32, + }, + + /// The direct command was written and flushed; terminal response timing starts now. + GraphicsTransmissionStarted { transfer_id: u64, image_id: u32 }, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -696,6 +717,8 @@ pub enum ServerMessage { MouseCapture { /// True when Herdr mouse UI is enabled or the focused pane app requests mouse reporting. enabled: bool, + /// True only while the focused pane requests DEC SGR pixel mode 1016. + sgr_pixels: bool, }, /// Whether the focused terminal requests Kitty report-all keyboard input. @@ -717,6 +740,19 @@ pub enum ServerMessage { /// Number of BEL characters parsed from one PTY read. count: u16, }, + + /// One validated Herdr-owned Kitty regular-file RGBA transmission. + GraphicsFile { + path: String, + expected_len: u64, + image_id: u32, + transfer_id: u64, + leading: Vec, + control: String, + }, + + /// Suppress a direct command that expired before terminal delivery. + GraphicsTransmissionRetired { transfer_id: u64, image_id: u32 }, } // --------------------------------------------------------------------------- @@ -1148,7 +1184,7 @@ mod tests { ], }; let encoded = bincode::serde::encode_to_vec(&msg, bincode::config::standard()).unwrap(); - // Freeze the protocol 19 input envelope before it is published. + // Freeze the protocol 20 input envelope before it is published. assert_eq!( encoded, vec![ @@ -1534,7 +1570,10 @@ mod tests { #[test] fn server_mouse_capture_roundtrip() { - let msg = ServerMessage::MouseCapture { enabled: true }; + let msg = ServerMessage::MouseCapture { + enabled: true, + sgr_pixels: true, + }; let encoded = bincode::serde::encode_to_vec(&msg, bincode::config::standard()).unwrap(); let (decoded, _): (ServerMessage, _) = bincode::serde::decode_from_slice(&encoded, bincode::config::standard()).unwrap(); @@ -1550,6 +1589,44 @@ mod tests { assert_eq!(msg, decoded); } + #[test] + fn direct_graphics_messages_roundtrip() { + let client = ClientMessage::GraphicsTransmissionResult { + transfer_id: 7, + image_id: 42, + success: false, + }; + let encoded = bincode::serde::encode_to_vec(&client, bincode::config::standard()).unwrap(); + let (decoded, _): (ClientMessage, _) = + bincode::serde::decode_from_slice(&encoded, bincode::config::standard()).unwrap(); + assert_eq!(client, decoded); + + let server = ServerMessage::GraphicsFile { + path: "/run/user/1000/herdr/source/frame".into(), + expected_len: 4, + image_id: 42, + transfer_id: 7, + leading: b"\x1b[2;3H".to_vec(), + control: "a=T,f=32,i=42,q=0".into(), + }; + let encoded = bincode::serde::encode_to_vec(&server, bincode::config::standard()).unwrap(); + let (decoded, _): (ServerMessage, _) = + bincode::serde::decode_from_slice(&encoded, bincode::config::standard()).unwrap(); + assert_eq!(server, decoded); + + let pixels = ClientMessage::InputPixels { + data: b"\x1b[<35;321;241M".to_vec(), + cols: 80, + rows: 24, + width_px: 800, + height_px: 480, + }; + let encoded = bincode::serde::encode_to_vec(&pixels, bincode::config::standard()).unwrap(); + let (decoded, _): (ClientMessage, _) = + bincode::serde::decode_from_slice(&encoded, bincode::config::standard()).unwrap(); + assert_eq!(pixels, decoded); + } + #[test] fn server_prefix_input_source_roundtrip() { for active in [true, false] { diff --git a/src/server/alt_screen_read.rs b/src/server/alt_screen_read.rs index 240e96de..425d4a41 100644 --- a/src/server/alt_screen_read.rs +++ b/src/server/alt_screen_read.rs @@ -373,7 +373,11 @@ fn send_wheel( let column = snapshot.cols.saturating_sub(1) / 2; let row = u16::try_from(snapshot.rows.len().saturating_sub(1) / 2).unwrap_or(0); let event = runtime - .encode_mouse_wheel(kind, column, row, KeyModifiers::empty()) + .encode_mouse_wheel( + kind, + crate::input::mouse::Position::Cell { column, row }, + KeyModifiers::empty(), + ) .ok_or(())?; let mut bytes = Vec::with_capacity(event.len().saturating_mul(events)); for _ in 0..events { diff --git a/src/server/client_transport.rs b/src/server/client_transport.rs index 2675b4c3..b5023b97 100644 --- a/src/server/client_transport.rs +++ b/src/server/client_transport.rs @@ -41,6 +41,8 @@ const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(4); const MAX_INPUT_PAYLOAD: usize = 1024 * 1024; // 1 MB /// Maximum structured input events accepted in one client message. const MAX_INPUT_EVENT_BATCH: usize = 4096; +/// Maximum encoded mouse report accepted with pixel geometry. +const MAX_PIXEL_MOUSE_PAYLOAD: usize = 128; /// Channels owned by the server side of a client writer thread. #[derive(Clone, Debug)] @@ -51,115 +53,99 @@ pub(crate) struct ClientWriter { pub(crate) render: ClientRenderWriter, } -#[cfg(test)] impl ClientWriter { + pub(crate) fn replace_with_cleanup(&self, data: Vec) { + self.render.queue.replace_with_cleanup(data); + } + + #[cfg(test)] + pub(crate) fn test_fill_render(&self, data: Vec) { + self.render.try_send(data).unwrap(); + } + + #[cfg(test)] + pub(crate) fn test_close(&self) { + self.render.queue.close_writer(); + } + + #[cfg(test)] pub(crate) fn test_channel( control: std::sync::mpsc::Sender>, render: std::sync::mpsc::SyncSender>, ) -> Self { - Self { - control: ClientControlWriter { - target: ClientControlTarget::Channel(control), - }, - render: ClientRenderWriter { - target: ClientRenderTarget::Channel(render), - }, - } + let queue = ClientWriterQueue::new(); + let drain = queue.clone(); + let control_writer = ClientControlWriter::queue(queue.clone()); + let mut render_writer = ClientRenderWriter::queue(queue); + render_writer.test_render = Some(render.clone()); + let writer = Self { + control: control_writer, + render: render_writer, + }; + std::thread::spawn(move || { + while let Some(item) = drain.recv() { + let sent = match item { + ClientWriteItem::Control(data) => control.send(data).is_ok(), + ClientWriteItem::Render(data) => render.send(data).is_ok(), + }; + if !sent { + break; + } + } + drain.close_writer(); + }); + writer } } #[derive(Debug)] pub(crate) struct ClientControlWriter { - target: ClientControlTarget, -} - -#[derive(Debug)] -enum ClientControlTarget { - Queue(Arc), + queue: Arc, #[cfg(test)] - Channel(std::sync::mpsc::Sender>), + test_render: Option>>, } #[derive(Debug)] pub(crate) struct ClientRenderWriter { - target: ClientRenderTarget, -} - -#[derive(Debug)] -enum ClientRenderTarget { - Queue(Arc), + queue: Arc, #[cfg(test)] - Channel(std::sync::mpsc::SyncSender>), + test_render: Option>>, } -impl Clone for ClientControlWriter { - fn clone(&self) -> Self { - match &self.target { - ClientControlTarget::Queue(queue) => { - queue.add_sender(); +macro_rules! writer_handle { + ($type:ty) => { + impl Clone for $type { + fn clone(&self) -> Self { + self.queue.add_sender(); Self { - target: ClientControlTarget::Queue(queue.clone()), + queue: self.queue.clone(), + #[cfg(test)] + test_render: self.test_render.clone(), } } - #[cfg(test)] - ClientControlTarget::Channel(sender) => Self { - target: ClientControlTarget::Channel(sender.clone()), - }, } - } -} - -impl Drop for ClientControlWriter { - fn drop(&mut self) { - match &self.target { - ClientControlTarget::Queue(queue) => queue.remove_sender(), - #[cfg(test)] - ClientControlTarget::Channel(_) => {} + impl Drop for $type { + fn drop(&mut self) { + self.queue.remove_sender(); + } } - } + }; } +writer_handle!(ClientControlWriter); +writer_handle!(ClientRenderWriter); impl ClientControlWriter { fn queue(queue: Arc) -> Self { queue.add_sender(); Self { - target: ClientControlTarget::Queue(queue), + queue, + #[cfg(test)] + test_render: None, } } pub(crate) fn send(&self, data: Vec) -> Result<(), SendError>> { - match &self.target { - ClientControlTarget::Queue(queue) => queue.send_control(data), - #[cfg(test)] - ClientControlTarget::Channel(sender) => sender.send(data), - } - } -} - -impl Clone for ClientRenderWriter { - fn clone(&self) -> Self { - match &self.target { - ClientRenderTarget::Queue(queue) => { - queue.add_sender(); - Self { - target: ClientRenderTarget::Queue(queue.clone()), - } - } - #[cfg(test)] - ClientRenderTarget::Channel(sender) => Self { - target: ClientRenderTarget::Channel(sender.clone()), - }, - } - } -} - -impl Drop for ClientRenderWriter { - fn drop(&mut self) { - match &self.target { - ClientRenderTarget::Queue(queue) => queue.remove_sender(), - #[cfg(test)] - ClientRenderTarget::Channel(_) => {} - } + self.queue.send_control(data) } } @@ -167,16 +153,22 @@ impl ClientRenderWriter { fn queue(queue: Arc) -> Self { queue.add_sender(); Self { - target: ClientRenderTarget::Queue(queue), + queue, + #[cfg(test)] + test_render: None, } } pub(crate) fn try_send(&self, data: Vec) -> Result<(), TrySendError>> { - match &self.target { - ClientRenderTarget::Queue(queue) => queue.try_send_render(data), - #[cfg(test)] - ClientRenderTarget::Channel(sender) => sender.try_send(data), + #[cfg(test)] + if let Some(sender) = &self.test_render { + return sender.try_send(data); } + self.queue.try_send_render(data) + } + + pub(crate) fn send_ordered(&self, data: Vec) -> Result<(), TrySendError>> { + self.queue.send_ordered(data) } } @@ -189,6 +181,7 @@ struct ClientWriterQueue { #[derive(Debug, Default)] struct ClientWriterQueueState { control: VecDeque>, + ordered: VecDeque>, render: Option>, senders: usize, writer_alive: bool, @@ -245,16 +238,45 @@ impl ClientWriterQueue { Ok(()) } + fn send_ordered(&self, data: Vec) -> Result<(), TrySendError>> { + let mut state = self.lock_state(); + if !state.writer_alive { + return Err(TrySendError::Disconnected(data)); + } + if !state.ordered.is_empty() { + return Err(TrySendError::Full(data)); + } + if let Some(older) = state.render.take() { + state.ordered.push_back(older); + } + state.ordered.push_back(data); + self.ready.notify_one(); + Ok(()) + } + + fn replace_with_cleanup(&self, data: Vec) { + let mut state = self.lock_state(); + state.render = None; + state.ordered.clear(); + if state.writer_alive { + state.control.push_back(data); + self.ready.notify_one(); + } + } + fn recv(&self) -> Option { let mut state = self.lock_state(); loop { if let Some(data) = state.control.pop_front() { return Some(ClientWriteItem::Control(data)); } - if let Some(data) = state.render.take() { + if let Some(data) = state.ordered.pop_front() { self.ready.notify_one(); return Some(ClientWriteItem::Render(data)); } + if let Some(data) = state.render.take() { + return Some(ClientWriteItem::Render(data)); + } if state.senders == 0 { return None; } @@ -268,6 +290,8 @@ impl ClientWriterQueue { fn close_writer(&self) { let mut state = self.lock_state(); state.writer_alive = false; + state.render = None; + state.ordered.clear(); self.ready.notify_all(); } @@ -291,10 +315,29 @@ pub(crate) enum ServerEvent { render_encoding: RenderEncoding, keybindings: Option>, direct_attach_requested: bool, + direct_graphics: bool, writer: ClientWriter, }, /// A client sent an input message. ClientInput { client_id: u64, data: Vec }, + /// A client reported the one armed Kitty regular-file response. + GraphicsTransmissionResult { + client_id: u64, + transfer_id: u64, + image_id: u32, + success: bool, + }, + GraphicsTransmissionStarted { + client_id: u64, + transfer_id: u64, + image_id: u32, + }, + /// One confirmed SGR pixel report with client read-time geometry. + ClientInputPixels { + client_id: u64, + data: Vec, + geometry: crate::input::mouse::HostGeometry, + }, /// A client sent structured input events. ClientInputEvents { client_id: u64, @@ -515,6 +558,7 @@ pub(crate) fn handle_client_handshake( render_encoding, keybindings, direct_attach_requested, + direct_graphics, ) = match hello { ClientMessage::Hello { version, @@ -564,6 +608,7 @@ pub(crate) fn handle_client_handshake( requested_encoding, keybindings, launch_mode == ClientLaunchMode::TerminalAttach, + launch_mode == ClientLaunchMode::AppDirectGraphics, ) } _ => { @@ -618,6 +663,7 @@ pub(crate) fn handle_client_handshake( render_encoding, keybindings, direct_attach_requested, + direct_graphics, writer, }); @@ -633,19 +679,17 @@ fn client_writer_loop( server_event_tx: mpsc::Sender, ) { while let Some(item) = writer_queue.recv() { - match item { - ClientWriteItem::Control(data) => { - if !write_framed_bytes(&mut stream, &data) { - break; - } - } + let written = match item { + ClientWriteItem::Control(data) => write_framed_bytes(&mut stream, &data), ClientWriteItem::Render(data) => { let _ = server_event_tx.blocking_send(ServerEvent::ClientWriterDrained { client_id }); - if !write_framed_bytes(&mut stream, &data) { - break; - } + write_framed_bytes(&mut stream, &data) } + }; + if !written { + let _ = server_event_tx.blocking_send(ServerEvent::ClientDisconnected { client_id }); + break; } } writer_queue.close_writer(); @@ -728,6 +772,47 @@ fn client_read_loop( ServerEvent::ClientInput { client_id, data } } } + ClientMessage::InputPixels { + data, + cols, + rows, + width_px, + height_px, + } => { + let Some(geometry) = + crate::input::mouse::HostGeometry::new(cols, rows, width_px, height_px) + else { + warn!( + client_id, + cols, + rows, + width_px, + height_px, + "invalid pixel mouse geometry from client, closing" + ); + let _ = server_event_tx + .blocking_send(ServerEvent::ClientDisconnected { client_id }); + break; + }; + if data.len() > MAX_PIXEL_MOUSE_PAYLOAD + || crate::input::mouse::parse_report(&data).is_none() + { + warn!( + client_id, + size = data.len(), + max = MAX_PIXEL_MOUSE_PAYLOAD, + "invalid pixel mouse report from client, closing" + ); + let _ = server_event_tx + .blocking_send(ServerEvent::ClientDisconnected { client_id }); + break; + } + ServerEvent::ClientInputPixels { + client_id, + data, + geometry, + } + } ClientMessage::InputEvents { events } => match input_event_limit(&events) { InputEventLimit::WithinLimits => { ServerEvent::ClientInputEvents { client_id, events } @@ -777,6 +862,24 @@ fn client_read_loop( takeover, } } + ClientMessage::GraphicsTransmissionResult { + transfer_id, + image_id, + success, + } => ServerEvent::GraphicsTransmissionResult { + client_id, + transfer_id, + image_id, + success, + }, + ClientMessage::GraphicsTransmissionStarted { + transfer_id, + image_id, + } => ServerEvent::GraphicsTransmissionStarted { + client_id, + transfer_id, + image_id, + }, ClientMessage::ClipboardImage { extension, data } => { if data.len() > MAX_CLIPBOARD_IMAGE_PAYLOAD { warn!( @@ -947,6 +1050,30 @@ mod tests { )); } + #[test] + fn ordered_direct_follows_older_render_and_stays_bounded() { + let (writer, queue) = test_queue_writer(); + writer.render.try_send(b"old".to_vec()).unwrap(); + writer.render.send_ordered(b"direct".to_vec()).unwrap(); + assert!(matches!( + writer.render.send_ordered(b"second".to_vec()), + Err(TrySendError::Full(_)) + )); + writer.render.try_send(b"new".to_vec()).unwrap(); + + for expected in [b"old".as_slice(), b"direct", b"new"] { + assert_eq!( + queue.recv(), + Some(ClientWriteItem::Render(expected.to_vec())) + ); + } + queue.close_writer(); + assert!(matches!( + writer.render.send_ordered(b"closed".to_vec()), + Err(TrySendError::Disconnected(_)) + )); + } + #[test] fn client_writer_prioritizes_control_and_reports_render_drain() { let (mut client_stream, server_stream, _path) = local_stream_pair("client-writer-priority"); @@ -1203,6 +1330,7 @@ new_tab = "ctrl+notakey" render_encoding, keybindings, direct_attach_requested, + direct_graphics, writer, } => { assert_eq!(client_id, 42); @@ -1211,6 +1339,7 @@ new_tab = "ctrl+notakey" assert_eq!(render_encoding, RenderEncoding::TerminalAnsi); assert!(keybindings.is_none()); assert!(!direct_attach_requested); + assert!(!direct_graphics); drop(writer); } other => panic!("expected ClientConnected, got {other:?}"), @@ -1394,6 +1523,76 @@ new_tab = "ctrl+notakey" .expect("read thread result"); } + #[test] + fn client_read_loop_disconnects_invalid_pixel_mouse_geometry() { + let (mut client_stream, server_stream, _path) = + local_stream_pair("client-read-invalid-pixel-geometry"); + let (server_event_tx, mut server_event_rx) = mpsc::channel(4); + let should_quit = Arc::new(AtomicBool::new(false)); + let read_quit = should_quit.clone(); + let handle = std::thread::spawn(move || { + client_read_loop(server_stream, 7, &server_event_tx, &read_quit) + }); + + protocol::write_message( + &mut client_stream, + &ClientMessage::InputPixels { + data: b"\x1b[<35;1;1M".to_vec(), + cols: 0, + rows: 24, + width_px: 800, + height_px: 480, + }, + ) + .expect("write invalid pixel geometry"); + + assert!(matches!( + recv_server_event(&mut server_event_rx, "invalid pixel geometry disconnect"), + ServerEvent::ClientDisconnected { client_id: 7 } + )); + drop(client_stream); + should_quit.store(true, Ordering::Release); + handle + .join() + .expect("read thread join") + .expect("read thread result"); + } + + #[test] + fn client_read_loop_disconnects_invalid_pixel_mouse_report() { + let (mut client_stream, server_stream, _path) = + local_stream_pair("client-read-invalid-pixel-report"); + let (server_event_tx, mut server_event_rx) = mpsc::channel(4); + let should_quit = Arc::new(AtomicBool::new(false)); + let read_quit = should_quit.clone(); + let handle = std::thread::spawn(move || { + client_read_loop(server_stream, 7, &server_event_tx, &read_quit) + }); + + protocol::write_message( + &mut client_stream, + &ClientMessage::InputPixels { + data: vec![b'x'; MAX_PIXEL_MOUSE_PAYLOAD + 1], + cols: 80, + rows: 24, + width_px: 800, + height_px: 480, + }, + ) + .expect("write invalid pixel report"); + + assert!(matches!( + recv_server_event(&mut server_event_rx, "invalid pixel report disconnect"), + ServerEvent::ClientDisconnected { client_id: 7 } + )); + drop(client_stream); + should_quit.store(true, Ordering::Release); + handle + .join() + .expect("read thread join") + .expect("read thread result"); + } + #[test] fn client_read_loop_disconnects_marker_wrapped_invalid_utf8() { let (mut client_stream, server_stream, _path) = diff --git a/src/server/clients.rs b/src/server/clients.rs index bb0f31c6..0d3d9ae0 100644 --- a/src/server/clients.rs +++ b/src/server/clients.rs @@ -24,7 +24,6 @@ pub(crate) type RenderTarget = ( pub(crate) enum DeferredRender { #[default] None, - Graphics, Full, } @@ -56,14 +55,18 @@ pub(crate) struct ClientConnection { pub(crate) render_state: ClientRenderState, /// Client-local host Kitty graphics cache. pub(crate) graphics_cache: crate::kitty_graphics::HostGraphicsCache, + /// Passive eligibility for audited local Kitty regular-file graphics. + pub(crate) direct_graphics: bool, + /// Whether this frontend preserves exact SGR pixel reports. + pub(crate) pixel_mouse: bool, /// Whether the next graphics frame must clear and rebuild host-side Kitty state. pub(crate) graphics_surface_reset_pending: bool, /// Whether an ordinary render was skipped because the render channel was full. pub(crate) render_pending: bool, - /// Whether a pane-graphics-only render was skipped because the channel was full. - pane_graphics_render_pending: bool, /// Last host mouse capture mode sent to this client. pub(crate) host_mouse_capture_active: Option, + /// Last SGR pixel provenance mode sent to this client. + pub(crate) host_sgr_pixels_active: Option, /// Last Kitty report-all mode sent to this client's host terminal. pub(crate) host_keyboard_report_all_active: Option, /// Temporary files staged from this client's local clipboard image pastes. @@ -125,10 +128,12 @@ impl ClientConnection { last_activity, render_state: ClientRenderState::new(render_encoding), graphics_cache: crate::kitty_graphics::HostGraphicsCache::default(), + direct_graphics: false, + pixel_mouse: false, graphics_surface_reset_pending: false, render_pending: false, - pane_graphics_render_pending: false, host_mouse_capture_active: None, + host_sgr_pixels_active: None, host_keyboard_report_all_active: None, staged_clipboard_files: Vec::new(), writer, @@ -137,14 +142,11 @@ impl ClientConnection { pub(crate) fn request_repaint(&mut self) { self.render_state.request_repaint(); - self.pane_graphics_render_pending = false; } pub(crate) fn deferred_render(&self) -> DeferredRender { if self.render_pending { DeferredRender::Full - } else if self.pane_graphics_render_pending { - DeferredRender::Graphics } else { DeferredRender::None } @@ -152,18 +154,10 @@ impl ClientConnection { pub(crate) fn clear_deferred_render(&mut self) { self.render_pending = false; - self.pane_graphics_render_pending = false; } pub(crate) fn defer_full_render(&mut self) { self.render_pending = true; - self.pane_graphics_render_pending = false; - } - - pub(crate) fn defer_pane_graphics_render(&mut self) { - if !self.render_pending { - self.pane_graphics_render_pending = true; - } } pub(crate) fn take_deferred_render(&mut self) -> DeferredRender { diff --git a/src/server/headless.rs b/src/server/headless.rs index 5a90177b..61c825b8 100644 --- a/src/server/headless.rs +++ b/src/server/headless.rs @@ -44,7 +44,6 @@ use crate::ipc::{ }; use crate::protocol::{ self, AttachScrollDirection, AttachScrollSource, FrameData, ServerMessage, MAX_FRAME_SIZE, - MAX_GRAPHICS_FRAME_SIZE, }; #[cfg(unix)] use crate::server::client_accept::{ @@ -66,8 +65,7 @@ use crate::server::terminal_attach::paste_payload_for_runtime; mod pane_graphics; -#[cfg(test)] -use pane_graphics::frame_pane_graphics_for_client; +use crate::protocol::MAX_GRAPHICS_FRAME_SIZE; use pane_graphics::RetainedGraphicsOutcome; #[cfg(test)] @@ -367,12 +365,13 @@ fn apply_terminal_attach_scroll( match runtime.wheel_routing() { Some(crate::pane::WheelRouting::MouseReport) => { runtime.scroll_reset(); - let column = column.unwrap_or(0); - let row = row.unwrap_or(0); + let position = crate::input::mouse::Position::Cell { + column: column.unwrap_or(0), + row: row.unwrap_or(0), + }; let Some(bytes) = runtime.encode_mouse_wheel( wheel_kind, - column, - row, + position, KeyModifiers::from_bits_truncate(modifiers), ) else { return Err(format!( @@ -671,7 +670,14 @@ impl HeadlessServer { crate::render_prof::event("full_render_cause.default_workspace"); } - self.cancel_inactive_pane_graphics_streams(); + if self.app.pane_graphics.retain_live_panes(&self.app.state) { + needs_render = true; + needs_graphics_render = true; + } + if self.expire_direct_graphics(now) { + needs_render = true; + needs_graphics_render = true; + } self.drain_client_config_reload_request(); self.stream_host_mouse_capture_mode(); @@ -1002,6 +1008,7 @@ impl HeadlessServer { }, respond_to, response_write_complete: None, + stream_active: None, }, true, ); @@ -1083,6 +1090,15 @@ impl HeadlessServer { } fn sync_foreground_client_state(&mut self) { + self.app.direct_graphics_available = self.direct_graphics_available(); + self.app.pixel_mouse_available = self.foreground_client_id.is_some_and(|id| { + self.clients + .get(&id) + .is_some_and(|client| client.pixel_mouse) + }); + if !self.app.direct_graphics_available { + self.retire_all_direct_graphics(); + } let Some(client_id) = self.foreground_client_id else { self.effective_size = (MIN_COLS, MIN_ROWS); self.app.state.outer_terminal_focus = None; @@ -1485,11 +1501,21 @@ impl HeadlessServer { .count() } + fn direct_graphics_available(&self) -> bool { + self.app_client_count() == 1 + && self.foreground_client_id.is_some_and(|id| { + self.clients.get(&id).is_some_and(|client| { + client.is_full_app_client() && client.writer.is_some() && client.direct_graphics + }) + }) + } + fn has_app_client(&self) -> bool { self.app_client_count() > 0 } fn remove_client(&mut self, client_id: u64) -> bool { + self.retire_direct_graphics_for_client(client_id); let was_foreground = self.foreground_client_id == Some(client_id); self.app.clear_input_source(client_id); self.send_client_graphics_cleanup(client_id); @@ -1551,7 +1577,7 @@ impl HeadlessServer { let Ok(serialized) = Self::frame_server_message(&ServerMessage::Graphics { bytes }) else { return; }; - let _ = writer.control.send(serialized); + writer.replace_with_cleanup(serialized); } fn send_all_clients_graphics_cleanup(&mut self) { @@ -2781,6 +2807,7 @@ impl HeadlessServer { writer, render_encoding, direct_attach_requested, + direct_graphics, } => { if self.handoff_in_progress { if let Ok(message) = @@ -2806,24 +2833,24 @@ impl HeadlessServer { "client connected" ); let last_activity = self.allocate_activity_stamp(); - self.clients.insert( - client_id, - ClientConnection::new_with_mode( - ClientConnectionMode::App, - keybindings, - (cols, rows), - crate::kitty_graphics::HostCellSize { - width_px: cell_width_px, - height_px: cell_height_px, - }, - crate::terminal_theme::TerminalTheme::default(), - None, - last_activity, - render_encoding, - direct_attach_requested, - Some(writer), - ), + let mut connection = ClientConnection::new_with_mode( + ClientConnectionMode::App, + keybindings, + (cols, rows), + crate::kitty_graphics::HostCellSize { + width_px: cell_width_px, + height_px: cell_height_px, + }, + crate::terminal_theme::TerminalTheme::default(), + None, + last_activity, + render_encoding, + direct_attach_requested, + Some(writer), ); + connection.direct_graphics = direct_graphics; + connection.pixel_mouse = direct_graphics; + self.clients.insert(client_id, connection); if !direct_attach_requested { self.foreground_client_id = Some(client_id); } @@ -2835,6 +2862,17 @@ impl HeadlessServer { self.nudge_handoff_panes_on_first_client_attach(); true } + ServerEvent::GraphicsTransmissionResult { + client_id, + transfer_id, + image_id, + success, + } => self.complete_direct_graphics(client_id, transfer_id, image_id, success), + ServerEvent::GraphicsTransmissionStarted { + client_id, + transfer_id, + image_id, + } => self.start_direct_graphics_response(client_id, transfer_id, image_id), ServerEvent::ClientAttachTerminal { client_id, terminal_id, @@ -2859,6 +2897,35 @@ impl HeadlessServer { } => self.handle_terminal_attach_scroll( client_id, source, direction, lines, column, row, modifiers, ), + ServerEvent::ClientInputPixels { + client_id, + data, + geometry, + } => { + let coordinates_valid = crate::input::mouse::parse_report(&data) + .and_then(|(x, y)| geometry.cell(x, y)) + .is_some(); + let valid = coordinates_valid + && self.clients.get(&client_id).is_some_and(|client| { + let cell = client.cell_size; + client.is_full_app_client() + && client.host_sgr_pixels_active == Some(true) + && client.terminal_size == (geometry.cols, geometry.rows) + && cell.is_known() + && cell.width_px == geometry.width_px / u32::from(geometry.cols) + && cell.height_px == geometry.height_px / u32::from(geometry.rows) + }); + if !valid || self.handoff_in_progress || !self.focused_pane_graphics_demand() { + return false; + } + let foreground_changed = self.promote_client_to_foreground(client_id); + if foreground_changed { + self.resize_shared_runtime_to_effective_size_before_input(); + } + self.app + .route_client_pixel_mouse(client_id, &data, geometry) + || foreground_changed + } ServerEvent::ClientInput { client_id, data } => { if self.handoff_in_progress { debug!( @@ -2987,18 +3054,21 @@ impl HeadlessServer { }) = self.clients.get_mut(&client_id) { *terminal_size = (cols, rows); - *cell_size = crate::kitty_graphics::HostCellSize { + let observed = crate::kitty_graphics::HostCellSize { width_px: cell_width_px, height_px: cell_height_px, }; + if observed.is_known() { + *cell_size = observed; + } render_state.request_repaint(); - Some(terminal_id.clone()) + Some((terminal_id.clone(), *cell_size)) } else { None }; - if let Some(terminal_id) = direct_terminal_id { + if let Some((terminal_id, cell_size)) = direct_terminal_id { if let Some(runtime) = self.runtime_for_terminal_id_string(&terminal_id) { - runtime.resize(rows, cols, cell_width_px, cell_height_px); + runtime.resize(rows, cols, cell_size.width_px, cell_size.height_px); } return true; } @@ -3011,19 +3081,25 @@ impl HeadlessServer { }) = self.clients.get_mut(&client_id) { *terminal_size = (cols, rows); - *cell_size = crate::kitty_graphics::HostCellSize { + let observed = crate::kitty_graphics::HostCellSize { width_px: cell_width_px, height_px: cell_height_px, }; + if observed.is_known() { + *cell_size = observed; + } render_state.request_repaint(); return true; } if let Some(client) = self.clients.get_mut(&client_id) { client.terminal_size = (cols, rows); - client.cell_size = crate::kitty_graphics::HostCellSize { + let observed = crate::kitty_graphics::HostCellSize { width_px: cell_width_px, height_px: cell_height_px, }; + if observed.is_known() { + client.cell_size = observed; + } } self.promote_client_to_foreground(client_id); self.resize_shared_runtime_to_effective_size(); @@ -3055,19 +3131,10 @@ impl HeadlessServer { } fn handle_server_event_with_render_impact(&mut self, ev: ServerEvent) -> RenderImpact { - let deferred_render = match &ev { - ServerEvent::ClientWriterDrained { client_id } => self - .clients - .get(client_id) - .map_or(DeferredRender::None, ClientConnection::deferred_render), - _ => DeferredRender::None, - }; - if !self.handle_server_event(ev) { - return RenderImpact::None; - } - match deferred_render { - DeferredRender::Graphics => RenderImpact::Graphics, - DeferredRender::None | DeferredRender::Full => RenderImpact::Full, + if self.handle_server_event(ev) { + RenderImpact::Full + } else { + RenderImpact::None } } @@ -3299,6 +3366,7 @@ impl HeadlessServer { if matches!( &msg.request.method, api::schema::Method::PaneGraphicsStreamSet(_) + | api::schema::Method::PaneGraphicsStreamDirect(_) ) { return self.handle_pane_graphics_stream_frame(msg); } @@ -3341,6 +3409,11 @@ impl HeadlessServer { }; let metadata_expired = self.app.expire_due_metadata(Instant::now()); + let stream_open = match &msg.request.method { + api::schema::Method::PaneGraphicsStreamOpen(params) => Some(params.clone()), + _ => None, + }; + let stream_active = msg.stream_active.clone(); if let api::schema::Method::ServerLiveHandoff(params) = &msg.request.method { let handoff_result = self.perform_live_handoff(params.clone()); @@ -3398,7 +3471,7 @@ impl HeadlessServer { | api::schema::Method::PaneGraphicsStreamOpen(_) | api::schema::Method::PaneGraphicsStreamClose(_) ) - .then_some(self.app.state.pane_graphics_revision); + .then_some(self.app.pane_graphics.revision()); let mut changed = metadata_expired | (pane_graphics_revision_before.is_none() && api::request_changes_ui(&msg.request)); let skip_default_workspace = skip_default_workspace_for_request @@ -3501,6 +3574,10 @@ impl HeadlessServer { } } } + if let (Some(params), Some(active)) = (stream_open.as_ref(), stream_active) { + self.app + .attach_pane_graphics_stream_active(params, active, &response); + } if let Some(spec) = alt_screen_read_spec { if let Ok(success) = serde_json::from_str::(&response) { if let api::schema::ResponseResult::PaneRead { read } = success.result { @@ -3523,7 +3600,7 @@ impl HeadlessServer { let _ = msg.respond_to.send(response); if let Some(revision_before) = pane_graphics_revision_before { - changed |= revision_before != self.app.state.pane_graphics_revision; + changed |= revision_before != self.app.pane_graphics.revision(); } // Forward new toast state only when a client-local delivery mode is selected. @@ -3671,32 +3748,73 @@ impl HeadlessServer { changed } + fn focused_pane_graphics_demand(&self) -> bool { + self.app + .state + .active + .and_then(|ws_idx| self.app.state.workspaces.get(ws_idx)) + .and_then(crate::workspace::Workspace::focused_pane_id) + .is_some_and(|pane_id| self.app.pane_graphics.active_for_pane(pane_id)) + } + fn stream_host_mouse_capture_mode(&mut self) { let enabled = self .app .state .should_capture_host_mouse_from(&self.app.terminal_runtimes); - let serialized = match Self::frame_server_message(&ServerMessage::MouseCapture { enabled }) - { - Ok(framed) => framed, - Err(err) => { - warn!(err = %err, "failed to serialize mouse capture mode for clients"); - return; - } - }; - + let pixel_mouse_requested = self + .clients + .values() + .any(|client| client.is_full_app_client() && client.pixel_mouse); + let sgr_pixels = pixel_mouse_requested + && self.focused_pane_graphics_demand() + && self + .app + .state + .active + .and_then(|ws_idx| { + self.app + .state + .workspaces + .get(ws_idx) + .and_then(crate::workspace::Workspace::focused_pane_id) + .and_then(|pane_id| { + self.app.state.runtime_for_pane_in_workspace( + &self.app.terminal_runtimes, + ws_idx, + pane_id, + ) + }) + }) + .and_then(crate::terminal::TerminalRuntime::input_state) + .is_some_and(|state| { + state.mouse_protocol_encoding == crate::input::MouseProtocolEncoding::SgrPixels + }); let mut broken_clients: Vec = Vec::new(); for (&client_id, client) in &mut self.clients { if !client.is_full_app_client() { continue; } - if client.host_mouse_capture_active == Some(enabled) { + let client_sgr_pixels = sgr_pixels && client.pixel_mouse; + if client.host_mouse_capture_active == Some(enabled) + && client.host_sgr_pixels_active == Some(client_sgr_pixels) + { continue; } let Some(writer) = &client.writer else { continue; }; - if writer.control.send(serialized.clone()).is_err() { + let serialized = match Self::frame_server_message(&ServerMessage::MouseCapture { + enabled, + sgr_pixels: client_sgr_pixels, + }) { + Ok(framed) => framed, + Err(err) => { + warn!(err = %err, "failed to serialize mouse capture mode for client"); + continue; + } + }; + if writer.control.send(serialized).is_err() { debug!( client_id, "client writer channel closed during mouse capture update" @@ -3705,6 +3823,7 @@ impl HeadlessServer { continue; } client.host_mouse_capture_active = Some(enabled); + client.host_sgr_pixels_active = Some(client_sgr_pixels); } for client_id in broken_clients { @@ -3877,6 +3996,7 @@ impl HeadlessServer { && cell_size.is_known() && crate::kitty_graphics::has_visible_pane_graphics( &self.app.state, + &self.app.pane_graphics, &self.app.terminal_runtimes, self.app.state.view.tab_surface(), *cell_size, @@ -4166,31 +4286,48 @@ impl HeadlessServer { continue; }; let mut next_graphics_cache = client.graphics_cache.clone(); - let graphics_surface_reset_pending = client.graphics_surface_reset_pending; - if is_app_client && self.app.state.kitty_graphics_enabled && cell_size.is_known() { - if graphics_surface_reset_pending { - frame.graphics = next_graphics_cache.clear_bytes(); + let mut reset_graphics = Vec::new(); + let mut encoded = if is_app_client + && self.app.state.kitty_graphics_enabled + && cell_size.is_known() + { + if client.graphics_surface_reset_pending { + if self.app.pane_graphics.slots.is_empty() { + reset_graphics = next_graphics_cache.clear_bytes(); + } else { + next_graphics_cache = crate::kitty_graphics::HostGraphicsCache::default(); + } } let graphics_started = crate::render_prof::timer(); - frame - .graphics - .extend(crate::kitty_graphics::encode_local_pane_graphics( - &self.app.state, - &self.app.terminal_runtimes, - self.app.state.view.tab_surface(), - cell_size, - &mut next_graphics_cache, - )); + let encoded = crate::kitty_graphics::encode_local_pane_graphics( + &self.app.state, + &self.app.pane_graphics, + &self.app.terminal_runtimes, + self.app.state.view.tab_surface(), + cell_size, + Some(crate::kitty_graphics::HEADLESS_GRAPHICS_TRANSACTION_BUDGET), + &mut next_graphics_cache, + ); crate::render_prof::duration_since("full_render.graphics_encode", graphics_started); + encoded + } else if self.app.pane_graphics.slots.is_empty() { + crate::kitty_graphics::EncodedGraphics { + bytes: next_graphics_cache.clear_bytes(), + incomplete: false, + } } else { - frame.graphics = next_graphics_cache.clear_bytes(); + next_graphics_cache.clear_next() + }; + if !reset_graphics.is_empty() { + reset_graphics.extend(encoded.bytes); + encoded.bytes = reset_graphics; } + frame.graphics = encoded.bytes; let Some(writer) = client.writer.as_ref().cloned() else { crate::render_prof::event("full_render.writer_missing"); continue; }; - let mut commit_graphics_cache = true; if frame.graphics.len() > MAX_GRAPHICS_FRAME_SIZE { warn!( @@ -4201,32 +4338,30 @@ impl HeadlessServer { ); frame.graphics.clear(); commit_graphics_cache = false; + encoded.incomplete = false; } - - let max_frame_size = if frame.graphics.is_empty() { - MAX_FRAME_SIZE - } else { - MAX_GRAPHICS_FRAME_SIZE - }; let has_graphics = !frame.graphics.is_empty(); - let prepare_started = crate::render_prof::timer(); let Some(mut prepared) = client.render_state.prepare_frame(frame) else { - client.clear_deferred_render(); + if commit_graphics_cache { + client.graphics_cache = next_graphics_cache; + client.graphics_surface_reset_pending = false; + } + if encoded.incomplete { + client.defer_full_render(); + deferred_frame = true; + } else { + client.clear_deferred_render(); + } crate::render_prof::event("full_render.skip_identical"); - crate::render_prof::duration_since("full_render.prepare_frame", prepare_started); continue; }; - crate::render_prof::duration_since("full_render.prepare_frame", prepare_started); - - let serialize_started = crate::render_prof::timer(); - let serialized = match Self::frame_server_message_with_max( - prepared.message(), - max_frame_size, - ) { - Ok(framed) => { - crate::render_prof::duration_since("full_render.serialize", serialize_started); - framed - } + let max = if has_graphics { + MAX_GRAPHICS_FRAME_SIZE + } else { + crate::protocol::MAX_FRAME_SIZE + }; + let serialized = match Self::frame_server_message_with_max(prepared.message(), max) { + Ok(frame) => frame, Err(protocol::FramingError::Oversized { claimed, max }) if has_graphics => { warn!( client_id, @@ -4234,10 +4369,6 @@ impl HeadlessServer { ); let Some(mut text_only_frame) = prepared.into_frame() else { crate::render_prof::event("full_render.serialize_error"); - crate::render_prof::duration_since( - "full_render.serialize", - serialize_started, - ); continue; }; text_only_frame.graphics.clear(); @@ -4246,10 +4377,6 @@ impl HeadlessServer { else { client.clear_deferred_render(); crate::render_prof::event("full_render.skip_identical_text_only"); - crate::render_prof::duration_since( - "full_render.serialize", - serialize_started, - ); continue; }; let framed = match Self::frame_server_message(text_only_prepared.message()) { @@ -4258,16 +4385,12 @@ impl HeadlessServer { warn!(client_id, err = %err, "failed to serialize text-only frame for client"); broken_clients.push(client_id); crate::render_prof::event("full_render.serialize_error"); - crate::render_prof::duration_since( - "full_render.serialize", - serialize_started, - ); continue; } }; prepared = text_only_prepared; commit_graphics_cache = false; - crate::render_prof::duration_since("full_render.serialize", serialize_started); + encoded.incomplete = false; framed } Err(protocol::FramingError::Oversized { claimed, max }) => { @@ -4276,45 +4399,36 @@ impl HeadlessServer { claimed, max, "skipping oversized frame for client" ); crate::render_prof::event("full_render.serialize_oversized"); - crate::render_prof::duration_since("full_render.serialize", serialize_started); continue; } Err(err) => { - warn!(client_id, err = %err, "failed to serialize frame for client"); + warn!(client_id, err = %err, "failed to serialize frame"); broken_clients.push(client_id); crate::render_prof::event("full_render.serialize_error"); - crate::render_prof::duration_since("full_render.serialize", serialize_started); continue; } }; - crate::render_prof::counter("full_render.bytes", serialized.len() as u64); - - let send_started = crate::render_prof::timer(); match writer.render.try_send(serialized) { Ok(()) => { - client.clear_deferred_render(); if commit_graphics_cache { client.graphics_cache = next_graphics_cache; client.graphics_surface_reset_pending = false; } client.render_state.commit_sent_frame(prepared); + if encoded.incomplete { + client.defer_full_render(); + deferred_frame = true; + } else { + client.clear_deferred_render(); + } crate::render_prof::event("full_render.sent"); - crate::render_prof::duration_since("full_render.try_send", send_started); } Err(std::sync::mpsc::TrySendError::Full(_)) => { client.defer_full_render(); deferred_frame = true; - crate::render_prof::event("full_render.queue_full"); - crate::render_prof::duration_since("full_render.try_send", send_started); - debug!(client_id, "render queue full, deferring latest frame"); - continue; } Err(std::sync::mpsc::TrySendError::Disconnected(_)) => { - debug!(client_id, "client writer channel closed, marking as broken"); broken_clients.push(client_id); - crate::render_prof::event("full_render.writer_disconnected"); - crate::render_prof::duration_since("full_render.try_send", send_started); - continue; } } } @@ -5044,7 +5158,9 @@ mod tests { } fn read_server_frame(bytes: Vec) -> FrameData { - match read_server_message(bytes) { + match protocol::read_message(&mut std::io::Cursor::new(bytes), MAX_GRAPHICS_FRAME_SIZE) + .expect("decode server frame") + { ServerMessage::Frame(frame) => frame, other => panic!("expected frame, got {other:?}"), } @@ -5139,6 +5255,7 @@ mod tests { }, respond_to, response_write_complete: None, + stream_active: None, }); let response: api::schema::SuccessResponse = serde_json::from_str(&response_rx.recv().unwrap()).unwrap(); @@ -5179,6 +5296,7 @@ mod tests { }, respond_to, response_write_complete: None, + stream_active: None, }) ); let response = response_rx @@ -5439,6 +5557,42 @@ mod tests { && matches!(mark, '\u{ff9e}' | '\u{ff9f}') } + #[test] + fn direct_graphics_requires_one_negotiated_app_client() { + let mut server = test_headless_server(); + let (writer_a, _control_a, _render_a) = test_client_writer(); + assert!(server.handle_server_event(ServerEvent::ClientConnected { + client_id: 1, + cols: 80, + rows: 24, + cell_width_px: 10, + cell_height_px: 20, + render_encoding: RenderEncoding::SemanticFrame, + keybindings: None, + direct_attach_requested: false, + direct_graphics: true, + writer: writer_a, + })); + assert!(server.clients[&1].direct_graphics); + assert!(server.clients[&1].pixel_mouse); + assert!(server.direct_graphics_available()); + + let (writer_b, _control_b, _render_b) = test_client_writer(); + assert!(server.handle_server_event(ServerEvent::ClientConnected { + client_id: 2, + cols: 80, + rows: 24, + cell_width_px: 10, + cell_height_px: 20, + render_encoding: RenderEncoding::SemanticFrame, + keybindings: None, + direct_attach_requested: false, + direct_graphics: false, + writer: writer_b, + })); + assert!(!server.direct_graphics_available()); + } + #[test] fn foreground_client_applies_client_keybindings() { let mut server = test_headless_server(); @@ -5463,6 +5617,7 @@ new_tab = "prefix+t" render_encoding: RenderEncoding::SemanticFrame, keybindings: Some(Box::new(local_keybindings)), direct_attach_requested: false, + direct_graphics: false, writer: writer_a, })); assert_eq!( @@ -5487,6 +5642,7 @@ new_tab = "prefix+t" render_encoding: RenderEncoding::SemanticFrame, keybindings: None, direct_attach_requested: false, + direct_graphics: false, writer: writer_b, })); assert_eq!( @@ -5527,6 +5683,7 @@ new_tab = "prefix+t" render_encoding: RenderEncoding::SemanticFrame, keybindings: Some(Box::new(local_keybindings)), direct_attach_requested: false, + direct_graphics: false, writer: writer_a, })); assert_eq!(server.app.state.config_diagnostic, without_keybindings); @@ -5540,6 +5697,7 @@ new_tab = "prefix+t" render_encoding: RenderEncoding::SemanticFrame, keybindings: None, direct_attach_requested: false, + direct_graphics: false, writer: writer_b, })); assert_eq!( @@ -5583,6 +5741,7 @@ next_tab = "" render_encoding: RenderEncoding::SemanticFrame, keybindings: Some(Box::new(local_keybindings)), direct_attach_requested: false, + direct_graphics: false, writer, })); server.app.state.mode = crate::app::Mode::Settings; @@ -5658,6 +5817,7 @@ next_tab = "" render_encoding: RenderEncoding::SemanticFrame, keybindings: Some(Box::new(local_config.live_keybinds().unwrap())), direct_attach_requested: false, + direct_graphics: false, writer: writer_a, })); server.app.state.mode = crate::app::Mode::Settings; @@ -5678,6 +5838,7 @@ next_tab = "" render_encoding: RenderEncoding::SemanticFrame, keybindings: None, direct_attach_requested: false, + direct_graphics: false, writer: writer_b, })); assert_eq!( @@ -5712,6 +5873,7 @@ next_tab = "" render_encoding: RenderEncoding::TerminalAnsi, keybindings: None, direct_attach_requested: true, + direct_graphics: false, writer, })); assert!(server.clients.contains_key(&7)); @@ -5777,6 +5939,7 @@ next_tab = "" render_encoding: RenderEncoding::TerminalAnsi, keybindings: None, direct_attach_requested: true, + direct_graphics: false, writer, })); control_rx @@ -6189,6 +6352,7 @@ next_tab = "" render_encoding, keybindings: None, direct_attach_requested: false, + direct_graphics: false, writer, })); @@ -6223,6 +6387,7 @@ next_tab = "" render_encoding: RenderEncoding::TerminalAnsi, keybindings: None, direct_attach_requested: true, + direct_graphics: false, writer, })); @@ -6256,6 +6421,7 @@ next_tab = "" render_encoding: RenderEncoding::SemanticFrame, keybindings: None, direct_attach_requested: false, + direct_graphics: false, writer, })); assert!(server.has_app_client()); @@ -6356,6 +6522,7 @@ next_tab = "" render_encoding: RenderEncoding::TerminalAnsi, keybindings: None, direct_attach_requested: true, + direct_graphics: false, writer, })); assert!( @@ -7778,6 +7945,51 @@ next_tab = "" assert_eq!(server.foreground_client_id, Some(3)); } + #[test] + fn terminal_attach_resize_preserves_known_cell_size_when_pixels_are_omitted() { + with_terminal_session_test_server(|server, _terminal_id, terminal_id, _pane_id| { + let mut client = ClientConnection::new( + (80, 24), + crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }, + crate::terminal_theme::TerminalTheme::default(), + Some(true), + 1, + RenderEncoding::SemanticFrame, + None, + ); + client.mode = ClientConnectionMode::TerminalAttach { + terminal_id: terminal_id.clone(), + }; + server.clients.insert(1, client); + + assert!(server.handle_server_event(ServerEvent::ClientResize { + client_id: 1, + cols: 100, + rows: 30, + cell_width_px: 0, + cell_height_px: 0, + })); + + assert_eq!( + server + .runtime_for_terminal_id_string(&terminal_id) + .unwrap() + .pixel_size(), + Some((1_000, 600)) + ); + assert_eq!( + server.clients[&1].cell_size, + crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + } + ); + }); + } + #[tokio::test] async fn passive_mouse_motion_forwards_without_requesting_render() { let mut server = test_headless_server(); @@ -8335,6 +8547,7 @@ next_tab = "" render_encoding: RenderEncoding::TerminalAnsi, keybindings: None, direct_attach_requested: true, + direct_graphics: false, writer, })); assert!( @@ -8531,16 +8744,11 @@ next_tab = "" let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig) .expect("serialize dummy message"); - server - .clients - .get(&1) - .unwrap() + server.clients[&1] .writer .as_ref() .unwrap() - .render - .try_send(queued) - .expect("pre-fill render queue"); + .test_fill_render(queued); assert!(server.handle_server_event(ServerEvent::ClientInput { client_id: 1, @@ -8671,10 +8879,7 @@ next_tab = "" let (client_tx, _client_control_rx, client_rx) = test_client_writer(); let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig) .expect("serialize dummy message"); - client_tx - .render - .try_send(queued) - .expect("pre-fill render queue"); + client_tx.test_fill_render(queued); server.clients.insert( 1, @@ -8715,10 +8920,7 @@ next_tab = "" let (client_tx, _client_control_rx, client_rx) = test_client_writer(); let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig) .expect("serialize dummy message"); - client_tx - .render - .try_send(queued) - .expect("pre-fill render queue"); + client_tx.test_fill_render(queued); server.clients.insert( 1, @@ -9003,7 +9205,7 @@ next_tab = "" .recv_timeout(Duration::from_millis(100)) .expect("mouse capture message") ), - ServerMessage::MouseCapture { enabled: true } + ServerMessage::MouseCapture { enabled: true, .. } )); } @@ -9525,16 +9727,11 @@ next_tab = "" let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa"); let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig) .expect("serialize dummy message"); - server - .clients - .get(&1) - .unwrap() + server.clients[&1] .writer .as_ref() .unwrap() - .render - .try_send(queued) - .expect("pre-fill render queue"); + .test_fill_render(queued); server.app.full_redraw_pending = true; server.render_and_stream(); @@ -9742,6 +9939,7 @@ next_tab = "" let mut server = test_headless_server(); let (foreground_tx, foreground_control_rx, _foreground_rx) = test_client_writer(); drop(foreground_control_rx); + foreground_tx.test_close(); server.clients.insert( 1, @@ -10125,6 +10323,7 @@ next_tab = "" }, respond_to, response_write_complete: None, + stream_active: None, }); assert!(changed); @@ -10211,6 +10410,7 @@ next_tab = "" }, respond_to, response_write_complete: None, + stream_active: None, }); assert!(changed); @@ -10263,6 +10463,7 @@ next_tab = "" }, respond_to, response_write_complete: None, + stream_active: None, }); assert!(changed); @@ -10295,6 +10496,7 @@ next_tab = "" }, respond_to, response_write_complete: None, + stream_active: None, }); assert!(changed); @@ -10332,6 +10534,7 @@ next_tab = "" }, respond_to, response_write_complete: None, + stream_active: None, }) ); @@ -10388,6 +10591,7 @@ next_tab = "" }, respond_to, response_write_complete: None, + stream_active: None, }) ); @@ -10688,6 +10892,7 @@ next_tab = "" }, respond_to, response_write_complete: None, + stream_active: None, }); assert!(changed); diff --git a/src/server/headless/pane_graphics.rs b/src/server/headless/pane_graphics.rs index ed28eab0..d027f79b 100644 --- a/src/server/headless/pane_graphics.rs +++ b/src/server/headless/pane_graphics.rs @@ -1,5 +1,3 @@ -use tracing::warn; - use super::{HeadlessServer, RenderImpact}; use crate::api; use crate::protocol::{ServerMessage, MAX_GRAPHICS_FRAME_SIZE}; @@ -12,31 +10,16 @@ pub(super) enum RetainedGraphicsOutcome { Fallback, } -pub(super) fn frame_pane_graphics_for_client(bytes: Vec) -> Vec { +pub(super) fn frame_pane_graphics(bytes: Vec) -> Vec { if bytes.is_empty() { return bytes; } - let mut framed = Vec::with_capacity(bytes.len() + 4); - framed.extend_from_slice(b"\x1b7"); - framed.extend_from_slice(&bytes); - framed.extend_from_slice(b"\x1b8"); - framed + [b"\x1b7".as_slice(), &bytes, b"\x1b8"].concat() } impl HeadlessServer { pub(super) fn pane_graphics_runtime_active(&self) -> bool { - !self.app.state.pane_graphics_layers.is_empty() - || !self.app.state.pane_graphics_streams.is_empty() - } - - pub(super) fn cancel_inactive_pane_graphics_streams(&self) { - api::cancel_inactive_pane_graphics_streams(|owner| { - self.app - .state - .pane_graphics_streams - .values() - .any(|active_owner| active_owner == owner) - }); + !self.app.pane_graphics.slots.is_empty() } pub(super) fn handle_pane_graphics_stream_frame( @@ -49,12 +32,164 @@ impl HeadlessServer { } let internal_changed = self.drain_all_internal_events_with_forwarding(); + let direct = match &msg.request.method { + api::schema::Method::PaneGraphicsStreamDirect(params) => Some(params.clone()), + _ => None, + }; + let direct_key = direct.as_ref().and_then(|params| { + self.app.parse_pane_id(¶ms.pane_id).map(|(_, pane_id)| { + ( + pane_id, + params + .layer_id + .clone() + .unwrap_or_else(|| api::schema::PANE_GRAPHICS_PRIMARY_LAYER_ID.into()), + ) + }) + }); + let direct_client = self + .direct_graphics_available() + .then_some(self.foreground_client_id) + .flatten(); + let gate_busy = self + .app + .pane_graphics + .slots + .values() + .any(|slot| slot.direct_gate.is_some()); + self.app.direct_graphics_available = direct_client.is_some() && !gate_busy; let response = self .app .handle_api_request_after_internal_events_drained(msg.request); + self.app.direct_graphics_available = self.direct_graphics_available(); let succeeded = serde_json::from_str::(&response).is_ok(); - let _ = msg.respond_to.send(response); + if succeeded { + if let (Some(params), Some(client_id)) = (direct, direct_client) { + let direct_frame = direct_key.as_ref().and_then(|key| { + self.app.pane_graphics.slots.get(key).and_then(|slot| { + let active_owner = slot.stream_owner.as_deref() == Some(¶ms.owner) + && slot.stream_is_active(); + active_owner.then_some(())?; + slot.layer + .as_ref() + .and_then(crate::app::pane_graphics::Layer::direct_lease) + .map(|lease| { + ( + slot.host_image_id, + lease.path().to_string_lossy().into_owned(), + lease.len() as u64, + lease.fingerprint(), + ) + }) + }) + }); + if let (Some(key), Some((image_id, path, expected_len, transfer_id))) = + (direct_key.clone(), direct_frame) + { + let mut next_cache = self + .clients + .get(&client_id) + .map(|client| client.graphics_cache.clone()) + .unwrap_or_default(); + let command = (!internal_changed) + .then(|| { + crate::kitty_graphics::prepare_direct_file( + &self.app.state, + &self.app.pane_graphics, + self.app.state.view.tab_surface(), + self.clients + .get(&client_id) + .map(|client| client.cell_size) + .unwrap_or_default(), + &mut next_cache, + &key, + ) + }) + .flatten(); + let Some(command) = command else { + if self.install_inline_fallback(&key) { + if msg.respond_to.send(response).is_err() { + self.retire_direct_gate(&key); + } + return if internal_changed { + RenderImpact::Full + } else { + RenderImpact::Graphics + }; + } + self.retire_direct_gate(&key); + return if internal_changed { + RenderImpact::Full + } else { + RenderImpact::Graphics + }; + }; + let message = ServerMessage::GraphicsFile { + path, + expected_len, + image_id, + transfer_id, + leading: command.leading, + control: command.control, + }; + let send = + Self::frame_server_message_with_max(&message, MAX_GRAPHICS_FRAME_SIZE) + .map_err(|_| std::sync::mpsc::TrySendError::Full(Vec::new())) + .and_then(|framed| { + let Some(writer) = self + .clients + .get(&client_id) + .and_then(|client| client.writer.as_ref()) + else { + return Err(std::sync::mpsc::TrySendError::Disconnected( + framed, + )); + }; + writer.render.send_ordered(framed) + }); + match send { + Ok(()) => { + if let Some(slot) = self.app.pane_graphics.slots.get_mut(&key) { + slot.direct_gate = Some(crate::app::pane_graphics::DirectGate { + transfer_id, + client_id, + deadline: std::time::Instant::now() + + crate::app::pane_graphics::DIRECT_DELIVERY_TIMEOUT, + written: false, + success_response: response, + respond_to: msg.respond_to, + }); + } + if let Some(client) = self.clients.get_mut(&client_id) { + client.graphics_cache = next_cache; + } + return if internal_changed { + RenderImpact::Full + } else { + RenderImpact::None + }; + } + Err(error) => { + self.handle_unwritten_direct_failure( + &key, + response, + msg.respond_to, + error, + ); + return RenderImpact::Graphics; + } + } + } + } + } + + let response_failed = msg.respond_to.send(response).is_err(); + if succeeded && response_failed { + if let Some(key) = direct_key { + self.retire_direct_gate(&key); + } + } if internal_changed { RenderImpact::Full } else if succeeded { @@ -64,6 +199,276 @@ impl HeadlessServer { } } + fn install_inline_fallback(&mut self, key: &crate::app::pane_graphics::Key) -> bool { + let len = self + .app + .pane_graphics + .slots + .get(key) + .and_then(|slot| slot.layer.as_ref()) + .and_then(crate::app::pane_graphics::Layer::direct_lease) + .map(crate::pane_graphics_files::Lease::len); + let Some(len) = len else { + return false; + }; + if !self.app.pane_graphics.can_store_inline(key, len) { + return false; + } + let data = self + .app + .pane_graphics + .slots + .get(key) + .and_then(|slot| slot.layer.as_ref()) + .and_then(crate::app::pane_graphics::Layer::direct_lease) + .and_then(|lease| lease.copy_rgba().ok()); + let Some(data) = data else { + return false; + }; + let slot = self + .app + .pane_graphics + .slots + .get_mut(key) + .expect("matched slot"); + if !slot.stream_is_active() { + return false; + } + let layer = slot.layer.take().expect("direct layer"); + slot.layer = Some(crate::app::pane_graphics::Layer::inline( + layer.format, + layer.image_width, + layer.image_height, + data, + layer.render, + layer.z_index, + )); + self.app.pane_graphics.mark_changed(); + true + } + + pub(super) fn handle_unwritten_direct_failure( + &mut self, + key: &crate::app::pane_graphics::Key, + response: String, + respond_to: std::sync::mpsc::Sender, + error: std::sync::mpsc::TrySendError>, + ) -> bool { + if matches!(error, std::sync::mpsc::TrySendError::Full(_)) + && self.install_inline_fallback(key) + && respond_to.send(response).is_ok() + { + return true; + } + self.retire_direct_gate(key); + false + } + + pub(super) fn retire_direct_gate(&mut self, key: &crate::app::pane_graphics::Key) { + if self.app.pane_graphics.slots.remove(key).is_some() { + self.app.pane_graphics.mark_changed(); + } + } + + pub(super) fn start_direct_graphics_response( + &mut self, + client_id: u64, + transfer_id: u64, + image_id: u32, + ) -> bool { + if let Some(gate) = self.app.pane_graphics.slots.values_mut().find_map(|slot| { + (slot.host_image_id == image_id && slot.stream_is_active()) + .then_some(slot.direct_gate.as_mut()) + .flatten() + .filter(|gate| gate.client_id == client_id && gate.transfer_id == transfer_id) + }) { + gate.written = true; + gate.deadline = + std::time::Instant::now() + crate::app::pane_graphics::DIRECT_RESPONSE_TIMEOUT; + } + false + } + + pub(super) fn complete_direct_graphics( + &mut self, + client_id: u64, + transfer_id: u64, + image_id: u32, + success: bool, + ) -> bool { + if self.shutting_down { + self.retire_all_direct_graphics(); + return false; + } + let key = self.app.pane_graphics.slots.iter().find_map(|(key, slot)| { + slot.direct_gate + .as_ref() + .is_some_and(|gate| { + slot.stream_is_active() + && gate.client_id == client_id + && gate.transfer_id == transfer_id + && slot.host_image_id == image_id + && (!success || gate.written) + }) + .then(|| key.clone()) + }); + let Some(key) = key else { + return false; + }; + let pane_is_live = self + .app + .state + .workspaces + .iter() + .any(|workspace| workspace.pane_state(key.0).is_some()); + if !pane_is_live { + self.retire_direct_gate(&key); + return false; + } + if success { + let (gate, host_image_id) = { + let slot = self + .app + .pane_graphics + .slots + .get_mut(&key) + .expect("matched slot"); + if !slot.stream_is_active() { + return false; + } + if let Some(layer) = slot.layer.as_mut() { + layer.mark_resident(client_id); + } + ( + slot.direct_gate.take().expect("matched gate"), + slot.host_image_id, + ) + }; + if let (Some(client), Some(layer)) = ( + self.clients.get_mut(&client_id), + self.app + .pane_graphics + .slots + .get(&key) + .and_then(|slot| slot.layer.as_ref()), + ) { + client + .graphics_cache + .trust_pane_layer(&key, host_image_id, layer); + } + if gate.respond_to.send(gate.success_response).is_err() { + self.retire_direct_gate(&key); + return true; + } + return true; + } + + if let Some(client) = self.clients.get_mut(&client_id) { + client.direct_graphics = false; + client.graphics_cache.forget_pane_layer(&key, image_id); + } + self.app.direct_graphics_available = false; + if !self.install_inline_fallback(&key) { + self.retire_direct_gate(&key); + self.retire_all_direct_graphics(); + return true; + } + let gate = self + .app + .pane_graphics + .slots + .get_mut(&key) + .filter(|slot| slot.stream_is_active()) + .and_then(|slot| slot.direct_gate.take()); + let Some(gate) = gate else { + self.retire_direct_gate(&key); + return true; + }; + if gate.respond_to.send(gate.success_response).is_err() { + self.retire_direct_gate(&key); + return true; + } + self.retire_all_direct_graphics(); + true + } + + pub(super) fn expire_direct_graphics(&mut self, now: std::time::Instant) -> bool { + let expired = self + .app + .pane_graphics + .slots + .iter() + .filter_map(|(key, slot)| { + slot.direct_gate + .as_ref() + .filter(|gate| gate.deadline <= now) + .map(|gate| (key.clone(), gate.client_id)) + }) + .collect::>(); + for (key, client_id) in &expired { + if let Some(slot) = self.app.pane_graphics.slots.get(key) { + if let Some(gate) = &slot.direct_gate { + self.send_to_client( + *client_id, + ServerMessage::GraphicsTransmissionRetired { + transfer_id: gate.transfer_id, + image_id: slot.host_image_id, + }, + ); + } + } + if let Some(client) = self.clients.get_mut(client_id) { + client.direct_graphics = false; + } + self.retire_direct_gate(key); + } + if !expired.is_empty() { + self.app.direct_graphics_available = false; + self.retire_all_direct_graphics(); + } + !expired.is_empty() + } + + pub(super) fn retire_all_direct_graphics(&mut self) { + let keys = self + .app + .pane_graphics + .slots + .iter() + .filter(|(_, slot)| { + slot.layer + .as_ref() + .is_some_and(crate::app::pane_graphics::Layer::terminal_only) + }) + .map(|(key, _)| key.clone()) + .collect::>(); + for key in keys { + self.retire_direct_gate(&key); + } + } + + pub(super) fn retire_direct_graphics_for_client(&mut self, client_id: u64) { + let keys = self + .app + .pane_graphics + .slots + .iter() + .filter_map(|(key, slot)| { + let layer = slot.layer.as_ref()?; + let owned = layer.resident_client() == Some(client_id) + || (layer.direct_lease().is_some() + && slot + .direct_gate + .as_ref() + .is_some_and(|gate| gate.client_id == client_id)); + (layer.terminal_only() && owned).then(|| key.clone()) + }) + .collect::>(); + for key in keys { + self.retire_direct_gate(&key); + } + } + pub(super) fn render_retained_graphics_update_and_stream(&mut self) -> RetainedGraphicsOutcome { crate::render_prof::event("retained_graphics.attempt"); if self.app.full_redraw_pending { @@ -117,74 +522,62 @@ impl HeadlessServer { let mut next_graphics_cache = client.graphics_cache.clone(); let encode_started = crate::render_prof::timer(); - let bytes = - frame_pane_graphics_for_client(crate::kitty_graphics::encode_local_pane_graphics( - &self.app.state, - &self.app.terminal_runtimes, - self.app.state.view.tab_surface(), - cell_size, - &mut next_graphics_cache, - )); + let encoded = crate::kitty_graphics::encode_local_pane_graphics( + &self.app.state, + &self.app.pane_graphics, + &self.app.terminal_runtimes, + self.app.state.view.tab_surface(), + cell_size, + Some(crate::kitty_graphics::HEADLESS_GRAPHICS_TRANSACTION_BUDGET), + &mut next_graphics_cache, + ); crate::render_prof::duration_since("retained_graphics.graphics_encode", encode_started); - if bytes.len() > MAX_GRAPHICS_FRAME_SIZE { - warn!( - client_id, - graphics_bytes = bytes.len(), - max = MAX_GRAPHICS_FRAME_SIZE, - "dropping oversized retained graphics payload" - ); - continue; - } - let serialized = if bytes.is_empty() { - None - } else { - let serialize_started = crate::render_prof::timer(); - let framed = match Self::frame_server_message_with_max( - &ServerMessage::Graphics { bytes }, - MAX_GRAPHICS_FRAME_SIZE, - ) { - Ok(framed) => framed, - Err(err) => { - warn!(client_id, err = %err, "failed to serialize retained graphics"); - return RetainedGraphicsOutcome::Fallback; - } - }; - crate::render_prof::duration_since( - "retained_graphics.serialize", - serialize_started, - ); - Some(framed) - }; - prepared.push((client_id, serialized, next_graphics_cache)); + prepared.push((client_id, encoded, next_graphics_cache)); } let mut broken_clients = Vec::new(); - for (client_id, serialized, next_graphics_cache) in prepared { + for (client_id, encoded, next_graphics_cache) in prepared { let Some(client) = self.clients.get_mut(&client_id) else { continue; }; - let Some(serialized) = serialized else { - client.graphics_cache = next_graphics_cache; - client.clear_deferred_render(); - continue; + let serialized = if encoded.bytes.is_empty() { + None + } else { + match Self::frame_server_message_with_max( + &ServerMessage::Graphics { + bytes: frame_pane_graphics(encoded.bytes), + }, + MAX_GRAPHICS_FRAME_SIZE, + ) { + Ok(serialized) => Some(serialized), + Err(_) => { + crate::render_prof::event("retained_graphics_fallback.oversized"); + return RetainedGraphicsOutcome::Fallback; + } + } }; - let Some(writer) = client.writer.as_ref().cloned() else { - broken_clients.push(client_id); - continue; + let result = match (serialized, client.writer.as_ref()) { + (None, _) => Ok(()), + (Some(bytes), Some(writer)) => writer.render.try_send(bytes), + (Some(bytes), None) => Err(std::sync::mpsc::TrySendError::Disconnected(bytes)), }; - match writer.render.try_send(serialized) { + match result { Ok(()) => { client.graphics_cache = next_graphics_cache; - client.clear_deferred_render(); + if encoded.incomplete { + client.defer_full_render(); + deferred = true; + } else { + client.clear_deferred_render(); + } crate::render_prof::event("retained_graphics.sent"); } Err(std::sync::mpsc::TrySendError::Full(_)) => { - client.defer_pane_graphics_render(); + client.defer_full_render(); deferred = true; - crate::render_prof::event("retained_graphics.deferred"); } Err(std::sync::mpsc::TrySendError::Disconnected(_)) => { - broken_clients.push(client_id); + broken_clients.push(client_id) } } } diff --git a/src/server/headless/tests/pane_graphics.rs b/src/server/headless/tests/pane_graphics.rs index 1eba9816..f9bd0a01 100644 --- a/src/server/headless/tests/pane_graphics.rs +++ b/src/server/headless/tests/pane_graphics.rs @@ -1,12 +1,52 @@ use super::*; -#[test] -fn frames_preserve_cursor_without_changing_ordinary_messages() { - assert_eq!(frame_pane_graphics_for_client(Vec::new()), Vec::::new()); - assert_eq!( - frame_pane_graphics_for_client(b"graphics".to_vec()), - b"\x1b7graphics\x1b8" - ); +fn receive_render(receiver: &std::sync::mpsc::Receiver>, timeout: Duration) -> Vec { + receiver.recv_timeout(timeout).unwrap() +} + +#[tokio::test] +async fn cold_redraw_advances_one_bounded_layer_after_each_send() { + let (mut server, client_rx, pane_id) = retained_test_server(b"cold redraw"); + server.app.state.kitty_graphics_enabled = true; + server.clients.get_mut(&1).unwrap().cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + const LAYERS: usize = 8; + for index in 0..LAYERS { + set_named_graphics_layer( + &mut server, + pane_id, + &format!("layer-{index:02}"), + vec![index as u8; 1024 * 1024], + index as i32, + ); + } + + fill_render_lane(&server); + server.render_and_stream(); + assert!(server.clients[&1].graphics_cache.is_empty()); + let _older = client_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + for expected in 1..=LAYERS { + server.render_and_stream(); + let bytes = client_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!(bytes.len() <= MAX_GRAPHICS_FRAME_SIZE + 4); + let frame = read_server_frame(bytes); + assert_eq!( + frame + .graphics + .windows(4) + .filter(|part| *part == b"a=t,") + .count(), + 1 + ); + assert_eq!( + server.clients[&1].graphics_cache.test_image_count(), + expected + ); + } + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); } fn enable_graphics_and_render( @@ -19,39 +59,72 @@ fn enable_graphics_and_render( height_px: 20, }; server.render_and_stream(); - read_server_frame( - client_rx - .recv_timeout(Duration::from_millis(100)) - .expect("initial frame"), - ) + read_server_frame(receive_render(client_rx, Duration::from_millis(100))) +} + +fn graphics_key(pane_id: crate::layout::PaneId) -> crate::app::pane_graphics::Key { + (pane_id, api::schema::PANE_GRAPHICS_PRIMARY_LAYER_ID.into()) +} + +fn active_gate() -> std::sync::Arc { + std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)) } fn set_graphics_layer(server: &mut HeadlessServer, pane_id: crate::layout::PaneId, data: Vec) { - server.app.state.pane_graphics_layers.insert( + set_named_graphics_layer( + server, pane_id, - crate::app::state::PaneGraphicsLayer::new( - api::schema::PaneGraphicsFormat::Png, - 1, - 1, - data, - api::schema::PaneGraphicsPlacementParams::default(), - ), + api::schema::PANE_GRAPHICS_PRIMARY_LAYER_ID, + data, + 0, ); } +fn set_named_graphics_layer( + server: &mut HeadlessServer, + pane_id: crate::layout::PaneId, + layer_id: &str, + data: Vec, + z_index: i32, +) { + let key = (pane_id, layer_id.into()); + let host_image_id = server.app.pane_graphics.reserve_image_id(&key).unwrap(); + let layer = crate::app::pane_graphics::Layer::inline( + api::schema::PaneGraphicsFormat::Png, + 1, + 1, + data, + Default::default(), + z_index, + ); + server.app.pane_graphics.slots.insert( + key, + crate::app::pane_graphics::Slot::test(host_image_id, Some(layer)), + ); +} + +fn set_stream_owner(server: &mut HeadlessServer, pane_id: crate::layout::PaneId, owner: &str) { + let key = graphics_key(pane_id); + if let Some(slot) = server.app.pane_graphics.slots.get_mut(&key) { + slot.stream_owner = Some(owner.into()); + slot.stream_active = Some(active_gate()); + } else { + let host_image_id = server.app.pane_graphics.reserve_image_id(&key).unwrap(); + let mut slot = crate::app::pane_graphics::Slot::test(host_image_id, None); + slot.stream_owner = Some(owner.into()); + slot.stream_active = Some(active_gate()); + server.app.pane_graphics.slots.insert(key, slot); + } +} + fn fill_render_lane(server: &HeadlessServer) { let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig) .expect("dummy frame"); - server - .clients - .get(&1) - .unwrap() + server.clients[&1] .writer .as_ref() .unwrap() - .render - .try_send(queued) - .expect("pre-fill render lane"); + .test_fill_render(queued); } fn stream_set_message( @@ -68,6 +141,8 @@ fn stream_set_message( method: api::schema::Method::PaneGraphicsStreamSet( api::schema::PaneGraphicsSetParams { pane_id: pane_id.into(), + layer_id: None, + z_index: 0, owner: owner.into(), format: api::schema::PaneGraphicsFormat::Png, image_width: 1, @@ -80,11 +155,114 @@ fn stream_set_message( }, respond_to, response_write_complete: None, + stream_active: None, }, response_rx, ) } +#[tokio::test] +async fn pixel_mouse_activation_requires_graphics_demand_not_direct_transport() { + let (mut server, _client_rx, pane_id) = + retained_test_server(b"\x1b[?1003h\x1b[?1006h\x1b[?1016h"); + let (writer, control_rx, _render_rx) = test_client_writer(); + let client = server.clients.get_mut(&1).unwrap(); + client.writer = Some(writer); + client.direct_graphics = false; + client.pixel_mouse = true; + client.host_mouse_capture_active = None; + client.host_sgr_pixels_active = None; + server.app.direct_graphics_available = false; + + server.stream_host_mouse_capture_mode(); + assert!(matches!( + read_server_message(control_rx.recv_timeout(Duration::from_millis(100)).unwrap()), + ServerMessage::MouseCapture { + enabled: true, + sgr_pixels: false + } + )); + + set_graphics_layer(&mut server, pane_id, vec![1, 2, 3]); + server.stream_host_mouse_capture_mode(); + assert!(matches!( + read_server_message(control_rx.recv_timeout(Duration::from_millis(100)).unwrap()), + ServerMessage::MouseCapture { + enabled: true, + sgr_pixels: true + } + )); +} + +#[tokio::test] +async fn pixel_input_metadata_cannot_resize_authoritative_client_state() { + let (mut server, _client_rx, pane_id) = + retained_test_server(b"\x1b[?1003h\x1b[?1006h\x1b[?1016h"); + set_graphics_layer(&mut server, pane_id, vec![1]); + let client = server.clients.get_mut(&1).unwrap(); + client.pixel_mouse = true; + client.host_sgr_pixels_active = Some(true); + server.foreground_client_id = None; + assert!(!server.handle_server_event(ServerEvent::ClientInputPixels { + client_id: 1, + data: b"\x1b[<0;500;300M".to_vec(), + geometry: crate::input::mouse::HostGeometry::new(80, 24, 800, 480).unwrap(), + })); + server.clients.get_mut(&1).unwrap().cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + for (geometry, data) in [ + ( + crate::input::mouse::HostGeometry::new(100, 30, 1_000, 600).unwrap(), + b"\x1b[<0;500;300M".as_slice(), + ), + ( + crate::input::mouse::HostGeometry::new(80, 24, 960, 480).unwrap(), + b"\x1b[<0;500;300M", + ), + ( + crate::input::mouse::HostGeometry::new(80, 24, 800, 480).unwrap(), + b"\x1b[<0;0;1M", + ), + ] { + assert!(!server.handle_server_event(ServerEvent::ClientInputPixels { + client_id: 1, + data: data.to_vec(), + geometry, + })); + } + assert_eq!(server.clients[&1].terminal_size, (80, 24)); + assert_eq!( + (server.effective_size, server.foreground_client_id), + ((80, 24), None) + ); +} + +#[test] +fn direct_eligibility_is_installed_with_the_client_connection() { + let mut server = test_headless_server(); + let (writer, _control_rx, _render_rx) = test_client_writer(); + + assert!(server.handle_server_event(ServerEvent::ClientConnected { + client_id: 7, + cols: 80, + rows: 24, + cell_width_px: 10, + cell_height_px: 20, + render_encoding: RenderEncoding::SemanticFrame, + keybindings: None, + direct_attach_requested: false, + direct_graphics: true, + writer, + })); + + let client = server.clients.get(&7).expect("connected client"); + assert!(client.direct_graphics); + assert_eq!(server.foreground_client_id, Some(7)); + assert!(server.app.direct_graphics_available); +} + #[tokio::test] async fn focus_repaint_preserves_uploaded_graphics() { let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa"); @@ -109,11 +287,8 @@ async fn focus_repaint_preserves_uploaded_graphics() { let initial_graphics = String::from_utf8_lossy(&initial.graphics); assert!(initial_graphics.contains("a=t")); assert!(initial_graphics.contains("a=p")); - let client_2_initial = read_server_frame( - client_2_rx - .recv_timeout(Duration::from_millis(100)) - .expect("second client initial frame"), - ); + let client_2_initial = + read_server_frame(receive_render(&client_2_rx, Duration::from_millis(100))); assert!(String::from_utf8_lossy(&client_2_initial.graphics).contains("a=t")); assert!(server.handle_server_event(ServerEvent::ClientInput { @@ -123,12 +298,68 @@ async fn focus_repaint_preserves_uploaded_graphics() { assert_eq!(server.foreground_client_id, Some(2)); server.render_and_stream(); - let focused = read_server_frame( - client_2_rx - .recv_timeout(Duration::from_millis(100)) - .expect("focus redraw"), + let focused = read_server_frame(receive_render(&client_2_rx, Duration::from_millis(100))); + let focused_graphics = String::from_utf8_lossy(&focused.graphics); + assert!(focused_graphics.contains("a=p")); + assert!(!focused_graphics.contains("a=t")); +} + +#[tokio::test] +async fn resize_replays_placement_without_retransmitting_or_closing_stream() { + let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa"); + set_graphics_layer(&mut server, pane_id, vec![1, 2, 3]); + set_stream_owner(&mut server, pane_id, "owner-resize"); + let initial = enable_graphics_and_render(&mut server, &client_rx); + assert!(String::from_utf8_lossy(&initial.graphics).contains("a=t")); + + for (cols, rows, cell_width_px, cell_height_px) in + [(100, 30, 10, 20), (100, 30, 12, 24), (100, 30, 12, 24)] + { + assert!(server.handle_server_event(ServerEvent::ClientResize { + client_id: 1, + cols, + rows, + cell_width_px, + cell_height_px, + })); + server.render_and_stream(); + let frame = read_server_frame(receive_render(&client_rx, Duration::from_millis(100))); + let graphics = String::from_utf8_lossy(&frame.graphics); + assert!(!graphics.contains("a=t")); + assert!(graphics.contains("a=p")); + } + assert_eq!( + server + .app + .pane_graphics + .slots + .get(&graphics_key(pane_id)) + .and_then(|slot| slot.stream_owner.as_deref()), + Some("owner-resize") ); - assert!(focused.graphics.is_empty()); +} + +#[tokio::test] +async fn graphics_pruning_preserves_live_panes_and_removes_closed_panes() { + let (mut server, _client_rx, pane_id) = retained_test_server(b"aaaa"); + set_graphics_layer(&mut server, pane_id, vec![1, 2, 3]); + + assert!(!server + .app + .pane_graphics + .retain_live_panes(&server.app.state)); + assert!(server + .app + .pane_graphics + .slots + .contains_key(&graphics_key(pane_id))); + + server.app.state.workspaces.clear(); + assert!(server + .app + .pane_graphics + .retain_live_panes(&server.app.state)); + assert!(server.app.pane_graphics.slots.is_empty()); } #[tokio::test] @@ -164,58 +395,25 @@ async fn retained_update_sends_only_graphics_message() { } #[tokio::test] -async fn retained_update_defers_on_full_render_lane() { +async fn retained_graphics_stays_ordered_after_an_older_render() { let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa"); let _ = enable_graphics_and_render(&mut server, &client_rx); fill_render_lane(&server); set_graphics_layer(&mut server, pane_id, vec![4, 5, 6]); - + let older = client_rx.recv_timeout(Duration::from_secs(1)).unwrap(); assert_eq!( server.render_retained_graphics_update_and_stream(), - RetainedGraphicsOutcome::Deferred + RetainedGraphicsOutcome::Sent ); - let client = server.clients.get(&1).unwrap(); - assert_eq!(client.deferred_render(), DeferredRender::Graphics); + let graphics = client_rx.recv_timeout(Duration::from_secs(1)).unwrap(); assert!(matches!( - read_server_message(client_rx.recv_timeout(Duration::from_millis(100)).unwrap()), + read_server_message(older), ServerMessage::ReloadSoundConfig )); - assert_eq!( - server.handle_server_event_with_render_impact(ServerEvent::ClientWriterDrained { - client_id: 1 - }), - RenderImpact::Graphics - ); -} - -#[tokio::test] -async fn retained_update_does_not_downgrade_pending_full_render() { - let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa"); - let _ = enable_graphics_and_render(&mut server, &client_rx); - fill_render_lane(&server); - let client = server.clients.get_mut(&1).unwrap(); - client.request_repaint(); - server.render_and_stream(); - assert_eq!( - server.clients.get(&1).unwrap().deferred_render(), - DeferredRender::Full - ); - - set_graphics_layer(&mut server, pane_id, vec![7, 8, 9]); - assert_eq!( - server.render_retained_graphics_update_and_stream(), - RetainedGraphicsOutcome::Deferred - ); - assert_eq!( - server.clients.get(&1).unwrap().deferred_render(), - DeferredRender::Full - ); - assert_eq!( - server.handle_server_event_with_render_impact(ServerEvent::ClientWriterDrained { - client_id: 1 - }), - RenderImpact::Full - ); + assert!(matches!( + read_server_message(graphics), + ServerMessage::Graphics { .. } + )); } #[tokio::test] @@ -246,6 +444,64 @@ async fn retained_update_falls_back_for_mixed_app_geometry() { ); } +#[test] +fn stream_open_gate_is_owned_by_the_layer_and_cancels_on_removal() { + let mut server = test_headless_server(); + server.app.state.kitty_graphics_enabled = true; + let workspace = crate::workspace::Workspace::test_new("gated"); + let pane_id = workspace.tabs[0].root_pane; + let public = format!("{}:p1", workspace.id); + server.app.state.workspaces = vec![workspace]; + server.app.state.active = Some(0); + let active = active_gate(); + let (respond_to, response_rx) = std::sync::mpsc::channel(); + + server.handle_api_request_with_shutdown_check(api::ApiRequestMessage { + request: api::schema::Request { + id: "open-gated".into(), + method: api::schema::Method::PaneGraphicsStreamOpen( + api::schema::PaneGraphicsStreamParams { + pane_id: public.clone(), + layer_id: None, + z_index: 0, + owner: "worker-1".into(), + }, + ), + }, + respond_to, + response_write_complete: None, + stream_active: Some(active.clone()), + }); + assert!( + serde_json::from_str::(&response_rx.recv().unwrap()).is_ok() + ); + let (frame, frame_response) = + stream_set_message("gated-frame", &public, "worker-1", vec![1, 2, 3]); + assert_eq!( + server.handle_api_request_with_render_impact(frame), + RenderImpact::Graphics + ); + assert!(frame_response.recv().is_ok()); + assert!(active.load(std::sync::atomic::Ordering::Acquire)); + active.store(false, std::sync::atomic::Ordering::Release); + let (delayed, delayed_response) = + stream_set_message("delayed-frame", &public, "worker-1", vec![4, 5, 6]); + assert_eq!( + server.handle_api_request_with_render_impact(delayed), + RenderImpact::None + ); + let error: api::schema::ErrorResponse = + serde_json::from_str(&delayed_response.recv().unwrap()).unwrap(); + assert_eq!(error.error.code, "stream_closed"); + assert!(server + .app + .pane_graphics + .slots + .remove(&graphics_key(pane_id)) + .is_some()); + assert!(!active.load(std::sync::atomic::Ordering::Acquire)); +} + #[test] fn stream_set_has_graphics_only_render_impact() { let mut server = test_headless_server(); @@ -256,11 +512,7 @@ fn stream_set_has_graphics_only_render_impact() { server.app.state.active = Some(0); server.app.state.selected = 0; server.app.state.kitty_graphics_enabled = true; - server - .app - .state - .pane_graphics_streams - .insert(pane_id, "owner-a".into()); + set_stream_owner(&mut server, pane_id, "owner-a"); let (request, response_rx) = stream_set_message("wrong-owner", &public_pane_id, "owner-b", vec![1, 2, 3]); @@ -307,13 +559,15 @@ fn stream_set_has_graphics_only_render_impact() { RenderImpact::Full ); - server.app.state.pane_graphics_streams.clear(); + server.app.pane_graphics.clear(); let (respond_to, _response_rx) = std::sync::mpsc::channel(); let impact = server.handle_api_request_with_render_impact(api::ApiRequestMessage { request: api::schema::Request { id: "direct-frame".into(), method: api::schema::Method::PaneGraphicsSet(api::schema::PaneGraphicsSetParams { pane_id: public_pane_id, + layer_id: None, + z_index: 0, owner: String::new(), format: api::schema::PaneGraphicsFormat::Png, image_width: 1, @@ -325,6 +579,7 @@ fn stream_set_has_graphics_only_render_impact() { }, respond_to, response_write_complete: None, + stream_active: None, }); assert_eq!(impact, RenderImpact::Full); } @@ -345,6 +600,8 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() { id: "disabled-set".into(), method: api::schema::Method::PaneGraphicsSet(api::schema::PaneGraphicsSetParams { pane_id: public_pane_id.clone(), + layer_id: None, + z_index: 0, owner: String::new(), format: api::schema::PaneGraphicsFormat::Png, image_width: 1, @@ -356,6 +613,7 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() { }, respond_to, response_write_complete: None, + stream_active: None, }); assert!(!changed); let response = response_rx @@ -370,11 +628,7 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() { ); server.app.state.kitty_graphics_enabled = true; - server - .app - .state - .pane_graphics_streams - .insert(pane_id, "current-owner".into()); + set_stream_owner(&mut server, pane_id, "current-owner"); let (respond_to, response_rx) = std::sync::mpsc::channel(); let impact = server.handle_api_request_with_render_impact(api::ApiRequestMessage { request: api::schema::Request { @@ -382,17 +636,25 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() { method: api::schema::Method::PaneGraphicsStreamClose( api::schema::PaneGraphicsStreamParams { pane_id: public_pane_id, + layer_id: None, + z_index: 0, owner: "stale-owner".into(), }, ), }, respond_to, response_write_complete: None, + stream_active: None, }); assert_eq!(impact, RenderImpact::None); assert_eq!( - server.app.state.pane_graphics_streams.get(&pane_id), - Some(&"current-owner".to_string()) + server + .app + .pane_graphics + .slots + .get(&graphics_key(pane_id)) + .and_then(|slot| slot.stream_owner.as_deref()), + Some("current-owner") ); assert!(serde_json::from_str::( &response_rx @@ -401,3 +663,282 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() { ) .is_ok()); } + +#[cfg(unix)] +fn direct_gate_server( + data: &[u8], +) -> ( + HeadlessServer, + crate::app::pane_graphics::Key, + std::sync::mpsc::Receiver, +) { + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + let mut server = test_headless_server(); + let workspace = crate::workspace::Workspace::test_new("direct-gate"); + let pane_id = workspace.tabs[0].root_pane; + server.app.state.workspaces = vec![workspace]; + server.app.state.active = Some(0); + let key = graphics_key(pane_id); + let path = server + .app + .pane_graphics_files + .source_directory() + .unwrap() + .join("gate-frame"); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .unwrap(); + file.write_all(data).unwrap(); + drop(file); + let lease = server + .app + .pane_graphics_files + .lease(&path, data.len()) + .unwrap(); + let (respond_to, response_rx) = std::sync::mpsc::channel(); + let layer = + crate::app::pane_graphics::Layer::direct(1, 1, lease.clone(), Default::default(), 0); + let mut slot = crate::app::pane_graphics::Slot::test((1 << 31) | 900, Some(layer)); + slot.stream_owner = Some("owner".into()); + slot.stream_active = Some(active_gate()); + slot.direct_gate = Some(crate::app::pane_graphics::DirectGate { + transfer_id: lease.fingerprint(), + client_id: 7, + deadline: std::time::Instant::now() + Duration::from_secs(1), + written: true, + success_response: "ack".into(), + respond_to, + }); + server.app.pane_graphics.slots.insert(key.clone(), slot); + (server, key, response_rx) +} + +#[cfg(unix)] +fn direct_ids(server: &HeadlessServer, key: &crate::app::pane_graphics::Key) -> (u64, u32) { + let slot = &server.app.pane_graphics.slots[key]; + ( + slot.direct_gate.as_ref().unwrap().transfer_id, + slot.host_image_id, + ) +} + +#[cfg(unix)] +fn add_direct_client(server: &mut HeadlessServer, client_id: u64) { + let (writer, control_rx, render_rx) = test_client_writer(); + std::mem::forget((control_rx, render_rx)); + let mut client = ClientConnection::new( + (80, 24), + crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }, + crate::terminal_theme::TerminalTheme::default(), + None, + 1, + RenderEncoding::SemanticFrame, + Some(writer), + ); + client.direct_graphics = true; + client.pixel_mouse = true; + server.clients.insert(client_id, client); +} + +#[cfg(unix)] +#[test] +fn terminal_response_deadline_starts_only_after_client_flush() { + let (mut server, key, _response_rx) = direct_gate_server(&[1, 2, 3, 4]); + let slot = server.app.pane_graphics.slots.get_mut(&key).unwrap(); + let gate = slot.direct_gate.as_mut().unwrap(); + gate.written = false; + let (transfer_id, image_id) = (gate.transfer_id, slot.host_image_id); + assert!(!server.complete_direct_graphics(7, transfer_id, image_id, true)); + assert!(!server.start_direct_graphics_response(7, transfer_id, image_id)); + let gate = server.app.pane_graphics.slots[&key] + .direct_gate + .as_ref() + .unwrap(); + assert!(gate.written && gate.deadline > std::time::Instant::now()); +} + +#[cfg(unix)] +#[test] +fn outer_timeout_covers_both_direct_phases_and_cancellation_blocks_late_results() { + assert!( + crate::app::pane_graphics::DIRECT_OUTER_TIMEOUT + > crate::app::pane_graphics::DIRECT_DELIVERY_TIMEOUT + + crate::app::pane_graphics::DIRECT_RESPONSE_TIMEOUT + ); + let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); + let slot = server.app.pane_graphics.slots.get_mut(&key).unwrap(); + slot.stream_active + .as_ref() + .unwrap() + .store(false, std::sync::atomic::Ordering::Release); + let (transfer_id, image_id) = ( + slot.direct_gate.as_ref().unwrap().transfer_id, + slot.host_image_id, + ); + + assert!(!server.complete_direct_graphics(7, transfer_id, image_id, true)); + assert!(response_rx.try_recv().is_err()); + assert!(server.app.pane_graphics.slots[&key] + .layer + .as_ref() + .unwrap() + .direct_lease() + .is_some()); +} + +#[cfg(unix)] +#[test] +fn matching_terminal_ok_releases_producer_and_acknowledges() { + let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); + let (transfer_id, image_id) = direct_ids(&server, &key); + + assert!(server.complete_direct_graphics(7, transfer_id, image_id, true)); + + assert_eq!(response_rx.recv().unwrap(), "ack"); + let layer = server.app.pane_graphics.slots[&key].layer.as_ref().unwrap(); + assert!(layer.terminal_only()); + assert!(layer.direct_lease().is_none()); +} + +#[cfg(unix)] +#[test] +fn explicit_terminal_error_acks_only_after_owned_inline_fallback() { + let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); + add_direct_client(&mut server, 7); + let (transfer_id, image_id) = direct_ids(&server, &key); + let layer = server.app.pane_graphics.slots[&key].layer.as_ref().unwrap(); + server + .clients + .get_mut(&7) + .unwrap() + .graphics_cache + .trust_pane_layer(&key, image_id, layer); + assert!(server.complete_direct_graphics(7, transfer_id, image_id, false)); + + let layer = server.app.pane_graphics.slots[&key].layer.as_ref().unwrap(); + assert_eq!( + ( + response_rx.recv().unwrap(), + layer.inline_data(), + server.clients[&7].direct_graphics, + server.clients[&7].pixel_mouse, + ), + ("ack".into(), Some([1, 2, 3, 4].as_slice()), false, true) + ); + assert!(server.clients[&7].graphics_cache.is_empty()); +} + +#[cfg(unix)] +#[test] +fn unwritten_direct_full_falls_back_without_stickiness_but_disconnect_retires() { + for error in [ + std::sync::mpsc::TrySendError::Full(Vec::new()), + std::sync::mpsc::TrySendError::Disconnected(Vec::new()), + ] { + let should_ack = matches!(error, std::sync::mpsc::TrySendError::Full(_)); + let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); + add_direct_client(&mut server, 7); + let gate = server + .app + .pane_graphics + .slots + .get_mut(&key) + .and_then(|slot| slot.direct_gate.take()) + .unwrap(); + let result = server.handle_unwritten_direct_failure( + &key, + gate.success_response, + gate.respond_to, + error, + ); + let inline = server + .app + .pane_graphics + .slots + .get(&key) + .and_then(|slot| slot.layer.as_ref()?.inline_data()) + .is_some(); + assert_eq!( + ( + result, + response_rx.try_recv().ok().as_deref() == Some("ack"), + inline, + server.clients[&7].direct_graphics, + ), + (should_ack, should_ack, should_ack, true) + ); + } +} + +#[cfg(unix)] +#[test] +fn client_loss_retires_only_its_direct_stream() { + let (mut pending, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); + pending.retire_direct_graphics_for_client(8); + assert!(pending.app.pane_graphics.slots.contains_key(&key)); + pending.retire_direct_graphics_for_client(7); + assert!(!pending.app.pane_graphics.slots.contains_key(&key)); + assert!(response_rx.recv().is_err()); + + let (mut resident, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); + let slot = resident.app.pane_graphics.slots.get(&key).unwrap(); + assert!(resident.complete_direct_graphics( + 7, + slot.direct_gate.as_ref().unwrap().transfer_id, + slot.host_image_id, + true, + )); + assert_eq!(response_rx.recv().unwrap(), "ack"); + resident.retire_direct_graphics_for_client(8); + assert!(resident.app.pane_graphics.slots.contains_key(&key)); + resident.retire_direct_graphics_for_client(7); + assert!(!resident.app.pane_graphics.slots.contains_key(&key)); +} + +#[cfg(unix)] +#[test] +fn pane_removal_and_shutdown_drop_direct_without_ack() { + let setups: [fn(&mut HeadlessServer); 2] = [ + |server| server.app.state.workspaces.clear(), + |server| server.shutting_down = true, + ]; + for setup in setups { + let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); + let (transfer_id, image_id) = direct_ids(&server, &key); + setup(&mut server); + assert!(!server.complete_direct_graphics(7, transfer_id, image_id, true)); + assert!(response_rx.recv().is_err()); + assert!(!server.app.pane_graphics.slots.contains_key(&key)); + } +} + +#[cfg(unix)] +#[test] +fn timeout_retires_stream_without_producer_ack() { + let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); + add_direct_client(&mut server, 7); + server + .app + .pane_graphics + .slots + .get_mut(&key) + .unwrap() + .direct_gate + .as_mut() + .unwrap() + .deadline = std::time::Instant::now() - Duration::from_millis(1); + + assert!(server.expire_direct_graphics(std::time::Instant::now())); + + assert!(response_rx.recv().is_err()); + assert!(!server.app.pane_graphics.slots.contains_key(&key)); + assert!(!server.clients[&7].direct_graphics); + assert!(server.clients[&7].pixel_mouse); +} diff --git a/src/terminal/runtime.rs b/src/terminal/runtime.rs index 8a3da876..13e29690 100644 --- a/src/terminal/runtime.rs +++ b/src/terminal/runtime.rs @@ -465,31 +465,32 @@ impl TerminalRuntime { pub fn encode_mouse_button( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { - self.0.encode_mouse_button(kind, column, row, modifiers) + self.0.encode_mouse_button(kind, position, modifiers) } - pub fn encode_mouse_motion( + pub(crate) fn encode_mouse_motion( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { - self.0.encode_mouse_motion(kind, column, row, modifiers) + self.0.encode_mouse_motion(kind, position, modifiers) } - pub fn encode_mouse_wheel( + pub(crate) fn encode_mouse_wheel( &self, kind: crossterm::event::MouseEventKind, - column: u16, - row: u16, + position: crate::input::mouse::Position, modifiers: crossterm::event::KeyModifiers, ) -> Option> { - self.0.encode_mouse_wheel(kind, column, row, modifiers) + self.0.encode_mouse_wheel(kind, position, modifiers) + } + + pub(crate) fn pixel_size(&self) -> Option<(u32, u32)> { + self.0.pixel_size() } pub fn encode_alternate_scroll(