diff options
| author | Can Celik <ogulcancelik@gmail.com> | 2026-08-09 02:04:39 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-08-09 02:04:39 +0300 |
| commit | 1777e9bba32b953ed1ad203b4a16d01105539000 (patch) | |
| tree | 22096973812c9baba822453d8470bdcfe64fc2a6 /src/app | |
| parent | 36a8d36a7258e08b40c01bf521b1f4006525f143 (diff) | |
feat(graphics): add direct pane frame streaming (#2523)
* feat(graphics): add direct pane frame streaming
* feat(graphics): expose pane visibility
* test(graphics): preserve legacy terminal image output
* fix(graphics): harden streaming lifecycle
* fix(input): preserve mouse releases outside panes
Diffstat (limited to 'src/app')
| -rw-r--r-- | src/app/actions.rs | 34 | ||||
| -rw-r--r-- | src/app/api.rs | 3 | ||||
| -rw-r--r-- | src/app/api/pane_graphics.rs | 1512 | ||||
| -rw-r--r-- | src/app/api/plugins/mod.rs | 4 | ||||
| -rw-r--r-- | src/app/ids.rs | 2 | ||||
| -rw-r--r-- | src/app/input/mod.rs | 14 | ||||
| -rw-r--r-- | src/app/input/mouse.rs | 152 | ||||
| -rw-r--r-- | src/app/mod.rs | 61 | ||||
| -rw-r--r-- | src/app/pane_graphics.rs | 347 | ||||
| -rw-r--r-- | src/app/runtime.rs | 8 | ||||
| -rw-r--r-- | src/app/state.rs | 54 |
11 files changed, 1566 insertions, 625 deletions
diff --git a/src/app/actions.rs b/src/app/actions.rs index 40707cee..20deab2a 100644 --- a/src/app/actions.rs +++ b/src/app/actions.rs @@ -1662,8 +1662,6 @@ impl AppState { } for pane_id in pane_ids { self.plugin_panes.remove(&pane_id); - self.pane_graphics_layers.remove(&pane_id); - self.pane_graphics_streams.remove(&pane_id); } } @@ -3402,26 +3400,6 @@ mod tests { state } - fn insert_test_pane_graphics_layer(state: &mut AppState, pane_id: PaneId) { - state.pane_graphics_layers.insert( - pane_id, - crate::app::state::PaneGraphicsLayer::new( - crate::api::schema::PaneGraphicsFormat::Rgba, - 1, - 1, - vec![1, 2, 3, 4], - crate::api::schema::PaneGraphicsPlacementParams::default(), - ), - ); - } - - fn insert_test_pane_graphics_state(state: &mut AppState, pane_id: PaneId) { - insert_test_pane_graphics_layer(state, pane_id); - state - .pane_graphics_streams - .insert(pane_id, "test-stream".into()); - } - fn mark_linked_worktree(state: &mut AppState, ws_idx: usize) { state.workspaces[ws_idx].worktree_space = Some(crate::workspace::WorktreeSpaceMembership { key: "repo-key".into(), @@ -5834,13 +5812,9 @@ mod tests { entrypoint: "board".into(), }, ); - insert_test_pane_graphics_state(&mut state, closed); - state.close_pane(); assert_eq!(state.workspaces[0].panes.len(), 1); assert!(!state.plugin_panes.contains_key(&closed)); - assert!(!state.pane_graphics_layers.contains_key(&closed)); - assert!(!state.pane_graphics_streams.contains_key(&closed)); state.assert_invariants_for_test(); } @@ -5910,14 +5884,10 @@ mod tests { entrypoint: "board".into(), }, ); - insert_test_pane_graphics_state(&mut state, pane_id); - state.close_tab(); assert!(!state.terminals.contains_key(&terminal_id)); assert!(!state.plugin_panes.contains_key(&pane_id)); - assert!(!state.pane_graphics_layers.contains_key(&pane_id)); - assert!(!state.pane_graphics_streams.contains_key(&pane_id)); state.assert_invariants_for_test(); } @@ -5933,14 +5903,10 @@ mod tests { entrypoint: "board".into(), }, ); - insert_test_pane_graphics_state(&mut state, pane_id); - state.close_selected_workspace(); assert!(!state.terminals.contains_key(&terminal_id)); assert!(!state.plugin_panes.contains_key(&pane_id)); - assert!(!state.pane_graphics_layers.contains_key(&pane_id)); - assert!(!state.pane_graphics_streams.contains_key(&pane_id)); state.assert_invariants_for_test(); } diff --git a/src/app/api.rs b/src/app/api.rs index 6dc40c3f..0ccd9b21 100644 --- a/src/app/api.rs +++ b/src/app/api.rs @@ -1102,6 +1102,9 @@ impl App { Method::PaneGraphicsStreamSet(params) => { return self.handle_pane_graphics_stream_set(request.id, params); } + Method::PaneGraphicsStreamDirect(params) => { + return self.handle_pane_graphics_stream_direct(request.id, params); + } Method::PaneGraphicsStreamOpen(params) => { return self.handle_pane_graphics_stream_open(request.id, params); } diff --git a/src/app/api/pane_graphics.rs b/src/app/api/pane_graphics.rs index 7851f69a..f5f5f2b7 100644 --- a/src/app/api/pane_graphics.rs +++ b/src/app/api/pane_graphics.rs @@ -2,121 +2,126 @@ use base64::Engine; use crate::api::schema::{ PaneGraphicsClearParams, PaneGraphicsSetParams, PaneGraphicsStreamParams, ResponseResult, - PANE_GRAPHICS_SET_MAX_BYTES, PANE_GRAPHICS_STREAM_MAX_BYTES, + PANE_GRAPHICS_MAX_LAYERS_PER_PANE, PANE_GRAPHICS_PRIMARY_LAYER_ID, PANE_GRAPHICS_SET_MAX_BYTES, + PANE_GRAPHICS_STREAM_MAX_BYTES, }; -use crate::app::state::PaneGraphicsLayer; +use crate::app::pane_graphics::{Key as PaneGraphicsKey, Layer, Slot}; use crate::app::App; use crate::layout::PaneId; use super::responses::{encode_error, encode_success}; impl App { + /// Whether graphics for a pane can be placed on the surface rendered by this client. + /// + /// This deliberately follows the pane layout rather than pane existence: inactive + /// workspaces/tabs and panes hidden by zoom are not placeable. Short-lived UI modes do not + /// suspend the producer because the pane becomes visible again without a layout event. + fn pane_graphics_visible(&self, ws_idx: usize, pane_id: PaneId) -> bool { + if self.state.active != Some(ws_idx) { + return false; + } + let Some(tab) = self.state.workspaces[ws_idx].active_tab() else { + return false; + }; + if tab.zoomed { + tab.layout.focused() == pane_id + } else { + tab.layout.pane_ids().contains(&pane_id) + } + } + pub(super) fn handle_pane_graphics_info( &mut self, id: String, target: crate::api::schema::PaneTarget, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } - if self.parse_pane_id(&target.pane_id).is_none() { + let Some((ws_idx, pane_id)) = self.parse_pane_id(&target.pane_id) else { return pane_not_found(id, &target.pane_id); - } + }; + let pane_visible = self.pane_graphics_visible(ws_idx, pane_id); if !self.state.host_cell_size.is_known() { return encode_error(id, "cell_size_unavailable", "host cell size is unavailable"); } + let file_frame_directory = self + .direct_graphics_available + .then(|| self.pane_graphics_files.source_directory().ok()) + .flatten(); + let direct = file_frame_directory.is_some(); encode_success( id, ResponseResult::PaneGraphicsInfo { cell_width_px: self.state.host_cell_size.width_px, cell_height_px: self.state.host_cell_size.height_px, + pane_visible, + file_frame_directory: file_frame_directory + .map(|directory| directory.to_string_lossy().into_owned()), + file_frame_formats: if direct { + vec!["rgba".into(), "bgra".into()] + } else { + Vec::new() + }, + file_frame_max_bytes: direct.then_some(PANE_GRAPHICS_STREAM_MAX_BYTES), + // Damage metadata never changes the complete canonical frame contract. + file_frame_damage: true, + max_layers_per_pane: PANE_GRAPHICS_MAX_LAYERS_PER_PANE, + pixel_mouse: self.pixel_mouse_available, + file_frame_transport: direct.then(|| "direct-kitty".into()), }, ) } + pub(crate) fn attach_pane_graphics_stream_active( + &mut self, + params: &PaneGraphicsStreamParams, + active: std::sync::Arc<std::sync::atomic::AtomicBool>, + response: &str, + ) { + if serde_json::from_str::<crate::api::schema::SuccessResponse>(response).is_err() { + return; + } + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + return; + }; + let Ok(key) = graphics_key(pane_id, params.layer_id.as_deref()) else { + return; + }; + self.pane_graphics + .attach_stream_active(&key, ¶ms.owner, active); + } + pub(super) fn handle_pane_graphics_set( &mut self, id: String, params: PaneGraphicsSetParams, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - if self.state.pane_graphics_streams.contains_key(&pane_id) { + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + self.reclaim_inactive_stream(&key); + if self + .pane_graphics + .slots + .get(&key) + .is_some_and(|slot| slot.stream_owner.is_some()) + { return encode_error( id, "stream_conflict", - "pane already has an active graphics stream", - ); - } - self.set_pane_graphics_layer(id, pane_id, params) - } - - fn set_pane_graphics_layer( - &mut self, - id: String, - pane_id: PaneId, - params: PaneGraphicsSetParams, - ) -> String { - if params.image_width == 0 || params.image_height == 0 { - return encode_error( - id, - "invalid_image", - "image_width and image_height must be greater than zero", + "pane graphics layer has an active stream", ); } - let max_bytes = if params.data.is_some() { - PANE_GRAPHICS_STREAM_MAX_BYTES - } else { - PANE_GRAPHICS_SET_MAX_BYTES - }; - let data = match params.data { - Some(data) => data, - None => match base64::engine::general_purpose::STANDARD.decode(params.data_base64) { - Ok(data) => data, - Err(_) => { - return encode_error(id, "invalid_image", "data_base64 is not valid base64"); - } - }, - }; - if data.is_empty() { - return encode_error(id, "invalid_image", "image data must not be empty"); - } - if data.len() > max_bytes { - return encode_error(id, "image_too_large", "image data is too large"); - } - match pane_graphics_expected_data_len( - params.format, - params.image_width, - params.image_height, - ) { - Ok(Some(expected_len)) if data.len() != expected_len => { - return encode_error( - id, - "invalid_image", - "image data length does not match format and dimensions", - ); - } - Ok(_) => {} - Err(()) => { - return encode_error(id, "invalid_image", "image dimensions are too large"); - } - } - - let layer = PaneGraphicsLayer::new( - params.format, - params.image_width, - params.image_height, - data, - params.placement, - ); - self.state.pane_graphics_layers.insert(pane_id, layer); - self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); - - encode_success(id, ResponseResult::Ok {}) + self.set_layer(id, key, params) } pub(super) fn handle_pane_graphics_clear( @@ -124,23 +129,32 @@ impl App { id: String, params: PaneGraphicsClearParams, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - if self.state.pane_graphics_streams.contains_key(&pane_id) { + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + self.reclaim_inactive_stream(&key); + if self + .pane_graphics + .slots + .get(&key) + .is_some_and(|slot| slot.stream_owner.is_some()) + { return encode_error( id, "stream_conflict", - "pane already has an active graphics stream", + "pane graphics layer has an active stream", ); } - if self.state.pane_graphics_layers.remove(&pane_id).is_some() { - self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); + if self.pane_graphics.slots.remove(&key).is_some() { + self.pane_graphics.mark_changed(); } - encode_success(id, ResponseResult::Ok {}) } @@ -149,24 +163,32 @@ impl App { id: String, params: PaneGraphicsSetParams, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - match self.state.pane_graphics_streams.get(&pane_id) { - Some(owner) if owner == ¶ms.owner => {} - Some(_) => { - return encode_error( - id, - "stream_conflict", - "pane graphics stream owner does not match active stream", - ); + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + match self.pane_graphics.slots.get(&key) { + Some(slot) + if slot.stream_owner.as_ref() == Some(¶ms.owner) && slot.stream_is_active() => + { + self.set_layer(id, key, params) + } + Some(slot) if slot.stream_owner.as_ref() == Some(¶ms.owner) => { + encode_error(id, "stream_closed", "pane graphics stream is not active") } - None => return encode_error(id, "stream_closed", "pane graphics stream is not active"), + Some(_) => encode_error( + id, + "stream_conflict", + "pane graphics stream owner does not match active stream", + ), + None => encode_error(id, "stream_closed", "pane graphics stream is not active"), } - self.set_pane_graphics_layer(id, pane_id, params) } pub(super) fn handle_pane_graphics_stream_open( @@ -174,7 +196,7 @@ impl App { id: String, params: PaneGraphicsStreamParams, ) -> String { - if let Err(response) = require_pane_graphics_enabled(self, &id) { + if let Err(response) = require_enabled(self, &id) { return response; } if params.owner.is_empty() { @@ -184,22 +206,57 @@ impl App { "pane graphics stream owner is required", ); } - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - if self.state.pane_graphics_streams.contains_key(&pane_id) { + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + let stale = self + .pane_graphics + .slots + .get(&key) + .is_some_and(|slot| slot.stream_owner.is_some() && !slot.stream_is_active()); + if stale { + self.pane_graphics.slots.remove(&key); + } + if self + .pane_graphics + .slots + .get(&key) + .and_then(|slot| slot.stream_owner.as_ref()) + .is_some() + { return encode_error( id, "stream_conflict", - "pane already has an active graphics stream", + "pane graphics layer has an active stream", ); } - self.state - .pane_graphics_streams - .insert(pane_id, params.owner); - self.state.pane_graphics_layers.remove(&pane_id); - self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); - + let exists = self.pane_graphics.slots.contains_key(&key); + if !self.pane_graphics.can_add_slot(&key) + || (!exists + && self.pane_graphics.layer_count(pane_id) >= PANE_GRAPHICS_MAX_LAYERS_PER_PANE) + { + return encode_error(id, "layer_limit", "pane graphics layer limit reached"); + } + let Some(host_image_id) = self.pane_graphics.reserve_image_id(&key) else { + return encode_error(id, "layer_limit", "pane graphics image ids are exhausted"); + }; + self.pane_graphics.slots.insert( + key, + Slot { + host_image_id, + layer: None, + stream_owner: Some(params.owner), + stream_active: Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new( + true, + ))), + direct_gate: None, + }, + ); + self.pane_graphics.mark_changed(); encode_success(id, ResponseResult::Ok {}) } @@ -208,55 +265,286 @@ impl App { id: String, params: PaneGraphicsStreamParams, ) -> String { - let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { return pane_not_found(id, ¶ms.pane_id); }; - if self - .state - .pane_graphics_streams - .get(&pane_id) - .is_some_and(|owner| owner == ¶ms.owner) + if let Ok(key) = graphics_key(pane_id, params.layer_id.as_deref()) { + if self + .pane_graphics + .slots + .get(&key) + .and_then(|slot| slot.stream_owner.as_ref()) + .is_some_and(|owner| owner == ¶ms.owner) + { + self.pane_graphics.slots.remove(&key); + self.pane_graphics.mark_changed(); + } + } + encode_success(id, ResponseResult::Ok {}) + } + + pub(super) fn handle_pane_graphics_stream_direct( + &mut self, + id: String, + params: crate::api::schema::PaneGraphicsDirectParams, + ) -> String { + if let Err(response) = require_enabled(self, &id) { + return response; + } + let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + return pane_not_found(id, ¶ms.pane_id); + }; + let key = match graphics_key(pane_id, params.layer_id.as_deref()) { + Ok(key) => key, + Err(message) => return encode_error(id, "invalid_layer_id", message), + }; + let owner_matches = self.pane_graphics.slots.get(&key).is_some_and(|slot| { + slot.stream_owner.as_deref() == Some(params.owner.as_str()) && slot.stream_is_active() + }); + if !owner_matches { + return encode_error(id, "stream_closed", "pane graphics stream is not active"); + } + if params.image_width == 0 || params.image_height == 0 { + return encode_error( + id, + "invalid_image", + "image_width and image_height must be greater than zero", + ); + } + if !matches!( + params.format, + crate::api::schema::PaneGraphicsFormat::Rgba + | crate::api::schema::PaneGraphicsFormat::Bgra + ) { + return encode_error(id, "invalid_image", "direct frames require rgba or bgra"); + } + let expected_len = + match expected_len(params.format, params.image_width, params.image_height) { + Ok(Some(len)) if len <= PANE_GRAPHICS_STREAM_MAX_BYTES => len, + _ => return encode_error(id, "invalid_image", "invalid direct RGBA dimensions"), + }; + let lease = match self + .pane_graphics_files + .lease(std::path::Path::new(¶ms.path), expected_len) { - self.state.pane_graphics_streams.remove(&pane_id); - self.state.pane_graphics_layers.remove(&pane_id); - self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); + Ok(lease) => lease, + Err(err) => return encode_error(id, "invalid_frame_file", err.to_string()), + }; + let primary = key.1 == PANE_GRAPHICS_PRIMARY_LAYER_ID; + let direct = self.direct_graphics_available + && primary + && params.format == crate::api::schema::PaneGraphicsFormat::Rgba; + if !direct && !self.pane_graphics.can_store_inline(&key, expected_len) { + return encode_error( + id, + "graphics_budget_exceeded", + "pane graphics inline memory limit reached", + ); + } + let layer = if direct { + Layer::direct( + params.image_width, + params.image_height, + lease, + params.placement, + params.z_index, + ) + } else { + let mut data = match lease.copy_rgba() { + Ok(data) => data, + Err(err) => return encode_error(id, "invalid_frame_file", err.to_string()), + }; + canonicalize_bgra(params.format, &mut data); + Layer::inline( + crate::api::schema::PaneGraphicsFormat::Rgba, + params.image_width, + params.image_height, + data, + params.placement, + params.z_index, + ) + }; + let Some(slot) = self.pane_graphics.slots.get_mut(&key) else { + return encode_error(id, "stream_closed", "pane graphics stream is not active"); + }; + if !slot.stream_is_active() { + return encode_error(id, "stream_closed", "pane graphics stream is not active"); } + slot.layer = Some(layer); + slot.direct_gate = None; + self.pane_graphics.mark_changed(); + encode_success( + id, + ResponseResult::PaneGraphicsFrameAck { + sequence: params.sequence, + revision: params.revision, + }, + ) + } + fn set_layer( + &mut self, + id: String, + key: PaneGraphicsKey, + params: PaneGraphicsSetParams, + ) -> String { + if !self.pane_graphics.can_add_slot(&key) + || (!self.pane_graphics.slots.contains_key(&key) + && self.pane_graphics.layer_count(key.0) >= PANE_GRAPHICS_MAX_LAYERS_PER_PANE) + { + return encode_error(id, "layer_limit", "pane graphics layer limit reached"); + } + if params.image_width == 0 || params.image_height == 0 { + return encode_error( + id, + "invalid_image", + "image_width and image_height must be greater than zero", + ); + } + let expected = match expected_len(params.format, params.image_width, params.image_height) { + Ok(expected) => expected, + Err(()) => return encode_error(id, "invalid_image", "image dimensions are too large"), + }; + let streamed = params.data.is_some(); + let mut data = match params.data { + Some(data) => data, + None => match base64::engine::general_purpose::STANDARD.decode(params.data_base64) { + Ok(data) => data, + Err(_) => { + return encode_error(id, "invalid_image", "data_base64 is not valid base64") + } + }, + }; + let limit = if streamed { + PANE_GRAPHICS_STREAM_MAX_BYTES + } else { + PANE_GRAPHICS_SET_MAX_BYTES + }; + if data.is_empty() { + return encode_error(id, "invalid_image", "image data must not be empty"); + } + if data.len() > limit { + return encode_error(id, "image_too_large", "image data is too large"); + } + if expected.is_some_and(|size| size != data.len()) { + return encode_error( + id, + "invalid_image", + "image data does not match the frame contract", + ); + } + let format = canonicalize_bgra(params.format, &mut data); + if !self.pane_graphics.can_store_inline(&key, data.len()) { + return encode_error( + id, + "graphics_budget_exceeded", + "pane graphics inline memory limit reached", + ); + } + let Some(host_image_id) = self.pane_graphics.reserve_image_id(&key) else { + return encode_error(id, "layer_limit", "pane graphics image ids are exhausted"); + }; + // Move the liveness handle out before replacing the slot so Slot::drop cannot + // deactivate the stream that the replacement still owns. + let (stream_owner, stream_active) = self + .pane_graphics + .slots + .get_mut(&key) + .map(|slot| (slot.stream_owner.take(), slot.stream_active.take())) + .unwrap_or_default(); + self.pane_graphics.slots.insert( + key, + Slot { + host_image_id, + layer: Some(Layer::inline( + format, + params.image_width, + params.image_height, + data, + params.placement, + params.z_index, + )), + stream_owner, + stream_active, + direct_gate: None, + }, + ); + self.pane_graphics.mark_changed(); encode_success(id, ResponseResult::Ok {}) } + + fn reclaim_inactive_stream(&mut self, key: &PaneGraphicsKey) { + let stale = self + .pane_graphics + .slots + .get(key) + .is_some_and(|slot| slot.stream_owner.is_some() && !slot.stream_is_active()); + if stale { + self.pane_graphics.slots.remove(key); + self.pane_graphics.mark_changed(); + } + } +} + +fn canonicalize_bgra( + format: crate::api::schema::PaneGraphicsFormat, + data: &mut [u8], +) -> crate::api::schema::PaneGraphicsFormat { + if format != crate::api::schema::PaneGraphicsFormat::Bgra { + return format; + } + for pixel in data.chunks_exact_mut(4) { + pixel.swap(0, 2); + } + crate::api::schema::PaneGraphicsFormat::Rgba } -fn require_pane_graphics_enabled(app: &App, id: &str) -> Result<(), String> { - if app.state.kitty_graphics_enabled { - return Ok(()); +fn graphics_key(pane_id: PaneId, layer_id: Option<&str>) -> Result<PaneGraphicsKey, &'static str> { + let layer_id = layer_id.unwrap_or(PANE_GRAPHICS_PRIMARY_LAYER_ID); + if layer_id.is_empty() || layer_id.len() > 64 { + return Err("layer_id must contain between 1 and 64 characters"); } - Err(encode_error( - id.to_owned(), - "feature_disabled", - "pane graphics require experimental.kitty_graphics", - )) + if !layer_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) + { + return Err("layer_id contains unsupported characters"); + } + Ok((pane_id, layer_id.to_owned())) } -fn pane_graphics_expected_data_len( +fn expected_len( format: crate::api::schema::PaneGraphicsFormat, - image_width: u32, - image_height: u32, + width: u32, + height: u32, ) -> Result<Option<usize>, ()> { - let bytes_per_pixel = match format { + let bytes = match format { crate::api::schema::PaneGraphicsFormat::Png => return Ok(None), crate::api::schema::PaneGraphicsFormat::Rgb => 3_u64, - crate::api::schema::PaneGraphicsFormat::Rgba => 4_u64, + crate::api::schema::PaneGraphicsFormat::Rgba + | crate::api::schema::PaneGraphicsFormat::Bgra => 4_u64, }; - let pixels = u64::from(image_width) - .checked_mul(u64::from(image_height)) - .ok_or(())?; - pixels - .checked_mul(bytes_per_pixel) - .and_then(|bytes| usize::try_from(bytes).ok()) + u64::from(width) + .checked_mul(u64::from(height)) + .and_then(|pixels| pixels.checked_mul(bytes)) + .and_then(|size| usize::try_from(size).ok()) .map(Some) .ok_or(()) } +fn require_enabled(app: &App, id: &str) -> Result<(), String> { + app.state + .kitty_graphics_enabled + .then_some(()) + .ok_or_else(|| { + encode_error( + id.to_owned(), + "feature_disabled", + "pane graphics require experimental.kitty_graphics", + ) + }) +} + fn pane_not_found(id: String, pane_id: &str) -> String { encode_error(id, "pane_not_found", format!("pane {pane_id} not found")) } @@ -268,435 +556,667 @@ mod tests { use crate::config::Config; use crate::workspace::Workspace; - fn app_with_test_workspace() -> (App, String) { - let (_api_tx, api_rx) = tokio::sync::mpsc::unbounded_channel(); + fn app() -> (App, String) { + let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); let mut app = App::new( &Config::default(), true, None, - api_rx, + rx, crate::api::EventHub::default(), ); - app.state.workspaces = vec![Workspace::test_new("pane-graphics")]; + app.state.workspaces = vec![Workspace::test_new("graphics")]; app.state.ensure_test_terminals(); - let pane_id = app.state.workspaces[0].tabs[0].root_pane; - let public_pane_id = app.public_pane_id(0, pane_id).unwrap(); - (app, public_pane_id) + let pane = app.state.workspaces[0].tabs[0].root_pane; + let public = app.public_pane_id(0, pane).unwrap(); + app.state.kitty_graphics_enabled = true; + (app, public) + } + + fn frame( + pane_id: String, + layer_id: Option<&str>, + owner: &str, + value: u8, + ) -> PaneGraphicsSetParams { + PaneGraphicsSetParams { + pane_id, + layer_id: layer_id.map(str::to_owned), + z_index: 2, + owner: owner.to_owned(), + format: crate::api::schema::PaneGraphicsFormat::Rgba, + image_width: 1, + image_height: 1, + data: Some(vec![value; 4]), + data_base64: String::new(), + placement: Default::default(), + } + } + + fn pane_visible(response: &str) -> bool { + serde_json::from_str::<serde_json::Value>(response).unwrap()["result"]["pane_visible"] + .as_bool() + .unwrap() } #[test] - fn api_pane_graphics_info_is_explicit_and_side_effect_free() { - let (mut app, public_pane_id) = app_with_test_workspace(); - let request = |pane_id: String| crate::api::schema::Request { - id: "graphics-info".into(), - method: crate::api::schema::Method::PaneGraphicsInfo(crate::api::schema::PaneTarget { - pane_id, - }), + fn info_reports_visibility_for_terminal_surface_workspace_tab_and_zoom() { + let (mut app, pane_id) = app(); + app.state.mode = crate::app::Mode::Terminal; + app.state.active = Some(0); + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, }; + let visible = app.handle_pane_graphics_info( + "visible".into(), + crate::api::schema::PaneTarget { + pane_id: pane_id.clone(), + }, + ); + assert!(pane_visible(&visible)); - let response = app.handle_api_request(request(public_pane_id.clone())); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); + let hidden_workspace = Workspace::test_new("hidden"); + let hidden_workspace_pane = hidden_workspace.tabs[0].root_pane; + app.state.workspaces.push(hidden_workspace); + let hidden_workspace_id = app.public_pane_id(1, hidden_workspace_pane).unwrap(); + let hidden = app.handle_pane_graphics_info( + "hidden-workspace".into(), + crate::api::schema::PaneTarget { + pane_id: hidden_workspace_id, + }, + ); + assert!(!pane_visible(&hidden)); - app.state.kitty_graphics_enabled = true; + let inactive_tab = app.state.workspaces[0].test_add_tab(Some("inactive")); + let inactive_pane = app.state.workspaces[0].tabs[inactive_tab].root_pane; + let inactive_id = app.public_pane_id(0, inactive_pane).unwrap(); + let hidden_tab = app.handle_pane_graphics_info( + "hidden-tab".into(), + crate::api::schema::PaneTarget { + pane_id: inactive_id, + }, + ); + assert!(!pane_visible(&hidden_tab)); + + app.state.workspaces[0].test_split(ratatui::layout::Direction::Horizontal); + app.state.workspaces[0].tabs[0].zoomed = true; + let zoomed_away = app.handle_pane_graphics_info( + "zoomed-away".into(), + crate::api::schema::PaneTarget { + pane_id: pane_id.clone(), + }, + ); + assert!(!pane_visible(&zoomed_away)); + + app.state.workspaces[0].tabs[0].zoomed = false; + app.state.mode = crate::app::Mode::Navigate; + let short_lived_mode = app.handle_pane_graphics_info( + "navigate".into(), + crate::api::schema::PaneTarget { pane_id }, + ); + assert!(pane_visible(&short_lived_mode)); + } + + #[test] + fn monolithic_info_keeps_fast_transport_and_exact_pixels_disabled() { + let (mut app, pane_id) = app(); app.state.host_cell_size = crate::kitty_graphics::HostCellSize { - width_px: 11, - height_px: 22, + width_px: 10, + height_px: 20, }; - let response = app.handle_api_request(request(public_pane_id)); + assert!(!app.pane_graphics_files.is_initialized()); + let response = app + .handle_pane_graphics_info("info".into(), crate::api::schema::PaneTarget { pane_id }); let success: SuccessResponse = serde_json::from_str(&response).unwrap(); assert!(matches!( success.result, ResponseResult::PaneGraphicsInfo { - cell_width_px: 11, - cell_height_px: 22, + file_frame_damage: true, + max_layers_per_pane: 16, + pixel_mouse: false, + .. } )); - assert!(app.state.pane_graphics_layers.is_empty()); - assert!(app.state.pane_graphics_streams.is_empty()); + assert!(app.pane_graphics.slots.is_empty()); + assert!(!app.pane_graphics_files.is_initialized()); } + #[cfg(unix)] #[test] - fn api_pane_graphics_set_and_clear_updates_runtime_layers() { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: String::new(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: data, - placement: crate::api::schema::PaneGraphicsPlacementParams { - grid_cols: 10, - grid_rows: 4, - ..Default::default() - }, - }), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); + fn info_advertises_rgba_direct_and_bgra_fallback_file_formats_on_demand() { + let (mut app, pane_id) = app(); + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + app.direct_graphics_available = true; + let response = app + .handle_pane_graphics_info("info".into(), crate::api::schema::PaneTarget { pane_id }); + let value: serde_json::Value = serde_json::from_str(&response).unwrap(); + assert!(app.pane_graphics_files.is_initialized()); + assert_eq!( + value["result"]["file_frame_formats"], + serde_json::json!(["rgba", "bgra"]) + ); + assert_eq!(value["result"]["file_frame_damage"], true); + assert_eq!(value["result"]["file_frame_transport"], "direct-kitty"); + } - let layer = app - .state - .pane_graphics_layers - .get(&pane_id) - .expect("graphics layer stored"); - assert_eq!(layer.image_width, 1); - assert_eq!(layer.image_height, 1); - assert_eq!(layer.render.grid_cols, 10); - assert_eq!(layer.render.grid_rows, 4); - assert!(!app.state.session_dirty); + #[test] + fn layers_are_isolated_and_stream_owned() { + let (mut app, pane_id) = app(); + for (layer, z, value) in [(None, 0, 1), (Some("chrome"), 10, 2)] { + let mut request = frame(pane_id.clone(), layer, "", value); + request.z_index = z; + let response = app.handle_pane_graphics_set(format!("set-{value}"), request); + assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok()); + } + let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); + assert_eq!( + app.pane_graphics + .slots + .get(&(pane, PANE_GRAPHICS_PRIMARY_LAYER_ID.into())) + .and_then(|slot| slot.layer.as_ref()) + .unwrap() + .z_index, + 0 + ); + assert_eq!( + app.pane_graphics + .slots + .get(&(pane, "chrome".into())) + .and_then(|slot| slot.layer.as_ref()) + .unwrap() + .z_index, + 10 + ); + let response = app.handle_pane_graphics_stream_open( + "open".into(), + PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some("chrome".into()), + z_index: 10, + owner: "stream".into(), + }, + ); + assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok()); + let response = + app.handle_pane_graphics_set("conflict".into(), frame(pane_id, Some("chrome"), "", 3)); + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "stream_conflict"); + } - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-clear".into(), - method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams { - pane_id: public_pane_id, - }), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + fn error_code(response: &str) -> String { + serde_json::from_str::<ErrorResponse>(response) + .unwrap() + .error + .code } #[test] - fn api_pane_graphics_rejects_when_kitty_graphics_disabled() { - let (mut app, public_pane_id) = app_with_test_workspace(); - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: String::new(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: data, - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-clear".into(), - method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams { - pane_id: public_pane_id, - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + fn validates_layer_and_frame_contracts() { + let cases = [ + (Some("bad space"), 1, 1, vec![1; 4], "invalid_layer_id"), + (None, 0, 1, vec![1; 4], "invalid_image"), + (None, 1, 1, Vec::new(), "invalid_image"), + (None, 1, 1, vec![1; 3], "invalid_image"), + ]; + for (layer, width, height, data, expected) in cases { + let (mut app, pane_id) = app(); + let mut params = frame(pane_id, layer, "", 1); + params.image_width = width; + params.image_height = height; + params.data = Some(data); + let response = app.handle_pane_graphics_set("invalid".into(), params); + assert_eq!(error_code(&response), expected); + assert!(app.pane_graphics.slots.is_empty()); + } } #[test] - fn api_pane_graphics_stream_claim_uses_resolved_pane_identity() { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let alias = format!("{public_pane_id}:alias"); - app.state - .public_pane_id_aliases - .insert(alias.clone(), pane_id); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-open".into(), - method: crate::api::schema::Method::PaneGraphicsStreamOpen( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: public_pane_id, - owner: "stream-1".into(), - }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); + fn disabled_requests_have_no_graphics_side_effects() { + let (mut app, pane_id) = app(); + app.state.kitty_graphics_enabled = false; + let revision = app.pane_graphics.revision(); + + let response = app.handle_pane_graphics_set("disabled".into(), frame(pane_id, None, "", 1)); + + assert_eq!(error_code(&response), "feature_disabled"); + assert!(app.pane_graphics.slots.is_empty()); + assert_eq!(app.pane_graphics.revision(), revision); + } + + #[test] + fn stream_owner_controls_only_its_named_layer() { + let (mut app, pane_id) = app(); + let params = |owner: &str| PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some("chrome".into()), + z_index: 7, + owner: owner.into(), + }; + assert!(serde_json::from_str::<SuccessResponse>( + &app.handle_pane_graphics_stream_open("open".into(), params("owner")) + ) + .is_ok()); assert_eq!( - app.state.pane_graphics_streams.get(&pane_id), - Some(&"stream-1".to_string()) + error_code(&app.handle_pane_graphics_stream_set( + "wrong".into(), + frame(pane_id.clone(), Some("chrome"), "other", 2), + )), + "stream_conflict" + ); + assert!( + serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_stream_set( + "frame".into(), + frame(pane_id.clone(), Some("chrome"), "owner", 3), + )) + .is_ok() ); + app.handle_pane_graphics_stream_close("stale".into(), params("other")); + assert_eq!(app.pane_graphics.slots.len(), 1); + app.handle_pane_graphics_stream_close("close".into(), params("owner")); + assert!(app.pane_graphics.slots.is_empty()); + } + + #[test] + fn static_set_and_clear_reclaim_inactive_stream_layers() { + let (mut app, pane_id) = app(); + let open = |layer: &str| PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some(layer.into()), + z_index: 0, + owner: format!("owner-{layer}"), + }; + for layer in ["set", "clear"] { + assert!(serde_json::from_str::<SuccessResponse>( + &app.handle_pane_graphics_stream_open(layer.into(), open(layer)) + ) + .is_ok()); + let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); + app.pane_graphics.slots[&(pane, layer.into())] + .stream_active + .as_ref() + .unwrap() + .store(false, std::sync::atomic::Ordering::Release); + } - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-open-alias".into(), - method: crate::api::schema::Method::PaneGraphicsStreamOpen( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: alias.clone(), - owner: "stream-2".into(), + assert!( + serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_set( + "set-static".into(), + frame(pane_id.clone(), Some("set"), "", 4), + )) + .is_ok() + ); + assert!( + serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_clear( + "clear-static".into(), + PaneGraphicsClearParams { + pane_id, + layer_id: Some("clear".into()), }, - ), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "stream_conflict"); - assert_eq!( - app.state.pane_graphics_streams.get(&pane_id), - Some(&"stream-1".to_string()) + )) + .is_ok() ); + assert_eq!(app.pane_graphics.slots.len(), 1); + assert!(app + .pane_graphics + .slots + .values() + .all(|slot| slot.stream_owner.is_none())); + } - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-close-wrong-owner".into(), - method: crate::api::schema::Method::PaneGraphicsStreamClose( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: alias.clone(), - owner: "stream-2".into(), + #[test] + fn layer_limit_counts_empty_and_populated_stream_slots() { + let (mut app, pane_id) = app(); + for index in 0..PANE_GRAPHICS_MAX_LAYERS_PER_PANE { + let layer_id = format!("layer-{index}"); + let response = app.handle_pane_graphics_stream_open( + layer_id.clone(), + PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some(layer_id), + z_index: index as i32, + owner: format!("owner-{index}"), }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert_eq!( - app.state.pane_graphics_streams.get(&pane_id), - Some(&"stream-1".to_string()) + ); + assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok()); + } + let response = app.handle_pane_graphics_set( + "set-overflow".into(), + frame(pane_id.clone(), Some("set-overflow"), "", 1), + ); + assert_eq!(error_code(&response), "layer_limit"); + let response = app.handle_pane_graphics_stream_open( + "overflow".into(), + PaneGraphicsStreamParams { + pane_id, + layer_id: Some("overflow".into()), + z_index: 0, + owner: "overflow".into(), + }, ); + assert_eq!(error_code(&response), "layer_limit"); + assert_eq!(app.pane_graphics.slots.len(), 16); + } + + #[test] + fn set_and_clear_preserve_runtime_only_layer_details() { + let (mut app, pane_id) = app(); + let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); + let mut params = frame(pane_id.clone(), None, "", 9); + params.placement.grid_cols = 10; + params.placement.grid_rows = 4; + assert!(serde_json::from_str::<SuccessResponse>( + &app.handle_pane_graphics_set("set".into(), params) + ) + .is_ok()); + let layer = app + .pane_graphics + .slots + .get(&(pane, PANE_GRAPHICS_PRIMARY_LAYER_ID.into())) + .and_then(|slot| slot.layer.as_ref()) + .unwrap(); + assert_eq!(layer.inline_data(), Some([9; 4].as_slice())); + assert_eq!((layer.render.grid_cols, layer.render.grid_rows), (10, 4)); + assert!(!app.state.session_dirty); - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-close-alias".into(), - method: crate::api::schema::Method::PaneGraphicsStreamClose( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: alias, - owner: "stream-1".into(), + assert!( + serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_clear( + "clear".into(), + PaneGraphicsClearParams { + pane_id, + layer_id: None, }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(!app.state.pane_graphics_streams.contains_key(&pane_id)); + )) + .is_ok() + ); + assert!(app.pane_graphics.slots.is_empty()); } #[test] - fn api_pane_graphics_stream_owns_layer_while_active() { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let public_data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-open".into(), - method: crate::api::schema::Method::PaneGraphicsStreamOpen( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: public_pane_id.clone(), - owner: "stream-1".into(), - }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set-public".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: String::new(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: public_data.clone(), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "stream_conflict"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set-stream".into(), - method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: "stream-1".into(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: public_data, - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(app.state.pane_graphics_layers.contains_key(&pane_id)); - - let wrong_owner_data = base64::engine::general_purpose::STANDARD.encode([5_u8, 6, 7, 8]); - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-set-stream-wrong-owner".into(), - method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - owner: "stream-2".into(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: None, - data_base64: wrong_owner_data, - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "stream_conflict"); + fn raw_formats_require_exact_non_overflowing_lengths() { + for (format, exact) in [ + (crate::api::schema::PaneGraphicsFormat::Rgb, 6), + (crate::api::schema::PaneGraphicsFormat::Rgba, 8), + ] { + for (length, expected) in [(exact, "ok"), (exact - 1, "invalid_image")] { + let (mut app, pane_id) = app(); + let mut params = frame(pane_id, None, "", 1); + params.format = format; + params.image_width = 2; + params.data = Some(vec![1; length]); + let response = app.handle_pane_graphics_set("length".into(), params); + if expected == "ok" { + assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok()); + } else { + assert_eq!(error_code(&response), expected); + } + } + let (mut app, pane_id) = app(); + let mut params = frame(pane_id, None, "", 1); + params.format = format; + params.image_width = u32::MAX; + params.image_height = u32::MAX; + assert_eq!( + error_code(&app.handle_pane_graphics_set("overflow".into(), params)), + "invalid_image" + ); + } + } + + #[test] + fn public_frames_validate_base64_and_public_size_limit() { + let (mut app, pane_id) = app(); + let mut params = frame(pane_id.clone(), None, "", 1); + params.format = crate::api::schema::PaneGraphicsFormat::Png; + params.data = None; + params.data_base64 = "not base64".into(); assert_eq!( - app.state.pane_graphics_layers.get(&pane_id).unwrap().data, - vec![1_u8, 2, 3, 4] + error_code(&app.handle_pane_graphics_set("base64".into(), params)), + "invalid_image" ); - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-clear-public".into(), - method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams { - pane_id: public_pane_id.clone(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "stream_conflict"); - assert!(app.state.pane_graphics_layers.contains_key(&pane_id)); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-close".into(), - method: crate::api::schema::Method::PaneGraphicsStreamClose( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: public_pane_id, - owner: "stream-1".into(), - }, - ), - }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + let mut params = frame(pane_id, None, "", 1); + params.format = crate::api::schema::PaneGraphicsFormat::Png; + params.data = None; + params.data_base64 = + base64::engine::general_purpose::STANDARD + .encode(vec![1; PANE_GRAPHICS_SET_MAX_BYTES + 1]); + assert_eq!( + error_code(&app.handle_pane_graphics_set("large".into(), params)), + "image_too_large" + ); } #[test] - fn api_pane_graphics_rejects_overflowing_raw_dimensions() { - for format in [ - crate::api::schema::PaneGraphicsFormat::Rgb, - crate::api::schema::PaneGraphicsFormat::Rgba, - ] { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-overflow".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id, - owner: String::new(), - format, - image_width: u32::MAX, - image_height: u32::MAX, - data: None, - data_base64: base64::engine::general_purpose::STANDARD.encode([1_u8]), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); + fn aliases_resolve_to_the_same_layer_identity() { + let (mut app, pane_id) = app(); + let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); + let alias = format!("{pane_id}:alias"); + app.state.public_pane_id_aliases.insert(alias.clone(), pane); + let params = |target: String, owner: &str| PaneGraphicsStreamParams { + pane_id: target, + layer_id: None, + z_index: 0, + owner: owner.into(), + }; + assert!(serde_json::from_str::<SuccessResponse>( + &app.handle_pane_graphics_stream_open("open".into(), params(pane_id.clone(), "owner"),) + ) + .is_ok()); + assert_eq!( + error_code(&app.handle_pane_graphics_stream_open( + "conflict".into(), + params(alias.clone(), "other"), + )), + "stream_conflict" + ); + app.handle_pane_graphics_stream_close("stale".into(), params(alias.clone(), "other")); + assert_eq!(app.pane_graphics.slots.len(), 1); + app.handle_pane_graphics_stream_close("close".into(), params(alias, "owner")); + assert!(app.pane_graphics.slots.is_empty()); + } + + #[cfg(unix)] + fn direct_file(app: &App, name: &str, data: &[u8]) -> String { + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + let path = app + .pane_graphics_files + .source_directory() + .unwrap() + .join(name); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path) + .unwrap(); + file.write_all(data).unwrap(); + path.to_string_lossy().into_owned() + } - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "invalid_image"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + #[cfg(unix)] + fn direct_params( + pane_id: String, + owner: &str, + path: String, + ) -> crate::api::schema::PaneGraphicsDirectParams { + crate::api::schema::PaneGraphicsDirectParams { + pane_id, + layer_id: None, + z_index: 0, + owner: owner.into(), + image_width: 1, + image_height: 1, + format: crate::api::schema::PaneGraphicsFormat::Rgba, + path, + sequence: 1, + revision: 1, + placement: Default::default(), } } + #[cfg(unix)] #[test] - fn api_pane_graphics_validates_exact_raw_lengths() { - for (format, exact_len) in [ - (crate::api::schema::PaneGraphicsFormat::Rgb, 6_usize), - (crate::api::schema::PaneGraphicsFormat::Rgba, 8_usize), + fn direct_file_is_leased_or_safely_copied_for_inline_fallback() { + let (mut app, pane_id) = app(); + let stream = PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: None, + z_index: 0, + owner: "owner".into(), + }; + app.handle_pane_graphics_stream_open("open".into(), stream.clone()); + for (width, height) in [(0, 1), (1, 0)] { + let mut params = direct_params(pane_id.clone(), "owner", "unused".into()); + (params.image_width, params.image_height) = (width, height); + assert_eq!( + error_code(&app.handle_pane_graphics_stream_direct("invalid".into(), params)), + "invalid_image" + ); + } + for format in [ + crate::api::schema::PaneGraphicsFormat::Rgb, + crate::api::schema::PaneGraphicsFormat::Png, ] { - for data_len in [exact_len, exact_len - 1] { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-raw-length".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id, - owner: String::new(), - format, - image_width: 2, - image_height: 1, - data: None, - data_base64: base64::engine::general_purpose::STANDARD - .encode(vec![1_u8; data_len]), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - - if data_len == exact_len { - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!(success.result, ResponseResult::Ok {})); - assert!(app.state.pane_graphics_layers.contains_key(&pane_id)); - } else { - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "invalid_image"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); - } - } + let mut params = direct_params(pane_id.clone(), "owner", "unused".into()); + params.format = format; + assert_eq!( + error_code(&app.handle_pane_graphics_stream_direct("invalid".into(), params)), + "invalid_image" + ); } + app.direct_graphics_available = true; + let path = direct_file(&app, "direct", &[1, 2, 3, 4]); + let response = app.handle_pane_graphics_stream_direct( + "direct".into(), + crate::api::schema::PaneGraphicsDirectParams { + sequence: 7, + revision: 8, + ..direct_params(pane_id.clone(), "owner", path) + }, + ); + let ack: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!( + ack.result, + ResponseResult::PaneGraphicsFrameAck { + sequence: 7, + revision: 8 + } + )); + assert!(app + .pane_graphics + .slots + .values() + .next() + .unwrap() + .layer + .as_ref() + .unwrap() + .direct_lease() + .is_some()); + + app.handle_pane_graphics_stream_close("close".into(), stream.clone()); + app.handle_pane_graphics_stream_open("reopen".into(), stream); + app.direct_graphics_available = false; + let path = direct_file(&app, "fallback", &[5, 6, 7, 8]); + app.handle_pane_graphics_stream_direct( + "fallback".into(), + crate::api::schema::PaneGraphicsDirectParams { + sequence: 9, + revision: 10, + ..direct_params(pane_id, "owner", path) + }, + ); + assert_eq!( + app.pane_graphics + .slots + .values() + .next() + .unwrap() + .layer + .as_ref() + .unwrap() + .inline_data(), + Some([5, 6, 7, 8].as_slice()) + ); } + #[cfg(unix)] #[test] - fn api_pane_graphics_uses_transport_specific_limits() { - let (mut app, public_pane_id) = app_with_test_workspace(); - app.state.kitty_graphics_enabled = true; - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let response = app.handle_api_request(crate::api::schema::Request { - id: "graphics-public-too-large".into(), - method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { - pane_id: public_pane_id, - owner: String::new(), - format: crate::api::schema::PaneGraphicsFormat::Png, - image_width: 1, - image_height: 1, - data: None, - data_base64: base64::engine::general_purpose::STANDARD.encode(vec![ - 1_u8; - PANE_GRAPHICS_SET_MAX_BYTES - + 1 - ]), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "image_too_large"); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + fn bgra_and_secondary_file_frames_are_canonical_owned_rgba() { + let (mut app, pane_id) = app(); + app.direct_graphics_available = true; + for (layer, format, source) in [ + ( + "browser-chrome", + crate::api::schema::PaneGraphicsFormat::Rgba, + [1, 2, 3, 4], + ), + ( + "primary", + crate::api::schema::PaneGraphicsFormat::Bgra, + [3, 2, 1, 4], + ), + ] { + app.handle_pane_graphics_stream_open( + format!("open-{layer}"), + PaneGraphicsStreamParams { + pane_id: pane_id.clone(), + layer_id: Some(layer.into()), + z_index: 0, + owner: layer.into(), + }, + ); + let path = direct_file(&app, layer, &source); + app.handle_pane_graphics_stream_direct( + format!("frame-{layer}"), + crate::api::schema::PaneGraphicsDirectParams { + layer_id: Some(layer.into()), + format, + ..direct_params(pane_id.clone(), layer, path) + }, + ); + let key = (app.parse_pane_id(&pane_id).unwrap().1, layer.into()); + assert_eq!( + app.pane_graphics.slots[&key] + .layer + .as_ref() + .unwrap() + .inline_data(), + Some([1, 2, 3, 4].as_slice()) + ); + } } #[test] - fn api_pane_graphics_stream_methods_require_kitty_graphics() { - let (mut app, public_pane_id) = app_with_test_workspace(); - let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); - let open = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-open".into(), - method: crate::api::schema::Method::PaneGraphicsStreamOpen( - crate::api::schema::PaneGraphicsStreamParams { - pane_id: public_pane_id.clone(), - owner: "stream-1".into(), + fn disabled_info_and_stream_requests_are_rejected() { + let (mut app, pane_id) = app(); + app.state.kitty_graphics_enabled = false; + assert_eq!( + error_code(&app.handle_pane_graphics_info( + "info".into(), + crate::api::schema::PaneTarget { + pane_id: pane_id.clone(), }, - ), - }); - let error: ErrorResponse = serde_json::from_str(&open).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); - - let set = app.handle_api_request(crate::api::schema::Request { - id: "graphics-stream-set".into(), - method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { - pane_id: public_pane_id, - owner: "stream-1".into(), - format: crate::api::schema::PaneGraphicsFormat::Png, - image_width: 1, - image_height: 1, - data: Some(vec![1_u8]), - data_base64: String::new(), - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }); - let error: ErrorResponse = serde_json::from_str(&set).unwrap(); - assert_eq!(error.error.code, "feature_disabled"); - assert!(!app.state.pane_graphics_streams.contains_key(&pane_id)); - assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + )), + "feature_disabled" + ); + assert_eq!( + error_code(&app.handle_pane_graphics_stream_open( + "open".into(), + PaneGraphicsStreamParams { + pane_id, + layer_id: None, + z_index: 0, + owner: "owner".into(), + }, + )), + "feature_disabled" + ); + assert!(app.pane_graphics.slots.is_empty()); } } diff --git a/src/app/api/plugins/mod.rs b/src/app/api/plugins/mod.rs index a8cb4405..ddf3fba6 100644 --- a/src/app/api/plugins/mod.rs +++ b/src/app/api/plugins/mod.rs @@ -705,8 +705,10 @@ fn manifest_actions( #[cfg(test)] mod tests { use super::*; + #[cfg(unix)] + use crate::api::schema::PaneListParams; use crate::api::schema::{ - Method, PaneListParams, PluginSourceInfo, PluginSourceKind, Request, SuccessResponse, + Method, PluginSourceInfo, PluginSourceKind, Request, SuccessResponse, }; use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/src/app/ids.rs b/src/app/ids.rs index 4bebedc6..c2fc0a4d 100644 --- a/src/app/ids.rs +++ b/src/app/ids.rs @@ -103,7 +103,7 @@ impl App { None } - pub(super) fn parse_pane_id(&self, id: &str) -> Option<(usize, crate::layout::PaneId)> { + pub(crate) fn parse_pane_id(&self, id: &str) -> Option<(usize, crate::layout::PaneId)> { if let Some(alias) = self.state.public_pane_id_aliases.get(id).copied() { return self.find_pane(alias).map(|(ws_idx, _)| (ws_idx, alias)); } diff --git a/src/app/input/mod.rs b/src/app/input/mod.rs index f77821c9..3202ae26 100644 --- a/src/app/input/mod.rs +++ b/src/app/input/mod.rs @@ -493,15 +493,17 @@ impl App { self.close_popup_pane(); return; }; - let column = mouse.column.saturating_sub(inner.x); - let row = mouse.row.saturating_sub(inner.y); + let position = crate::input::mouse::Position::Cell { + column: mouse.column.saturating_sub(inner.x), + row: mouse.row.saturating_sub(inner.y), + }; let bytes = match mouse.kind { MouseEventKind::ScrollUp | MouseEventKind::ScrollDown | MouseEventKind::ScrollLeft | MouseEventKind::ScrollRight => match rt.wheel_routing() { Some(crate::pane::WheelRouting::MouseReport) => { - rt.encode_mouse_wheel(mouse.kind, column, row, mouse.modifiers) + rt.encode_mouse_wheel(mouse.kind, position, mouse.modifiers) } Some(crate::pane::WheelRouting::AlternateScroll) => { rt.encode_alternate_scroll(mouse.kind) @@ -517,11 +519,9 @@ impl App { } }, MouseEventKind::Down(_) | MouseEventKind::Up(_) | MouseEventKind::Drag(_) => { - rt.encode_mouse_button(mouse.kind, column, row, mouse.modifiers) - } - MouseEventKind::Moved => { - rt.encode_mouse_motion(mouse.kind, column, row, mouse.modifiers) + rt.encode_mouse_button(mouse.kind, position, mouse.modifiers) } + MouseEventKind::Moved => rt.encode_mouse_motion(mouse.kind, position, mouse.modifiers), }; let Some(bytes) = bytes else { return; diff --git a/src/app/input/mouse.rs b/src/app/input/mouse.rs index 2555d95d..8d8988f9 100644 --- a/src/app/input/mouse.rs +++ b/src/app/input/mouse.rs @@ -1667,6 +1667,33 @@ impl AppState { } } + fn pane_mouse_position( + &self, + runtime: &crate::terminal::TerminalRuntime, + inner: Rect, + mouse: MouseEvent, + ) -> Option<crate::input::mouse::Position> { + let column = mouse.column.saturating_sub(inner.x); + let row = mouse.row.saturating_sub(inner.y); + let cell = crate::input::mouse::Position::Cell { column, row }; + let Some(host) = self.host_mouse_pixels else { + return Some(cell); + }; + let wants_pixels = runtime.input_state().is_some_and(|state| { + state.mouse_protocol_encoding == crate::input::MouseProtocolEncoding::SgrPixels + }); + if !wants_pixels { + return Some(cell); + } + let Some((width_px, height_px)) = runtime.pixel_size() else { + return Some(cell); + }; + Some( + host.pane_position(inner, width_px, height_px) + .unwrap_or(cell), + ) + } + pub(super) fn forward_pane_mouse_button( &self, terminal_runtimes: &TerminalRuntimeRegistry, @@ -1680,9 +1707,10 @@ impl AppState { else { return false; }; - let column = mouse.column.saturating_sub(info.inner_rect.x); - let row = mouse.row.saturating_sub(info.inner_rect.y); - let Some(bytes) = rt.encode_mouse_button(mouse.kind, column, row, mouse.modifiers) else { + let Some(position) = self.pane_mouse_position(rt, info.inner_rect, mouse) else { + return false; + }; + let Some(bytes) = rt.encode_mouse_button(mouse.kind, position, mouse.modifiers) else { return false; }; rt.scroll_reset(); @@ -1705,9 +1733,10 @@ impl AppState { else { return false; }; - let column = mouse.column.saturating_sub(info.inner_rect.x); - let row = mouse.row.saturating_sub(info.inner_rect.y); - let Some(bytes) = rt.encode_mouse_motion(mouse.kind, column, row, mouse.modifiers) else { + let Some(position) = self.pane_mouse_position(rt, info.inner_rect, mouse) else { + return false; + }; + let Some(bytes) = rt.encode_mouse_motion(mouse.kind, position, mouse.modifiers) else { return false; }; if let Err(err) = rt.try_send_bytes(Bytes::from(bytes)) { @@ -1733,9 +1762,10 @@ impl AppState { return false; } rt.scroll_reset(); - let column = mouse.column.saturating_sub(info.inner_rect.x); - let row = mouse.row.saturating_sub(info.inner_rect.y); - let Some(bytes) = rt.encode_mouse_wheel(mouse.kind, column, row, mouse.modifiers) else { + let Some(position) = self.pane_mouse_position(rt, info.inner_rect, mouse) else { + return false; + }; + let Some(bytes) = rt.encode_mouse_wheel(mouse.kind, position, mouse.modifiers) else { warn!(pane = info.id.raw(), kind = ?mouse.kind, "failed to encode mouse wheel event"); return true; }; @@ -1761,18 +1791,7 @@ impl AppState { match rt.wheel_routing() { Some(crate::pane::WheelRouting::HostScroll) | None => false, Some(crate::pane::WheelRouting::MouseReport) => { - rt.scroll_reset(); - let column = mouse.column.saturating_sub(info.inner_rect.x); - let row = mouse.row.saturating_sub(info.inner_rect.y); - let Some(bytes) = rt.encode_mouse_wheel(mouse.kind, column, row, mouse.modifiers) - else { - warn!(pane = info.id.raw(), kind = ?mouse.kind, "failed to encode mouse wheel event"); - return true; - }; - if let Err(err) = rt.try_send_bytes(Bytes::from(bytes)) { - warn!(pane = info.id.raw(), err = %err, "failed to forward mouse wheel event"); - } - true + self.forward_pane_reported_wheel(terminal_runtimes, info, mouse) } Some(crate::pane::WheelRouting::AlternateScroll) => { rt.scroll_reset(); @@ -2146,7 +2165,7 @@ mod tests { } #[tokio::test] - async fn configured_right_click_passthrough_forwards_full_gesture_to_pane() { + async fn configured_right_click_passthrough_forwards_gesture_outside_pane() { let mut app = app_for_mouse_test(); let mut ws = Workspace::test_new("test"); let pane_id = ws.tabs[0].root_pane; @@ -2177,11 +2196,11 @@ mod tests { }); app.handle_mouse(MouseEvent { modifiers: KeyModifiers::CONTROL, - ..mouse(MouseEventKind::Drag(MouseButton::Right), col + 1, row + 1) + ..mouse(MouseEventKind::Drag(MouseButton::Right), 0, 0) }); app.handle_mouse(MouseEvent { modifiers: KeyModifiers::CONTROL, - ..mouse(MouseEventKind::Up(MouseButton::Right), col + 1, row + 1) + ..mouse(MouseEventKind::Up(MouseButton::Right), 0, 0) }); assert_eq!(app.state.mode, Mode::Terminal); @@ -2193,11 +2212,11 @@ mod tests { ); assert_eq!( input_rx.try_recv().expect("forwarded right mouse drag"), - Bytes::from_static(b"\x1b[<34;4;5M") + Bytes::from_static(b"\x1b[<34;1;1M") ); assert_eq!( input_rx.try_recv().expect("forwarded right mouse up"), - Bytes::from_static(b"\x1b[<2;4;5m") + Bytes::from_static(b"\x1b[<2;1;1m") ); assert!(input_rx.try_recv().is_err()); } @@ -2321,7 +2340,7 @@ mod tests { } #[tokio::test] - async fn mouse_dispatcher_downgrades_sgr_pixel_motion_to_cell_coordinates() { + async fn ordinary_cell_mouse_downgrades_pixel_mode_to_cell_coordinates() { let mut app = app_for_mouse_test(); let mut ws = Workspace::test_new("test"); let pane_id = ws.tabs[0].root_pane; @@ -2339,8 +2358,16 @@ mod tests { app.state.active = Some(0); app.state.selected = 0; app.state.mode = Mode::Terminal; + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; crate::ui::compute_view(&mut app.state, Rect::new(0, 0, 106, 20)); let info = app.state.view.pane_infos[0].clone(); + app.state + .runtime_for_pane_in_workspace(&app.terminal_runtimes, 0, pane_id) + .unwrap() + .resize(info.inner_rect.height, info.inner_rect.width, 10, 20); assert!(info.inner_rect.x > 0, "sidebar offset should be present"); assert!(info.inner_rect.y > 0, "tab bar offset should be present"); @@ -2358,6 +2385,77 @@ mod tests { } #[tokio::test] + async fn dedicated_client_pixel_mouse_preserves_subcell_position() { + let mut app = app_for_mouse_test(); + let mut ws = Workspace::test_new("test"); + let pane_id = ws.tabs[0].root_pane; + let (runtime, mut input_rx) = + crate::terminal::TerminalRuntime::test_with_channel_and_scrollback_bytes( + 80, + 18, + 0, + b"\x1b[?1003h\x1b[?1006h\x1b[?1016h", + 4, + ); + ws.insert_test_runtime(pane_id, runtime); + app.state.workspaces = vec![ws]; + app.state.active = Some(0); + app.state.selected = 0; + app.state.mode = Mode::Terminal; + app.state.mouse_capture = false; + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + crate::ui::compute_view(&mut app.state, Rect::new(0, 0, 106, 20)); + let inner = app.state.view.pane_infos[0].inner_rect; + let geometry = crate::input::mouse::HostGeometry::new(106, 20, 1_060, 400).unwrap(); + let x = u32::from(inner.x + 2) * 10 + 8; + let y = u32::from(inner.y + 3) * 20 + 9; + let report = format!("\x1b[<35;{x};{y}M"); + app.state.host_mouse_pixels = Some(crate::input::mouse::HostPixels { x, y, geometry }); + let runtime = app + .state + .runtime_for_pane_in_workspace(&app.terminal_runtimes, 0, pane_id) + .unwrap(); + assert_eq!(runtime.pixel_size(), None); + assert_eq!( + app.state.pane_mouse_position( + runtime, + inner, + mouse(MouseEventKind::Moved, inner.x + 2, inner.y + 3), + ), + Some(crate::input::mouse::Position::Cell { column: 2, row: 3 }) + ); + runtime.resize(inner.height, inner.width, 10, 20); + let runtime = app + .state + .runtime_for_pane_in_workspace(&app.terminal_runtimes, 0, pane_id) + .unwrap(); + assert_eq!( + runtime.pixel_size(), + Some((u32::from(inner.width) * 10, u32::from(inner.height) * 20)) + ); + assert_eq!( + app.state.pane_mouse_position( + runtime, + inner, + mouse(MouseEventKind::Moved, inner.x + 2, inner.y + 3), + ), + Some(crate::input::mouse::Position::Pixels { x: 28, y: 69 }) + ); + app.state.host_mouse_pixels = None; + + assert!(app.route_client_pixel_mouse(7, report.as_bytes(), geometry)); + assert_eq!( + input_rx.try_recv().expect("forwarded exact mouse motion"), + Bytes::from_static(b"\x1b[<35;28;69M") + ); + assert!(input_rx.try_recv().is_err()); + assert!(app.state.host_mouse_pixels.is_none()); + } + + #[tokio::test] async fn mouse_dispatcher_does_not_forward_motion_behind_herdr_modes() { let mut app = app_for_mouse_test(); let mut ws = Workspace::test_new("test"); diff --git a/src/app/mod.rs b/src/app/mod.rs index 70d5fe93..15f352af 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -16,6 +16,7 @@ mod creation; mod git_refresh; mod ids; mod input; +pub(crate) mod pane_graphics; mod popup; mod runtime; mod runtime_mutations; @@ -95,6 +96,10 @@ impl PaneClickState { pub struct App { pub state: AppState, + pub(crate) pane_graphics: pane_graphics::Runtime, + pub(crate) pane_graphics_files: Arc<crate::pane_graphics_files::FileStore>, + pub(crate) direct_graphics_available: bool, + pub(crate) pixel_mouse_available: bool, pub(crate) terminal_runtimes: crate::terminal::TerminalRuntimeRegistry, pub event_tx: mpsc::Sender<AppEvent>, pub(crate) event_rx: mpsc::Receiver<AppEvent>, @@ -672,9 +677,6 @@ impl App { integration_install_messages: Vec::new(), installed_plugins: load_plugin_registry(no_session), plugin_panes: std::collections::HashMap::new(), - pane_graphics_layers: std::collections::HashMap::new(), - pane_graphics_streams: std::collections::HashMap::new(), - pane_graphics_revision: 0, popup_pane: None, plugin_command_logs: Vec::new(), next_plugin_command_log_id: 1, @@ -682,6 +684,7 @@ impl App { global_menu: state::MenuListState::new(0), host_terminal_theme: crate::terminal_theme::TerminalTheme::default(), host_cell_size: crate::kitty_graphics::HostCellSize::default(), + host_mouse_pixels: None, session_dirty: false, terminal_runtime_shutdowns: Vec::new(), }; @@ -726,6 +729,10 @@ impl App { copy_feedback_deadline: None, last_api_notification_at: None, state, + pane_graphics: pane_graphics::Runtime::default(), + pane_graphics_files: Arc::new(crate::pane_graphics_files::FileStore::default()), + direct_graphics_available: false, + pixel_mouse_available: false, terminal_runtimes: restored_terminal_runtimes, event_tx, event_rx, @@ -1033,6 +1040,10 @@ impl App { needs_render = true; } + if self.pane_graphics.retain_live_panes(&self.state) { + needs_render = true; + } + let now = Instant::now(); self.sync_host_mouse_capture(&mut host_mouse_capture_active)?; self.sync_host_keyboard_report_all(&mut host_keyboard_report_all_active)?; @@ -1048,18 +1059,19 @@ impl App { terminal.swap_buffers(); self.full_redraw_pending = false; } - let mut cell_size = crate::kitty_graphics::HostCellSize::default(); + let mut cell_size = self.state.host_cell_size; terminal.draw(|frame| { let area = frame.area(); if kitty_graphics_enabled { - let observed_cell_size = - crate::kitty_graphics::HostCellSize::try_from_terminal(area); - if let Some(observed_cell_size) = observed_cell_size { + if let Some(observed_cell_size) = + crate::kitty_graphics::HostCellSize::try_from_terminal(area) + { self.state.host_cell_size = observed_cell_size; + cell_size = observed_cell_size; + } else if !cell_size.is_known() { + cell_size = + crate::kitty_graphics::HostCellSize::fallback_for_area(area); } - cell_size = observed_cell_size.unwrap_or_else(|| { - crate::kitty_graphics::HostCellSize::fallback_for_area(area) - }); crate::ui::compute_view_with_cell_size( &mut self.state, &self.terminal_runtimes, @@ -1082,6 +1094,7 @@ impl App { if kitty_graphics_enabled { crate::kitty_graphics::paint_local_pane_graphics( &self.state, + &self.pane_graphics, &self.terminal_runtimes, cell_size, )?; @@ -1468,8 +1481,7 @@ impl App { crate::kitty_graphics::set_enabled(config.experimental.kitty_graphics); if was_kitty_graphics_enabled && !config.experimental.kitty_graphics { let _ = crate::kitty_graphics::clear_all_host_graphics(); - self.state.pane_graphics_layers.clear(); - self.state.pane_graphics_streams.clear(); + self.pane_graphics.clear(); self.state.host_cell_size = crate::kitty_graphics::HostCellSize::default(); } self.state.reveal_hidden_cursor_for_cjk_ime = @@ -1663,6 +1675,31 @@ impl App { self.route_client_events(events, true); } + pub(crate) fn route_client_pixel_mouse( + &mut self, + source_id: InputSourceId, + data: &[u8], + geometry: crate::input::mouse::HostGeometry, + ) -> bool { + let Some((x, y)) = crate::input::mouse::parse_report(data) else { + return false; + }; + let Some((column, row)) = geometry.cell(x, y) else { + return false; + }; + let Some(cell_report) = crate::input::mouse::report_at_cell(data, column, row) else { + return false; + }; + let mut events = crate::raw_input::parse_raw_input_bytes_sync(&cell_report); + if events.len() != 1 || !matches!(events[0], crate::raw_input::RawInputEvent::Mouse(_)) { + return false; + } + self.state.host_mouse_pixels = Some(crate::input::mouse::HostPixels { x, y, geometry }); + self.route_client_events_from(source_id, std::mem::take(&mut events), false); + self.state.host_mouse_pixels = None; + true + } + pub(crate) fn route_client_events( &mut self, events: Vec<crate::raw_input::RawInputEvent>, diff --git a/src/app/pane_graphics.rs b/src/app/pane_graphics.rs new file mode 100644 index 00000000..c1d4841a --- /dev/null +++ b/src/app/pane_graphics.rs @@ -0,0 +1,347 @@ +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; + +use crate::api::schema::{PaneGraphicsFormat, PaneGraphicsPlacementParams}; +use crate::layout::PaneId; + +pub(crate) type Key = (PaneId, String); + +pub(crate) const DIRECT_DELIVERY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +pub(crate) const DIRECT_RESPONSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3); +pub(crate) const DIRECT_OUTER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(9); + +#[derive(Debug)] +pub(crate) enum Backing { + Inline(Vec<u8>), + Direct(crate::pane_graphics_files::Lease), + Resident { len: usize, client_id: u64 }, +} + +#[derive(Debug)] +pub(crate) struct Layer { + pub(crate) format: PaneGraphicsFormat, + pub(crate) image_width: u32, + pub(crate) image_height: u32, + pub(crate) backing: Backing, + pub(crate) data_fingerprint: u64, + pub(crate) render: PaneGraphicsPlacementParams, + pub(crate) z_index: i32, +} + +#[derive(Debug)] +pub(crate) struct DirectGate { + pub(crate) transfer_id: u64, + pub(crate) client_id: u64, + pub(crate) deadline: std::time::Instant, + pub(crate) written: bool, + pub(crate) success_response: String, + pub(crate) respond_to: std::sync::mpsc::Sender<String>, +} + +#[derive(Debug)] +pub(crate) struct Slot { + pub(crate) host_image_id: u32, + pub(crate) layer: Option<Layer>, + pub(crate) stream_owner: Option<String>, + pub(crate) stream_active: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>, + pub(crate) direct_gate: Option<DirectGate>, +} + +impl Slot { + #[cfg(test)] + pub(crate) fn test(host_image_id: u32, layer: Option<Layer>) -> Self { + Self { + host_image_id, + layer, + stream_owner: None, + stream_active: None, + direct_gate: None, + } + } + + pub(crate) fn stream_is_active(&self) -> bool { + self.stream_active + .as_ref() + .is_some_and(|active| active.load(std::sync::atomic::Ordering::Acquire)) + } +} + +impl Drop for Slot { + fn drop(&mut self) { + if let Some(active) = &self.stream_active { + active.store(false, std::sync::atomic::Ordering::Release); + } + } +} + +#[derive(Debug)] +pub(crate) struct Runtime { + pub(crate) slots: HashMap<Key, Slot>, + revision: u64, + next_host_image_id: u32, +} + +impl Default for Runtime { + fn default() -> Self { + Self { + slots: HashMap::new(), + revision: 0, + next_host_image_id: 1 << 31, + } + } +} + +impl Runtime { + pub(crate) fn revision(&self) -> u64 { + self.revision + } + + pub(crate) fn mark_changed(&mut self) { + self.revision = self.revision.wrapping_add(1); + } + + pub(crate) fn clear(&mut self) { + self.slots.clear(); + self.mark_changed(); + } + + pub(crate) fn reserve_image_id(&mut self, key: &Key) -> Option<u32> { + if let Some(id) = self.slots.get(key).map(|slot| slot.host_image_id) { + return Some(id); + } + for _ in 0..=self.slots.len() { + let id = self.next_host_image_id | (1 << 31); + self.next_host_image_id = id.wrapping_add(1) | (1 << 31); + if self.slots.values().all(|slot| slot.host_image_id != id) { + return Some(id); + } + } + None + } + + pub(crate) fn layer_count(&self, pane_id: PaneId) -> usize { + self.slots.keys().filter(|(id, _)| *id == pane_id).count() + } + + pub(crate) fn can_add_slot(&self, key: &Key) -> bool { + self.slots.contains_key(key) + || self.slots.len() < crate::api::schema::panes::PANE_GRAPHICS_MAX_LAYERS_TOTAL + } + + pub(crate) fn can_store_inline(&self, key: &Key, len: usize) -> bool { + let replaced = self + .slots + .get(key) + .and_then(|slot| slot.layer.as_ref()) + .and_then(Layer::inline_data) + .map_or(0, <[u8]>::len); + self.inline_bytes() + .saturating_sub(replaced) + .saturating_add(len) + <= crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL + } + + fn inline_bytes(&self) -> usize { + self.slots + .values() + .filter_map(|slot| slot.layer.as_ref()) + .filter_map(Layer::inline_data) + .map(<[u8]>::len) + .sum() + } + + pub(crate) fn active_for_pane(&self, pane_id: PaneId) -> bool { + self.slots + .iter() + .any(|((id, _), slot)| *id == pane_id && slot.layer.is_some()) + } + + pub(crate) fn attach_stream_active( + &mut self, + key: &Key, + owner: &str, + active: std::sync::Arc<std::sync::atomic::AtomicBool>, + ) { + if let Some(slot) = self + .slots + .get_mut(key) + .filter(|slot| slot.stream_owner.as_deref() == Some(owner)) + { + slot.stream_active = Some(active); + } + } + + pub(crate) fn retain_live_panes(&mut self, state: &crate::app::state::AppState) -> bool { + if self.slots.is_empty() { + return false; + } + let before = self.slots.len(); + self.slots.retain(|(pane_id, _), _| { + state + .workspaces + .iter() + .any(|workspace| workspace.pane_state(*pane_id).is_some()) + }); + if self.slots.len() == before { + return false; + } + self.mark_changed(); + true + } +} + +impl Layer { + pub(crate) fn inline( + format: PaneGraphicsFormat, + image_width: u32, + image_height: u32, + data: Vec<u8>, + render: PaneGraphicsPlacementParams, + z_index: i32, + ) -> Self { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + data.hash(&mut hasher); + Self { + format, + image_width, + image_height, + backing: Backing::Inline(data), + data_fingerprint: hasher.finish(), + render, + z_index, + } + } + + pub(crate) fn direct( + image_width: u32, + image_height: u32, + lease: crate::pane_graphics_files::Lease, + render: PaneGraphicsPlacementParams, + z_index: i32, + ) -> Self { + let data_fingerprint = lease.fingerprint(); + Self { + format: PaneGraphicsFormat::Rgba, + image_width, + image_height, + backing: Backing::Direct(lease), + data_fingerprint, + render, + z_index, + } + } + + pub(crate) fn data_len(&self) -> usize { + match &self.backing { + Backing::Inline(data) => data.len(), + Backing::Direct(lease) => lease.len(), + Backing::Resident { len, .. } => *len, + } + } + + pub(crate) fn inline_data(&self) -> Option<&[u8]> { + match &self.backing { + Backing::Inline(data) => Some(data), + Backing::Direct(_) | Backing::Resident { .. } => None, + } + } + + pub(crate) fn direct_lease(&self) -> Option<&crate::pane_graphics_files::Lease> { + match &self.backing { + Backing::Direct(lease) => Some(lease), + Backing::Inline(_) | Backing::Resident { .. } => None, + } + } + + pub(crate) fn mark_resident(&mut self, client_id: u64) -> bool { + let Backing::Direct(lease) = &self.backing else { + return false; + }; + self.backing = Backing::Resident { + len: lease.len(), + client_id, + }; + true + } + + pub(crate) fn resident_client(&self) -> Option<u64> { + match self.backing { + Backing::Resident { client_id, .. } => Some(client_id), + Backing::Inline(_) | Backing::Direct(_) => None, + } + } + + pub(crate) fn terminal_only(&self) -> bool { + matches!(self.backing, Backing::Direct(_) | Backing::Resident { .. }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stream_liveness_attaches_only_to_the_exact_owned_layer() { + let mut runtime = Runtime::default(); + let first = (PaneId::from_raw(1), "first".into()); + let second = (PaneId::from_raw(2), "second".into()); + for (key, id) in [(&first, 1), (&second, 2)] { + runtime.slots.insert( + key.clone(), + Slot { + host_image_id: (1 << 31) | id, + layer: None, + stream_owner: Some("shared-owner".into()), + stream_active: None, + direct_gate: None, + }, + ); + } + let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + + runtime.attach_stream_active(&second, "shared-owner", active.clone()); + + assert!(runtime.slots[&first].stream_active.is_none()); + assert!(std::sync::Arc::ptr_eq( + runtime.slots[&second].stream_active.as_ref().unwrap(), + &active + )); + } + + #[test] + fn runtime_bounds_total_slots_and_inline_bytes() { + let mut runtime = Runtime::default(); + for index in 0..crate::api::schema::panes::PANE_GRAPHICS_MAX_LAYERS_TOTAL { + let key = (PaneId::from_raw(1), format!("layer-{index}")); + runtime + .slots + .insert(key, Slot::test((1 << 31) | index as u32, None)); + } + assert!(!runtime.can_add_slot(&(PaneId::from_raw(2), "extra".into()))); + + let mut runtime = Runtime::default(); + let key = (PaneId::from_raw(1), "primary".into()); + runtime.slots.insert( + key.clone(), + Slot::test( + 1 << 31, + Some(Layer::inline( + PaneGraphicsFormat::Rgba, + 1, + 1, + vec![0; 4], + PaneGraphicsPlacementParams::default(), + 0, + )), + ), + ); + assert!(runtime.can_store_inline( + &key, + crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL, + )); + assert!(!runtime.can_store_inline( + &(PaneId::from_raw(2), "primary".into()), + crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL, + )); + } +} diff --git a/src/app/runtime.rs b/src/app/runtime.rs index 49420381..31426f87 100644 --- a/src/app/runtime.rs +++ b/src/app/runtime.rs @@ -61,6 +61,11 @@ impl App { msg: crate::api::ApiRequestMessage, ) -> bool { let previous_mode = self.state.mode; + let stream_open = match &msg.request.method { + crate::api::schema::Method::PaneGraphicsStreamOpen(params) => Some(params.clone()), + _ => None, + }; + let stream_active = msg.stream_active.clone(); let mut changed = self.expire_due_metadata(Instant::now()); changed |= crate::api::request_changes_ui(&msg.request); let skip_default_workspace = matches!( @@ -83,6 +88,9 @@ impl App { return changed | deferred_changed; } let response = self.handle_api_request(msg.request); + if let (Some(params), Some(active)) = (stream_open.as_ref(), stream_active) { + self.attach_pane_graphics_stream_active(params, active, &response); + } if !skip_default_workspace { changed |= self.ensure_default_workspace(); } diff --git a/src/app/state.rs b/src/app/state.rs index 04876290..e460f62b 100644 --- a/src/app/state.rs +++ b/src/app/state.rs @@ -4,7 +4,6 @@ use crate::config::{ use crossterm::event::{KeyCode, KeyModifiers}; use ratatui::layout::{Direction, Rect}; use ratatui::style::Color; -use std::hash::{Hash, Hasher}; use crate::detect::AgentState; use crate::layout::{PaneId, PaneInfo, SplitBorder}; @@ -12,7 +11,6 @@ use crate::selection::Selection; pub(crate) type InstalledPluginRegistry = std::collections::HashMap<String, crate::api::schema::InstalledPluginInfo>; - #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct PluginPaneRecord { pub plugin_id: String, @@ -20,42 +18,6 @@ pub(crate) struct PluginPaneRecord { } #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct PaneGraphicsLayer { - pub format: crate::api::schema::PaneGraphicsFormat, - pub image_width: u32, - pub image_height: u32, - pub data: Vec<u8>, - pub data_fingerprint: u64, - pub render: crate::api::schema::PaneGraphicsPlacementParams, -} - -impl PaneGraphicsLayer { - pub(crate) fn new( - format: crate::api::schema::PaneGraphicsFormat, - image_width: u32, - image_height: u32, - data: Vec<u8>, - render: crate::api::schema::PaneGraphicsPlacementParams, - ) -> Self { - let data_fingerprint = pane_graphics_data_fingerprint(&data); - Self { - format, - image_width, - image_height, - data, - data_fingerprint, - render, - } - } -} - -fn pane_graphics_data_fingerprint(data: &[u8]) -> u64 { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - data.hash(&mut hasher); - hasher.finish() -} - -#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct PopupPaneState { pub pane_id: PaneId, pub terminal_id: crate::terminal::TerminalId, @@ -1520,12 +1482,6 @@ pub struct AppState { pub(crate) installed_plugins: InstalledPluginRegistry, /// Pane ids opened through the plugin pane API. pub(crate) plugin_panes: std::collections::HashMap<PaneId, PluginPaneRecord>, - /// Runtime image layers owned by API clients and composited over panes. - pub(crate) pane_graphics_layers: std::collections::HashMap<PaneId, PaneGraphicsLayer>, - /// Active streaming graphics owner token by pane id. - pub(crate) pane_graphics_streams: std::collections::HashMap<PaneId, String>, - /// Monotonic marker for accepted pane graphics mutations. - pub(crate) pane_graphics_revision: u64, /// Session-modal terminal popup. This is intentionally outside workspace layouts. pub(crate) popup_pane: Option<PopupPaneState>, /// Recent plugin action/event command executions. @@ -1538,6 +1494,8 @@ pub struct AppState { pub host_terminal_theme: TerminalTheme, /// Last known foreground host terminal cell size in pixels. pub(crate) host_cell_size: crate::kitty_graphics::HostCellSize, + /// Exact pixel provenance only while one confirmed SGR report is dispatched. + pub(crate) host_mouse_pixels: Option<crate::input::mouse::HostPixels>, /// Set when a persisted session snapshot would change. pub session_dirty: bool, /// Terminal runtimes that should be shut down by the app/runtime layer @@ -1884,9 +1842,6 @@ impl AppState { integration_install_messages: Vec::new(), installed_plugins: std::collections::HashMap::new(), plugin_panes: std::collections::HashMap::new(), - pane_graphics_layers: std::collections::HashMap::new(), - pane_graphics_streams: std::collections::HashMap::new(), - pane_graphics_revision: 0, popup_pane: None, plugin_command_logs: Vec::new(), next_plugin_command_log_id: 1, @@ -1894,6 +1849,7 @@ impl AppState { global_menu: MenuListState::new(0), host_terminal_theme: TerminalTheme::default(), host_cell_size: crate::kitty_graphics::HostCellSize::default(), + host_mouse_pixels: None, session_dirty: false, terminal_runtime_shutdowns: Vec::new(), } @@ -2000,6 +1956,10 @@ impl AppState { self.context_menu.is_none(), "empty app state must not keep context menu" ); + assert!( + self.host_mouse_pixels.is_none(), + "empty app state must not keep host mouse pixel provenance" + ); return; } |
