diff options
| author | Ogulcan Celik <ogulcancelik@gmail.com> | 2026-07-15 02:32:52 +0300 |
|---|---|---|
| committer | Ogulcan Celik <ogulcancelik@gmail.com> | 2026-07-15 02:32:52 +0300 |
| commit | 88370e1516182ab519d22a12fa82d5780a40fee6 (patch) | |
| tree | b07be9b77abe14cdd3c1ff0e9acc1268729d8723 /src/app | |
| parent | 5d24d0d214d05858e344a9e15a63856dc1328eae (diff) | |
feat: add isolated pane graphics streaming
Diffstat (limited to 'src/app')
| -rw-r--r-- | src/app/actions.rs | 31 | ||||
| -rw-r--r-- | src/app/api.rs | 26 | ||||
| -rw-r--r-- | src/app/api/pane_graphics.rs | 702 | ||||
| -rw-r--r-- | src/app/api/plugins/mod.rs | 9 | ||||
| -rw-r--r-- | src/app/mod.rs | 16 | ||||
| -rw-r--r-- | src/app/state.rs | 49 |
6 files changed, 831 insertions, 2 deletions
diff --git a/src/app/actions.rs b/src/app/actions.rs index 7cf43e79..68286952 100644 --- a/src/app/actions.rs +++ b/src/app/actions.rs @@ -1490,6 +1490,8 @@ impl AppState { } for pane_id in pane_ids { self.plugin_panes.remove(&pane_id); + self.pane_graphics_layers.remove(&pane_id); + self.pane_graphics_streams.remove(&pane_id); } } @@ -3113,6 +3115,26 @@ mod tests { state } + fn insert_test_pane_graphics_layer(state: &mut AppState, pane_id: PaneId) { + state.pane_graphics_layers.insert( + pane_id, + crate::app::state::PaneGraphicsLayer::new( + crate::api::schema::PaneGraphicsFormat::Rgba, + 1, + 1, + vec![1, 2, 3, 4], + crate::api::schema::PaneGraphicsPlacementParams::default(), + ), + ); + } + + fn insert_test_pane_graphics_state(state: &mut AppState, pane_id: PaneId) { + insert_test_pane_graphics_layer(state, pane_id); + state + .pane_graphics_streams + .insert(pane_id, "test-stream".into()); + } + fn mark_linked_worktree(state: &mut AppState, ws_idx: usize) { state.workspaces[ws_idx].worktree_space = Some(crate::workspace::WorktreeSpaceMembership { key: "repo-key".into(), @@ -5196,10 +5218,13 @@ mod tests { entrypoint: "board".into(), }, ); + insert_test_pane_graphics_state(&mut state, closed); state.close_pane(); assert_eq!(state.workspaces[0].panes.len(), 1); assert!(!state.plugin_panes.contains_key(&closed)); + assert!(!state.pane_graphics_layers.contains_key(&closed)); + assert!(!state.pane_graphics_streams.contains_key(&closed)); state.assert_invariants_for_test(); } @@ -5264,11 +5289,14 @@ mod tests { entrypoint: "board".into(), }, ); + insert_test_pane_graphics_state(&mut state, pane_id); state.close_tab(); assert!(!state.terminals.contains_key(&terminal_id)); assert!(!state.plugin_panes.contains_key(&pane_id)); + assert!(!state.pane_graphics_layers.contains_key(&pane_id)); + assert!(!state.pane_graphics_streams.contains_key(&pane_id)); state.assert_invariants_for_test(); } @@ -5284,11 +5312,14 @@ mod tests { entrypoint: "board".into(), }, ); + insert_test_pane_graphics_state(&mut state, pane_id); state.close_selected_workspace(); assert!(!state.terminals.contains_key(&terminal_id)); assert!(!state.plugin_panes.contains_key(&pane_id)); + assert!(!state.pane_graphics_layers.contains_key(&pane_id)); + assert!(!state.pane_graphics_streams.contains_key(&pane_id)); state.assert_invariants_for_test(); } diff --git a/src/app/api.rs b/src/app/api.rs index d6d657b7..d2c2858f 100644 --- a/src/app/api.rs +++ b/src/app/api.rs @@ -5,6 +5,7 @@ mod agents; mod env; mod integrations; mod layouts; +mod pane_graphics; mod panes; pub(crate) mod plugins; mod responses; @@ -1013,6 +1014,31 @@ impl App { Method::PaneFocus(target) => return self.handle_pane_focus(request.id, target), Method::PaneRename(params) => return self.handle_pane_rename(request.id, params), Method::PaneRead(params) => return self.handle_pane_read(request.id, params), + Method::PaneGraphicsSet(params) => { + return self.handle_pane_graphics_set(request.id, params); + } + Method::PaneGraphicsClear(params) => { + return self.handle_pane_graphics_clear(request.id, params); + } + Method::PaneGraphicsInfo(params) => { + return self.handle_pane_graphics_info(request.id, params); + } + Method::PaneGraphicsStream(_) => { + return responses::encode_error( + request.id, + "stream_transport_required", + "pane.graphics.stream requires the streaming socket transport", + ); + } + Method::PaneGraphicsStreamSet(params) => { + return self.handle_pane_graphics_stream_set(request.id, params); + } + Method::PaneGraphicsStreamOpen(params) => { + return self.handle_pane_graphics_stream_open(request.id, params); + } + Method::PaneGraphicsStreamClose(params) => { + return self.handle_pane_graphics_stream_close(request.id, params); + } Method::PaneReportAgent(params) => { return self.handle_pane_report_agent(request.id, params); } diff --git a/src/app/api/pane_graphics.rs b/src/app/api/pane_graphics.rs new file mode 100644 index 00000000..7851f69a --- /dev/null +++ b/src/app/api/pane_graphics.rs @@ -0,0 +1,702 @@ +use base64::Engine; + +use crate::api::schema::{ + PaneGraphicsClearParams, PaneGraphicsSetParams, PaneGraphicsStreamParams, ResponseResult, + PANE_GRAPHICS_SET_MAX_BYTES, PANE_GRAPHICS_STREAM_MAX_BYTES, +}; +use crate::app::state::PaneGraphicsLayer; +use crate::app::App; +use crate::layout::PaneId; + +use super::responses::{encode_error, encode_success}; + +impl App { + pub(super) fn handle_pane_graphics_info( + &mut self, + id: String, + target: crate::api::schema::PaneTarget, + ) -> String { + if let Err(response) = require_pane_graphics_enabled(self, &id) { + return response; + } + if self.parse_pane_id(&target.pane_id).is_none() { + return pane_not_found(id, &target.pane_id); + } + if !self.state.host_cell_size.is_known() { + return encode_error(id, "cell_size_unavailable", "host cell size is unavailable"); + } + encode_success( + id, + ResponseResult::PaneGraphicsInfo { + cell_width_px: self.state.host_cell_size.width_px, + cell_height_px: self.state.host_cell_size.height_px, + }, + ) + } + + pub(super) fn handle_pane_graphics_set( + &mut self, + id: String, + params: PaneGraphicsSetParams, + ) -> String { + if let Err(response) = require_pane_graphics_enabled(self, &id) { + return response; + } + let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + return pane_not_found(id, ¶ms.pane_id); + }; + if self.state.pane_graphics_streams.contains_key(&pane_id) { + return encode_error( + id, + "stream_conflict", + "pane already has an active graphics stream", + ); + } + self.set_pane_graphics_layer(id, pane_id, params) + } + + fn set_pane_graphics_layer( + &mut self, + id: String, + pane_id: PaneId, + params: PaneGraphicsSetParams, + ) -> String { + if params.image_width == 0 || params.image_height == 0 { + return encode_error( + id, + "invalid_image", + "image_width and image_height must be greater than zero", + ); + } + let max_bytes = if params.data.is_some() { + PANE_GRAPHICS_STREAM_MAX_BYTES + } else { + PANE_GRAPHICS_SET_MAX_BYTES + }; + let data = match params.data { + Some(data) => data, + None => match base64::engine::general_purpose::STANDARD.decode(params.data_base64) { + Ok(data) => data, + Err(_) => { + return encode_error(id, "invalid_image", "data_base64 is not valid base64"); + } + }, + }; + if data.is_empty() { + return encode_error(id, "invalid_image", "image data must not be empty"); + } + if data.len() > max_bytes { + return encode_error(id, "image_too_large", "image data is too large"); + } + match pane_graphics_expected_data_len( + params.format, + params.image_width, + params.image_height, + ) { + Ok(Some(expected_len)) if data.len() != expected_len => { + return encode_error( + id, + "invalid_image", + "image data length does not match format and dimensions", + ); + } + Ok(_) => {} + Err(()) => { + return encode_error(id, "invalid_image", "image dimensions are too large"); + } + } + + let layer = PaneGraphicsLayer::new( + params.format, + params.image_width, + params.image_height, + data, + params.placement, + ); + self.state.pane_graphics_layers.insert(pane_id, layer); + self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); + + encode_success(id, ResponseResult::Ok {}) + } + + pub(super) fn handle_pane_graphics_clear( + &mut self, + id: String, + params: PaneGraphicsClearParams, + ) -> String { + if let Err(response) = require_pane_graphics_enabled(self, &id) { + return response; + } + let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + return pane_not_found(id, ¶ms.pane_id); + }; + if self.state.pane_graphics_streams.contains_key(&pane_id) { + return encode_error( + id, + "stream_conflict", + "pane already has an active graphics stream", + ); + } + if self.state.pane_graphics_layers.remove(&pane_id).is_some() { + self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); + } + + encode_success(id, ResponseResult::Ok {}) + } + + pub(super) fn handle_pane_graphics_stream_set( + &mut self, + id: String, + params: PaneGraphicsSetParams, + ) -> String { + if let Err(response) = require_pane_graphics_enabled(self, &id) { + return response; + } + let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + return pane_not_found(id, ¶ms.pane_id); + }; + match self.state.pane_graphics_streams.get(&pane_id) { + Some(owner) if owner == ¶ms.owner => {} + Some(_) => { + return encode_error( + id, + "stream_conflict", + "pane graphics stream owner does not match active stream", + ); + } + None => return encode_error(id, "stream_closed", "pane graphics stream is not active"), + } + self.set_pane_graphics_layer(id, pane_id, params) + } + + pub(super) fn handle_pane_graphics_stream_open( + &mut self, + id: String, + params: PaneGraphicsStreamParams, + ) -> String { + if let Err(response) = require_pane_graphics_enabled(self, &id) { + return response; + } + if params.owner.is_empty() { + return encode_error( + id, + "invalid_stream", + "pane graphics stream owner is required", + ); + } + let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + return pane_not_found(id, ¶ms.pane_id); + }; + if self.state.pane_graphics_streams.contains_key(&pane_id) { + return encode_error( + id, + "stream_conflict", + "pane already has an active graphics stream", + ); + } + self.state + .pane_graphics_streams + .insert(pane_id, params.owner); + self.state.pane_graphics_layers.remove(&pane_id); + self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); + + encode_success(id, ResponseResult::Ok {}) + } + + pub(super) fn handle_pane_graphics_stream_close( + &mut self, + id: String, + params: PaneGraphicsStreamParams, + ) -> String { + let Some((_ws_idx, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { + return pane_not_found(id, ¶ms.pane_id); + }; + if self + .state + .pane_graphics_streams + .get(&pane_id) + .is_some_and(|owner| owner == ¶ms.owner) + { + self.state.pane_graphics_streams.remove(&pane_id); + self.state.pane_graphics_layers.remove(&pane_id); + self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1); + } + + encode_success(id, ResponseResult::Ok {}) + } +} + +fn require_pane_graphics_enabled(app: &App, id: &str) -> Result<(), String> { + if app.state.kitty_graphics_enabled { + return Ok(()); + } + Err(encode_error( + id.to_owned(), + "feature_disabled", + "pane graphics require experimental.kitty_graphics", + )) +} + +fn pane_graphics_expected_data_len( + format: crate::api::schema::PaneGraphicsFormat, + image_width: u32, + image_height: u32, +) -> Result<Option<usize>, ()> { + let bytes_per_pixel = match format { + crate::api::schema::PaneGraphicsFormat::Png => return Ok(None), + crate::api::schema::PaneGraphicsFormat::Rgb => 3_u64, + crate::api::schema::PaneGraphicsFormat::Rgba => 4_u64, + }; + let pixels = u64::from(image_width) + .checked_mul(u64::from(image_height)) + .ok_or(())?; + pixels + .checked_mul(bytes_per_pixel) + .and_then(|bytes| usize::try_from(bytes).ok()) + .map(Some) + .ok_or(()) +} + +fn pane_not_found(id: String, pane_id: &str) -> String { + encode_error(id, "pane_not_found", format!("pane {pane_id} not found")) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::api::schema::{ErrorResponse, SuccessResponse}; + use crate::config::Config; + use crate::workspace::Workspace; + + fn app_with_test_workspace() -> (App, String) { + let (_api_tx, api_rx) = tokio::sync::mpsc::unbounded_channel(); + let mut app = App::new( + &Config::default(), + true, + None, + api_rx, + crate::api::EventHub::default(), + ); + app.state.workspaces = vec![Workspace::test_new("pane-graphics")]; + app.state.ensure_test_terminals(); + let pane_id = app.state.workspaces[0].tabs[0].root_pane; + let public_pane_id = app.public_pane_id(0, pane_id).unwrap(); + (app, public_pane_id) + } + + #[test] + fn api_pane_graphics_info_is_explicit_and_side_effect_free() { + let (mut app, public_pane_id) = app_with_test_workspace(); + let request = |pane_id: String| crate::api::schema::Request { + id: "graphics-info".into(), + method: crate::api::schema::Method::PaneGraphicsInfo(crate::api::schema::PaneTarget { + pane_id, + }), + }; + + let response = app.handle_api_request(request(public_pane_id.clone())); + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "feature_disabled"); + + app.state.kitty_graphics_enabled = true; + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 11, + height_px: 22, + }; + let response = app.handle_api_request(request(public_pane_id)); + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!( + success.result, + ResponseResult::PaneGraphicsInfo { + cell_width_px: 11, + cell_height_px: 22, + } + )); + assert!(app.state.pane_graphics_layers.is_empty()); + assert!(app.state.pane_graphics_streams.is_empty()); + } + + #[test] + fn api_pane_graphics_set_and_clear_updates_runtime_layers() { + let (mut app, public_pane_id) = app_with_test_workspace(); + app.state.kitty_graphics_enabled = true; + let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); + let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-set".into(), + method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { + pane_id: public_pane_id.clone(), + owner: String::new(), + format: crate::api::schema::PaneGraphicsFormat::Rgba, + image_width: 1, + image_height: 1, + data: None, + data_base64: data, + placement: crate::api::schema::PaneGraphicsPlacementParams { + grid_cols: 10, + grid_rows: 4, + ..Default::default() + }, + }), + }); + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!(success.result, ResponseResult::Ok {})); + + let layer = app + .state + .pane_graphics_layers + .get(&pane_id) + .expect("graphics layer stored"); + assert_eq!(layer.image_width, 1); + assert_eq!(layer.image_height, 1); + assert_eq!(layer.render.grid_cols, 10); + assert_eq!(layer.render.grid_rows, 4); + assert!(!app.state.session_dirty); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-clear".into(), + method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams { + pane_id: public_pane_id, + }), + }); + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!(success.result, ResponseResult::Ok {})); + assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + } + + #[test] + fn api_pane_graphics_rejects_when_kitty_graphics_disabled() { + let (mut app, public_pane_id) = app_with_test_workspace(); + let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); + let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-set".into(), + method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { + pane_id: public_pane_id.clone(), + owner: String::new(), + format: crate::api::schema::PaneGraphicsFormat::Rgba, + image_width: 1, + image_height: 1, + data: None, + data_base64: data, + placement: crate::api::schema::PaneGraphicsPlacementParams::default(), + }), + }); + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "feature_disabled"); + assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-clear".into(), + method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams { + pane_id: public_pane_id, + }), + }); + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "feature_disabled"); + assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + } + + #[test] + fn api_pane_graphics_stream_claim_uses_resolved_pane_identity() { + let (mut app, public_pane_id) = app_with_test_workspace(); + app.state.kitty_graphics_enabled = true; + let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); + let alias = format!("{public_pane_id}:alias"); + app.state + .public_pane_id_aliases + .insert(alias.clone(), pane_id); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-stream-open".into(), + method: crate::api::schema::Method::PaneGraphicsStreamOpen( + crate::api::schema::PaneGraphicsStreamParams { + pane_id: public_pane_id, + owner: "stream-1".into(), + }, + ), + }); + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!(success.result, ResponseResult::Ok {})); + assert_eq!( + app.state.pane_graphics_streams.get(&pane_id), + Some(&"stream-1".to_string()) + ); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-stream-open-alias".into(), + method: crate::api::schema::Method::PaneGraphicsStreamOpen( + crate::api::schema::PaneGraphicsStreamParams { + pane_id: alias.clone(), + owner: "stream-2".into(), + }, + ), + }); + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "stream_conflict"); + assert_eq!( + app.state.pane_graphics_streams.get(&pane_id), + Some(&"stream-1".to_string()) + ); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-stream-close-wrong-owner".into(), + method: crate::api::schema::Method::PaneGraphicsStreamClose( + crate::api::schema::PaneGraphicsStreamParams { + pane_id: alias.clone(), + owner: "stream-2".into(), + }, + ), + }); + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!(success.result, ResponseResult::Ok {})); + assert_eq!( + app.state.pane_graphics_streams.get(&pane_id), + Some(&"stream-1".to_string()) + ); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-stream-close-alias".into(), + method: crate::api::schema::Method::PaneGraphicsStreamClose( + crate::api::schema::PaneGraphicsStreamParams { + pane_id: alias, + owner: "stream-1".into(), + }, + ), + }); + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!(success.result, ResponseResult::Ok {})); + assert!(!app.state.pane_graphics_streams.contains_key(&pane_id)); + } + + #[test] + fn api_pane_graphics_stream_owns_layer_while_active() { + let (mut app, public_pane_id) = app_with_test_workspace(); + app.state.kitty_graphics_enabled = true; + let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); + let public_data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-stream-open".into(), + method: crate::api::schema::Method::PaneGraphicsStreamOpen( + crate::api::schema::PaneGraphicsStreamParams { + pane_id: public_pane_id.clone(), + owner: "stream-1".into(), + }, + ), + }); + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!(success.result, ResponseResult::Ok {})); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-set-public".into(), + method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { + pane_id: public_pane_id.clone(), + owner: String::new(), + format: crate::api::schema::PaneGraphicsFormat::Rgba, + image_width: 1, + image_height: 1, + data: None, + data_base64: public_data.clone(), + placement: crate::api::schema::PaneGraphicsPlacementParams::default(), + }), + }); + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "stream_conflict"); + assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-set-stream".into(), + method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { + pane_id: public_pane_id.clone(), + owner: "stream-1".into(), + format: crate::api::schema::PaneGraphicsFormat::Rgba, + image_width: 1, + image_height: 1, + data: None, + data_base64: public_data, + placement: crate::api::schema::PaneGraphicsPlacementParams::default(), + }), + }); + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!(success.result, ResponseResult::Ok {})); + assert!(app.state.pane_graphics_layers.contains_key(&pane_id)); + + let wrong_owner_data = base64::engine::general_purpose::STANDARD.encode([5_u8, 6, 7, 8]); + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-set-stream-wrong-owner".into(), + method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { + pane_id: public_pane_id.clone(), + owner: "stream-2".into(), + format: crate::api::schema::PaneGraphicsFormat::Rgba, + image_width: 1, + image_height: 1, + data: None, + data_base64: wrong_owner_data, + placement: crate::api::schema::PaneGraphicsPlacementParams::default(), + }), + }); + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "stream_conflict"); + assert_eq!( + app.state.pane_graphics_layers.get(&pane_id).unwrap().data, + vec![1_u8, 2, 3, 4] + ); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-clear-public".into(), + method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams { + pane_id: public_pane_id.clone(), + }), + }); + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "stream_conflict"); + assert!(app.state.pane_graphics_layers.contains_key(&pane_id)); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-stream-close".into(), + method: crate::api::schema::Method::PaneGraphicsStreamClose( + crate::api::schema::PaneGraphicsStreamParams { + pane_id: public_pane_id, + owner: "stream-1".into(), + }, + ), + }); + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!(success.result, ResponseResult::Ok {})); + assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + } + + #[test] + fn api_pane_graphics_rejects_overflowing_raw_dimensions() { + for format in [ + crate::api::schema::PaneGraphicsFormat::Rgb, + crate::api::schema::PaneGraphicsFormat::Rgba, + ] { + let (mut app, public_pane_id) = app_with_test_workspace(); + app.state.kitty_graphics_enabled = true; + let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); + + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-overflow".into(), + method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { + pane_id: public_pane_id, + owner: String::new(), + format, + image_width: u32::MAX, + image_height: u32::MAX, + data: None, + data_base64: base64::engine::general_purpose::STANDARD.encode([1_u8]), + placement: crate::api::schema::PaneGraphicsPlacementParams::default(), + }), + }); + + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "invalid_image"); + assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + } + } + + #[test] + fn api_pane_graphics_validates_exact_raw_lengths() { + for (format, exact_len) in [ + (crate::api::schema::PaneGraphicsFormat::Rgb, 6_usize), + (crate::api::schema::PaneGraphicsFormat::Rgba, 8_usize), + ] { + for data_len in [exact_len, exact_len - 1] { + let (mut app, public_pane_id) = app_with_test_workspace(); + app.state.kitty_graphics_enabled = true; + let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-raw-length".into(), + method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { + pane_id: public_pane_id, + owner: String::new(), + format, + image_width: 2, + image_height: 1, + data: None, + data_base64: base64::engine::general_purpose::STANDARD + .encode(vec![1_u8; data_len]), + placement: crate::api::schema::PaneGraphicsPlacementParams::default(), + }), + }); + + if data_len == exact_len { + let success: SuccessResponse = serde_json::from_str(&response).unwrap(); + assert!(matches!(success.result, ResponseResult::Ok {})); + assert!(app.state.pane_graphics_layers.contains_key(&pane_id)); + } else { + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "invalid_image"); + assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + } + } + } + } + + #[test] + fn api_pane_graphics_uses_transport_specific_limits() { + let (mut app, public_pane_id) = app_with_test_workspace(); + app.state.kitty_graphics_enabled = true; + let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); + let response = app.handle_api_request(crate::api::schema::Request { + id: "graphics-public-too-large".into(), + method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams { + pane_id: public_pane_id, + owner: String::new(), + format: crate::api::schema::PaneGraphicsFormat::Png, + image_width: 1, + image_height: 1, + data: None, + data_base64: base64::engine::general_purpose::STANDARD.encode(vec![ + 1_u8; + PANE_GRAPHICS_SET_MAX_BYTES + + 1 + ]), + placement: crate::api::schema::PaneGraphicsPlacementParams::default(), + }), + }); + + let error: ErrorResponse = serde_json::from_str(&response).unwrap(); + assert_eq!(error.error.code, "image_too_large"); + assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + } + + #[test] + fn api_pane_graphics_stream_methods_require_kitty_graphics() { + let (mut app, public_pane_id) = app_with_test_workspace(); + let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap(); + let open = app.handle_api_request(crate::api::schema::Request { + id: "graphics-stream-open".into(), + method: crate::api::schema::Method::PaneGraphicsStreamOpen( + crate::api::schema::PaneGraphicsStreamParams { + pane_id: public_pane_id.clone(), + owner: "stream-1".into(), + }, + ), + }); + let error: ErrorResponse = serde_json::from_str(&open).unwrap(); + assert_eq!(error.error.code, "feature_disabled"); + + let set = app.handle_api_request(crate::api::schema::Request { + id: "graphics-stream-set".into(), + method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { + pane_id: public_pane_id, + owner: "stream-1".into(), + format: crate::api::schema::PaneGraphicsFormat::Png, + image_width: 1, + image_height: 1, + data: Some(vec![1_u8]), + data_base64: String::new(), + placement: crate::api::schema::PaneGraphicsPlacementParams::default(), + }), + }); + let error: ErrorResponse = serde_json::from_str(&set).unwrap(); + assert_eq!(error.error.code, "feature_disabled"); + assert!(!app.state.pane_graphics_streams.contains_key(&pane_id)); + assert!(!app.state.pane_graphics_layers.contains_key(&pane_id)); + } +} diff --git a/src/app/api/plugins/mod.rs b/src/app/api/plugins/mod.rs index 1e0ec7d8..b7f90df9 100644 --- a/src/app/api/plugins/mod.rs +++ b/src/app/api/plugins/mod.rs @@ -1164,6 +1164,11 @@ command = ["echo", "b"] app.state.active = Some(0); app.state.selected = 0; app.state.mode = crate::app::Mode::Terminal; + app.state.kitty_graphics_enabled = true; + app.state.host_cell_size = crate::kitty_graphics::HostCellSize { + width_px: 11, + height_px: 22, + }; app.state.terminals.get_mut(&root_terminal).unwrap().cwd = "/tmp".into(); let target_public_pane_id = app.public_pane_id(0, root_pane).unwrap(); @@ -1182,7 +1187,7 @@ platforms = ["linux", "macos"] [[panes]] id = "board" title = "Plugin Board" -command = ["sh", "-c", "printf '%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \"$PWD\" \"$HERDR_PLUGIN_ID\" \"$HERDR_PLUGIN_ENTRYPOINT_ID\" \"$HERDR_WORKSPACE_ID\" \"$HERDR_PANE_ID\" \"$HERDR_BIN_PATH\" \"$HERDR_PLUGIN_CONTEXT_JSON\" > {}"] +command = ["sh", "-c", "printf '%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \"$PWD\" \"$HERDR_PLUGIN_ID\" \"$HERDR_PLUGIN_ENTRYPOINT_ID\" \"$HERDR_WORKSPACE_ID\" \"$HERDR_PANE_ID\" \"$HERDR_BIN_PATH\" \"$HERDR_PLUGIN_CONTEXT_JSON\" \"${{HERDR_CELL_WIDTH_PX-unset}}\" \"${{HERDR_CELL_HEIGHT_PX-unset}}\" > {}"] "#, capture.display() ), @@ -1254,6 +1259,8 @@ command = ["sh", "-c", "printf '%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \"$PWD\" \"$HERDR_ context.focused_pane_id.as_deref(), Some(target_public_pane_id.as_str()) ); + assert_eq!(lines.next(), Some("unset")); + assert_eq!(lines.next(), Some("unset")); for (_, runtime) in app.terminal_runtimes.drain() { runtime.shutdown(); diff --git a/src/app/mod.rs b/src/app/mod.rs index f32d7785..7391d7d3 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -656,11 +656,15 @@ impl App { integration_install_messages: Vec::new(), installed_plugins: load_plugin_registry(no_session), plugin_panes: std::collections::HashMap::new(), + pane_graphics_layers: std::collections::HashMap::new(), + pane_graphics_streams: std::collections::HashMap::new(), + pane_graphics_revision: 0, plugin_command_logs: Vec::new(), next_plugin_command_log_id: 1, plugin_commands_in_flight: 0, global_menu: state::MenuListState::new(0), host_terminal_theme: crate::terminal_theme::TerminalTheme::default(), + host_cell_size: crate::kitty_graphics::HostCellSize::default(), session_dirty: false, terminal_runtime_shutdowns: Vec::new(), }; @@ -1026,7 +1030,14 @@ impl App { terminal.draw(|frame| { let area = frame.area(); if kitty_graphics_enabled { - cell_size = crate::kitty_graphics::HostCellSize::from_terminal(area); + let observed_cell_size = + crate::kitty_graphics::HostCellSize::try_from_terminal(area); + if let Some(observed_cell_size) = observed_cell_size { + self.state.host_cell_size = observed_cell_size; + } + cell_size = observed_cell_size.unwrap_or_else(|| { + crate::kitty_graphics::HostCellSize::fallback_for_area(area) + }); crate::ui::compute_view_with_cell_size( &mut self.state, &self.terminal_runtimes, @@ -1426,6 +1437,9 @@ impl App { crate::kitty_graphics::set_enabled(config.experimental.kitty_graphics); if was_kitty_graphics_enabled && !config.experimental.kitty_graphics { let _ = crate::kitty_graphics::clear_all_host_graphics(); + self.state.pane_graphics_layers.clear(); + self.state.pane_graphics_streams.clear(); + self.state.host_cell_size = crate::kitty_graphics::HostCellSize::default(); } self.state.reveal_hidden_cursor_for_cjk_ime = config.experimental.reveal_hidden_cursor_for_cjk_ime; diff --git a/src/app/state.rs b/src/app/state.rs index d2315422..c0f2c8bb 100644 --- a/src/app/state.rs +++ b/src/app/state.rs @@ -2,6 +2,7 @@ use crate::config::{Keybinds, NewTerminalCwdConfig, SoundConfig, ToastConfig, To use crossterm::event::{KeyCode, KeyModifiers}; use ratatui::layout::{Direction, Rect}; use ratatui::style::Color; +use std::hash::{Hash, Hasher}; use crate::detect::AgentState; use crate::layout::{PaneId, PaneInfo, SplitBorder}; @@ -16,6 +17,42 @@ pub(crate) struct PluginPaneRecord { pub entrypoint: String, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PaneGraphicsLayer { + pub format: crate::api::schema::PaneGraphicsFormat, + pub image_width: u32, + pub image_height: u32, + pub data: Vec<u8>, + pub data_fingerprint: u64, + pub render: crate::api::schema::PaneGraphicsPlacementParams, +} + +impl PaneGraphicsLayer { + pub(crate) fn new( + format: crate::api::schema::PaneGraphicsFormat, + image_width: u32, + image_height: u32, + data: Vec<u8>, + render: crate::api::schema::PaneGraphicsPlacementParams, + ) -> Self { + let data_fingerprint = pane_graphics_data_fingerprint(&data); + Self { + format, + image_width, + image_height, + data, + data_fingerprint, + render, + } + } +} + +fn pane_graphics_data_fingerprint(data: &[u8]) -> u64 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + data.hash(&mut hasher); + hasher.finish() +} + // --------------------------------------------------------------------------- // Selection autoscroll types // --------------------------------------------------------------------------- @@ -1472,6 +1509,12 @@ pub struct AppState { pub(crate) installed_plugins: InstalledPluginRegistry, /// Pane ids opened through the plugin pane API. pub(crate) plugin_panes: std::collections::HashMap<PaneId, PluginPaneRecord>, + /// Runtime image layers owned by API clients and composited over panes. + pub(crate) pane_graphics_layers: std::collections::HashMap<PaneId, PaneGraphicsLayer>, + /// Active streaming graphics owner token by pane id. + pub(crate) pane_graphics_streams: std::collections::HashMap<PaneId, String>, + /// Monotonic marker for accepted pane graphics mutations. + pub(crate) pane_graphics_revision: u64, /// Recent plugin action/event command executions. pub(crate) plugin_command_logs: Vec<crate::api::schema::PluginCommandLogInfo>, pub(crate) next_plugin_command_log_id: u64, @@ -1480,6 +1523,8 @@ pub struct AppState { pub global_menu: MenuListState, /// Resolved host terminal default colors for theming embedded panes. pub host_terminal_theme: TerminalTheme, + /// Last known foreground host terminal cell size in pixels. + pub(crate) host_cell_size: crate::kitty_graphics::HostCellSize, /// Set when a persisted session snapshot would change. pub session_dirty: bool, /// Terminal runtimes that should be shut down by the app/runtime layer @@ -1826,11 +1871,15 @@ impl AppState { integration_install_messages: Vec::new(), installed_plugins: std::collections::HashMap::new(), plugin_panes: std::collections::HashMap::new(), + pane_graphics_layers: std::collections::HashMap::new(), + pane_graphics_streams: std::collections::HashMap::new(), + pane_graphics_revision: 0, plugin_command_logs: Vec::new(), next_plugin_command_log_id: 1, plugin_commands_in_flight: 0, global_menu: MenuListState::new(0), host_terminal_theme: TerminalTheme::default(), + host_cell_size: crate::kitty_graphics::HostCellSize::default(), session_dirty: false, terminal_runtime_shutdowns: Vec::new(), } |
