aboutsummaryrefslogtreecommitdiff
path: root/src/app
diff options
context:
space:
mode:
authorOgulcan Celik <ogulcancelik@gmail.com>2026-07-15 02:32:52 +0300
committerOgulcan Celik <ogulcancelik@gmail.com>2026-07-15 02:32:52 +0300
commit88370e1516182ab519d22a12fa82d5780a40fee6 (patch)
treeb07be9b77abe14cdd3c1ff0e9acc1268729d8723 /src/app
parent5d24d0d214d05858e344a9e15a63856dc1328eae (diff)
feat: add isolated pane graphics streaming
Diffstat (limited to 'src/app')
-rw-r--r--src/app/actions.rs31
-rw-r--r--src/app/api.rs26
-rw-r--r--src/app/api/pane_graphics.rs702
-rw-r--r--src/app/api/plugins/mod.rs9
-rw-r--r--src/app/mod.rs16
-rw-r--r--src/app/state.rs49
6 files changed, 831 insertions, 2 deletions
diff --git a/src/app/actions.rs b/src/app/actions.rs
index 7cf43e79..68286952 100644
--- a/src/app/actions.rs
+++ b/src/app/actions.rs
@@ -1490,6 +1490,8 @@ impl AppState {
}
for pane_id in pane_ids {
self.plugin_panes.remove(&pane_id);
+ self.pane_graphics_layers.remove(&pane_id);
+ self.pane_graphics_streams.remove(&pane_id);
}
}
@@ -3113,6 +3115,26 @@ mod tests {
state
}
+ fn insert_test_pane_graphics_layer(state: &mut AppState, pane_id: PaneId) {
+ state.pane_graphics_layers.insert(
+ pane_id,
+ crate::app::state::PaneGraphicsLayer::new(
+ crate::api::schema::PaneGraphicsFormat::Rgba,
+ 1,
+ 1,
+ vec![1, 2, 3, 4],
+ crate::api::schema::PaneGraphicsPlacementParams::default(),
+ ),
+ );
+ }
+
+ fn insert_test_pane_graphics_state(state: &mut AppState, pane_id: PaneId) {
+ insert_test_pane_graphics_layer(state, pane_id);
+ state
+ .pane_graphics_streams
+ .insert(pane_id, "test-stream".into());
+ }
+
fn mark_linked_worktree(state: &mut AppState, ws_idx: usize) {
state.workspaces[ws_idx].worktree_space = Some(crate::workspace::WorktreeSpaceMembership {
key: "repo-key".into(),
@@ -5196,10 +5218,13 @@ mod tests {
entrypoint: "board".into(),
},
);
+ insert_test_pane_graphics_state(&mut state, closed);
state.close_pane();
assert_eq!(state.workspaces[0].panes.len(), 1);
assert!(!state.plugin_panes.contains_key(&closed));
+ assert!(!state.pane_graphics_layers.contains_key(&closed));
+ assert!(!state.pane_graphics_streams.contains_key(&closed));
state.assert_invariants_for_test();
}
@@ -5264,11 +5289,14 @@ mod tests {
entrypoint: "board".into(),
},
);
+ insert_test_pane_graphics_state(&mut state, pane_id);
state.close_tab();
assert!(!state.terminals.contains_key(&terminal_id));
assert!(!state.plugin_panes.contains_key(&pane_id));
+ assert!(!state.pane_graphics_layers.contains_key(&pane_id));
+ assert!(!state.pane_graphics_streams.contains_key(&pane_id));
state.assert_invariants_for_test();
}
@@ -5284,11 +5312,14 @@ mod tests {
entrypoint: "board".into(),
},
);
+ insert_test_pane_graphics_state(&mut state, pane_id);
state.close_selected_workspace();
assert!(!state.terminals.contains_key(&terminal_id));
assert!(!state.plugin_panes.contains_key(&pane_id));
+ assert!(!state.pane_graphics_layers.contains_key(&pane_id));
+ assert!(!state.pane_graphics_streams.contains_key(&pane_id));
state.assert_invariants_for_test();
}
diff --git a/src/app/api.rs b/src/app/api.rs
index d6d657b7..d2c2858f 100644
--- a/src/app/api.rs
+++ b/src/app/api.rs
@@ -5,6 +5,7 @@ mod agents;
mod env;
mod integrations;
mod layouts;
+mod pane_graphics;
mod panes;
pub(crate) mod plugins;
mod responses;
@@ -1013,6 +1014,31 @@ impl App {
Method::PaneFocus(target) => return self.handle_pane_focus(request.id, target),
Method::PaneRename(params) => return self.handle_pane_rename(request.id, params),
Method::PaneRead(params) => return self.handle_pane_read(request.id, params),
+ Method::PaneGraphicsSet(params) => {
+ return self.handle_pane_graphics_set(request.id, params);
+ }
+ Method::PaneGraphicsClear(params) => {
+ return self.handle_pane_graphics_clear(request.id, params);
+ }
+ Method::PaneGraphicsInfo(params) => {
+ return self.handle_pane_graphics_info(request.id, params);
+ }
+ Method::PaneGraphicsStream(_) => {
+ return responses::encode_error(
+ request.id,
+ "stream_transport_required",
+ "pane.graphics.stream requires the streaming socket transport",
+ );
+ }
+ Method::PaneGraphicsStreamSet(params) => {
+ return self.handle_pane_graphics_stream_set(request.id, params);
+ }
+ Method::PaneGraphicsStreamOpen(params) => {
+ return self.handle_pane_graphics_stream_open(request.id, params);
+ }
+ Method::PaneGraphicsStreamClose(params) => {
+ return self.handle_pane_graphics_stream_close(request.id, params);
+ }
Method::PaneReportAgent(params) => {
return self.handle_pane_report_agent(request.id, params);
}
diff --git a/src/app/api/pane_graphics.rs b/src/app/api/pane_graphics.rs
new file mode 100644
index 00000000..7851f69a
--- /dev/null
+++ b/src/app/api/pane_graphics.rs
@@ -0,0 +1,702 @@
+use base64::Engine;
+
+use crate::api::schema::{
+ PaneGraphicsClearParams, PaneGraphicsSetParams, PaneGraphicsStreamParams, ResponseResult,
+ PANE_GRAPHICS_SET_MAX_BYTES, PANE_GRAPHICS_STREAM_MAX_BYTES,
+};
+use crate::app::state::PaneGraphicsLayer;
+use crate::app::App;
+use crate::layout::PaneId;
+
+use super::responses::{encode_error, encode_success};
+
+impl App {
+ pub(super) fn handle_pane_graphics_info(
+ &mut self,
+ id: String,
+ target: crate::api::schema::PaneTarget,
+ ) -> String {
+ if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ return response;
+ }
+ if self.parse_pane_id(&target.pane_id).is_none() {
+ return pane_not_found(id, &target.pane_id);
+ }
+ if !self.state.host_cell_size.is_known() {
+ return encode_error(id, "cell_size_unavailable", "host cell size is unavailable");
+ }
+ encode_success(
+ id,
+ ResponseResult::PaneGraphicsInfo {
+ cell_width_px: self.state.host_cell_size.width_px,
+ cell_height_px: self.state.host_cell_size.height_px,
+ },
+ )
+ }
+
+ pub(super) fn handle_pane_graphics_set(
+ &mut self,
+ id: String,
+ params: PaneGraphicsSetParams,
+ ) -> String {
+ if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ return response;
+ }
+ let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ return pane_not_found(id, &params.pane_id);
+ };
+ if self.state.pane_graphics_streams.contains_key(&pane_id) {
+ return encode_error(
+ id,
+ "stream_conflict",
+ "pane already has an active graphics stream",
+ );
+ }
+ self.set_pane_graphics_layer(id, pane_id, params)
+ }
+
+ fn set_pane_graphics_layer(
+ &mut self,
+ id: String,
+ pane_id: PaneId,
+ params: PaneGraphicsSetParams,
+ ) -> String {
+ if params.image_width == 0 || params.image_height == 0 {
+ return encode_error(
+ id,
+ "invalid_image",
+ "image_width and image_height must be greater than zero",
+ );
+ }
+ let max_bytes = if params.data.is_some() {
+ PANE_GRAPHICS_STREAM_MAX_BYTES
+ } else {
+ PANE_GRAPHICS_SET_MAX_BYTES
+ };
+ let data = match params.data {
+ Some(data) => data,
+ None => match base64::engine::general_purpose::STANDARD.decode(params.data_base64) {
+ Ok(data) => data,
+ Err(_) => {
+ return encode_error(id, "invalid_image", "data_base64 is not valid base64");
+ }
+ },
+ };
+ if data.is_empty() {
+ return encode_error(id, "invalid_image", "image data must not be empty");
+ }
+ if data.len() > max_bytes {
+ return encode_error(id, "image_too_large", "image data is too large");
+ }
+ match pane_graphics_expected_data_len(
+ params.format,
+ params.image_width,
+ params.image_height,
+ ) {
+ Ok(Some(expected_len)) if data.len() != expected_len => {
+ return encode_error(
+ id,
+ "invalid_image",
+ "image data length does not match format and dimensions",
+ );
+ }
+ Ok(_) => {}
+ Err(()) => {
+ return encode_error(id, "invalid_image", "image dimensions are too large");
+ }
+ }
+
+ let layer = PaneGraphicsLayer::new(
+ params.format,
+ params.image_width,
+ params.image_height,
+ data,
+ params.placement,
+ );
+ self.state.pane_graphics_layers.insert(pane_id, layer);
+ self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1);
+
+ encode_success(id, ResponseResult::Ok {})
+ }
+
+ pub(super) fn handle_pane_graphics_clear(
+ &mut self,
+ id: String,
+ params: PaneGraphicsClearParams,
+ ) -> String {
+ if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ return response;
+ }
+ let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ return pane_not_found(id, &params.pane_id);
+ };
+ if self.state.pane_graphics_streams.contains_key(&pane_id) {
+ return encode_error(
+ id,
+ "stream_conflict",
+ "pane already has an active graphics stream",
+ );
+ }
+ if self.state.pane_graphics_layers.remove(&pane_id).is_some() {
+ self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1);
+ }
+
+ encode_success(id, ResponseResult::Ok {})
+ }
+
+ pub(super) fn handle_pane_graphics_stream_set(
+ &mut self,
+ id: String,
+ params: PaneGraphicsSetParams,
+ ) -> String {
+ if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ return response;
+ }
+ let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ return pane_not_found(id, &params.pane_id);
+ };
+ match self.state.pane_graphics_streams.get(&pane_id) {
+ Some(owner) if owner == &params.owner => {}
+ Some(_) => {
+ return encode_error(
+ id,
+ "stream_conflict",
+ "pane graphics stream owner does not match active stream",
+ );
+ }
+ None => return encode_error(id, "stream_closed", "pane graphics stream is not active"),
+ }
+ self.set_pane_graphics_layer(id, pane_id, params)
+ }
+
+ pub(super) fn handle_pane_graphics_stream_open(
+ &mut self,
+ id: String,
+ params: PaneGraphicsStreamParams,
+ ) -> String {
+ if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ return response;
+ }
+ if params.owner.is_empty() {
+ return encode_error(
+ id,
+ "invalid_stream",
+ "pane graphics stream owner is required",
+ );
+ }
+ let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ return pane_not_found(id, &params.pane_id);
+ };
+ if self.state.pane_graphics_streams.contains_key(&pane_id) {
+ return encode_error(
+ id,
+ "stream_conflict",
+ "pane already has an active graphics stream",
+ );
+ }
+ self.state
+ .pane_graphics_streams
+ .insert(pane_id, params.owner);
+ self.state.pane_graphics_layers.remove(&pane_id);
+ self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1);
+
+ encode_success(id, ResponseResult::Ok {})
+ }
+
+ pub(super) fn handle_pane_graphics_stream_close(
+ &mut self,
+ id: String,
+ params: PaneGraphicsStreamParams,
+ ) -> String {
+ let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ return pane_not_found(id, &params.pane_id);
+ };
+ if self
+ .state
+ .pane_graphics_streams
+ .get(&pane_id)
+ .is_some_and(|owner| owner == &params.owner)
+ {
+ self.state.pane_graphics_streams.remove(&pane_id);
+ self.state.pane_graphics_layers.remove(&pane_id);
+ self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1);
+ }
+
+ encode_success(id, ResponseResult::Ok {})
+ }
+}
+
+fn require_pane_graphics_enabled(app: &App, id: &str) -> Result<(), String> {
+ if app.state.kitty_graphics_enabled {
+ return Ok(());
+ }
+ Err(encode_error(
+ id.to_owned(),
+ "feature_disabled",
+ "pane graphics require experimental.kitty_graphics",
+ ))
+}
+
+fn pane_graphics_expected_data_len(
+ format: crate::api::schema::PaneGraphicsFormat,
+ image_width: u32,
+ image_height: u32,
+) -> Result<Option<usize>, ()> {
+ let bytes_per_pixel = match format {
+ crate::api::schema::PaneGraphicsFormat::Png => return Ok(None),
+ crate::api::schema::PaneGraphicsFormat::Rgb => 3_u64,
+ crate::api::schema::PaneGraphicsFormat::Rgba => 4_u64,
+ };
+ let pixels = u64::from(image_width)
+ .checked_mul(u64::from(image_height))
+ .ok_or(())?;
+ pixels
+ .checked_mul(bytes_per_pixel)
+ .and_then(|bytes| usize::try_from(bytes).ok())
+ .map(Some)
+ .ok_or(())
+}
+
+fn pane_not_found(id: String, pane_id: &str) -> String {
+ encode_error(id, "pane_not_found", format!("pane {pane_id} not found"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::api::schema::{ErrorResponse, SuccessResponse};
+ use crate::config::Config;
+ use crate::workspace::Workspace;
+
+ fn app_with_test_workspace() -> (App, String) {
+ let (_api_tx, api_rx) = tokio::sync::mpsc::unbounded_channel();
+ let mut app = App::new(
+ &Config::default(),
+ true,
+ None,
+ api_rx,
+ crate::api::EventHub::default(),
+ );
+ app.state.workspaces = vec![Workspace::test_new("pane-graphics")];
+ app.state.ensure_test_terminals();
+ let pane_id = app.state.workspaces[0].tabs[0].root_pane;
+ let public_pane_id = app.public_pane_id(0, pane_id).unwrap();
+ (app, public_pane_id)
+ }
+
+ #[test]
+ fn api_pane_graphics_info_is_explicit_and_side_effect_free() {
+ let (mut app, public_pane_id) = app_with_test_workspace();
+ let request = |pane_id: String| crate::api::schema::Request {
+ id: "graphics-info".into(),
+ method: crate::api::schema::Method::PaneGraphicsInfo(crate::api::schema::PaneTarget {
+ pane_id,
+ }),
+ };
+
+ let response = app.handle_api_request(request(public_pane_id.clone()));
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "feature_disabled");
+
+ app.state.kitty_graphics_enabled = true;
+ app.state.host_cell_size = crate::kitty_graphics::HostCellSize {
+ width_px: 11,
+ height_px: 22,
+ };
+ let response = app.handle_api_request(request(public_pane_id));
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(
+ success.result,
+ ResponseResult::PaneGraphicsInfo {
+ cell_width_px: 11,
+ cell_height_px: 22,
+ }
+ ));
+ assert!(app.state.pane_graphics_layers.is_empty());
+ assert!(app.state.pane_graphics_streams.is_empty());
+ }
+
+ #[test]
+ fn api_pane_graphics_set_and_clear_updates_runtime_layers() {
+ let (mut app, public_pane_id) = app_with_test_workspace();
+ app.state.kitty_graphics_enabled = true;
+ let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
+ let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]);
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-set".into(),
+ method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
+ pane_id: public_pane_id.clone(),
+ owner: String::new(),
+ format: crate::api::schema::PaneGraphicsFormat::Rgba,
+ image_width: 1,
+ image_height: 1,
+ data: None,
+ data_base64: data,
+ placement: crate::api::schema::PaneGraphicsPlacementParams {
+ grid_cols: 10,
+ grid_rows: 4,
+ ..Default::default()
+ },
+ }),
+ });
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(success.result, ResponseResult::Ok {}));
+
+ let layer = app
+ .state
+ .pane_graphics_layers
+ .get(&pane_id)
+ .expect("graphics layer stored");
+ assert_eq!(layer.image_width, 1);
+ assert_eq!(layer.image_height, 1);
+ assert_eq!(layer.render.grid_cols, 10);
+ assert_eq!(layer.render.grid_rows, 4);
+ assert!(!app.state.session_dirty);
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-clear".into(),
+ method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams {
+ pane_id: public_pane_id,
+ }),
+ });
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(success.result, ResponseResult::Ok {}));
+ assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ }
+
+ #[test]
+ fn api_pane_graphics_rejects_when_kitty_graphics_disabled() {
+ let (mut app, public_pane_id) = app_with_test_workspace();
+ let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
+ let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]);
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-set".into(),
+ method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
+ pane_id: public_pane_id.clone(),
+ owner: String::new(),
+ format: crate::api::schema::PaneGraphicsFormat::Rgba,
+ image_width: 1,
+ image_height: 1,
+ data: None,
+ data_base64: data,
+ placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
+ }),
+ });
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "feature_disabled");
+ assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-clear".into(),
+ method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams {
+ pane_id: public_pane_id,
+ }),
+ });
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "feature_disabled");
+ assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ }
+
+ #[test]
+ fn api_pane_graphics_stream_claim_uses_resolved_pane_identity() {
+ let (mut app, public_pane_id) = app_with_test_workspace();
+ app.state.kitty_graphics_enabled = true;
+ let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
+ let alias = format!("{public_pane_id}:alias");
+ app.state
+ .public_pane_id_aliases
+ .insert(alias.clone(), pane_id);
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-stream-open".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamOpen(
+ crate::api::schema::PaneGraphicsStreamParams {
+ pane_id: public_pane_id,
+ owner: "stream-1".into(),
+ },
+ ),
+ });
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(success.result, ResponseResult::Ok {}));
+ assert_eq!(
+ app.state.pane_graphics_streams.get(&pane_id),
+ Some(&"stream-1".to_string())
+ );
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-stream-open-alias".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamOpen(
+ crate::api::schema::PaneGraphicsStreamParams {
+ pane_id: alias.clone(),
+ owner: "stream-2".into(),
+ },
+ ),
+ });
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "stream_conflict");
+ assert_eq!(
+ app.state.pane_graphics_streams.get(&pane_id),
+ Some(&"stream-1".to_string())
+ );
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-stream-close-wrong-owner".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamClose(
+ crate::api::schema::PaneGraphicsStreamParams {
+ pane_id: alias.clone(),
+ owner: "stream-2".into(),
+ },
+ ),
+ });
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(success.result, ResponseResult::Ok {}));
+ assert_eq!(
+ app.state.pane_graphics_streams.get(&pane_id),
+ Some(&"stream-1".to_string())
+ );
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-stream-close-alias".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamClose(
+ crate::api::schema::PaneGraphicsStreamParams {
+ pane_id: alias,
+ owner: "stream-1".into(),
+ },
+ ),
+ });
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(success.result, ResponseResult::Ok {}));
+ assert!(!app.state.pane_graphics_streams.contains_key(&pane_id));
+ }
+
+ #[test]
+ fn api_pane_graphics_stream_owns_layer_while_active() {
+ let (mut app, public_pane_id) = app_with_test_workspace();
+ app.state.kitty_graphics_enabled = true;
+ let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
+ let public_data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]);
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-stream-open".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamOpen(
+ crate::api::schema::PaneGraphicsStreamParams {
+ pane_id: public_pane_id.clone(),
+ owner: "stream-1".into(),
+ },
+ ),
+ });
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(success.result, ResponseResult::Ok {}));
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-set-public".into(),
+ method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
+ pane_id: public_pane_id.clone(),
+ owner: String::new(),
+ format: crate::api::schema::PaneGraphicsFormat::Rgba,
+ image_width: 1,
+ image_height: 1,
+ data: None,
+ data_base64: public_data.clone(),
+ placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
+ }),
+ });
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "stream_conflict");
+ assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-set-stream".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams {
+ pane_id: public_pane_id.clone(),
+ owner: "stream-1".into(),
+ format: crate::api::schema::PaneGraphicsFormat::Rgba,
+ image_width: 1,
+ image_height: 1,
+ data: None,
+ data_base64: public_data,
+ placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
+ }),
+ });
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(success.result, ResponseResult::Ok {}));
+ assert!(app.state.pane_graphics_layers.contains_key(&pane_id));
+
+ let wrong_owner_data = base64::engine::general_purpose::STANDARD.encode([5_u8, 6, 7, 8]);
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-set-stream-wrong-owner".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams {
+ pane_id: public_pane_id.clone(),
+ owner: "stream-2".into(),
+ format: crate::api::schema::PaneGraphicsFormat::Rgba,
+ image_width: 1,
+ image_height: 1,
+ data: None,
+ data_base64: wrong_owner_data,
+ placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
+ }),
+ });
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "stream_conflict");
+ assert_eq!(
+ app.state.pane_graphics_layers.get(&pane_id).unwrap().data,
+ vec![1_u8, 2, 3, 4]
+ );
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-clear-public".into(),
+ method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams {
+ pane_id: public_pane_id.clone(),
+ }),
+ });
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "stream_conflict");
+ assert!(app.state.pane_graphics_layers.contains_key(&pane_id));
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-stream-close".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamClose(
+ crate::api::schema::PaneGraphicsStreamParams {
+ pane_id: public_pane_id,
+ owner: "stream-1".into(),
+ },
+ ),
+ });
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(success.result, ResponseResult::Ok {}));
+ assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ }
+
+ #[test]
+ fn api_pane_graphics_rejects_overflowing_raw_dimensions() {
+ for format in [
+ crate::api::schema::PaneGraphicsFormat::Rgb,
+ crate::api::schema::PaneGraphicsFormat::Rgba,
+ ] {
+ let (mut app, public_pane_id) = app_with_test_workspace();
+ app.state.kitty_graphics_enabled = true;
+ let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
+
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-overflow".into(),
+ method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
+ pane_id: public_pane_id,
+ owner: String::new(),
+ format,
+ image_width: u32::MAX,
+ image_height: u32::MAX,
+ data: None,
+ data_base64: base64::engine::general_purpose::STANDARD.encode([1_u8]),
+ placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
+ }),
+ });
+
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "invalid_image");
+ assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ }
+ }
+
+ #[test]
+ fn api_pane_graphics_validates_exact_raw_lengths() {
+ for (format, exact_len) in [
+ (crate::api::schema::PaneGraphicsFormat::Rgb, 6_usize),
+ (crate::api::schema::PaneGraphicsFormat::Rgba, 8_usize),
+ ] {
+ for data_len in [exact_len, exact_len - 1] {
+ let (mut app, public_pane_id) = app_with_test_workspace();
+ app.state.kitty_graphics_enabled = true;
+ let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-raw-length".into(),
+ method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
+ pane_id: public_pane_id,
+ owner: String::new(),
+ format,
+ image_width: 2,
+ image_height: 1,
+ data: None,
+ data_base64: base64::engine::general_purpose::STANDARD
+ .encode(vec![1_u8; data_len]),
+ placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
+ }),
+ });
+
+ if data_len == exact_len {
+ let success: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(success.result, ResponseResult::Ok {}));
+ assert!(app.state.pane_graphics_layers.contains_key(&pane_id));
+ } else {
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "invalid_image");
+ assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn api_pane_graphics_uses_transport_specific_limits() {
+ let (mut app, public_pane_id) = app_with_test_workspace();
+ app.state.kitty_graphics_enabled = true;
+ let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
+ let response = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-public-too-large".into(),
+ method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
+ pane_id: public_pane_id,
+ owner: String::new(),
+ format: crate::api::schema::PaneGraphicsFormat::Png,
+ image_width: 1,
+ image_height: 1,
+ data: None,
+ data_base64: base64::engine::general_purpose::STANDARD.encode(vec![
+ 1_u8;
+ PANE_GRAPHICS_SET_MAX_BYTES
+ + 1
+ ]),
+ placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
+ }),
+ });
+
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "image_too_large");
+ assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ }
+
+ #[test]
+ fn api_pane_graphics_stream_methods_require_kitty_graphics() {
+ let (mut app, public_pane_id) = app_with_test_workspace();
+ let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
+ let open = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-stream-open".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamOpen(
+ crate::api::schema::PaneGraphicsStreamParams {
+ pane_id: public_pane_id.clone(),
+ owner: "stream-1".into(),
+ },
+ ),
+ });
+ let error: ErrorResponse = serde_json::from_str(&open).unwrap();
+ assert_eq!(error.error.code, "feature_disabled");
+
+ let set = app.handle_api_request(crate::api::schema::Request {
+ id: "graphics-stream-set".into(),
+ method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams {
+ pane_id: public_pane_id,
+ owner: "stream-1".into(),
+ format: crate::api::schema::PaneGraphicsFormat::Png,
+ image_width: 1,
+ image_height: 1,
+ data: Some(vec![1_u8]),
+ data_base64: String::new(),
+ placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
+ }),
+ });
+ let error: ErrorResponse = serde_json::from_str(&set).unwrap();
+ assert_eq!(error.error.code, "feature_disabled");
+ assert!(!app.state.pane_graphics_streams.contains_key(&pane_id));
+ assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ }
+}
diff --git a/src/app/api/plugins/mod.rs b/src/app/api/plugins/mod.rs
index 1e0ec7d8..b7f90df9 100644
--- a/src/app/api/plugins/mod.rs
+++ b/src/app/api/plugins/mod.rs
@@ -1164,6 +1164,11 @@ command = ["echo", "b"]
app.state.active = Some(0);
app.state.selected = 0;
app.state.mode = crate::app::Mode::Terminal;
+ app.state.kitty_graphics_enabled = true;
+ app.state.host_cell_size = crate::kitty_graphics::HostCellSize {
+ width_px: 11,
+ height_px: 22,
+ };
app.state.terminals.get_mut(&root_terminal).unwrap().cwd = "/tmp".into();
let target_public_pane_id = app.public_pane_id(0, root_pane).unwrap();
@@ -1182,7 +1187,7 @@ platforms = ["linux", "macos"]
[[panes]]
id = "board"
title = "Plugin Board"
-command = ["sh", "-c", "printf '%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \"$PWD\" \"$HERDR_PLUGIN_ID\" \"$HERDR_PLUGIN_ENTRYPOINT_ID\" \"$HERDR_WORKSPACE_ID\" \"$HERDR_PANE_ID\" \"$HERDR_BIN_PATH\" \"$HERDR_PLUGIN_CONTEXT_JSON\" > {}"]
+command = ["sh", "-c", "printf '%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \"$PWD\" \"$HERDR_PLUGIN_ID\" \"$HERDR_PLUGIN_ENTRYPOINT_ID\" \"$HERDR_WORKSPACE_ID\" \"$HERDR_PANE_ID\" \"$HERDR_BIN_PATH\" \"$HERDR_PLUGIN_CONTEXT_JSON\" \"${{HERDR_CELL_WIDTH_PX-unset}}\" \"${{HERDR_CELL_HEIGHT_PX-unset}}\" > {}"]
"#,
capture.display()
),
@@ -1254,6 +1259,8 @@ command = ["sh", "-c", "printf '%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \"$PWD\" \"$HERDR_
context.focused_pane_id.as_deref(),
Some(target_public_pane_id.as_str())
);
+ assert_eq!(lines.next(), Some("unset"));
+ assert_eq!(lines.next(), Some("unset"));
for (_, runtime) in app.terminal_runtimes.drain() {
runtime.shutdown();
diff --git a/src/app/mod.rs b/src/app/mod.rs
index f32d7785..7391d7d3 100644
--- a/src/app/mod.rs
+++ b/src/app/mod.rs
@@ -656,11 +656,15 @@ impl App {
integration_install_messages: Vec::new(),
installed_plugins: load_plugin_registry(no_session),
plugin_panes: std::collections::HashMap::new(),
+ pane_graphics_layers: std::collections::HashMap::new(),
+ pane_graphics_streams: std::collections::HashMap::new(),
+ pane_graphics_revision: 0,
plugin_command_logs: Vec::new(),
next_plugin_command_log_id: 1,
plugin_commands_in_flight: 0,
global_menu: state::MenuListState::new(0),
host_terminal_theme: crate::terminal_theme::TerminalTheme::default(),
+ host_cell_size: crate::kitty_graphics::HostCellSize::default(),
session_dirty: false,
terminal_runtime_shutdowns: Vec::new(),
};
@@ -1026,7 +1030,14 @@ impl App {
terminal.draw(|frame| {
let area = frame.area();
if kitty_graphics_enabled {
- cell_size = crate::kitty_graphics::HostCellSize::from_terminal(area);
+ let observed_cell_size =
+ crate::kitty_graphics::HostCellSize::try_from_terminal(area);
+ if let Some(observed_cell_size) = observed_cell_size {
+ self.state.host_cell_size = observed_cell_size;
+ }
+ cell_size = observed_cell_size.unwrap_or_else(|| {
+ crate::kitty_graphics::HostCellSize::fallback_for_area(area)
+ });
crate::ui::compute_view_with_cell_size(
&mut self.state,
&self.terminal_runtimes,
@@ -1426,6 +1437,9 @@ impl App {
crate::kitty_graphics::set_enabled(config.experimental.kitty_graphics);
if was_kitty_graphics_enabled && !config.experimental.kitty_graphics {
let _ = crate::kitty_graphics::clear_all_host_graphics();
+ self.state.pane_graphics_layers.clear();
+ self.state.pane_graphics_streams.clear();
+ self.state.host_cell_size = crate::kitty_graphics::HostCellSize::default();
}
self.state.reveal_hidden_cursor_for_cjk_ime =
config.experimental.reveal_hidden_cursor_for_cjk_ime;
diff --git a/src/app/state.rs b/src/app/state.rs
index d2315422..c0f2c8bb 100644
--- a/src/app/state.rs
+++ b/src/app/state.rs
@@ -2,6 +2,7 @@ use crate::config::{Keybinds, NewTerminalCwdConfig, SoundConfig, ToastConfig, To
use crossterm::event::{KeyCode, KeyModifiers};
use ratatui::layout::{Direction, Rect};
use ratatui::style::Color;
+use std::hash::{Hash, Hasher};
use crate::detect::AgentState;
use crate::layout::{PaneId, PaneInfo, SplitBorder};
@@ -16,6 +17,42 @@ pub(crate) struct PluginPaneRecord {
pub entrypoint: String,
}
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct PaneGraphicsLayer {
+ pub format: crate::api::schema::PaneGraphicsFormat,
+ pub image_width: u32,
+ pub image_height: u32,
+ pub data: Vec<u8>,
+ pub data_fingerprint: u64,
+ pub render: crate::api::schema::PaneGraphicsPlacementParams,
+}
+
+impl PaneGraphicsLayer {
+ pub(crate) fn new(
+ format: crate::api::schema::PaneGraphicsFormat,
+ image_width: u32,
+ image_height: u32,
+ data: Vec<u8>,
+ render: crate::api::schema::PaneGraphicsPlacementParams,
+ ) -> Self {
+ let data_fingerprint = pane_graphics_data_fingerprint(&data);
+ Self {
+ format,
+ image_width,
+ image_height,
+ data,
+ data_fingerprint,
+ render,
+ }
+ }
+}
+
+fn pane_graphics_data_fingerprint(data: &[u8]) -> u64 {
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
+ data.hash(&mut hasher);
+ hasher.finish()
+}
+
// ---------------------------------------------------------------------------
// Selection autoscroll types
// ---------------------------------------------------------------------------
@@ -1472,6 +1509,12 @@ pub struct AppState {
pub(crate) installed_plugins: InstalledPluginRegistry,
/// Pane ids opened through the plugin pane API.
pub(crate) plugin_panes: std::collections::HashMap<PaneId, PluginPaneRecord>,
+ /// Runtime image layers owned by API clients and composited over panes.
+ pub(crate) pane_graphics_layers: std::collections::HashMap<PaneId, PaneGraphicsLayer>,
+ /// Active streaming graphics owner token by pane id.
+ pub(crate) pane_graphics_streams: std::collections::HashMap<PaneId, String>,
+ /// Monotonic marker for accepted pane graphics mutations.
+ pub(crate) pane_graphics_revision: u64,
/// Recent plugin action/event command executions.
pub(crate) plugin_command_logs: Vec<crate::api::schema::PluginCommandLogInfo>,
pub(crate) next_plugin_command_log_id: u64,
@@ -1480,6 +1523,8 @@ pub struct AppState {
pub global_menu: MenuListState,
/// Resolved host terminal default colors for theming embedded panes.
pub host_terminal_theme: TerminalTheme,
+ /// Last known foreground host terminal cell size in pixels.
+ pub(crate) host_cell_size: crate::kitty_graphics::HostCellSize,
/// Set when a persisted session snapshot would change.
pub session_dirty: bool,
/// Terminal runtimes that should be shut down by the app/runtime layer
@@ -1826,11 +1871,15 @@ impl AppState {
integration_install_messages: Vec::new(),
installed_plugins: std::collections::HashMap::new(),
plugin_panes: std::collections::HashMap::new(),
+ pane_graphics_layers: std::collections::HashMap::new(),
+ pane_graphics_streams: std::collections::HashMap::new(),
+ pane_graphics_revision: 0,
plugin_command_logs: Vec::new(),
next_plugin_command_log_id: 1,
plugin_commands_in_flight: 0,
global_menu: MenuListState::new(0),
host_terminal_theme: TerminalTheme::default(),
+ host_cell_size: crate::kitty_graphics::HostCellSize::default(),
session_dirty: false,
terminal_runtime_shutdowns: Vec::new(),
}