From 1777e9bba32b953ed1ad203b4a16d01105539000 Mon Sep 17 00:00:00 2001
From: Can Celik <ogulcancelik@gmail.com>
Date: Sun, 9 Aug 2026 02:04:39 +0300
Subject: feat(graphics): add direct pane frame streaming (#2523)

* feat(graphics): add direct pane frame streaming

* feat(graphics): expose pane visibility

* test(graphics): preserve legacy terminal image output

* fix(graphics): harden streaming lifecycle

* fix(input): preserve mouse releases outside panes
---
 src/api/schema/panes.rs    | 30 ++++++++++++++++++++++++++++++
 src/api/schema/response.rs | 21 +++++++++++++++++++++
 2 files changed, 51 insertions(+)

(limited to 'src/api/schema')

diff --git a/src/api/schema/panes.rs b/src/api/schema/panes.rs
index 2dffd7f9..c8d89c48 100644
--- a/src/api/schema/panes.rs
+++ b/src/api/schema/panes.rs
@@ -4,6 +4,10 @@ use serde::{Deserialize, Serialize};
 
 pub(crate) const PANE_GRAPHICS_SET_MAX_BYTES: usize = 512 * 1024;
 pub(crate) const PANE_GRAPHICS_STREAM_MAX_BYTES: usize = 16 * 1024 * 1024;
+pub(crate) const PANE_GRAPHICS_MAX_LAYERS_PER_PANE: usize = 16;
+pub(crate) const PANE_GRAPHICS_MAX_LAYERS_TOTAL: usize = 64;
+pub(crate) const PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL: usize = 64 * 1024 * 1024;
+pub(crate) const PANE_GRAPHICS_PRIMARY_LAYER_ID: &str = "primary";
 
 use super::agents::AgentSessionInfo;
 use super::common::{AgentStatus, PaneAgentState, ReadFormat, ReadSource, SplitDirection};
@@ -287,11 +291,16 @@ pub enum PaneGraphicsFormat {
     Png,
     Rgb,
     Rgba,
+    Bgra,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
 pub struct PaneGraphicsSetParams {
     pub pane_id: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub layer_id: Option<String>,
+    #[serde(default)]
+    pub z_index: i32,
     #[serde(skip)]
     #[schemars(skip)]
     pub owner: String,
@@ -323,11 +332,32 @@ pub struct PaneGraphicsPlacementParams {
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
 pub struct PaneGraphicsClearParams {
     pub pane_id: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub layer_id: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PaneGraphicsDirectParams {
+    pub pane_id: String,
+    pub layer_id: Option<String>,
+    pub z_index: i32,
+    pub owner: String,
+    pub image_width: u32,
+    pub image_height: u32,
+    pub format: PaneGraphicsFormat,
+    pub path: String,
+    pub sequence: u64,
+    pub revision: u64,
+    pub placement: PaneGraphicsPlacementParams,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
 pub struct PaneGraphicsStreamParams {
     pub pane_id: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub layer_id: Option<String>,
+    #[serde(default)]
+    pub z_index: i32,
     #[serde(skip)]
     #[schemars(skip)]
     pub owner: String,
diff --git a/src/api/schema/response.rs b/src/api/schema/response.rs
index db8cfc7b..ae5a32e0 100644
--- a/src/api/schema/response.rs
+++ b/src/api/schema/response.rs
@@ -162,9 +162,30 @@ pub enum ResponseResult {
     PaneRead {
         read: PaneReadResult,
     },
+    PaneGraphicsFrameAck {
+        sequence: u64,
+        revision: u64,
+    },
     PaneGraphicsInfo {
         cell_width_px: u32,
         cell_height_px: u32,
+        /// True only when this pane is on the currently rendered terminal surface.
+        pane_visible: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        file_frame_directory: Option<String>,
+        #[serde(default, skip_serializing_if = "Vec::is_empty")]
+        file_frame_formats: Vec<String>,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        file_frame_max_bytes: Option<usize>,
+        /// Accepts damage metadata while still consuming a complete canonical file.
+        #[serde(default)]
+        file_frame_damage: bool,
+        #[serde(default)]
+        max_layers_per_pane: usize,
+        #[serde(default)]
+        pixel_mouse: bool,
+        #[serde(default, skip_serializing_if = "Option::is_none")]
+        file_frame_transport: Option<String>,
     },
     AgentExplain {
         explain: serde_json::Value,
-- 
cgit v1.2.3