feat(graphics): add direct pane frame streaming (#2523)

* feat(graphics): add direct pane frame streaming

* feat(graphics): expose pane visibility

* test(graphics): preserve legacy terminal image output

* fix(graphics): harden streaming lifecycle

* fix(input): preserve mouse releases outside panes

1 files changed, 320 insertions, 34 deletions

diff --git a/src/client/mod.rs b/src/client/mod.rs
index 7932cb79..301377c9 100644
--- a/src/client/mod.rs
+++ b/src/client/mod.rs
@@ -12,9 +12,13 @@
 //! - Forwards OSC 52 clipboard writes from server to its own stdout
 //! - Displays sound/toast notifications forwarded from server
 
+#[cfg(unix)]
+mod direct_graphics;
 mod input;
 
 use std::collections::HashSet;
+#[cfg(unix)]
+use std::io::IsTerminal as _;
 use std::io::{self, BufRead, Write as _};
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex, OnceLock};
@@ -75,6 +79,12 @@ struct ClientState {
     sound_config: crate::config::SoundConfig,
     /// Whether this client may write Kitty graphics bytes to its host terminal.
     kitty_graphics_enabled: bool,
+    /// One bounded matcher, inactive unless a direct transmission is armed.
+    #[cfg(unix)]
+    direct_graphics_response: Arc<Mutex<direct_graphics::ResponseMatcher>>,
+    /// One server-retired direct transfer to suppress if it was still queued.
+    #[cfg(unix)]
+    retired_direct_graphics: Option<(u64, u32)>,
     /// Direct attach prefix escape state. None for full-app clients.
     attach_escape: Option<AttachEscapeState>,
     /// Rows scrolled for one direct-attach wheel notch.
@@ -356,9 +366,9 @@ fn setup_terminal_with_capabilities(
 
     if enable_client_protocols {
         if mouse_capture {
-            set_mouse_capture(true)?;
+            set_mouse_capture(true, false)?;
         } else {
-            set_mouse_capture(false)?;
+            set_mouse_capture(false, false)?;
         }
         execute!(io::stdout(), EnableBracketedPaste, EnableFocusChange)?;
         if host_color_scheme_reports {
@@ -370,9 +380,9 @@ fn setup_terminal_with_capabilities(
             write_host_color_scheme_report_mode(&mut io::stdout(), false)?;
         }
         if mouse_capture {
-            set_mouse_capture(true)?;
+            set_mouse_capture(true, false)?;
         } else {
-            set_mouse_capture(false)?;
+            set_mouse_capture(false, false)?;
         }
     }
 
@@ -549,10 +559,15 @@ fn restore_windows_input_mode_value(mode: u32) {
     }
 }
 
-fn set_mouse_capture(enabled: bool) -> io::Result<()> {
+fn set_mouse_capture(enabled: bool, sgr_pixels: bool) -> io::Result<()> {
     crate::terminal_modes::clear_host_mouse_reporting(&mut io::stdout())?;
     if enabled {
-        execute!(io::stdout(), EnableMouseCapture)
+        execute!(io::stdout(), EnableMouseCapture)?;
+        if sgr_pixels {
+            io::stdout().write_all(b"\x1b[?1016h")?;
+            io::stdout().flush()?;
+        }
+        Ok(())
     } else {
         match execute!(io::stdout(), DisableMouseCapture) {
             Ok(()) => Ok(()),
@@ -685,6 +700,44 @@ fn handshake_read_timeout() -> Duration {
     LOCAL_HANDSHAKE_READ_TIMEOUT
 }
 
+#[cfg(any(unix, test))]
+fn direct_graphics_profile_values(
+    term_program: &str,
+    term: &str,
+    kitty_window: bool,
+    blocked_transport: bool,
+    terminals: bool,
+) -> bool {
+    let supported = term_program.eq_ignore_ascii_case("ghostty")
+        || term_program.eq_ignore_ascii_case("wezterm")
+        || matches!(term, "xterm-ghostty" | "xterm-kitty" | "xterm-wezterm")
+        || kitty_window;
+    supported && !blocked_transport && terminals
+}
+
+#[cfg(unix)]
+fn direct_graphics_profile_allowed(direct_attach: bool) -> bool {
+    let term_program = std::env::var("TERM_PROGRAM").unwrap_or_default();
+    let term = std::env::var("TERM").unwrap_or_default();
+    direct_graphics_profile_values(
+        &term_program,
+        &term,
+        std::env::var_os("KITTY_WINDOW_ID").is_some(),
+        direct_attach
+            || is_remote_client_process()
+            || std::env::var_os("SSH_CONNECTION").is_some()
+            || std::env::var_os("SSH_TTY").is_some()
+            || std::env::var_os("TMUX").is_some()
+            || std::env::var_os("STY").is_some(),
+        io::stdin().is_terminal() && io::stdout().is_terminal(),
+    )
+}
+
+#[cfg(not(unix))]
+fn direct_graphics_profile_allowed(_direct_attach: bool) -> bool {
+    false
+}
+
 fn requested_keybindings() -> ClientKeybindings {
     match std::env::var(crate::remote::REMOTE_KEYBINDINGS_ENV_VAR)
         .ok()
@@ -726,6 +779,25 @@ fn set_handshake_recv_timeout(
         .map_err(ClientError::ConnectionFailed)
 }
 
+fn client_launch_mode(
+    direct_attach_requested: bool,
+    exact_cell_size: bool,
+    cell_width_px: u32,
+    cell_height_px: u32,
+) -> ClientLaunchMode {
+    if direct_attach_requested {
+        ClientLaunchMode::TerminalAttach
+    } else if exact_cell_size
+        && cell_width_px > 0
+        && cell_height_px > 0
+        && direct_graphics_profile_allowed(false)
+    {
+        ClientLaunchMode::AppDirectGraphics
+    } else {
+        ClientLaunchMode::App
+    }
+}
+
 /// Performs the client→server handshake.
 ///
 /// Sends Hello with the terminal size and protocol version, reads the Welcome
@@ -736,6 +808,7 @@ fn do_handshake(
     rows: u16,
     cell_width_px: u32,
     cell_height_px: u32,
+    exact_cell_size: bool,
     requested_encoding: RenderEncoding,
     direct_attach_requested: bool,
 ) -> Result<RenderEncoding, ClientError> {
@@ -752,11 +825,12 @@ fn do_handshake(
         cell_height_px,
         requested_encoding,
         keybindings: requested_keybindings(),
-        launch_mode: if direct_attach_requested {
-            ClientLaunchMode::TerminalAttach
-        } else {
-            ClientLaunchMode::App
-        },
+        launch_mode: client_launch_mode(
+            direct_attach_requested,
+            exact_cell_size,
+            cell_width_px,
+            cell_height_px,
+        ),
     };
     protocol::write_message(stream, &hello)
         .map_err(|e| ClientError::ConnectionFailed(io::Error::other(e.to_string())))?;
@@ -801,6 +875,11 @@ enum ClientLoopEvent {
     /// Raw input bytes from stdin.
     #[cfg(unix)]
     StdinInput(Vec<u8>),
+    /// One confirmed SGR pixel report with geometry captured by the reader.
+    #[cfg(unix)]
+    PixelMouse(Vec<u8>, crate::input::mouse::HostGeometry),
+    #[cfg(unix)]
+    DirectGraphicsResponse(direct_graphics::Response),
     /// Structured input events from platforms without Unix-style stdin bytes.
     #[cfg(windows)]
     StdinEvents(Vec<crate::protocol::ClientInputEvent>),
@@ -929,6 +1008,7 @@ fn connect_terminal_session_stream(
         rows,
         0,
         0,
+        false,
         RenderEncoding::TerminalAnsi,
         true,
     ) {
@@ -1149,7 +1229,7 @@ fn run_client_with_mode(
     };
 
     // Get the terminal geometry before handshake (before raw mode).
-    let (cols, rows, cell_width_px, cell_height_px) =
+    let (cols, rows, cell_width_px, cell_height_px, exact_cell_size) =
         initial_terminal_geometry(kitty_graphics_enabled);
 
     // Perform handshake while the stream is still in blocking mode.
@@ -1159,6 +1239,7 @@ fn run_client_with_mode(
         rows,
         cell_width_px,
         cell_height_px,
+        exact_cell_size,
         requested_encoding,
         direct_attach_requested,
     ) {
@@ -1296,6 +1377,10 @@ async fn run_client_loop(
         reported_size: (cols, rows),
         sound_config: config.sound_config,
         kitty_graphics_enabled: config.kitty_graphics_enabled,
+        #[cfg(unix)]
+        direct_graphics_response: Arc::new(Mutex::new(direct_graphics::ResponseMatcher::default())),
+        #[cfg(unix)]
+        retired_direct_graphics: None,
         attach_escape,
         #[cfg(unix)]
         mouse_scroll_lines: config.mouse_scroll_lines,
@@ -1309,6 +1394,7 @@ async fn run_client_loop(
     // Cell size reported by the host terminal, packed as width<<32 | height.
     // Zero means the host has not reported one.
     let reported_cell_size = Arc::new(AtomicU64::new(0));
+    let host_sgr_pixels_active = Arc::new(AtomicBool::new(false));
 
     // Channel for events from the stdin, resize, and server reader threads.
     let (event_tx, mut event_rx) = tokio::sync::mpsc::channel::<ClientLoopEvent>(256);
@@ -1323,6 +1409,14 @@ async fn run_client_loop(
     let stdin_quit = should_quit.clone();
     let stdin_tx = event_tx.clone();
     let stdin_mouse_capture_active = host_mouse_capture_active.clone();
+    let stdin_sgr_pixels_active = host_sgr_pixels_active.clone();
+    #[cfg(unix)]
+    let stdin_direct_response = state.direct_graphics_response.clone();
+    #[cfg(unix)]
+    let stdin_direct_response_active = stdin_direct_response
+        .lock()
+        .map(|matcher| matcher.active_handle())
+        .unwrap_or_default();
     std::thread::spawn(move || {
         input::stdin_reader_loop(
             stdin_tx,
@@ -1330,6 +1424,11 @@ async fn run_client_loop(
             will_query_host_terminal_theme,
             will_query_host_cell_size,
             stdin_mouse_capture_active,
+            stdin_sgr_pixels_active,
+            #[cfg(unix)]
+            stdin_direct_response,
+            #[cfg(unix)]
+            stdin_direct_response_active,
         );
     });
 
@@ -1475,6 +1574,30 @@ async fn run_client_loop(
                     return Err(ClientError::ConnectionLost(e));
                 }
             }
+            #[cfg(unix)]
+            ClientLoopEvent::DirectGraphicsResponse(response) => {
+                let message = ClientMessage::GraphicsTransmissionResult {
+                    transfer_id: response.transfer_id,
+                    image_id: response.image_id,
+                    success: response.success,
+                };
+                if let Err(err) = write_to_server(&mut write_stream, &message) {
+                    return Err(ClientError::ConnectionLost(err));
+                }
+            }
+            #[cfg(unix)]
+            ClientLoopEvent::PixelMouse(data, geometry) => {
+                let message = ClientMessage::InputPixels {
+                    data,
+                    cols: geometry.cols,
+                    rows: geometry.rows,
+                    width_px: geometry.width_px,
+                    height_px: geometry.height_px,
+                };
+                if let Err(err) = write_to_server(&mut write_stream, &message) {
+                    return Err(ClientError::ConnectionLost(err));
+                }
+            }
             #[cfg(windows)]
             ClientLoopEvent::StdinEvents(events) => {
                 if state.attach_escape.is_some() {
@@ -1578,6 +1701,98 @@ async fn run_client_loop(
                         warn!(err = %err, "failed to emit terminal bell");
                     }
                 }
+                ServerMessage::GraphicsFile {
+                    path,
+                    expected_len,
+                    image_id,
+                    transfer_id,
+                    leading,
+                    control,
+                } => {
+                    #[cfg(unix)]
+                    {
+                        if state.retired_direct_graphics.take() == Some((transfer_id, image_id)) {
+                            continue;
+                        }
+                        let valid = state.kitty_graphics_enabled
+                            && usize::try_from(expected_len).ok().is_some_and(|len| {
+                                crate::pane_graphics_files::validate_direct_source(
+                                    std::path::Path::new(&path),
+                                    len,
+                                )
+                                .is_ok()
+                            })
+                            && direct_graphics::valid_control(&control, image_id)
+                            && state
+                                .direct_graphics_response
+                                .lock()
+                                .is_ok_and(|mut matcher| matcher.arm(transfer_id, image_id));
+                        let sent = if valid {
+                            let mut command = Vec::new();
+                            crate::kitty_graphics::encode_kitty_regular_file(
+                                &mut command,
+                                &leading,
+                                &control,
+                                &path,
+                            );
+                            let mut stdout = io::stdout();
+                            let written = stdout
+                                .write_all(&command)
+                                .and_then(|()| stdout.flush())
+                                .is_ok();
+                            if written {
+                                record_received_kitty_graphics(&command);
+                            }
+                            written
+                        } else {
+                            false
+                        };
+                        if sent {
+                            if let Ok(mut matcher) = state.direct_graphics_response.lock() {
+                                matcher.start(transfer_id);
+                            }
+                            let started = ClientMessage::GraphicsTransmissionStarted {
+                                transfer_id,
+                                image_id,
+                            };
+                            if let Err(err) = write_to_server(&mut write_stream, &started) {
+                                return Err(ClientError::ConnectionLost(err));
+                            }
+                        } else {
+                            if let Ok(mut matcher) = state.direct_graphics_response.lock() {
+                                if valid {
+                                    matcher.retire(transfer_id);
+                                } else {
+                                    matcher.cancel(transfer_id);
+                                }
+                            }
+                            let result = ClientMessage::GraphicsTransmissionResult {
+                                transfer_id,
+                                image_id,
+                                success: false,
+                            };
+                            if let Err(err) = write_to_server(&mut write_stream, &result) {
+                                return Err(ClientError::ConnectionLost(err));
+                            }
+                        }
+                    }
+                    #[cfg(not(unix))]
+                    let _ = (path, expected_len, image_id, transfer_id, leading, control);
+                }
+                ServerMessage::GraphicsTransmissionRetired {
+                    transfer_id,
+                    image_id,
+                } => {
+                    #[cfg(unix)]
+                    {
+                        state.retired_direct_graphics = Some((transfer_id, image_id));
+                        if let Ok(mut matcher) = state.direct_graphics_response.lock() {
+                            matcher.retire(transfer_id);
+                        }
+                    }
+                    #[cfg(not(unix))]
+                    let _ = (transfer_id, image_id);
+                }
                 ServerMessage::ServerShutdown { reason } => {
                     return Err(ClientError::ServerShutdown { reason });
                 }
@@ -1604,17 +1819,24 @@ async fn run_client_loop(
                         &mut state.remote_image_paste_key,
                     );
                 }
-                ServerMessage::MouseCapture { enabled } => {
-                    let desired = enabled;
-                    if desired != state.mouse_capture_active {
-                        set_mouse_capture(desired).map_err(ClientError::ConnectionFailed)?;
+                ServerMessage::MouseCapture {
+                    enabled,
+                    sgr_pixels,
+                } => {
+                    let next_sgr_pixels = enabled && sgr_pixels;
+                    let mouse_mode_changed = enabled != state.mouse_capture_active
+                        || next_sgr_pixels != host_sgr_pixels_active.load(Ordering::Acquire);
+                    if mouse_mode_changed {
+                        set_mouse_capture(enabled, next_sgr_pixels)
+                            .map_err(ClientError::ConnectionFailed)?;
                         #[cfg(windows)]
-                        if windows_vti_input_backend_enabled() {
+                        if enabled && windows_vti_input_backend_enabled() {
                             let _ = enable_windows_virtual_terminal_input();
                         }
-                        state.mouse_capture_active = desired;
-                        host_mouse_capture_active.store(desired, Ordering::Release);
                     }
+                    state.mouse_capture_active = enabled;
+                    host_mouse_capture_active.store(enabled, Ordering::Release);
+                    host_sgr_pixels_active.store(next_sgr_pixels, Ordering::Release);
                 }
                 ServerMessage::KittyKeyboardReportAll { enabled } => {
                     if enabled != state.keyboard_report_all_active {
@@ -1643,7 +1865,12 @@ async fn run_client_loop(
                     "server closed connection",
                 )));
             }
-            ClientLoopEvent::Timer => {}
+            ClientLoopEvent::Timer => {
+                #[cfg(unix)]
+                if let Ok(mut matcher) = state.direct_graphics_response.lock() {
+                    matcher.expire();
+                }
+            }
         }
     }
 
@@ -2204,8 +2431,10 @@ fn ioctl_cell_size() -> Option<(u32, u32)> {
 }
 
 /// Cell size used when the ioctl reports no pixels.
-fn cell_size_fallback(reported: u64) -> (u32, u32) {
-    unpack_cell_size(reported).unwrap_or((DEFAULT_CELL_WIDTH_PX, DEFAULT_CELL_HEIGHT_PX))
+fn cell_size_fallback(reported: u64, last: Option<(u32, u32)>) -> (u32, u32) {
+    unpack_cell_size(reported)
+        .or(last.filter(|(width, height)| *width > 0 && *height > 0))
+        .unwrap_or((DEFAULT_CELL_WIDTH_PX, DEFAULT_CELL_HEIGHT_PX))
 }
 
 #[cfg(any(unix, test))]
@@ -2222,20 +2451,35 @@ fn unpack_cell_size(packed: u64) -> Option<(u32, u32)> {
 fn current_terminal_geometry(
     kitty_graphics_enabled: bool,
     reported_cell_size: &AtomicU64,
+    last_cell_size: Option<(u32, u32)>,
 ) -> (u16, u16, u32, u32) {
     let (cols, rows) = crossterm::terminal::size().unwrap_or((80, 24));
     if !kitty_graphics_enabled {
         return (cols, rows, 0, 0);
     }
-    let (cell_width_px, cell_height_px) = ioctl_cell_size()
-        .unwrap_or_else(|| cell_size_fallback(reported_cell_size.load(Ordering::Acquire)));
+    let (cell_width_px, cell_height_px) = ioctl_cell_size().unwrap_or_else(|| {
+        cell_size_fallback(reported_cell_size.load(Ordering::Acquire), last_cell_size)
+    });
     (cols, rows, cell_width_px, cell_height_px)
 }
 
-/// Reads the terminal geometry before the handshake, before any host cell
-/// size report can exist.
-fn initial_terminal_geometry(kitty_graphics_enabled: bool) -> (u16, u16, u32, u32) {
-    current_terminal_geometry(kitty_graphics_enabled, &AtomicU64::new(0))
+/// Reads terminal geometry before the handshake. Direct graphics is eligible
+/// only when the host supplied exact pixel dimensions through the ioctl.
+fn initial_terminal_geometry(kitty_graphics_enabled: bool) -> (u16, u16, u32, u32, bool) {
+    let (cols, rows) = crossterm::terminal::size().unwrap_or((80, 24));
+    if !kitty_graphics_enabled {
+        return (cols, rows, 0, 0, false);
+    }
+    match ioctl_cell_size() {
+        Some((width, height)) => (cols, rows, width, height, true),
+        None => (
+            cols,
+            rows,
+            DEFAULT_CELL_WIDTH_PX,
+            DEFAULT_CELL_HEIGHT_PX,
+            false,
+        ),
+    }
 }
 
 /// Reports polled changes and signalled resizes that return to the same size.
@@ -2272,7 +2516,11 @@ fn resize_poll_loop(
     while !should_quit.load(Ordering::Acquire) {
         std::thread::sleep(Duration::from_millis(100));
         let signalled = crate::platform::take_terminal_resize_signal();
-        let new_size = current_terminal_geometry(kitty_graphics_enabled, reported_cell_size);
+        let new_size = current_terminal_geometry(
+            kitty_graphics_enabled,
+            reported_cell_size,
+            Some((last_size.2, last_size.3)),
+        );
         if resize_report_required(signalled, new_size, last_size) {
             last_size = new_size;
             if resize_tx
@@ -2388,6 +2636,40 @@ mod tests {
         assert!(resize_report_required(false, (120, 40, 9, 18), size));
     }
 
+    #[test]
+    fn approximate_cell_size_never_enables_direct_graphics() {
+        assert_eq!(
+            client_launch_mode(false, false, 8, 16),
+            ClientLaunchMode::App
+        );
+        assert_eq!(
+            client_launch_mode(true, false, 8, 16),
+            ClientLaunchMode::TerminalAttach
+        );
+    }
+
+    #[test]
+    fn direct_graphics_profile_is_narrow_and_transport_safe() {
+        for (program, term, kitty, expected) in [
+            ("ghostty", "", false, true),
+            ("WezTerm", "", false, true),
+            ("", "xterm-kitty", false, true),
+            ("", "xterm-256color", true, true),
+            ("", "xterm-256color", false, false),
+        ] {
+            assert_eq!(
+                direct_graphics_profile_values(program, term, kitty, false, true),
+                expected
+            );
+        }
+        assert!(!direct_graphics_profile_values(
+            "ghostty", "", false, true, true
+        ));
+        assert!(!direct_graphics_profile_values(
+            "ghostty", "", false, false, false
+        ));
+    }
+
     fn restore_env_var(key: &str, value: Option<OsString>) {
         if let Some(value) = value {
             std::env::set_var(key, value);
@@ -2758,11 +3040,15 @@ mod tests {
     }
 
     #[test]
-    fn cell_size_fallback_prefers_reported_size_over_default() {
-        assert_eq!(cell_size_fallback(0), (8, 16));
-        assert_eq!(cell_size_fallback(pack_cell_size(10, 21)), (10, 21));
-        assert_eq!(cell_size_fallback(pack_cell_size(10, 0)), (8, 16));
-        assert_eq!(cell_size_fallback(pack_cell_size(0, 21)), (8, 16));
+    fn cell_size_fallback_prefers_reported_then_previous_size() {
+        assert_eq!(cell_size_fallback(0, None), (8, 16));
+        assert_eq!(cell_size_fallback(0, Some((11, 22))), (11, 22));
+        assert_eq!(
+            cell_size_fallback(pack_cell_size(10, 21), Some((11, 22))),
+            (10, 21)
+        );
+        assert_eq!(cell_size_fallback(pack_cell_size(10, 0), None), (8, 16));
+        assert_eq!(cell_size_fallback(pack_cell_size(0, 21), None), (8, 16));
     }
 
     #[test]