aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorCan Celik <ogulcancelik@gmail.com>2026-08-09 02:04:39 +0300
committerGitHub <noreply@github.com>2026-08-09 02:04:39 +0300
commit1777e9bba32b953ed1ad203b4a16d01105539000 (patch)
tree22096973812c9baba822453d8470bdcfe64fc2a6 /src
parent36a8d36a7258e08b40c01bf521b1f4006525f143 (diff)
feat(graphics): add direct pane frame streaming (#2523)
* feat(graphics): add direct pane frame streaming * feat(graphics): expose pane visibility * test(graphics): preserve legacy terminal image output * fix(graphics): harden streaming lifecycle * fix(input): preserve mouse releases outside panes
Diffstat (limited to 'src')
-rw-r--r--src/api/mod.rs3
-rw-r--r--src/api/schema.rs3
-rw-r--r--src/api/schema/panes.rs30
-rw-r--r--src/api/schema/response.rs21
-rw-r--r--src/api/server.rs59
-rw-r--r--src/api/server/pane_graphics_stream.rs423
-rw-r--r--src/app/actions.rs34
-rw-r--r--src/app/api.rs3
-rw-r--r--src/app/api/pane_graphics.rs1512
-rw-r--r--src/app/api/plugins/mod.rs4
-rw-r--r--src/app/ids.rs2
-rw-r--r--src/app/input/mod.rs14
-rw-r--r--src/app/input/mouse.rs152
-rw-r--r--src/app/mod.rs61
-rw-r--r--src/app/pane_graphics.rs347
-rw-r--r--src/app/runtime.rs8
-rw-r--r--src/app/state.rs54
-rw-r--r--src/client/direct_graphics.rs393
-rw-r--r--src/client/input.rs193
-rw-r--r--src/client/mod.rs354
-rw-r--r--src/ghostty/mod.rs4
-rw-r--r--src/input/encode.rs4
-rw-r--r--src/input/mod.rs1
-rw-r--r--src/input/model.rs1
-rw-r--r--src/input/mouse.rs194
-rw-r--r--src/kitty_graphics.rs1543
-rw-r--r--src/main.rs1
-rw-r--r--src/pane.rs29
-rw-r--r--src/pane/input.rs38
-rw-r--r--src/pane/terminal.rs140
-rw-r--r--src/pane_graphics_files.rs529
-rw-r--r--src/protocol/wire.rs81
-rw-r--r--src/server/alt_screen_read.rs6
-rw-r--r--src/server/client_transport.rs381
-rw-r--r--src/server/clients.rs24
-rw-r--r--src/server/headless.rs497
-rw-r--r--src/server/headless/pane_graphics.rs535
-rw-r--r--src/server/headless/tests/pane_graphics.rs725
-rw-r--r--src/terminal/runtime.rs23
39 files changed, 6579 insertions, 1847 deletions
diff --git a/src/api/mod.rs b/src/api/mod.rs
index 27e98d2b..a44899da 100644
--- a/src/api/mod.rs
+++ b/src/api/mod.rs
@@ -7,7 +7,6 @@ mod subscriptions;
mod wait;
pub use event_hub::EventHub;
-pub(crate) use server::cancel_inactive_pane_graphics_streams;
pub use server::{start_server, start_server_with_capabilities, ServerHandle};
pub use status::{read_runtime_status_at, RuntimeStatus};
@@ -62,6 +61,7 @@ pub(crate) fn request_changes_ui(request: &Request) -> bool {
| Method::PaneGraphicsClear(_)
| Method::PaneGraphicsStream(_)
| Method::PaneGraphicsStreamSet(_)
+ | Method::PaneGraphicsStreamDirect(_)
| Method::PaneGraphicsStreamOpen(_)
| Method::PaneGraphicsStreamClose(_)
| Method::PaneReportAgent(_)
@@ -84,6 +84,7 @@ pub struct ApiRequestMessage {
pub request: Request,
pub respond_to: std::sync::mpsc::Sender<String>,
pub response_write_complete: Option<std::sync::mpsc::Receiver<()>>,
+ pub stream_active: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
}
pub type ApiRequestSender = mpsc::UnboundedSender<ApiRequestMessage>;
diff --git a/src/api/schema.rs b/src/api/schema.rs
index 447ea42c..6f76e3d3 100644
--- a/src/api/schema.rs
+++ b/src/api/schema.rs
@@ -187,6 +187,9 @@ pub enum Method {
PaneGraphicsStreamSet(PaneGraphicsSetParams),
#[serde(skip)]
#[schemars(skip)]
+ PaneGraphicsStreamDirect(PaneGraphicsDirectParams),
+ #[serde(skip)]
+ #[schemars(skip)]
PaneGraphicsStreamOpen(PaneGraphicsStreamParams),
#[serde(skip)]
#[schemars(skip)]
diff --git a/src/api/schema/panes.rs b/src/api/schema/panes.rs
index 2dffd7f9..c8d89c48 100644
--- a/src/api/schema/panes.rs
+++ b/src/api/schema/panes.rs
@@ -4,6 +4,10 @@ use serde::{Deserialize, Serialize};
pub(crate) const PANE_GRAPHICS_SET_MAX_BYTES: usize = 512 * 1024;
pub(crate) const PANE_GRAPHICS_STREAM_MAX_BYTES: usize = 16 * 1024 * 1024;
+pub(crate) const PANE_GRAPHICS_MAX_LAYERS_PER_PANE: usize = 16;
+pub(crate) const PANE_GRAPHICS_MAX_LAYERS_TOTAL: usize = 64;
+pub(crate) const PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL: usize = 64 * 1024 * 1024;
+pub(crate) const PANE_GRAPHICS_PRIMARY_LAYER_ID: &str = "primary";
use super::agents::AgentSessionInfo;
use super::common::{AgentStatus, PaneAgentState, ReadFormat, ReadSource, SplitDirection};
@@ -287,11 +291,16 @@ pub enum PaneGraphicsFormat {
Png,
Rgb,
Rgba,
+ Bgra,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct PaneGraphicsSetParams {
pub pane_id: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub layer_id: Option<String>,
+ #[serde(default)]
+ pub z_index: i32,
#[serde(skip)]
#[schemars(skip)]
pub owner: String,
@@ -323,11 +332,32 @@ pub struct PaneGraphicsPlacementParams {
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct PaneGraphicsClearParams {
pub pane_id: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub layer_id: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PaneGraphicsDirectParams {
+ pub pane_id: String,
+ pub layer_id: Option<String>,
+ pub z_index: i32,
+ pub owner: String,
+ pub image_width: u32,
+ pub image_height: u32,
+ pub format: PaneGraphicsFormat,
+ pub path: String,
+ pub sequence: u64,
+ pub revision: u64,
+ pub placement: PaneGraphicsPlacementParams,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct PaneGraphicsStreamParams {
pub pane_id: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub layer_id: Option<String>,
+ #[serde(default)]
+ pub z_index: i32,
#[serde(skip)]
#[schemars(skip)]
pub owner: String,
diff --git a/src/api/schema/response.rs b/src/api/schema/response.rs
index db8cfc7b..ae5a32e0 100644
--- a/src/api/schema/response.rs
+++ b/src/api/schema/response.rs
@@ -162,9 +162,30 @@ pub enum ResponseResult {
PaneRead {
read: PaneReadResult,
},
+ PaneGraphicsFrameAck {
+ sequence: u64,
+ revision: u64,
+ },
PaneGraphicsInfo {
cell_width_px: u32,
cell_height_px: u32,
+ /// True only when this pane is on the currently rendered terminal surface.
+ pane_visible: bool,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ file_frame_directory: Option<String>,
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ file_frame_formats: Vec<String>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ file_frame_max_bytes: Option<usize>,
+ /// Accepts damage metadata while still consuming a complete canonical file.
+ #[serde(default)]
+ file_frame_damage: bool,
+ #[serde(default)]
+ max_layers_per_pane: usize,
+ #[serde(default)]
+ pixel_mouse: bool,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ file_frame_transport: Option<String>,
},
AgentExplain {
explain: serde_json::Value,
diff --git a/src/api/server.rs b/src/api/server.rs
index 3932b1ea..e8b73167 100644
--- a/src/api/server.rs
+++ b/src/api/server.rs
@@ -23,7 +23,6 @@ use crate::ipc::{
};
mod pane_graphics_stream;
-pub(crate) use pane_graphics_stream::cancel_inactive_streams as cancel_inactive_pane_graphics_streams;
const SOCKET_PERMISSION_MODE: u32 = 0o600;
pub(super) const CONNECTION_POLL_INTERVAL: Duration = Duration::from_millis(100);
@@ -332,12 +331,7 @@ fn handle_request(
r#"{"id":"","error":{"code":"internal_error","message":"failed to encode response"}}"#
.to_string()
}),
- _ => dispatch_to_app_with_timeout_and_write_completion(
- request,
- api_tx,
- None,
- response_write_complete,
- ),
+ _ => dispatch_to_app(request, api_tx, None, response_write_complete, None),
}
}
@@ -413,6 +407,7 @@ fn api_method_name(method: &Method) -> &'static str {
Method::PaneGraphicsInfo(_) => "pane.graphics.info",
Method::PaneGraphicsStream(_) => "pane.graphics.stream",
Method::PaneGraphicsStreamSet(_) => "pane.graphics.stream.set",
+ Method::PaneGraphicsStreamDirect(_) => "pane.graphics.stream.direct",
Method::PaneGraphicsStreamOpen(_) => "pane.graphics.stream.open",
Method::PaneGraphicsStreamClose(_) => "pane.graphics.stream.close",
Method::PaneReportAgent(_) => "pane.report_agent",
@@ -749,22 +744,51 @@ pub(super) fn dispatch_to_app_with_timeout(
api_tx: &ApiRequestSender,
timeout: Option<Duration>,
) -> String {
- dispatch_to_app_with_timeout_and_write_completion(request, api_tx, timeout, None)
+ dispatch_to_app(request, api_tx, timeout, None, None)
+}
+
+pub(super) fn dispatch_stream_open(
+ request: Request,
+ api_tx: &ApiRequestSender,
+ timeout: Duration,
+ active: Arc<AtomicBool>,
+) -> String {
+ dispatch_to_app(request, api_tx, Some(timeout), None, Some(active))
}
-fn dispatch_to_app_with_timeout_and_write_completion(
+pub(super) fn dispatch_stream_frame(
+ request: Request,
+ api_tx: &ApiRequestSender,
+ active: Arc<AtomicBool>,
+) -> String {
+ dispatch_to_app(
+ request,
+ api_tx,
+ Some(crate::app::pane_graphics::DIRECT_OUTER_TIMEOUT),
+ None,
+ Some(active),
+ )
+}
+
+fn dispatch_to_app(
request: Request,
api_tx: &ApiRequestSender,
timeout: Option<Duration>,
response_write_complete: Option<std::sync::mpsc::Receiver<()>>,
+ stream_active: Option<Arc<AtomicBool>>,
) -> String {
let request_id = request.id.clone();
+ let request_active = stream_active.clone();
let (respond_to, response_rx) = std::sync::mpsc::channel();
if let Err(err) = api_tx.send(ApiRequestMessage {
request,
respond_to,
response_write_complete,
+ stream_active,
}) {
+ if let Some(active) = request_active {
+ active.store(false, Ordering::Release);
+ }
return error_response_json(
request_id,
"server_unavailable",
@@ -793,11 +817,16 @@ fn dispatch_to_app_with_timeout_and_write_completion(
match response {
Ok(response) => response,
- Err(err) => error_response_json(
- request_id,
- "server_unavailable",
- format!("request handling failed: {err}"),
- ),
+ Err(err) => {
+ if let Some(active) = request_active {
+ active.store(false, Ordering::Release);
+ }
+ error_response_json(
+ request_id,
+ "server_unavailable",
+ format!("request handling failed: {err}"),
+ )
+ }
}
}
@@ -1335,6 +1364,8 @@ mod pane_graphics_request_tests {
id: "graphics-max".into(),
method: Method::PaneGraphicsSet(crate::api::schema::PaneGraphicsSetParams {
pane_id: "pane_1".into(),
+ layer_id: None,
+ z_index: 0,
owner: String::new(),
format: crate::api::schema::PaneGraphicsFormat::Png,
image_width: 1,
diff --git a/src/api/server/pane_graphics_stream.rs b/src/api/server/pane_graphics_stream.rs
index 747c0205..b6420f95 100644
--- a/src/api/server/pane_graphics_stream.rs
+++ b/src/api/server/pane_graphics_stream.rs
@@ -1,7 +1,6 @@
-use std::collections::HashMap;
use std::io::{self, Read};
-use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
-use std::sync::{Arc, Mutex, OnceLock, Weak};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::Arc;
use std::time::{Duration, Instant};
use interprocess::local_socket::traits::Stream as _;
@@ -14,9 +13,9 @@ use crate::api::ApiRequestSender;
use crate::ipc::{is_connection_closed_error, LocalStream};
use super::{
- api_response_outcome, dispatch_to_app_with_timeout, write_json_line,
- write_json_line_allow_disconnect, write_text_line_allow_disconnect, APP_RESPONSE_TIMEOUT,
- CONNECTION_POLL_INTERVAL,
+ api_response_outcome, dispatch_stream_frame, dispatch_stream_open,
+ dispatch_to_app_with_timeout, write_json_line, write_json_line_allow_disconnect,
+ write_text_line_allow_disconnect, APP_RESPONSE_TIMEOUT, CONNECTION_POLL_INTERVAL,
};
const MAX_STREAM_FRAME_HEADER_BYTES: usize = 64 * 1024;
@@ -28,15 +27,25 @@ const STREAM_FRAME_BODY_TIMEOUT: Duration = Duration::from_secs(30);
const STREAM_FALLBACK_POLL_INTERVAL: Duration = Duration::from_millis(1);
const STREAM_FALLBACK_FAST_POLLS: u8 = 32;
static NEXT_PANE_GRAPHICS_STREAM_OWNER: AtomicU64 = AtomicU64::new(1);
-static REGISTERED_STREAM_COUNT: AtomicUsize = AtomicUsize::new(0);
-static REGISTERED_STREAMS: OnceLock<Mutex<HashMap<String, Weak<AtomicBool>>>> = OnceLock::new();
+
+#[derive(serde::Deserialize)]
+struct FrameFile {
+ path: String,
+}
#[derive(serde::Deserialize)]
struct FrameHeader {
format: crate::api::schema::PaneGraphicsFormat,
image_width: u32,
image_height: u32,
- data_length: usize,
+ #[serde(default)]
+ data_length: Option<usize>,
+ #[serde(default)]
+ file: Option<FrameFile>,
+ #[serde(default)]
+ sequence: u64,
+ #[serde(default)]
+ revision: u64,
#[serde(default)]
placement: crate::api::schema::PaneGraphicsPlacementParams,
}
@@ -102,19 +111,24 @@ fn serve_with_timeouts(
read_timeouts: ReadTimeouts,
) -> std::io::Result<()> {
let pane_id = params.pane_id.clone();
+ let layer_id = params.layer_id.clone();
+ let z_index = params.z_index;
let owner = next_owner();
params.owner = owner.clone();
- let open_response = dispatch_to_app_with_timeout(
+ let stream_active = Arc::new(AtomicBool::new(true));
+ let open_response = dispatch_stream_open(
Request {
id: request_id.clone(),
method: Method::PaneGraphicsStreamOpen(params),
},
api_tx,
- Some(open_timeout),
+ open_timeout,
+ Arc::clone(&stream_active),
);
if api_response_outcome(&open_response) != "ok" {
+ stream_active.store(false, Ordering::Release);
let write_result = write_text_line_allow_disconnect(&mut stream, &open_response);
- clear_layer(&pane_id, &owner, api_tx);
+ clear_layer(&pane_id, layer_id.as_deref(), z_index, &owner, api_tx);
write_result?;
return Ok(());
}
@@ -126,27 +140,28 @@ fn serve_with_timeouts(
result: ResponseResult::Ok {},
},
) {
- clear_layer(&pane_id, &owner, api_tx);
+ stream_active.store(false, Ordering::Release);
+ clear_layer(&pane_id, layer_id.as_deref(), z_index, &owner, api_tx);
if is_connection_closed_error(&err) {
return Ok(());
}
return Err(err);
}
- let stream_active = Arc::new(AtomicBool::new(true));
- register_stream(&owner, &stream_active);
let result = serve_frames(
&mut stream,
&request_id,
&owner,
&pane_id,
+ layer_id.as_deref(),
+ z_index,
api_tx,
running,
&stream_active,
read_timeouts,
);
- unregister_stream(&owner);
- clear_layer(&pane_id, &owner, api_tx);
+ stream_active.store(false, Ordering::Release);
+ clear_layer(&pane_id, layer_id.as_deref(), z_index, &owner, api_tx);
result
}
@@ -155,6 +170,8 @@ fn serve_frames(
request_id: &str,
owner: &str,
pane_id: &str,
+ layer_id: Option<&str>,
+ z_index: i32,
api_tx: &ApiRequestSender,
running: &Arc<AtomicBool>,
stream_active: &Arc<AtomicBool>,
@@ -193,7 +210,67 @@ fn serve_frames(
return Ok(());
}
};
- if header.data_length == 0 {
+ if let Some(file) = header.file {
+ if !matches!(
+ header.format,
+ crate::api::schema::PaneGraphicsFormat::Rgba
+ | crate::api::schema::PaneGraphicsFormat::Bgra
+ ) {
+ write_json_line_allow_disconnect(
+ stream,
+ &ErrorResponse {
+ id: request_id.to_string(),
+ error: ErrorBody {
+ code: "invalid_frame".into(),
+ message: "file frames require rgba or bgra".into(),
+ },
+ },
+ )?;
+ return Ok(());
+ }
+ let response = dispatch_stream_frame(
+ Request {
+ id: format!("{request_id}:file:{}", header.sequence),
+ method: Method::PaneGraphicsStreamDirect(
+ crate::api::schema::PaneGraphicsDirectParams {
+ pane_id: pane_id.to_owned(),
+ layer_id: layer_id.map(str::to_owned),
+ z_index,
+ owner: owner.to_owned(),
+ image_width: header.image_width,
+ image_height: header.image_height,
+ format: header.format,
+ path: file.path,
+ sequence: header.sequence,
+ revision: header.revision,
+ placement: header.placement,
+ },
+ ),
+ },
+ api_tx,
+ Arc::clone(stream_active),
+ );
+ write_text_line_allow_disconnect(stream, &response)?;
+ if api_response_outcome(&response) != "ok" {
+ return Ok(());
+ }
+ continue;
+ }
+
+ let Some(data_length) = header.data_length else {
+ write_json_line_allow_disconnect(
+ stream,
+ &ErrorResponse {
+ id: request_id.to_string(),
+ error: ErrorBody {
+ code: "invalid_frame".into(),
+ message: "frame requires data_length or file".into(),
+ },
+ },
+ )?;
+ return Ok(());
+ };
+ if data_length == 0 {
write_json_line_allow_disconnect(
stream,
&ErrorResponse {
@@ -206,7 +283,7 @@ fn serve_frames(
)?;
return Ok(());
}
- if header.data_length > crate::api::schema::PANE_GRAPHICS_STREAM_MAX_BYTES {
+ if data_length > crate::api::schema::PANE_GRAPHICS_STREAM_MAX_BYTES {
write_json_line_allow_disconnect(
stream,
&ErrorResponse {
@@ -222,7 +299,7 @@ fn serve_frames(
let Some(data) = read_exact(
stream,
- header.data_length,
+ data_length,
running,
stream_active,
timeouts.body_idle,
@@ -239,6 +316,8 @@ fn serve_frames(
id: frame_id,
method: Method::PaneGraphicsStreamSet(PaneGraphicsSetParams {
pane_id: pane_id.to_string(),
+ layer_id: layer_id.map(str::to_owned),
+ z_index,
owner: owner.to_string(),
format: header.format,
image_width: header.image_width,
@@ -260,52 +339,6 @@ fn serve_frames(
Ok(())
}
-fn stream_registry() -> &'static Mutex<HashMap<String, Weak<AtomicBool>>> {
- REGISTERED_STREAMS.get_or_init(|| Mutex::new(HashMap::new()))
-}
-
-fn register_stream(owner: &str, active: &Arc<AtomicBool>) {
- let Ok(mut streams) = stream_registry().lock() else {
- active.store(false, Ordering::Release);
- return;
- };
- if streams
- .insert(owner.to_string(), Arc::downgrade(active))
- .is_none()
- {
- REGISTERED_STREAM_COUNT.fetch_add(1, Ordering::Release);
- }
-}
-
-fn unregister_stream(owner: &str) {
- let Ok(mut streams) = stream_registry().lock() else {
- return;
- };
- if streams.remove(owner).is_some() {
- REGISTERED_STREAM_COUNT.fetch_sub(1, Ordering::Release);
- }
-}
-
-pub(crate) fn cancel_inactive_streams(mut is_active: impl FnMut(&str) -> bool) {
- if REGISTERED_STREAM_COUNT.load(Ordering::Acquire) == 0 {
- return;
- }
- let Ok(mut streams) = stream_registry().lock() else {
- return;
- };
- let before = streams.len();
- streams.retain(|owner, active| {
- let keep = is_active(owner);
- if !keep {
- if let Some(active) = active.upgrade() {
- active.store(false, Ordering::Release);
- }
- }
- keep
- });
- REGISTERED_STREAM_COUNT.fetch_sub(before.saturating_sub(streams.len()), Ordering::Release);
-}
-
fn stream_is_running(running: &AtomicBool, stream_active: &AtomicBool) -> bool {
running.load(Ordering::Relaxed) && stream_active.load(Ordering::Acquire)
}
@@ -315,12 +348,20 @@ fn next_owner() -> String {
format!("pane.graphics.stream:{}:{id}", std::process::id())
}
-fn clear_layer(pane_id: &str, owner: &str, api_tx: &ApiRequestSender) {
+fn clear_layer(
+ pane_id: &str,
+ layer_id: Option<&str>,
+ z_index: i32,
+ owner: &str,
+ api_tx: &ApiRequestSender,
+) {
let _response = dispatch_to_app_with_timeout(
Request {
id: format!("pane.graphics.stream.clear:{pane_id}"),
method: Method::PaneGraphicsStreamClose(PaneGraphicsStreamParams {
pane_id: pane_id.to_string(),
+ layer_id: layer_id.map(str::to_owned),
+ z_index,
owner: owner.to_string(),
}),
},
@@ -574,7 +615,9 @@ fn read_should_retry(err: &io::Error) -> bool {
mod tests {
use super::*;
use crate::api::schema::{ErrorResponse, Method, ResponseResult, SuccessResponse};
- use crate::api::{ApiRequestMessage, EventHub};
+ use crate::api::ApiRequestMessage;
+ #[cfg(unix)]
+ use crate::api::EventHub;
use crate::ipc::LocalStream;
use interprocess::local_socket::traits::Listener as _;
use std::io::{BufRead, BufReader, Write};
@@ -610,6 +653,42 @@ mod tests {
assert!(owner.starts_with("pane.graphics.stream:"));
}
+ fn respond_ok(message: ApiRequestMessage) {
+ let response = serde_json::to_string(&SuccessResponse {
+ id: message.request.id,
+ result: ResponseResult::Ok {},
+ })
+ .unwrap();
+ message.respond_to.send(response).unwrap();
+ }
+
+ fn assert_close_and_respond(message: ApiRequestMessage, pane_id: &str, owner: &str) {
+ match &message.request.method {
+ Method::PaneGraphicsStreamClose(params) => {
+ assert_eq!(params.pane_id, pane_id);
+ assert_eq!(params.owner, owner);
+ }
+ other => panic!("unexpected close request: {other:?}"),
+ }
+ respond_ok(message);
+ }
+
+ #[test]
+ fn browser_file_header_accepts_damage_but_keeps_full_canonical_frame() {
+ let header: FrameHeader = serde_json::from_str(
+ r#"{"format":"rgba","image_width":2,"image_height":3,"sequence":7,"revision":8,"file":{"path":"/private/frame"},"damage":{"x":1,"y":1,"width":1,"height":1},"transport":"direct-kitty","placement":{"grid_cols":2,"grid_rows":3}}"#,
+ )
+ .unwrap();
+ assert_eq!(header.data_length, None);
+ assert_eq!(header.file.unwrap().path, "/private/frame");
+ assert_eq!((header.sequence, header.revision), (7, 8));
+ assert_eq!((header.image_width, header.image_height), (2, 3));
+ assert_eq!(
+ (header.placement.grid_cols, header.placement.grid_rows),
+ (2, 3)
+ );
+ }
+
#[test]
fn timed_read_skips_reset_after_stream_ends() {
let mut reset_called = false;
@@ -650,15 +729,7 @@ mod tests {
}
other => panic!("unexpected open request: {other:?}"),
};
- open.respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: open.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
+ respond_ok(open);
let ack: SuccessResponse = serde_json::from_str(&read_response_line(&mut client)).unwrap();
assert_eq!(ack.id, "stream_1");
@@ -687,36 +758,11 @@ mod tests {
}
other => panic!("unexpected request: {other:?}"),
}
- msg.respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: msg.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
+ respond_ok(msg);
drop(client);
running.store(false, Ordering::Relaxed);
- let clear = api_rx.blocking_recv().unwrap();
- match &clear.request.method {
- Method::PaneGraphicsStreamClose(params) => {
- assert_eq!(params.pane_id, "pane_1");
- assert_eq!(params.owner, stream_owner);
- }
- other => panic!("unexpected clear request: {other:?}"),
- }
- clear
- .respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: clear.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
+ assert_close_and_respond(api_rx.blocking_recv().unwrap(), "pane_1", &stream_owner);
assert!(server_thread.join().unwrap().is_ok());
}
@@ -761,24 +807,7 @@ mod tests {
assert_eq!(response.id, "stream_2");
assert_eq!(response.error.code, "feature_disabled");
- let close = api_rx.blocking_recv().unwrap();
- match &close.request.method {
- Method::PaneGraphicsStreamClose(params) => {
- assert_eq!(params.pane_id, "pane_1");
- assert_eq!(params.owner, stream_owner);
- }
- other => panic!("unexpected close request: {other:?}"),
- }
- close
- .respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: close.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
+ assert_close_and_respond(api_rx.blocking_recv().unwrap(), "pane_1", &stream_owner);
drop(client);
running.store(false, Ordering::Relaxed);
@@ -798,6 +827,8 @@ mod tests {
"stream_timeout".into(),
PaneGraphicsStreamParams {
pane_id: "pane_1".into(),
+ layer_id: None,
+ z_index: 0,
owner: String::new(),
},
&api_tx,
@@ -823,24 +854,7 @@ mod tests {
assert_eq!(response.error.code, "server_unavailable");
assert!(response.error.message.contains("timed out"));
- let close = api_rx.blocking_recv().unwrap();
- match &close.request.method {
- Method::PaneGraphicsStreamClose(params) => {
- assert_eq!(params.pane_id, "pane_1");
- assert_eq!(params.owner, stream_owner);
- }
- other => panic!("unexpected close request: {other:?}"),
- }
- close
- .respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: close.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
+ assert_close_and_respond(api_rx.blocking_recv().unwrap(), "pane_1", &stream_owner);
drop(open);
drop(client);
@@ -879,15 +893,7 @@ mod tests {
};
drop(client);
- open.respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: open.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
+ respond_ok(open);
let (close_tx, close_rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
@@ -897,23 +903,7 @@ mod tests {
.recv_timeout(Duration::from_secs(2))
.unwrap()
.unwrap();
- match &close.request.method {
- Method::PaneGraphicsStreamClose(params) => {
- assert_eq!(params.pane_id, "pane_1");
- assert_eq!(params.owner, stream_owner);
- }
- other => panic!("unexpected close request: {other:?}"),
- }
- close
- .respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: close.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
+ assert_close_and_respond(close, "pane_1", &stream_owner);
running.store(false, Ordering::Relaxed);
assert!(server_thread.join().unwrap().is_ok());
@@ -945,75 +935,6 @@ mod tests {
}
#[test]
- fn inactive_owner_cancels_idle_stream_and_dispatches_close() {
- let (mut client, server, _path) = local_stream_pair("graphics-owner-cancel");
- let (api_tx, mut api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>();
- let running = Arc::new(AtomicBool::new(true));
- let server_running = Arc::clone(&running);
- let server_thread = std::thread::spawn(move || {
- serve_with_timeouts(
- server,
- "stream-cancel".into(),
- PaneGraphicsStreamParams {
- pane_id: "pane_1".into(),
- owner: String::new(),
- },
- &api_tx,
- &server_running,
- Duration::from_secs(1),
- READ_TIMEOUTS,
- )
- });
-
- let open = api_rx.blocking_recv().unwrap();
- let owner = match &open.request.method {
- Method::PaneGraphicsStreamOpen(params) => params.owner.clone(),
- other => panic!("unexpected open request: {other:?}"),
- };
- open.respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: open.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
- let ack: SuccessResponse = serde_json::from_str(&read_response_line(&mut client)).unwrap();
- assert_eq!(ack.id, "stream-cancel");
-
- cancel_inactive_streams(|registered| registered != owner);
-
- let (close_tx, close_rx) = std::sync::mpsc::channel();
- std::thread::spawn(move || close_tx.send(api_rx.blocking_recv()).unwrap());
- let close = close_rx
- .recv_timeout(Duration::from_secs(1))
- .expect("canceled idle stream should dispatch a close")
- .expect("API request channel should remain open");
- match &close.request.method {
- Method::PaneGraphicsStreamClose(params) => {
- assert_eq!(params.pane_id, "pane_1");
- assert_eq!(params.owner, owner);
- }
- other => panic!("unexpected close request: {other:?}"),
- }
- close
- .respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: close.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
-
- drop(client);
- running.store(false, Ordering::Relaxed);
- assert!(server_thread.join().unwrap().is_ok());
- }
-
- #[test]
fn fallback_poll_backoff_preserves_fast_window_then_reaches_poll_ceiling() {
let mut backoff = PollBackoff::new();
for _ in 0..STREAM_FALLBACK_FAST_POLLS {
@@ -1102,6 +1023,8 @@ mod tests {
"stream-timeout".into(),
PaneGraphicsStreamParams {
pane_id: "pane_1".into(),
+ layer_id: None,
+ z_index: 0,
owner: String::new(),
},
&api_tx,
@@ -1121,15 +1044,7 @@ mod tests {
Method::PaneGraphicsStreamOpen(params) => params.owner.clone(),
other => panic!("unexpected open request: {other:?}"),
};
- open.respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: open.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
+ respond_ok(open);
let ack: SuccessResponse = serde_json::from_str(&read_response_line(&mut client)).unwrap();
assert_eq!(ack.id, "stream-timeout");
client.write_all(b"{").unwrap();
@@ -1141,23 +1056,7 @@ mod tests {
.recv_timeout(Duration::from_secs(1))
.unwrap()
.unwrap();
- match &close.request.method {
- Method::PaneGraphicsStreamClose(params) => {
- assert_eq!(params.pane_id, "pane_1");
- assert_eq!(params.owner, owner);
- }
- other => panic!("unexpected close request: {other:?}"),
- }
- close
- .respond_to
- .send(
- serde_json::to_string(&SuccessResponse {
- id: close.request.id,
- result: ResponseResult::Ok {},
- })
- .unwrap(),
- )
- .unwrap();
+ assert_close_and_respond(close, "pane_1", &owner);
let error = server_thread.join().unwrap().unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::TimedOut);
@@ -1176,6 +1075,8 @@ mod tests {
"stream-oversized",
"owner-1",
"pane_1",
+ None,
+ 0,
&api_tx,
&server_running,
&stream_active,
diff --git a/src/app/actions.rs b/src/app/actions.rs
index 40707cee..20deab2a 100644
--- a/src/app/actions.rs
+++ b/src/app/actions.rs
@@ -1662,8 +1662,6 @@ impl AppState {
}
for pane_id in pane_ids {
self.plugin_panes.remove(&pane_id);
- self.pane_graphics_layers.remove(&pane_id);
- self.pane_graphics_streams.remove(&pane_id);
}
}
@@ -3402,26 +3400,6 @@ mod tests {
state
}
- fn insert_test_pane_graphics_layer(state: &mut AppState, pane_id: PaneId) {
- state.pane_graphics_layers.insert(
- pane_id,
- crate::app::state::PaneGraphicsLayer::new(
- crate::api::schema::PaneGraphicsFormat::Rgba,
- 1,
- 1,
- vec![1, 2, 3, 4],
- crate::api::schema::PaneGraphicsPlacementParams::default(),
- ),
- );
- }
-
- fn insert_test_pane_graphics_state(state: &mut AppState, pane_id: PaneId) {
- insert_test_pane_graphics_layer(state, pane_id);
- state
- .pane_graphics_streams
- .insert(pane_id, "test-stream".into());
- }
-
fn mark_linked_worktree(state: &mut AppState, ws_idx: usize) {
state.workspaces[ws_idx].worktree_space = Some(crate::workspace::WorktreeSpaceMembership {
key: "repo-key".into(),
@@ -5834,13 +5812,9 @@ mod tests {
entrypoint: "board".into(),
},
);
- insert_test_pane_graphics_state(&mut state, closed);
-
state.close_pane();
assert_eq!(state.workspaces[0].panes.len(), 1);
assert!(!state.plugin_panes.contains_key(&closed));
- assert!(!state.pane_graphics_layers.contains_key(&closed));
- assert!(!state.pane_graphics_streams.contains_key(&closed));
state.assert_invariants_for_test();
}
@@ -5910,14 +5884,10 @@ mod tests {
entrypoint: "board".into(),
},
);
- insert_test_pane_graphics_state(&mut state, pane_id);
-
state.close_tab();
assert!(!state.terminals.contains_key(&terminal_id));
assert!(!state.plugin_panes.contains_key(&pane_id));
- assert!(!state.pane_graphics_layers.contains_key(&pane_id));
- assert!(!state.pane_graphics_streams.contains_key(&pane_id));
state.assert_invariants_for_test();
}
@@ -5933,14 +5903,10 @@ mod tests {
entrypoint: "board".into(),
},
);
- insert_test_pane_graphics_state(&mut state, pane_id);
-
state.close_selected_workspace();
assert!(!state.terminals.contains_key(&terminal_id));
assert!(!state.plugin_panes.contains_key(&pane_id));
- assert!(!state.pane_graphics_layers.contains_key(&pane_id));
- assert!(!state.pane_graphics_streams.contains_key(&pane_id));
state.assert_invariants_for_test();
}
diff --git a/src/app/api.rs b/src/app/api.rs
index 6dc40c3f..0ccd9b21 100644
--- a/src/app/api.rs
+++ b/src/app/api.rs
@@ -1102,6 +1102,9 @@ impl App {
Method::PaneGraphicsStreamSet(params) => {
return self.handle_pane_graphics_stream_set(request.id, params);
}
+ Method::PaneGraphicsStreamDirect(params) => {
+ return self.handle_pane_graphics_stream_direct(request.id, params);
+ }
Method::PaneGraphicsStreamOpen(params) => {
return self.handle_pane_graphics_stream_open(request.id, params);
}
diff --git a/src/app/api/pane_graphics.rs b/src/app/api/pane_graphics.rs
index 7851f69a..f5f5f2b7 100644
--- a/src/app/api/pane_graphics.rs
+++ b/src/app/api/pane_graphics.rs
@@ -2,121 +2,126 @@ use base64::Engine;
use crate::api::schema::{
PaneGraphicsClearParams, PaneGraphicsSetParams, PaneGraphicsStreamParams, ResponseResult,
- PANE_GRAPHICS_SET_MAX_BYTES, PANE_GRAPHICS_STREAM_MAX_BYTES,
+ PANE_GRAPHICS_MAX_LAYERS_PER_PANE, PANE_GRAPHICS_PRIMARY_LAYER_ID, PANE_GRAPHICS_SET_MAX_BYTES,
+ PANE_GRAPHICS_STREAM_MAX_BYTES,
};
-use crate::app::state::PaneGraphicsLayer;
+use crate::app::pane_graphics::{Key as PaneGraphicsKey, Layer, Slot};
use crate::app::App;
use crate::layout::PaneId;
use super::responses::{encode_error, encode_success};
impl App {
+ /// Whether graphics for a pane can be placed on the surface rendered by this client.
+ ///
+ /// This deliberately follows the pane layout rather than pane existence: inactive
+ /// workspaces/tabs and panes hidden by zoom are not placeable. Short-lived UI modes do not
+ /// suspend the producer because the pane becomes visible again without a layout event.
+ fn pane_graphics_visible(&self, ws_idx: usize, pane_id: PaneId) -> bool {
+ if self.state.active != Some(ws_idx) {
+ return false;
+ }
+ let Some(tab) = self.state.workspaces[ws_idx].active_tab() else {
+ return false;
+ };
+ if tab.zoomed {
+ tab.layout.focused() == pane_id
+ } else {
+ tab.layout.pane_ids().contains(&pane_id)
+ }
+ }
+
pub(super) fn handle_pane_graphics_info(
&mut self,
id: String,
target: crate::api::schema::PaneTarget,
) -> String {
- if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ if let Err(response) = require_enabled(self, &id) {
return response;
}
- if self.parse_pane_id(&target.pane_id).is_none() {
+ let Some((ws_idx, pane_id)) = self.parse_pane_id(&target.pane_id) else {
return pane_not_found(id, &target.pane_id);
- }
+ };
+ let pane_visible = self.pane_graphics_visible(ws_idx, pane_id);
if !self.state.host_cell_size.is_known() {
return encode_error(id, "cell_size_unavailable", "host cell size is unavailable");
}
+ let file_frame_directory = self
+ .direct_graphics_available
+ .then(|| self.pane_graphics_files.source_directory().ok())
+ .flatten();
+ let direct = file_frame_directory.is_some();
encode_success(
id,
ResponseResult::PaneGraphicsInfo {
cell_width_px: self.state.host_cell_size.width_px,
cell_height_px: self.state.host_cell_size.height_px,
+ pane_visible,
+ file_frame_directory: file_frame_directory
+ .map(|directory| directory.to_string_lossy().into_owned()),
+ file_frame_formats: if direct {
+ vec!["rgba".into(), "bgra".into()]
+ } else {
+ Vec::new()
+ },
+ file_frame_max_bytes: direct.then_some(PANE_GRAPHICS_STREAM_MAX_BYTES),
+ // Damage metadata never changes the complete canonical frame contract.
+ file_frame_damage: true,
+ max_layers_per_pane: PANE_GRAPHICS_MAX_LAYERS_PER_PANE,
+ pixel_mouse: self.pixel_mouse_available,
+ file_frame_transport: direct.then(|| "direct-kitty".into()),
},
)
}
+ pub(crate) fn attach_pane_graphics_stream_active(
+ &mut self,
+ params: &PaneGraphicsStreamParams,
+ active: std::sync::Arc<std::sync::atomic::AtomicBool>,
+ response: &str,
+ ) {
+ if serde_json::from_str::<crate::api::schema::SuccessResponse>(response).is_err() {
+ return;
+ }
+ let Some((_, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ return;
+ };
+ let Ok(key) = graphics_key(pane_id, params.layer_id.as_deref()) else {
+ return;
+ };
+ self.pane_graphics
+ .attach_stream_active(&key, &params.owner, active);
+ }
+
pub(super) fn handle_pane_graphics_set(
&mut self,
id: String,
params: PaneGraphicsSetParams,
) -> String {
- if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ if let Err(response) = require_enabled(self, &id) {
return response;
}
- let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ let Some((_, pane_id)) = self.parse_pane_id(&params.pane_id) else {
return pane_not_found(id, &params.pane_id);
};
- if self.state.pane_graphics_streams.contains_key(&pane_id) {
+ let key = match graphics_key(pane_id, params.layer_id.as_deref()) {
+ Ok(key) => key,
+ Err(message) => return encode_error(id, "invalid_layer_id", message),
+ };
+ self.reclaim_inactive_stream(&key);
+ if self
+ .pane_graphics
+ .slots
+ .get(&key)
+ .is_some_and(|slot| slot.stream_owner.is_some())
+ {
return encode_error(
id,
"stream_conflict",
- "pane already has an active graphics stream",
- );
- }
- self.set_pane_graphics_layer(id, pane_id, params)
- }
-
- fn set_pane_graphics_layer(
- &mut self,
- id: String,
- pane_id: PaneId,
- params: PaneGraphicsSetParams,
- ) -> String {
- if params.image_width == 0 || params.image_height == 0 {
- return encode_error(
- id,
- "invalid_image",
- "image_width and image_height must be greater than zero",
+ "pane graphics layer has an active stream",
);
}
- let max_bytes = if params.data.is_some() {
- PANE_GRAPHICS_STREAM_MAX_BYTES
- } else {
- PANE_GRAPHICS_SET_MAX_BYTES
- };
- let data = match params.data {
- Some(data) => data,
- None => match base64::engine::general_purpose::STANDARD.decode(params.data_base64) {
- Ok(data) => data,
- Err(_) => {
- return encode_error(id, "invalid_image", "data_base64 is not valid base64");
- }
- },
- };
- if data.is_empty() {
- return encode_error(id, "invalid_image", "image data must not be empty");
- }
- if data.len() > max_bytes {
- return encode_error(id, "image_too_large", "image data is too large");
- }
- match pane_graphics_expected_data_len(
- params.format,
- params.image_width,
- params.image_height,
- ) {
- Ok(Some(expected_len)) if data.len() != expected_len => {
- return encode_error(
- id,
- "invalid_image",
- "image data length does not match format and dimensions",
- );
- }
- Ok(_) => {}
- Err(()) => {
- return encode_error(id, "invalid_image", "image dimensions are too large");
- }
- }
-
- let layer = PaneGraphicsLayer::new(
- params.format,
- params.image_width,
- params.image_height,
- data,
- params.placement,
- );
- self.state.pane_graphics_layers.insert(pane_id, layer);
- self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1);
-
- encode_success(id, ResponseResult::Ok {})
+ self.set_layer(id, key, params)
}
pub(super) fn handle_pane_graphics_clear(
@@ -124,23 +129,32 @@ impl App {
id: String,
params: PaneGraphicsClearParams,
) -> String {
- if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ if let Err(response) = require_enabled(self, &id) {
return response;
}
- let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ let Some((_, pane_id)) = self.parse_pane_id(&params.pane_id) else {
return pane_not_found(id, &params.pane_id);
};
- if self.state.pane_graphics_streams.contains_key(&pane_id) {
+ let key = match graphics_key(pane_id, params.layer_id.as_deref()) {
+ Ok(key) => key,
+ Err(message) => return encode_error(id, "invalid_layer_id", message),
+ };
+ self.reclaim_inactive_stream(&key);
+ if self
+ .pane_graphics
+ .slots
+ .get(&key)
+ .is_some_and(|slot| slot.stream_owner.is_some())
+ {
return encode_error(
id,
"stream_conflict",
- "pane already has an active graphics stream",
+ "pane graphics layer has an active stream",
);
}
- if self.state.pane_graphics_layers.remove(&pane_id).is_some() {
- self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1);
+ if self.pane_graphics.slots.remove(&key).is_some() {
+ self.pane_graphics.mark_changed();
}
-
encode_success(id, ResponseResult::Ok {})
}
@@ -149,24 +163,32 @@ impl App {
id: String,
params: PaneGraphicsSetParams,
) -> String {
- if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ if let Err(response) = require_enabled(self, &id) {
return response;
}
- let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ let Some((_, pane_id)) = self.parse_pane_id(&params.pane_id) else {
return pane_not_found(id, &params.pane_id);
};
- match self.state.pane_graphics_streams.get(&pane_id) {
- Some(owner) if owner == &params.owner => {}
- Some(_) => {
- return encode_error(
- id,
- "stream_conflict",
- "pane graphics stream owner does not match active stream",
- );
+ let key = match graphics_key(pane_id, params.layer_id.as_deref()) {
+ Ok(key) => key,
+ Err(message) => return encode_error(id, "invalid_layer_id", message),
+ };
+ match self.pane_graphics.slots.get(&key) {
+ Some(slot)
+ if slot.stream_owner.as_ref() == Some(&params.owner) && slot.stream_is_active() =>
+ {
+ self.set_layer(id, key, params)
+ }
+ Some(slot) if slot.stream_owner.as_ref() == Some(&params.owner) => {
+ encode_error(id, "stream_closed", "pane graphics stream is not active")
}
- None => return encode_error(id, "stream_closed", "pane graphics stream is not active"),
+ Some(_) => encode_error(
+ id,
+ "stream_conflict",
+ "pane graphics stream owner does not match active stream",
+ ),
+ None => encode_error(id, "stream_closed", "pane graphics stream is not active"),
}
- self.set_pane_graphics_layer(id, pane_id, params)
}
pub(super) fn handle_pane_graphics_stream_open(
@@ -174,7 +196,7 @@ impl App {
id: String,
params: PaneGraphicsStreamParams,
) -> String {
- if let Err(response) = require_pane_graphics_enabled(self, &id) {
+ if let Err(response) = require_enabled(self, &id) {
return response;
}
if params.owner.is_empty() {
@@ -184,22 +206,57 @@ impl App {
"pane graphics stream owner is required",
);
}
- let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ let Some((_, pane_id)) = self.parse_pane_id(&params.pane_id) else {
return pane_not_found(id, &params.pane_id);
};
- if self.state.pane_graphics_streams.contains_key(&pane_id) {
+ let key = match graphics_key(pane_id, params.layer_id.as_deref()) {
+ Ok(key) => key,
+ Err(message) => return encode_error(id, "invalid_layer_id", message),
+ };
+ let stale = self
+ .pane_graphics
+ .slots
+ .get(&key)
+ .is_some_and(|slot| slot.stream_owner.is_some() && !slot.stream_is_active());
+ if stale {
+ self.pane_graphics.slots.remove(&key);
+ }
+ if self
+ .pane_graphics
+ .slots
+ .get(&key)
+ .and_then(|slot| slot.stream_owner.as_ref())
+ .is_some()
+ {
return encode_error(
id,
"stream_conflict",
- "pane already has an active graphics stream",
+ "pane graphics layer has an active stream",
);
}
- self.state
- .pane_graphics_streams
- .insert(pane_id, params.owner);
- self.state.pane_graphics_layers.remove(&pane_id);
- self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1);
-
+ let exists = self.pane_graphics.slots.contains_key(&key);
+ if !self.pane_graphics.can_add_slot(&key)
+ || (!exists
+ && self.pane_graphics.layer_count(pane_id) >= PANE_GRAPHICS_MAX_LAYERS_PER_PANE)
+ {
+ return encode_error(id, "layer_limit", "pane graphics layer limit reached");
+ }
+ let Some(host_image_id) = self.pane_graphics.reserve_image_id(&key) else {
+ return encode_error(id, "layer_limit", "pane graphics image ids are exhausted");
+ };
+ self.pane_graphics.slots.insert(
+ key,
+ Slot {
+ host_image_id,
+ layer: None,
+ stream_owner: Some(params.owner),
+ stream_active: Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new(
+ true,
+ ))),
+ direct_gate: None,
+ },
+ );
+ self.pane_graphics.mark_changed();
encode_success(id, ResponseResult::Ok {})
}
@@ -208,55 +265,286 @@ impl App {
id: String,
params: PaneGraphicsStreamParams,
) -> String {
- let Some((_ws_idx, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ let Some((_, pane_id)) = self.parse_pane_id(&params.pane_id) else {
return pane_not_found(id, &params.pane_id);
};
- if self
- .state
- .pane_graphics_streams
- .get(&pane_id)
- .is_some_and(|owner| owner == &params.owner)
+ if let Ok(key) = graphics_key(pane_id, params.layer_id.as_deref()) {
+ if self
+ .pane_graphics
+ .slots
+ .get(&key)
+ .and_then(|slot| slot.stream_owner.as_ref())
+ .is_some_and(|owner| owner == &params.owner)
+ {
+ self.pane_graphics.slots.remove(&key);
+ self.pane_graphics.mark_changed();
+ }
+ }
+ encode_success(id, ResponseResult::Ok {})
+ }
+
+ pub(super) fn handle_pane_graphics_stream_direct(
+ &mut self,
+ id: String,
+ params: crate::api::schema::PaneGraphicsDirectParams,
+ ) -> String {
+ if let Err(response) = require_enabled(self, &id) {
+ return response;
+ }
+ let Some((_, pane_id)) = self.parse_pane_id(&params.pane_id) else {
+ return pane_not_found(id, &params.pane_id);
+ };
+ let key = match graphics_key(pane_id, params.layer_id.as_deref()) {
+ Ok(key) => key,
+ Err(message) => return encode_error(id, "invalid_layer_id", message),
+ };
+ let owner_matches = self.pane_graphics.slots.get(&key).is_some_and(|slot| {
+ slot.stream_owner.as_deref() == Some(params.owner.as_str()) && slot.stream_is_active()
+ });
+ if !owner_matches {
+ return encode_error(id, "stream_closed", "pane graphics stream is not active");
+ }
+ if params.image_width == 0 || params.image_height == 0 {
+ return encode_error(
+ id,
+ "invalid_image",
+ "image_width and image_height must be greater than zero",
+ );
+ }
+ if !matches!(
+ params.format,
+ crate::api::schema::PaneGraphicsFormat::Rgba
+ | crate::api::schema::PaneGraphicsFormat::Bgra
+ ) {
+ return encode_error(id, "invalid_image", "direct frames require rgba or bgra");
+ }
+ let expected_len =
+ match expected_len(params.format, params.image_width, params.image_height) {
+ Ok(Some(len)) if len <= PANE_GRAPHICS_STREAM_MAX_BYTES => len,
+ _ => return encode_error(id, "invalid_image", "invalid direct RGBA dimensions"),
+ };
+ let lease = match self
+ .pane_graphics_files
+ .lease(std::path::Path::new(&params.path), expected_len)
{
- self.state.pane_graphics_streams.remove(&pane_id);
- self.state.pane_graphics_layers.remove(&pane_id);
- self.state.pane_graphics_revision = self.state.pane_graphics_revision.wrapping_add(1);
+ Ok(lease) => lease,
+ Err(err) => return encode_error(id, "invalid_frame_file", err.to_string()),
+ };
+ let primary = key.1 == PANE_GRAPHICS_PRIMARY_LAYER_ID;
+ let direct = self.direct_graphics_available
+ && primary
+ && params.format == crate::api::schema::PaneGraphicsFormat::Rgba;
+ if !direct && !self.pane_graphics.can_store_inline(&key, expected_len) {
+ return encode_error(
+ id,
+ "graphics_budget_exceeded",
+ "pane graphics inline memory limit reached",
+ );
+ }
+ let layer = if direct {
+ Layer::direct(
+ params.image_width,
+ params.image_height,
+ lease,
+ params.placement,
+ params.z_index,
+ )
+ } else {
+ let mut data = match lease.copy_rgba() {
+ Ok(data) => data,
+ Err(err) => return encode_error(id, "invalid_frame_file", err.to_string()),
+ };
+ canonicalize_bgra(params.format, &mut data);
+ Layer::inline(
+ crate::api::schema::PaneGraphicsFormat::Rgba,
+ params.image_width,
+ params.image_height,
+ data,
+ params.placement,
+ params.z_index,
+ )
+ };
+ let Some(slot) = self.pane_graphics.slots.get_mut(&key) else {
+ return encode_error(id, "stream_closed", "pane graphics stream is not active");
+ };
+ if !slot.stream_is_active() {
+ return encode_error(id, "stream_closed", "pane graphics stream is not active");
}
+ slot.layer = Some(layer);
+ slot.direct_gate = None;
+ self.pane_graphics.mark_changed();
+ encode_success(
+ id,
+ ResponseResult::PaneGraphicsFrameAck {
+ sequence: params.sequence,
+ revision: params.revision,
+ },
+ )
+ }
+ fn set_layer(
+ &mut self,
+ id: String,
+ key: PaneGraphicsKey,
+ params: PaneGraphicsSetParams,
+ ) -> String {
+ if !self.pane_graphics.can_add_slot(&key)
+ || (!self.pane_graphics.slots.contains_key(&key)
+ && self.pane_graphics.layer_count(key.0) >= PANE_GRAPHICS_MAX_LAYERS_PER_PANE)
+ {
+ return encode_error(id, "layer_limit", "pane graphics layer limit reached");
+ }
+ if params.image_width == 0 || params.image_height == 0 {
+ return encode_error(
+ id,
+ "invalid_image",
+ "image_width and image_height must be greater than zero",
+ );
+ }
+ let expected = match expected_len(params.format, params.image_width, params.image_height) {
+ Ok(expected) => expected,
+ Err(()) => return encode_error(id, "invalid_image", "image dimensions are too large"),
+ };
+ let streamed = params.data.is_some();
+ let mut data = match params.data {
+ Some(data) => data,
+ None => match base64::engine::general_purpose::STANDARD.decode(params.data_base64) {
+ Ok(data) => data,
+ Err(_) => {
+ return encode_error(id, "invalid_image", "data_base64 is not valid base64")
+ }
+ },
+ };
+ let limit = if streamed {
+ PANE_GRAPHICS_STREAM_MAX_BYTES
+ } else {
+ PANE_GRAPHICS_SET_MAX_BYTES
+ };
+ if data.is_empty() {
+ return encode_error(id, "invalid_image", "image data must not be empty");
+ }
+ if data.len() > limit {
+ return encode_error(id, "image_too_large", "image data is too large");
+ }
+ if expected.is_some_and(|size| size != data.len()) {
+ return encode_error(
+ id,
+ "invalid_image",
+ "image data does not match the frame contract",
+ );
+ }
+ let format = canonicalize_bgra(params.format, &mut data);
+ if !self.pane_graphics.can_store_inline(&key, data.len()) {
+ return encode_error(
+ id,
+ "graphics_budget_exceeded",
+ "pane graphics inline memory limit reached",
+ );
+ }
+ let Some(host_image_id) = self.pane_graphics.reserve_image_id(&key) else {
+ return encode_error(id, "layer_limit", "pane graphics image ids are exhausted");
+ };
+ // Move the liveness handle out before replacing the slot so Slot::drop cannot
+ // deactivate the stream that the replacement still owns.
+ let (stream_owner, stream_active) = self
+ .pane_graphics
+ .slots
+ .get_mut(&key)
+ .map(|slot| (slot.stream_owner.take(), slot.stream_active.take()))
+ .unwrap_or_default();
+ self.pane_graphics.slots.insert(
+ key,
+ Slot {
+ host_image_id,
+ layer: Some(Layer::inline(
+ format,
+ params.image_width,
+ params.image_height,
+ data,
+ params.placement,
+ params.z_index,
+ )),
+ stream_owner,
+ stream_active,
+ direct_gate: None,
+ },
+ );
+ self.pane_graphics.mark_changed();
encode_success(id, ResponseResult::Ok {})
}
+
+ fn reclaim_inactive_stream(&mut self, key: &PaneGraphicsKey) {
+ let stale = self
+ .pane_graphics
+ .slots
+ .get(key)
+ .is_some_and(|slot| slot.stream_owner.is_some() && !slot.stream_is_active());
+ if stale {
+ self.pane_graphics.slots.remove(key);
+ self.pane_graphics.mark_changed();
+ }
+ }
+}
+
+fn canonicalize_bgra(
+ format: crate::api::schema::PaneGraphicsFormat,
+ data: &mut [u8],
+) -> crate::api::schema::PaneGraphicsFormat {
+ if format != crate::api::schema::PaneGraphicsFormat::Bgra {
+ return format;
+ }
+ for pixel in data.chunks_exact_mut(4) {
+ pixel.swap(0, 2);
+ }
+ crate::api::schema::PaneGraphicsFormat::Rgba
}
-fn require_pane_graphics_enabled(app: &App, id: &str) -> Result<(), String> {
- if app.state.kitty_graphics_enabled {
- return Ok(());
+fn graphics_key(pane_id: PaneId, layer_id: Option<&str>) -> Result<PaneGraphicsKey, &'static str> {
+ let layer_id = layer_id.unwrap_or(PANE_GRAPHICS_PRIMARY_LAYER_ID);
+ if layer_id.is_empty() || layer_id.len() > 64 {
+ return Err("layer_id must contain between 1 and 64 characters");
}
- Err(encode_error(
- id.to_owned(),
- "feature_disabled",
- "pane graphics require experimental.kitty_graphics",
- ))
+ if !layer_id
+ .bytes()
+ .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-'))
+ {
+ return Err("layer_id contains unsupported characters");
+ }
+ Ok((pane_id, layer_id.to_owned()))
}
-fn pane_graphics_expected_data_len(
+fn expected_len(
format: crate::api::schema::PaneGraphicsFormat,
- image_width: u32,
- image_height: u32,
+ width: u32,
+ height: u32,
) -> Result<Option<usize>, ()> {
- let bytes_per_pixel = match format {
+ let bytes = match format {
crate::api::schema::PaneGraphicsFormat::Png => return Ok(None),
crate::api::schema::PaneGraphicsFormat::Rgb => 3_u64,
- crate::api::schema::PaneGraphicsFormat::Rgba => 4_u64,
+ crate::api::schema::PaneGraphicsFormat::Rgba
+ | crate::api::schema::PaneGraphicsFormat::Bgra => 4_u64,
};
- let pixels = u64::from(image_width)
- .checked_mul(u64::from(image_height))
- .ok_or(())?;
- pixels
- .checked_mul(bytes_per_pixel)
- .and_then(|bytes| usize::try_from(bytes).ok())
+ u64::from(width)
+ .checked_mul(u64::from(height))
+ .and_then(|pixels| pixels.checked_mul(bytes))
+ .and_then(|size| usize::try_from(size).ok())
.map(Some)
.ok_or(())
}
+fn require_enabled(app: &App, id: &str) -> Result<(), String> {
+ app.state
+ .kitty_graphics_enabled
+ .then_some(())
+ .ok_or_else(|| {
+ encode_error(
+ id.to_owned(),
+ "feature_disabled",
+ "pane graphics require experimental.kitty_graphics",
+ )
+ })
+}
+
fn pane_not_found(id: String, pane_id: &str) -> String {
encode_error(id, "pane_not_found", format!("pane {pane_id} not found"))
}
@@ -268,435 +556,667 @@ mod tests {
use crate::config::Config;
use crate::workspace::Workspace;
- fn app_with_test_workspace() -> (App, String) {
- let (_api_tx, api_rx) = tokio::sync::mpsc::unbounded_channel();
+ fn app() -> (App, String) {
+ let (_tx, rx) = tokio::sync::mpsc::unbounded_channel();
let mut app = App::new(
&Config::default(),
true,
None,
- api_rx,
+ rx,
crate::api::EventHub::default(),
);
- app.state.workspaces = vec![Workspace::test_new("pane-graphics")];
+ app.state.workspaces = vec![Workspace::test_new("graphics")];
app.state.ensure_test_terminals();
- let pane_id = app.state.workspaces[0].tabs[0].root_pane;
- let public_pane_id = app.public_pane_id(0, pane_id).unwrap();
- (app, public_pane_id)
+ let pane = app.state.workspaces[0].tabs[0].root_pane;
+ let public = app.public_pane_id(0, pane).unwrap();
+ app.state.kitty_graphics_enabled = true;
+ (app, public)
+ }
+
+ fn frame(
+ pane_id: String,
+ layer_id: Option<&str>,
+ owner: &str,
+ value: u8,
+ ) -> PaneGraphicsSetParams {
+ PaneGraphicsSetParams {
+ pane_id,
+ layer_id: layer_id.map(str::to_owned),
+ z_index: 2,
+ owner: owner.to_owned(),
+ format: crate::api::schema::PaneGraphicsFormat::Rgba,
+ image_width: 1,
+ image_height: 1,
+ data: Some(vec![value; 4]),
+ data_base64: String::new(),
+ placement: Default::default(),
+ }
+ }
+
+ fn pane_visible(response: &str) -> bool {
+ serde_json::from_str::<serde_json::Value>(response).unwrap()["result"]["pane_visible"]
+ .as_bool()
+ .unwrap()
}
#[test]
- fn api_pane_graphics_info_is_explicit_and_side_effect_free() {
- let (mut app, public_pane_id) = app_with_test_workspace();
- let request = |pane_id: String| crate::api::schema::Request {
- id: "graphics-info".into(),
- method: crate::api::schema::Method::PaneGraphicsInfo(crate::api::schema::PaneTarget {
- pane_id,
- }),
+ fn info_reports_visibility_for_terminal_surface_workspace_tab_and_zoom() {
+ let (mut app, pane_id) = app();
+ app.state.mode = crate::app::Mode::Terminal;
+ app.state.active = Some(0);
+ app.state.host_cell_size = crate::kitty_graphics::HostCellSize {
+ width_px: 10,
+ height_px: 20,
};
+ let visible = app.handle_pane_graphics_info(
+ "visible".into(),
+ crate::api::schema::PaneTarget {
+ pane_id: pane_id.clone(),
+ },
+ );
+ assert!(pane_visible(&visible));
- let response = app.handle_api_request(request(public_pane_id.clone()));
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "feature_disabled");
+ let hidden_workspace = Workspace::test_new("hidden");
+ let hidden_workspace_pane = hidden_workspace.tabs[0].root_pane;
+ app.state.workspaces.push(hidden_workspace);
+ let hidden_workspace_id = app.public_pane_id(1, hidden_workspace_pane).unwrap();
+ let hidden = app.handle_pane_graphics_info(
+ "hidden-workspace".into(),
+ crate::api::schema::PaneTarget {
+ pane_id: hidden_workspace_id,
+ },
+ );
+ assert!(!pane_visible(&hidden));
- app.state.kitty_graphics_enabled = true;
+ let inactive_tab = app.state.workspaces[0].test_add_tab(Some("inactive"));
+ let inactive_pane = app.state.workspaces[0].tabs[inactive_tab].root_pane;
+ let inactive_id = app.public_pane_id(0, inactive_pane).unwrap();
+ let hidden_tab = app.handle_pane_graphics_info(
+ "hidden-tab".into(),
+ crate::api::schema::PaneTarget {
+ pane_id: inactive_id,
+ },
+ );
+ assert!(!pane_visible(&hidden_tab));
+
+ app.state.workspaces[0].test_split(ratatui::layout::Direction::Horizontal);
+ app.state.workspaces[0].tabs[0].zoomed = true;
+ let zoomed_away = app.handle_pane_graphics_info(
+ "zoomed-away".into(),
+ crate::api::schema::PaneTarget {
+ pane_id: pane_id.clone(),
+ },
+ );
+ assert!(!pane_visible(&zoomed_away));
+
+ app.state.workspaces[0].tabs[0].zoomed = false;
+ app.state.mode = crate::app::Mode::Navigate;
+ let short_lived_mode = app.handle_pane_graphics_info(
+ "navigate".into(),
+ crate::api::schema::PaneTarget { pane_id },
+ );
+ assert!(pane_visible(&short_lived_mode));
+ }
+
+ #[test]
+ fn monolithic_info_keeps_fast_transport_and_exact_pixels_disabled() {
+ let (mut app, pane_id) = app();
app.state.host_cell_size = crate::kitty_graphics::HostCellSize {
- width_px: 11,
- height_px: 22,
+ width_px: 10,
+ height_px: 20,
};
- let response = app.handle_api_request(request(public_pane_id));
+ assert!(!app.pane_graphics_files.is_initialized());
+ let response = app
+ .handle_pane_graphics_info("info".into(), crate::api::schema::PaneTarget { pane_id });
let success: SuccessResponse = serde_json::from_str(&response).unwrap();
assert!(matches!(
success.result,
ResponseResult::PaneGraphicsInfo {
- cell_width_px: 11,
- cell_height_px: 22,
+ file_frame_damage: true,
+ max_layers_per_pane: 16,
+ pixel_mouse: false,
+ ..
}
));
- assert!(app.state.pane_graphics_layers.is_empty());
- assert!(app.state.pane_graphics_streams.is_empty());
+ assert!(app.pane_graphics.slots.is_empty());
+ assert!(!app.pane_graphics_files.is_initialized());
}
+ #[cfg(unix)]
#[test]
- fn api_pane_graphics_set_and_clear_updates_runtime_layers() {
- let (mut app, public_pane_id) = app_with_test_workspace();
- app.state.kitty_graphics_enabled = true;
- let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
- let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]);
-
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-set".into(),
- method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
- pane_id: public_pane_id.clone(),
- owner: String::new(),
- format: crate::api::schema::PaneGraphicsFormat::Rgba,
- image_width: 1,
- image_height: 1,
- data: None,
- data_base64: data,
- placement: crate::api::schema::PaneGraphicsPlacementParams {
- grid_cols: 10,
- grid_rows: 4,
- ..Default::default()
- },
- }),
- });
- let success: SuccessResponse = serde_json::from_str(&response).unwrap();
- assert!(matches!(success.result, ResponseResult::Ok {}));
+ fn info_advertises_rgba_direct_and_bgra_fallback_file_formats_on_demand() {
+ let (mut app, pane_id) = app();
+ app.state.host_cell_size = crate::kitty_graphics::HostCellSize {
+ width_px: 10,
+ height_px: 20,
+ };
+ app.direct_graphics_available = true;
+ let response = app
+ .handle_pane_graphics_info("info".into(), crate::api::schema::PaneTarget { pane_id });
+ let value: serde_json::Value = serde_json::from_str(&response).unwrap();
+ assert!(app.pane_graphics_files.is_initialized());
+ assert_eq!(
+ value["result"]["file_frame_formats"],
+ serde_json::json!(["rgba", "bgra"])
+ );
+ assert_eq!(value["result"]["file_frame_damage"], true);
+ assert_eq!(value["result"]["file_frame_transport"], "direct-kitty");
+ }
- let layer = app
- .state
- .pane_graphics_layers
- .get(&pane_id)
- .expect("graphics layer stored");
- assert_eq!(layer.image_width, 1);
- assert_eq!(layer.image_height, 1);
- assert_eq!(layer.render.grid_cols, 10);
- assert_eq!(layer.render.grid_rows, 4);
- assert!(!app.state.session_dirty);
+ #[test]
+ fn layers_are_isolated_and_stream_owned() {
+ let (mut app, pane_id) = app();
+ for (layer, z, value) in [(None, 0, 1), (Some("chrome"), 10, 2)] {
+ let mut request = frame(pane_id.clone(), layer, "", value);
+ request.z_index = z;
+ let response = app.handle_pane_graphics_set(format!("set-{value}"), request);
+ assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok());
+ }
+ let (_, pane) = app.parse_pane_id(&pane_id).unwrap();
+ assert_eq!(
+ app.pane_graphics
+ .slots
+ .get(&(pane, PANE_GRAPHICS_PRIMARY_LAYER_ID.into()))
+ .and_then(|slot| slot.layer.as_ref())
+ .unwrap()
+ .z_index,
+ 0
+ );
+ assert_eq!(
+ app.pane_graphics
+ .slots
+ .get(&(pane, "chrome".into()))
+ .and_then(|slot| slot.layer.as_ref())
+ .unwrap()
+ .z_index,
+ 10
+ );
+ let response = app.handle_pane_graphics_stream_open(
+ "open".into(),
+ PaneGraphicsStreamParams {
+ pane_id: pane_id.clone(),
+ layer_id: Some("chrome".into()),
+ z_index: 10,
+ owner: "stream".into(),
+ },
+ );
+ assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok());
+ let response =
+ app.handle_pane_graphics_set("conflict".into(), frame(pane_id, Some("chrome"), "", 3));
+ let error: ErrorResponse = serde_json::from_str(&response).unwrap();
+ assert_eq!(error.error.code, "stream_conflict");
+ }
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-clear".into(),
- method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams {
- pane_id: public_pane_id,
- }),
- });
- let success: SuccessResponse = serde_json::from_str(&response).unwrap();
- assert!(matches!(success.result, ResponseResult::Ok {}));
- assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ fn error_code(response: &str) -> String {
+ serde_json::from_str::<ErrorResponse>(response)
+ .unwrap()
+ .error
+ .code
}
#[test]
- fn api_pane_graphics_rejects_when_kitty_graphics_disabled() {
- let (mut app, public_pane_id) = app_with_test_workspace();
- let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
- let data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]);
-
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-set".into(),
- method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
- pane_id: public_pane_id.clone(),
- owner: String::new(),
- format: crate::api::schema::PaneGraphicsFormat::Rgba,
- image_width: 1,
- image_height: 1,
- data: None,
- data_base64: data,
- placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
- }),
- });
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "feature_disabled");
- assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
-
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-clear".into(),
- method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams {
- pane_id: public_pane_id,
- }),
- });
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "feature_disabled");
- assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ fn validates_layer_and_frame_contracts() {
+ let cases = [
+ (Some("bad space"), 1, 1, vec![1; 4], "invalid_layer_id"),
+ (None, 0, 1, vec![1; 4], "invalid_image"),
+ (None, 1, 1, Vec::new(), "invalid_image"),
+ (None, 1, 1, vec![1; 3], "invalid_image"),
+ ];
+ for (layer, width, height, data, expected) in cases {
+ let (mut app, pane_id) = app();
+ let mut params = frame(pane_id, layer, "", 1);
+ params.image_width = width;
+ params.image_height = height;
+ params.data = Some(data);
+ let response = app.handle_pane_graphics_set("invalid".into(), params);
+ assert_eq!(error_code(&response), expected);
+ assert!(app.pane_graphics.slots.is_empty());
+ }
}
#[test]
- fn api_pane_graphics_stream_claim_uses_resolved_pane_identity() {
- let (mut app, public_pane_id) = app_with_test_workspace();
- app.state.kitty_graphics_enabled = true;
- let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
- let alias = format!("{public_pane_id}:alias");
- app.state
- .public_pane_id_aliases
- .insert(alias.clone(), pane_id);
-
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-stream-open".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamOpen(
- crate::api::schema::PaneGraphicsStreamParams {
- pane_id: public_pane_id,
- owner: "stream-1".into(),
- },
- ),
- });
- let success: SuccessResponse = serde_json::from_str(&response).unwrap();
- assert!(matches!(success.result, ResponseResult::Ok {}));
+ fn disabled_requests_have_no_graphics_side_effects() {
+ let (mut app, pane_id) = app();
+ app.state.kitty_graphics_enabled = false;
+ let revision = app.pane_graphics.revision();
+
+ let response = app.handle_pane_graphics_set("disabled".into(), frame(pane_id, None, "", 1));
+
+ assert_eq!(error_code(&response), "feature_disabled");
+ assert!(app.pane_graphics.slots.is_empty());
+ assert_eq!(app.pane_graphics.revision(), revision);
+ }
+
+ #[test]
+ fn stream_owner_controls_only_its_named_layer() {
+ let (mut app, pane_id) = app();
+ let params = |owner: &str| PaneGraphicsStreamParams {
+ pane_id: pane_id.clone(),
+ layer_id: Some("chrome".into()),
+ z_index: 7,
+ owner: owner.into(),
+ };
+ assert!(serde_json::from_str::<SuccessResponse>(
+ &app.handle_pane_graphics_stream_open("open".into(), params("owner"))
+ )
+ .is_ok());
assert_eq!(
- app.state.pane_graphics_streams.get(&pane_id),
- Some(&"stream-1".to_string())
+ error_code(&app.handle_pane_graphics_stream_set(
+ "wrong".into(),
+ frame(pane_id.clone(), Some("chrome"), "other", 2),
+ )),
+ "stream_conflict"
+ );
+ assert!(
+ serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_stream_set(
+ "frame".into(),
+ frame(pane_id.clone(), Some("chrome"), "owner", 3),
+ ))
+ .is_ok()
);
+ app.handle_pane_graphics_stream_close("stale".into(), params("other"));
+ assert_eq!(app.pane_graphics.slots.len(), 1);
+ app.handle_pane_graphics_stream_close("close".into(), params("owner"));
+ assert!(app.pane_graphics.slots.is_empty());
+ }
+
+ #[test]
+ fn static_set_and_clear_reclaim_inactive_stream_layers() {
+ let (mut app, pane_id) = app();
+ let open = |layer: &str| PaneGraphicsStreamParams {
+ pane_id: pane_id.clone(),
+ layer_id: Some(layer.into()),
+ z_index: 0,
+ owner: format!("owner-{layer}"),
+ };
+ for layer in ["set", "clear"] {
+ assert!(serde_json::from_str::<SuccessResponse>(
+ &app.handle_pane_graphics_stream_open(layer.into(), open(layer))
+ )
+ .is_ok());
+ let (_, pane) = app.parse_pane_id(&pane_id).unwrap();
+ app.pane_graphics.slots[&(pane, layer.into())]
+ .stream_active
+ .as_ref()
+ .unwrap()
+ .store(false, std::sync::atomic::Ordering::Release);
+ }
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-stream-open-alias".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamOpen(
- crate::api::schema::PaneGraphicsStreamParams {
- pane_id: alias.clone(),
- owner: "stream-2".into(),
+ assert!(
+ serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_set(
+ "set-static".into(),
+ frame(pane_id.clone(), Some("set"), "", 4),
+ ))
+ .is_ok()
+ );
+ assert!(
+ serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_clear(
+ "clear-static".into(),
+ PaneGraphicsClearParams {
+ pane_id,
+ layer_id: Some("clear".into()),
},
- ),
- });
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "stream_conflict");
- assert_eq!(
- app.state.pane_graphics_streams.get(&pane_id),
- Some(&"stream-1".to_string())
+ ))
+ .is_ok()
);
+ assert_eq!(app.pane_graphics.slots.len(), 1);
+ assert!(app
+ .pane_graphics
+ .slots
+ .values()
+ .all(|slot| slot.stream_owner.is_none()));
+ }
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-stream-close-wrong-owner".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamClose(
- crate::api::schema::PaneGraphicsStreamParams {
- pane_id: alias.clone(),
- owner: "stream-2".into(),
+ #[test]
+ fn layer_limit_counts_empty_and_populated_stream_slots() {
+ let (mut app, pane_id) = app();
+ for index in 0..PANE_GRAPHICS_MAX_LAYERS_PER_PANE {
+ let layer_id = format!("layer-{index}");
+ let response = app.handle_pane_graphics_stream_open(
+ layer_id.clone(),
+ PaneGraphicsStreamParams {
+ pane_id: pane_id.clone(),
+ layer_id: Some(layer_id),
+ z_index: index as i32,
+ owner: format!("owner-{index}"),
},
- ),
- });
- let success: SuccessResponse = serde_json::from_str(&response).unwrap();
- assert!(matches!(success.result, ResponseResult::Ok {}));
- assert_eq!(
- app.state.pane_graphics_streams.get(&pane_id),
- Some(&"stream-1".to_string())
+ );
+ assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok());
+ }
+ let response = app.handle_pane_graphics_set(
+ "set-overflow".into(),
+ frame(pane_id.clone(), Some("set-overflow"), "", 1),
+ );
+ assert_eq!(error_code(&response), "layer_limit");
+ let response = app.handle_pane_graphics_stream_open(
+ "overflow".into(),
+ PaneGraphicsStreamParams {
+ pane_id,
+ layer_id: Some("overflow".into()),
+ z_index: 0,
+ owner: "overflow".into(),
+ },
);
+ assert_eq!(error_code(&response), "layer_limit");
+ assert_eq!(app.pane_graphics.slots.len(), 16);
+ }
+
+ #[test]
+ fn set_and_clear_preserve_runtime_only_layer_details() {
+ let (mut app, pane_id) = app();
+ let (_, pane) = app.parse_pane_id(&pane_id).unwrap();
+ let mut params = frame(pane_id.clone(), None, "", 9);
+ params.placement.grid_cols = 10;
+ params.placement.grid_rows = 4;
+ assert!(serde_json::from_str::<SuccessResponse>(
+ &app.handle_pane_graphics_set("set".into(), params)
+ )
+ .is_ok());
+ let layer = app
+ .pane_graphics
+ .slots
+ .get(&(pane, PANE_GRAPHICS_PRIMARY_LAYER_ID.into()))
+ .and_then(|slot| slot.layer.as_ref())
+ .unwrap();
+ assert_eq!(layer.inline_data(), Some([9; 4].as_slice()));
+ assert_eq!((layer.render.grid_cols, layer.render.grid_rows), (10, 4));
+ assert!(!app.state.session_dirty);
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-stream-close-alias".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamClose(
- crate::api::schema::PaneGraphicsStreamParams {
- pane_id: alias,
- owner: "stream-1".into(),
+ assert!(
+ serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_clear(
+ "clear".into(),
+ PaneGraphicsClearParams {
+ pane_id,
+ layer_id: None,
},
- ),
- });
- let success: SuccessResponse = serde_json::from_str(&response).unwrap();
- assert!(matches!(success.result, ResponseResult::Ok {}));
- assert!(!app.state.pane_graphics_streams.contains_key(&pane_id));
+ ))
+ .is_ok()
+ );
+ assert!(app.pane_graphics.slots.is_empty());
}
#[test]
- fn api_pane_graphics_stream_owns_layer_while_active() {
- let (mut app, public_pane_id) = app_with_test_workspace();
- app.state.kitty_graphics_enabled = true;
- let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
- let public_data = base64::engine::general_purpose::STANDARD.encode([1_u8, 2, 3, 4]);
-
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-stream-open".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamOpen(
- crate::api::schema::PaneGraphicsStreamParams {
- pane_id: public_pane_id.clone(),
- owner: "stream-1".into(),
- },
- ),
- });
- let success: SuccessResponse = serde_json::from_str(&response).unwrap();
- assert!(matches!(success.result, ResponseResult::Ok {}));
-
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-set-public".into(),
- method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
- pane_id: public_pane_id.clone(),
- owner: String::new(),
- format: crate::api::schema::PaneGraphicsFormat::Rgba,
- image_width: 1,
- image_height: 1,
- data: None,
- data_base64: public_data.clone(),
- placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
- }),
- });
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "stream_conflict");
- assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
-
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-set-stream".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams {
- pane_id: public_pane_id.clone(),
- owner: "stream-1".into(),
- format: crate::api::schema::PaneGraphicsFormat::Rgba,
- image_width: 1,
- image_height: 1,
- data: None,
- data_base64: public_data,
- placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
- }),
- });
- let success: SuccessResponse = serde_json::from_str(&response).unwrap();
- assert!(matches!(success.result, ResponseResult::Ok {}));
- assert!(app.state.pane_graphics_layers.contains_key(&pane_id));
-
- let wrong_owner_data = base64::engine::general_purpose::STANDARD.encode([5_u8, 6, 7, 8]);
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-set-stream-wrong-owner".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams {
- pane_id: public_pane_id.clone(),
- owner: "stream-2".into(),
- format: crate::api::schema::PaneGraphicsFormat::Rgba,
- image_width: 1,
- image_height: 1,
- data: None,
- data_base64: wrong_owner_data,
- placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
- }),
- });
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "stream_conflict");
+ fn raw_formats_require_exact_non_overflowing_lengths() {
+ for (format, exact) in [
+ (crate::api::schema::PaneGraphicsFormat::Rgb, 6),
+ (crate::api::schema::PaneGraphicsFormat::Rgba, 8),
+ ] {
+ for (length, expected) in [(exact, "ok"), (exact - 1, "invalid_image")] {
+ let (mut app, pane_id) = app();
+ let mut params = frame(pane_id, None, "", 1);
+ params.format = format;
+ params.image_width = 2;
+ params.data = Some(vec![1; length]);
+ let response = app.handle_pane_graphics_set("length".into(), params);
+ if expected == "ok" {
+ assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok());
+ } else {
+ assert_eq!(error_code(&response), expected);
+ }
+ }
+ let (mut app, pane_id) = app();
+ let mut params = frame(pane_id, None, "", 1);
+ params.format = format;
+ params.image_width = u32::MAX;
+ params.image_height = u32::MAX;
+ assert_eq!(
+ error_code(&app.handle_pane_graphics_set("overflow".into(), params)),
+ "invalid_image"
+ );
+ }
+ }
+
+ #[test]
+ fn public_frames_validate_base64_and_public_size_limit() {
+ let (mut app, pane_id) = app();
+ let mut params = frame(pane_id.clone(), None, "", 1);
+ params.format = crate::api::schema::PaneGraphicsFormat::Png;
+ params.data = None;
+ params.data_base64 = "not base64".into();
assert_eq!(
- app.state.pane_graphics_layers.get(&pane_id).unwrap().data,
- vec![1_u8, 2, 3, 4]
+ error_code(&app.handle_pane_graphics_set("base64".into(), params)),
+ "invalid_image"
);
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-clear-public".into(),
- method: crate::api::schema::Method::PaneGraphicsClear(PaneGraphicsClearParams {
- pane_id: public_pane_id.clone(),
- }),
- });
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "stream_conflict");
- assert!(app.state.pane_graphics_layers.contains_key(&pane_id));
-
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-stream-close".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamClose(
- crate::api::schema::PaneGraphicsStreamParams {
- pane_id: public_pane_id,
- owner: "stream-1".into(),
- },
- ),
- });
- let success: SuccessResponse = serde_json::from_str(&response).unwrap();
- assert!(matches!(success.result, ResponseResult::Ok {}));
- assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ let mut params = frame(pane_id, None, "", 1);
+ params.format = crate::api::schema::PaneGraphicsFormat::Png;
+ params.data = None;
+ params.data_base64 =
+ base64::engine::general_purpose::STANDARD
+ .encode(vec![1; PANE_GRAPHICS_SET_MAX_BYTES + 1]);
+ assert_eq!(
+ error_code(&app.handle_pane_graphics_set("large".into(), params)),
+ "image_too_large"
+ );
}
#[test]
- fn api_pane_graphics_rejects_overflowing_raw_dimensions() {
- for format in [
- crate::api::schema::PaneGraphicsFormat::Rgb,
- crate::api::schema::PaneGraphicsFormat::Rgba,
- ] {
- let (mut app, public_pane_id) = app_with_test_workspace();
- app.state.kitty_graphics_enabled = true;
- let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
-
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-overflow".into(),
- method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
- pane_id: public_pane_id,
- owner: String::new(),
- format,
- image_width: u32::MAX,
- image_height: u32::MAX,
- data: None,
- data_base64: base64::engine::general_purpose::STANDARD.encode([1_u8]),
- placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
- }),
- });
+ fn aliases_resolve_to_the_same_layer_identity() {
+ let (mut app, pane_id) = app();
+ let (_, pane) = app.parse_pane_id(&pane_id).unwrap();
+ let alias = format!("{pane_id}:alias");
+ app.state.public_pane_id_aliases.insert(alias.clone(), pane);
+ let params = |target: String, owner: &str| PaneGraphicsStreamParams {
+ pane_id: target,
+ layer_id: None,
+ z_index: 0,
+ owner: owner.into(),
+ };
+ assert!(serde_json::from_str::<SuccessResponse>(
+ &app.handle_pane_graphics_stream_open("open".into(), params(pane_id.clone(), "owner"),)
+ )
+ .is_ok());
+ assert_eq!(
+ error_code(&app.handle_pane_graphics_stream_open(
+ "conflict".into(),
+ params(alias.clone(), "other"),
+ )),
+ "stream_conflict"
+ );
+ app.handle_pane_graphics_stream_close("stale".into(), params(alias.clone(), "other"));
+ assert_eq!(app.pane_graphics.slots.len(), 1);
+ app.handle_pane_graphics_stream_close("close".into(), params(alias, "owner"));
+ assert!(app.pane_graphics.slots.is_empty());
+ }
+
+ #[cfg(unix)]
+ fn direct_file(app: &App, name: &str, data: &[u8]) -> String {
+ use std::io::Write as _;
+ use std::os::unix::fs::OpenOptionsExt as _;
+ let path = app
+ .pane_graphics_files
+ .source_directory()
+ .unwrap()
+ .join(name);
+ let mut file = std::fs::OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .mode(0o600)
+ .open(&path)
+ .unwrap();
+ file.write_all(data).unwrap();
+ path.to_string_lossy().into_owned()
+ }
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "invalid_image");
- assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ #[cfg(unix)]
+ fn direct_params(
+ pane_id: String,
+ owner: &str,
+ path: String,
+ ) -> crate::api::schema::PaneGraphicsDirectParams {
+ crate::api::schema::PaneGraphicsDirectParams {
+ pane_id,
+ layer_id: None,
+ z_index: 0,
+ owner: owner.into(),
+ image_width: 1,
+ image_height: 1,
+ format: crate::api::schema::PaneGraphicsFormat::Rgba,
+ path,
+ sequence: 1,
+ revision: 1,
+ placement: Default::default(),
}
}
+ #[cfg(unix)]
#[test]
- fn api_pane_graphics_validates_exact_raw_lengths() {
- for (format, exact_len) in [
- (crate::api::schema::PaneGraphicsFormat::Rgb, 6_usize),
- (crate::api::schema::PaneGraphicsFormat::Rgba, 8_usize),
+ fn direct_file_is_leased_or_safely_copied_for_inline_fallback() {
+ let (mut app, pane_id) = app();
+ let stream = PaneGraphicsStreamParams {
+ pane_id: pane_id.clone(),
+ layer_id: None,
+ z_index: 0,
+ owner: "owner".into(),
+ };
+ app.handle_pane_graphics_stream_open("open".into(), stream.clone());
+ for (width, height) in [(0, 1), (1, 0)] {
+ let mut params = direct_params(pane_id.clone(), "owner", "unused".into());
+ (params.image_width, params.image_height) = (width, height);
+ assert_eq!(
+ error_code(&app.handle_pane_graphics_stream_direct("invalid".into(), params)),
+ "invalid_image"
+ );
+ }
+ for format in [
+ crate::api::schema::PaneGraphicsFormat::Rgb,
+ crate::api::schema::PaneGraphicsFormat::Png,
] {
- for data_len in [exact_len, exact_len - 1] {
- let (mut app, public_pane_id) = app_with_test_workspace();
- app.state.kitty_graphics_enabled = true;
- let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-raw-length".into(),
- method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
- pane_id: public_pane_id,
- owner: String::new(),
- format,
- image_width: 2,
- image_height: 1,
- data: None,
- data_base64: base64::engine::general_purpose::STANDARD
- .encode(vec![1_u8; data_len]),
- placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
- }),
- });
-
- if data_len == exact_len {
- let success: SuccessResponse = serde_json::from_str(&response).unwrap();
- assert!(matches!(success.result, ResponseResult::Ok {}));
- assert!(app.state.pane_graphics_layers.contains_key(&pane_id));
- } else {
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "invalid_image");
- assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
- }
- }
+ let mut params = direct_params(pane_id.clone(), "owner", "unused".into());
+ params.format = format;
+ assert_eq!(
+ error_code(&app.handle_pane_graphics_stream_direct("invalid".into(), params)),
+ "invalid_image"
+ );
}
+ app.direct_graphics_available = true;
+ let path = direct_file(&app, "direct", &[1, 2, 3, 4]);
+ let response = app.handle_pane_graphics_stream_direct(
+ "direct".into(),
+ crate::api::schema::PaneGraphicsDirectParams {
+ sequence: 7,
+ revision: 8,
+ ..direct_params(pane_id.clone(), "owner", path)
+ },
+ );
+ let ack: SuccessResponse = serde_json::from_str(&response).unwrap();
+ assert!(matches!(
+ ack.result,
+ ResponseResult::PaneGraphicsFrameAck {
+ sequence: 7,
+ revision: 8
+ }
+ ));
+ assert!(app
+ .pane_graphics
+ .slots
+ .values()
+ .next()
+ .unwrap()
+ .layer
+ .as_ref()
+ .unwrap()
+ .direct_lease()
+ .is_some());
+
+ app.handle_pane_graphics_stream_close("close".into(), stream.clone());
+ app.handle_pane_graphics_stream_open("reopen".into(), stream);
+ app.direct_graphics_available = false;
+ let path = direct_file(&app, "fallback", &[5, 6, 7, 8]);
+ app.handle_pane_graphics_stream_direct(
+ "fallback".into(),
+ crate::api::schema::PaneGraphicsDirectParams {
+ sequence: 9,
+ revision: 10,
+ ..direct_params(pane_id, "owner", path)
+ },
+ );
+ assert_eq!(
+ app.pane_graphics
+ .slots
+ .values()
+ .next()
+ .unwrap()
+ .layer
+ .as_ref()
+ .unwrap()
+ .inline_data(),
+ Some([5, 6, 7, 8].as_slice())
+ );
}
+ #[cfg(unix)]
#[test]
- fn api_pane_graphics_uses_transport_specific_limits() {
- let (mut app, public_pane_id) = app_with_test_workspace();
- app.state.kitty_graphics_enabled = true;
- let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
- let response = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-public-too-large".into(),
- method: crate::api::schema::Method::PaneGraphicsSet(PaneGraphicsSetParams {
- pane_id: public_pane_id,
- owner: String::new(),
- format: crate::api::schema::PaneGraphicsFormat::Png,
- image_width: 1,
- image_height: 1,
- data: None,
- data_base64: base64::engine::general_purpose::STANDARD.encode(vec![
- 1_u8;
- PANE_GRAPHICS_SET_MAX_BYTES
- + 1
- ]),
- placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
- }),
- });
-
- let error: ErrorResponse = serde_json::from_str(&response).unwrap();
- assert_eq!(error.error.code, "image_too_large");
- assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ fn bgra_and_secondary_file_frames_are_canonical_owned_rgba() {
+ let (mut app, pane_id) = app();
+ app.direct_graphics_available = true;
+ for (layer, format, source) in [
+ (
+ "browser-chrome",
+ crate::api::schema::PaneGraphicsFormat::Rgba,
+ [1, 2, 3, 4],
+ ),
+ (
+ "primary",
+ crate::api::schema::PaneGraphicsFormat::Bgra,
+ [3, 2, 1, 4],
+ ),
+ ] {
+ app.handle_pane_graphics_stream_open(
+ format!("open-{layer}"),
+ PaneGraphicsStreamParams {
+ pane_id: pane_id.clone(),
+ layer_id: Some(layer.into()),
+ z_index: 0,
+ owner: layer.into(),
+ },
+ );
+ let path = direct_file(&app, layer, &source);
+ app.handle_pane_graphics_stream_direct(
+ format!("frame-{layer}"),
+ crate::api::schema::PaneGraphicsDirectParams {
+ layer_id: Some(layer.into()),
+ format,
+ ..direct_params(pane_id.clone(), layer, path)
+ },
+ );
+ let key = (app.parse_pane_id(&pane_id).unwrap().1, layer.into());
+ assert_eq!(
+ app.pane_graphics.slots[&key]
+ .layer
+ .as_ref()
+ .unwrap()
+ .inline_data(),
+ Some([1, 2, 3, 4].as_slice())
+ );
+ }
}
#[test]
- fn api_pane_graphics_stream_methods_require_kitty_graphics() {
- let (mut app, public_pane_id) = app_with_test_workspace();
- let (_, pane_id) = app.parse_pane_id(&public_pane_id).unwrap();
- let open = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-stream-open".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamOpen(
- crate::api::schema::PaneGraphicsStreamParams {
- pane_id: public_pane_id.clone(),
- owner: "stream-1".into(),
+ fn disabled_info_and_stream_requests_are_rejected() {
+ let (mut app, pane_id) = app();
+ app.state.kitty_graphics_enabled = false;
+ assert_eq!(
+ error_code(&app.handle_pane_graphics_info(
+ "info".into(),
+ crate::api::schema::PaneTarget {
+ pane_id: pane_id.clone(),
},
- ),
- });
- let error: ErrorResponse = serde_json::from_str(&open).unwrap();
- assert_eq!(error.error.code, "feature_disabled");
-
- let set = app.handle_api_request(crate::api::schema::Request {
- id: "graphics-stream-set".into(),
- method: crate::api::schema::Method::PaneGraphicsStreamSet(PaneGraphicsSetParams {
- pane_id: public_pane_id,
- owner: "stream-1".into(),
- format: crate::api::schema::PaneGraphicsFormat::Png,
- image_width: 1,
- image_height: 1,
- data: Some(vec![1_u8]),
- data_base64: String::new(),
- placement: crate::api::schema::PaneGraphicsPlacementParams::default(),
- }),
- });
- let error: ErrorResponse = serde_json::from_str(&set).unwrap();
- assert_eq!(error.error.code, "feature_disabled");
- assert!(!app.state.pane_graphics_streams.contains_key(&pane_id));
- assert!(!app.state.pane_graphics_layers.contains_key(&pane_id));
+ )),
+ "feature_disabled"
+ );
+ assert_eq!(
+ error_code(&app.handle_pane_graphics_stream_open(
+ "open".into(),
+ PaneGraphicsStreamParams {
+ pane_id,
+ layer_id: None,
+ z_index: 0,
+ owner: "owner".into(),
+ },
+ )),
+ "feature_disabled"
+ );
+ assert!(app.pane_graphics.slots.is_empty());
}
}
diff --git a/src/app/api/plugins/mod.rs b/src/app/api/plugins/mod.rs
index a8cb4405..ddf3fba6 100644
--- a/src/app/api/plugins/mod.rs
+++ b/src/app/api/plugins/mod.rs
@@ -705,8 +705,10 @@ fn manifest_actions(
#[cfg(test)]
mod tests {
use super::*;
+ #[cfg(unix)]
+ use crate::api::schema::PaneListParams;
use crate::api::schema::{
- Method, PaneListParams, PluginSourceInfo, PluginSourceKind, Request, SuccessResponse,
+ Method, PluginSourceInfo, PluginSourceKind, Request, SuccessResponse,
};
use std::time::{SystemTime, UNIX_EPOCH};
diff --git a/src/app/ids.rs b/src/app/ids.rs
index 4bebedc6..c2fc0a4d 100644
--- a/src/app/ids.rs
+++ b/src/app/ids.rs
@@ -103,7 +103,7 @@ impl App {
None
}
- pub(super) fn parse_pane_id(&self, id: &str) -> Option<(usize, crate::layout::PaneId)> {
+ pub(crate) fn parse_pane_id(&self, id: &str) -> Option<(usize, crate::layout::PaneId)> {
if let Some(alias) = self.state.public_pane_id_aliases.get(id).copied() {
return self.find_pane(alias).map(|(ws_idx, _)| (ws_idx, alias));
}
diff --git a/src/app/input/mod.rs b/src/app/input/mod.rs
index f77821c9..3202ae26 100644
--- a/src/app/input/mod.rs
+++ b/src/app/input/mod.rs
@@ -493,15 +493,17 @@ impl App {
self.close_popup_pane();
return;
};
- let column = mouse.column.saturating_sub(inner.x);
- let row = mouse.row.saturating_sub(inner.y);
+ let position = crate::input::mouse::Position::Cell {
+ column: mouse.column.saturating_sub(inner.x),
+ row: mouse.row.saturating_sub(inner.y),
+ };
let bytes = match mouse.kind {
MouseEventKind::ScrollUp
| MouseEventKind::ScrollDown
| MouseEventKind::ScrollLeft
| MouseEventKind::ScrollRight => match rt.wheel_routing() {
Some(crate::pane::WheelRouting::MouseReport) => {
- rt.encode_mouse_wheel(mouse.kind, column, row, mouse.modifiers)
+ rt.encode_mouse_wheel(mouse.kind, position, mouse.modifiers)
}
Some(crate::pane::WheelRouting::AlternateScroll) => {
rt.encode_alternate_scroll(mouse.kind)
@@ -517,11 +519,9 @@ impl App {
}
},
MouseEventKind::Down(_) | MouseEventKind::Up(_) | MouseEventKind::Drag(_) => {
- rt.encode_mouse_button(mouse.kind, column, row, mouse.modifiers)
- }
- MouseEventKind::Moved => {
- rt.encode_mouse_motion(mouse.kind, column, row, mouse.modifiers)
+ rt.encode_mouse_button(mouse.kind, position, mouse.modifiers)
}
+ MouseEventKind::Moved => rt.encode_mouse_motion(mouse.kind, position, mouse.modifiers),
};
let Some(bytes) = bytes else {
return;
diff --git a/src/app/input/mouse.rs b/src/app/input/mouse.rs
index 2555d95d..8d8988f9 100644
--- a/src/app/input/mouse.rs
+++ b/src/app/input/mouse.rs
@@ -1667,6 +1667,33 @@ impl AppState {
}
}
+ fn pane_mouse_position(
+ &self,
+ runtime: &crate::terminal::TerminalRuntime,
+ inner: Rect,
+ mouse: MouseEvent,
+ ) -> Option<crate::input::mouse::Position> {
+ let column = mouse.column.saturating_sub(inner.x);
+ let row = mouse.row.saturating_sub(inner.y);
+ let cell = crate::input::mouse::Position::Cell { column, row };
+ let Some(host) = self.host_mouse_pixels else {
+ return Some(cell);
+ };
+ let wants_pixels = runtime.input_state().is_some_and(|state| {
+ state.mouse_protocol_encoding == crate::input::MouseProtocolEncoding::SgrPixels
+ });
+ if !wants_pixels {
+ return Some(cell);
+ }
+ let Some((width_px, height_px)) = runtime.pixel_size() else {
+ return Some(cell);
+ };
+ Some(
+ host.pane_position(inner, width_px, height_px)
+ .unwrap_or(cell),
+ )
+ }
+
pub(super) fn forward_pane_mouse_button(
&self,
terminal_runtimes: &TerminalRuntimeRegistry,
@@ -1680,9 +1707,10 @@ impl AppState {
else {
return false;
};
- let column = mouse.column.saturating_sub(info.inner_rect.x);
- let row = mouse.row.saturating_sub(info.inner_rect.y);
- let Some(bytes) = rt.encode_mouse_button(mouse.kind, column, row, mouse.modifiers) else {
+ let Some(position) = self.pane_mouse_position(rt, info.inner_rect, mouse) else {
+ return false;
+ };
+ let Some(bytes) = rt.encode_mouse_button(mouse.kind, position, mouse.modifiers) else {
return false;
};
rt.scroll_reset();
@@ -1705,9 +1733,10 @@ impl AppState {
else {
return false;
};
- let column = mouse.column.saturating_sub(info.inner_rect.x);
- let row = mouse.row.saturating_sub(info.inner_rect.y);
- let Some(bytes) = rt.encode_mouse_motion(mouse.kind, column, row, mouse.modifiers) else {
+ let Some(position) = self.pane_mouse_position(rt, info.inner_rect, mouse) else {
+ return false;
+ };
+ let Some(bytes) = rt.encode_mouse_motion(mouse.kind, position, mouse.modifiers) else {
return false;
};
if let Err(err) = rt.try_send_bytes(Bytes::from(bytes)) {
@@ -1733,9 +1762,10 @@ impl AppState {
return false;
}
rt.scroll_reset();
- let column = mouse.column.saturating_sub(info.inner_rect.x);
- let row = mouse.row.saturating_sub(info.inner_rect.y);
- let Some(bytes) = rt.encode_mouse_wheel(mouse.kind, column, row, mouse.modifiers) else {
+ let Some(position) = self.pane_mouse_position(rt, info.inner_rect, mouse) else {
+ return false;
+ };
+ let Some(bytes) = rt.encode_mouse_wheel(mouse.kind, position, mouse.modifiers) else {
warn!(pane = info.id.raw(), kind = ?mouse.kind, "failed to encode mouse wheel event");
return true;
};
@@ -1761,18 +1791,7 @@ impl AppState {
match rt.wheel_routing() {
Some(crate::pane::WheelRouting::HostScroll) | None => false,
Some(crate::pane::WheelRouting::MouseReport) => {
- rt.scroll_reset();
- let column = mouse.column.saturating_sub(info.inner_rect.x);
- let row = mouse.row.saturating_sub(info.inner_rect.y);
- let Some(bytes) = rt.encode_mouse_wheel(mouse.kind, column, row, mouse.modifiers)
- else {
- warn!(pane = info.id.raw(), kind = ?mouse.kind, "failed to encode mouse wheel event");
- return true;
- };
- if let Err(err) = rt.try_send_bytes(Bytes::from(bytes)) {
- warn!(pane = info.id.raw(), err = %err, "failed to forward mouse wheel event");
- }
- true
+ self.forward_pane_reported_wheel(terminal_runtimes, info, mouse)
}
Some(crate::pane::WheelRouting::AlternateScroll) => {
rt.scroll_reset();
@@ -2146,7 +2165,7 @@ mod tests {
}
#[tokio::test]
- async fn configured_right_click_passthrough_forwards_full_gesture_to_pane() {
+ async fn configured_right_click_passthrough_forwards_gesture_outside_pane() {
let mut app = app_for_mouse_test();
let mut ws = Workspace::test_new("test");
let pane_id = ws.tabs[0].root_pane;
@@ -2177,11 +2196,11 @@ mod tests {
});
app.handle_mouse(MouseEvent {
modifiers: KeyModifiers::CONTROL,
- ..mouse(MouseEventKind::Drag(MouseButton::Right), col + 1, row + 1)
+ ..mouse(MouseEventKind::Drag(MouseButton::Right), 0, 0)
});
app.handle_mouse(MouseEvent {
modifiers: KeyModifiers::CONTROL,
- ..mouse(MouseEventKind::Up(MouseButton::Right), col + 1, row + 1)
+ ..mouse(MouseEventKind::Up(MouseButton::Right), 0, 0)
});
assert_eq!(app.state.mode, Mode::Terminal);
@@ -2193,11 +2212,11 @@ mod tests {
);
assert_eq!(
input_rx.try_recv().expect("forwarded right mouse drag"),
- Bytes::from_static(b"\x1b[<34;4;5M")
+ Bytes::from_static(b"\x1b[<34;1;1M")
);
assert_eq!(
input_rx.try_recv().expect("forwarded right mouse up"),
- Bytes::from_static(b"\x1b[<2;4;5m")
+ Bytes::from_static(b"\x1b[<2;1;1m")
);
assert!(input_rx.try_recv().is_err());
}
@@ -2321,7 +2340,7 @@ mod tests {
}
#[tokio::test]
- async fn mouse_dispatcher_downgrades_sgr_pixel_motion_to_cell_coordinates() {
+ async fn ordinary_cell_mouse_downgrades_pixel_mode_to_cell_coordinates() {
let mut app = app_for_mouse_test();
let mut ws = Workspace::test_new("test");
let pane_id = ws.tabs[0].root_pane;
@@ -2339,8 +2358,16 @@ mod tests {
app.state.active = Some(0);
app.state.selected = 0;
app.state.mode = Mode::Terminal;
+ app.state.host_cell_size = crate::kitty_graphics::HostCellSize {
+ width_px: 10,
+ height_px: 20,
+ };
crate::ui::compute_view(&mut app.state, Rect::new(0, 0, 106, 20));
let info = app.state.view.pane_infos[0].clone();
+ app.state
+ .runtime_for_pane_in_workspace(&app.terminal_runtimes, 0, pane_id)
+ .unwrap()
+ .resize(info.inner_rect.height, info.inner_rect.width, 10, 20);
assert!(info.inner_rect.x > 0, "sidebar offset should be present");
assert!(info.inner_rect.y > 0, "tab bar offset should be present");
@@ -2358,6 +2385,77 @@ mod tests {
}
#[tokio::test]
+ async fn dedicated_client_pixel_mouse_preserves_subcell_position() {
+ let mut app = app_for_mouse_test();
+ let mut ws = Workspace::test_new("test");
+ let pane_id = ws.tabs[0].root_pane;
+ let (runtime, mut input_rx) =
+ crate::terminal::TerminalRuntime::test_with_channel_and_scrollback_bytes(
+ 80,
+ 18,
+ 0,
+ b"\x1b[?1003h\x1b[?1006h\x1b[?1016h",
+ 4,
+ );
+ ws.insert_test_runtime(pane_id, runtime);
+ app.state.workspaces = vec![ws];
+ app.state.active = Some(0);
+ app.state.selected = 0;
+ app.state.mode = Mode::Terminal;
+ app.state.mouse_capture = false;
+ app.state.host_cell_size = crate::kitty_graphics::HostCellSize {
+ width_px: 10,
+ height_px: 20,
+ };
+ crate::ui::compute_view(&mut app.state, Rect::new(0, 0, 106, 20));
+ let inner = app.state.view.pane_infos[0].inner_rect;
+ let geometry = crate::input::mouse::HostGeometry::new(106, 20, 1_060, 400).unwrap();
+ let x = u32::from(inner.x + 2) * 10 + 8;
+ let y = u32::from(inner.y + 3) * 20 + 9;
+ let report = format!("\x1b[<35;{x};{y}M");
+ app.state.host_mouse_pixels = Some(crate::input::mouse::HostPixels { x, y, geometry });
+ let runtime = app
+ .state
+ .runtime_for_pane_in_workspace(&app.terminal_runtimes, 0, pane_id)
+ .unwrap();
+ assert_eq!(runtime.pixel_size(), None);
+ assert_eq!(
+ app.state.pane_mouse_position(
+ runtime,
+ inner,
+ mouse(MouseEventKind::Moved, inner.x + 2, inner.y + 3),
+ ),
+ Some(crate::input::mouse::Position::Cell { column: 2, row: 3 })
+ );
+ runtime.resize(inner.height, inner.width, 10, 20);
+ let runtime = app
+ .state
+ .runtime_for_pane_in_workspace(&app.terminal_runtimes, 0, pane_id)
+ .unwrap();
+ assert_eq!(
+ runtime.pixel_size(),
+ Some((u32::from(inner.width) * 10, u32::from(inner.height) * 20))
+ );
+ assert_eq!(
+ app.state.pane_mouse_position(
+ runtime,
+ inner,
+ mouse(MouseEventKind::Moved, inner.x + 2, inner.y + 3),
+ ),
+ Some(crate::input::mouse::Position::Pixels { x: 28, y: 69 })
+ );
+ app.state.host_mouse_pixels = None;
+
+ assert!(app.route_client_pixel_mouse(7, report.as_bytes(), geometry));
+ assert_eq!(
+ input_rx.try_recv().expect("forwarded exact mouse motion"),
+ Bytes::from_static(b"\x1b[<35;28;69M")
+ );
+ assert!(input_rx.try_recv().is_err());
+ assert!(app.state.host_mouse_pixels.is_none());
+ }
+
+ #[tokio::test]
async fn mouse_dispatcher_does_not_forward_motion_behind_herdr_modes() {
let mut app = app_for_mouse_test();
let mut ws = Workspace::test_new("test");
diff --git a/src/app/mod.rs b/src/app/mod.rs
index 70d5fe93..15f352af 100644
--- a/src/app/mod.rs
+++ b/src/app/mod.rs
@@ -16,6 +16,7 @@ mod creation;
mod git_refresh;
mod ids;
mod input;
+pub(crate) mod pane_graphics;
mod popup;
mod runtime;
mod runtime_mutations;
@@ -95,6 +96,10 @@ impl PaneClickState {
pub struct App {
pub state: AppState,
+ pub(crate) pane_graphics: pane_graphics::Runtime,
+ pub(crate) pane_graphics_files: Arc<crate::pane_graphics_files::FileStore>,
+ pub(crate) direct_graphics_available: bool,
+ pub(crate) pixel_mouse_available: bool,
pub(crate) terminal_runtimes: crate::terminal::TerminalRuntimeRegistry,
pub event_tx: mpsc::Sender<AppEvent>,
pub(crate) event_rx: mpsc::Receiver<AppEvent>,
@@ -672,9 +677,6 @@ impl App {
integration_install_messages: Vec::new(),
installed_plugins: load_plugin_registry(no_session),
plugin_panes: std::collections::HashMap::new(),
- pane_graphics_layers: std::collections::HashMap::new(),
- pane_graphics_streams: std::collections::HashMap::new(),
- pane_graphics_revision: 0,
popup_pane: None,
plugin_command_logs: Vec::new(),
next_plugin_command_log_id: 1,
@@ -682,6 +684,7 @@ impl App {
global_menu: state::MenuListState::new(0),
host_terminal_theme: crate::terminal_theme::TerminalTheme::default(),
host_cell_size: crate::kitty_graphics::HostCellSize::default(),
+ host_mouse_pixels: None,
session_dirty: false,
terminal_runtime_shutdowns: Vec::new(),
};
@@ -726,6 +729,10 @@ impl App {
copy_feedback_deadline: None,
last_api_notification_at: None,
state,
+ pane_graphics: pane_graphics::Runtime::default(),
+ pane_graphics_files: Arc::new(crate::pane_graphics_files::FileStore::default()),
+ direct_graphics_available: false,
+ pixel_mouse_available: false,
terminal_runtimes: restored_terminal_runtimes,
event_tx,
event_rx,
@@ -1033,6 +1040,10 @@ impl App {
needs_render = true;
}
+ if self.pane_graphics.retain_live_panes(&self.state) {
+ needs_render = true;
+ }
+
let now = Instant::now();
self.sync_host_mouse_capture(&mut host_mouse_capture_active)?;
self.sync_host_keyboard_report_all(&mut host_keyboard_report_all_active)?;
@@ -1048,18 +1059,19 @@ impl App {
terminal.swap_buffers();
self.full_redraw_pending = false;
}
- let mut cell_size = crate::kitty_graphics::HostCellSize::default();
+ let mut cell_size = self.state.host_cell_size;
terminal.draw(|frame| {
let area = frame.area();
if kitty_graphics_enabled {
- let observed_cell_size =
- crate::kitty_graphics::HostCellSize::try_from_terminal(area);
- if let Some(observed_cell_size) = observed_cell_size {
+ if let Some(observed_cell_size) =
+ crate::kitty_graphics::HostCellSize::try_from_terminal(area)
+ {
self.state.host_cell_size = observed_cell_size;
+ cell_size = observed_cell_size;
+ } else if !cell_size.is_known() {
+ cell_size =
+ crate::kitty_graphics::HostCellSize::fallback_for_area(area);
}
- cell_size = observed_cell_size.unwrap_or_else(|| {
- crate::kitty_graphics::HostCellSize::fallback_for_area(area)
- });
crate::ui::compute_view_with_cell_size(
&mut self.state,
&self.terminal_runtimes,
@@ -1082,6 +1094,7 @@ impl App {
if kitty_graphics_enabled {
crate::kitty_graphics::paint_local_pane_graphics(
&self.state,
+ &self.pane_graphics,
&self.terminal_runtimes,
cell_size,
)?;
@@ -1468,8 +1481,7 @@ impl App {
crate::kitty_graphics::set_enabled(config.experimental.kitty_graphics);
if was_kitty_graphics_enabled && !config.experimental.kitty_graphics {
let _ = crate::kitty_graphics::clear_all_host_graphics();
- self.state.pane_graphics_layers.clear();
- self.state.pane_graphics_streams.clear();
+ self.pane_graphics.clear();
self.state.host_cell_size = crate::kitty_graphics::HostCellSize::default();
}
self.state.reveal_hidden_cursor_for_cjk_ime =
@@ -1663,6 +1675,31 @@ impl App {
self.route_client_events(events, true);
}
+ pub(crate) fn route_client_pixel_mouse(
+ &mut self,
+ source_id: InputSourceId,
+ data: &[u8],
+ geometry: crate::input::mouse::HostGeometry,
+ ) -> bool {
+ let Some((x, y)) = crate::input::mouse::parse_report(data) else {
+ return false;
+ };
+ let Some((column, row)) = geometry.cell(x, y) else {
+ return false;
+ };
+ let Some(cell_report) = crate::input::mouse::report_at_cell(data, column, row) else {
+ return false;
+ };
+ let mut events = crate::raw_input::parse_raw_input_bytes_sync(&cell_report);
+ if events.len() != 1 || !matches!(events[0], crate::raw_input::RawInputEvent::Mouse(_)) {
+ return false;
+ }
+ self.state.host_mouse_pixels = Some(crate::input::mouse::HostPixels { x, y, geometry });
+ self.route_client_events_from(source_id, std::mem::take(&mut events), false);
+ self.state.host_mouse_pixels = None;
+ true
+ }
+
pub(crate) fn route_client_events(
&mut self,
events: Vec<crate::raw_input::RawInputEvent>,
diff --git a/src/app/pane_graphics.rs b/src/app/pane_graphics.rs
new file mode 100644
index 00000000..c1d4841a
--- /dev/null
+++ b/src/app/pane_graphics.rs
@@ -0,0 +1,347 @@
+use std::collections::HashMap;
+use std::hash::{Hash, Hasher};
+
+use crate::api::schema::{PaneGraphicsFormat, PaneGraphicsPlacementParams};
+use crate::layout::PaneId;
+
+pub(crate) type Key = (PaneId, String);
+
+pub(crate) const DIRECT_DELIVERY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
+pub(crate) const DIRECT_RESPONSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3);
+pub(crate) const DIRECT_OUTER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(9);
+
+#[derive(Debug)]
+pub(crate) enum Backing {
+ Inline(Vec<u8>),
+ Direct(crate::pane_graphics_files::Lease),
+ Resident { len: usize, client_id: u64 },
+}
+
+#[derive(Debug)]
+pub(crate) struct Layer {
+ pub(crate) format: PaneGraphicsFormat,
+ pub(crate) image_width: u32,
+ pub(crate) image_height: u32,
+ pub(crate) backing: Backing,
+ pub(crate) data_fingerprint: u64,
+ pub(crate) render: PaneGraphicsPlacementParams,
+ pub(crate) z_index: i32,
+}
+
+#[derive(Debug)]
+pub(crate) struct DirectGate {
+ pub(crate) transfer_id: u64,
+ pub(crate) client_id: u64,
+ pub(crate) deadline: std::time::Instant,
+ pub(crate) written: bool,
+ pub(crate) success_response: String,
+ pub(crate) respond_to: std::sync::mpsc::Sender<String>,
+}
+
+#[derive(Debug)]
+pub(crate) struct Slot {
+ pub(crate) host_image_id: u32,
+ pub(crate) layer: Option<Layer>,
+ pub(crate) stream_owner: Option<String>,
+ pub(crate) stream_active: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>,
+ pub(crate) direct_gate: Option<DirectGate>,
+}
+
+impl Slot {
+ #[cfg(test)]
+ pub(crate) fn test(host_image_id: u32, layer: Option<Layer>) -> Self {
+ Self {
+ host_image_id,
+ layer,
+ stream_owner: None,
+ stream_active: None,
+ direct_gate: None,
+ }
+ }
+
+ pub(crate) fn stream_is_active(&self) -> bool {
+ self.stream_active
+ .as_ref()
+ .is_some_and(|active| active.load(std::sync::atomic::Ordering::Acquire))
+ }
+}
+
+impl Drop for Slot {
+ fn drop(&mut self) {
+ if let Some(active) = &self.stream_active {
+ active.store(false, std::sync::atomic::Ordering::Release);
+ }
+ }
+}
+
+#[derive(Debug)]
+pub(crate) struct Runtime {
+ pub(crate) slots: HashMap<Key, Slot>,
+ revision: u64,
+ next_host_image_id: u32,
+}
+
+impl Default for Runtime {
+ fn default() -> Self {
+ Self {
+ slots: HashMap::new(),
+ revision: 0,
+ next_host_image_id: 1 << 31,
+ }
+ }
+}
+
+impl Runtime {
+ pub(crate) fn revision(&self) -> u64 {
+ self.revision
+ }
+
+ pub(crate) fn mark_changed(&mut self) {
+ self.revision = self.revision.wrapping_add(1);
+ }
+
+ pub(crate) fn clear(&mut self) {
+ self.slots.clear();
+ self.mark_changed();
+ }
+
+ pub(crate) fn reserve_image_id(&mut self, key: &Key) -> Option<u32> {
+ if let Some(id) = self.slots.get(key).map(|slot| slot.host_image_id) {
+ return Some(id);
+ }
+ for _ in 0..=self.slots.len() {
+ let id = self.next_host_image_id | (1 << 31);
+ self.next_host_image_id = id.wrapping_add(1) | (1 << 31);
+ if self.slots.values().all(|slot| slot.host_image_id != id) {
+ return Some(id);
+ }
+ }
+ None
+ }
+
+ pub(crate) fn layer_count(&self, pane_id: PaneId) -> usize {
+ self.slots.keys().filter(|(id, _)| *id == pane_id).count()
+ }
+
+ pub(crate) fn can_add_slot(&self, key: &Key) -> bool {
+ self.slots.contains_key(key)
+ || self.slots.len() < crate::api::schema::panes::PANE_GRAPHICS_MAX_LAYERS_TOTAL
+ }
+
+ pub(crate) fn can_store_inline(&self, key: &Key, len: usize) -> bool {
+ let replaced = self
+ .slots
+ .get(key)
+ .and_then(|slot| slot.layer.as_ref())
+ .and_then(Layer::inline_data)
+ .map_or(0, <[u8]>::len);
+ self.inline_bytes()
+ .saturating_sub(replaced)
+ .saturating_add(len)
+ <= crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL
+ }
+
+ fn inline_bytes(&self) -> usize {
+ self.slots
+ .values()
+ .filter_map(|slot| slot.layer.as_ref())
+ .filter_map(Layer::inline_data)
+ .map(<[u8]>::len)
+ .sum()
+ }
+
+ pub(crate) fn active_for_pane(&self, pane_id: PaneId) -> bool {
+ self.slots
+ .iter()
+ .any(|((id, _), slot)| *id == pane_id && slot.layer.is_some())
+ }
+
+ pub(crate) fn attach_stream_active(
+ &mut self,
+ key: &Key,
+ owner: &str,
+ active: std::sync::Arc<std::sync::atomic::AtomicBool>,
+ ) {
+ if let Some(slot) = self
+ .slots
+ .get_mut(key)
+ .filter(|slot| slot.stream_owner.as_deref() == Some(owner))
+ {
+ slot.stream_active = Some(active);
+ }
+ }
+
+ pub(crate) fn retain_live_panes(&mut self, state: &crate::app::state::AppState) -> bool {
+ if self.slots.is_empty() {
+ return false;
+ }
+ let before = self.slots.len();
+ self.slots.retain(|(pane_id, _), _| {
+ state
+ .workspaces
+ .iter()
+ .any(|workspace| workspace.pane_state(*pane_id).is_some())
+ });
+ if self.slots.len() == before {
+ return false;
+ }
+ self.mark_changed();
+ true
+ }
+}
+
+impl Layer {
+ pub(crate) fn inline(
+ format: PaneGraphicsFormat,
+ image_width: u32,
+ image_height: u32,
+ data: Vec<u8>,
+ render: PaneGraphicsPlacementParams,
+ z_index: i32,
+ ) -> Self {
+ let mut hasher = std::collections::hash_map::DefaultHasher::new();
+ data.hash(&mut hasher);
+ Self {
+ format,
+ image_width,
+ image_height,
+ backing: Backing::Inline(data),
+ data_fingerprint: hasher.finish(),
+ render,
+ z_index,
+ }
+ }
+
+ pub(crate) fn direct(
+ image_width: u32,
+ image_height: u32,
+ lease: crate::pane_graphics_files::Lease,
+ render: PaneGraphicsPlacementParams,
+ z_index: i32,
+ ) -> Self {
+ let data_fingerprint = lease.fingerprint();
+ Self {
+ format: PaneGraphicsFormat::Rgba,
+ image_width,
+ image_height,
+ backing: Backing::Direct(lease),
+ data_fingerprint,
+ render,
+ z_index,
+ }
+ }
+
+ pub(crate) fn data_len(&self) -> usize {
+ match &self.backing {
+ Backing::Inline(data) => data.len(),
+ Backing::Direct(lease) => lease.len(),
+ Backing::Resident { len, .. } => *len,
+ }
+ }
+
+ pub(crate) fn inline_data(&self) -> Option<&[u8]> {
+ match &self.backing {
+ Backing::Inline(data) => Some(data),
+ Backing::Direct(_) | Backing::Resident { .. } => None,
+ }
+ }
+
+ pub(crate) fn direct_lease(&self) -> Option<&crate::pane_graphics_files::Lease> {
+ match &self.backing {
+ Backing::Direct(lease) => Some(lease),
+ Backing::Inline(_) | Backing::Resident { .. } => None,
+ }
+ }
+
+ pub(crate) fn mark_resident(&mut self, client_id: u64) -> bool {
+ let Backing::Direct(lease) = &self.backing else {
+ return false;
+ };
+ self.backing = Backing::Resident {
+ len: lease.len(),
+ client_id,
+ };
+ true
+ }
+
+ pub(crate) fn resident_client(&self) -> Option<u64> {
+ match self.backing {
+ Backing::Resident { client_id, .. } => Some(client_id),
+ Backing::Inline(_) | Backing::Direct(_) => None,
+ }
+ }
+
+ pub(crate) fn terminal_only(&self) -> bool {
+ matches!(self.backing, Backing::Direct(_) | Backing::Resident { .. })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn stream_liveness_attaches_only_to_the_exact_owned_layer() {
+ let mut runtime = Runtime::default();
+ let first = (PaneId::from_raw(1), "first".into());
+ let second = (PaneId::from_raw(2), "second".into());
+ for (key, id) in [(&first, 1), (&second, 2)] {
+ runtime.slots.insert(
+ key.clone(),
+ Slot {
+ host_image_id: (1 << 31) | id,
+ layer: None,
+ stream_owner: Some("shared-owner".into()),
+ stream_active: None,
+ direct_gate: None,
+ },
+ );
+ }
+ let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true));
+
+ runtime.attach_stream_active(&second, "shared-owner", active.clone());
+
+ assert!(runtime.slots[&first].stream_active.is_none());
+ assert!(std::sync::Arc::ptr_eq(
+ runtime.slots[&second].stream_active.as_ref().unwrap(),
+ &active
+ ));
+ }
+
+ #[test]
+ fn runtime_bounds_total_slots_and_inline_bytes() {
+ let mut runtime = Runtime::default();
+ for index in 0..crate::api::schema::panes::PANE_GRAPHICS_MAX_LAYERS_TOTAL {
+ let key = (PaneId::from_raw(1), format!("layer-{index}"));
+ runtime
+ .slots
+ .insert(key, Slot::test((1 << 31) | index as u32, None));
+ }
+ assert!(!runtime.can_add_slot(&(PaneId::from_raw(2), "extra".into())));
+
+ let mut runtime = Runtime::default();
+ let key = (PaneId::from_raw(1), "primary".into());
+ runtime.slots.insert(
+ key.clone(),
+ Slot::test(
+ 1 << 31,
+ Some(Layer::inline(
+ PaneGraphicsFormat::Rgba,
+ 1,
+ 1,
+ vec![0; 4],
+ PaneGraphicsPlacementParams::default(),
+ 0,
+ )),
+ ),
+ );
+ assert!(runtime.can_store_inline(
+ &key,
+ crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL,
+ ));
+ assert!(!runtime.can_store_inline(
+ &(PaneId::from_raw(2), "primary".into()),
+ crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL,
+ ));
+ }
+}
diff --git a/src/app/runtime.rs b/src/app/runtime.rs
index 49420381..31426f87 100644
--- a/src/app/runtime.rs
+++ b/src/app/runtime.rs
@@ -61,6 +61,11 @@ impl App {
msg: crate::api::ApiRequestMessage,
) -> bool {
let previous_mode = self.state.mode;
+ let stream_open = match &msg.request.method {
+ crate::api::schema::Method::PaneGraphicsStreamOpen(params) => Some(params.clone()),
+ _ => None,
+ };
+ let stream_active = msg.stream_active.clone();
let mut changed = self.expire_due_metadata(Instant::now());
changed |= crate::api::request_changes_ui(&msg.request);
let skip_default_workspace = matches!(
@@ -83,6 +88,9 @@ impl App {
return changed | deferred_changed;
}
let response = self.handle_api_request(msg.request);
+ if let (Some(params), Some(active)) = (stream_open.as_ref(), stream_active) {
+ self.attach_pane_graphics_stream_active(params, active, &response);
+ }
if !skip_default_workspace {
changed |= self.ensure_default_workspace();
}
diff --git a/src/app/state.rs b/src/app/state.rs
index 04876290..e460f62b 100644
--- a/src/app/state.rs
+++ b/src/app/state.rs
@@ -4,7 +4,6 @@ use crate::config::{
use crossterm::event::{KeyCode, KeyModifiers};
use ratatui::layout::{Direction, Rect};
use ratatui::style::Color;
-use std::hash::{Hash, Hasher};
use crate::detect::AgentState;
use crate::layout::{PaneId, PaneInfo, SplitBorder};
@@ -12,7 +11,6 @@ use crate::selection::Selection;
pub(crate) type InstalledPluginRegistry =
std::collections::HashMap<String, crate::api::schema::InstalledPluginInfo>;
-
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct PluginPaneRecord {
pub plugin_id: String,
@@ -20,42 +18,6 @@ pub(crate) struct PluginPaneRecord {
}
#[derive(Clone, Debug, PartialEq, Eq)]
-pub(crate) struct PaneGraphicsLayer {
- pub format: crate::api::schema::PaneGraphicsFormat,
- pub image_width: u32,
- pub image_height: u32,
- pub data: Vec<u8>,
- pub data_fingerprint: u64,
- pub render: crate::api::schema::PaneGraphicsPlacementParams,
-}
-
-impl PaneGraphicsLayer {
- pub(crate) fn new(
- format: crate::api::schema::PaneGraphicsFormat,
- image_width: u32,
- image_height: u32,
- data: Vec<u8>,
- render: crate::api::schema::PaneGraphicsPlacementParams,
- ) -> Self {
- let data_fingerprint = pane_graphics_data_fingerprint(&data);
- Self {
- format,
- image_width,
- image_height,
- data,
- data_fingerprint,
- render,
- }
- }
-}
-
-fn pane_graphics_data_fingerprint(data: &[u8]) -> u64 {
- let mut hasher = std::collections::hash_map::DefaultHasher::new();
- data.hash(&mut hasher);
- hasher.finish()
-}
-
-#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct PopupPaneState {
pub pane_id: PaneId,
pub terminal_id: crate::terminal::TerminalId,
@@ -1520,12 +1482,6 @@ pub struct AppState {
pub(crate) installed_plugins: InstalledPluginRegistry,
/// Pane ids opened through the plugin pane API.
pub(crate) plugin_panes: std::collections::HashMap<PaneId, PluginPaneRecord>,
- /// Runtime image layers owned by API clients and composited over panes.
- pub(crate) pane_graphics_layers: std::collections::HashMap<PaneId, PaneGraphicsLayer>,
- /// Active streaming graphics owner token by pane id.
- pub(crate) pane_graphics_streams: std::collections::HashMap<PaneId, String>,
- /// Monotonic marker for accepted pane graphics mutations.
- pub(crate) pane_graphics_revision: u64,
/// Session-modal terminal popup. This is intentionally outside workspace layouts.
pub(crate) popup_pane: Option<PopupPaneState>,
/// Recent plugin action/event command executions.
@@ -1538,6 +1494,8 @@ pub struct AppState {
pub host_terminal_theme: TerminalTheme,
/// Last known foreground host terminal cell size in pixels.
pub(crate) host_cell_size: crate::kitty_graphics::HostCellSize,
+ /// Exact pixel provenance only while one confirmed SGR report is dispatched.
+ pub(crate) host_mouse_pixels: Option<crate::input::mouse::HostPixels>,
/// Set when a persisted session snapshot would change.
pub session_dirty: bool,
/// Terminal runtimes that should be shut down by the app/runtime layer
@@ -1884,9 +1842,6 @@ impl AppState {
integration_install_messages: Vec::new(),
installed_plugins: std::collections::HashMap::new(),
plugin_panes: std::collections::HashMap::new(),
- pane_graphics_layers: std::collections::HashMap::new(),
- pane_graphics_streams: std::collections::HashMap::new(),
- pane_graphics_revision: 0,
popup_pane: None,
plugin_command_logs: Vec::new(),
next_plugin_command_log_id: 1,
@@ -1894,6 +1849,7 @@ impl AppState {
global_menu: MenuListState::new(0),
host_terminal_theme: TerminalTheme::default(),
host_cell_size: crate::kitty_graphics::HostCellSize::default(),
+ host_mouse_pixels: None,
session_dirty: false,
terminal_runtime_shutdowns: Vec::new(),
}
@@ -2000,6 +1956,10 @@ impl AppState {
self.context_menu.is_none(),
"empty app state must not keep context menu"
);
+ assert!(
+ self.host_mouse_pixels.is_none(),
+ "empty app state must not keep host mouse pixel provenance"
+ );
return;
}
diff --git a/src/client/direct_graphics.rs b/src/client/direct_graphics.rs
new file mode 100644
index 00000000..1bf10041
--- /dev/null
+++ b/src/client/direct_graphics.rs
@@ -0,0 +1,393 @@
+const MAX_RESPONSE_BYTES: usize = 4096;
+const KITTY_PREFIX: &[u8] = b"\x1b_G";
+const KITTY_SUFFIX: &[u8] = b"\x1b\\";
+const RESPONSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3);
+const LATE_RESPONSE_DRAIN: std::time::Duration = std::time::Duration::from_secs(1);
+
+#[derive(Debug, PartialEq, Eq)]
+pub(super) struct Response {
+ pub(super) transfer_id: u64,
+ pub(super) image_id: u32,
+ pub(super) success: bool,
+}
+
+#[derive(Debug, Default)]
+pub(super) struct ResponseMatcher {
+ expected: Option<(u64, u32, Option<std::time::Instant>)>,
+ retired: Option<(u32, std::time::Instant)>,
+ active: std::sync::Arc<std::sync::atomic::AtomicBool>,
+}
+
+impl ResponseMatcher {
+ pub(super) fn active_handle(&self) -> std::sync::Arc<std::sync::atomic::AtomicBool> {
+ self.active.clone()
+ }
+
+ fn refresh_active(&self) {
+ self.active.store(
+ self.expected.is_some() || self.retired.is_some(),
+ std::sync::atomic::Ordering::Release,
+ );
+ }
+ pub(super) fn interested(&mut self) -> bool {
+ self.expire();
+ self.expected.is_some() || self.retired.is_some()
+ }
+ pub(super) fn arm(&mut self, transfer_id: u64, image_id: u32) -> bool {
+ self.expire();
+ if self.expected.is_some() {
+ return false;
+ }
+ self.expected = Some((transfer_id, image_id, None));
+ self.refresh_active();
+ true
+ }
+
+ pub(super) fn start(&mut self, transfer_id: u64) {
+ self.start_at(transfer_id, std::time::Instant::now());
+ }
+
+ fn start_at(&mut self, transfer_id: u64, now: std::time::Instant) {
+ if let Some((id, _, deadline)) = &mut self.expected {
+ if *id == transfer_id {
+ *deadline = Some(now + RESPONSE_TIMEOUT);
+ }
+ }
+ }
+
+ pub(super) fn cancel(&mut self, transfer_id: u64) {
+ if self.expected.is_some_and(|(id, _, _)| id == transfer_id) {
+ self.expected = None;
+ self.refresh_active();
+ }
+ }
+
+ pub(super) fn retire(&mut self, transfer_id: u64) {
+ if self.expected.is_some_and(|(id, _, _)| id == transfer_id) {
+ if let Some((_, image_id, _)) = self.expected.take() {
+ self.retired = Some((image_id, std::time::Instant::now() + LATE_RESPONSE_DRAIN));
+ self.refresh_active();
+ }
+ }
+ }
+
+ pub(super) fn expire(&mut self) {
+ self.expire_at(std::time::Instant::now());
+ }
+
+ fn expire_at(&mut self, now: std::time::Instant) {
+ if self
+ .expected
+ .is_some_and(|(_, _, deadline)| deadline.is_some_and(|deadline| deadline <= now))
+ {
+ if let Some((_, image_id, _)) = self.expected.take() {
+ self.retired = Some((image_id, now + LATE_RESPONSE_DRAIN));
+ }
+ }
+ if self.retired.is_some_and(|(_, deadline)| deadline <= now) {
+ self.retired = None;
+ }
+ self.refresh_active();
+ }
+
+ pub(super) fn consume(&mut self, bytes: &[u8]) -> Option<Option<Response>> {
+ self.expire();
+ if !bytes.starts_with(b"\x1b_G")
+ || bytes.len() > MAX_RESPONSE_BYTES
+ || !bytes.ends_with(b"\x1b\\")
+ {
+ return None;
+ }
+ let payload = &bytes[3..bytes.len() - 2];
+ let separator = payload.iter().position(|byte| *byte == b';')?;
+ if let Some((retired_id, _)) = self.retired {
+ if matching_response_controls(&payload[..separator], retired_id) {
+ self.retired = None;
+ self.refresh_active();
+ return Some(None);
+ }
+ }
+ let (transfer_id, image_id, _) = self.expected?;
+ if !matching_response_controls(&payload[..separator], image_id) {
+ return None;
+ }
+ self.expected = None;
+ self.refresh_active();
+ Some(Some(Response {
+ transfer_id,
+ image_id,
+ success: &payload[separator + 1..] == b"OK",
+ }))
+ }
+}
+
+#[derive(Default)]
+pub(super) struct InputFilter {
+ pending: Vec<u8>,
+}
+
+impl InputFilter {
+ pub(super) fn has_pending(&self) -> bool {
+ !self.pending.is_empty()
+ }
+
+ pub(super) fn flush_if_inactive(&mut self, matcher: &mut ResponseMatcher) -> Option<Vec<u8>> {
+ (self.has_pending() && !matcher.interested()).then(|| std::mem::take(&mut self.pending))
+ }
+
+ pub(super) fn push(
+ &mut self,
+ bytes: &[u8],
+ matcher: &mut ResponseMatcher,
+ ) -> (Vec<Vec<u8>>, Vec<Response>) {
+ self.pending.extend_from_slice(bytes);
+ let mut output = Vec::new();
+ let mut responses = Vec::new();
+ if !matcher.interested() {
+ output.push(std::mem::take(&mut self.pending));
+ return (output, responses);
+ }
+ loop {
+ let Some(start) = self
+ .pending
+ .windows(3)
+ .position(|window| window == KITTY_PREFIX)
+ else {
+ let keep = (1..KITTY_PREFIX.len())
+ .rev()
+ .find(|len| self.pending.ends_with(&KITTY_PREFIX[..*len]))
+ .unwrap_or(0);
+ let emit = self.pending.len().saturating_sub(keep);
+ if emit > 0 {
+ output.push(self.pending.drain(..emit).collect());
+ }
+ break;
+ };
+ if start > 0 {
+ output.push(self.pending.drain(..start).collect());
+ }
+ let end = self
+ .pending
+ .windows(2)
+ .position(|window| window == KITTY_SUFFIX);
+ let Some(end) = end else {
+ if self.pending.len() > MAX_RESPONSE_BYTES {
+ output.push(std::mem::take(&mut self.pending));
+ }
+ break;
+ };
+ let command: Vec<u8> = self.pending.drain(..end + 2).collect();
+ match matcher.consume(&command) {
+ Some(Some(response)) => responses.push(response),
+ Some(None) => {}
+ None => output.push(command),
+ }
+ }
+ (output, responses)
+ }
+}
+
+fn matching_response_controls(bytes: &[u8], expected: u32) -> bool {
+ let mut matched = false;
+ for field in bytes.split(|byte| *byte == b',') {
+ let Some(separator) = field.iter().position(|byte| *byte == b'=') else {
+ return false;
+ };
+ let (key, value) = field.split_at(separator);
+ let value = &value[1..];
+ if value.is_empty() || !value.iter().all(u8::is_ascii_digit) {
+ return false;
+ }
+ match key {
+ b"i" if !matched
+ && std::str::from_utf8(value).ok().and_then(|v| v.parse().ok())
+ == Some(expected) =>
+ {
+ matched = true
+ }
+ b"I" | b"p" => {}
+ _ => return false,
+ }
+ }
+ matched
+}
+
+pub(super) fn valid_control(control: &str, image_id: u32) -> bool {
+ if control.len() > 1024 || control.contains([';', '\x1b']) {
+ return false;
+ }
+ let mut action = false;
+ let mut format = false;
+ let mut image = false;
+ let mut quiet = false;
+ let mut cursor = false;
+ for field in control.split(',') {
+ let Some((key, value)) = field.split_once('=') else {
+ return false;
+ };
+ if key == "t"
+ || !matches!(
+ key,
+ "a" | "f"
+ | "s"
+ | "v"
+ | "i"
+ | "p"
+ | "c"
+ | "r"
+ | "z"
+ | "C"
+ | "q"
+ | "x"
+ | "y"
+ | "w"
+ | "h"
+ | "X"
+ | "Y"
+ )
+ {
+ return false;
+ }
+ let numeric = value
+ .strip_prefix('-')
+ .unwrap_or(value)
+ .bytes()
+ .all(|byte| byte.is_ascii_digit())
+ && !value.is_empty();
+ if key != "a" && !numeric {
+ return false;
+ }
+ match key {
+ "a" => action = value == "T",
+ "f" => format = value == "32",
+ "i" => image = value.parse() == Ok(image_id),
+ "q" => quiet = value == "0",
+ "C" => cursor = value == "1",
+ _ => {}
+ }
+ }
+ action && format && image && quiet && cursor
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn matches_only_the_armed_image_and_preserves_unrelated_input() {
+ let mut matcher = ResponseMatcher::default();
+ let active = matcher.active_handle();
+ assert!(matcher.arm(7, 42));
+ assert!(active.load(std::sync::atomic::Ordering::Acquire));
+ assert!(!matcher.arm(8, 43));
+ assert_eq!(matcher.consume(b"typed"), None);
+ assert_eq!(matcher.consume(b"\x1b_Gi=41;OK\x1b\\"), None);
+ assert_eq!(
+ matcher.consume(b"\x1b_Gi=42;OK\x1b\\"),
+ Some(Some(Response {
+ transfer_id: 7,
+ image_id: 42,
+ success: true,
+ }))
+ );
+ assert!(!active.load(std::sync::atomic::Ordering::Acquire));
+ }
+
+ #[test]
+ fn explicit_error_is_reported_and_malformed_responses_are_ignored() {
+ let mut matcher = ResponseMatcher::default();
+ matcher.arm(9, 44);
+ assert_eq!(
+ matcher.consume(b"\x1b_Gi=44;ENOENT\x1b\\"),
+ Some(Some(Response {
+ transfer_id: 9,
+ image_id: 44,
+ success: false,
+ }))
+ );
+ matcher.arm(10, 45);
+ assert_eq!(matcher.consume(b"\x1b_Gi=45;broken"), None);
+ matcher.cancel(10);
+ matcher.arm(11, 46);
+ assert_eq!(matcher.consume(b"\x1b_Gi=46oops;OK\x1b\\"), None);
+ }
+
+ #[test]
+ fn validated_control_is_one_owned_rgba_transmit_and_display() {
+ assert!(valid_control(
+ "a=T,f=32,s=10,v=20,i=42,p=7,c=5,r=6,z=-1,C=1,q=0,x=2",
+ 42
+ ));
+ for invalid in [
+ "a=T,f=24,i=42,C=1,q=0",
+ "a=T,f=32,i=41,C=1,q=0",
+ "a=T,t=f,f=32,i=42,C=1,q=0",
+ "a=p,f=32,i=42,C=1,q=0",
+ ] {
+ assert!(!valid_control(invalid, 42), "{invalid}");
+ }
+ }
+
+ #[test]
+ fn fragmented_response_is_held_before_generic_input_timeout() {
+ let mut matcher = ResponseMatcher::default();
+ let mut filter = InputFilter::default();
+ matcher.arm(19, 49);
+ for foreign in [
+ b"\x1b_Ga=p,i=49;OK\x1b\\".as_slice(),
+ b"\x1b_Gi=49oops;OK\x1b\\",
+ ] {
+ let (output, responses) = filter.push(foreign, &mut matcher);
+ assert_eq!(output, [foreign.to_vec()]);
+ assert!(responses.is_empty());
+ }
+ let (output, responses) = filter.push(b"typed\x1b_Gi=49;", &mut matcher);
+ assert_eq!(output, [b"typed".to_vec()]);
+ assert!(responses.is_empty());
+ let (output, responses) = filter.push(b"OK\x1b\\tail", &mut matcher);
+ assert_eq!(output, [b"tail".to_vec()]);
+ assert_eq!(responses[0].transfer_id, 19);
+ assert!(responses[0].success);
+ }
+
+ #[test]
+ fn attempted_output_failure_retires_response_without_forwarding_it() {
+ let mut matcher = ResponseMatcher::default();
+ let mut filter = InputFilter::default();
+ matcher.arm(20, 50);
+ matcher.retire(20);
+ for foreign in [
+ b"\x1b_Ga=p,i=50;OK\x1b\\".as_slice(),
+ b"\x1b_Gi=50oops;OK\x1b\\",
+ ] {
+ let (output, responses) = filter.push(foreign, &mut matcher);
+ assert_eq!(output, [foreign.to_vec()]);
+ assert!(responses.is_empty());
+ }
+ assert!(filter
+ .push(b"\x1b_Gi=50;EINVAL\x1b\\", &mut matcher)
+ .0
+ .is_empty());
+ }
+
+ #[test]
+ fn timed_out_expectation_requires_a_new_transfer_before_matching() {
+ let mut matcher = ResponseMatcher::default();
+ let started = std::time::Instant::now();
+ assert!(matcher.arm(12, 47));
+ matcher.start_at(12, started);
+ assert!(!matcher.arm(13, 48));
+ matcher.expire_at(started + RESPONSE_TIMEOUT);
+ assert!(matcher.arm(13, 48));
+ assert_eq!(matcher.consume(b"typed"), None);
+ assert_eq!(matcher.consume(b"\x1b_Gi=47;OK\x1b\\"), Some(None));
+ assert_eq!(
+ matcher.consume(b"\x1b_Gi=48;OK\x1b\\"),
+ Some(Some(Response {
+ transfer_id: 13,
+ image_id: 48,
+ success: true,
+ }))
+ );
+ }
+}
diff --git a/src/client/input.rs b/src/client/input.rs
index 4dc26eae..456f59b5 100644
--- a/src/client/input.rs
+++ b/src/client/input.rs
@@ -41,6 +41,9 @@ pub fn stdin_reader_loop(
host_color_query_sent: bool,
host_cell_size_query_sent: bool,
host_mouse_capture_active: Arc<AtomicBool>,
+ host_sgr_pixels_active: Arc<AtomicBool>,
+ #[cfg(unix)] direct_response: Arc<std::sync::Mutex<super::direct_graphics::ResponseMatcher>>,
+ #[cfg(unix)] direct_response_active: Arc<AtomicBool>,
) {
#[cfg(windows)]
{
@@ -48,6 +51,7 @@ pub fn stdin_reader_loop(
host_color_query_sent,
host_cell_size_query_sent,
host_mouse_capture_active,
+ host_sgr_pixels_active,
);
windows_stdin_reader_loop(event_tx, should_quit);
}
@@ -59,6 +63,9 @@ pub fn stdin_reader_loop(
host_color_query_sent,
host_cell_size_query_sent,
host_mouse_capture_active,
+ host_sgr_pixels_active,
+ direct_response,
+ direct_response_active,
);
}
@@ -69,6 +76,9 @@ fn unix_stdin_reader_loop(
host_color_query_sent: bool,
host_cell_size_query_sent: bool,
host_mouse_capture_active: Arc<AtomicBool>,
+ host_sgr_pixels_active: Arc<AtomicBool>,
+ direct_response: Arc<std::sync::Mutex<super::direct_graphics::ResponseMatcher>>,
+ direct_response_active: Arc<AtomicBool>,
) {
let stdin = io::stdin();
let mut reader = stdin.lock();
@@ -83,15 +93,71 @@ fn unix_stdin_reader_loop(
framer.host_cell_size_query_sent();
}
let mut pending_palette = Vec::new();
+ let mut pending_mode = None;
+ let mut last_geometry = None;
+ let mut direct_filter = super::direct_graphics::InputFilter::default();
while !should_quit.load(Ordering::Acquire) {
+ if direct_filter.has_pending()
+ && stdin_read_ready(&reader, crate::raw_input::RAW_INPUT_IDLE_FLUSH_TIMEOUT_MS)
+ == Some(false)
+ {
+ let released = direct_response
+ .lock()
+ .ok()
+ .and_then(|mut matcher| direct_filter.flush_if_inactive(&mut matcher));
+ if let Some(data) = released {
+ if event_tx
+ .blocking_send(ClientLoopEvent::StdinInput(data))
+ .is_err()
+ {
+ return;
+ }
+ }
+ continue;
+ }
match reader.read(&mut scratch) {
Ok(0) => break,
Ok(n) => {
+ let sgr_pixels = *pending_mode
+ .get_or_insert_with(|| host_sgr_pixels_active.load(Ordering::Acquire));
+ if sgr_pixels {
+ last_geometry = retain_geometry(
+ last_geometry,
+ crate::input::mouse::HostGeometry::current(),
+ );
+ }
+ let filtered = filter_direct_input(
+ &scratch[..n],
+ &mut direct_filter,
+ &direct_response,
+ &direct_response_active,
+ );
+ let chunks = if let Some((raw_chunks, responses)) = filtered {
+ for response in responses {
+ if event_tx
+ .blocking_send(ClientLoopEvent::DirectGraphicsResponse(response))
+ .is_err()
+ {
+ return;
+ }
+ }
+ raw_chunks
+ .into_iter()
+ .flat_map(|chunk| framer.push(&chunk))
+ .collect()
+ } else {
+ framer.push(&scratch[..n])
+ };
+ if !framer.has_pending_input() {
+ pending_mode = None;
+ }
if !send_unix_input_chunks(
- framer.push(&scratch[..n]),
+ chunks,
&event_tx,
&mut pending_palette,
+ sgr_pixels,
+ last_geometry,
) {
return;
}
@@ -104,8 +170,18 @@ fn unix_stdin_reader_loop(
let had_pending = framer.has_pending_input();
let chunks = framer.flush_timeout();
let held_escape = had_pending && chunks.is_empty();
- if !send_unix_input_chunks(chunks, &event_tx, &mut pending_palette)
- || !flush_unix_palette_input(&event_tx, &mut pending_palette)
+ let sgr_pixels = pending_mode
+ .unwrap_or_else(|| host_sgr_pixels_active.load(Ordering::Acquire));
+ if !framer.has_pending_input() {
+ pending_mode = None;
+ }
+ if !send_unix_input_chunks(
+ chunks,
+ &event_tx,
+ &mut pending_palette,
+ sgr_pixels,
+ last_geometry,
+ ) || !flush_unix_palette_input(&event_tx, &mut pending_palette)
{
return;
}
@@ -114,13 +190,20 @@ fn unix_stdin_reader_loop(
&reader,
crate::raw_input::RAW_INPUT_IDLE_FLUSH_TIMEOUT_MS,
) == Some(false)
- && !send_unix_input_chunks(
- framer.flush_timeout(),
+ {
+ let chunks = framer.flush_timeout();
+ if !framer.has_pending_input() {
+ pending_mode = None;
+ }
+ if !send_unix_input_chunks(
+ chunks,
&event_tx,
&mut pending_palette,
- )
- {
- return;
+ sgr_pixels,
+ last_geometry,
+ ) {
+ return;
+ }
}
}
}
@@ -135,10 +218,30 @@ fn unix_stdin_reader_loop(
}
#[cfg(unix)]
+fn filter_direct_input(
+ bytes: &[u8],
+ filter: &mut super::direct_graphics::InputFilter,
+ response: &std::sync::Mutex<super::direct_graphics::ResponseMatcher>,
+ active: &AtomicBool,
+) -> Option<(Vec<Vec<u8>>, Vec<super::direct_graphics::Response>)> {
+ if !active.load(Ordering::Acquire) && !filter.has_pending() {
+ return None;
+ }
+ Some(
+ response
+ .lock()
+ .map(|mut matcher| filter.push(bytes, &mut matcher))
+ .unwrap_or_else(|_| (vec![bytes.to_vec()], Vec::new())),
+ )
+}
+
+#[cfg(unix)]
fn send_unix_input_chunks(
chunks: Vec<Vec<u8>>,
event_tx: &mpsc::Sender<ClientLoopEvent>,
pending_palette: &mut Vec<Vec<u8>>,
+ sgr_pixels: bool,
+ geometry: Option<crate::input::mouse::HostGeometry>,
) -> bool {
for data in chunks {
let palette_response = std::str::from_utf8(&data)
@@ -160,10 +263,10 @@ fn send_unix_input_chunks(
if !default_color_response && !flush_unix_palette_input(event_tx, pending_palette) {
return false;
}
- if event_tx
- .blocking_send(ClientLoopEvent::StdinInput(data))
- .is_err()
- {
+ let Some(event) = classify_unix_input(data, sgr_pixels, geometry) else {
+ continue;
+ };
+ if event_tx.blocking_send(event).is_err() {
return false;
}
}
@@ -171,6 +274,26 @@ fn send_unix_input_chunks(
}
#[cfg(unix)]
+fn retain_geometry(
+ last: Option<crate::input::mouse::HostGeometry>,
+ observed: Option<crate::input::mouse::HostGeometry>,
+) -> Option<crate::input::mouse::HostGeometry> {
+ observed.or(last)
+}
+
+#[cfg(unix)]
+fn classify_unix_input(
+ data: Vec<u8>,
+ sgr_pixels: bool,
+ geometry: Option<crate::input::mouse::HostGeometry>,
+) -> Option<ClientLoopEvent> {
+ if sgr_pixels && crate::input::mouse::parse_report(&data).is_some() {
+ return geometry.map(|geometry| ClientLoopEvent::PixelMouse(data, geometry));
+ }
+ Some(ClientLoopEvent::StdinInput(data))
+}
+
+#[cfg(unix)]
fn flush_unix_palette_input(
event_tx: &mpsc::Sender<ClientLoopEvent>,
pending_palette: &mut Vec<Vec<u8>>,
@@ -496,6 +619,50 @@ mod tests {
}
#[test]
+ fn inactive_direct_input_bypasses_filter() {
+ let response =
+ std::sync::Mutex::new(super::super::direct_graphics::ResponseMatcher::default());
+ let active = response.lock().unwrap().active_handle();
+ let mut filter = super::super::direct_graphics::InputFilter::default();
+ assert!(filter_direct_input(b"typed", &mut filter, &response, &active).is_none());
+ assert!(!filter.has_pending());
+ }
+
+ #[test]
+ fn pixel_mouse_classification_is_narrow_and_uses_read_geometry() {
+ let geometry = crate::input::mouse::HostGeometry::new(80, 24, 800, 480).unwrap();
+ let report = b"\x1b[<35;321;241M".to_vec();
+ let Some(ClientLoopEvent::PixelMouse(data, captured)) =
+ classify_unix_input(report.clone(), true, Some(geometry))
+ else {
+ panic!("expected dedicated pixel mouse event");
+ };
+ assert_eq!(data, report);
+ assert_eq!(captured, geometry);
+ assert!(classify_unix_input(report, true, None).is_none());
+
+ for raw in [
+ b"key".as_slice(),
+ b"\x1b[200~paste\x1b[201~".as_slice(),
+ b"\x1b_Gi=7;unrelated\x1b\\".as_slice(),
+ b"\x1b[<35;2;3Mtail".as_slice(),
+ ] {
+ let Some(ClientLoopEvent::StdinInput(data)) =
+ classify_unix_input(raw.to_vec(), true, Some(geometry))
+ else {
+ panic!("unrelated input must remain raw");
+ };
+ assert_eq!(data, raw);
+ }
+ }
+
+ #[test]
+ fn transient_geometry_failure_keeps_last_real_value() {
+ let geometry = crate::input::mouse::HostGeometry::new(80, 24, 800, 480).unwrap();
+ assert_eq!(retain_geometry(Some(geometry), None), Some(geometry));
+ }
+
+ #[test]
fn palette_replies_are_forwarded_as_one_input_batch() {
let (tx, mut rx) = mpsc::channel(4);
let mut pending = Vec::new();
@@ -506,6 +673,8 @@ mod tests {
],
&tx,
&mut pending,
+ false,
+ None,
));
assert!(rx.try_recv().is_err());
diff --git a/src/client/mod.rs b/src/client/mod.rs
index 7932cb79..301377c9 100644
--- a/src/client/mod.rs
+++ b/src/client/mod.rs
@@ -12,9 +12,13 @@
//! - Forwards OSC 52 clipboard writes from server to its own stdout
//! - Displays sound/toast notifications forwarded from server
+#[cfg(unix)]
+mod direct_graphics;
mod input;
use std::collections::HashSet;
+#[cfg(unix)]
+use std::io::IsTerminal as _;
use std::io::{self, BufRead, Write as _};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
@@ -75,6 +79,12 @@ struct ClientState {
sound_config: crate::config::SoundConfig,
/// Whether this client may write Kitty graphics bytes to its host terminal.
kitty_graphics_enabled: bool,
+ /// One bounded matcher, inactive unless a direct transmission is armed.
+ #[cfg(unix)]
+ direct_graphics_response: Arc<Mutex<direct_graphics::ResponseMatcher>>,
+ /// One server-retired direct transfer to suppress if it was still queued.
+ #[cfg(unix)]
+ retired_direct_graphics: Option<(u64, u32)>,
/// Direct attach prefix escape state. None for full-app clients.
attach_escape: Option<AttachEscapeState>,
/// Rows scrolled for one direct-attach wheel notch.
@@ -356,9 +366,9 @@ fn setup_terminal_with_capabilities(
if enable_client_protocols {
if mouse_capture {
- set_mouse_capture(true)?;
+ set_mouse_capture(true, false)?;
} else {
- set_mouse_capture(false)?;
+ set_mouse_capture(false, false)?;
}
execute!(io::stdout(), EnableBracketedPaste, EnableFocusChange)?;
if host_color_scheme_reports {
@@ -370,9 +380,9 @@ fn setup_terminal_with_capabilities(
write_host_color_scheme_report_mode(&mut io::stdout(), false)?;
}
if mouse_capture {
- set_mouse_capture(true)?;
+ set_mouse_capture(true, false)?;
} else {
- set_mouse_capture(false)?;
+ set_mouse_capture(false, false)?;
}
}
@@ -549,10 +559,15 @@ fn restore_windows_input_mode_value(mode: u32) {
}
}
-fn set_mouse_capture(enabled: bool) -> io::Result<()> {
+fn set_mouse_capture(enabled: bool, sgr_pixels: bool) -> io::Result<()> {
crate::terminal_modes::clear_host_mouse_reporting(&mut io::stdout())?;
if enabled {
- execute!(io::stdout(), EnableMouseCapture)
+ execute!(io::stdout(), EnableMouseCapture)?;
+ if sgr_pixels {
+ io::stdout().write_all(b"\x1b[?1016h")?;
+ io::stdout().flush()?;
+ }
+ Ok(())
} else {
match execute!(io::stdout(), DisableMouseCapture) {
Ok(()) => Ok(()),
@@ -685,6 +700,44 @@ fn handshake_read_timeout() -> Duration {
LOCAL_HANDSHAKE_READ_TIMEOUT
}
+#[cfg(any(unix, test))]
+fn direct_graphics_profile_values(
+ term_program: &str,
+ term: &str,
+ kitty_window: bool,
+ blocked_transport: bool,
+ terminals: bool,
+) -> bool {
+ let supported = term_program.eq_ignore_ascii_case("ghostty")
+ || term_program.eq_ignore_ascii_case("wezterm")
+ || matches!(term, "xterm-ghostty" | "xterm-kitty" | "xterm-wezterm")
+ || kitty_window;
+ supported && !blocked_transport && terminals
+}
+
+#[cfg(unix)]
+fn direct_graphics_profile_allowed(direct_attach: bool) -> bool {
+ let term_program = std::env::var("TERM_PROGRAM").unwrap_or_default();
+ let term = std::env::var("TERM").unwrap_or_default();
+ direct_graphics_profile_values(
+ &term_program,
+ &term,
+ std::env::var_os("KITTY_WINDOW_ID").is_some(),
+ direct_attach
+ || is_remote_client_process()
+ || std::env::var_os("SSH_CONNECTION").is_some()
+ || std::env::var_os("SSH_TTY").is_some()
+ || std::env::var_os("TMUX").is_some()
+ || std::env::var_os("STY").is_some(),
+ io::stdin().is_terminal() && io::stdout().is_terminal(),
+ )
+}
+
+#[cfg(not(unix))]
+fn direct_graphics_profile_allowed(_direct_attach: bool) -> bool {
+ false
+}
+
fn requested_keybindings() -> ClientKeybindings {
match std::env::var(crate::remote::REMOTE_KEYBINDINGS_ENV_VAR)
.ok()
@@ -726,6 +779,25 @@ fn set_handshake_recv_timeout(
.map_err(ClientError::ConnectionFailed)
}
+fn client_launch_mode(
+ direct_attach_requested: bool,
+ exact_cell_size: bool,
+ cell_width_px: u32,
+ cell_height_px: u32,
+) -> ClientLaunchMode {
+ if direct_attach_requested {
+ ClientLaunchMode::TerminalAttach
+ } else if exact_cell_size
+ && cell_width_px > 0
+ && cell_height_px > 0
+ && direct_graphics_profile_allowed(false)
+ {
+ ClientLaunchMode::AppDirectGraphics
+ } else {
+ ClientLaunchMode::App
+ }
+}
+
/// Performs the client→server handshake.
///
/// Sends Hello with the terminal size and protocol version, reads the Welcome
@@ -736,6 +808,7 @@ fn do_handshake(
rows: u16,
cell_width_px: u32,
cell_height_px: u32,
+ exact_cell_size: bool,
requested_encoding: RenderEncoding,
direct_attach_requested: bool,
) -> Result<RenderEncoding, ClientError> {
@@ -752,11 +825,12 @@ fn do_handshake(
cell_height_px,
requested_encoding,
keybindings: requested_keybindings(),
- launch_mode: if direct_attach_requested {
- ClientLaunchMode::TerminalAttach
- } else {
- ClientLaunchMode::App
- },
+ launch_mode: client_launch_mode(
+ direct_attach_requested,
+ exact_cell_size,
+ cell_width_px,
+ cell_height_px,
+ ),
};
protocol::write_message(stream, &hello)
.map_err(|e| ClientError::ConnectionFailed(io::Error::other(e.to_string())))?;
@@ -801,6 +875,11 @@ enum ClientLoopEvent {
/// Raw input bytes from stdin.
#[cfg(unix)]
StdinInput(Vec<u8>),
+ /// One confirmed SGR pixel report with geometry captured by the reader.
+ #[cfg(unix)]
+ PixelMouse(Vec<u8>, crate::input::mouse::HostGeometry),
+ #[cfg(unix)]
+ DirectGraphicsResponse(direct_graphics::Response),
/// Structured input events from platforms without Unix-style stdin bytes.
#[cfg(windows)]
StdinEvents(Vec<crate::protocol::ClientInputEvent>),
@@ -929,6 +1008,7 @@ fn connect_terminal_session_stream(
rows,
0,
0,
+ false,
RenderEncoding::TerminalAnsi,
true,
) {
@@ -1149,7 +1229,7 @@ fn run_client_with_mode(
};
// Get the terminal geometry before handshake (before raw mode).
- let (cols, rows, cell_width_px, cell_height_px) =
+ let (cols, rows, cell_width_px, cell_height_px, exact_cell_size) =
initial_terminal_geometry(kitty_graphics_enabled);
// Perform handshake while the stream is still in blocking mode.
@@ -1159,6 +1239,7 @@ fn run_client_with_mode(
rows,
cell_width_px,
cell_height_px,
+ exact_cell_size,
requested_encoding,
direct_attach_requested,
) {
@@ -1296,6 +1377,10 @@ async fn run_client_loop(
reported_size: (cols, rows),
sound_config: config.sound_config,
kitty_graphics_enabled: config.kitty_graphics_enabled,
+ #[cfg(unix)]
+ direct_graphics_response: Arc::new(Mutex::new(direct_graphics::ResponseMatcher::default())),
+ #[cfg(unix)]
+ retired_direct_graphics: None,
attach_escape,
#[cfg(unix)]
mouse_scroll_lines: config.mouse_scroll_lines,
@@ -1309,6 +1394,7 @@ async fn run_client_loop(
// Cell size reported by the host terminal, packed as width<<32 | height.
// Zero means the host has not reported one.
let reported_cell_size = Arc::new(AtomicU64::new(0));
+ let host_sgr_pixels_active = Arc::new(AtomicBool::new(false));
// Channel for events from the stdin, resize, and server reader threads.
let (event_tx, mut event_rx) = tokio::sync::mpsc::channel::<ClientLoopEvent>(256);
@@ -1323,6 +1409,14 @@ async fn run_client_loop(
let stdin_quit = should_quit.clone();
let stdin_tx = event_tx.clone();
let stdin_mouse_capture_active = host_mouse_capture_active.clone();
+ let stdin_sgr_pixels_active = host_sgr_pixels_active.clone();
+ #[cfg(unix)]
+ let stdin_direct_response = state.direct_graphics_response.clone();
+ #[cfg(unix)]
+ let stdin_direct_response_active = stdin_direct_response
+ .lock()
+ .map(|matcher| matcher.active_handle())
+ .unwrap_or_default();
std::thread::spawn(move || {
input::stdin_reader_loop(
stdin_tx,
@@ -1330,6 +1424,11 @@ async fn run_client_loop(
will_query_host_terminal_theme,
will_query_host_cell_size,
stdin_mouse_capture_active,
+ stdin_sgr_pixels_active,
+ #[cfg(unix)]
+ stdin_direct_response,
+ #[cfg(unix)]
+ stdin_direct_response_active,
);
});
@@ -1475,6 +1574,30 @@ async fn run_client_loop(
return Err(ClientError::ConnectionLost(e));
}
}
+ #[cfg(unix)]
+ ClientLoopEvent::DirectGraphicsResponse(response) => {
+ let message = ClientMessage::GraphicsTransmissionResult {
+ transfer_id: response.transfer_id,
+ image_id: response.image_id,
+ success: response.success,
+ };
+ if let Err(err) = write_to_server(&mut write_stream, &message) {
+ return Err(ClientError::ConnectionLost(err));
+ }
+ }
+ #[cfg(unix)]
+ ClientLoopEvent::PixelMouse(data, geometry) => {
+ let message = ClientMessage::InputPixels {
+ data,
+ cols: geometry.cols,
+ rows: geometry.rows,
+ width_px: geometry.width_px,
+ height_px: geometry.height_px,
+ };
+ if let Err(err) = write_to_server(&mut write_stream, &message) {
+ return Err(ClientError::ConnectionLost(err));
+ }
+ }
#[cfg(windows)]
ClientLoopEvent::StdinEvents(events) => {
if state.attach_escape.is_some() {
@@ -1578,6 +1701,98 @@ async fn run_client_loop(
warn!(err = %err, "failed to emit terminal bell");
}
}
+ ServerMessage::GraphicsFile {
+ path,
+ expected_len,
+ image_id,
+ transfer_id,
+ leading,
+ control,
+ } => {
+ #[cfg(unix)]
+ {
+ if state.retired_direct_graphics.take() == Some((transfer_id, image_id)) {
+ continue;
+ }
+ let valid = state.kitty_graphics_enabled
+ && usize::try_from(expected_len).ok().is_some_and(|len| {
+ crate::pane_graphics_files::validate_direct_source(
+ std::path::Path::new(&path),
+ len,
+ )
+ .is_ok()
+ })
+ && direct_graphics::valid_control(&control, image_id)
+ && state
+ .direct_graphics_response
+ .lock()
+ .is_ok_and(|mut matcher| matcher.arm(transfer_id, image_id));
+ let sent = if valid {
+ let mut command = Vec::new();
+ crate::kitty_graphics::encode_kitty_regular_file(
+ &mut command,
+ &leading,
+ &control,
+ &path,
+ );
+ let mut stdout = io::stdout();
+ let written = stdout
+ .write_all(&command)
+ .and_then(|()| stdout.flush())
+ .is_ok();
+ if written {
+ record_received_kitty_graphics(&command);
+ }
+ written
+ } else {
+ false
+ };
+ if sent {
+ if let Ok(mut matcher) = state.direct_graphics_response.lock() {
+ matcher.start(transfer_id);
+ }
+ let started = ClientMessage::GraphicsTransmissionStarted {
+ transfer_id,
+ image_id,
+ };
+ if let Err(err) = write_to_server(&mut write_stream, &started) {
+ return Err(ClientError::ConnectionLost(err));
+ }
+ } else {
+ if let Ok(mut matcher) = state.direct_graphics_response.lock() {
+ if valid {
+ matcher.retire(transfer_id);
+ } else {
+ matcher.cancel(transfer_id);
+ }
+ }
+ let result = ClientMessage::GraphicsTransmissionResult {
+ transfer_id,
+ image_id,
+ success: false,
+ };
+ if let Err(err) = write_to_server(&mut write_stream, &result) {
+ return Err(ClientError::ConnectionLost(err));
+ }
+ }
+ }
+ #[cfg(not(unix))]
+ let _ = (path, expected_len, image_id, transfer_id, leading, control);
+ }
+ ServerMessage::GraphicsTransmissionRetired {
+ transfer_id,
+ image_id,
+ } => {
+ #[cfg(unix)]
+ {
+ state.retired_direct_graphics = Some((transfer_id, image_id));
+ if let Ok(mut matcher) = state.direct_graphics_response.lock() {
+ matcher.retire(transfer_id);
+ }
+ }
+ #[cfg(not(unix))]
+ let _ = (transfer_id, image_id);
+ }
ServerMessage::ServerShutdown { reason } => {
return Err(ClientError::ServerShutdown { reason });
}
@@ -1604,17 +1819,24 @@ async fn run_client_loop(
&mut state.remote_image_paste_key,
);
}
- ServerMessage::MouseCapture { enabled } => {
- let desired = enabled;
- if desired != state.mouse_capture_active {
- set_mouse_capture(desired).map_err(ClientError::ConnectionFailed)?;
+ ServerMessage::MouseCapture {
+ enabled,
+ sgr_pixels,
+ } => {
+ let next_sgr_pixels = enabled && sgr_pixels;
+ let mouse_mode_changed = enabled != state.mouse_capture_active
+ || next_sgr_pixels != host_sgr_pixels_active.load(Ordering::Acquire);
+ if mouse_mode_changed {
+ set_mouse_capture(enabled, next_sgr_pixels)
+ .map_err(ClientError::ConnectionFailed)?;
#[cfg(windows)]
- if windows_vti_input_backend_enabled() {
+ if enabled && windows_vti_input_backend_enabled() {
let _ = enable_windows_virtual_terminal_input();
}
- state.mouse_capture_active = desired;
- host_mouse_capture_active.store(desired, Ordering::Release);
}
+ state.mouse_capture_active = enabled;
+ host_mouse_capture_active.store(enabled, Ordering::Release);
+ host_sgr_pixels_active.store(next_sgr_pixels, Ordering::Release);
}
ServerMessage::KittyKeyboardReportAll { enabled } => {
if enabled != state.keyboard_report_all_active {
@@ -1643,7 +1865,12 @@ async fn run_client_loop(
"server closed connection",
)));
}
- ClientLoopEvent::Timer => {}
+ ClientLoopEvent::Timer => {
+ #[cfg(unix)]
+ if let Ok(mut matcher) = state.direct_graphics_response.lock() {
+ matcher.expire();
+ }
+ }
}
}
@@ -2204,8 +2431,10 @@ fn ioctl_cell_size() -> Option<(u32, u32)> {
}
/// Cell size used when the ioctl reports no pixels.
-fn cell_size_fallback(reported: u64) -> (u32, u32) {
- unpack_cell_size(reported).unwrap_or((DEFAULT_CELL_WIDTH_PX, DEFAULT_CELL_HEIGHT_PX))
+fn cell_size_fallback(reported: u64, last: Option<(u32, u32)>) -> (u32, u32) {
+ unpack_cell_size(reported)
+ .or(last.filter(|(width, height)| *width > 0 && *height > 0))
+ .unwrap_or((DEFAULT_CELL_WIDTH_PX, DEFAULT_CELL_HEIGHT_PX))
}
#[cfg(any(unix, test))]
@@ -2222,20 +2451,35 @@ fn unpack_cell_size(packed: u64) -> Option<(u32, u32)> {
fn current_terminal_geometry(
kitty_graphics_enabled: bool,
reported_cell_size: &AtomicU64,
+ last_cell_size: Option<(u32, u32)>,
) -> (u16, u16, u32, u32) {
let (cols, rows) = crossterm::terminal::size().unwrap_or((80, 24));
if !kitty_graphics_enabled {
return (cols, rows, 0, 0);
}
- let (cell_width_px, cell_height_px) = ioctl_cell_size()
- .unwrap_or_else(|| cell_size_fallback(reported_cell_size.load(Ordering::Acquire)));
+ let (cell_width_px, cell_height_px) = ioctl_cell_size().unwrap_or_else(|| {
+ cell_size_fallback(reported_cell_size.load(Ordering::Acquire), last_cell_size)
+ });
(cols, rows, cell_width_px, cell_height_px)
}
-/// Reads the terminal geometry before the handshake, before any host cell
-/// size report can exist.
-fn initial_terminal_geometry(kitty_graphics_enabled: bool) -> (u16, u16, u32, u32) {
- current_terminal_geometry(kitty_graphics_enabled, &AtomicU64::new(0))
+/// Reads terminal geometry before the handshake. Direct graphics is eligible
+/// only when the host supplied exact pixel dimensions through the ioctl.
+fn initial_terminal_geometry(kitty_graphics_enabled: bool) -> (u16, u16, u32, u32, bool) {
+ let (cols, rows) = crossterm::terminal::size().unwrap_or((80, 24));
+ if !kitty_graphics_enabled {
+ return (cols, rows, 0, 0, false);
+ }
+ match ioctl_cell_size() {
+ Some((width, height)) => (cols, rows, width, height, true),
+ None => (
+ cols,
+ rows,
+ DEFAULT_CELL_WIDTH_PX,
+ DEFAULT_CELL_HEIGHT_PX,
+ false,
+ ),
+ }
}
/// Reports polled changes and signalled resizes that return to the same size.
@@ -2272,7 +2516,11 @@ fn resize_poll_loop(
while !should_quit.load(Ordering::Acquire) {
std::thread::sleep(Duration::from_millis(100));
let signalled = crate::platform::take_terminal_resize_signal();
- let new_size = current_terminal_geometry(kitty_graphics_enabled, reported_cell_size);
+ let new_size = current_terminal_geometry(
+ kitty_graphics_enabled,
+ reported_cell_size,
+ Some((last_size.2, last_size.3)),
+ );
if resize_report_required(signalled, new_size, last_size) {
last_size = new_size;
if resize_tx
@@ -2388,6 +2636,40 @@ mod tests {
assert!(resize_report_required(false, (120, 40, 9, 18), size));
}
+ #[test]
+ fn approximate_cell_size_never_enables_direct_graphics() {
+ assert_eq!(
+ client_launch_mode(false, false, 8, 16),
+ ClientLaunchMode::App
+ );
+ assert_eq!(
+ client_launch_mode(true, false, 8, 16),
+ ClientLaunchMode::TerminalAttach
+ );
+ }
+
+ #[test]
+ fn direct_graphics_profile_is_narrow_and_transport_safe() {
+ for (program, term, kitty, expected) in [
+ ("ghostty", "", false, true),
+ ("WezTerm", "", false, true),
+ ("", "xterm-kitty", false, true),
+ ("", "xterm-256color", true, true),
+ ("", "xterm-256color", false, false),
+ ] {
+ assert_eq!(
+ direct_graphics_profile_values(program, term, kitty, false, true),
+ expected
+ );
+ }
+ assert!(!direct_graphics_profile_values(
+ "ghostty", "", false, true, true
+ ));
+ assert!(!direct_graphics_profile_values(
+ "ghostty", "", false, false, false
+ ));
+ }
+
fn restore_env_var(key: &str, value: Option<OsString>) {
if let Some(value) = value {
std::env::set_var(key, value);
@@ -2758,11 +3040,15 @@ mod tests {
}
#[test]
- fn cell_size_fallback_prefers_reported_size_over_default() {
- assert_eq!(cell_size_fallback(0), (8, 16));
- assert_eq!(cell_size_fallback(pack_cell_size(10, 21)), (10, 21));
- assert_eq!(cell_size_fallback(pack_cell_size(10, 0)), (8, 16));
- assert_eq!(cell_size_fallback(pack_cell_size(0, 21)), (8, 16));
+ fn cell_size_fallback_prefers_reported_then_previous_size() {
+ assert_eq!(cell_size_fallback(0, None), (8, 16));
+ assert_eq!(cell_size_fallback(0, Some((11, 22))), (11, 22));
+ assert_eq!(
+ cell_size_fallback(pack_cell_size(10, 21), Some((11, 22))),
+ (10, 21)
+ );
+ assert_eq!(cell_size_fallback(pack_cell_size(10, 0), None), (8, 16));
+ assert_eq!(cell_size_fallback(pack_cell_size(0, 21), None), (8, 16));
}
#[test]
diff --git a/src/ghostty/mod.rs b/src/ghostty/mod.rs
index 93efbd76..4dead632 100644
--- a/src/ghostty/mod.rs
+++ b/src/ghostty/mod.rs
@@ -1416,11 +1416,11 @@ impl Terminal {
self.get_optional_rgb_color(TERMINAL_DATA_COLOR_CURSOR)
}
- fn width_px(&self) -> Result<u32, Error> {
+ pub(crate) fn width_px(&self) -> Result<u32, Error> {
self.get_u32(ffi::GhosttyTerminalData_GHOSTTY_TERMINAL_DATA_WIDTH_PX)
}
- fn height_px(&self) -> Result<u32, Error> {
+ pub(crate) fn height_px(&self) -> Result<u32, Error> {
self.get_u32(ffi::GhosttyTerminalData_GHOSTTY_TERMINAL_DATA_HEIGHT_PX)
}
diff --git a/src/input/encode.rs b/src/input/encode.rs
index c475e0da..8ec8a1c5 100644
--- a/src/input/encode.rs
+++ b/src/input/encode.rs
@@ -125,7 +125,7 @@ fn encode_mouse_cb(
encoding: MouseProtocolEncoding,
) -> Option<Vec<u8>> {
let mut cb = match (encoding, release) {
- (MouseProtocolEncoding::Sgr, true) => base_button,
+ (MouseProtocolEncoding::Sgr | MouseProtocolEncoding::SgrPixels, true) => base_button,
(_, true) => 3,
(_, false) => base_button,
};
@@ -143,7 +143,7 @@ fn encode_mouse_cb(
let row = row as u32 + 1;
match encoding {
- MouseProtocolEncoding::Sgr => Some(
+ MouseProtocolEncoding::Sgr | MouseProtocolEncoding::SgrPixels => Some(
format!(
"\x1b[<{cb};{column};{row}{}",
if release { 'm' } else { 'M' }
diff --git a/src/input/mod.rs b/src/input/mod.rs
index 0a716f9c..21893303 100644
--- a/src/input/mod.rs
+++ b/src/input/mod.rs
@@ -1,5 +1,6 @@
mod encode;
mod model;
+pub(crate) mod mouse;
mod parse;
#[allow(unused_imports)]
diff --git a/src/input/model.rs b/src/input/model.rs
index 9c3c14e7..21252d2d 100644
--- a/src/input/model.rs
+++ b/src/input/model.rs
@@ -317,6 +317,7 @@ pub enum MouseProtocolEncoding {
Default,
Utf8,
Sgr,
+ SgrPixels,
}
#[cfg(test)]
diff --git a/src/input/mouse.rs b/src/input/mouse.rs
new file mode 100644
index 00000000..62b5f6c2
--- /dev/null
+++ b/src/input/mouse.rs
@@ -0,0 +1,194 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum Position {
+ Cell { column: u16, row: u16 },
+ Pixels { x: u32, y: u32 },
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct HostGeometry {
+ pub(crate) cols: u16,
+ pub(crate) rows: u16,
+ pub(crate) width_px: u32,
+ pub(crate) height_px: u32,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) struct HostPixels {
+ pub(crate) x: u32,
+ pub(crate) y: u32,
+ pub(crate) geometry: HostGeometry,
+}
+
+impl HostGeometry {
+ pub(crate) fn new(cols: u16, rows: u16, width_px: u32, height_px: u32) -> Option<Self> {
+ (cols > 0 && rows > 0 && width_px > 0 && height_px > 0).then_some(Self {
+ cols,
+ rows,
+ width_px,
+ height_px,
+ })
+ }
+
+ #[cfg(unix)]
+ pub(crate) fn current() -> Option<Self> {
+ let size = crossterm::terminal::window_size().ok()?;
+ Self::new(
+ size.columns,
+ size.rows,
+ u32::from(size.width),
+ u32::from(size.height),
+ )
+ }
+
+ pub(crate) fn cell(self, x: u32, y: u32) -> Option<(u16, u16)> {
+ Some((
+ grid_cell(x.checked_sub(1)?, self.cols, self.width_px)?,
+ grid_cell(y.checked_sub(1)?, self.rows, self.height_px)?,
+ ))
+ }
+
+ fn column_boundary(self, column: u16) -> Option<u32> {
+ boundary(column, self.cols, self.width_px)
+ }
+
+ fn row_boundary(self, row: u16) -> Option<u32> {
+ boundary(row, self.rows, self.height_px)
+ }
+}
+
+impl HostPixels {
+ pub(crate) fn pane_position(
+ self,
+ inner: ratatui::layout::Rect,
+ child_width_px: u32,
+ child_height_px: u32,
+ ) -> Option<Position> {
+ let start_x = self.geometry.column_boundary(inner.x)?;
+ let start_y = self.geometry.row_boundary(inner.y)?;
+ let end_x = self
+ .geometry
+ .column_boundary(inner.x.checked_add(inner.width)?)?;
+ let end_y = self
+ .geometry
+ .row_boundary(inner.y.checked_add(inner.height)?)?;
+ let x = self.x.checked_sub(1)?.checked_sub(start_x)?;
+ let y = self.y.checked_sub(1)?.checked_sub(start_y)?;
+ let source_width = end_x.checked_sub(start_x)?;
+ let source_height = end_y.checked_sub(start_y)?;
+ if x >= source_width || y >= source_height || child_width_px == 0 || child_height_px == 0 {
+ return None;
+ }
+ Some(Position::Pixels {
+ x: scale(x, source_width, child_width_px).checked_add(1)?,
+ y: scale(y, source_height, child_height_px).checked_add(1)?,
+ })
+ }
+}
+
+pub(crate) fn parse_report(data: &[u8]) -> Option<(u32, u32)> {
+ let body = data.strip_prefix(b"\x1b[<")?;
+ let body = body
+ .strip_suffix(b"M")
+ .or_else(|| body.strip_suffix(b"m"))?;
+ let mut fields = body.split(|byte| *byte == b';');
+ parse_number(fields.next()?)?;
+ let x = parse_number(fields.next()?)?;
+ let y = parse_number(fields.next()?)?;
+ fields.next().is_none().then_some((x, y))
+}
+
+pub(crate) fn report_at_cell(data: &[u8], column: u16, row: u16) -> Option<Vec<u8>> {
+ let body = data.strip_prefix(b"\x1b[<")?;
+ let suffix = if body.ends_with(b"M") { 'M' } else { 'm' };
+ let body = body.strip_suffix(&[suffix as u8])?;
+ let buttons = body.split(|byte| *byte == b';').next()?;
+ Some(
+ format!(
+ "\x1b[<{};{};{}{}",
+ std::str::from_utf8(buttons).ok()?,
+ u32::from(column) + 1,
+ u32::from(row) + 1,
+ suffix
+ )
+ .into_bytes(),
+ )
+}
+
+fn parse_number(value: &[u8]) -> Option<u32> {
+ (!value.is_empty() && value.iter().all(u8::is_ascii_digit))
+ .then(|| std::str::from_utf8(value).ok()?.parse().ok())
+ .flatten()
+}
+
+fn boundary(index: u16, count: u16, extent: u32) -> Option<u32> {
+ (count > 0 && index <= count && extent > 0)
+ .then(|| (u64::from(index) * u64::from(extent) / u64::from(count)) as u32)
+}
+
+fn grid_cell(pixel: u32, count: u16, extent: u32) -> Option<u16> {
+ if count == 0 || extent == 0 || pixel >= extent {
+ return None;
+ }
+ let cell = ((u64::from(pixel) + 1) * u64::from(count) - 1) / u64::from(extent);
+ u16::try_from(cell).ok().filter(|cell| *cell < count)
+}
+
+fn scale(pixel: u32, source: u32, target: u32) -> u32 {
+ ((u64::from(pixel) * u64::from(target)) / u64::from(source))
+ .min(u64::from(target.saturating_sub(1))) as u32
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn parser_accepts_only_complete_sgr_mouse_reports() {
+ for (input, expected) in [
+ (b"\x1b[<35;321;241M".as_slice(), Some((321, 241))),
+ (b"\x1b[<0;1;2m".as_slice(), Some((1, 2))),
+ (b"key".as_slice(), None),
+ (b"\x1b[<0;1;2Mkey".as_slice(), None),
+ (b"\x1b[<0;1M".as_slice(), None),
+ ] {
+ assert_eq!(parse_report(input), expected);
+ }
+ }
+
+ #[test]
+ fn fractional_geometry_maps_exactly_to_pane_pixels() {
+ let geometry = HostGeometry::new(211, 57, 2_537, 1_429).unwrap();
+ let inner = ratatui::layout::Rect::new(157, 7, 53, 49);
+ let start_x = geometry.column_boundary(inner.x).unwrap();
+ let end_x = geometry.column_boundary(inner.x + inner.width).unwrap();
+ let start_y = geometry.row_boundary(inner.y).unwrap();
+ let end_y = geometry.row_boundary(inner.y + inner.height).unwrap();
+ assert_eq!(
+ HostPixels {
+ x: start_x + 1,
+ y: start_y + 1,
+ geometry,
+ }
+ .pane_position(inner, 636, 1_225),
+ Some(Position::Pixels { x: 1, y: 1 })
+ );
+ assert_eq!(
+ HostPixels {
+ x: end_x,
+ y: end_y,
+ geometry,
+ }
+ .pane_position(inner, 636, 1_225),
+ Some(Position::Pixels { x: 636, y: 1_225 })
+ );
+ }
+
+ #[test]
+ fn geometry_rejects_outside_pixels_and_maps_cells() {
+ let geometry = HostGeometry::new(80, 24, 800, 480).unwrap();
+ assert_eq!(geometry.cell(1, 1), Some((0, 0)));
+ assert_eq!(geometry.cell(800, 480), Some((79, 23)));
+ assert_eq!(geometry.cell(801, 1), None);
+ assert_eq!(geometry.cell(0, 1), None);
+ }
+}
diff --git a/src/kitty_graphics.rs b/src/kitty_graphics.rs
index 98577ef8..726e1d8b 100644
--- a/src/kitty_graphics.rs
+++ b/src/kitty_graphics.rs
@@ -18,7 +18,10 @@ use crate::layout::{PaneId, PaneInfo};
use crate::terminal::TerminalRuntimeRegistry;
const KITTY_CHUNK_BYTES: usize = 3072;
+pub(crate) const HEADLESS_GRAPHICS_TRANSACTION_BUDGET: usize =
+ crate::protocol::MAX_GRAPHICS_FRAME_SIZE - crate::protocol::MAX_FRAME_SIZE;
const HOST_IMAGE_ID_BASE: u32 = 10_000;
+#[cfg(test)]
const PANE_GRAPHICS_IMAGE_ID_BIT: u32 = 1 << 31;
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
@@ -73,6 +76,7 @@ struct HostViewKey {
#[derive(Debug)]
struct HostPlacement {
pane_id: PaneId,
+ host_image_id: Option<u32>,
area: Rect,
cell_size: HostCellSize,
source_key: HostSourceKey,
@@ -80,10 +84,10 @@ struct HostPlacement {
scrollback_offset: u32,
}
-#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
enum HostSourceKey {
Terminal { pane_id: PaneId, image_id: u32 },
- PaneLayer { pane_id: PaneId },
+ PaneLayer { pane_id: PaneId, layer_id: String },
}
#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
@@ -131,7 +135,11 @@ pub(crate) struct HostGraphicsCache {
placements: HashMap<(u32, u32), PlacementSignature>,
/// Host image currently backing each (pane, source image id) pair.
sources: HashMap<HostSourceKey, u32>,
+ oversized: HashMap<HostSourceKey, ImageSignature>,
+ continuation: Option<(HostSourceKey, u32, usize)>,
view: Option<HostViewKey>,
+ replay_placements: bool,
+ replayed_placements: HashSet<(u32, u32)>,
}
static KITTY_GRAPHICS_ENABLED: AtomicBool = AtomicBool::new(false);
@@ -147,94 +155,130 @@ pub(crate) fn is_enabled() -> bool {
pub(crate) fn paint_local_pane_graphics(
app: &AppState,
+ graphics: &crate::app::pane_graphics::Runtime,
terminal_runtimes: &TerminalRuntimeRegistry,
cell_size: HostCellSize,
) -> io::Result<()> {
let cache = LOCAL_HOST_GRAPHICS.get_or_init(|| Mutex::new(HostGraphicsCache::default()));
- let mut bytes = Vec::new();
- if let Ok(mut cache) = cache.lock() {
- bytes = encode_local_pane_graphics(
+ let Ok(mut cache) = cache.lock() else {
+ return Ok(());
+ };
+ if graphics.slots.is_empty() && !cache.has_pane_sources() {
+ let encoded = encode_local_pane_graphics(
app,
+ graphics,
terminal_runtimes,
app.view.tab_surface(),
cell_size,
+ None,
&mut cache,
);
- }
- if bytes.is_empty() {
- return Ok(());
+ drop(cache);
+ if encoded.bytes.is_empty() {
+ return Ok(());
+ }
+ let mut stdout = io::stdout().lock();
+ stdout.write_all(b"\x1b7")?;
+ stdout.write_all(&encoded.bytes)?;
+ stdout.write_all(b"\x1b8")?;
+ return stdout.flush();
}
- let mut framed = Vec::with_capacity(bytes.len() + 8);
- framed.extend_from_slice(b"\x1b7");
- framed.extend_from_slice(&bytes);
- framed.extend_from_slice(b"\x1b8");
-
let mut stdout = io::stdout().lock();
- stdout.write_all(&framed)?;
+ loop {
+ let encoded = encode_local_pane_graphics(
+ app,
+ graphics,
+ terminal_runtimes,
+ app.view.tab_surface(),
+ cell_size,
+ None,
+ &mut cache,
+ );
+ if !encoded.bytes.is_empty() {
+ stdout.write_all(b"\x1b7")?;
+ stdout.write_all(&encoded.bytes)?;
+ stdout.write_all(b"\x1b8")?;
+ }
+ if !encoded.incomplete {
+ break;
+ }
+ }
stdout.flush()
}
+pub(crate) struct EncodedGraphics {
+ pub(crate) bytes: Vec<u8>,
+ pub(crate) incomplete: bool,
+}
+
pub(crate) fn encode_local_pane_graphics(
app: &AppState,
+ graphics: &crate::app::pane_graphics::Runtime,
terminal_runtimes: &TerminalRuntimeRegistry,
surface: crate::ui::TabSurfaceView<'_>,
cell_size: HostCellSize,
+ transaction_budget: Option<usize>,
cache: &mut HostGraphicsCache,
-) -> Vec<u8> {
- let mode_ok = app.mode == Mode::Terminal;
- let cell_ok = cell_size.is_known();
- tracing::debug!(
- mode_ok,
- cell_ok,
- cell_width_px = cell_size.width_px,
- cell_height_px = cell_size.height_px,
- active = ?app.active,
- pane_infos_len = surface.pane_infos.len(),
- "paint_local_pane_graphics entry"
- );
- if !mode_ok || !cell_ok {
- tracing::debug!(
- reason = if !mode_ok {
- "not terminal mode"
- } else {
- "cell size unknown"
- },
- "paint_local_pane_graphics early return"
+) -> EncodedGraphics {
+ let visible = app.mode == Mode::Terminal && cell_size.is_known();
+ if graphics.slots.is_empty() {
+ let mut bytes = cache.clear_pane_sources();
+ if !visible {
+ bytes.extend(cache.clear_bytes());
+ return EncodedGraphics {
+ bytes,
+ incomplete: false,
+ };
+ }
+ let placements = collect_visible_placements(
+ app,
+ graphics,
+ terminal_runtimes,
+ surface,
+ cell_size,
+ &cache.images,
);
- return cache.clear_bytes();
+ let view_changed = cache.update_view(active_view_key(app));
+ cache.reset_incremental_state();
+ encode_terminal_graphics_update_legacy(&mut bytes, &placements, view_changed, cache);
+ return EncodedGraphics {
+ bytes,
+ incomplete: false,
+ };
}
- let view_key = active_view_key(app);
- let placements =
- collect_visible_placements(app, terminal_runtimes, surface, cell_size, &cache.images);
- tracing::debug!(
- placements_collected = placements.len(),
- "collect_visible_placements result"
- );
-
- let mut bytes = Vec::new();
- let view_changed = cache.update_view(view_key);
- encode_graphics_update(
- &mut bytes,
- &placements,
- view_changed,
- &mut cache.images,
- &mut cache.placements,
- &mut cache.sources,
- );
- tracing::debug!(
- placements = placements.len(),
- bytes = bytes.len(),
- cell_width_px = cell_size.width_px,
- cell_height_px = cell_size.height_px,
- "painting kitty graphics placements"
- );
- bytes
+ let live_pane_sources = graphics
+ .slots
+ .iter()
+ .filter(|(_, slot)| slot.layer.is_some())
+ .map(|((pane_id, layer_id), _)| HostSourceKey::PaneLayer {
+ pane_id: *pane_id,
+ layer_id: layer_id.clone(),
+ })
+ .collect::<HashSet<_>>();
+ let placements = if visible {
+ collect_visible_placements(
+ app,
+ graphics,
+ terminal_runtimes,
+ surface,
+ cell_size,
+ &cache.images,
+ )
+ } else {
+ Vec::new()
+ };
+ cache.update_view(visible.then(|| active_view_key(app)).flatten());
+ // The host text blit overwrites Kitty placements, so every rendered frame must
+ // display cached images again even when their data and geometry are unchanged.
+ cache.request_placement_replay();
+ encode_graphics_update_incremental(cache, &placements, &live_pane_sources, transaction_budget)
}
pub(crate) fn has_visible_pane_graphics(
app: &AppState,
+ graphics: &crate::app::pane_graphics::Runtime,
terminal_runtimes: &TerminalRuntimeRegistry,
surface: crate::ui::TabSurfaceView<'_>,
cell_size: HostCellSize,
@@ -257,10 +301,20 @@ pub(crate) fn has_visible_pane_graphics(
for info in surface.pane_infos {
let empty_uploaded = HashMap::new();
- if app.pane_graphics_layers.get(&info.id).is_some_and(|layer| {
- let host_placement =
- pane_graphics_host_placement(info, cell_size, layer, &empty_uploaded, false);
- clipped_placement(&host_placement).is_some()
+ if graphics.slots.iter().any(|((pane_id, layer_id), slot)| {
+ *pane_id == info.id
+ && slot.layer.as_ref().is_some_and(|layer| {
+ clipped_placement(&pane_graphics_host_placement(
+ info,
+ layer_id,
+ slot.host_image_id,
+ cell_size,
+ layer,
+ &empty_uploaded,
+ false,
+ ))
+ .is_some()
+ })
}) {
return true;
}
@@ -274,6 +328,7 @@ pub(crate) fn has_visible_pane_graphics(
for placement in runtime.kitty_image_placements_with_data_filter(|_| false) {
let host_placement = HostPlacement {
pane_id: info.id,
+ host_image_id: None,
area: info.inner_rect,
cell_size,
source_key: HostSourceKey::Terminal {
@@ -292,73 +347,41 @@ pub(crate) fn has_visible_pane_graphics(
false
}
-fn encode_graphics_update(
+fn encode_terminal_graphics_update_legacy(
bytes: &mut Vec<u8>,
placements: &[HostPlacement],
view_changed: bool,
- host_images: &mut HashMap<u32, ImageSignature>,
- host_placements: &mut HashMap<(u32, u32), PlacementSignature>,
- sources: &mut HashMap<HostSourceKey, u32>,
+ cache: &mut HostGraphicsCache,
) {
- // Prune sources that are no longer visible: a stale entry would keep its
- // old host image referenced and block the superseded-image delete.
- let current_sources: HashSet<HostSourceKey> = placements
+ let current_sources = placements
.iter()
- .filter(|placement| {
- matches!(placement.source_key, HostSourceKey::Terminal { .. })
- || clipped_placement(placement).is_some()
- })
- .map(|placement| placement.source_key)
- .collect();
- let mut removed_pane_layer_images = HashSet::new();
- sources.retain(|source, host_id| {
- let retain = current_sources.contains(source);
- if !retain && matches!(source, HostSourceKey::PaneLayer { .. }) {
- removed_pane_layer_images.insert(*host_id);
- }
- retain
- });
- for host_id in removed_pane_layer_images {
- if sources.values().any(|id| *id == host_id) {
- continue;
- }
- encode_delete_image(bytes, host_id);
- host_images.remove(&host_id);
- host_placements.retain(|(image_id, _), _| *image_id != host_id);
- }
+ .filter(|placement| matches!(placement.source_key, HostSourceKey::Terminal { .. }))
+ .map(|placement| placement.source_key.clone())
+ .collect::<HashSet<_>>();
+ cache
+ .sources
+ .retain(|source, _| current_sources.contains(source));
let mut current_placements = HashSet::new();
for placement in placements {
- let clipped = clipped_placement(placement);
- tracing::debug!(
- pane_id = ?placement.pane_id,
- has_clipped = clipped.is_some(),
- grid_cols = placement.placement.render.grid_cols,
- grid_rows = placement.placement.render.grid_rows,
- viewport_col = placement.placement.render.viewport_col,
- viewport_row = placement.placement.render.viewport_row,
- area_w = placement.area.width,
- area_h = placement.area.height,
- "clipped_placement result"
- );
- let Some((clipped, format_code)) = clipped else {
+ let Some((clipped, format_code)) = clipped_placement(placement) else {
continue;
};
let host_id = host_image_id(placement.pane_id, &placement.placement);
- let host_placement_id = host_placement_id(placement.source_key, &placement.placement);
+ let placement_id = host_placement_id(&placement.source_key, &placement.placement);
let image_signature = image_signature(placement, format_code);
let placement_signature =
placement_signature(clipped, placement.placement.z, placement.scrollback_offset);
- let placement_key = (host_id, host_placement_id);
+ let placement_key = (host_id, placement_id);
current_placements.insert(placement_key);
- match host_images.get(&host_id).copied() {
+ match cache.images.get(&host_id).copied() {
Some(existing) if existing == image_signature => {}
Some(_) => {
encode_delete_image(bytes, host_id);
- host_placements.retain(|(image_id, placement_id), _| {
+ cache.placements.retain(|(image_id, id), _| {
if *image_id == host_id {
- current_placements.remove(&(*image_id, *placement_id));
+ current_placements.remove(&(*image_id, *id));
false
} else {
true
@@ -367,37 +390,32 @@ fn encode_graphics_update(
if !encode_upload_image(bytes, placement, format_code, host_id) {
continue;
}
- host_images.insert(host_id, image_signature);
+ cache.images.insert(host_id, image_signature);
}
None => {
if !encode_upload_image(bytes, placement, format_code, host_id) {
continue;
}
- host_images.insert(host_id, image_signature);
+ cache.images.insert(host_id, image_signature);
}
}
- release_superseded_source_image(
+ release_superseded_terminal_image_legacy(
bytes,
- sources,
- host_images,
- host_placements,
+ cache,
&mut current_placements,
- placement.source_key,
+ placement.source_key.clone(),
host_id,
);
- // A different view can repaint the same cells with text or overlays and
- // leave the host-side Kitty placement state out of sync with this cache.
- // Re-emit the placement even when its geometry signature is unchanged.
- match host_placements.get_mut(&placement_key) {
+ match cache.placements.get_mut(&placement_key) {
Some(existing) if !view_changed && *existing == placement_signature => {}
Some(existing) => {
encode_display_placement(
bytes,
clipped,
host_id,
- host_placement_id,
+ placement_id,
placement.placement.z,
);
*existing = placement_signature;
@@ -407,48 +425,42 @@ fn encode_graphics_update(
bytes,
clipped,
host_id,
- host_placement_id,
+ placement_id,
placement.placement.z,
);
- host_placements.insert(placement_key, placement_signature);
+ cache.placements.insert(placement_key, placement_signature);
}
}
}
- let mut stale_placements = Vec::new();
- for key in host_placements.keys() {
- if current_placements.contains(key) {
- continue;
- }
- stale_placements.push(*key);
- }
- for (host_id, host_placement_id) in stale_placements {
- encode_delete_placement(bytes, host_id, host_placement_id);
- host_placements.remove(&(host_id, host_placement_id));
+ let stale = cache
+ .placements
+ .keys()
+ .filter(|key| !current_placements.contains(key))
+ .copied()
+ .collect::<Vec<_>>();
+ for (host_id, placement_id) in stale {
+ encode_delete_placement(bytes, host_id, placement_id);
+ cache.placements.remove(&(host_id, placement_id));
}
}
-/// Records that `source` is now backed by `host_id` and deletes the host
-/// image it previously pointed at once no other source references it.
-fn release_superseded_source_image(
+fn release_superseded_terminal_image_legacy(
bytes: &mut Vec<u8>,
- sources: &mut HashMap<HostSourceKey, u32>,
- host_images: &mut HashMap<u32, ImageSignature>,
- host_placements: &mut HashMap<(u32, u32), PlacementSignature>,
+ cache: &mut HostGraphicsCache,
current_placements: &mut HashSet<(u32, u32)>,
source: HostSourceKey,
host_id: u32,
) {
- let Some(previous) = sources.insert(source, host_id) else {
+ let Some(previous) = cache.sources.insert(source, host_id) else {
return;
};
- if previous == host_id || sources.values().any(|id| *id == previous) {
+ if previous == host_id || cache.sources.values().any(|id| *id == previous) {
return;
}
encode_delete_image(bytes, previous);
- host_images.remove(&previous);
- // The `d=I` delete also removes the image's placements host-side.
- host_placements.retain(|(image_id, placement_id), _| {
+ cache.images.remove(&previous);
+ cache.placements.retain(|(image_id, placement_id), _| {
if *image_id == previous {
current_placements.remove(&(*image_id, *placement_id));
false
@@ -458,6 +470,320 @@ fn release_superseded_source_image(
});
}
+fn encode_graphics_update_incremental(
+ cache: &mut HostGraphicsCache,
+ placements: &[HostPlacement],
+ live_pane_sources: &HashSet<HostSourceKey>,
+ transaction_budget: Option<usize>,
+) -> EncodedGraphics {
+ let desired_sources = placements
+ .iter()
+ .map(|placement| placement.source_key.clone())
+ .collect::<HashSet<_>>();
+ let desired_placements = placements
+ .iter()
+ .filter_map(|placement| {
+ clipped_placement(placement).map(|_| {
+ let host_id = placement
+ .host_image_id
+ .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement));
+ (
+ host_id,
+ host_placement_id(&placement.source_key, &placement.placement),
+ )
+ })
+ })
+ .collect::<HashSet<_>>();
+ let start = cache
+ .continuation
+ .as_ref()
+ .and_then(|(source, id, _)| {
+ placements
+ .iter()
+ .position(|placement| placement_identity(placement) == (source.clone(), *id))
+ })
+ .map(|index| index + 1)
+ .or_else(|| cache.continuation.as_ref().map(|cursor| cursor.2))
+ .map_or(0, |index| index % placements.len().max(1));
+ let mut bytes = Vec::new();
+ let mut emitted = false;
+
+ let mut dead_sources = cache
+ .sources
+ .keys()
+ .filter(|source| {
+ matches!(source, HostSourceKey::PaneLayer { .. })
+ && !live_pane_sources.contains(*source)
+ })
+ .cloned()
+ .collect::<Vec<_>>();
+ dead_sources.sort_by_key(source_order);
+ for source in dead_sources {
+ let host_id = cache.sources[&source];
+ let last_reference = !cache
+ .sources
+ .iter()
+ .any(|(other, id)| *other != source && *id == host_id);
+ if emitted && last_reference {
+ return EncodedGraphics {
+ bytes,
+ incomplete: true,
+ };
+ }
+ cache.sources.remove(&source);
+ if last_reference {
+ encode_delete_image(&mut bytes, host_id);
+ cache.images.remove(&host_id);
+ cache.placements.retain(|(id, _), _| *id != host_id);
+ cache.replayed_placements.retain(|(id, _)| *id != host_id);
+ emitted = true;
+ }
+ }
+ cache.sources.retain(|source, _| {
+ matches!(source, HostSourceKey::PaneLayer { .. }) || desired_sources.contains(source)
+ });
+ cache
+ .oversized
+ .retain(|source, _| live_pane_sources.contains(source) || desired_sources.contains(source));
+
+ let mut stale = cache
+ .placements
+ .keys()
+ .filter(|key| !desired_placements.contains(key))
+ .copied()
+ .collect::<Vec<_>>();
+ stale.sort_unstable();
+ for key @ (host_id, placement_id) in stale {
+ if emitted {
+ return EncodedGraphics {
+ bytes,
+ incomplete: true,
+ };
+ }
+ encode_delete_placement(&mut bytes, host_id, placement_id);
+ cache.placements.remove(&key);
+ cache.replayed_placements.remove(&key);
+ emitted = true;
+ }
+
+ for offset in 0..placements.len() {
+ let index = (start + offset) % placements.len();
+ let placement = &placements[index];
+ let signature = image_signature(placement, kitty_format_code(placement.placement.format));
+ if transaction_budget.is_some()
+ && cache.oversized.get(&placement.source_key) == Some(&signature)
+ {
+ continue;
+ }
+ cache.oversized.remove(&placement.source_key);
+ let host_id = placement
+ .host_image_id
+ .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement));
+ if cache.images.get(&host_id) != Some(&signature)
+ && !image_transaction_fits(placement, transaction_budget)
+ {
+ cache
+ .oversized
+ .insert(placement.source_key.clone(), signature);
+ continue;
+ }
+ let mut candidate = cache.clone();
+ let Some(transaction) = encode_placement_update(&mut candidate, placement) else {
+ continue;
+ };
+ if transaction.is_empty() {
+ *cache = candidate;
+ continue;
+ }
+ if emitted {
+ return EncodedGraphics {
+ bytes,
+ incomplete: true,
+ };
+ }
+ *cache = candidate;
+ let (source, id) = placement_identity(placement);
+ cache.continuation = Some((source, id, (index + 1) % placements.len()));
+ bytes = transaction;
+ emitted = true;
+ }
+
+ cache.replay_placements = false;
+ cache.replayed_placements.clear();
+ EncodedGraphics {
+ bytes,
+ incomplete: false,
+ }
+}
+
+fn image_transaction_fits(placement: &HostPlacement, budget: Option<usize>) -> bool {
+ let Some(budget) = budget else {
+ return true;
+ };
+ let data = placement.placement.data_len;
+ let encoded = data.div_ceil(3).saturating_mul(4);
+ let command_overhead = data.div_ceil(KITTY_CHUNK_BYTES).saturating_mul(16) + 1024;
+ encoded.saturating_add(command_overhead) <= budget
+}
+
+fn placement_identity(placement: &HostPlacement) -> (HostSourceKey, u32) {
+ (
+ placement.source_key.clone(),
+ host_placement_id(&placement.source_key, &placement.placement),
+ )
+}
+
+fn source_order(source: &HostSourceKey) -> (u32, String) {
+ match source {
+ HostSourceKey::Terminal { pane_id, .. } => (pane_id.raw(), String::new()),
+ HostSourceKey::PaneLayer { pane_id, layer_id } => (pane_id.raw(), layer_id.clone()),
+ }
+}
+
+fn encode_placement_update(
+ cache: &mut HostGraphicsCache,
+ placement: &HostPlacement,
+) -> Option<Vec<u8>> {
+ let (clipped, format_code) = clipped_placement(placement)?;
+ let host_id = placement
+ .host_image_id
+ .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement));
+ let placement_id = host_placement_id(&placement.source_key, &placement.placement);
+ let key = (host_id, placement_id);
+ let image_signature = image_signature(placement, format_code);
+ let placement_signature =
+ placement_signature(clipped, placement.placement.z, placement.scrollback_offset);
+ let image_current = cache.images.get(&host_id) == Some(&image_signature);
+ let placement_current = cache.placements.get(&key) == Some(&placement_signature)
+ && (!cache.replay_placements || cache.replayed_placements.contains(&key));
+ if image_current
+ && placement_current
+ && cache.sources.get(&placement.source_key) == Some(&host_id)
+ {
+ return None;
+ }
+
+ let mut bytes = Vec::new();
+ let mut displayed = false;
+ if !image_current {
+ if cache.images.contains_key(&host_id)
+ && matches!(placement.source_key, HostSourceKey::PaneLayer { .. })
+ {
+ if !encode_transmit_and_display(
+ &mut bytes,
+ placement,
+ clipped,
+ format_code,
+ host_id,
+ placement_id,
+ ) {
+ return None;
+ }
+ displayed = true;
+ } else {
+ if cache.images.contains_key(&host_id) {
+ encode_delete_image(&mut bytes, host_id);
+ cache.placements.retain(|(id, _), _| *id != host_id);
+ cache.replayed_placements.retain(|(id, _)| *id != host_id);
+ }
+ if !encode_upload_image(&mut bytes, placement, format_code, host_id) {
+ return None;
+ }
+ }
+ cache.images.insert(host_id, image_signature);
+ }
+
+ release_superseded_source_image(&mut bytes, cache, placement.source_key.clone(), host_id);
+ if !displayed && !placement_current {
+ encode_display_placement(
+ &mut bytes,
+ clipped,
+ host_id,
+ placement_id,
+ placement.placement.z,
+ );
+ }
+ cache.placements.insert(key, placement_signature);
+ if cache.replay_placements {
+ cache.replayed_placements.insert(key);
+ }
+ Some(bytes)
+}
+
+fn release_superseded_source_image(
+ bytes: &mut Vec<u8>,
+ cache: &mut HostGraphicsCache,
+ source: HostSourceKey,
+ host_id: u32,
+) {
+ let Some(previous) = cache.sources.insert(source, host_id) else {
+ return;
+ };
+ if previous == host_id || cache.sources.values().any(|id| *id == previous) {
+ return;
+ }
+ encode_delete_image(bytes, previous);
+ cache.images.remove(&previous);
+ cache.placements.retain(|(id, _), _| *id != previous);
+ cache.replayed_placements.retain(|(id, _)| *id != previous);
+}
+
+#[cfg(test)]
+fn drain_graphics_updates(
+ cache: &mut HostGraphicsCache,
+ placements: &[HostPlacement],
+ live: &HashSet<HostSourceKey>,
+) -> Vec<u8> {
+ let mut bytes = Vec::new();
+ loop {
+ let encoded = encode_graphics_update_incremental(cache, placements, live, None);
+ bytes.extend(encoded.bytes);
+ if !encoded.incomplete {
+ return bytes;
+ }
+ }
+}
+
+#[cfg(test)]
+fn encode_graphics_update(
+ bytes: &mut Vec<u8>,
+ placements: &[HostPlacement],
+ replay: bool,
+ images: &mut HashMap<u32, ImageSignature>,
+ host_placements: &mut HashMap<(u32, u32), PlacementSignature>,
+ sources: &mut HashMap<HostSourceKey, u32>,
+) {
+ let mut cache = HostGraphicsCache {
+ images: std::mem::take(images),
+ placements: std::mem::take(host_placements),
+ sources: std::mem::take(sources),
+ ..HostGraphicsCache::default()
+ };
+ let mut live = cache
+ .sources
+ .keys()
+ .filter(|source| matches!(source, HostSourceKey::PaneLayer { .. }))
+ .cloned()
+ .collect::<HashSet<_>>();
+ live.extend(
+ placements
+ .iter()
+ .filter(|placement| matches!(placement.source_key, HostSourceKey::PaneLayer { .. }))
+ .map(|placement| placement.source_key.clone()),
+ );
+ if live.is_empty() {
+ encode_terminal_graphics_update_legacy(bytes, placements, replay, &mut cache);
+ } else {
+ if replay {
+ cache.request_placement_replay();
+ }
+ bytes.extend(drain_graphics_updates(&mut cache, placements, &live));
+ }
+ *images = cache.images;
+ *host_placements = cache.placements;
+ *sources = cache.sources;
+}
+
pub(crate) fn clear_all_host_graphics() -> io::Result<()> {
let cache = LOCAL_HOST_GRAPHICS.get_or_init(|| Mutex::new(HostGraphicsCache::default()));
let mut bytes = Vec::new();
@@ -473,10 +799,97 @@ pub(crate) fn clear_all_host_graphics() -> io::Result<()> {
}
impl HostGraphicsCache {
+ fn clear_pane_sources(&mut self) -> Vec<u8> {
+ let pane_sources = self
+ .sources
+ .keys()
+ .filter(|source| matches!(source, HostSourceKey::PaneLayer { .. }))
+ .cloned()
+ .collect::<Vec<_>>();
+ let mut removed_images = HashSet::new();
+ for source in pane_sources {
+ if let Some(image_id) = self.sources.remove(&source) {
+ removed_images.insert(image_id);
+ }
+ self.oversized.remove(&source);
+ }
+
+ let mut bytes = Vec::new();
+ for image_id in removed_images {
+ if self.sources.values().any(|id| *id == image_id) {
+ continue;
+ }
+ encode_delete_image(&mut bytes, image_id);
+ self.images.remove(&image_id);
+ self.placements.retain(|(id, _), _| *id != image_id);
+ self.replayed_placements.retain(|(id, _)| *id != image_id);
+ }
+ self.reset_incremental_state();
+ bytes
+ }
+
+ fn has_pane_sources(&self) -> bool {
+ self.sources
+ .keys()
+ .any(|source| matches!(source, HostSourceKey::PaneLayer { .. }))
+ }
+
+ fn reset_incremental_state(&mut self) {
+ self.oversized.clear();
+ self.continuation = None;
+ self.replay_placements = false;
+ self.replayed_placements.clear();
+ }
+
+ pub(crate) fn trust_pane_layer(
+ &mut self,
+ key: &crate::app::pane_graphics::Key,
+ host_id: u32,
+ layer: &crate::app::pane_graphics::Layer,
+ ) {
+ let source = HostSourceKey::PaneLayer {
+ pane_id: key.0,
+ layer_id: key.1.clone(),
+ };
+ self.oversized.remove(&source);
+ self.sources.insert(source, host_id);
+ self.images
+ .insert(host_id, pane_layer_image_signature(layer));
+ }
+
+ pub(crate) fn forget_pane_layer(&mut self, key: &crate::app::pane_graphics::Key, host_id: u32) {
+ let source = HostSourceKey::PaneLayer {
+ pane_id: key.0,
+ layer_id: key.1.clone(),
+ };
+ self.sources.remove(&source);
+ self.oversized.remove(&source);
+ self.images.remove(&host_id);
+ self.placements.retain(|(id, _), _| *id != host_id);
+ self.replayed_placements.retain(|(id, _)| *id != host_id);
+ }
+
pub(crate) fn is_empty(&self) -> bool {
self.images.is_empty() && self.placements.is_empty()
}
+ pub(crate) fn request_placement_replay(&mut self) {
+ if !self.replay_placements {
+ self.replay_placements = true;
+ self.replayed_placements.clear();
+ }
+ }
+
+ #[cfg(test)]
+ fn hide_except_live_pane_layers(&mut self, live: &HashSet<HostSourceKey>) -> Vec<u8> {
+ drain_graphics_updates(self, &[], live)
+ }
+
+ #[cfg(test)]
+ pub(crate) fn test_image_count(&self) -> usize {
+ self.images.len()
+ }
+
#[cfg(test)]
pub(crate) fn test_mark_non_empty(&mut self) {
self.images.insert(
@@ -499,15 +912,43 @@ impl HostGraphicsCache {
self.images.clear();
self.placements.clear();
self.sources.clear();
+ self.reset_incremental_state();
self.view = None;
bytes
}
+ pub(crate) fn clear_next(&mut self) -> EncodedGraphics {
+ self.continuation = None;
+ let mut bytes = Vec::new();
+ if let Some(id) = self.images.keys().copied().min() {
+ encode_delete_image(&mut bytes, id);
+ self.images.remove(&id);
+ self.placements.retain(|(image, _), _| *image != id);
+ self.sources.retain(|_, image| *image != id);
+ self.replayed_placements.retain(|(image, _)| *image != id);
+ } else if let Some(key) = self.placements.keys().copied().min() {
+ encode_delete_placement(&mut bytes, key.0, key.1);
+ self.placements.remove(&key);
+ self.replayed_placements.remove(&key);
+ } else {
+ self.sources.clear();
+ self.oversized.clear();
+ self.view = None;
+ self.replay_placements = false;
+ self.replayed_placements.clear();
+ }
+ EncodedGraphics {
+ bytes,
+ incomplete: !self.is_empty(),
+ }
+ }
+
fn update_view(&mut self, view_key: Option<HostViewKey>) -> bool {
if self.view == view_key {
return false;
}
self.view = view_key;
+ self.continuation = None;
true
}
}
@@ -523,6 +964,7 @@ fn active_view_key(app: &AppState) -> Option<HostViewKey> {
fn collect_visible_placements(
app: &AppState,
+ graphics: &crate::app::pane_graphics::Runtime,
terminal_runtimes: &TerminalRuntimeRegistry,
surface: crate::ui::TabSurfaceView<'_>,
cell_size: HostCellSize,
@@ -553,9 +995,25 @@ fn collect_visible_placements(
);
let mut placements = Vec::new();
for info in surface.pane_infos {
- if let Some(layer) = app.pane_graphics_layers.get(&info.id) {
+ let mut pane_layers = graphics
+ .slots
+ .iter()
+ .filter_map(|((pane_id, layer_id), slot)| {
+ (*pane_id == info.id)
+ .then(|| {
+ slot.layer
+ .as_ref()
+ .map(|layer| (layer_id, slot.host_image_id, layer))
+ })
+ .flatten()
+ })
+ .collect::<Vec<_>>();
+ pane_layers.sort_by_key(|(layer_id, _, layer)| (layer.z_index, layer_id.as_str()));
+ for (layer_id, host_image_id, layer) in pane_layers {
placements.push(pane_graphics_host_placement(
info,
+ layer_id,
+ host_image_id,
cell_size,
layer,
uploaded_images,
@@ -582,6 +1040,7 @@ fn collect_visible_placements(
.unwrap_or(0);
placements.push(HostPlacement {
pane_id: info.id,
+ host_image_id: None,
area: info.inner_rect,
cell_size,
source_key: HostSourceKey::Terminal {
@@ -602,25 +1061,19 @@ fn collect_visible_placements(
fn pane_graphics_host_placement(
info: &PaneInfo,
+ layer_id: &str,
+ host_id: u32,
cell_size: HostCellSize,
- layer: &crate::app::state::PaneGraphicsLayer,
+ layer: &crate::app::pane_graphics::Layer,
uploaded_images: &HashMap<u32, ImageSignature>,
include_data: bool,
) -> HostPlacement {
let format = pane_graphics_kitty_format(layer.format);
- let format_code = kitty_format_code(format);
- let signature = ImageSignature {
- image_width: layer.image_width,
- image_height: layer.image_height,
- format_code,
- data_len: layer.data.len(),
- data_fingerprint: layer.data_fingerprint,
- };
- let host_id = pane_graphics_host_image_id(info.id, signature);
+ let signature = pane_layer_image_signature(layer);
let data = if !include_data || uploaded_images.get(&host_id).copied() == Some(signature) {
Vec::new()
} else {
- layer.data.clone()
+ layer.inline_data().map(<[u8]>::to_vec).unwrap_or_default()
};
let render = layer.render;
let grid_cols = if render.grid_cols == 0 {
@@ -636,20 +1089,24 @@ fn pane_graphics_host_placement(
HostPlacement {
pane_id: info.id,
+ host_image_id: Some(host_id),
area: info.inner_rect,
cell_size,
- source_key: HostSourceKey::PaneLayer { pane_id: info.id },
+ source_key: HostSourceKey::PaneLayer {
+ pane_id: info.id,
+ layer_id: layer_id.to_owned(),
+ },
scrollback_offset: 0,
placement: KittyImagePlacement {
image_id: 1,
placement_id: 1,
- z: 0,
+ z: layer.z_index,
x_offset: 0,
y_offset: 0,
image_width: layer.image_width,
image_height: layer.image_height,
format,
- data_len: layer.data.len(),
+ data_len: layer.data_len(),
data_fingerprint: layer.data_fingerprint,
data,
render: KittyPlacementRenderInfo {
@@ -672,7 +1129,8 @@ fn pane_graphics_kitty_format(format: crate::api::schema::PaneGraphicsFormat) ->
match format {
crate::api::schema::PaneGraphicsFormat::Png => KittyImageFormat::Png,
crate::api::schema::PaneGraphicsFormat::Rgb => KittyImageFormat::Rgb,
- crate::api::schema::PaneGraphicsFormat::Rgba => KittyImageFormat::Rgba,
+ crate::api::schema::PaneGraphicsFormat::Rgba
+ | crate::api::schema::PaneGraphicsFormat::Bgra => KittyImageFormat::Rgba,
}
}
@@ -697,20 +1155,14 @@ fn host_image_id_for_signature(pane_id: PaneId, signature: ImageSignature) -> u3
HOST_IMAGE_ID_BASE + ((hasher.finish() as u32) % 900_000)
}
-fn pane_graphics_host_image_id(pane_id: PaneId, signature: ImageSignature) -> u32 {
- let mut hasher = DefaultHasher::new();
- pane_id.raw().hash(&mut hasher);
- signature.hash(&mut hasher);
- PANE_GRAPHICS_IMAGE_ID_BIT | ((hasher.finish() as u32) & !PANE_GRAPHICS_IMAGE_ID_BIT)
-}
-
-fn host_placement_id(source_key: HostSourceKey, placement: &KittyImagePlacement) -> u32 {
+fn host_placement_id(source_key: &HostSourceKey, placement: &KittyImagePlacement) -> u32 {
let mut hasher = DefaultHasher::new();
match source_key {
HostSourceKey::Terminal { pane_id, .. } => pane_id.raw().hash(&mut hasher),
- HostSourceKey::PaneLayer { pane_id } => {
+ HostSourceKey::PaneLayer { pane_id, layer_id } => {
"pane.graphics".hash(&mut hasher);
pane_id.raw().hash(&mut hasher);
+ layer_id.hash(&mut hasher);
}
}
placement.image_id.hash(&mut hasher);
@@ -718,6 +1170,90 @@ fn host_placement_id(source_key: HostSourceKey, placement: &KittyImagePlacement)
1 + ((hasher.finish() as u32) % 900_000)
}
+pub(crate) struct DirectFileCommand {
+ pub(crate) leading: Vec<u8>,
+ pub(crate) control: String,
+}
+
+pub(crate) fn prepare_direct_file(
+ app: &AppState,
+ graphics: &crate::app::pane_graphics::Runtime,
+ surface: crate::ui::TabSurfaceView<'_>,
+ cell_size: HostCellSize,
+ cache: &mut HostGraphicsCache,
+ key: &crate::app::pane_graphics::Key,
+) -> Option<DirectFileCommand> {
+ if app.mode != Mode::Terminal || !cell_size.is_known() || app.active.is_none() {
+ return None;
+ }
+ let info = surface.pane_infos.iter().find(|info| info.id == key.0)?;
+ let slot = graphics.slots.get(key)?;
+ let layer = slot.layer.as_ref()?;
+ layer.direct_lease()?;
+ let placement = pane_graphics_host_placement(
+ info,
+ &key.1,
+ slot.host_image_id,
+ cell_size,
+ layer,
+ &cache.images,
+ false,
+ );
+ let (command, clipped, format_code, placement_id) =
+ direct_file_command(&placement, slot.host_image_id)?;
+ cache
+ .images
+ .insert(slot.host_image_id, image_signature(&placement, format_code));
+ cache.placements.insert(
+ (slot.host_image_id, placement_id),
+ placement_signature(clipped, layer.z_index, 0),
+ );
+ cache
+ .sources
+ .insert(placement.source_key, slot.host_image_id);
+ Some(command)
+}
+
+fn direct_file_command(
+ placement: &HostPlacement,
+ host_image_id: u32,
+) -> Option<(DirectFileCommand, ClippedPlacement, u32, u32)> {
+ let (clipped, format_code) = clipped_placement(placement)?;
+ let placement_id = host_placement_id(&placement.source_key, &placement.placement);
+ let mut control = format!(
+ "a=T,f={format_code},s={},v={},i={host_image_id},p={placement_id},c={},r={},z={},C=1,q=0",
+ placement.placement.image_width,
+ placement.placement.image_height,
+ clipped.cols,
+ clipped.rows,
+ placement.placement.z,
+ );
+ append_placement_controls(&mut control, clipped);
+ Some((
+ DirectFileCommand {
+ leading: format!("\x1b[{};{}H", clipped.y + 1, clipped.x + 1).into_bytes(),
+ control,
+ },
+ clipped,
+ format_code,
+ placement_id,
+ ))
+}
+
+#[cfg(unix)]
+pub(crate) fn encode_kitty_regular_file(
+ out: &mut Vec<u8>,
+ leading: &[u8],
+ control: &str,
+ path: &str,
+) {
+ let payload = base64::engine::general_purpose::STANDARD.encode(path.as_bytes());
+ out.extend_from_slice(b"\x1b7");
+ out.extend_from_slice(leading);
+ let _ = write!(out, "\x1b_G{control},t=f;{payload}\x1b\\");
+ out.extend_from_slice(b"\x1b8");
+}
+
fn encode_delete_image(out: &mut Vec<u8>, id: u32) {
let _ = write!(out, "\x1b_Ga=d,d=I,i={id},q=2;\x1b\\");
}
@@ -747,6 +1283,31 @@ fn encode_upload_image(
true
}
+fn encode_transmit_and_display(
+ out: &mut Vec<u8>,
+ placement: &HostPlacement,
+ clipped: ClippedPlacement,
+ format_code: u32,
+ host_id: u32,
+ host_placement_id: u32,
+) -> bool {
+ if placement.placement.data.is_empty() {
+ return false;
+ }
+ let _ = write!(out, "\x1b[{};{}H", clipped.y + 1, clipped.x + 1);
+ let mut control = format!(
+ "a=T,t=d,f={format_code},s={},v={},i={host_id},p={host_placement_id},c={},r={},z={},C=1,q=2",
+ placement.placement.image_width,
+ placement.placement.image_height,
+ clipped.cols,
+ clipped.rows,
+ placement.placement.z,
+ );
+ append_placement_controls(&mut control, clipped);
+ encode_kitty_data(out, &control, &placement.placement.data);
+ true
+}
+
fn encode_display_placement(
out: &mut Vec<u8>,
clipped: ClippedPlacement,
@@ -759,6 +1320,11 @@ fn encode_display_placement(
"a=p,i={host_id},p={host_placement_id},c={},r={},z={z},C=1,q=2",
clipped.cols, clipped.rows,
);
+ append_placement_controls(&mut control, clipped);
+ let _ = write!(out, "\x1b_G{control};\x1b\\");
+}
+
+fn append_placement_controls(control: &mut String, clipped: ClippedPlacement) {
if clipped.source_x > 0 {
let _ = write!(control, ",x={}", clipped.source_x);
}
@@ -777,8 +1343,6 @@ fn encode_display_placement(
if clipped.y_offset > 0 {
let _ = write!(control, ",Y={}", clipped.y_offset);
}
-
- let _ = write!(out, "\x1b_G{control};\x1b\\");
}
fn clipped_placement(placement: &HostPlacement) -> Option<(ClippedPlacement, u32)> {
@@ -929,6 +1493,16 @@ fn scale_pixels(value: u32, source: u32, dest: u32) -> u32 {
((value as u64).saturating_mul(source as u64) / dest.max(1) as u64).min(u32::MAX as u64) as u32
}
+fn pane_layer_image_signature(layer: &crate::app::pane_graphics::Layer) -> ImageSignature {
+ ImageSignature {
+ image_width: layer.image_width,
+ image_height: layer.image_height,
+ format_code: kitty_format_code(pane_graphics_kitty_format(layer.format)),
+ data_len: layer.data_len(),
+ data_fingerprint: layer.data_fingerprint,
+ }
+}
+
fn image_signature(placement: &HostPlacement, format_code: u32) -> ImageSignature {
ImageSignature {
image_width: placement.placement.image_width,
@@ -1001,9 +1575,22 @@ fn encode_kitty_data(out: &mut Vec<u8>, control: &str, data: &[u8]) {
mod tests {
use super::*;
+ #[test]
+ fn fallback_cell_size_is_usable_only_for_nonempty_areas() {
+ assert_eq!(
+ HostCellSize::fallback_for_area(Rect::new(0, 0, 80, 24)),
+ HostCellSize {
+ width_px: 8,
+ height_px: 16,
+ }
+ );
+ assert!(!HostCellSize::fallback_for_area(Rect::default()).is_known());
+ }
+
fn test_placement(viewport_col: i32, viewport_row: i32) -> HostPlacement {
HostPlacement {
pane_id: PaneId::from_raw(1),
+ host_image_id: None,
area: Rect::new(0, 0, 20, 10),
cell_size: HostCellSize {
width_px: 10,
@@ -1046,10 +1633,70 @@ mod tests {
let mut placement = test_placement(viewport_col, viewport_row);
placement.source_key = HostSourceKey::PaneLayer {
pane_id: placement.pane_id,
+ layer_id: "primary".into(),
};
placement
}
+ fn update(
+ cache: &mut HostGraphicsCache,
+ placements: &[HostPlacement],
+ replay: bool,
+ ) -> Vec<u8> {
+ let mut bytes = Vec::new();
+ encode_graphics_update(
+ &mut bytes,
+ placements,
+ replay,
+ &mut cache.images,
+ &mut cache.placements,
+ &mut cache.sources,
+ );
+ bytes
+ }
+
+ #[test]
+ fn terminal_graphics_without_pane_layers_preserves_legacy_transcript() {
+ fn record(transcript: &mut Vec<u8>, bytes: &[u8]) {
+ transcript.extend_from_slice(&(bytes.len() as u64).to_le_bytes());
+ transcript.extend_from_slice(bytes);
+ }
+
+ fn fnv1a(bytes: &[u8]) -> u64 {
+ bytes.iter().fold(0xcbf29ce484222325, |hash, byte| {
+ (hash ^ u64::from(*byte)).wrapping_mul(0x100000001b3)
+ })
+ }
+
+ let mut cache = HostGraphicsCache::default();
+ let mut transcript = Vec::new();
+
+ record(
+ &mut transcript,
+ &update(&mut cache, &[test_placement(0, 0)], false),
+ );
+ record(
+ &mut transcript,
+ &update(&mut cache, &[test_placement(0, 0)], false),
+ );
+ record(
+ &mut transcript,
+ &update(&mut cache, &[test_placement(-1, 2)], false),
+ );
+ record(
+ &mut transcript,
+ &update(&mut cache, &[test_placement(-1, 2)], true),
+ );
+
+ let mut changed = test_placement(-1, 2);
+ changed.placement.data_fingerprint = 43;
+ record(&mut transcript, &update(&mut cache, &[changed], false));
+ record(&mut transcript, &update(&mut cache, &[], false));
+
+ assert_eq!(transcript.len(), 10_084);
+ assert_eq!(fnv1a(&transcript), 0xc5bd_83e4_b039_870e);
+ }
+
#[test]
fn terminal_placement_id_preserves_legacy_identity() {
let placement = test_placement(0, 0);
@@ -1060,13 +1707,14 @@ mod tests {
let expected = 1 + ((legacy.finish() as u32) % 900_000);
assert_eq!(
- host_placement_id(placement.source_key, &placement.placement),
+ host_placement_id(&placement.source_key, &placement.placement),
expected
);
assert_ne!(
host_placement_id(
- HostSourceKey::PaneLayer {
+ &HostSourceKey::PaneLayer {
pane_id: placement.pane_id,
+ layer_id: "primary".into(),
},
&placement.placement,
),
@@ -1074,15 +1722,60 @@ mod tests {
);
}
+ #[cfg(unix)]
+ #[test]
+ fn regular_file_command_is_rgba_quiet_zero_and_path_encoded() {
+ let mut bytes = Vec::new();
+ encode_kitty_regular_file(
+ &mut bytes,
+ b"\x1b[2;3H",
+ "a=T,f=32,s=3,v=2,i=42,p=7,c=3,r=2,z=0,C=1,q=0",
+ "/private/frame",
+ );
+ let text = String::from_utf8(bytes).unwrap();
+ assert!(text.starts_with("\x1b7\x1b[2;3H\x1b_Ga=T,f=32"));
+ assert!(text.contains(",C=1,q=0,t=f;L3ByaXZhdGUvZnJhbWU="));
+ assert!(text.ends_with("\x1b\\\x1b8"));
+ }
+
+ #[test]
+ fn direct_file_uses_one_clipped_transmit_and_display_at_the_final_position() {
+ let mut placement = pane_layer_placement(-1, 2);
+ placement.area = Rect::new(10, 4, 8, 6);
+ let (command, _, _, _) = direct_file_command(&placement, (1 << 31) | 9).unwrap();
+ let control = command.control;
+
+ assert_eq!(command.leading, b"\x1b[7;11H");
+ assert!(control.starts_with("a=T,f=32,s=30,v=30,i=2147483657,p="));
+ assert!(control.contains(",c=2,r=3,z=0,C=1,q=0,x=10,w=20,h=30"));
+ assert!(direct_file_command(&pane_layer_placement(30, 0), 9).is_none());
+ }
+
#[test]
fn pane_graphics_image_ids_are_disjoint_from_terminal_image_ids() {
let placement = test_placement(0, 0);
let signature = image_signature(&placement, kitty_format_code(placement.placement.format));
let terminal_id = host_image_id_for_signature(placement.pane_id, signature);
- let pane_graphics_id = pane_graphics_host_image_id(placement.pane_id, signature);
+ let mut graphics = crate::app::pane_graphics::Runtime::default();
+ let primary = (placement.pane_id, "primary".into());
+ let pane_graphics_id = graphics.reserve_image_id(&primary).unwrap();
+ graphics.slots.insert(
+ primary.clone(),
+ crate::app::pane_graphics::Slot::test(pane_graphics_id, None),
+ );
assert_eq!(terminal_id & PANE_GRAPHICS_IMAGE_ID_BIT, 0);
assert_ne!(pane_graphics_id & PANE_GRAPHICS_IMAGE_ID_BIT, 0);
+ assert_eq!(
+ pane_graphics_id,
+ graphics.reserve_image_id(&primary).unwrap()
+ );
+ assert_ne!(
+ pane_graphics_id,
+ graphics
+ .reserve_image_id(&(placement.pane_id, "toolbar".into()))
+ .unwrap()
+ );
}
#[test]
@@ -1121,16 +1814,19 @@ mod tests {
borders: ratatui::widgets::Borders::NONE,
is_focused: true,
};
- let layer = crate::app::state::PaneGraphicsLayer::new(
+ let layer = crate::app::pane_graphics::Layer::inline(
crate::api::schema::PaneGraphicsFormat::Rgba,
80,
30,
vec![255; 80 * 30 * 4],
crate::api::schema::PaneGraphicsPlacementParams::default(),
+ 0,
);
let placement = pane_graphics_host_placement(
&info,
+ "primary",
+ PANE_GRAPHICS_IMAGE_ID_BIT | 1,
HostCellSize {
width_px: 10,
height_px: 10,
@@ -1151,166 +1847,55 @@ mod tests {
#[test]
fn graphics_update_uploads_once_then_repositions_only() {
- let mut images = HashMap::new();
- let mut placements = HashMap::new();
- let mut sources = HashMap::new();
- let mut bytes = Vec::new();
- let placement = test_placement(0, 0);
-
- encode_graphics_update(
- &mut bytes,
- &[placement],
- false,
- &mut images,
- &mut placements,
- &mut sources,
- );
- let first = String::from_utf8_lossy(&bytes);
- assert!(first.contains("a=t"));
- assert!(first.contains("a=p"));
-
- bytes.clear();
- let same = test_placement(0, 0);
- encode_graphics_update(
- &mut bytes,
- &[same],
- false,
- &mut images,
- &mut placements,
- &mut sources,
- );
- assert!(bytes.is_empty());
-
- let mut z_changed = test_placement(0, 0);
- z_changed.placement.z = 1;
- encode_graphics_update(
- &mut bytes,
- &[z_changed],
- false,
- &mut images,
- &mut placements,
- &mut sources,
- );
- let z_changed_bytes = String::from_utf8_lossy(&bytes);
- assert!(!z_changed_bytes.contains("a=t"));
- assert!(z_changed_bytes.contains("a=p"));
+ let mut cache = HostGraphicsCache::default();
+ let first = update(&mut cache, &[test_placement(0, 0)], false);
+ assert!(String::from_utf8_lossy(&first).contains("a=t"));
+ assert!(String::from_utf8_lossy(&first).contains("a=p"));
+ assert!(update(&mut cache, &[test_placement(0, 0)], false).is_empty());
- bytes.clear();
- let moved = test_placement(0, 1);
- encode_graphics_update(
- &mut bytes,
- &[moved],
- false,
- &mut images,
- &mut placements,
- &mut sources,
- );
- let moved_bytes = String::from_utf8_lossy(&bytes);
- assert!(!moved_bytes.contains("a=t"));
- assert!(moved_bytes.contains("a=p"));
+ let mut changed = test_placement(0, 0);
+ changed.placement.z = 1;
+ for placement in [changed, test_placement(0, 1)] {
+ let bytes = update(&mut cache, &[placement], false);
+ assert!(!String::from_utf8_lossy(&bytes).contains("a=t"));
+ assert!(String::from_utf8_lossy(&bytes).contains("a=p"));
+ }
}
#[test]
fn view_change_redisplays_unchanged_visible_placement() {
- let mut images = HashMap::new();
- let mut placements = HashMap::new();
- let mut sources = HashMap::new();
- let mut bytes = Vec::new();
- let placement = test_placement(0, 0);
-
- encode_graphics_update(
- &mut bytes,
- &[placement],
- false,
- &mut images,
- &mut placements,
- &mut sources,
- );
- assert_eq!(placements.len(), 1);
-
- bytes.clear();
- let same = test_placement(0, 0);
- encode_graphics_update(
- &mut bytes,
- &[same],
- true,
- &mut images,
- &mut placements,
- &mut sources,
- );
- let redisplay = String::from_utf8_lossy(&bytes);
- assert!(!redisplay.contains("a=t"));
- assert!(redisplay.contains("a=p"));
- assert_eq!(placements.len(), 1);
+ let mut cache = HostGraphicsCache::default();
+ update(&mut cache, &[test_placement(0, 0)], false);
+ assert_eq!(cache.placements.len(), 1);
+ let bytes = update(&mut cache, &[test_placement(0, 0)], true);
+ assert!(!String::from_utf8_lossy(&bytes).contains("a=t"));
+ assert!(String::from_utf8_lossy(&bytes).contains("a=p"));
+ assert_eq!(cache.placements.len(), 1);
}
#[test]
fn surface_reset_deletes_then_reuploads_and_redisplays_placement() {
let mut cache = HostGraphicsCache::default();
- let mut bytes = Vec::new();
- let placement = test_placement(0, 0);
-
- encode_graphics_update(
- &mut bytes,
- &[placement],
- false,
- &mut cache.images,
- &mut cache.placements,
- &mut cache.sources,
- );
- assert_eq!(cache.images.len(), 1);
- assert_eq!(cache.placements.len(), 1);
-
- bytes = cache.clear_bytes();
- let same = test_placement(0, 0);
- encode_graphics_update(
- &mut bytes,
- &[same],
- false,
- &mut cache.images,
- &mut cache.placements,
- &mut cache.sources,
- );
-
+ update(&mut cache, &[test_placement(0, 0)], false);
+ assert_eq!((cache.images.len(), cache.placements.len()), (1, 1));
+ let mut bytes = cache.clear_bytes();
+ bytes.extend(update(&mut cache, &[test_placement(0, 0)], false));
let redisplay = String::from_utf8_lossy(&bytes);
assert!(redisplay.contains("a=d,d=I"));
assert!(redisplay.contains("a=t"));
assert!(redisplay.contains("a=p"));
- assert_eq!(cache.images.len(), 1);
- assert_eq!(cache.placements.len(), 1);
+ assert_eq!((cache.images.len(), cache.placements.len()), (1, 1));
}
#[test]
fn scrollback_offset_change_redisplays_placement() {
- let mut images = HashMap::new();
- let mut placements = HashMap::new();
- let mut sources = HashMap::new();
- let mut bytes = Vec::new();
- let placement = test_placement(0, 0);
-
- encode_graphics_update(
- &mut bytes,
- &[placement],
- false,
- &mut images,
- &mut placements,
- &mut sources,
- );
-
- bytes.clear();
+ let mut cache = HostGraphicsCache::default();
+ update(&mut cache, &[test_placement(0, 0)], false);
let mut scrolled = test_placement(0, 0);
scrolled.scrollback_offset = 3;
- encode_graphics_update(
- &mut bytes,
- &[scrolled],
- false,
- &mut images,
- &mut placements,
- &mut sources,
- );
- let redisplay = String::from_utf8_lossy(&bytes);
- assert!(!redisplay.contains("a=t"));
- assert!(redisplay.contains("a=p"));
+ let bytes = update(&mut cache, &[scrolled], false);
+ assert!(!String::from_utf8_lossy(&bytes).contains("a=t"));
+ assert!(String::from_utf8_lossy(&bytes).contains("a=p"));
}
#[test]
@@ -1377,6 +1962,42 @@ mod tests {
}
#[test]
+ fn pane_layer_replacement_is_atomic_without_delete_to_blank() {
+ let mut images = HashMap::new();
+ let mut placements = HashMap::new();
+ let mut sources = HashMap::new();
+ let mut bytes = Vec::new();
+ let mut first = pane_layer_placement(0, 0);
+ first.host_image_id = Some((1 << 31) | 7);
+ encode_graphics_update(
+ &mut bytes,
+ &[first],
+ false,
+ &mut images,
+ &mut placements,
+ &mut sources,
+ );
+
+ bytes.clear();
+ let mut changed = pane_layer_placement(0, 0);
+ changed.host_image_id = Some((1 << 31) | 7);
+ changed.placement.data_fingerprint += 1;
+ encode_graphics_update(
+ &mut bytes,
+ &[changed],
+ false,
+ &mut images,
+ &mut placements,
+ &mut sources,
+ );
+
+ let update = String::from_utf8_lossy(&bytes);
+ assert!(update.contains("a=T,t=d"));
+ assert!(update.contains(",p=") && update.contains(",C=1,q=2"));
+ assert!(!update.contains("a=d"));
+ }
+
+ #[test]
fn replaced_image_content_deletes_superseded_host_image() {
let mut images = HashMap::new();
let mut placements = HashMap::new();
@@ -1555,40 +2176,143 @@ mod tests {
}
#[test]
- fn removed_pane_layer_deletes_unreferenced_host_image() {
+ fn trusted_direct_image_uses_reserved_id_for_placement_without_upload() {
+ let key = (PaneId::from_raw(1), "primary".to_owned());
+ let layer = crate::app::pane_graphics::Layer::inline(
+ crate::api::schema::PaneGraphicsFormat::Rgba,
+ 30,
+ 30,
+ vec![255; 30 * 30 * 4],
+ Default::default(),
+ 0,
+ );
+ let reserved_id = (1 << 31) | 77;
+ let mut cache = HostGraphicsCache::default();
+ cache.trust_pane_layer(&key, reserved_id, &layer);
+ let mut placement = pane_layer_placement(0, 0);
+ placement.host_image_id = Some(reserved_id);
+ placement.placement.data.clear();
+ placement.placement.data_len = layer.data_len();
+ placement.placement.data_fingerprint = layer.data_fingerprint;
+ let mut bytes = Vec::new();
+
+ encode_graphics_update(
+ &mut bytes,
+ &[placement],
+ false,
+ &mut cache.images,
+ &mut cache.placements,
+ &mut cache.sources,
+ );
+
+ let update = String::from_utf8_lossy(&bytes);
+ assert!(update.contains(&format!("a=p,i={reserved_id}")));
+ assert!(!update.contains("a=t"));
+
+ let live = HashSet::from([HostSourceKey::PaneLayer {
+ pane_id: key.0,
+ layer_id: key.1.clone(),
+ }]);
+ let hidden = String::from_utf8(cache.hide_except_live_pane_layers(&live)).unwrap();
+ assert!(hidden.contains("a=d,d=i"));
+ assert!(!hidden.contains("a=d,d=I"));
+ assert!(cache.images.contains_key(&reserved_id));
+ assert!(cache.placements.is_empty());
+
+ let mut returning = pane_layer_placement(0, 0);
+ returning.host_image_id = Some(reserved_id);
+ returning.placement.data.clear();
+ returning.placement.data_len = layer.data_len();
+ returning.placement.data_fingerprint = layer.data_fingerprint;
+ let mut replay = Vec::new();
+ encode_graphics_update(
+ &mut replay,
+ &[returning],
+ false,
+ &mut cache.images,
+ &mut cache.placements,
+ &mut cache.sources,
+ );
+ let replay = String::from_utf8(replay).unwrap();
+ assert!(replay.contains(&format!("a=p,i={reserved_id}")));
+ assert!(!replay.contains("a=t"));
+
+ cache.forget_pane_layer(&key, reserved_id);
+ let mut fallback = pane_layer_placement(0, 0);
+ fallback.host_image_id = Some(reserved_id);
+ let mut retransmit = Vec::new();
+ encode_graphics_update(
+ &mut retransmit,
+ &[fallback],
+ false,
+ &mut cache.images,
+ &mut cache.placements,
+ &mut cache.sources,
+ );
+ assert!(String::from_utf8_lossy(&retransmit).contains("a=t"));
+ }
+
+ #[test]
+ fn hidden_layer_and_full_redraw_replay_placement_without_pixels() {
let mut images = HashMap::new();
let mut placements = HashMap::new();
let mut sources = HashMap::new();
let mut bytes = Vec::new();
+ let placement = pane_layer_placement(0, 0);
encode_graphics_update(
&mut bytes,
- &[pane_layer_placement(0, 0)],
+ &[placement],
false,
&mut images,
&mut placements,
&mut sources,
);
- let host_id = *images.keys().next().expect("uploaded pane layer");
- bytes.clear();
+ for (visible, replay) in [(false, false), (true, false), (true, true)] {
+ bytes.clear();
+ let current = visible.then(|| pane_layer_placement(0, 0));
+ encode_graphics_update(
+ &mut bytes,
+ current.as_slice(),
+ replay,
+ &mut images,
+ &mut placements,
+ &mut sources,
+ );
+ let update = String::from_utf8_lossy(&bytes);
+ assert!(!update.contains("a=t"));
+ assert!(!update.contains("a=d,d=I"));
+ assert_eq!(update.contains("a=p"), visible);
+ }
+ assert_eq!(images.len(), 1);
+ assert_eq!(sources.len(), 1);
+ }
+
+ #[test]
+ fn removed_pane_layer_deletes_unreferenced_host_image() {
+ let mut cache = HostGraphicsCache::default();
+ let mut bytes = Vec::new();
encode_graphics_update(
&mut bytes,
- &[],
+ &[pane_layer_placement(0, 0)],
false,
- &mut images,
- &mut placements,
- &mut sources,
+ &mut cache.images,
+ &mut cache.placements,
+ &mut cache.sources,
);
+ let host_id = *cache.images.keys().next().expect("uploaded pane layer");
+
+ bytes = drain_graphics_updates(&mut cache, &[], &HashSet::new());
let delete = String::from_utf8_lossy(&bytes);
assert!(delete.contains(&format!("a=d,d=I,i={host_id}")));
- assert!(images.is_empty());
- assert!(placements.is_empty());
- assert!(sources.is_empty());
+ assert!(cache.images.is_empty());
+ assert!(cache.placements.is_empty());
+ assert!(cache.sources.is_empty());
}
#[test]
- fn clipped_pane_layer_deletes_unreferenced_host_image() {
+ fn hidden_pane_layer_retains_image_and_removes_only_placement() {
let mut images = HashMap::new();
let mut placements = HashMap::new();
let mut sources = HashMap::new();
@@ -1613,11 +2337,12 @@ mod tests {
&mut sources,
);
- let delete = String::from_utf8_lossy(&bytes);
- assert!(delete.contains(&format!("a=d,d=I,i={host_id}")));
- assert!(images.is_empty());
+ let update = String::from_utf8_lossy(&bytes);
+ assert!(update.contains("a=d,d=i"));
+ assert!(!update.contains(&format!("a=d,d=I,i={host_id}")));
+ assert_eq!(images.len(), 1);
assert!(placements.is_empty());
- assert!(sources.is_empty());
+ assert_eq!(sources.len(), 1);
}
#[test]
@@ -1668,35 +2393,118 @@ mod tests {
#[test]
fn removed_pane_layer_preserves_image_shared_with_terminal_source() {
- let mut images = HashMap::new();
- let mut placements = HashMap::new();
- let mut sources = HashMap::new();
+ let mut cache = HostGraphicsCache::default();
let mut bytes = Vec::new();
encode_graphics_update(
&mut bytes,
&[pane_layer_placement(0, 0), test_placement(4, 0)],
false,
- &mut images,
- &mut placements,
- &mut sources,
+ &mut cache.images,
+ &mut cache.placements,
+ &mut cache.sources,
);
- assert_eq!(images.len(), 1);
+ assert_eq!(cache.images.len(), 1);
- bytes.clear();
+ bytes = drain_graphics_updates(&mut cache, &[], &HashSet::new());
encode_graphics_update(
&mut bytes,
&[test_placement(4, 0)],
false,
- &mut images,
- &mut placements,
- &mut sources,
+ &mut cache.images,
+ &mut cache.placements,
+ &mut cache.sources,
);
let update = String::from_utf8_lossy(&bytes);
assert!(!update.contains("a=d,d=I"));
- assert_eq!(images.len(), 1);
- assert_eq!(placements.len(), 1);
- assert_eq!(sources.len(), 1);
+ assert_eq!(cache.images.len(), 1);
+ assert_eq!(cache.placements.len(), 1);
+ assert_eq!(cache.sources.len(), 1);
+ }
+
+ #[test]
+ fn changing_first_source_does_not_starve_second_source() {
+ let layers = |first| {
+ [(1, "a", first), (2, "b", 80)].map(|(id, name, fingerprint)| {
+ let mut placement = pane_layer_placement(0, 0);
+ placement.host_image_id = Some(PANE_GRAPHICS_IMAGE_ID_BIT | id);
+ placement.source_key = HostSourceKey::PaneLayer {
+ pane_id: placement.pane_id,
+ layer_id: name.into(),
+ };
+ placement.placement.data_fingerprint = fingerprint;
+ placement
+ })
+ };
+ let initial = layers(42);
+ let live = initial.iter().map(|p| p.source_key.clone()).collect();
+ let mut cache = HostGraphicsCache::default();
+ assert!(encode_graphics_update_incremental(&mut cache, &initial, &live, None).incomplete);
+ assert!(
+ encode_graphics_update_incremental(&mut cache, &layers(43), &live, None).incomplete
+ );
+ assert_eq!(cache.images.len(), 2, "second source uploaded next");
+
+ let terminal = |id| {
+ let mut placement = test_placement(0, 0);
+ placement.placement.image_id = id;
+ placement.placement.data_fingerprint = u64::from(id);
+ placement.source_key = HostSourceKey::Terminal {
+ pane_id: placement.pane_id,
+ image_id: id,
+ };
+ placement
+ };
+ let second = terminal(99).source_key;
+ let mut cache = HostGraphicsCache::default();
+ for id in 1..=3 {
+ assert!(
+ encode_graphics_update_incremental(
+ &mut cache,
+ &[terminal(id), terminal(99)],
+ &HashSet::new(),
+ None,
+ )
+ .incomplete
+ );
+ }
+ assert!(cache.sources.contains_key(&second));
+ }
+
+ #[test]
+ fn large_terminal_image_is_local_but_quarantined_headless() {
+ let placements = || {
+ let mut large = test_placement(0, 0);
+ large.placement.data_len = 24 * 1024 * 1024;
+ let mut later = test_placement(4, 0);
+ later.placement.image_id = 8;
+ later.source_key = HostSourceKey::Terminal {
+ pane_id: later.pane_id,
+ image_id: 8,
+ };
+ [large, later]
+ };
+ for (budget, expected) in [
+ (None, (true, 1, 0)),
+ (Some(HEADLESS_GRAPHICS_TRANSACTION_BUDGET), (false, 1, 1)),
+ ] {
+ let mut cache = HostGraphicsCache::default();
+ let encoded = encode_graphics_update_incremental(
+ &mut cache,
+ &placements(),
+ &HashSet::new(),
+ budget,
+ );
+ assert!(String::from_utf8_lossy(&encoded.bytes).contains("a=t"));
+ assert_eq!(
+ (
+ encoded.incomplete,
+ cache.images.len(),
+ cache.oversized.len()
+ ),
+ expected
+ );
+ }
}
#[test]
@@ -1719,37 +2527,12 @@ mod tests {
));
encode_display_placement(&mut encoded, clipped, host_id, 1, 0);
- assert!(encoded.len() < crate::protocol::MAX_GRAPHICS_FRAME_SIZE);
- }
-
- #[test]
- fn view_change_deletes_stale_placement_immediately() {
- let mut images = HashMap::new();
- let mut placements = HashMap::new();
- let mut sources = HashMap::new();
- let mut bytes = Vec::new();
- let placement = test_placement(0, 0);
-
- encode_graphics_update(
- &mut bytes,
- &[placement],
- false,
- &mut images,
- &mut placements,
- &mut sources,
- );
- bytes.clear();
- encode_graphics_update(
- &mut bytes,
- &[],
- true,
- &mut images,
- &mut placements,
- &mut sources,
- );
-
- let delete = String::from_utf8_lossy(&bytes);
- assert!(delete.contains("a=d,d=i"));
- assert!(placements.is_empty());
+ let mut framed = Vec::new();
+ crate::protocol::write_message(
+ &mut framed,
+ &crate::protocol::ServerMessage::Graphics { bytes: encoded },
+ )
+ .unwrap();
+ assert!(framed.len() <= crate::protocol::MAX_GRAPHICS_FRAME_SIZE + 4);
}
}
diff --git a/src/main.rs b/src/main.rs
index c3269675..2db4f8aa 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -75,6 +75,7 @@ mod logging;
mod metadata_tokens;
mod noninteractive_process;
mod pane;
+mod pane_graphics_files;
mod persist;
mod platform;
mod plugin_command;
diff --git a/src/pane.rs b/src/pane.rs
index ab984946..da99f55e 100644
--- a/src/pane.rs
+++ b/src/pane.rs
@@ -2780,40 +2780,41 @@ impl PaneRuntime {
pub fn encode_mouse_button(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
if !self.input_state()?.mouse_protocol_mode.reporting_enabled() {
return None;
}
- self.terminal
- .encode_mouse_button(kind, column, row, modifiers)
+ self.terminal.encode_mouse_button(kind, position, modifiers)
}
- pub fn encode_mouse_motion(
+ pub(crate) fn encode_mouse_motion(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- self.terminal
- .encode_mouse_motion(kind, column, row, modifiers)
+ self.terminal.encode_mouse_motion(kind, position, modifiers)
}
- pub fn encode_mouse_wheel(
+ pub(crate) fn encode_mouse_wheel(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
if self.wheel_routing()? != WheelRouting::MouseReport {
return None;
}
- self.terminal
- .encode_mouse_wheel(kind, column, row, modifiers)
+ self.terminal.encode_mouse_wheel(kind, position, modifiers)
+ }
+
+ pub(crate) fn pixel_size(&self) -> Option<(u32, u32)> {
+ let (rows, cols, cell_width_px, cell_height_px) = self.current_size.get();
+ let width = u32::from(cols).checked_mul(cell_width_px)?;
+ let height = u32::from(rows).checked_mul(cell_height_px)?;
+ (width > 0 && height > 0).then_some((width, height))
}
pub fn encode_alternate_scroll(
diff --git a/src/pane/input.rs b/src/pane/input.rs
index 2e60bfc8..2ff299c9 100644
--- a/src/pane/input.rs
+++ b/src/pane/input.rs
@@ -60,23 +60,43 @@ pub(super) fn ghostty_mods_from_key_modifiers(modifiers: crossterm::event::KeyMo
pub(super) fn ghostty_mouse_encoder_for_terminal(
terminal: &crate::ghostty::Terminal,
+ position: crate::input::mouse::Position,
) -> Option<crate::ghostty::MouseEncoder> {
let mut encoder = crate::ghostty::MouseEncoder::new().ok()?;
encoder.set_from_terminal(terminal);
- if terminal
- .mode_get(crate::ghostty::MODE_MOUSE_SGR_PIXELS)
- .ok()?
- {
- // Herdr receives host mouse positions in terminal cells. Downgrade
- // SGR-pixels to normal SGR so forwarded coordinates stay cell-local.
- encoder.set_format(crate::ghostty::MOUSE_FORMAT_SGR);
- }
let cols = terminal.cols().ok()? as u32;
let rows = terminal.rows().ok()? as u32;
- encoder.set_size(cols, rows, 1, 1);
+ match position {
+ crate::input::mouse::Position::Cell { .. } => {
+ if terminal
+ .mode_get(crate::ghostty::MODE_MOUSE_SGR_PIXELS)
+ .ok()?
+ {
+ encoder.set_format(crate::ghostty::MOUSE_FORMAT_SGR);
+ }
+ encoder.set_size(cols, rows, 1, 1);
+ }
+ crate::input::mouse::Position::Pixels { .. } => {
+ let width_px = terminal.width_px().ok()?;
+ let height_px = terminal.height_px().ok()?;
+ if width_px == 0 || height_px == 0 || cols == 0 || rows == 0 {
+ return None;
+ }
+ encoder.set_size(width_px, height_px, width_px / cols, height_px / rows);
+ }
+ }
Some(encoder)
}
+pub(super) fn ghostty_mouse_position_for_terminal(
+ position: crate::input::mouse::Position,
+) -> Option<(f32, f32)> {
+ match position {
+ crate::input::mouse::Position::Pixels { x, y } => Some((x as f32, y as f32)),
+ crate::input::mouse::Position::Cell { column, row } => Some((column as f32, row as f32)),
+ }
+}
+
pub(super) fn ghostty_mouse_event_from_button_kind(
kind: crossterm::event::MouseEventKind,
column: u16,
diff --git a/src/pane/terminal.rs b/src/pane/terminal.rs
index 8de84af3..91fcb4fe 100644
--- a/src/pane/terminal.rs
+++ b/src/pane/terminal.rs
@@ -23,7 +23,8 @@ use super::{
input::{
ghostty_key_event_from_terminal_key, ghostty_mouse_encoder_for_terminal,
ghostty_mouse_event_from_button_kind, ghostty_mouse_event_from_motion_kind,
- ghostty_mouse_event_from_wheel_kind, ghostty_prefers_herdr_text_encoding,
+ ghostty_mouse_event_from_wheel_kind, ghostty_mouse_position_for_terminal,
+ ghostty_prefers_herdr_text_encoding,
},
kitty_keyboard::KittyKeyboardTracker,
osc::{
@@ -537,37 +538,31 @@ impl PaneTerminal {
self.ghostty.encode_terminal_key(key, protocol)
}
- pub fn encode_mouse_button(
+ pub(crate) fn encode_mouse_button(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- self.ghostty
- .encode_mouse_button(kind, column, row, modifiers)
+ self.ghostty.encode_mouse_button(kind, position, modifiers)
}
- pub fn encode_mouse_motion(
+ pub(crate) fn encode_mouse_motion(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- self.ghostty
- .encode_mouse_motion(kind, column, row, modifiers)
+ self.ghostty.encode_mouse_motion(kind, position, modifiers)
}
- pub fn encode_mouse_wheel(
+ pub(crate) fn encode_mouse_wheel(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- self.ghostty
- .encode_mouse_wheel(kind, column, row, modifiers)
+ self.ghostty.encode_mouse_wheel(kind, position, modifiers)
}
}
@@ -1488,6 +1483,9 @@ impl GhosttyPaneTerminal {
let _ = core
.terminal
.mode_set(crate::ghostty::MODE_MOUSE_SGR, false);
+ let _ = core
+ .terminal
+ .mode_set(crate::ghostty::MODE_MOUSE_SGR_PIXELS, false);
match input_state.mouse_protocol_encoding {
crate::input::MouseProtocolEncoding::Default => {}
crate::input::MouseProtocolEncoding::Utf8 => {
@@ -1498,6 +1496,11 @@ impl GhosttyPaneTerminal {
crate::input::MouseProtocolEncoding::Sgr => {
let _ = core.terminal.mode_set(crate::ghostty::MODE_MOUSE_SGR, true);
}
+ crate::input::MouseProtocolEncoding::SgrPixels => {
+ let _ = core
+ .terminal
+ .mode_set(crate::ghostty::MODE_MOUSE_SGR_PIXELS, true);
+ }
}
if input_state.modify_other_keys {
@@ -1688,6 +1691,10 @@ impl GhosttyPaneTerminal {
.terminal
.mode_get(crate::ghostty::MODE_MOUSE_UTF8)
.ok()?;
+ let mouse_sgr_pixels = core
+ .terminal
+ .mode_get(crate::ghostty::MODE_MOUSE_SGR_PIXELS)
+ .ok()?;
let mouse_alternate_scroll = core
.terminal
.mode_get(crate::ghostty::MODE_MOUSE_ALTERNATE_SCROLL)
@@ -1703,7 +1710,9 @@ impl GhosttyPaneTerminal {
} else {
crate::input::MouseProtocolMode::None
};
- let mouse_protocol_encoding = if mouse_sgr {
+ let mouse_protocol_encoding = if mouse_sgr_pixels {
+ crate::input::MouseProtocolEncoding::SgrPixels
+ } else if mouse_sgr {
crate::input::MouseProtocolEncoding::Sgr
} else if mouse_utf8 {
crate::input::MouseProtocolEncoding::Utf8
@@ -1838,57 +1847,58 @@ impl GhosttyPaneTerminal {
}
}
- pub fn encode_mouse_button(
+ pub(crate) fn encode_mouse_button(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- let Ok(core) = self.core.lock() else {
- return None;
- };
- let mut encoder = ghostty_mouse_encoder_for_terminal(&core.terminal)?;
- let event = ghostty_mouse_event_from_button_kind(kind, column, row, modifiers)?;
- encoder
- .encode(&event)
- .ok()
- .filter(|bytes| !bytes.is_empty())
+ self.encode_mouse_event(
+ ghostty_mouse_event_from_button_kind(kind, 0, 0, modifiers)?,
+ position,
+ false,
+ )
}
- pub fn encode_mouse_motion(
+ pub(crate) fn encode_mouse_motion(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- let Ok(core) = self.core.lock() else {
- return None;
- };
- if !core.terminal.mode_get(MODE_MOUSE_ANY_MOTION).ok()? {
- return None;
- }
- let mut encoder = ghostty_mouse_encoder_for_terminal(&core.terminal)?;
- let event = ghostty_mouse_event_from_motion_kind(kind, column, row, modifiers)?;
- encoder
- .encode(&event)
- .ok()
- .filter(|bytes| !bytes.is_empty())
+ self.encode_mouse_event(
+ ghostty_mouse_event_from_motion_kind(kind, 0, 0, modifiers)?,
+ position,
+ true,
+ )
}
- pub fn encode_mouse_wheel(
+ pub(crate) fn encode_mouse_wheel(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- let Ok(core) = self.core.lock() else {
+ self.encode_mouse_event(
+ ghostty_mouse_event_from_wheel_kind(kind, 0, 0, modifiers)?,
+ position,
+ false,
+ )
+ }
+
+ fn encode_mouse_event(
+ &self,
+ mut event: crate::ghostty::MouseEvent,
+ position: crate::input::mouse::Position,
+ require_any_motion: bool,
+ ) -> Option<Vec<u8>> {
+ let core = self.core.lock().ok()?;
+ if require_any_motion && !core.terminal.mode_get(MODE_MOUSE_ANY_MOTION).ok()? {
return None;
- };
- let mut encoder = ghostty_mouse_encoder_for_terminal(&core.terminal)?;
- let event = ghostty_mouse_event_from_wheel_kind(kind, column, row, modifiers)?;
+ }
+ let mut encoder = ghostty_mouse_encoder_for_terminal(&core.terminal, position)?;
+ let (x, y) = ghostty_mouse_position_for_terminal(position)?;
+ event.set_position(x, y);
encoder
.encode(&event)
.ok()
@@ -4670,8 +4680,7 @@ mod tests {
let encoded = pane.encode_mouse_button(
crossterm::event::MouseEventKind::Up(crossterm::event::MouseButton::Left),
- 11,
- 9,
+ crate::input::mouse::Position::Cell { column: 11, row: 9 },
crossterm::event::KeyModifiers::empty(),
);
@@ -4687,8 +4696,7 @@ mod tests {
let encoded = pane.encode_mouse_button(
crossterm::event::MouseEventKind::Drag(crossterm::event::MouseButton::Left),
- 4,
- 6,
+ crate::input::mouse::Position::Cell { column: 4, row: 6 },
crossterm::event::KeyModifiers::SHIFT,
);
@@ -4704,8 +4712,7 @@ mod tests {
let encoded = pane.encode_mouse_button(
crossterm::event::MouseEventKind::Drag(crossterm::event::MouseButton::Left),
- 4,
- 6,
+ crate::input::mouse::Position::Cell { column: 4, row: 6 },
crossterm::event::KeyModifiers::empty(),
);
@@ -4721,8 +4728,7 @@ mod tests {
let encoded = pane.encode_mouse_motion(
crossterm::event::MouseEventKind::Moved,
- 4,
- 6,
+ crate::input::mouse::Position::Cell { column: 4, row: 6 },
crossterm::event::KeyModifiers::empty(),
);
@@ -4730,20 +4736,26 @@ mod tests {
}
#[test]
- fn ghostty_mouse_sgr_pixels_downgrades_to_cell_coordinates() {
+ fn ghostty_mouse_sgr_pixels_preserves_exact_and_downgrades_cell_input() {
let (tx, _rx) = mpsc::channel(4);
let mut terminal = crate::ghostty::Terminal::new(80, 24, 0).unwrap();
+ terminal.resize(80, 24, 10, 20).unwrap();
terminal.write(b"\x1b[?1003h\x1b[?1006h\x1b[?1016h");
let pane = GhosttyPaneTerminal::new(terminal, tx).unwrap();
- let encoded = pane.encode_mouse_motion(
+ let exact = pane.encode_mouse_motion(
crossterm::event::MouseEventKind::Moved,
- 4,
- 6,
+ crate::input::mouse::Position::Pixels { x: 48, y: 139 },
+ crossterm::event::KeyModifiers::empty(),
+ );
+ let fallback = pane.encode_mouse_motion(
+ crossterm::event::MouseEventKind::Moved,
+ crate::input::mouse::Position::Cell { column: 4, row: 6 },
crossterm::event::KeyModifiers::empty(),
);
- assert_eq!(encoded.as_deref(), Some(&b"\x1b[<35;5;7M"[..]));
+ assert_eq!(exact.as_deref(), Some(&b"\x1b[<35;48;139M"[..]));
+ assert_eq!(fallback.as_deref(), Some(&b"\x1b[<35;5;7M"[..]));
}
#[test]
diff --git a/src/pane_graphics_files.rs b/src/pane_graphics_files.rs
new file mode 100644
index 00000000..527be1d5
--- /dev/null
+++ b/src/pane_graphics_files.rs
@@ -0,0 +1,529 @@
+#[cfg(unix)]
+use std::fs::OpenOptions;
+use std::fs::{self, File};
+use std::io;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, OnceLock};
+
+#[cfg(unix)]
+use std::os::unix::fs::{FileExt, MetadataExt, OpenOptionsExt, PermissionsExt};
+
+#[cfg(unix)]
+const DIRECTORY_MODE: u32 = 0o700;
+#[cfg(unix)]
+const FILE_MODE: u32 = 0o600;
+
+#[derive(Debug)]
+pub(crate) struct FileStore {
+ base: PathBuf,
+ generation: OnceLock<Arc<Generation>>,
+ next_fingerprint: AtomicU64,
+}
+
+#[derive(Debug)]
+struct Generation {
+ root: PathBuf,
+ source: PathBuf,
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct Lease {
+ inner: Arc<LeaseInner>,
+}
+
+#[derive(Debug)]
+struct LeaseInner {
+ path: PathBuf,
+ file: File,
+ generation: Arc<Generation>,
+ metadata: fs::Metadata,
+ len: usize,
+ fingerprint: u64,
+}
+
+impl Default for FileStore {
+ fn default() -> Self {
+ Self::new(runtime_base())
+ }
+}
+
+impl FileStore {
+ fn new(base: PathBuf) -> Self {
+ Self {
+ base,
+ generation: OnceLock::new(),
+ next_fingerprint: AtomicU64::new(1),
+ }
+ }
+
+ pub(crate) fn source_directory(&self) -> io::Result<PathBuf> {
+ Ok(self.generation()?.source.clone())
+ }
+
+ pub(crate) fn lease(&self, path: &Path, expected_len: usize) -> io::Result<Lease> {
+ let generation = self.generation()?;
+ validate_child(path, &generation.source)?;
+ let file = open_no_follow(path)?;
+ let metadata = file.metadata()?;
+ validate_metadata(&metadata, expected_len)?;
+ validate_path_identity(path, &metadata)?;
+ let fingerprint = self.next_fingerprint.fetch_add(1, Ordering::Relaxed);
+ Ok(Lease {
+ inner: Arc::new(LeaseInner {
+ path: path.to_owned(),
+ file,
+ generation,
+ metadata,
+ len: expected_len,
+ fingerprint,
+ }),
+ })
+ }
+
+ fn generation(&self) -> io::Result<Arc<Generation>> {
+ if let Some(generation) = self.generation.get() {
+ return Ok(Arc::clone(generation));
+ }
+ let generation = Arc::new(create_generation(&self.base)?);
+ match self.generation.set(Arc::clone(&generation)) {
+ Ok(()) => Ok(generation),
+ Err(_) => self
+ .generation
+ .get()
+ .cloned()
+ .ok_or_else(|| io::Error::other("pane graphics generation was lost")),
+ }
+ }
+
+ #[cfg(test)]
+ pub(crate) fn is_initialized(&self) -> bool {
+ self.generation.get().is_some()
+ }
+}
+
+impl Lease {
+ pub(crate) fn path(&self) -> &Path {
+ &self.inner.path
+ }
+
+ pub(crate) fn len(&self) -> usize {
+ self.inner.len
+ }
+
+ pub(crate) fn fingerprint(&self) -> u64 {
+ self.inner.fingerprint
+ }
+
+ pub(crate) fn copy_rgba(&self) -> io::Result<Vec<u8>> {
+ let _keep_generation_alive = &self.inner.generation;
+ validate_path_identity(&self.inner.path, &self.inner.metadata)?;
+ let mut data = vec![0; self.inner.len];
+ read_exact_at(&self.inner.file, &mut data)?;
+ if has_byte_at(&self.inner.file, self.inner.len as u64)? {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ "frame length changed while leased",
+ ));
+ }
+ validate_metadata(&self.inner.file.metadata()?, self.inner.len)?;
+ validate_path_identity(&self.inner.path, &self.inner.metadata)?;
+ Ok(data)
+ }
+}
+
+impl Drop for Generation {
+ fn drop(&mut self) {
+ if let Err(err) = fs::remove_dir_all(&self.root) {
+ if err.kind() != io::ErrorKind::NotFound {
+ tracing::warn!(path = %self.root.display(), err = %err, "failed to remove pane graphics directory");
+ }
+ }
+ }
+}
+
+#[cfg(unix)]
+pub(crate) fn validate_direct_source(path: &Path, expected_len: usize) -> io::Result<()> {
+ let source = path.parent().ok_or_else(invalid_path)?;
+ let generation = source.parent().ok_or_else(invalid_path)?;
+ if !path.is_absolute()
+ || path.file_name().is_none()
+ || source.file_name().and_then(|name| name.to_str()) != Some("source")
+ || generation.parent() != Some(runtime_base().as_path())
+ || !generation
+ .file_name()
+ .and_then(|name| name.to_str())
+ .is_some_and(|name| name.starts_with("server-"))
+ {
+ return Err(invalid_path());
+ }
+ for directory in [runtime_base(), generation.to_owned(), source.to_owned()] {
+ validate_directory(&directory)?;
+ }
+ let file = open_no_follow(path)?;
+ let metadata = file.metadata()?;
+ validate_metadata(&metadata, expected_len)?;
+ validate_path_identity(path, &metadata)
+}
+
+fn create_generation(base: &Path) -> io::Result<Generation> {
+ #[cfg(not(unix))]
+ {
+ let _ = base;
+ Err(io::Error::new(
+ io::ErrorKind::Unsupported,
+ "file-backed pane graphics require Unix",
+ ))
+ }
+ #[cfg(unix)]
+ {
+ fs::create_dir_all(base)?;
+ fs::set_permissions(base, fs::Permissions::from_mode(DIRECTORY_MODE))?;
+ validate_directory(base)?;
+ remove_stale_generations(base);
+ let nonce = std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_nanos();
+ let root = base.join(format!("server-{}-{nonce}", std::process::id()));
+ fs::create_dir(&root)?;
+ fs::set_permissions(&root, fs::Permissions::from_mode(DIRECTORY_MODE))?;
+ let source = root.join("source");
+ fs::create_dir(&source)?;
+ fs::set_permissions(&source, fs::Permissions::from_mode(DIRECTORY_MODE))?;
+ validate_directory(&root)?;
+ validate_directory(&source)?;
+ Ok(Generation { root, source })
+ }
+}
+
+fn runtime_base() -> PathBuf {
+ #[cfg(unix)]
+ {
+ let root = std::env::var_os("XDG_RUNTIME_DIR")
+ .map(PathBuf::from)
+ .filter(|path| path.is_absolute())
+ .unwrap_or_else(|| PathBuf::from("/var/tmp"));
+ root.join(format!("herdr-pane-graphics-{}", effective_uid()))
+ }
+ #[cfg(not(unix))]
+ {
+ PathBuf::from("pane-graphics-unavailable")
+ }
+}
+
+#[cfg(unix)]
+fn read_exact_at(file: &File, data: &mut [u8]) -> io::Result<()> {
+ file.read_exact_at(data, 0)
+}
+
+#[cfg(not(unix))]
+fn read_exact_at(_file: &File, _data: &mut [u8]) -> io::Result<()> {
+ Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only"))
+}
+
+#[cfg(unix)]
+fn has_byte_at(file: &File, offset: u64) -> io::Result<bool> {
+ Ok(file.read_at(&mut [0], offset)? != 0)
+}
+
+#[cfg(not(unix))]
+fn has_byte_at(_file: &File, _offset: u64) -> io::Result<bool> {
+ Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only"))
+}
+
+#[cfg(unix)]
+fn remove_stale_generations(base: &Path) {
+ let Ok(entries) = fs::read_dir(base) else {
+ return;
+ };
+ for entry in entries.flatten() {
+ let Some(pid) = entry
+ .file_name()
+ .to_str()
+ .and_then(|name| name.strip_prefix("server-"))
+ .and_then(|name| name.split('-').next())
+ .and_then(|pid| pid.parse::<i32>().ok())
+ .filter(|pid| *pid > 0)
+ else {
+ continue;
+ };
+ let Ok(file_type) = entry.file_type() else {
+ continue;
+ };
+ if !file_type.is_dir() {
+ continue;
+ }
+ // SAFETY: kill with signal 0 probes process existence without sending a signal.
+ let alive = unsafe { libc::kill(pid, 0) } == 0;
+ if alive || io::Error::last_os_error().raw_os_error() != Some(libc::ESRCH) {
+ continue;
+ }
+ if let Err(err) = fs::remove_dir_all(entry.path()) {
+ tracing::warn!(path = %entry.path().display(), err = %err, "failed to remove stale pane graphics directory");
+ }
+ }
+}
+
+fn validate_child(path: &Path, directory: &Path) -> io::Result<()> {
+ if !path.is_absolute() || path.parent() != Some(directory) || path.file_name().is_none() {
+ return Err(io::Error::new(
+ io::ErrorKind::PermissionDenied,
+ "frame path must be a direct child of the advertised source directory",
+ ));
+ }
+ Ok(())
+}
+
+fn open_no_follow(path: &Path) -> io::Result<File> {
+ #[cfg(unix)]
+ {
+ OpenOptions::new()
+ .read(true)
+ .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC)
+ .open(path)
+ }
+ #[cfg(not(unix))]
+ {
+ let _ = path;
+ Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only"))
+ }
+}
+
+fn validate_metadata(metadata: &fs::Metadata, expected_len: usize) -> io::Result<()> {
+ #[cfg(unix)]
+ {
+ if !metadata.file_type().is_file()
+ || metadata.uid() != effective_uid()
+ || metadata.mode() & 0o777 != FILE_MODE
+ || metadata.nlink() != 1
+ || metadata.len() != expected_len as u64
+ {
+ return Err(invalid(
+ "frame must be same-uid regular 0600 single-link exact-length file",
+ ));
+ }
+ Ok(())
+ }
+ #[cfg(not(unix))]
+ {
+ let _ = (metadata, expected_len);
+ Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only"))
+ }
+}
+
+#[cfg(unix)]
+fn validate_directory(path: &Path) -> io::Result<()> {
+ let metadata = fs::symlink_metadata(path)?;
+ if !metadata.file_type().is_dir()
+ || metadata.uid() != effective_uid()
+ || metadata.mode() & 0o777 != DIRECTORY_MODE
+ {
+ return Err(invalid("pane graphics directory is not private"));
+ }
+ Ok(())
+}
+
+fn validate_path_identity(path: &Path, expected: &fs::Metadata) -> io::Result<()> {
+ #[cfg(unix)]
+ {
+ let actual = fs::symlink_metadata(path)?;
+ if actual.dev() != expected.dev() || actual.ino() != expected.ino() {
+ return Err(invalid("frame path changed while leased"));
+ }
+ Ok(())
+ }
+ #[cfg(not(unix))]
+ {
+ let _ = (path, expected);
+ Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only"))
+ }
+}
+
+#[cfg(unix)]
+fn effective_uid() -> u32 {
+ // SAFETY: geteuid takes no arguments and has no preconditions.
+ unsafe { libc::geteuid() }
+}
+
+#[cfg(unix)]
+fn invalid(message: &str) -> io::Error {
+ io::Error::new(io::ErrorKind::InvalidData, message)
+}
+
+#[cfg(unix)]
+fn invalid_path() -> io::Error {
+ io::Error::new(
+ io::ErrorKind::PermissionDenied,
+ "invalid pane graphics source path",
+ )
+}
+
+#[cfg(all(test, unix))]
+mod tests {
+ use super::*;
+ use std::io::Write;
+ use std::os::unix::fs::{symlink, PermissionsExt};
+ use std::sync::atomic::AtomicU64;
+
+ static NEXT: AtomicU64 = AtomicU64::new(1);
+
+ fn store() -> (FileStore, PathBuf) {
+ let base = PathBuf::from(format!(
+ "/var/tmp/herdr-graphics-test-{}-{}",
+ std::process::id(),
+ NEXT.fetch_add(1, Ordering::Relaxed)
+ ));
+ (FileStore::new(base.clone()), base)
+ }
+
+ fn frame(store: &FileStore, name: &str, data: &[u8]) -> PathBuf {
+ let path = store.source_directory().unwrap().join(name);
+ let mut file = OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .mode(FILE_MODE)
+ .open(&path)
+ .unwrap();
+ file.write_all(data).unwrap();
+ path
+ }
+
+ #[test]
+ fn source_directory_is_lazy_private_and_removed_with_store() {
+ let (store, base) = store();
+ assert!(!store.is_initialized());
+ let source = store.source_directory().unwrap();
+ assert!(store.is_initialized());
+ validate_directory(&source).unwrap();
+ drop(store);
+ assert!(!source.exists());
+ let _ = fs::remove_dir(base);
+ }
+
+ #[test]
+ fn lease_holds_exact_file_and_copies_from_validated_fd() {
+ let (store, base) = store();
+ let path = frame(&store, "frame", &[1, 2, 3, 4]);
+ let lease = store.lease(&path, 4).unwrap();
+ assert_eq!(lease.path(), path);
+ assert_eq!(lease.len(), 4);
+ assert!(lease.fingerprint() > 0);
+ assert_eq!(lease.copy_rgba().unwrap(), [1, 2, 3, 4]);
+ drop(lease);
+ drop(store);
+ let _ = fs::remove_dir(base);
+ }
+
+ #[test]
+ fn validation_rejects_links_modes_lengths_and_outside_paths() {
+ let cases = ["mode", "length", "hard-link", "symlink", "outside"];
+ for case in cases {
+ let (store, base) = store();
+ let path = frame(&store, "frame", &[1, 2, 3, 4]);
+ let result = match case {
+ "mode" => {
+ fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap();
+ store.lease(&path, 4)
+ }
+ "length" => store.lease(&path, 8),
+ "hard-link" => {
+ fs::hard_link(&path, store.source_directory().unwrap().join("other")).unwrap();
+ store.lease(&path, 4)
+ }
+ "symlink" => {
+ let link = store.source_directory().unwrap().join("link");
+ symlink(&path, &link).unwrap();
+ store.lease(&link, 4)
+ }
+ "outside" => store.lease(&base.join("outside"), 4),
+ _ => unreachable!(),
+ };
+ assert!(result.is_err(), "{case}");
+ drop(store);
+ let _ = fs::remove_dir_all(base);
+ }
+ }
+
+ #[test]
+ fn direct_source_validation_accepts_only_the_runtime_generation() {
+ let runtime_store = FileStore::default();
+ let runtime_path = frame(&runtime_store, "direct", &[1, 2, 3, 4]);
+ validate_direct_source(&runtime_path, 4).unwrap();
+ assert!(validate_direct_source(&runtime_path, 8).is_err());
+ assert!(validate_direct_source(Path::new("relative"), 4).is_err());
+
+ let generation = runtime_path.parent().unwrap().parent().unwrap();
+ let wrong_source = generation.join("frames");
+ fs::create_dir(&wrong_source).unwrap();
+ fs::set_permissions(&wrong_source, fs::Permissions::from_mode(DIRECTORY_MODE)).unwrap();
+ let wrong_source_path = wrong_source.join("frame");
+ let mut file = OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .mode(FILE_MODE)
+ .open(&wrong_source_path)
+ .unwrap();
+ file.write_all(&[1, 2, 3, 4]).unwrap();
+ assert!(validate_direct_source(&wrong_source_path, 4).is_err());
+
+ let (other_store, other_base) = store();
+ let other_path = frame(&other_store, "other", &[1, 2, 3, 4]);
+ assert!(validate_direct_source(&other_path, 4).is_err());
+ drop(other_store);
+ let _ = fs::remove_dir_all(other_base);
+
+ let invalid_generation = runtime_base().join(format!(
+ "invalid-generation-{}-{}",
+ std::process::id(),
+ NEXT.fetch_add(1, Ordering::Relaxed)
+ ));
+ let invalid_source = invalid_generation.join("source");
+ fs::create_dir_all(&invalid_source).unwrap();
+ fs::set_permissions(
+ &invalid_generation,
+ fs::Permissions::from_mode(DIRECTORY_MODE),
+ )
+ .unwrap();
+ fs::set_permissions(&invalid_source, fs::Permissions::from_mode(DIRECTORY_MODE)).unwrap();
+ let invalid_path = invalid_source.join("frame");
+ let mut file = OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .mode(FILE_MODE)
+ .open(&invalid_path)
+ .unwrap();
+ file.write_all(&[1, 2, 3, 4]).unwrap();
+ assert!(validate_direct_source(&invalid_path, 4).is_err());
+ let _ = fs::remove_dir_all(invalid_generation);
+ }
+
+ #[test]
+ fn generation_creation_removes_dead_server_directories() {
+ let (store, base) = store();
+ fs::create_dir_all(&base).unwrap();
+ fs::set_permissions(&base, fs::Permissions::from_mode(DIRECTORY_MODE)).unwrap();
+ let stale = base.join("server-2147483647-stale");
+ fs::create_dir(&stale).unwrap();
+ fs::set_permissions(&stale, fs::Permissions::from_mode(DIRECTORY_MODE)).unwrap();
+
+ store.source_directory().unwrap();
+
+ assert!(!stale.exists());
+ drop(store);
+ let _ = fs::remove_dir_all(base);
+ }
+
+ #[test]
+ fn replacement_after_lease_is_detected_before_fallback_copy() {
+ let (store, base) = store();
+ let path = frame(&store, "frame", &[1, 2, 3, 4]);
+ let lease = store.lease(&path, 4).unwrap();
+ fs::remove_file(&path).unwrap();
+ let _replacement = frame(&store, "frame", &[5, 6, 7, 8]);
+ assert!(lease.copy_rgba().is_err());
+ drop(lease);
+ drop(store);
+ let _ = fs::remove_dir_all(base);
+ }
+}
diff --git a/src/protocol/wire.rs b/src/protocol/wire.rs
index 4af2c794..4a445361 100644
--- a/src/protocol/wire.rs
+++ b/src/protocol/wire.rs
@@ -57,6 +57,8 @@ pub enum ClientKeybindings {
pub enum ClientLaunchMode {
/// Full app client.
App,
+ /// Full app client eligible for audited local direct graphics.
+ AppDirectGraphics,
/// Direct terminal attach client.
TerminalAttach,
}
@@ -428,6 +430,25 @@ pub enum ClientMessage {
/// Replace an existing writable controller for this terminal.
takeover: bool,
},
+
+ /// Result of the one armed Herdr-owned direct Kitty transmission.
+ GraphicsTransmissionResult {
+ transfer_id: u64,
+ image_id: u32,
+ success: bool,
+ },
+
+ /// One confirmed SGR 1016 mouse report with read-time host geometry.
+ InputPixels {
+ data: Vec<u8>,
+ cols: u16,
+ rows: u16,
+ width_px: u32,
+ height_px: u32,
+ },
+
+ /// The direct command was written and flushed; terminal response timing starts now.
+ GraphicsTransmissionStarted { transfer_id: u64, image_id: u32 },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -696,6 +717,8 @@ pub enum ServerMessage {
MouseCapture {
/// True when Herdr mouse UI is enabled or the focused pane app requests mouse reporting.
enabled: bool,
+ /// True only while the focused pane requests DEC SGR pixel mode 1016.
+ sgr_pixels: bool,
},
/// Whether the focused terminal requests Kitty report-all keyboard input.
@@ -717,6 +740,19 @@ pub enum ServerMessage {
/// Number of BEL characters parsed from one PTY read.
count: u16,
},
+
+ /// One validated Herdr-owned Kitty regular-file RGBA transmission.
+ GraphicsFile {
+ path: String,
+ expected_len: u64,
+ image_id: u32,
+ transfer_id: u64,
+ leading: Vec<u8>,
+ control: String,
+ },
+
+ /// Suppress a direct command that expired before terminal delivery.
+ GraphicsTransmissionRetired { transfer_id: u64, image_id: u32 },
}
// ---------------------------------------------------------------------------
@@ -1148,7 +1184,7 @@ mod tests {
],
};
let encoded = bincode::serde::encode_to_vec(&msg, bincode::config::standard()).unwrap();
- // Freeze the protocol 19 input envelope before it is published.
+ // Freeze the protocol 20 input envelope before it is published.
assert_eq!(
encoded,
vec![
@@ -1534,7 +1570,10 @@ mod tests {
#[test]
fn server_mouse_capture_roundtrip() {
- let msg = ServerMessage::MouseCapture { enabled: true };
+ let msg = ServerMessage::MouseCapture {
+ enabled: true,
+ sgr_pixels: true,
+ };
let encoded = bincode::serde::encode_to_vec(&msg, bincode::config::standard()).unwrap();
let (decoded, _): (ServerMessage, _) =
bincode::serde::decode_from_slice(&encoded, bincode::config::standard()).unwrap();
@@ -1551,6 +1590,44 @@ mod tests {
}
#[test]
+ fn direct_graphics_messages_roundtrip() {
+ let client = ClientMessage::GraphicsTransmissionResult {
+ transfer_id: 7,
+ image_id: 42,
+ success: false,
+ };
+ let encoded = bincode::serde::encode_to_vec(&client, bincode::config::standard()).unwrap();
+ let (decoded, _): (ClientMessage, _) =
+ bincode::serde::decode_from_slice(&encoded, bincode::config::standard()).unwrap();
+ assert_eq!(client, decoded);
+
+ let server = ServerMessage::GraphicsFile {
+ path: "/run/user/1000/herdr/source/frame".into(),
+ expected_len: 4,
+ image_id: 42,
+ transfer_id: 7,
+ leading: b"\x1b[2;3H".to_vec(),
+ control: "a=T,f=32,i=42,q=0".into(),
+ };
+ let encoded = bincode::serde::encode_to_vec(&server, bincode::config::standard()).unwrap();
+ let (decoded, _): (ServerMessage, _) =
+ bincode::serde::decode_from_slice(&encoded, bincode::config::standard()).unwrap();
+ assert_eq!(server, decoded);
+
+ let pixels = ClientMessage::InputPixels {
+ data: b"\x1b[<35;321;241M".to_vec(),
+ cols: 80,
+ rows: 24,
+ width_px: 800,
+ height_px: 480,
+ };
+ let encoded = bincode::serde::encode_to_vec(&pixels, bincode::config::standard()).unwrap();
+ let (decoded, _): (ClientMessage, _) =
+ bincode::serde::decode_from_slice(&encoded, bincode::config::standard()).unwrap();
+ assert_eq!(pixels, decoded);
+ }
+
+ #[test]
fn server_prefix_input_source_roundtrip() {
for active in [true, false] {
let msg = ServerMessage::PrefixInputSource { active };
diff --git a/src/server/alt_screen_read.rs b/src/server/alt_screen_read.rs
index 240e96de..425d4a41 100644
--- a/src/server/alt_screen_read.rs
+++ b/src/server/alt_screen_read.rs
@@ -373,7 +373,11 @@ fn send_wheel(
let column = snapshot.cols.saturating_sub(1) / 2;
let row = u16::try_from(snapshot.rows.len().saturating_sub(1) / 2).unwrap_or(0);
let event = runtime
- .encode_mouse_wheel(kind, column, row, KeyModifiers::empty())
+ .encode_mouse_wheel(
+ kind,
+ crate::input::mouse::Position::Cell { column, row },
+ KeyModifiers::empty(),
+ )
.ok_or(())?;
let mut bytes = Vec::with_capacity(event.len().saturating_mul(events));
for _ in 0..events {
diff --git a/src/server/client_transport.rs b/src/server/client_transport.rs
index 2675b4c3..b5023b97 100644
--- a/src/server/client_transport.rs
+++ b/src/server/client_transport.rs
@@ -41,6 +41,8 @@ const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(4);
const MAX_INPUT_PAYLOAD: usize = 1024 * 1024; // 1 MB
/// Maximum structured input events accepted in one client message.
const MAX_INPUT_EVENT_BATCH: usize = 4096;
+/// Maximum encoded mouse report accepted with pixel geometry.
+const MAX_PIXEL_MOUSE_PAYLOAD: usize = 128;
/// Channels owned by the server side of a client writer thread.
#[derive(Clone, Debug)]
@@ -51,115 +53,99 @@ pub(crate) struct ClientWriter {
pub(crate) render: ClientRenderWriter,
}
-#[cfg(test)]
impl ClientWriter {
+ pub(crate) fn replace_with_cleanup(&self, data: Vec<u8>) {
+ self.render.queue.replace_with_cleanup(data);
+ }
+
+ #[cfg(test)]
+ pub(crate) fn test_fill_render(&self, data: Vec<u8>) {
+ self.render.try_send(data).unwrap();
+ }
+
+ #[cfg(test)]
+ pub(crate) fn test_close(&self) {
+ self.render.queue.close_writer();
+ }
+
+ #[cfg(test)]
pub(crate) fn test_channel(
control: std::sync::mpsc::Sender<Vec<u8>>,
render: std::sync::mpsc::SyncSender<Vec<u8>>,
) -> Self {
- Self {
- control: ClientControlWriter {
- target: ClientControlTarget::Channel(control),
- },
- render: ClientRenderWriter {
- target: ClientRenderTarget::Channel(render),
- },
- }
+ let queue = ClientWriterQueue::new();
+ let drain = queue.clone();
+ let control_writer = ClientControlWriter::queue(queue.clone());
+ let mut render_writer = ClientRenderWriter::queue(queue);
+ render_writer.test_render = Some(render.clone());
+ let writer = Self {
+ control: control_writer,
+ render: render_writer,
+ };
+ std::thread::spawn(move || {
+ while let Some(item) = drain.recv() {
+ let sent = match item {
+ ClientWriteItem::Control(data) => control.send(data).is_ok(),
+ ClientWriteItem::Render(data) => render.send(data).is_ok(),
+ };
+ if !sent {
+ break;
+ }
+ }
+ drain.close_writer();
+ });
+ writer
}
}
#[derive(Debug)]
pub(crate) struct ClientControlWriter {
- target: ClientControlTarget,
-}
-
-#[derive(Debug)]
-enum ClientControlTarget {
- Queue(Arc<ClientWriterQueue>),
+ queue: Arc<ClientWriterQueue>,
#[cfg(test)]
- Channel(std::sync::mpsc::Sender<Vec<u8>>),
+ test_render: Option<std::sync::mpsc::SyncSender<Vec<u8>>>,
}
#[derive(Debug)]
pub(crate) struct ClientRenderWriter {
- target: ClientRenderTarget,
-}
-
-#[derive(Debug)]
-enum ClientRenderTarget {
- Queue(Arc<ClientWriterQueue>),
+ queue: Arc<ClientWriterQueue>,
#[cfg(test)]
- Channel(std::sync::mpsc::SyncSender<Vec<u8>>),
+ test_render: Option<std::sync::mpsc::SyncSender<Vec<u8>>>,
}
-impl Clone for ClientControlWriter {
- fn clone(&self) -> Self {
- match &self.target {
- ClientControlTarget::Queue(queue) => {
- queue.add_sender();
+macro_rules! writer_handle {
+ ($type:ty) => {
+ impl Clone for $type {
+ fn clone(&self) -> Self {
+ self.queue.add_sender();
Self {
- target: ClientControlTarget::Queue(queue.clone()),
+ queue: self.queue.clone(),
+ #[cfg(test)]
+ test_render: self.test_render.clone(),
}
}
- #[cfg(test)]
- ClientControlTarget::Channel(sender) => Self {
- target: ClientControlTarget::Channel(sender.clone()),
- },
}
- }
-}
-
-impl Drop for ClientControlWriter {
- fn drop(&mut self) {
- match &self.target {
- ClientControlTarget::Queue(queue) => queue.remove_sender(),
- #[cfg(test)]
- ClientControlTarget::Channel(_) => {}
+ impl Drop for $type {
+ fn drop(&mut self) {
+ self.queue.remove_sender();
+ }
}
- }
+ };
}
+writer_handle!(ClientControlWriter);
+writer_handle!(ClientRenderWriter);
impl ClientControlWriter {
fn queue(queue: Arc<ClientWriterQueue>) -> Self {
queue.add_sender();
Self {
- target: ClientControlTarget::Queue(queue),
- }
- }
-
- pub(crate) fn send(&self, data: Vec<u8>) -> Result<(), SendError<Vec<u8>>> {
- match &self.target {
- ClientControlTarget::Queue(queue) => queue.send_control(data),
- #[cfg(test)]
- ClientControlTarget::Channel(sender) => sender.send(data),
- }
- }
-}
-
-impl Clone for ClientRenderWriter {
- fn clone(&self) -> Self {
- match &self.target {
- ClientRenderTarget::Queue(queue) => {
- queue.add_sender();
- Self {
- target: ClientRenderTarget::Queue(queue.clone()),
- }
- }
+ queue,
#[cfg(test)]
- ClientRenderTarget::Channel(sender) => Self {
- target: ClientRenderTarget::Channel(sender.clone()),
- },
+ test_render: None,
}
}
-}
-impl Drop for ClientRenderWriter {
- fn drop(&mut self) {
- match &self.target {
- ClientRenderTarget::Queue(queue) => queue.remove_sender(),
- #[cfg(test)]
- ClientRenderTarget::Channel(_) => {}
- }
+ pub(crate) fn send(&self, data: Vec<u8>) -> Result<(), SendError<Vec<u8>>> {
+ self.queue.send_control(data)
}
}
@@ -167,16 +153,22 @@ impl ClientRenderWriter {
fn queue(queue: Arc<ClientWriterQueue>) -> Self {
queue.add_sender();
Self {
- target: ClientRenderTarget::Queue(queue),
+ queue,
+ #[cfg(test)]
+ test_render: None,
}
}
pub(crate) fn try_send(&self, data: Vec<u8>) -> Result<(), TrySendError<Vec<u8>>> {
- match &self.target {
- ClientRenderTarget::Queue(queue) => queue.try_send_render(data),
- #[cfg(test)]
- ClientRenderTarget::Channel(sender) => sender.try_send(data),
+ #[cfg(test)]
+ if let Some(sender) = &self.test_render {
+ return sender.try_send(data);
}
+ self.queue.try_send_render(data)
+ }
+
+ pub(crate) fn send_ordered(&self, data: Vec<u8>) -> Result<(), TrySendError<Vec<u8>>> {
+ self.queue.send_ordered(data)
}
}
@@ -189,6 +181,7 @@ struct ClientWriterQueue {
#[derive(Debug, Default)]
struct ClientWriterQueueState {
control: VecDeque<Vec<u8>>,
+ ordered: VecDeque<Vec<u8>>,
render: Option<Vec<u8>>,
senders: usize,
writer_alive: bool,
@@ -245,16 +238,45 @@ impl ClientWriterQueue {
Ok(())
}
+ fn send_ordered(&self, data: Vec<u8>) -> Result<(), TrySendError<Vec<u8>>> {
+ let mut state = self.lock_state();
+ if !state.writer_alive {
+ return Err(TrySendError::Disconnected(data));
+ }
+ if !state.ordered.is_empty() {
+ return Err(TrySendError::Full(data));
+ }
+ if let Some(older) = state.render.take() {
+ state.ordered.push_back(older);
+ }
+ state.ordered.push_back(data);
+ self.ready.notify_one();
+ Ok(())
+ }
+
+ fn replace_with_cleanup(&self, data: Vec<u8>) {
+ let mut state = self.lock_state();
+ state.render = None;
+ state.ordered.clear();
+ if state.writer_alive {
+ state.control.push_back(data);
+ self.ready.notify_one();
+ }
+ }
+
fn recv(&self) -> Option<ClientWriteItem> {
let mut state = self.lock_state();
loop {
if let Some(data) = state.control.pop_front() {
return Some(ClientWriteItem::Control(data));
}
- if let Some(data) = state.render.take() {
+ if let Some(data) = state.ordered.pop_front() {
self.ready.notify_one();
return Some(ClientWriteItem::Render(data));
}
+ if let Some(data) = state.render.take() {
+ return Some(ClientWriteItem::Render(data));
+ }
if state.senders == 0 {
return None;
}
@@ -268,6 +290,8 @@ impl ClientWriterQueue {
fn close_writer(&self) {
let mut state = self.lock_state();
state.writer_alive = false;
+ state.render = None;
+ state.ordered.clear();
self.ready.notify_all();
}
@@ -291,10 +315,29 @@ pub(crate) enum ServerEvent {
render_encoding: RenderEncoding,
keybindings: Option<Box<crate::config::LiveKeybindConfig>>,
direct_attach_requested: bool,
+ direct_graphics: bool,
writer: ClientWriter,
},
/// A client sent an input message.
ClientInput { client_id: u64, data: Vec<u8> },
+ /// A client reported the one armed Kitty regular-file response.
+ GraphicsTransmissionResult {
+ client_id: u64,
+ transfer_id: u64,
+ image_id: u32,
+ success: bool,
+ },
+ GraphicsTransmissionStarted {
+ client_id: u64,
+ transfer_id: u64,
+ image_id: u32,
+ },
+ /// One confirmed SGR pixel report with client read-time geometry.
+ ClientInputPixels {
+ client_id: u64,
+ data: Vec<u8>,
+ geometry: crate::input::mouse::HostGeometry,
+ },
/// A client sent structured input events.
ClientInputEvents {
client_id: u64,
@@ -515,6 +558,7 @@ pub(crate) fn handle_client_handshake(
render_encoding,
keybindings,
direct_attach_requested,
+ direct_graphics,
) = match hello {
ClientMessage::Hello {
version,
@@ -564,6 +608,7 @@ pub(crate) fn handle_client_handshake(
requested_encoding,
keybindings,
launch_mode == ClientLaunchMode::TerminalAttach,
+ launch_mode == ClientLaunchMode::AppDirectGraphics,
)
}
_ => {
@@ -618,6 +663,7 @@ pub(crate) fn handle_client_handshake(
render_encoding,
keybindings,
direct_attach_requested,
+ direct_graphics,
writer,
});
@@ -633,19 +679,17 @@ fn client_writer_loop(
server_event_tx: mpsc::Sender<ServerEvent>,
) {
while let Some(item) = writer_queue.recv() {
- match item {
- ClientWriteItem::Control(data) => {
- if !write_framed_bytes(&mut stream, &data) {
- break;
- }
- }
+ let written = match item {
+ ClientWriteItem::Control(data) => write_framed_bytes(&mut stream, &data),
ClientWriteItem::Render(data) => {
let _ =
server_event_tx.blocking_send(ServerEvent::ClientWriterDrained { client_id });
- if !write_framed_bytes(&mut stream, &data) {
- break;
- }
+ write_framed_bytes(&mut stream, &data)
}
+ };
+ if !written {
+ let _ = server_event_tx.blocking_send(ServerEvent::ClientDisconnected { client_id });
+ break;
}
}
writer_queue.close_writer();
@@ -728,6 +772,47 @@ fn client_read_loop(
ServerEvent::ClientInput { client_id, data }
}
}
+ ClientMessage::InputPixels {
+ data,
+ cols,
+ rows,
+ width_px,
+ height_px,
+ } => {
+ let Some(geometry) =
+ crate::input::mouse::HostGeometry::new(cols, rows, width_px, height_px)
+ else {
+ warn!(
+ client_id,
+ cols,
+ rows,
+ width_px,
+ height_px,
+ "invalid pixel mouse geometry from client, closing"
+ );
+ let _ = server_event_tx
+ .blocking_send(ServerEvent::ClientDisconnected { client_id });
+ break;
+ };
+ if data.len() > MAX_PIXEL_MOUSE_PAYLOAD
+ || crate::input::mouse::parse_report(&data).is_none()
+ {
+ warn!(
+ client_id,
+ size = data.len(),
+ max = MAX_PIXEL_MOUSE_PAYLOAD,
+ "invalid pixel mouse report from client, closing"
+ );
+ let _ = server_event_tx
+ .blocking_send(ServerEvent::ClientDisconnected { client_id });
+ break;
+ }
+ ServerEvent::ClientInputPixels {
+ client_id,
+ data,
+ geometry,
+ }
+ }
ClientMessage::InputEvents { events } => match input_event_limit(&events) {
InputEventLimit::WithinLimits => {
ServerEvent::ClientInputEvents { client_id, events }
@@ -777,6 +862,24 @@ fn client_read_loop(
takeover,
}
}
+ ClientMessage::GraphicsTransmissionResult {
+ transfer_id,
+ image_id,
+ success,
+ } => ServerEvent::GraphicsTransmissionResult {
+ client_id,
+ transfer_id,
+ image_id,
+ success,
+ },
+ ClientMessage::GraphicsTransmissionStarted {
+ transfer_id,
+ image_id,
+ } => ServerEvent::GraphicsTransmissionStarted {
+ client_id,
+ transfer_id,
+ image_id,
+ },
ClientMessage::ClipboardImage { extension, data } => {
if data.len() > MAX_CLIPBOARD_IMAGE_PAYLOAD {
warn!(
@@ -948,6 +1051,30 @@ mod tests {
}
#[test]
+ fn ordered_direct_follows_older_render_and_stays_bounded() {
+ let (writer, queue) = test_queue_writer();
+ writer.render.try_send(b"old".to_vec()).unwrap();
+ writer.render.send_ordered(b"direct".to_vec()).unwrap();
+ assert!(matches!(
+ writer.render.send_ordered(b"second".to_vec()),
+ Err(TrySendError::Full(_))
+ ));
+ writer.render.try_send(b"new".to_vec()).unwrap();
+
+ for expected in [b"old".as_slice(), b"direct", b"new"] {
+ assert_eq!(
+ queue.recv(),
+ Some(ClientWriteItem::Render(expected.to_vec()))
+ );
+ }
+ queue.close_writer();
+ assert!(matches!(
+ writer.render.send_ordered(b"closed".to_vec()),
+ Err(TrySendError::Disconnected(_))
+ ));
+ }
+
+ #[test]
fn client_writer_prioritizes_control_and_reports_render_drain() {
let (mut client_stream, server_stream, _path) = local_stream_pair("client-writer-priority");
let (writer, queue) = test_queue_writer();
@@ -1203,6 +1330,7 @@ new_tab = "ctrl+notakey"
render_encoding,
keybindings,
direct_attach_requested,
+ direct_graphics,
writer,
} => {
assert_eq!(client_id, 42);
@@ -1211,6 +1339,7 @@ new_tab = "ctrl+notakey"
assert_eq!(render_encoding, RenderEncoding::TerminalAnsi);
assert!(keybindings.is_none());
assert!(!direct_attach_requested);
+ assert!(!direct_graphics);
drop(writer);
}
other => panic!("expected ClientConnected, got {other:?}"),
@@ -1395,6 +1524,76 @@ new_tab = "ctrl+notakey"
}
#[test]
+ fn client_read_loop_disconnects_invalid_pixel_mouse_geometry() {
+ let (mut client_stream, server_stream, _path) =
+ local_stream_pair("client-read-invalid-pixel-geometry");
+ let (server_event_tx, mut server_event_rx) = mpsc::channel(4);
+ let should_quit = Arc::new(AtomicBool::new(false));
+ let read_quit = should_quit.clone();
+ let handle = std::thread::spawn(move || {
+ client_read_loop(server_stream, 7, &server_event_tx, &read_quit)
+ });
+
+ protocol::write_message(
+ &mut client_stream,
+ &ClientMessage::InputPixels {
+ data: b"\x1b[<35;1;1M".to_vec(),
+ cols: 0,
+ rows: 24,
+ width_px: 800,
+ height_px: 480,
+ },
+ )
+ .expect("write invalid pixel geometry");
+
+ assert!(matches!(
+ recv_server_event(&mut server_event_rx, "invalid pixel geometry disconnect"),
+ ServerEvent::ClientDisconnected { client_id: 7 }
+ ));
+ drop(client_stream);
+ should_quit.store(true, Ordering::Release);
+ handle
+ .join()
+ .expect("read thread join")
+ .expect("read thread result");
+ }
+
+ #[test]
+ fn client_read_loop_disconnects_invalid_pixel_mouse_report() {
+ let (mut client_stream, server_stream, _path) =
+ local_stream_pair("client-read-invalid-pixel-report");
+ let (server_event_tx, mut server_event_rx) = mpsc::channel(4);
+ let should_quit = Arc::new(AtomicBool::new(false));
+ let read_quit = should_quit.clone();
+ let handle = std::thread::spawn(move || {
+ client_read_loop(server_stream, 7, &server_event_tx, &read_quit)
+ });
+
+ protocol::write_message(
+ &mut client_stream,
+ &ClientMessage::InputPixels {
+ data: vec![b'x'; MAX_PIXEL_MOUSE_PAYLOAD + 1],
+ cols: 80,
+ rows: 24,
+ width_px: 800,
+ height_px: 480,
+ },
+ )
+ .expect("write invalid pixel report");
+
+ assert!(matches!(
+ recv_server_event(&mut server_event_rx, "invalid pixel report disconnect"),
+ ServerEvent::ClientDisconnected { client_id: 7 }
+ ));
+ drop(client_stream);
+ should_quit.store(true, Ordering::Release);
+ handle
+ .join()
+ .expect("read thread join")
+ .expect("read thread result");
+ }
+
+ #[test]
fn client_read_loop_disconnects_marker_wrapped_invalid_utf8() {
let (mut client_stream, server_stream, _path) =
local_stream_pair("client-read-invalid-utf8-paste");
diff --git a/src/server/clients.rs b/src/server/clients.rs
index bb0f31c6..0d3d9ae0 100644
--- a/src/server/clients.rs
+++ b/src/server/clients.rs
@@ -24,7 +24,6 @@ pub(crate) type RenderTarget = (
pub(crate) enum DeferredRender {
#[default]
None,
- Graphics,
Full,
}
@@ -56,14 +55,18 @@ pub(crate) struct ClientConnection {
pub(crate) render_state: ClientRenderState,
/// Client-local host Kitty graphics cache.
pub(crate) graphics_cache: crate::kitty_graphics::HostGraphicsCache,
+ /// Passive eligibility for audited local Kitty regular-file graphics.
+ pub(crate) direct_graphics: bool,
+ /// Whether this frontend preserves exact SGR pixel reports.
+ pub(crate) pixel_mouse: bool,
/// Whether the next graphics frame must clear and rebuild host-side Kitty state.
pub(crate) graphics_surface_reset_pending: bool,
/// Whether an ordinary render was skipped because the render channel was full.
pub(crate) render_pending: bool,
- /// Whether a pane-graphics-only render was skipped because the channel was full.
- pane_graphics_render_pending: bool,
/// Last host mouse capture mode sent to this client.
pub(crate) host_mouse_capture_active: Option<bool>,
+ /// Last SGR pixel provenance mode sent to this client.
+ pub(crate) host_sgr_pixels_active: Option<bool>,
/// Last Kitty report-all mode sent to this client's host terminal.
pub(crate) host_keyboard_report_all_active: Option<bool>,
/// Temporary files staged from this client's local clipboard image pastes.
@@ -125,10 +128,12 @@ impl ClientConnection {
last_activity,
render_state: ClientRenderState::new(render_encoding),
graphics_cache: crate::kitty_graphics::HostGraphicsCache::default(),
+ direct_graphics: false,
+ pixel_mouse: false,
graphics_surface_reset_pending: false,
render_pending: false,
- pane_graphics_render_pending: false,
host_mouse_capture_active: None,
+ host_sgr_pixels_active: None,
host_keyboard_report_all_active: None,
staged_clipboard_files: Vec::new(),
writer,
@@ -137,14 +142,11 @@ impl ClientConnection {
pub(crate) fn request_repaint(&mut self) {
self.render_state.request_repaint();
- self.pane_graphics_render_pending = false;
}
pub(crate) fn deferred_render(&self) -> DeferredRender {
if self.render_pending {
DeferredRender::Full
- } else if self.pane_graphics_render_pending {
- DeferredRender::Graphics
} else {
DeferredRender::None
}
@@ -152,18 +154,10 @@ impl ClientConnection {
pub(crate) fn clear_deferred_render(&mut self) {
self.render_pending = false;
- self.pane_graphics_render_pending = false;
}
pub(crate) fn defer_full_render(&mut self) {
self.render_pending = true;
- self.pane_graphics_render_pending = false;
- }
-
- pub(crate) fn defer_pane_graphics_render(&mut self) {
- if !self.render_pending {
- self.pane_graphics_render_pending = true;
- }
}
pub(crate) fn take_deferred_render(&mut self) -> DeferredRender {
diff --git a/src/server/headless.rs b/src/server/headless.rs
index 5a90177b..61c825b8 100644
--- a/src/server/headless.rs
+++ b/src/server/headless.rs
@@ -44,7 +44,6 @@ use crate::ipc::{
};
use crate::protocol::{
self, AttachScrollDirection, AttachScrollSource, FrameData, ServerMessage, MAX_FRAME_SIZE,
- MAX_GRAPHICS_FRAME_SIZE,
};
#[cfg(unix)]
use crate::server::client_accept::{
@@ -66,8 +65,7 @@ use crate::server::terminal_attach::paste_payload_for_runtime;
mod pane_graphics;
-#[cfg(test)]
-use pane_graphics::frame_pane_graphics_for_client;
+use crate::protocol::MAX_GRAPHICS_FRAME_SIZE;
use pane_graphics::RetainedGraphicsOutcome;
#[cfg(test)]
@@ -367,12 +365,13 @@ fn apply_terminal_attach_scroll(
match runtime.wheel_routing() {
Some(crate::pane::WheelRouting::MouseReport) => {
runtime.scroll_reset();
- let column = column.unwrap_or(0);
- let row = row.unwrap_or(0);
+ let position = crate::input::mouse::Position::Cell {
+ column: column.unwrap_or(0),
+ row: row.unwrap_or(0),
+ };
let Some(bytes) = runtime.encode_mouse_wheel(
wheel_kind,
- column,
- row,
+ position,
KeyModifiers::from_bits_truncate(modifiers),
) else {
return Err(format!(
@@ -671,7 +670,14 @@ impl HeadlessServer {
crate::render_prof::event("full_render_cause.default_workspace");
}
- self.cancel_inactive_pane_graphics_streams();
+ if self.app.pane_graphics.retain_live_panes(&self.app.state) {
+ needs_render = true;
+ needs_graphics_render = true;
+ }
+ if self.expire_direct_graphics(now) {
+ needs_render = true;
+ needs_graphics_render = true;
+ }
self.drain_client_config_reload_request();
self.stream_host_mouse_capture_mode();
@@ -1002,6 +1008,7 @@ impl HeadlessServer {
},
respond_to,
response_write_complete: None,
+ stream_active: None,
},
true,
);
@@ -1083,6 +1090,15 @@ impl HeadlessServer {
}
fn sync_foreground_client_state(&mut self) {
+ self.app.direct_graphics_available = self.direct_graphics_available();
+ self.app.pixel_mouse_available = self.foreground_client_id.is_some_and(|id| {
+ self.clients
+ .get(&id)
+ .is_some_and(|client| client.pixel_mouse)
+ });
+ if !self.app.direct_graphics_available {
+ self.retire_all_direct_graphics();
+ }
let Some(client_id) = self.foreground_client_id else {
self.effective_size = (MIN_COLS, MIN_ROWS);
self.app.state.outer_terminal_focus = None;
@@ -1485,11 +1501,21 @@ impl HeadlessServer {
.count()
}
+ fn direct_graphics_available(&self) -> bool {
+ self.app_client_count() == 1
+ && self.foreground_client_id.is_some_and(|id| {
+ self.clients.get(&id).is_some_and(|client| {
+ client.is_full_app_client() && client.writer.is_some() && client.direct_graphics
+ })
+ })
+ }
+
fn has_app_client(&self) -> bool {
self.app_client_count() > 0
}
fn remove_client(&mut self, client_id: u64) -> bool {
+ self.retire_direct_graphics_for_client(client_id);
let was_foreground = self.foreground_client_id == Some(client_id);
self.app.clear_input_source(client_id);
self.send_client_graphics_cleanup(client_id);
@@ -1551,7 +1577,7 @@ impl HeadlessServer {
let Ok(serialized) = Self::frame_server_message(&ServerMessage::Graphics { bytes }) else {
return;
};
- let _ = writer.control.send(serialized);
+ writer.replace_with_cleanup(serialized);
}
fn send_all_clients_graphics_cleanup(&mut self) {
@@ -2781,6 +2807,7 @@ impl HeadlessServer {
writer,
render_encoding,
direct_attach_requested,
+ direct_graphics,
} => {
if self.handoff_in_progress {
if let Ok(message) =
@@ -2806,24 +2833,24 @@ impl HeadlessServer {
"client connected"
);
let last_activity = self.allocate_activity_stamp();
- self.clients.insert(
- client_id,
- ClientConnection::new_with_mode(
- ClientConnectionMode::App,
- keybindings,
- (cols, rows),
- crate::kitty_graphics::HostCellSize {
- width_px: cell_width_px,
- height_px: cell_height_px,
- },
- crate::terminal_theme::TerminalTheme::default(),
- None,
- last_activity,
- render_encoding,
- direct_attach_requested,
- Some(writer),
- ),
+ let mut connection = ClientConnection::new_with_mode(
+ ClientConnectionMode::App,
+ keybindings,
+ (cols, rows),
+ crate::kitty_graphics::HostCellSize {
+ width_px: cell_width_px,
+ height_px: cell_height_px,
+ },
+ crate::terminal_theme::TerminalTheme::default(),
+ None,
+ last_activity,
+ render_encoding,
+ direct_attach_requested,
+ Some(writer),
);
+ connection.direct_graphics = direct_graphics;
+ connection.pixel_mouse = direct_graphics;
+ self.clients.insert(client_id, connection);
if !direct_attach_requested {
self.foreground_client_id = Some(client_id);
}
@@ -2835,6 +2862,17 @@ impl HeadlessServer {
self.nudge_handoff_panes_on_first_client_attach();
true
}
+ ServerEvent::GraphicsTransmissionResult {
+ client_id,
+ transfer_id,
+ image_id,
+ success,
+ } => self.complete_direct_graphics(client_id, transfer_id, image_id, success),
+ ServerEvent::GraphicsTransmissionStarted {
+ client_id,
+ transfer_id,
+ image_id,
+ } => self.start_direct_graphics_response(client_id, transfer_id, image_id),
ServerEvent::ClientAttachTerminal {
client_id,
terminal_id,
@@ -2859,6 +2897,35 @@ impl HeadlessServer {
} => self.handle_terminal_attach_scroll(
client_id, source, direction, lines, column, row, modifiers,
),
+ ServerEvent::ClientInputPixels {
+ client_id,
+ data,
+ geometry,
+ } => {
+ let coordinates_valid = crate::input::mouse::parse_report(&data)
+ .and_then(|(x, y)| geometry.cell(x, y))
+ .is_some();
+ let valid = coordinates_valid
+ && self.clients.get(&client_id).is_some_and(|client| {
+ let cell = client.cell_size;
+ client.is_full_app_client()
+ && client.host_sgr_pixels_active == Some(true)
+ && client.terminal_size == (geometry.cols, geometry.rows)
+ && cell.is_known()
+ && cell.width_px == geometry.width_px / u32::from(geometry.cols)
+ && cell.height_px == geometry.height_px / u32::from(geometry.rows)
+ });
+ if !valid || self.handoff_in_progress || !self.focused_pane_graphics_demand() {
+ return false;
+ }
+ let foreground_changed = self.promote_client_to_foreground(client_id);
+ if foreground_changed {
+ self.resize_shared_runtime_to_effective_size_before_input();
+ }
+ self.app
+ .route_client_pixel_mouse(client_id, &data, geometry)
+ || foreground_changed
+ }
ServerEvent::ClientInput { client_id, data } => {
if self.handoff_in_progress {
debug!(
@@ -2987,18 +3054,21 @@ impl HeadlessServer {
}) = self.clients.get_mut(&client_id)
{
*terminal_size = (cols, rows);
- *cell_size = crate::kitty_graphics::HostCellSize {
+ let observed = crate::kitty_graphics::HostCellSize {
width_px: cell_width_px,
height_px: cell_height_px,
};
+ if observed.is_known() {
+ *cell_size = observed;
+ }
render_state.request_repaint();
- Some(terminal_id.clone())
+ Some((terminal_id.clone(), *cell_size))
} else {
None
};
- if let Some(terminal_id) = direct_terminal_id {
+ if let Some((terminal_id, cell_size)) = direct_terminal_id {
if let Some(runtime) = self.runtime_for_terminal_id_string(&terminal_id) {
- runtime.resize(rows, cols, cell_width_px, cell_height_px);
+ runtime.resize(rows, cols, cell_size.width_px, cell_size.height_px);
}
return true;
}
@@ -3011,19 +3081,25 @@ impl HeadlessServer {
}) = self.clients.get_mut(&client_id)
{
*terminal_size = (cols, rows);
- *cell_size = crate::kitty_graphics::HostCellSize {
+ let observed = crate::kitty_graphics::HostCellSize {
width_px: cell_width_px,
height_px: cell_height_px,
};
+ if observed.is_known() {
+ *cell_size = observed;
+ }
render_state.request_repaint();
return true;
}
if let Some(client) = self.clients.get_mut(&client_id) {
client.terminal_size = (cols, rows);
- client.cell_size = crate::kitty_graphics::HostCellSize {
+ let observed = crate::kitty_graphics::HostCellSize {
width_px: cell_width_px,
height_px: cell_height_px,
};
+ if observed.is_known() {
+ client.cell_size = observed;
+ }
}
self.promote_client_to_foreground(client_id);
self.resize_shared_runtime_to_effective_size();
@@ -3055,19 +3131,10 @@ impl HeadlessServer {
}
fn handle_server_event_with_render_impact(&mut self, ev: ServerEvent) -> RenderImpact {
- let deferred_render = match &ev {
- ServerEvent::ClientWriterDrained { client_id } => self
- .clients
- .get(client_id)
- .map_or(DeferredRender::None, ClientConnection::deferred_render),
- _ => DeferredRender::None,
- };
- if !self.handle_server_event(ev) {
- return RenderImpact::None;
- }
- match deferred_render {
- DeferredRender::Graphics => RenderImpact::Graphics,
- DeferredRender::None | DeferredRender::Full => RenderImpact::Full,
+ if self.handle_server_event(ev) {
+ RenderImpact::Full
+ } else {
+ RenderImpact::None
}
}
@@ -3299,6 +3366,7 @@ impl HeadlessServer {
if matches!(
&msg.request.method,
api::schema::Method::PaneGraphicsStreamSet(_)
+ | api::schema::Method::PaneGraphicsStreamDirect(_)
) {
return self.handle_pane_graphics_stream_frame(msg);
}
@@ -3341,6 +3409,11 @@ impl HeadlessServer {
};
let metadata_expired = self.app.expire_due_metadata(Instant::now());
+ let stream_open = match &msg.request.method {
+ api::schema::Method::PaneGraphicsStreamOpen(params) => Some(params.clone()),
+ _ => None,
+ };
+ let stream_active = msg.stream_active.clone();
if let api::schema::Method::ServerLiveHandoff(params) = &msg.request.method {
let handoff_result = self.perform_live_handoff(params.clone());
@@ -3398,7 +3471,7 @@ impl HeadlessServer {
| api::schema::Method::PaneGraphicsStreamOpen(_)
| api::schema::Method::PaneGraphicsStreamClose(_)
)
- .then_some(self.app.state.pane_graphics_revision);
+ .then_some(self.app.pane_graphics.revision());
let mut changed = metadata_expired
| (pane_graphics_revision_before.is_none() && api::request_changes_ui(&msg.request));
let skip_default_workspace = skip_default_workspace_for_request
@@ -3501,6 +3574,10 @@ impl HeadlessServer {
}
}
}
+ if let (Some(params), Some(active)) = (stream_open.as_ref(), stream_active) {
+ self.app
+ .attach_pane_graphics_stream_active(params, active, &response);
+ }
if let Some(spec) = alt_screen_read_spec {
if let Ok(success) = serde_json::from_str::<api::schema::SuccessResponse>(&response) {
if let api::schema::ResponseResult::PaneRead { read } = success.result {
@@ -3523,7 +3600,7 @@ impl HeadlessServer {
let _ = msg.respond_to.send(response);
if let Some(revision_before) = pane_graphics_revision_before {
- changed |= revision_before != self.app.state.pane_graphics_revision;
+ changed |= revision_before != self.app.pane_graphics.revision();
}
// Forward new toast state only when a client-local delivery mode is selected.
@@ -3671,32 +3748,73 @@ impl HeadlessServer {
changed
}
+ fn focused_pane_graphics_demand(&self) -> bool {
+ self.app
+ .state
+ .active
+ .and_then(|ws_idx| self.app.state.workspaces.get(ws_idx))
+ .and_then(crate::workspace::Workspace::focused_pane_id)
+ .is_some_and(|pane_id| self.app.pane_graphics.active_for_pane(pane_id))
+ }
+
fn stream_host_mouse_capture_mode(&mut self) {
let enabled = self
.app
.state
.should_capture_host_mouse_from(&self.app.terminal_runtimes);
- let serialized = match Self::frame_server_message(&ServerMessage::MouseCapture { enabled })
- {
- Ok(framed) => framed,
- Err(err) => {
- warn!(err = %err, "failed to serialize mouse capture mode for clients");
- return;
- }
- };
-
+ let pixel_mouse_requested = self
+ .clients
+ .values()
+ .any(|client| client.is_full_app_client() && client.pixel_mouse);
+ let sgr_pixels = pixel_mouse_requested
+ && self.focused_pane_graphics_demand()
+ && self
+ .app
+ .state
+ .active
+ .and_then(|ws_idx| {
+ self.app
+ .state
+ .workspaces
+ .get(ws_idx)
+ .and_then(crate::workspace::Workspace::focused_pane_id)
+ .and_then(|pane_id| {
+ self.app.state.runtime_for_pane_in_workspace(
+ &self.app.terminal_runtimes,
+ ws_idx,
+ pane_id,
+ )
+ })
+ })
+ .and_then(crate::terminal::TerminalRuntime::input_state)
+ .is_some_and(|state| {
+ state.mouse_protocol_encoding == crate::input::MouseProtocolEncoding::SgrPixels
+ });
let mut broken_clients: Vec<u64> = Vec::new();
for (&client_id, client) in &mut self.clients {
if !client.is_full_app_client() {
continue;
}
- if client.host_mouse_capture_active == Some(enabled) {
+ let client_sgr_pixels = sgr_pixels && client.pixel_mouse;
+ if client.host_mouse_capture_active == Some(enabled)
+ && client.host_sgr_pixels_active == Some(client_sgr_pixels)
+ {
continue;
}
let Some(writer) = &client.writer else {
continue;
};
- if writer.control.send(serialized.clone()).is_err() {
+ let serialized = match Self::frame_server_message(&ServerMessage::MouseCapture {
+ enabled,
+ sgr_pixels: client_sgr_pixels,
+ }) {
+ Ok(framed) => framed,
+ Err(err) => {
+ warn!(err = %err, "failed to serialize mouse capture mode for client");
+ continue;
+ }
+ };
+ if writer.control.send(serialized).is_err() {
debug!(
client_id,
"client writer channel closed during mouse capture update"
@@ -3705,6 +3823,7 @@ impl HeadlessServer {
continue;
}
client.host_mouse_capture_active = Some(enabled);
+ client.host_sgr_pixels_active = Some(client_sgr_pixels);
}
for client_id in broken_clients {
@@ -3877,6 +3996,7 @@ impl HeadlessServer {
&& cell_size.is_known()
&& crate::kitty_graphics::has_visible_pane_graphics(
&self.app.state,
+ &self.app.pane_graphics,
&self.app.terminal_runtimes,
self.app.state.view.tab_surface(),
*cell_size,
@@ -4166,31 +4286,48 @@ impl HeadlessServer {
continue;
};
let mut next_graphics_cache = client.graphics_cache.clone();
- let graphics_surface_reset_pending = client.graphics_surface_reset_pending;
- if is_app_client && self.app.state.kitty_graphics_enabled && cell_size.is_known() {
- if graphics_surface_reset_pending {
- frame.graphics = next_graphics_cache.clear_bytes();
+ let mut reset_graphics = Vec::new();
+ let mut encoded = if is_app_client
+ && self.app.state.kitty_graphics_enabled
+ && cell_size.is_known()
+ {
+ if client.graphics_surface_reset_pending {
+ if self.app.pane_graphics.slots.is_empty() {
+ reset_graphics = next_graphics_cache.clear_bytes();
+ } else {
+ next_graphics_cache = crate::kitty_graphics::HostGraphicsCache::default();
+ }
}
let graphics_started = crate::render_prof::timer();
- frame
- .graphics
- .extend(crate::kitty_graphics::encode_local_pane_graphics(
- &self.app.state,
- &self.app.terminal_runtimes,
- self.app.state.view.tab_surface(),
- cell_size,
- &mut next_graphics_cache,
- ));
+ let encoded = crate::kitty_graphics::encode_local_pane_graphics(
+ &self.app.state,
+ &self.app.pane_graphics,
+ &self.app.terminal_runtimes,
+ self.app.state.view.tab_surface(),
+ cell_size,
+ Some(crate::kitty_graphics::HEADLESS_GRAPHICS_TRANSACTION_BUDGET),
+ &mut next_graphics_cache,
+ );
crate::render_prof::duration_since("full_render.graphics_encode", graphics_started);
+ encoded
+ } else if self.app.pane_graphics.slots.is_empty() {
+ crate::kitty_graphics::EncodedGraphics {
+ bytes: next_graphics_cache.clear_bytes(),
+ incomplete: false,
+ }
} else {
- frame.graphics = next_graphics_cache.clear_bytes();
+ next_graphics_cache.clear_next()
+ };
+ if !reset_graphics.is_empty() {
+ reset_graphics.extend(encoded.bytes);
+ encoded.bytes = reset_graphics;
}
+ frame.graphics = encoded.bytes;
let Some(writer) = client.writer.as_ref().cloned() else {
crate::render_prof::event("full_render.writer_missing");
continue;
};
-
let mut commit_graphics_cache = true;
if frame.graphics.len() > MAX_GRAPHICS_FRAME_SIZE {
warn!(
@@ -4201,32 +4338,30 @@ impl HeadlessServer {
);
frame.graphics.clear();
commit_graphics_cache = false;
+ encoded.incomplete = false;
}
-
- let max_frame_size = if frame.graphics.is_empty() {
- MAX_FRAME_SIZE
- } else {
- MAX_GRAPHICS_FRAME_SIZE
- };
let has_graphics = !frame.graphics.is_empty();
- let prepare_started = crate::render_prof::timer();
let Some(mut prepared) = client.render_state.prepare_frame(frame) else {
- client.clear_deferred_render();
+ if commit_graphics_cache {
+ client.graphics_cache = next_graphics_cache;
+ client.graphics_surface_reset_pending = false;
+ }
+ if encoded.incomplete {
+ client.defer_full_render();
+ deferred_frame = true;
+ } else {
+ client.clear_deferred_render();
+ }
crate::render_prof::event("full_render.skip_identical");
- crate::render_prof::duration_since("full_render.prepare_frame", prepare_started);
continue;
};
- crate::render_prof::duration_since("full_render.prepare_frame", prepare_started);
-
- let serialize_started = crate::render_prof::timer();
- let serialized = match Self::frame_server_message_with_max(
- prepared.message(),
- max_frame_size,
- ) {
- Ok(framed) => {
- crate::render_prof::duration_since("full_render.serialize", serialize_started);
- framed
- }
+ let max = if has_graphics {
+ MAX_GRAPHICS_FRAME_SIZE
+ } else {
+ crate::protocol::MAX_FRAME_SIZE
+ };
+ let serialized = match Self::frame_server_message_with_max(prepared.message(), max) {
+ Ok(frame) => frame,
Err(protocol::FramingError::Oversized { claimed, max }) if has_graphics => {
warn!(
client_id,
@@ -4234,10 +4369,6 @@ impl HeadlessServer {
);
let Some(mut text_only_frame) = prepared.into_frame() else {
crate::render_prof::event("full_render.serialize_error");
- crate::render_prof::duration_since(
- "full_render.serialize",
- serialize_started,
- );
continue;
};
text_only_frame.graphics.clear();
@@ -4246,10 +4377,6 @@ impl HeadlessServer {
else {
client.clear_deferred_render();
crate::render_prof::event("full_render.skip_identical_text_only");
- crate::render_prof::duration_since(
- "full_render.serialize",
- serialize_started,
- );
continue;
};
let framed = match Self::frame_server_message(text_only_prepared.message()) {
@@ -4258,16 +4385,12 @@ impl HeadlessServer {
warn!(client_id, err = %err, "failed to serialize text-only frame for client");
broken_clients.push(client_id);
crate::render_prof::event("full_render.serialize_error");
- crate::render_prof::duration_since(
- "full_render.serialize",
- serialize_started,
- );
continue;
}
};
prepared = text_only_prepared;
commit_graphics_cache = false;
- crate::render_prof::duration_since("full_render.serialize", serialize_started);
+ encoded.incomplete = false;
framed
}
Err(protocol::FramingError::Oversized { claimed, max }) => {
@@ -4276,45 +4399,36 @@ impl HeadlessServer {
claimed, max, "skipping oversized frame for client"
);
crate::render_prof::event("full_render.serialize_oversized");
- crate::render_prof::duration_since("full_render.serialize", serialize_started);
continue;
}
Err(err) => {
- warn!(client_id, err = %err, "failed to serialize frame for client");
+ warn!(client_id, err = %err, "failed to serialize frame");
broken_clients.push(client_id);
crate::render_prof::event("full_render.serialize_error");
- crate::render_prof::duration_since("full_render.serialize", serialize_started);
continue;
}
};
- crate::render_prof::counter("full_render.bytes", serialized.len() as u64);
-
- let send_started = crate::render_prof::timer();
match writer.render.try_send(serialized) {
Ok(()) => {
- client.clear_deferred_render();
if commit_graphics_cache {
client.graphics_cache = next_graphics_cache;
client.graphics_surface_reset_pending = false;
}
client.render_state.commit_sent_frame(prepared);
+ if encoded.incomplete {
+ client.defer_full_render();
+ deferred_frame = true;
+ } else {
+ client.clear_deferred_render();
+ }
crate::render_prof::event("full_render.sent");
- crate::render_prof::duration_since("full_render.try_send", send_started);
}
Err(std::sync::mpsc::TrySendError::Full(_)) => {
client.defer_full_render();
deferred_frame = true;
- crate::render_prof::event("full_render.queue_full");
- crate::render_prof::duration_since("full_render.try_send", send_started);
- debug!(client_id, "render queue full, deferring latest frame");
- continue;
}
Err(std::sync::mpsc::TrySendError::Disconnected(_)) => {
- debug!(client_id, "client writer channel closed, marking as broken");
broken_clients.push(client_id);
- crate::render_prof::event("full_render.writer_disconnected");
- crate::render_prof::duration_since("full_render.try_send", send_started);
- continue;
}
}
}
@@ -5044,7 +5158,9 @@ mod tests {
}
fn read_server_frame(bytes: Vec<u8>) -> FrameData {
- match read_server_message(bytes) {
+ match protocol::read_message(&mut std::io::Cursor::new(bytes), MAX_GRAPHICS_FRAME_SIZE)
+ .expect("decode server frame")
+ {
ServerMessage::Frame(frame) => frame,
other => panic!("expected frame, got {other:?}"),
}
@@ -5139,6 +5255,7 @@ mod tests {
},
respond_to,
response_write_complete: None,
+ stream_active: None,
});
let response: api::schema::SuccessResponse =
serde_json::from_str(&response_rx.recv().unwrap()).unwrap();
@@ -5179,6 +5296,7 @@ mod tests {
},
respond_to,
response_write_complete: None,
+ stream_active: None,
})
);
let response = response_rx
@@ -5440,6 +5558,42 @@ mod tests {
}
#[test]
+ fn direct_graphics_requires_one_negotiated_app_client() {
+ let mut server = test_headless_server();
+ let (writer_a, _control_a, _render_a) = test_client_writer();
+ assert!(server.handle_server_event(ServerEvent::ClientConnected {
+ client_id: 1,
+ cols: 80,
+ rows: 24,
+ cell_width_px: 10,
+ cell_height_px: 20,
+ render_encoding: RenderEncoding::SemanticFrame,
+ keybindings: None,
+ direct_attach_requested: false,
+ direct_graphics: true,
+ writer: writer_a,
+ }));
+ assert!(server.clients[&1].direct_graphics);
+ assert!(server.clients[&1].pixel_mouse);
+ assert!(server.direct_graphics_available());
+
+ let (writer_b, _control_b, _render_b) = test_client_writer();
+ assert!(server.handle_server_event(ServerEvent::ClientConnected {
+ client_id: 2,
+ cols: 80,
+ rows: 24,
+ cell_width_px: 10,
+ cell_height_px: 20,
+ render_encoding: RenderEncoding::SemanticFrame,
+ keybindings: None,
+ direct_attach_requested: false,
+ direct_graphics: false,
+ writer: writer_b,
+ }));
+ assert!(!server.direct_graphics_available());
+ }
+
+ #[test]
fn foreground_client_applies_client_keybindings() {
let mut server = test_headless_server();
let local_config: crate::config::Config = toml::from_str(
@@ -5463,6 +5617,7 @@ new_tab = "prefix+t"
render_encoding: RenderEncoding::SemanticFrame,
keybindings: Some(Box::new(local_keybindings)),
direct_attach_requested: false,
+ direct_graphics: false,
writer: writer_a,
}));
assert_eq!(
@@ -5487,6 +5642,7 @@ new_tab = "prefix+t"
render_encoding: RenderEncoding::SemanticFrame,
keybindings: None,
direct_attach_requested: false,
+ direct_graphics: false,
writer: writer_b,
}));
assert_eq!(
@@ -5527,6 +5683,7 @@ new_tab = "prefix+t"
render_encoding: RenderEncoding::SemanticFrame,
keybindings: Some(Box::new(local_keybindings)),
direct_attach_requested: false,
+ direct_graphics: false,
writer: writer_a,
}));
assert_eq!(server.app.state.config_diagnostic, without_keybindings);
@@ -5540,6 +5697,7 @@ new_tab = "prefix+t"
render_encoding: RenderEncoding::SemanticFrame,
keybindings: None,
direct_attach_requested: false,
+ direct_graphics: false,
writer: writer_b,
}));
assert_eq!(
@@ -5583,6 +5741,7 @@ next_tab = ""
render_encoding: RenderEncoding::SemanticFrame,
keybindings: Some(Box::new(local_keybindings)),
direct_attach_requested: false,
+ direct_graphics: false,
writer,
}));
server.app.state.mode = crate::app::Mode::Settings;
@@ -5658,6 +5817,7 @@ next_tab = ""
render_encoding: RenderEncoding::SemanticFrame,
keybindings: Some(Box::new(local_config.live_keybinds().unwrap())),
direct_attach_requested: false,
+ direct_graphics: false,
writer: writer_a,
}));
server.app.state.mode = crate::app::Mode::Settings;
@@ -5678,6 +5838,7 @@ next_tab = ""
render_encoding: RenderEncoding::SemanticFrame,
keybindings: None,
direct_attach_requested: false,
+ direct_graphics: false,
writer: writer_b,
}));
assert_eq!(
@@ -5712,6 +5873,7 @@ next_tab = ""
render_encoding: RenderEncoding::TerminalAnsi,
keybindings: None,
direct_attach_requested: true,
+ direct_graphics: false,
writer,
}));
assert!(server.clients.contains_key(&7));
@@ -5777,6 +5939,7 @@ next_tab = ""
render_encoding: RenderEncoding::TerminalAnsi,
keybindings: None,
direct_attach_requested: true,
+ direct_graphics: false,
writer,
}));
control_rx
@@ -6189,6 +6352,7 @@ next_tab = ""
render_encoding,
keybindings: None,
direct_attach_requested: false,
+ direct_graphics: false,
writer,
}));
@@ -6223,6 +6387,7 @@ next_tab = ""
render_encoding: RenderEncoding::TerminalAnsi,
keybindings: None,
direct_attach_requested: true,
+ direct_graphics: false,
writer,
}));
@@ -6256,6 +6421,7 @@ next_tab = ""
render_encoding: RenderEncoding::SemanticFrame,
keybindings: None,
direct_attach_requested: false,
+ direct_graphics: false,
writer,
}));
assert!(server.has_app_client());
@@ -6356,6 +6522,7 @@ next_tab = ""
render_encoding: RenderEncoding::TerminalAnsi,
keybindings: None,
direct_attach_requested: true,
+ direct_graphics: false,
writer,
}));
assert!(
@@ -7778,6 +7945,51 @@ next_tab = ""
assert_eq!(server.foreground_client_id, Some(3));
}
+ #[test]
+ fn terminal_attach_resize_preserves_known_cell_size_when_pixels_are_omitted() {
+ with_terminal_session_test_server(|server, _terminal_id, terminal_id, _pane_id| {
+ let mut client = ClientConnection::new(
+ (80, 24),
+ crate::kitty_graphics::HostCellSize {
+ width_px: 10,
+ height_px: 20,
+ },
+ crate::terminal_theme::TerminalTheme::default(),
+ Some(true),
+ 1,
+ RenderEncoding::SemanticFrame,
+ None,
+ );
+ client.mode = ClientConnectionMode::TerminalAttach {
+ terminal_id: terminal_id.clone(),
+ };
+ server.clients.insert(1, client);
+
+ assert!(server.handle_server_event(ServerEvent::ClientResize {
+ client_id: 1,
+ cols: 100,
+ rows: 30,
+ cell_width_px: 0,
+ cell_height_px: 0,
+ }));
+
+ assert_eq!(
+ server
+ .runtime_for_terminal_id_string(&terminal_id)
+ .unwrap()
+ .pixel_size(),
+ Some((1_000, 600))
+ );
+ assert_eq!(
+ server.clients[&1].cell_size,
+ crate::kitty_graphics::HostCellSize {
+ width_px: 10,
+ height_px: 20,
+ }
+ );
+ });
+ }
+
#[tokio::test]
async fn passive_mouse_motion_forwards_without_requesting_render() {
let mut server = test_headless_server();
@@ -8335,6 +8547,7 @@ next_tab = ""
render_encoding: RenderEncoding::TerminalAnsi,
keybindings: None,
direct_attach_requested: true,
+ direct_graphics: false,
writer,
}));
assert!(
@@ -8531,16 +8744,11 @@ next_tab = ""
let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig)
.expect("serialize dummy message");
- server
- .clients
- .get(&1)
- .unwrap()
+ server.clients[&1]
.writer
.as_ref()
.unwrap()
- .render
- .try_send(queued)
- .expect("pre-fill render queue");
+ .test_fill_render(queued);
assert!(server.handle_server_event(ServerEvent::ClientInput {
client_id: 1,
@@ -8671,10 +8879,7 @@ next_tab = ""
let (client_tx, _client_control_rx, client_rx) = test_client_writer();
let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig)
.expect("serialize dummy message");
- client_tx
- .render
- .try_send(queued)
- .expect("pre-fill render queue");
+ client_tx.test_fill_render(queued);
server.clients.insert(
1,
@@ -8715,10 +8920,7 @@ next_tab = ""
let (client_tx, _client_control_rx, client_rx) = test_client_writer();
let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig)
.expect("serialize dummy message");
- client_tx
- .render
- .try_send(queued)
- .expect("pre-fill render queue");
+ client_tx.test_fill_render(queued);
server.clients.insert(
1,
@@ -9003,7 +9205,7 @@ next_tab = ""
.recv_timeout(Duration::from_millis(100))
.expect("mouse capture message")
),
- ServerMessage::MouseCapture { enabled: true }
+ ServerMessage::MouseCapture { enabled: true, .. }
));
}
@@ -9525,16 +9727,11 @@ next_tab = ""
let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa");
let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig)
.expect("serialize dummy message");
- server
- .clients
- .get(&1)
- .unwrap()
+ server.clients[&1]
.writer
.as_ref()
.unwrap()
- .render
- .try_send(queued)
- .expect("pre-fill render queue");
+ .test_fill_render(queued);
server.app.full_redraw_pending = true;
server.render_and_stream();
@@ -9742,6 +9939,7 @@ next_tab = ""
let mut server = test_headless_server();
let (foreground_tx, foreground_control_rx, _foreground_rx) = test_client_writer();
drop(foreground_control_rx);
+ foreground_tx.test_close();
server.clients.insert(
1,
@@ -10125,6 +10323,7 @@ next_tab = ""
},
respond_to,
response_write_complete: None,
+ stream_active: None,
});
assert!(changed);
@@ -10211,6 +10410,7 @@ next_tab = ""
},
respond_to,
response_write_complete: None,
+ stream_active: None,
});
assert!(changed);
@@ -10263,6 +10463,7 @@ next_tab = ""
},
respond_to,
response_write_complete: None,
+ stream_active: None,
});
assert!(changed);
@@ -10295,6 +10496,7 @@ next_tab = ""
},
respond_to,
response_write_complete: None,
+ stream_active: None,
});
assert!(changed);
@@ -10332,6 +10534,7 @@ next_tab = ""
},
respond_to,
response_write_complete: None,
+ stream_active: None,
})
);
@@ -10388,6 +10591,7 @@ next_tab = ""
},
respond_to,
response_write_complete: None,
+ stream_active: None,
})
);
@@ -10688,6 +10892,7 @@ next_tab = ""
},
respond_to,
response_write_complete: None,
+ stream_active: None,
});
assert!(changed);
diff --git a/src/server/headless/pane_graphics.rs b/src/server/headless/pane_graphics.rs
index ed28eab0..d027f79b 100644
--- a/src/server/headless/pane_graphics.rs
+++ b/src/server/headless/pane_graphics.rs
@@ -1,5 +1,3 @@
-use tracing::warn;
-
use super::{HeadlessServer, RenderImpact};
use crate::api;
use crate::protocol::{ServerMessage, MAX_GRAPHICS_FRAME_SIZE};
@@ -12,31 +10,16 @@ pub(super) enum RetainedGraphicsOutcome {
Fallback,
}
-pub(super) fn frame_pane_graphics_for_client(bytes: Vec<u8>) -> Vec<u8> {
+pub(super) fn frame_pane_graphics(bytes: Vec<u8>) -> Vec<u8> {
if bytes.is_empty() {
return bytes;
}
- let mut framed = Vec::with_capacity(bytes.len() + 4);
- framed.extend_from_slice(b"\x1b7");
- framed.extend_from_slice(&bytes);
- framed.extend_from_slice(b"\x1b8");
- framed
+ [b"\x1b7".as_slice(), &bytes, b"\x1b8"].concat()
}
impl HeadlessServer {
pub(super) fn pane_graphics_runtime_active(&self) -> bool {
- !self.app.state.pane_graphics_layers.is_empty()
- || !self.app.state.pane_graphics_streams.is_empty()
- }
-
- pub(super) fn cancel_inactive_pane_graphics_streams(&self) {
- api::cancel_inactive_pane_graphics_streams(|owner| {
- self.app
- .state
- .pane_graphics_streams
- .values()
- .any(|active_owner| active_owner == owner)
- });
+ !self.app.pane_graphics.slots.is_empty()
}
pub(super) fn handle_pane_graphics_stream_frame(
@@ -49,12 +32,164 @@ impl HeadlessServer {
}
let internal_changed = self.drain_all_internal_events_with_forwarding();
+ let direct = match &msg.request.method {
+ api::schema::Method::PaneGraphicsStreamDirect(params) => Some(params.clone()),
+ _ => None,
+ };
+ let direct_key = direct.as_ref().and_then(|params| {
+ self.app.parse_pane_id(&params.pane_id).map(|(_, pane_id)| {
+ (
+ pane_id,
+ params
+ .layer_id
+ .clone()
+ .unwrap_or_else(|| api::schema::PANE_GRAPHICS_PRIMARY_LAYER_ID.into()),
+ )
+ })
+ });
+ let direct_client = self
+ .direct_graphics_available()
+ .then_some(self.foreground_client_id)
+ .flatten();
+ let gate_busy = self
+ .app
+ .pane_graphics
+ .slots
+ .values()
+ .any(|slot| slot.direct_gate.is_some());
+ self.app.direct_graphics_available = direct_client.is_some() && !gate_busy;
let response = self
.app
.handle_api_request_after_internal_events_drained(msg.request);
+ self.app.direct_graphics_available = self.direct_graphics_available();
let succeeded = serde_json::from_str::<api::schema::SuccessResponse>(&response).is_ok();
- let _ = msg.respond_to.send(response);
+ if succeeded {
+ if let (Some(params), Some(client_id)) = (direct, direct_client) {
+ let direct_frame = direct_key.as_ref().and_then(|key| {
+ self.app.pane_graphics.slots.get(key).and_then(|slot| {
+ let active_owner = slot.stream_owner.as_deref() == Some(&params.owner)
+ && slot.stream_is_active();
+ active_owner.then_some(())?;
+ slot.layer
+ .as_ref()
+ .and_then(crate::app::pane_graphics::Layer::direct_lease)
+ .map(|lease| {
+ (
+ slot.host_image_id,
+ lease.path().to_string_lossy().into_owned(),
+ lease.len() as u64,
+ lease.fingerprint(),
+ )
+ })
+ })
+ });
+ if let (Some(key), Some((image_id, path, expected_len, transfer_id))) =
+ (direct_key.clone(), direct_frame)
+ {
+ let mut next_cache = self
+ .clients
+ .get(&client_id)
+ .map(|client| client.graphics_cache.clone())
+ .unwrap_or_default();
+ let command = (!internal_changed)
+ .then(|| {
+ crate::kitty_graphics::prepare_direct_file(
+ &self.app.state,
+ &self.app.pane_graphics,
+ self.app.state.view.tab_surface(),
+ self.clients
+ .get(&client_id)
+ .map(|client| client.cell_size)
+ .unwrap_or_default(),
+ &mut next_cache,
+ &key,
+ )
+ })
+ .flatten();
+ let Some(command) = command else {
+ if self.install_inline_fallback(&key) {
+ if msg.respond_to.send(response).is_err() {
+ self.retire_direct_gate(&key);
+ }
+ return if internal_changed {
+ RenderImpact::Full
+ } else {
+ RenderImpact::Graphics
+ };
+ }
+ self.retire_direct_gate(&key);
+ return if internal_changed {
+ RenderImpact::Full
+ } else {
+ RenderImpact::Graphics
+ };
+ };
+ let message = ServerMessage::GraphicsFile {
+ path,
+ expected_len,
+ image_id,
+ transfer_id,
+ leading: command.leading,
+ control: command.control,
+ };
+ let send =
+ Self::frame_server_message_with_max(&message, MAX_GRAPHICS_FRAME_SIZE)
+ .map_err(|_| std::sync::mpsc::TrySendError::Full(Vec::new()))
+ .and_then(|framed| {
+ let Some(writer) = self
+ .clients
+ .get(&client_id)
+ .and_then(|client| client.writer.as_ref())
+ else {
+ return Err(std::sync::mpsc::TrySendError::Disconnected(
+ framed,
+ ));
+ };
+ writer.render.send_ordered(framed)
+ });
+ match send {
+ Ok(()) => {
+ if let Some(slot) = self.app.pane_graphics.slots.get_mut(&key) {
+ slot.direct_gate = Some(crate::app::pane_graphics::DirectGate {
+ transfer_id,
+ client_id,
+ deadline: std::time::Instant::now()
+ + crate::app::pane_graphics::DIRECT_DELIVERY_TIMEOUT,
+ written: false,
+ success_response: response,
+ respond_to: msg.respond_to,
+ });
+ }
+ if let Some(client) = self.clients.get_mut(&client_id) {
+ client.graphics_cache = next_cache;
+ }
+ return if internal_changed {
+ RenderImpact::Full
+ } else {
+ RenderImpact::None
+ };
+ }
+ Err(error) => {
+ self.handle_unwritten_direct_failure(
+ &key,
+ response,
+ msg.respond_to,
+ error,
+ );
+ return RenderImpact::Graphics;
+ }
+ }
+ }
+ }
+ }
+
+ let response_failed = msg.respond_to.send(response).is_err();
+ if succeeded && response_failed {
+ if let Some(key) = direct_key {
+ self.retire_direct_gate(&key);
+ }
+ }
if internal_changed {
RenderImpact::Full
} else if succeeded {
@@ -64,6 +199,276 @@ impl HeadlessServer {
}
}
+ fn install_inline_fallback(&mut self, key: &crate::app::pane_graphics::Key) -> bool {
+ let len = self
+ .app
+ .pane_graphics
+ .slots
+ .get(key)
+ .and_then(|slot| slot.layer.as_ref())
+ .and_then(crate::app::pane_graphics::Layer::direct_lease)
+ .map(crate::pane_graphics_files::Lease::len);
+ let Some(len) = len else {
+ return false;
+ };
+ if !self.app.pane_graphics.can_store_inline(key, len) {
+ return false;
+ }
+ let data = self
+ .app
+ .pane_graphics
+ .slots
+ .get(key)
+ .and_then(|slot| slot.layer.as_ref())
+ .and_then(crate::app::pane_graphics::Layer::direct_lease)
+ .and_then(|lease| lease.copy_rgba().ok());
+ let Some(data) = data else {
+ return false;
+ };
+ let slot = self
+ .app
+ .pane_graphics
+ .slots
+ .get_mut(key)
+ .expect("matched slot");
+ if !slot.stream_is_active() {
+ return false;
+ }
+ let layer = slot.layer.take().expect("direct layer");
+ slot.layer = Some(crate::app::pane_graphics::Layer::inline(
+ layer.format,
+ layer.image_width,
+ layer.image_height,
+ data,
+ layer.render,
+ layer.z_index,
+ ));
+ self.app.pane_graphics.mark_changed();
+ true
+ }
+
+ pub(super) fn handle_unwritten_direct_failure(
+ &mut self,
+ key: &crate::app::pane_graphics::Key,
+ response: String,
+ respond_to: std::sync::mpsc::Sender<String>,
+ error: std::sync::mpsc::TrySendError<Vec<u8>>,
+ ) -> bool {
+ if matches!(error, std::sync::mpsc::TrySendError::Full(_))
+ && self.install_inline_fallback(key)
+ && respond_to.send(response).is_ok()
+ {
+ return true;
+ }
+ self.retire_direct_gate(key);
+ false
+ }
+
+ pub(super) fn retire_direct_gate(&mut self, key: &crate::app::pane_graphics::Key) {
+ if self.app.pane_graphics.slots.remove(key).is_some() {
+ self.app.pane_graphics.mark_changed();
+ }
+ }
+
+ pub(super) fn start_direct_graphics_response(
+ &mut self,
+ client_id: u64,
+ transfer_id: u64,
+ image_id: u32,
+ ) -> bool {
+ if let Some(gate) = self.app.pane_graphics.slots.values_mut().find_map(|slot| {
+ (slot.host_image_id == image_id && slot.stream_is_active())
+ .then_some(slot.direct_gate.as_mut())
+ .flatten()
+ .filter(|gate| gate.client_id == client_id && gate.transfer_id == transfer_id)
+ }) {
+ gate.written = true;
+ gate.deadline =
+ std::time::Instant::now() + crate::app::pane_graphics::DIRECT_RESPONSE_TIMEOUT;
+ }
+ false
+ }
+
+ pub(super) fn complete_direct_graphics(
+ &mut self,
+ client_id: u64,
+ transfer_id: u64,
+ image_id: u32,
+ success: bool,
+ ) -> bool {
+ if self.shutting_down {
+ self.retire_all_direct_graphics();
+ return false;
+ }
+ let key = self.app.pane_graphics.slots.iter().find_map(|(key, slot)| {
+ slot.direct_gate
+ .as_ref()
+ .is_some_and(|gate| {
+ slot.stream_is_active()
+ && gate.client_id == client_id
+ && gate.transfer_id == transfer_id
+ && slot.host_image_id == image_id
+ && (!success || gate.written)
+ })
+ .then(|| key.clone())
+ });
+ let Some(key) = key else {
+ return false;
+ };
+ let pane_is_live = self
+ .app
+ .state
+ .workspaces
+ .iter()
+ .any(|workspace| workspace.pane_state(key.0).is_some());
+ if !pane_is_live {
+ self.retire_direct_gate(&key);
+ return false;
+ }
+ if success {
+ let (gate, host_image_id) = {
+ let slot = self
+ .app
+ .pane_graphics
+ .slots
+ .get_mut(&key)
+ .expect("matched slot");
+ if !slot.stream_is_active() {
+ return false;
+ }
+ if let Some(layer) = slot.layer.as_mut() {
+ layer.mark_resident(client_id);
+ }
+ (
+ slot.direct_gate.take().expect("matched gate"),
+ slot.host_image_id,
+ )
+ };
+ if let (Some(client), Some(layer)) = (
+ self.clients.get_mut(&client_id),
+ self.app
+ .pane_graphics
+ .slots
+ .get(&key)
+ .and_then(|slot| slot.layer.as_ref()),
+ ) {
+ client
+ .graphics_cache
+ .trust_pane_layer(&key, host_image_id, layer);
+ }
+ if gate.respond_to.send(gate.success_response).is_err() {
+ self.retire_direct_gate(&key);
+ return true;
+ }
+ return true;
+ }
+
+ if let Some(client) = self.clients.get_mut(&client_id) {
+ client.direct_graphics = false;
+ client.graphics_cache.forget_pane_layer(&key, image_id);
+ }
+ self.app.direct_graphics_available = false;
+ if !self.install_inline_fallback(&key) {
+ self.retire_direct_gate(&key);
+ self.retire_all_direct_graphics();
+ return true;
+ }
+ let gate = self
+ .app
+ .pane_graphics
+ .slots
+ .get_mut(&key)
+ .filter(|slot| slot.stream_is_active())
+ .and_then(|slot| slot.direct_gate.take());
+ let Some(gate) = gate else {
+ self.retire_direct_gate(&key);
+ return true;
+ };
+ if gate.respond_to.send(gate.success_response).is_err() {
+ self.retire_direct_gate(&key);
+ return true;
+ }
+ self.retire_all_direct_graphics();
+ true
+ }
+
+ pub(super) fn expire_direct_graphics(&mut self, now: std::time::Instant) -> bool {
+ let expired = self
+ .app
+ .pane_graphics
+ .slots
+ .iter()
+ .filter_map(|(key, slot)| {
+ slot.direct_gate
+ .as_ref()
+ .filter(|gate| gate.deadline <= now)
+ .map(|gate| (key.clone(), gate.client_id))
+ })
+ .collect::<Vec<_>>();
+ for (key, client_id) in &expired {
+ if let Some(slot) = self.app.pane_graphics.slots.get(key) {
+ if let Some(gate) = &slot.direct_gate {
+ self.send_to_client(
+ *client_id,
+ ServerMessage::GraphicsTransmissionRetired {
+ transfer_id: gate.transfer_id,
+ image_id: slot.host_image_id,
+ },
+ );
+ }
+ }
+ if let Some(client) = self.clients.get_mut(client_id) {
+ client.direct_graphics = false;
+ }
+ self.retire_direct_gate(key);
+ }
+ if !expired.is_empty() {
+ self.app.direct_graphics_available = false;
+ self.retire_all_direct_graphics();
+ }
+ !expired.is_empty()
+ }
+
+ pub(super) fn retire_all_direct_graphics(&mut self) {
+ let keys = self
+ .app
+ .pane_graphics
+ .slots
+ .iter()
+ .filter(|(_, slot)| {
+ slot.layer
+ .as_ref()
+ .is_some_and(crate::app::pane_graphics::Layer::terminal_only)
+ })
+ .map(|(key, _)| key.clone())
+ .collect::<Vec<_>>();
+ for key in keys {
+ self.retire_direct_gate(&key);
+ }
+ }
+
+ pub(super) fn retire_direct_graphics_for_client(&mut self, client_id: u64) {
+ let keys = self
+ .app
+ .pane_graphics
+ .slots
+ .iter()
+ .filter_map(|(key, slot)| {
+ let layer = slot.layer.as_ref()?;
+ let owned = layer.resident_client() == Some(client_id)
+ || (layer.direct_lease().is_some()
+ && slot
+ .direct_gate
+ .as_ref()
+ .is_some_and(|gate| gate.client_id == client_id));
+ (layer.terminal_only() && owned).then(|| key.clone())
+ })
+ .collect::<Vec<_>>();
+ for key in keys {
+ self.retire_direct_gate(&key);
+ }
+ }
+
pub(super) fn render_retained_graphics_update_and_stream(&mut self) -> RetainedGraphicsOutcome {
crate::render_prof::event("retained_graphics.attempt");
if self.app.full_redraw_pending {
@@ -117,74 +522,62 @@ impl HeadlessServer {
let mut next_graphics_cache = client.graphics_cache.clone();
let encode_started = crate::render_prof::timer();
- let bytes =
- frame_pane_graphics_for_client(crate::kitty_graphics::encode_local_pane_graphics(
- &self.app.state,
- &self.app.terminal_runtimes,
- self.app.state.view.tab_surface(),
- cell_size,
- &mut next_graphics_cache,
- ));
+ let encoded = crate::kitty_graphics::encode_local_pane_graphics(
+ &self.app.state,
+ &self.app.pane_graphics,
+ &self.app.terminal_runtimes,
+ self.app.state.view.tab_surface(),
+ cell_size,
+ Some(crate::kitty_graphics::HEADLESS_GRAPHICS_TRANSACTION_BUDGET),
+ &mut next_graphics_cache,
+ );
crate::render_prof::duration_since("retained_graphics.graphics_encode", encode_started);
- if bytes.len() > MAX_GRAPHICS_FRAME_SIZE {
- warn!(
- client_id,
- graphics_bytes = bytes.len(),
- max = MAX_GRAPHICS_FRAME_SIZE,
- "dropping oversized retained graphics payload"
- );
+ prepared.push((client_id, encoded, next_graphics_cache));
+ }
+
+ let mut broken_clients = Vec::new();
+ for (client_id, encoded, next_graphics_cache) in prepared {
+ let Some(client) = self.clients.get_mut(&client_id) else {
continue;
- }
- let serialized = if bytes.is_empty() {
+ };
+ let serialized = if encoded.bytes.is_empty() {
None
} else {
- let serialize_started = crate::render_prof::timer();
- let framed = match Self::frame_server_message_with_max(
- &ServerMessage::Graphics { bytes },
+ match Self::frame_server_message_with_max(
+ &ServerMessage::Graphics {
+ bytes: frame_pane_graphics(encoded.bytes),
+ },
MAX_GRAPHICS_FRAME_SIZE,
) {
- Ok(framed) => framed,
- Err(err) => {
- warn!(client_id, err = %err, "failed to serialize retained graphics");
+ Ok(serialized) => Some(serialized),
+ Err(_) => {
+ crate::render_prof::event("retained_graphics_fallback.oversized");
return RetainedGraphicsOutcome::Fallback;
}
- };
- crate::render_prof::duration_since(
- "retained_graphics.serialize",
- serialize_started,
- );
- Some(framed)
- };
- prepared.push((client_id, serialized, next_graphics_cache));
- }
-
- let mut broken_clients = Vec::new();
- for (client_id, serialized, next_graphics_cache) in prepared {
- let Some(client) = self.clients.get_mut(&client_id) else {
- continue;
+ }
};
- let Some(serialized) = serialized else {
- client.graphics_cache = next_graphics_cache;
- client.clear_deferred_render();
- continue;
+ let result = match (serialized, client.writer.as_ref()) {
+ (None, _) => Ok(()),
+ (Some(bytes), Some(writer)) => writer.render.try_send(bytes),
+ (Some(bytes), None) => Err(std::sync::mpsc::TrySendError::Disconnected(bytes)),
};
- let Some(writer) = client.writer.as_ref().cloned() else {
- broken_clients.push(client_id);
- continue;
- };
- match writer.render.try_send(serialized) {
+ match result {
Ok(()) => {
client.graphics_cache = next_graphics_cache;
- client.clear_deferred_render();
+ if encoded.incomplete {
+ client.defer_full_render();
+ deferred = true;
+ } else {
+ client.clear_deferred_render();
+ }
crate::render_prof::event("retained_graphics.sent");
}
Err(std::sync::mpsc::TrySendError::Full(_)) => {
- client.defer_pane_graphics_render();
+ client.defer_full_render();
deferred = true;
- crate::render_prof::event("retained_graphics.deferred");
}
Err(std::sync::mpsc::TrySendError::Disconnected(_)) => {
- broken_clients.push(client_id);
+ broken_clients.push(client_id)
}
}
}
diff --git a/src/server/headless/tests/pane_graphics.rs b/src/server/headless/tests/pane_graphics.rs
index 1eba9816..f9bd0a01 100644
--- a/src/server/headless/tests/pane_graphics.rs
+++ b/src/server/headless/tests/pane_graphics.rs
@@ -1,12 +1,52 @@
use super::*;
-#[test]
-fn frames_preserve_cursor_without_changing_ordinary_messages() {
- assert_eq!(frame_pane_graphics_for_client(Vec::new()), Vec::<u8>::new());
- assert_eq!(
- frame_pane_graphics_for_client(b"graphics".to_vec()),
- b"\x1b7graphics\x1b8"
- );
+fn receive_render(receiver: &std::sync::mpsc::Receiver<Vec<u8>>, timeout: Duration) -> Vec<u8> {
+ receiver.recv_timeout(timeout).unwrap()
+}
+
+#[tokio::test]
+async fn cold_redraw_advances_one_bounded_layer_after_each_send() {
+ let (mut server, client_rx, pane_id) = retained_test_server(b"cold redraw");
+ server.app.state.kitty_graphics_enabled = true;
+ server.clients.get_mut(&1).unwrap().cell_size = crate::kitty_graphics::HostCellSize {
+ width_px: 10,
+ height_px: 20,
+ };
+ const LAYERS: usize = 8;
+ for index in 0..LAYERS {
+ set_named_graphics_layer(
+ &mut server,
+ pane_id,
+ &format!("layer-{index:02}"),
+ vec![index as u8; 1024 * 1024],
+ index as i32,
+ );
+ }
+
+ fill_render_lane(&server);
+ server.render_and_stream();
+ assert!(server.clients[&1].graphics_cache.is_empty());
+ let _older = client_rx.recv_timeout(Duration::from_secs(1)).unwrap();
+
+ for expected in 1..=LAYERS {
+ server.render_and_stream();
+ let bytes = client_rx.recv_timeout(Duration::from_secs(5)).unwrap();
+ assert!(bytes.len() <= MAX_GRAPHICS_FRAME_SIZE + 4);
+ let frame = read_server_frame(bytes);
+ assert_eq!(
+ frame
+ .graphics
+ .windows(4)
+ .filter(|part| *part == b"a=t,")
+ .count(),
+ 1
+ );
+ assert_eq!(
+ server.clients[&1].graphics_cache.test_image_count(),
+ expected
+ );
+ }
+ assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None);
}
fn enable_graphics_and_render(
@@ -19,39 +59,72 @@ fn enable_graphics_and_render(
height_px: 20,
};
server.render_and_stream();
- read_server_frame(
- client_rx
- .recv_timeout(Duration::from_millis(100))
- .expect("initial frame"),
- )
+ read_server_frame(receive_render(client_rx, Duration::from_millis(100)))
+}
+
+fn graphics_key(pane_id: crate::layout::PaneId) -> crate::app::pane_graphics::Key {
+ (pane_id, api::schema::PANE_GRAPHICS_PRIMARY_LAYER_ID.into())
+}
+
+fn active_gate() -> std::sync::Arc<std::sync::atomic::AtomicBool> {
+ std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true))
}
fn set_graphics_layer(server: &mut HeadlessServer, pane_id: crate::layout::PaneId, data: Vec<u8>) {
- server.app.state.pane_graphics_layers.insert(
+ set_named_graphics_layer(
+ server,
pane_id,
- crate::app::state::PaneGraphicsLayer::new(
- api::schema::PaneGraphicsFormat::Png,
- 1,
- 1,
- data,
- api::schema::PaneGraphicsPlacementParams::default(),
- ),
+ api::schema::PANE_GRAPHICS_PRIMARY_LAYER_ID,
+ data,
+ 0,
);
}
+fn set_named_graphics_layer(
+ server: &mut HeadlessServer,
+ pane_id: crate::layout::PaneId,
+ layer_id: &str,
+ data: Vec<u8>,
+ z_index: i32,
+) {
+ let key = (pane_id, layer_id.into());
+ let host_image_id = server.app.pane_graphics.reserve_image_id(&key).unwrap();
+ let layer = crate::app::pane_graphics::Layer::inline(
+ api::schema::PaneGraphicsFormat::Png,
+ 1,
+ 1,
+ data,
+ Default::default(),
+ z_index,
+ );
+ server.app.pane_graphics.slots.insert(
+ key,
+ crate::app::pane_graphics::Slot::test(host_image_id, Some(layer)),
+ );
+}
+
+fn set_stream_owner(server: &mut HeadlessServer, pane_id: crate::layout::PaneId, owner: &str) {
+ let key = graphics_key(pane_id);
+ if let Some(slot) = server.app.pane_graphics.slots.get_mut(&key) {
+ slot.stream_owner = Some(owner.into());
+ slot.stream_active = Some(active_gate());
+ } else {
+ let host_image_id = server.app.pane_graphics.reserve_image_id(&key).unwrap();
+ let mut slot = crate::app::pane_graphics::Slot::test(host_image_id, None);
+ slot.stream_owner = Some(owner.into());
+ slot.stream_active = Some(active_gate());
+ server.app.pane_graphics.slots.insert(key, slot);
+ }
+}
+
fn fill_render_lane(server: &HeadlessServer) {
let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig)
.expect("dummy frame");
- server
- .clients
- .get(&1)
- .unwrap()
+ server.clients[&1]
.writer
.as_ref()
.unwrap()
- .render
- .try_send(queued)
- .expect("pre-fill render lane");
+ .test_fill_render(queued);
}
fn stream_set_message(
@@ -68,6 +141,8 @@ fn stream_set_message(
method: api::schema::Method::PaneGraphicsStreamSet(
api::schema::PaneGraphicsSetParams {
pane_id: pane_id.into(),
+ layer_id: None,
+ z_index: 0,
owner: owner.into(),
format: api::schema::PaneGraphicsFormat::Png,
image_width: 1,
@@ -80,12 +155,115 @@ fn stream_set_message(
},
respond_to,
response_write_complete: None,
+ stream_active: None,
},
response_rx,
)
}
#[tokio::test]
+async fn pixel_mouse_activation_requires_graphics_demand_not_direct_transport() {
+ let (mut server, _client_rx, pane_id) =
+ retained_test_server(b"\x1b[?1003h\x1b[?1006h\x1b[?1016h");
+ let (writer, control_rx, _render_rx) = test_client_writer();
+ let client = server.clients.get_mut(&1).unwrap();
+ client.writer = Some(writer);
+ client.direct_graphics = false;
+ client.pixel_mouse = true;
+ client.host_mouse_capture_active = None;
+ client.host_sgr_pixels_active = None;
+ server.app.direct_graphics_available = false;
+
+ server.stream_host_mouse_capture_mode();
+ assert!(matches!(
+ read_server_message(control_rx.recv_timeout(Duration::from_millis(100)).unwrap()),
+ ServerMessage::MouseCapture {
+ enabled: true,
+ sgr_pixels: false
+ }
+ ));
+
+ set_graphics_layer(&mut server, pane_id, vec![1, 2, 3]);
+ server.stream_host_mouse_capture_mode();
+ assert!(matches!(
+ read_server_message(control_rx.recv_timeout(Duration::from_millis(100)).unwrap()),
+ ServerMessage::MouseCapture {
+ enabled: true,
+ sgr_pixels: true
+ }
+ ));
+}
+
+#[tokio::test]
+async fn pixel_input_metadata_cannot_resize_authoritative_client_state() {
+ let (mut server, _client_rx, pane_id) =
+ retained_test_server(b"\x1b[?1003h\x1b[?1006h\x1b[?1016h");
+ set_graphics_layer(&mut server, pane_id, vec![1]);
+ let client = server.clients.get_mut(&1).unwrap();
+ client.pixel_mouse = true;
+ client.host_sgr_pixels_active = Some(true);
+ server.foreground_client_id = None;
+ assert!(!server.handle_server_event(ServerEvent::ClientInputPixels {
+ client_id: 1,
+ data: b"\x1b[<0;500;300M".to_vec(),
+ geometry: crate::input::mouse::HostGeometry::new(80, 24, 800, 480).unwrap(),
+ }));
+ server.clients.get_mut(&1).unwrap().cell_size = crate::kitty_graphics::HostCellSize {
+ width_px: 10,
+ height_px: 20,
+ };
+ for (geometry, data) in [
+ (
+ crate::input::mouse::HostGeometry::new(100, 30, 1_000, 600).unwrap(),
+ b"\x1b[<0;500;300M".as_slice(),
+ ),
+ (
+ crate::input::mouse::HostGeometry::new(80, 24, 960, 480).unwrap(),
+ b"\x1b[<0;500;300M",
+ ),
+ (
+ crate::input::mouse::HostGeometry::new(80, 24, 800, 480).unwrap(),
+ b"\x1b[<0;0;1M",
+ ),
+ ] {
+ assert!(!server.handle_server_event(ServerEvent::ClientInputPixels {
+ client_id: 1,
+ data: data.to_vec(),
+ geometry,
+ }));
+ }
+ assert_eq!(server.clients[&1].terminal_size, (80, 24));
+ assert_eq!(
+ (server.effective_size, server.foreground_client_id),
+ ((80, 24), None)
+ );
+}
+
+#[test]
+fn direct_eligibility_is_installed_with_the_client_connection() {
+ let mut server = test_headless_server();
+ let (writer, _control_rx, _render_rx) = test_client_writer();
+
+ assert!(server.handle_server_event(ServerEvent::ClientConnected {
+ client_id: 7,
+ cols: 80,
+ rows: 24,
+ cell_width_px: 10,
+ cell_height_px: 20,
+ render_encoding: RenderEncoding::SemanticFrame,
+ keybindings: None,
+ direct_attach_requested: false,
+ direct_graphics: true,
+ writer,
+ }));
+
+ let client = server.clients.get(&7).expect("connected client");
+ assert!(client.direct_graphics);
+ assert_eq!(server.foreground_client_id, Some(7));
+ assert!(server.app.direct_graphics_available);
+}
+
+#[tokio::test]
async fn focus_repaint_preserves_uploaded_graphics() {
let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa");
let (client_2_writer, _client_2_control_rx, client_2_rx) = test_client_writer();
@@ -109,11 +287,8 @@ async fn focus_repaint_preserves_uploaded_graphics() {
let initial_graphics = String::from_utf8_lossy(&initial.graphics);
assert!(initial_graphics.contains("a=t"));
assert!(initial_graphics.contains("a=p"));
- let client_2_initial = read_server_frame(
- client_2_rx
- .recv_timeout(Duration::from_millis(100))
- .expect("second client initial frame"),
- );
+ let client_2_initial =
+ read_server_frame(receive_render(&client_2_rx, Duration::from_millis(100)));
assert!(String::from_utf8_lossy(&client_2_initial.graphics).contains("a=t"));
assert!(server.handle_server_event(ServerEvent::ClientInput {
@@ -123,12 +298,68 @@ async fn focus_repaint_preserves_uploaded_graphics() {
assert_eq!(server.foreground_client_id, Some(2));
server.render_and_stream();
- let focused = read_server_frame(
- client_2_rx
- .recv_timeout(Duration::from_millis(100))
- .expect("focus redraw"),
+ let focused = read_server_frame(receive_render(&client_2_rx, Duration::from_millis(100)));
+ let focused_graphics = String::from_utf8_lossy(&focused.graphics);
+ assert!(focused_graphics.contains("a=p"));
+ assert!(!focused_graphics.contains("a=t"));
+}
+
+#[tokio::test]
+async fn resize_replays_placement_without_retransmitting_or_closing_stream() {
+ let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa");
+ set_graphics_layer(&mut server, pane_id, vec![1, 2, 3]);
+ set_stream_owner(&mut server, pane_id, "owner-resize");
+ let initial = enable_graphics_and_render(&mut server, &client_rx);
+ assert!(String::from_utf8_lossy(&initial.graphics).contains("a=t"));
+
+ for (cols, rows, cell_width_px, cell_height_px) in
+ [(100, 30, 10, 20), (100, 30, 12, 24), (100, 30, 12, 24)]
+ {
+ assert!(server.handle_server_event(ServerEvent::ClientResize {
+ client_id: 1,
+ cols,
+ rows,
+ cell_width_px,
+ cell_height_px,
+ }));
+ server.render_and_stream();
+ let frame = read_server_frame(receive_render(&client_rx, Duration::from_millis(100)));
+ let graphics = String::from_utf8_lossy(&frame.graphics);
+ assert!(!graphics.contains("a=t"));
+ assert!(graphics.contains("a=p"));
+ }
+ assert_eq!(
+ server
+ .app
+ .pane_graphics
+ .slots
+ .get(&graphics_key(pane_id))
+ .and_then(|slot| slot.stream_owner.as_deref()),
+ Some("owner-resize")
);
- assert!(focused.graphics.is_empty());
+}
+
+#[tokio::test]
+async fn graphics_pruning_preserves_live_panes_and_removes_closed_panes() {
+ let (mut server, _client_rx, pane_id) = retained_test_server(b"aaaa");
+ set_graphics_layer(&mut server, pane_id, vec![1, 2, 3]);
+
+ assert!(!server
+ .app
+ .pane_graphics
+ .retain_live_panes(&server.app.state));
+ assert!(server
+ .app
+ .pane_graphics
+ .slots
+ .contains_key(&graphics_key(pane_id)));
+
+ server.app.state.workspaces.clear();
+ assert!(server
+ .app
+ .pane_graphics
+ .retain_live_panes(&server.app.state));
+ assert!(server.app.pane_graphics.slots.is_empty());
}
#[tokio::test]
@@ -164,58 +395,25 @@ async fn retained_update_sends_only_graphics_message() {
}
#[tokio::test]
-async fn retained_update_defers_on_full_render_lane() {
+async fn retained_graphics_stays_ordered_after_an_older_render() {
let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa");
let _ = enable_graphics_and_render(&mut server, &client_rx);
fill_render_lane(&server);
set_graphics_layer(&mut server, pane_id, vec![4, 5, 6]);
-
+ let older = client_rx.recv_timeout(Duration::from_secs(1)).unwrap();
assert_eq!(
server.render_retained_graphics_update_and_stream(),
- RetainedGraphicsOutcome::Deferred
+ RetainedGraphicsOutcome::Sent
);
- let client = server.clients.get(&1).unwrap();
- assert_eq!(client.deferred_render(), DeferredRender::Graphics);
+ let graphics = client_rx.recv_timeout(Duration::from_secs(1)).unwrap();
assert!(matches!(
- read_server_message(client_rx.recv_timeout(Duration::from_millis(100)).unwrap()),
+ read_server_message(older),
ServerMessage::ReloadSoundConfig
));
- assert_eq!(
- server.handle_server_event_with_render_impact(ServerEvent::ClientWriterDrained {
- client_id: 1
- }),
- RenderImpact::Graphics
- );
-}
-
-#[tokio::test]
-async fn retained_update_does_not_downgrade_pending_full_render() {
- let (mut server, client_rx, pane_id) = retained_test_server(b"aaaa");
- let _ = enable_graphics_and_render(&mut server, &client_rx);
- fill_render_lane(&server);
- let client = server.clients.get_mut(&1).unwrap();
- client.request_repaint();
- server.render_and_stream();
- assert_eq!(
- server.clients.get(&1).unwrap().deferred_render(),
- DeferredRender::Full
- );
-
- set_graphics_layer(&mut server, pane_id, vec![7, 8, 9]);
- assert_eq!(
- server.render_retained_graphics_update_and_stream(),
- RetainedGraphicsOutcome::Deferred
- );
- assert_eq!(
- server.clients.get(&1).unwrap().deferred_render(),
- DeferredRender::Full
- );
- assert_eq!(
- server.handle_server_event_with_render_impact(ServerEvent::ClientWriterDrained {
- client_id: 1
- }),
- RenderImpact::Full
- );
+ assert!(matches!(
+ read_server_message(graphics),
+ ServerMessage::Graphics { .. }
+ ));
}
#[tokio::test]
@@ -247,6 +445,64 @@ async fn retained_update_falls_back_for_mixed_app_geometry() {
}
#[test]
+fn stream_open_gate_is_owned_by_the_layer_and_cancels_on_removal() {
+ let mut server = test_headless_server();
+ server.app.state.kitty_graphics_enabled = true;
+ let workspace = crate::workspace::Workspace::test_new("gated");
+ let pane_id = workspace.tabs[0].root_pane;
+ let public = format!("{}:p1", workspace.id);
+ server.app.state.workspaces = vec![workspace];
+ server.app.state.active = Some(0);
+ let active = active_gate();
+ let (respond_to, response_rx) = std::sync::mpsc::channel();
+
+ server.handle_api_request_with_shutdown_check(api::ApiRequestMessage {
+ request: api::schema::Request {
+ id: "open-gated".into(),
+ method: api::schema::Method::PaneGraphicsStreamOpen(
+ api::schema::PaneGraphicsStreamParams {
+ pane_id: public.clone(),
+ layer_id: None,
+ z_index: 0,
+ owner: "worker-1".into(),
+ },
+ ),
+ },
+ respond_to,
+ response_write_complete: None,
+ stream_active: Some(active.clone()),
+ });
+ assert!(
+ serde_json::from_str::<api::schema::SuccessResponse>(&response_rx.recv().unwrap()).is_ok()
+ );
+ let (frame, frame_response) =
+ stream_set_message("gated-frame", &public, "worker-1", vec![1, 2, 3]);
+ assert_eq!(
+ server.handle_api_request_with_render_impact(frame),
+ RenderImpact::Graphics
+ );
+ assert!(frame_response.recv().is_ok());
+ assert!(active.load(std::sync::atomic::Ordering::Acquire));
+ active.store(false, std::sync::atomic::Ordering::Release);
+ let (delayed, delayed_response) =
+ stream_set_message("delayed-frame", &public, "worker-1", vec![4, 5, 6]);
+ assert_eq!(
+ server.handle_api_request_with_render_impact(delayed),
+ RenderImpact::None
+ );
+ let error: api::schema::ErrorResponse =
+ serde_json::from_str(&delayed_response.recv().unwrap()).unwrap();
+ assert_eq!(error.error.code, "stream_closed");
+ assert!(server
+ .app
+ .pane_graphics
+ .slots
+ .remove(&graphics_key(pane_id))
+ .is_some());
+ assert!(!active.load(std::sync::atomic::Ordering::Acquire));
+}
+
+#[test]
fn stream_set_has_graphics_only_render_impact() {
let mut server = test_headless_server();
let workspace = crate::workspace::Workspace::test_new("graphics");
@@ -256,11 +512,7 @@ fn stream_set_has_graphics_only_render_impact() {
server.app.state.active = Some(0);
server.app.state.selected = 0;
server.app.state.kitty_graphics_enabled = true;
- server
- .app
- .state
- .pane_graphics_streams
- .insert(pane_id, "owner-a".into());
+ set_stream_owner(&mut server, pane_id, "owner-a");
let (request, response_rx) =
stream_set_message("wrong-owner", &public_pane_id, "owner-b", vec![1, 2, 3]);
@@ -307,13 +559,15 @@ fn stream_set_has_graphics_only_render_impact() {
RenderImpact::Full
);
- server.app.state.pane_graphics_streams.clear();
+ server.app.pane_graphics.clear();
let (respond_to, _response_rx) = std::sync::mpsc::channel();
let impact = server.handle_api_request_with_render_impact(api::ApiRequestMessage {
request: api::schema::Request {
id: "direct-frame".into(),
method: api::schema::Method::PaneGraphicsSet(api::schema::PaneGraphicsSetParams {
pane_id: public_pane_id,
+ layer_id: None,
+ z_index: 0,
owner: String::new(),
format: api::schema::PaneGraphicsFormat::Png,
image_width: 1,
@@ -325,6 +579,7 @@ fn stream_set_has_graphics_only_render_impact() {
},
respond_to,
response_write_complete: None,
+ stream_active: None,
});
assert_eq!(impact, RenderImpact::Full);
}
@@ -345,6 +600,8 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() {
id: "disabled-set".into(),
method: api::schema::Method::PaneGraphicsSet(api::schema::PaneGraphicsSetParams {
pane_id: public_pane_id.clone(),
+ layer_id: None,
+ z_index: 0,
owner: String::new(),
format: api::schema::PaneGraphicsFormat::Png,
image_width: 1,
@@ -356,6 +613,7 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() {
},
respond_to,
response_write_complete: None,
+ stream_active: None,
});
assert!(!changed);
let response = response_rx
@@ -370,11 +628,7 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() {
);
server.app.state.kitty_graphics_enabled = true;
- server
- .app
- .state
- .pane_graphics_streams
- .insert(pane_id, "current-owner".into());
+ set_stream_owner(&mut server, pane_id, "current-owner");
let (respond_to, response_rx) = std::sync::mpsc::channel();
let impact = server.handle_api_request_with_render_impact(api::ApiRequestMessage {
request: api::schema::Request {
@@ -382,17 +636,25 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() {
method: api::schema::Method::PaneGraphicsStreamClose(
api::schema::PaneGraphicsStreamParams {
pane_id: public_pane_id,
+ layer_id: None,
+ z_index: 0,
owner: "stale-owner".into(),
},
),
},
respond_to,
response_write_complete: None,
+ stream_active: None,
});
assert_eq!(impact, RenderImpact::None);
assert_eq!(
- server.app.state.pane_graphics_streams.get(&pane_id),
- Some(&"current-owner".to_string())
+ server
+ .app
+ .pane_graphics
+ .slots
+ .get(&graphics_key(pane_id))
+ .and_then(|slot| slot.stream_owner.as_deref()),
+ Some("current-owner")
);
assert!(serde_json::from_str::<api::schema::SuccessResponse>(
&response_rx
@@ -401,3 +663,282 @@ fn rejected_or_stale_requests_do_not_schedule_rendering() {
)
.is_ok());
}
+
+#[cfg(unix)]
+fn direct_gate_server(
+ data: &[u8],
+) -> (
+ HeadlessServer,
+ crate::app::pane_graphics::Key,
+ std::sync::mpsc::Receiver<String>,
+) {
+ use std::io::Write as _;
+ use std::os::unix::fs::OpenOptionsExt as _;
+ let mut server = test_headless_server();
+ let workspace = crate::workspace::Workspace::test_new("direct-gate");
+ let pane_id = workspace.tabs[0].root_pane;
+ server.app.state.workspaces = vec![workspace];
+ server.app.state.active = Some(0);
+ let key = graphics_key(pane_id);
+ let path = server
+ .app
+ .pane_graphics_files
+ .source_directory()
+ .unwrap()
+ .join("gate-frame");
+ let mut file = std::fs::OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .mode(0o600)
+ .open(&path)
+ .unwrap();
+ file.write_all(data).unwrap();
+ drop(file);
+ let lease = server
+ .app
+ .pane_graphics_files
+ .lease(&path, data.len())
+ .unwrap();
+ let (respond_to, response_rx) = std::sync::mpsc::channel();
+ let layer =
+ crate::app::pane_graphics::Layer::direct(1, 1, lease.clone(), Default::default(), 0);
+ let mut slot = crate::app::pane_graphics::Slot::test((1 << 31) | 900, Some(layer));
+ slot.stream_owner = Some("owner".into());
+ slot.stream_active = Some(active_gate());
+ slot.direct_gate = Some(crate::app::pane_graphics::DirectGate {
+ transfer_id: lease.fingerprint(),
+ client_id: 7,
+ deadline: std::time::Instant::now() + Duration::from_secs(1),
+ written: true,
+ success_response: "ack".into(),
+ respond_to,
+ });
+ server.app.pane_graphics.slots.insert(key.clone(), slot);
+ (server, key, response_rx)
+}
+
+#[cfg(unix)]
+fn direct_ids(server: &HeadlessServer, key: &crate::app::pane_graphics::Key) -> (u64, u32) {
+ let slot = &server.app.pane_graphics.slots[key];
+ (
+ slot.direct_gate.as_ref().unwrap().transfer_id,
+ slot.host_image_id,
+ )
+}
+
+#[cfg(unix)]
+fn add_direct_client(server: &mut HeadlessServer, client_id: u64) {
+ let (writer, control_rx, render_rx) = test_client_writer();
+ std::mem::forget((control_rx, render_rx));
+ let mut client = ClientConnection::new(
+ (80, 24),
+ crate::kitty_graphics::HostCellSize {
+ width_px: 10,
+ height_px: 20,
+ },
+ crate::terminal_theme::TerminalTheme::default(),
+ None,
+ 1,
+ RenderEncoding::SemanticFrame,
+ Some(writer),
+ );
+ client.direct_graphics = true;
+ client.pixel_mouse = true;
+ server.clients.insert(client_id, client);
+}
+
+#[cfg(unix)]
+#[test]
+fn terminal_response_deadline_starts_only_after_client_flush() {
+ let (mut server, key, _response_rx) = direct_gate_server(&[1, 2, 3, 4]);
+ let slot = server.app.pane_graphics.slots.get_mut(&key).unwrap();
+ let gate = slot.direct_gate.as_mut().unwrap();
+ gate.written = false;
+ let (transfer_id, image_id) = (gate.transfer_id, slot.host_image_id);
+ assert!(!server.complete_direct_graphics(7, transfer_id, image_id, true));
+ assert!(!server.start_direct_graphics_response(7, transfer_id, image_id));
+ let gate = server.app.pane_graphics.slots[&key]
+ .direct_gate
+ .as_ref()
+ .unwrap();
+ assert!(gate.written && gate.deadline > std::time::Instant::now());
+}
+
+#[cfg(unix)]
+#[test]
+fn outer_timeout_covers_both_direct_phases_and_cancellation_blocks_late_results() {
+ assert!(
+ crate::app::pane_graphics::DIRECT_OUTER_TIMEOUT
+ > crate::app::pane_graphics::DIRECT_DELIVERY_TIMEOUT
+ + crate::app::pane_graphics::DIRECT_RESPONSE_TIMEOUT
+ );
+ let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]);
+ let slot = server.app.pane_graphics.slots.get_mut(&key).unwrap();
+ slot.stream_active
+ .as_ref()
+ .unwrap()
+ .store(false, std::sync::atomic::Ordering::Release);
+ let (transfer_id, image_id) = (
+ slot.direct_gate.as_ref().unwrap().transfer_id,
+ slot.host_image_id,
+ );
+
+ assert!(!server.complete_direct_graphics(7, transfer_id, image_id, true));
+ assert!(response_rx.try_recv().is_err());
+ assert!(server.app.pane_graphics.slots[&key]
+ .layer
+ .as_ref()
+ .unwrap()
+ .direct_lease()
+ .is_some());
+}
+
+#[cfg(unix)]
+#[test]
+fn matching_terminal_ok_releases_producer_and_acknowledges() {
+ let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]);
+ let (transfer_id, image_id) = direct_ids(&server, &key);
+
+ assert!(server.complete_direct_graphics(7, transfer_id, image_id, true));
+
+ assert_eq!(response_rx.recv().unwrap(), "ack");
+ let layer = server.app.pane_graphics.slots[&key].layer.as_ref().unwrap();
+ assert!(layer.terminal_only());
+ assert!(layer.direct_lease().is_none());
+}
+
+#[cfg(unix)]
+#[test]
+fn explicit_terminal_error_acks_only_after_owned_inline_fallback() {
+ let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]);
+ add_direct_client(&mut server, 7);
+ let (transfer_id, image_id) = direct_ids(&server, &key);
+ let layer = server.app.pane_graphics.slots[&key].layer.as_ref().unwrap();
+ server
+ .clients
+ .get_mut(&7)
+ .unwrap()
+ .graphics_cache
+ .trust_pane_layer(&key, image_id, layer);
+ assert!(server.complete_direct_graphics(7, transfer_id, image_id, false));
+
+ let layer = server.app.pane_graphics.slots[&key].layer.as_ref().unwrap();
+ assert_eq!(
+ (
+ response_rx.recv().unwrap(),
+ layer.inline_data(),
+ server.clients[&7].direct_graphics,
+ server.clients[&7].pixel_mouse,
+ ),
+ ("ack".into(), Some([1, 2, 3, 4].as_slice()), false, true)
+ );
+ assert!(server.clients[&7].graphics_cache.is_empty());
+}
+
+#[cfg(unix)]
+#[test]
+fn unwritten_direct_full_falls_back_without_stickiness_but_disconnect_retires() {
+ for error in [
+ std::sync::mpsc::TrySendError::Full(Vec::new()),
+ std::sync::mpsc::TrySendError::Disconnected(Vec::new()),
+ ] {
+ let should_ack = matches!(error, std::sync::mpsc::TrySendError::Full(_));
+ let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]);
+ add_direct_client(&mut server, 7);
+ let gate = server
+ .app
+ .pane_graphics
+ .slots
+ .get_mut(&key)
+ .and_then(|slot| slot.direct_gate.take())
+ .unwrap();
+ let result = server.handle_unwritten_direct_failure(
+ &key,
+ gate.success_response,
+ gate.respond_to,
+ error,
+ );
+ let inline = server
+ .app
+ .pane_graphics
+ .slots
+ .get(&key)
+ .and_then(|slot| slot.layer.as_ref()?.inline_data())
+ .is_some();
+ assert_eq!(
+ (
+ result,
+ response_rx.try_recv().ok().as_deref() == Some("ack"),
+ inline,
+ server.clients[&7].direct_graphics,
+ ),
+ (should_ack, should_ack, should_ack, true)
+ );
+ }
+}
+
+#[cfg(unix)]
+#[test]
+fn client_loss_retires_only_its_direct_stream() {
+ let (mut pending, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]);
+ pending.retire_direct_graphics_for_client(8);
+ assert!(pending.app.pane_graphics.slots.contains_key(&key));
+ pending.retire_direct_graphics_for_client(7);
+ assert!(!pending.app.pane_graphics.slots.contains_key(&key));
+ assert!(response_rx.recv().is_err());
+
+ let (mut resident, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]);
+ let slot = resident.app.pane_graphics.slots.get(&key).unwrap();
+ assert!(resident.complete_direct_graphics(
+ 7,
+ slot.direct_gate.as_ref().unwrap().transfer_id,
+ slot.host_image_id,
+ true,
+ ));
+ assert_eq!(response_rx.recv().unwrap(), "ack");
+ resident.retire_direct_graphics_for_client(8);
+ assert!(resident.app.pane_graphics.slots.contains_key(&key));
+ resident.retire_direct_graphics_for_client(7);
+ assert!(!resident.app.pane_graphics.slots.contains_key(&key));
+}
+
+#[cfg(unix)]
+#[test]
+fn pane_removal_and_shutdown_drop_direct_without_ack() {
+ let setups: [fn(&mut HeadlessServer); 2] = [
+ |server| server.app.state.workspaces.clear(),
+ |server| server.shutting_down = true,
+ ];
+ for setup in setups {
+ let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]);
+ let (transfer_id, image_id) = direct_ids(&server, &key);
+ setup(&mut server);
+ assert!(!server.complete_direct_graphics(7, transfer_id, image_id, true));
+ assert!(response_rx.recv().is_err());
+ assert!(!server.app.pane_graphics.slots.contains_key(&key));
+ }
+}
+
+#[cfg(unix)]
+#[test]
+fn timeout_retires_stream_without_producer_ack() {
+ let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]);
+ add_direct_client(&mut server, 7);
+ server
+ .app
+ .pane_graphics
+ .slots
+ .get_mut(&key)
+ .unwrap()
+ .direct_gate
+ .as_mut()
+ .unwrap()
+ .deadline = std::time::Instant::now() - Duration::from_millis(1);
+
+ assert!(server.expire_direct_graphics(std::time::Instant::now()));
+
+ assert!(response_rx.recv().is_err());
+ assert!(!server.app.pane_graphics.slots.contains_key(&key));
+ assert!(!server.clients[&7].direct_graphics);
+ assert!(server.clients[&7].pixel_mouse);
+}
diff --git a/src/terminal/runtime.rs b/src/terminal/runtime.rs
index 8a3da876..13e29690 100644
--- a/src/terminal/runtime.rs
+++ b/src/terminal/runtime.rs
@@ -465,31 +465,32 @@ impl TerminalRuntime {
pub fn encode_mouse_button(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- self.0.encode_mouse_button(kind, column, row, modifiers)
+ self.0.encode_mouse_button(kind, position, modifiers)
}
- pub fn encode_mouse_motion(
+ pub(crate) fn encode_mouse_motion(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- self.0.encode_mouse_motion(kind, column, row, modifiers)
+ self.0.encode_mouse_motion(kind, position, modifiers)
}
- pub fn encode_mouse_wheel(
+ pub(crate) fn encode_mouse_wheel(
&self,
kind: crossterm::event::MouseEventKind,
- column: u16,
- row: u16,
+ position: crate::input::mouse::Position,
modifiers: crossterm::event::KeyModifiers,
) -> Option<Vec<u8>> {
- self.0.encode_mouse_wheel(kind, column, row, modifiers)
+ self.0.encode_mouse_wheel(kind, position, modifiers)
+ }
+
+ pub(crate) fn pixel_size(&self) -> Option<(u32, u32)> {
+ self.0.pixel_size()
}
pub fn encode_alternate_scroll(