diff options
| author | Can Celik <ogulcancelik@gmail.com> | 2026-08-09 02:04:39 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-08-09 02:04:39 +0300 |
| commit | 1777e9bba32b953ed1ad203b4a16d01105539000 (patch) | |
| tree | 22096973812c9baba822453d8470bdcfe64fc2a6 /src/kitty_graphics.rs | |
| parent | 36a8d36a7258e08b40c01bf521b1f4006525f143 (diff) | |
feat(graphics): add direct pane frame streaming (#2523)
* feat(graphics): add direct pane frame streaming
* feat(graphics): expose pane visibility
* test(graphics): preserve legacy terminal image output
* fix(graphics): harden streaming lifecycle
* fix(input): preserve mouse releases outside panes
Diffstat (limited to 'src/kitty_graphics.rs')
| -rw-r--r-- | src/kitty_graphics.rs | 1543 |
1 files changed, 1163 insertions, 380 deletions
diff --git a/src/kitty_graphics.rs b/src/kitty_graphics.rs index 98577ef8..726e1d8b 100644 --- a/src/kitty_graphics.rs +++ b/src/kitty_graphics.rs @@ -18,7 +18,10 @@ use crate::layout::{PaneId, PaneInfo}; use crate::terminal::TerminalRuntimeRegistry; const KITTY_CHUNK_BYTES: usize = 3072; +pub(crate) const HEADLESS_GRAPHICS_TRANSACTION_BUDGET: usize = + crate::protocol::MAX_GRAPHICS_FRAME_SIZE - crate::protocol::MAX_FRAME_SIZE; const HOST_IMAGE_ID_BASE: u32 = 10_000; +#[cfg(test)] const PANE_GRAPHICS_IMAGE_ID_BIT: u32 = 1 << 31; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -73,6 +76,7 @@ struct HostViewKey { #[derive(Debug)] struct HostPlacement { pane_id: PaneId, + host_image_id: Option<u32>, area: Rect, cell_size: HostCellSize, source_key: HostSourceKey, @@ -80,10 +84,10 @@ struct HostPlacement { scrollback_offset: u32, } -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] enum HostSourceKey { Terminal { pane_id: PaneId, image_id: u32 }, - PaneLayer { pane_id: PaneId }, + PaneLayer { pane_id: PaneId, layer_id: String }, } #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] @@ -131,7 +135,11 @@ pub(crate) struct HostGraphicsCache { placements: HashMap<(u32, u32), PlacementSignature>, /// Host image currently backing each (pane, source image id) pair. sources: HashMap<HostSourceKey, u32>, + oversized: HashMap<HostSourceKey, ImageSignature>, + continuation: Option<(HostSourceKey, u32, usize)>, view: Option<HostViewKey>, + replay_placements: bool, + replayed_placements: HashSet<(u32, u32)>, } static KITTY_GRAPHICS_ENABLED: AtomicBool = AtomicBool::new(false); @@ -147,94 +155,130 @@ pub(crate) fn is_enabled() -> bool { pub(crate) fn paint_local_pane_graphics( app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, terminal_runtimes: &TerminalRuntimeRegistry, cell_size: HostCellSize, ) -> io::Result<()> { let cache = LOCAL_HOST_GRAPHICS.get_or_init(|| Mutex::new(HostGraphicsCache::default())); - let mut bytes = Vec::new(); - if let Ok(mut cache) = cache.lock() { - bytes = encode_local_pane_graphics( + let Ok(mut cache) = cache.lock() else { + return Ok(()); + }; + if graphics.slots.is_empty() && !cache.has_pane_sources() { + let encoded = encode_local_pane_graphics( app, + graphics, terminal_runtimes, app.view.tab_surface(), cell_size, + None, &mut cache, ); - } - if bytes.is_empty() { - return Ok(()); + drop(cache); + if encoded.bytes.is_empty() { + return Ok(()); + } + let mut stdout = io::stdout().lock(); + stdout.write_all(b"\x1b7")?; + stdout.write_all(&encoded.bytes)?; + stdout.write_all(b"\x1b8")?; + return stdout.flush(); } - let mut framed = Vec::with_capacity(bytes.len() + 8); - framed.extend_from_slice(b"\x1b7"); - framed.extend_from_slice(&bytes); - framed.extend_from_slice(b"\x1b8"); - let mut stdout = io::stdout().lock(); - stdout.write_all(&framed)?; + loop { + let encoded = encode_local_pane_graphics( + app, + graphics, + terminal_runtimes, + app.view.tab_surface(), + cell_size, + None, + &mut cache, + ); + if !encoded.bytes.is_empty() { + stdout.write_all(b"\x1b7")?; + stdout.write_all(&encoded.bytes)?; + stdout.write_all(b"\x1b8")?; + } + if !encoded.incomplete { + break; + } + } stdout.flush() } +pub(crate) struct EncodedGraphics { + pub(crate) bytes: Vec<u8>, + pub(crate) incomplete: bool, +} + pub(crate) fn encode_local_pane_graphics( app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, terminal_runtimes: &TerminalRuntimeRegistry, surface: crate::ui::TabSurfaceView<'_>, cell_size: HostCellSize, + transaction_budget: Option<usize>, cache: &mut HostGraphicsCache, -) -> Vec<u8> { - let mode_ok = app.mode == Mode::Terminal; - let cell_ok = cell_size.is_known(); - tracing::debug!( - mode_ok, - cell_ok, - cell_width_px = cell_size.width_px, - cell_height_px = cell_size.height_px, - active = ?app.active, - pane_infos_len = surface.pane_infos.len(), - "paint_local_pane_graphics entry" - ); - if !mode_ok || !cell_ok { - tracing::debug!( - reason = if !mode_ok { - "not terminal mode" - } else { - "cell size unknown" - }, - "paint_local_pane_graphics early return" +) -> EncodedGraphics { + let visible = app.mode == Mode::Terminal && cell_size.is_known(); + if graphics.slots.is_empty() { + let mut bytes = cache.clear_pane_sources(); + if !visible { + bytes.extend(cache.clear_bytes()); + return EncodedGraphics { + bytes, + incomplete: false, + }; + } + let placements = collect_visible_placements( + app, + graphics, + terminal_runtimes, + surface, + cell_size, + &cache.images, ); - return cache.clear_bytes(); + let view_changed = cache.update_view(active_view_key(app)); + cache.reset_incremental_state(); + encode_terminal_graphics_update_legacy(&mut bytes, &placements, view_changed, cache); + return EncodedGraphics { + bytes, + incomplete: false, + }; } - let view_key = active_view_key(app); - let placements = - collect_visible_placements(app, terminal_runtimes, surface, cell_size, &cache.images); - tracing::debug!( - placements_collected = placements.len(), - "collect_visible_placements result" - ); - - let mut bytes = Vec::new(); - let view_changed = cache.update_view(view_key); - encode_graphics_update( - &mut bytes, - &placements, - view_changed, - &mut cache.images, - &mut cache.placements, - &mut cache.sources, - ); - tracing::debug!( - placements = placements.len(), - bytes = bytes.len(), - cell_width_px = cell_size.width_px, - cell_height_px = cell_size.height_px, - "painting kitty graphics placements" - ); - bytes + let live_pane_sources = graphics + .slots + .iter() + .filter(|(_, slot)| slot.layer.is_some()) + .map(|((pane_id, layer_id), _)| HostSourceKey::PaneLayer { + pane_id: *pane_id, + layer_id: layer_id.clone(), + }) + .collect::<HashSet<_>>(); + let placements = if visible { + collect_visible_placements( + app, + graphics, + terminal_runtimes, + surface, + cell_size, + &cache.images, + ) + } else { + Vec::new() + }; + cache.update_view(visible.then(|| active_view_key(app)).flatten()); + // The host text blit overwrites Kitty placements, so every rendered frame must + // display cached images again even when their data and geometry are unchanged. + cache.request_placement_replay(); + encode_graphics_update_incremental(cache, &placements, &live_pane_sources, transaction_budget) } pub(crate) fn has_visible_pane_graphics( app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, terminal_runtimes: &TerminalRuntimeRegistry, surface: crate::ui::TabSurfaceView<'_>, cell_size: HostCellSize, @@ -257,10 +301,20 @@ pub(crate) fn has_visible_pane_graphics( for info in surface.pane_infos { let empty_uploaded = HashMap::new(); - if app.pane_graphics_layers.get(&info.id).is_some_and(|layer| { - let host_placement = - pane_graphics_host_placement(info, cell_size, layer, &empty_uploaded, false); - clipped_placement(&host_placement).is_some() + if graphics.slots.iter().any(|((pane_id, layer_id), slot)| { + *pane_id == info.id + && slot.layer.as_ref().is_some_and(|layer| { + clipped_placement(&pane_graphics_host_placement( + info, + layer_id, + slot.host_image_id, + cell_size, + layer, + &empty_uploaded, + false, + )) + .is_some() + }) }) { return true; } @@ -274,6 +328,7 @@ pub(crate) fn has_visible_pane_graphics( for placement in runtime.kitty_image_placements_with_data_filter(|_| false) { let host_placement = HostPlacement { pane_id: info.id, + host_image_id: None, area: info.inner_rect, cell_size, source_key: HostSourceKey::Terminal { @@ -292,73 +347,41 @@ pub(crate) fn has_visible_pane_graphics( false } -fn encode_graphics_update( +fn encode_terminal_graphics_update_legacy( bytes: &mut Vec<u8>, placements: &[HostPlacement], view_changed: bool, - host_images: &mut HashMap<u32, ImageSignature>, - host_placements: &mut HashMap<(u32, u32), PlacementSignature>, - sources: &mut HashMap<HostSourceKey, u32>, + cache: &mut HostGraphicsCache, ) { - // Prune sources that are no longer visible: a stale entry would keep its - // old host image referenced and block the superseded-image delete. - let current_sources: HashSet<HostSourceKey> = placements + let current_sources = placements .iter() - .filter(|placement| { - matches!(placement.source_key, HostSourceKey::Terminal { .. }) - || clipped_placement(placement).is_some() - }) - .map(|placement| placement.source_key) - .collect(); - let mut removed_pane_layer_images = HashSet::new(); - sources.retain(|source, host_id| { - let retain = current_sources.contains(source); - if !retain && matches!(source, HostSourceKey::PaneLayer { .. }) { - removed_pane_layer_images.insert(*host_id); - } - retain - }); - for host_id in removed_pane_layer_images { - if sources.values().any(|id| *id == host_id) { - continue; - } - encode_delete_image(bytes, host_id); - host_images.remove(&host_id); - host_placements.retain(|(image_id, _), _| *image_id != host_id); - } + .filter(|placement| matches!(placement.source_key, HostSourceKey::Terminal { .. })) + .map(|placement| placement.source_key.clone()) + .collect::<HashSet<_>>(); + cache + .sources + .retain(|source, _| current_sources.contains(source)); let mut current_placements = HashSet::new(); for placement in placements { - let clipped = clipped_placement(placement); - tracing::debug!( - pane_id = ?placement.pane_id, - has_clipped = clipped.is_some(), - grid_cols = placement.placement.render.grid_cols, - grid_rows = placement.placement.render.grid_rows, - viewport_col = placement.placement.render.viewport_col, - viewport_row = placement.placement.render.viewport_row, - area_w = placement.area.width, - area_h = placement.area.height, - "clipped_placement result" - ); - let Some((clipped, format_code)) = clipped else { + let Some((clipped, format_code)) = clipped_placement(placement) else { continue; }; let host_id = host_image_id(placement.pane_id, &placement.placement); - let host_placement_id = host_placement_id(placement.source_key, &placement.placement); + let placement_id = host_placement_id(&placement.source_key, &placement.placement); let image_signature = image_signature(placement, format_code); let placement_signature = placement_signature(clipped, placement.placement.z, placement.scrollback_offset); - let placement_key = (host_id, host_placement_id); + let placement_key = (host_id, placement_id); current_placements.insert(placement_key); - match host_images.get(&host_id).copied() { + match cache.images.get(&host_id).copied() { Some(existing) if existing == image_signature => {} Some(_) => { encode_delete_image(bytes, host_id); - host_placements.retain(|(image_id, placement_id), _| { + cache.placements.retain(|(image_id, id), _| { if *image_id == host_id { - current_placements.remove(&(*image_id, *placement_id)); + current_placements.remove(&(*image_id, *id)); false } else { true @@ -367,37 +390,32 @@ fn encode_graphics_update( if !encode_upload_image(bytes, placement, format_code, host_id) { continue; } - host_images.insert(host_id, image_signature); + cache.images.insert(host_id, image_signature); } None => { if !encode_upload_image(bytes, placement, format_code, host_id) { continue; } - host_images.insert(host_id, image_signature); + cache.images.insert(host_id, image_signature); } } - release_superseded_source_image( + release_superseded_terminal_image_legacy( bytes, - sources, - host_images, - host_placements, + cache, &mut current_placements, - placement.source_key, + placement.source_key.clone(), host_id, ); - // A different view can repaint the same cells with text or overlays and - // leave the host-side Kitty placement state out of sync with this cache. - // Re-emit the placement even when its geometry signature is unchanged. - match host_placements.get_mut(&placement_key) { + match cache.placements.get_mut(&placement_key) { Some(existing) if !view_changed && *existing == placement_signature => {} Some(existing) => { encode_display_placement( bytes, clipped, host_id, - host_placement_id, + placement_id, placement.placement.z, ); *existing = placement_signature; @@ -407,48 +425,42 @@ fn encode_graphics_update( bytes, clipped, host_id, - host_placement_id, + placement_id, placement.placement.z, ); - host_placements.insert(placement_key, placement_signature); + cache.placements.insert(placement_key, placement_signature); } } } - let mut stale_placements = Vec::new(); - for key in host_placements.keys() { - if current_placements.contains(key) { - continue; - } - stale_placements.push(*key); - } - for (host_id, host_placement_id) in stale_placements { - encode_delete_placement(bytes, host_id, host_placement_id); - host_placements.remove(&(host_id, host_placement_id)); + let stale = cache + .placements + .keys() + .filter(|key| !current_placements.contains(key)) + .copied() + .collect::<Vec<_>>(); + for (host_id, placement_id) in stale { + encode_delete_placement(bytes, host_id, placement_id); + cache.placements.remove(&(host_id, placement_id)); } } -/// Records that `source` is now backed by `host_id` and deletes the host -/// image it previously pointed at once no other source references it. -fn release_superseded_source_image( +fn release_superseded_terminal_image_legacy( bytes: &mut Vec<u8>, - sources: &mut HashMap<HostSourceKey, u32>, - host_images: &mut HashMap<u32, ImageSignature>, - host_placements: &mut HashMap<(u32, u32), PlacementSignature>, + cache: &mut HostGraphicsCache, current_placements: &mut HashSet<(u32, u32)>, source: HostSourceKey, host_id: u32, ) { - let Some(previous) = sources.insert(source, host_id) else { + let Some(previous) = cache.sources.insert(source, host_id) else { return; }; - if previous == host_id || sources.values().any(|id| *id == previous) { + if previous == host_id || cache.sources.values().any(|id| *id == previous) { return; } encode_delete_image(bytes, previous); - host_images.remove(&previous); - // The `d=I` delete also removes the image's placements host-side. - host_placements.retain(|(image_id, placement_id), _| { + cache.images.remove(&previous); + cache.placements.retain(|(image_id, placement_id), _| { if *image_id == previous { current_placements.remove(&(*image_id, *placement_id)); false @@ -458,6 +470,320 @@ fn release_superseded_source_image( }); } +fn encode_graphics_update_incremental( + cache: &mut HostGraphicsCache, + placements: &[HostPlacement], + live_pane_sources: &HashSet<HostSourceKey>, + transaction_budget: Option<usize>, +) -> EncodedGraphics { + let desired_sources = placements + .iter() + .map(|placement| placement.source_key.clone()) + .collect::<HashSet<_>>(); + let desired_placements = placements + .iter() + .filter_map(|placement| { + clipped_placement(placement).map(|_| { + let host_id = placement + .host_image_id + .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement)); + ( + host_id, + host_placement_id(&placement.source_key, &placement.placement), + ) + }) + }) + .collect::<HashSet<_>>(); + let start = cache + .continuation + .as_ref() + .and_then(|(source, id, _)| { + placements + .iter() + .position(|placement| placement_identity(placement) == (source.clone(), *id)) + }) + .map(|index| index + 1) + .or_else(|| cache.continuation.as_ref().map(|cursor| cursor.2)) + .map_or(0, |index| index % placements.len().max(1)); + let mut bytes = Vec::new(); + let mut emitted = false; + + let mut dead_sources = cache + .sources + .keys() + .filter(|source| { + matches!(source, HostSourceKey::PaneLayer { .. }) + && !live_pane_sources.contains(*source) + }) + .cloned() + .collect::<Vec<_>>(); + dead_sources.sort_by_key(source_order); + for source in dead_sources { + let host_id = cache.sources[&source]; + let last_reference = !cache + .sources + .iter() + .any(|(other, id)| *other != source && *id == host_id); + if emitted && last_reference { + return EncodedGraphics { + bytes, + incomplete: true, + }; + } + cache.sources.remove(&source); + if last_reference { + encode_delete_image(&mut bytes, host_id); + cache.images.remove(&host_id); + cache.placements.retain(|(id, _), _| *id != host_id); + cache.replayed_placements.retain(|(id, _)| *id != host_id); + emitted = true; + } + } + cache.sources.retain(|source, _| { + matches!(source, HostSourceKey::PaneLayer { .. }) || desired_sources.contains(source) + }); + cache + .oversized + .retain(|source, _| live_pane_sources.contains(source) || desired_sources.contains(source)); + + let mut stale = cache + .placements + .keys() + .filter(|key| !desired_placements.contains(key)) + .copied() + .collect::<Vec<_>>(); + stale.sort_unstable(); + for key @ (host_id, placement_id) in stale { + if emitted { + return EncodedGraphics { + bytes, + incomplete: true, + }; + } + encode_delete_placement(&mut bytes, host_id, placement_id); + cache.placements.remove(&key); + cache.replayed_placements.remove(&key); + emitted = true; + } + + for offset in 0..placements.len() { + let index = (start + offset) % placements.len(); + let placement = &placements[index]; + let signature = image_signature(placement, kitty_format_code(placement.placement.format)); + if transaction_budget.is_some() + && cache.oversized.get(&placement.source_key) == Some(&signature) + { + continue; + } + cache.oversized.remove(&placement.source_key); + let host_id = placement + .host_image_id + .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement)); + if cache.images.get(&host_id) != Some(&signature) + && !image_transaction_fits(placement, transaction_budget) + { + cache + .oversized + .insert(placement.source_key.clone(), signature); + continue; + } + let mut candidate = cache.clone(); + let Some(transaction) = encode_placement_update(&mut candidate, placement) else { + continue; + }; + if transaction.is_empty() { + *cache = candidate; + continue; + } + if emitted { + return EncodedGraphics { + bytes, + incomplete: true, + }; + } + *cache = candidate; + let (source, id) = placement_identity(placement); + cache.continuation = Some((source, id, (index + 1) % placements.len())); + bytes = transaction; + emitted = true; + } + + cache.replay_placements = false; + cache.replayed_placements.clear(); + EncodedGraphics { + bytes, + incomplete: false, + } +} + +fn image_transaction_fits(placement: &HostPlacement, budget: Option<usize>) -> bool { + let Some(budget) = budget else { + return true; + }; + let data = placement.placement.data_len; + let encoded = data.div_ceil(3).saturating_mul(4); + let command_overhead = data.div_ceil(KITTY_CHUNK_BYTES).saturating_mul(16) + 1024; + encoded.saturating_add(command_overhead) <= budget +} + +fn placement_identity(placement: &HostPlacement) -> (HostSourceKey, u32) { + ( + placement.source_key.clone(), + host_placement_id(&placement.source_key, &placement.placement), + ) +} + +fn source_order(source: &HostSourceKey) -> (u32, String) { + match source { + HostSourceKey::Terminal { pane_id, .. } => (pane_id.raw(), String::new()), + HostSourceKey::PaneLayer { pane_id, layer_id } => (pane_id.raw(), layer_id.clone()), + } +} + +fn encode_placement_update( + cache: &mut HostGraphicsCache, + placement: &HostPlacement, +) -> Option<Vec<u8>> { + let (clipped, format_code) = clipped_placement(placement)?; + let host_id = placement + .host_image_id + .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement)); + let placement_id = host_placement_id(&placement.source_key, &placement.placement); + let key = (host_id, placement_id); + let image_signature = image_signature(placement, format_code); + let placement_signature = + placement_signature(clipped, placement.placement.z, placement.scrollback_offset); + let image_current = cache.images.get(&host_id) == Some(&image_signature); + let placement_current = cache.placements.get(&key) == Some(&placement_signature) + && (!cache.replay_placements || cache.replayed_placements.contains(&key)); + if image_current + && placement_current + && cache.sources.get(&placement.source_key) == Some(&host_id) + { + return None; + } + + let mut bytes = Vec::new(); + let mut displayed = false; + if !image_current { + if cache.images.contains_key(&host_id) + && matches!(placement.source_key, HostSourceKey::PaneLayer { .. }) + { + if !encode_transmit_and_display( + &mut bytes, + placement, + clipped, + format_code, + host_id, + placement_id, + ) { + return None; + } + displayed = true; + } else { + if cache.images.contains_key(&host_id) { + encode_delete_image(&mut bytes, host_id); + cache.placements.retain(|(id, _), _| *id != host_id); + cache.replayed_placements.retain(|(id, _)| *id != host_id); + } + if !encode_upload_image(&mut bytes, placement, format_code, host_id) { + return None; + } + } + cache.images.insert(host_id, image_signature); + } + + release_superseded_source_image(&mut bytes, cache, placement.source_key.clone(), host_id); + if !displayed && !placement_current { + encode_display_placement( + &mut bytes, + clipped, + host_id, + placement_id, + placement.placement.z, + ); + } + cache.placements.insert(key, placement_signature); + if cache.replay_placements { + cache.replayed_placements.insert(key); + } + Some(bytes) +} + +fn release_superseded_source_image( + bytes: &mut Vec<u8>, + cache: &mut HostGraphicsCache, + source: HostSourceKey, + host_id: u32, +) { + let Some(previous) = cache.sources.insert(source, host_id) else { + return; + }; + if previous == host_id || cache.sources.values().any(|id| *id == previous) { + return; + } + encode_delete_image(bytes, previous); + cache.images.remove(&previous); + cache.placements.retain(|(id, _), _| *id != previous); + cache.replayed_placements.retain(|(id, _)| *id != previous); +} + +#[cfg(test)] +fn drain_graphics_updates( + cache: &mut HostGraphicsCache, + placements: &[HostPlacement], + live: &HashSet<HostSourceKey>, +) -> Vec<u8> { + let mut bytes = Vec::new(); + loop { + let encoded = encode_graphics_update_incremental(cache, placements, live, None); + bytes.extend(encoded.bytes); + if !encoded.incomplete { + return bytes; + } + } +} + +#[cfg(test)] +fn encode_graphics_update( + bytes: &mut Vec<u8>, + placements: &[HostPlacement], + replay: bool, + images: &mut HashMap<u32, ImageSignature>, + host_placements: &mut HashMap<(u32, u32), PlacementSignature>, + sources: &mut HashMap<HostSourceKey, u32>, +) { + let mut cache = HostGraphicsCache { + images: std::mem::take(images), + placements: std::mem::take(host_placements), + sources: std::mem::take(sources), + ..HostGraphicsCache::default() + }; + let mut live = cache + .sources + .keys() + .filter(|source| matches!(source, HostSourceKey::PaneLayer { .. })) + .cloned() + .collect::<HashSet<_>>(); + live.extend( + placements + .iter() + .filter(|placement| matches!(placement.source_key, HostSourceKey::PaneLayer { .. })) + .map(|placement| placement.source_key.clone()), + ); + if live.is_empty() { + encode_terminal_graphics_update_legacy(bytes, placements, replay, &mut cache); + } else { + if replay { + cache.request_placement_replay(); + } + bytes.extend(drain_graphics_updates(&mut cache, placements, &live)); + } + *images = cache.images; + *host_placements = cache.placements; + *sources = cache.sources; +} + pub(crate) fn clear_all_host_graphics() -> io::Result<()> { let cache = LOCAL_HOST_GRAPHICS.get_or_init(|| Mutex::new(HostGraphicsCache::default())); let mut bytes = Vec::new(); @@ -473,10 +799,97 @@ pub(crate) fn clear_all_host_graphics() -> io::Result<()> { } impl HostGraphicsCache { + fn clear_pane_sources(&mut self) -> Vec<u8> { + let pane_sources = self + .sources + .keys() + .filter(|source| matches!(source, HostSourceKey::PaneLayer { .. })) + .cloned() + .collect::<Vec<_>>(); + let mut removed_images = HashSet::new(); + for source in pane_sources { + if let Some(image_id) = self.sources.remove(&source) { + removed_images.insert(image_id); + } + self.oversized.remove(&source); + } + + let mut bytes = Vec::new(); + for image_id in removed_images { + if self.sources.values().any(|id| *id == image_id) { + continue; + } + encode_delete_image(&mut bytes, image_id); + self.images.remove(&image_id); + self.placements.retain(|(id, _), _| *id != image_id); + self.replayed_placements.retain(|(id, _)| *id != image_id); + } + self.reset_incremental_state(); + bytes + } + + fn has_pane_sources(&self) -> bool { + self.sources + .keys() + .any(|source| matches!(source, HostSourceKey::PaneLayer { .. })) + } + + fn reset_incremental_state(&mut self) { + self.oversized.clear(); + self.continuation = None; + self.replay_placements = false; + self.replayed_placements.clear(); + } + + pub(crate) fn trust_pane_layer( + &mut self, + key: &crate::app::pane_graphics::Key, + host_id: u32, + layer: &crate::app::pane_graphics::Layer, + ) { + let source = HostSourceKey::PaneLayer { + pane_id: key.0, + layer_id: key.1.clone(), + }; + self.oversized.remove(&source); + self.sources.insert(source, host_id); + self.images + .insert(host_id, pane_layer_image_signature(layer)); + } + + pub(crate) fn forget_pane_layer(&mut self, key: &crate::app::pane_graphics::Key, host_id: u32) { + let source = HostSourceKey::PaneLayer { + pane_id: key.0, + layer_id: key.1.clone(), + }; + self.sources.remove(&source); + self.oversized.remove(&source); + self.images.remove(&host_id); + self.placements.retain(|(id, _), _| *id != host_id); + self.replayed_placements.retain(|(id, _)| *id != host_id); + } + pub(crate) fn is_empty(&self) -> bool { self.images.is_empty() && self.placements.is_empty() } + pub(crate) fn request_placement_replay(&mut self) { + if !self.replay_placements { + self.replay_placements = true; + self.replayed_placements.clear(); + } + } + + #[cfg(test)] + fn hide_except_live_pane_layers(&mut self, live: &HashSet<HostSourceKey>) -> Vec<u8> { + drain_graphics_updates(self, &[], live) + } + + #[cfg(test)] + pub(crate) fn test_image_count(&self) -> usize { + self.images.len() + } + #[cfg(test)] pub(crate) fn test_mark_non_empty(&mut self) { self.images.insert( @@ -499,15 +912,43 @@ impl HostGraphicsCache { self.images.clear(); self.placements.clear(); self.sources.clear(); + self.reset_incremental_state(); self.view = None; bytes } + pub(crate) fn clear_next(&mut self) -> EncodedGraphics { + self.continuation = None; + let mut bytes = Vec::new(); + if let Some(id) = self.images.keys().copied().min() { + encode_delete_image(&mut bytes, id); + self.images.remove(&id); + self.placements.retain(|(image, _), _| *image != id); + self.sources.retain(|_, image| *image != id); + self.replayed_placements.retain(|(image, _)| *image != id); + } else if let Some(key) = self.placements.keys().copied().min() { + encode_delete_placement(&mut bytes, key.0, key.1); + self.placements.remove(&key); + self.replayed_placements.remove(&key); + } else { + self.sources.clear(); + self.oversized.clear(); + self.view = None; + self.replay_placements = false; + self.replayed_placements.clear(); + } + EncodedGraphics { + bytes, + incomplete: !self.is_empty(), + } + } + fn update_view(&mut self, view_key: Option<HostViewKey>) -> bool { if self.view == view_key { return false; } self.view = view_key; + self.continuation = None; true } } @@ -523,6 +964,7 @@ fn active_view_key(app: &AppState) -> Option<HostViewKey> { fn collect_visible_placements( app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, terminal_runtimes: &TerminalRuntimeRegistry, surface: crate::ui::TabSurfaceView<'_>, cell_size: HostCellSize, @@ -553,9 +995,25 @@ fn collect_visible_placements( ); let mut placements = Vec::new(); for info in surface.pane_infos { - if let Some(layer) = app.pane_graphics_layers.get(&info.id) { + let mut pane_layers = graphics + .slots + .iter() + .filter_map(|((pane_id, layer_id), slot)| { + (*pane_id == info.id) + .then(|| { + slot.layer + .as_ref() + .map(|layer| (layer_id, slot.host_image_id, layer)) + }) + .flatten() + }) + .collect::<Vec<_>>(); + pane_layers.sort_by_key(|(layer_id, _, layer)| (layer.z_index, layer_id.as_str())); + for (layer_id, host_image_id, layer) in pane_layers { placements.push(pane_graphics_host_placement( info, + layer_id, + host_image_id, cell_size, layer, uploaded_images, @@ -582,6 +1040,7 @@ fn collect_visible_placements( .unwrap_or(0); placements.push(HostPlacement { pane_id: info.id, + host_image_id: None, area: info.inner_rect, cell_size, source_key: HostSourceKey::Terminal { @@ -602,25 +1061,19 @@ fn collect_visible_placements( fn pane_graphics_host_placement( info: &PaneInfo, + layer_id: &str, + host_id: u32, cell_size: HostCellSize, - layer: &crate::app::state::PaneGraphicsLayer, + layer: &crate::app::pane_graphics::Layer, uploaded_images: &HashMap<u32, ImageSignature>, include_data: bool, ) -> HostPlacement { let format = pane_graphics_kitty_format(layer.format); - let format_code = kitty_format_code(format); - let signature = ImageSignature { - image_width: layer.image_width, - image_height: layer.image_height, - format_code, - data_len: layer.data.len(), - data_fingerprint: layer.data_fingerprint, - }; - let host_id = pane_graphics_host_image_id(info.id, signature); + let signature = pane_layer_image_signature(layer); let data = if !include_data || uploaded_images.get(&host_id).copied() == Some(signature) { Vec::new() } else { - layer.data.clone() + layer.inline_data().map(<[u8]>::to_vec).unwrap_or_default() }; let render = layer.render; let grid_cols = if render.grid_cols == 0 { @@ -636,20 +1089,24 @@ fn pane_graphics_host_placement( HostPlacement { pane_id: info.id, + host_image_id: Some(host_id), area: info.inner_rect, cell_size, - source_key: HostSourceKey::PaneLayer { pane_id: info.id }, + source_key: HostSourceKey::PaneLayer { + pane_id: info.id, + layer_id: layer_id.to_owned(), + }, scrollback_offset: 0, placement: KittyImagePlacement { image_id: 1, placement_id: 1, - z: 0, + z: layer.z_index, x_offset: 0, y_offset: 0, image_width: layer.image_width, image_height: layer.image_height, format, - data_len: layer.data.len(), + data_len: layer.data_len(), data_fingerprint: layer.data_fingerprint, data, render: KittyPlacementRenderInfo { @@ -672,7 +1129,8 @@ fn pane_graphics_kitty_format(format: crate::api::schema::PaneGraphicsFormat) -> match format { crate::api::schema::PaneGraphicsFormat::Png => KittyImageFormat::Png, crate::api::schema::PaneGraphicsFormat::Rgb => KittyImageFormat::Rgb, - crate::api::schema::PaneGraphicsFormat::Rgba => KittyImageFormat::Rgba, + crate::api::schema::PaneGraphicsFormat::Rgba + | crate::api::schema::PaneGraphicsFormat::Bgra => KittyImageFormat::Rgba, } } @@ -697,20 +1155,14 @@ fn host_image_id_for_signature(pane_id: PaneId, signature: ImageSignature) -> u3 HOST_IMAGE_ID_BASE + ((hasher.finish() as u32) % 900_000) } -fn pane_graphics_host_image_id(pane_id: PaneId, signature: ImageSignature) -> u32 { - let mut hasher = DefaultHasher::new(); - pane_id.raw().hash(&mut hasher); - signature.hash(&mut hasher); - PANE_GRAPHICS_IMAGE_ID_BIT | ((hasher.finish() as u32) & !PANE_GRAPHICS_IMAGE_ID_BIT) -} - -fn host_placement_id(source_key: HostSourceKey, placement: &KittyImagePlacement) -> u32 { +fn host_placement_id(source_key: &HostSourceKey, placement: &KittyImagePlacement) -> u32 { let mut hasher = DefaultHasher::new(); match source_key { HostSourceKey::Terminal { pane_id, .. } => pane_id.raw().hash(&mut hasher), - HostSourceKey::PaneLayer { pane_id } => { + HostSourceKey::PaneLayer { pane_id, layer_id } => { "pane.graphics".hash(&mut hasher); pane_id.raw().hash(&mut hasher); + layer_id.hash(&mut hasher); } } placement.image_id.hash(&mut hasher); @@ -718,6 +1170,90 @@ fn host_placement_id(source_key: HostSourceKey, placement: &KittyImagePlacement) 1 + ((hasher.finish() as u32) % 900_000) } +pub(crate) struct DirectFileCommand { + pub(crate) leading: Vec<u8>, + pub(crate) control: String, +} + +pub(crate) fn prepare_direct_file( + app: &AppState, + graphics: &crate::app::pane_graphics::Runtime, + surface: crate::ui::TabSurfaceView<'_>, + cell_size: HostCellSize, + cache: &mut HostGraphicsCache, + key: &crate::app::pane_graphics::Key, +) -> Option<DirectFileCommand> { + if app.mode != Mode::Terminal || !cell_size.is_known() || app.active.is_none() { + return None; + } + let info = surface.pane_infos.iter().find(|info| info.id == key.0)?; + let slot = graphics.slots.get(key)?; + let layer = slot.layer.as_ref()?; + layer.direct_lease()?; + let placement = pane_graphics_host_placement( + info, + &key.1, + slot.host_image_id, + cell_size, + layer, + &cache.images, + false, + ); + let (command, clipped, format_code, placement_id) = + direct_file_command(&placement, slot.host_image_id)?; + cache + .images + .insert(slot.host_image_id, image_signature(&placement, format_code)); + cache.placements.insert( + (slot.host_image_id, placement_id), + placement_signature(clipped, layer.z_index, 0), + ); + cache + .sources + .insert(placement.source_key, slot.host_image_id); + Some(command) +} + +fn direct_file_command( + placement: &HostPlacement, + host_image_id: u32, +) -> Option<(DirectFileCommand, ClippedPlacement, u32, u32)> { + let (clipped, format_code) = clipped_placement(placement)?; + let placement_id = host_placement_id(&placement.source_key, &placement.placement); + let mut control = format!( + "a=T,f={format_code},s={},v={},i={host_image_id},p={placement_id},c={},r={},z={},C=1,q=0", + placement.placement.image_width, + placement.placement.image_height, + clipped.cols, + clipped.rows, + placement.placement.z, + ); + append_placement_controls(&mut control, clipped); + Some(( + DirectFileCommand { + leading: format!("\x1b[{};{}H", clipped.y + 1, clipped.x + 1).into_bytes(), + control, + }, + clipped, + format_code, + placement_id, + )) +} + +#[cfg(unix)] +pub(crate) fn encode_kitty_regular_file( + out: &mut Vec<u8>, + leading: &[u8], + control: &str, + path: &str, +) { + let payload = base64::engine::general_purpose::STANDARD.encode(path.as_bytes()); + out.extend_from_slice(b"\x1b7"); + out.extend_from_slice(leading); + let _ = write!(out, "\x1b_G{control},t=f;{payload}\x1b\\"); + out.extend_from_slice(b"\x1b8"); +} + fn encode_delete_image(out: &mut Vec<u8>, id: u32) { let _ = write!(out, "\x1b_Ga=d,d=I,i={id},q=2;\x1b\\"); } @@ -747,6 +1283,31 @@ fn encode_upload_image( true } +fn encode_transmit_and_display( + out: &mut Vec<u8>, + placement: &HostPlacement, + clipped: ClippedPlacement, + format_code: u32, + host_id: u32, + host_placement_id: u32, +) -> bool { + if placement.placement.data.is_empty() { + return false; + } + let _ = write!(out, "\x1b[{};{}H", clipped.y + 1, clipped.x + 1); + let mut control = format!( + "a=T,t=d,f={format_code},s={},v={},i={host_id},p={host_placement_id},c={},r={},z={},C=1,q=2", + placement.placement.image_width, + placement.placement.image_height, + clipped.cols, + clipped.rows, + placement.placement.z, + ); + append_placement_controls(&mut control, clipped); + encode_kitty_data(out, &control, &placement.placement.data); + true +} + fn encode_display_placement( out: &mut Vec<u8>, clipped: ClippedPlacement, @@ -759,6 +1320,11 @@ fn encode_display_placement( "a=p,i={host_id},p={host_placement_id},c={},r={},z={z},C=1,q=2", clipped.cols, clipped.rows, ); + append_placement_controls(&mut control, clipped); + let _ = write!(out, "\x1b_G{control};\x1b\\"); +} + +fn append_placement_controls(control: &mut String, clipped: ClippedPlacement) { if clipped.source_x > 0 { let _ = write!(control, ",x={}", clipped.source_x); } @@ -777,8 +1343,6 @@ fn encode_display_placement( if clipped.y_offset > 0 { let _ = write!(control, ",Y={}", clipped.y_offset); } - - let _ = write!(out, "\x1b_G{control};\x1b\\"); } fn clipped_placement(placement: &HostPlacement) -> Option<(ClippedPlacement, u32)> { @@ -929,6 +1493,16 @@ fn scale_pixels(value: u32, source: u32, dest: u32) -> u32 { ((value as u64).saturating_mul(source as u64) / dest.max(1) as u64).min(u32::MAX as u64) as u32 } +fn pane_layer_image_signature(layer: &crate::app::pane_graphics::Layer) -> ImageSignature { + ImageSignature { + image_width: layer.image_width, + image_height: layer.image_height, + format_code: kitty_format_code(pane_graphics_kitty_format(layer.format)), + data_len: layer.data_len(), + data_fingerprint: layer.data_fingerprint, + } +} + fn image_signature(placement: &HostPlacement, format_code: u32) -> ImageSignature { ImageSignature { image_width: placement.placement.image_width, @@ -1001,9 +1575,22 @@ fn encode_kitty_data(out: &mut Vec<u8>, control: &str, data: &[u8]) { mod tests { use super::*; + #[test] + fn fallback_cell_size_is_usable_only_for_nonempty_areas() { + assert_eq!( + HostCellSize::fallback_for_area(Rect::new(0, 0, 80, 24)), + HostCellSize { + width_px: 8, + height_px: 16, + } + ); + assert!(!HostCellSize::fallback_for_area(Rect::default()).is_known()); + } + fn test_placement(viewport_col: i32, viewport_row: i32) -> HostPlacement { HostPlacement { pane_id: PaneId::from_raw(1), + host_image_id: None, area: Rect::new(0, 0, 20, 10), cell_size: HostCellSize { width_px: 10, @@ -1046,10 +1633,70 @@ mod tests { let mut placement = test_placement(viewport_col, viewport_row); placement.source_key = HostSourceKey::PaneLayer { pane_id: placement.pane_id, + layer_id: "primary".into(), }; placement } + fn update( + cache: &mut HostGraphicsCache, + placements: &[HostPlacement], + replay: bool, + ) -> Vec<u8> { + let mut bytes = Vec::new(); + encode_graphics_update( + &mut bytes, + placements, + replay, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, + ); + bytes + } + + #[test] + fn terminal_graphics_without_pane_layers_preserves_legacy_transcript() { + fn record(transcript: &mut Vec<u8>, bytes: &[u8]) { + transcript.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + transcript.extend_from_slice(bytes); + } + + fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf29ce484222325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x100000001b3) + }) + } + + let mut cache = HostGraphicsCache::default(); + let mut transcript = Vec::new(); + + record( + &mut transcript, + &update(&mut cache, &[test_placement(0, 0)], false), + ); + record( + &mut transcript, + &update(&mut cache, &[test_placement(0, 0)], false), + ); + record( + &mut transcript, + &update(&mut cache, &[test_placement(-1, 2)], false), + ); + record( + &mut transcript, + &update(&mut cache, &[test_placement(-1, 2)], true), + ); + + let mut changed = test_placement(-1, 2); + changed.placement.data_fingerprint = 43; + record(&mut transcript, &update(&mut cache, &[changed], false)); + record(&mut transcript, &update(&mut cache, &[], false)); + + assert_eq!(transcript.len(), 10_084); + assert_eq!(fnv1a(&transcript), 0xc5bd_83e4_b039_870e); + } + #[test] fn terminal_placement_id_preserves_legacy_identity() { let placement = test_placement(0, 0); @@ -1060,13 +1707,14 @@ mod tests { let expected = 1 + ((legacy.finish() as u32) % 900_000); assert_eq!( - host_placement_id(placement.source_key, &placement.placement), + host_placement_id(&placement.source_key, &placement.placement), expected ); assert_ne!( host_placement_id( - HostSourceKey::PaneLayer { + &HostSourceKey::PaneLayer { pane_id: placement.pane_id, + layer_id: "primary".into(), }, &placement.placement, ), @@ -1074,15 +1722,60 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn regular_file_command_is_rgba_quiet_zero_and_path_encoded() { + let mut bytes = Vec::new(); + encode_kitty_regular_file( + &mut bytes, + b"\x1b[2;3H", + "a=T,f=32,s=3,v=2,i=42,p=7,c=3,r=2,z=0,C=1,q=0", + "/private/frame", + ); + let text = String::from_utf8(bytes).unwrap(); + assert!(text.starts_with("\x1b7\x1b[2;3H\x1b_Ga=T,f=32")); + assert!(text.contains(",C=1,q=0,t=f;L3ByaXZhdGUvZnJhbWU=")); + assert!(text.ends_with("\x1b\\\x1b8")); + } + + #[test] + fn direct_file_uses_one_clipped_transmit_and_display_at_the_final_position() { + let mut placement = pane_layer_placement(-1, 2); + placement.area = Rect::new(10, 4, 8, 6); + let (command, _, _, _) = direct_file_command(&placement, (1 << 31) | 9).unwrap(); + let control = command.control; + + assert_eq!(command.leading, b"\x1b[7;11H"); + assert!(control.starts_with("a=T,f=32,s=30,v=30,i=2147483657,p=")); + assert!(control.contains(",c=2,r=3,z=0,C=1,q=0,x=10,w=20,h=30")); + assert!(direct_file_command(&pane_layer_placement(30, 0), 9).is_none()); + } + #[test] fn pane_graphics_image_ids_are_disjoint_from_terminal_image_ids() { let placement = test_placement(0, 0); let signature = image_signature(&placement, kitty_format_code(placement.placement.format)); let terminal_id = host_image_id_for_signature(placement.pane_id, signature); - let pane_graphics_id = pane_graphics_host_image_id(placement.pane_id, signature); + let mut graphics = crate::app::pane_graphics::Runtime::default(); + let primary = (placement.pane_id, "primary".into()); + let pane_graphics_id = graphics.reserve_image_id(&primary).unwrap(); + graphics.slots.insert( + primary.clone(), + crate::app::pane_graphics::Slot::test(pane_graphics_id, None), + ); assert_eq!(terminal_id & PANE_GRAPHICS_IMAGE_ID_BIT, 0); assert_ne!(pane_graphics_id & PANE_GRAPHICS_IMAGE_ID_BIT, 0); + assert_eq!( + pane_graphics_id, + graphics.reserve_image_id(&primary).unwrap() + ); + assert_ne!( + pane_graphics_id, + graphics + .reserve_image_id(&(placement.pane_id, "toolbar".into())) + .unwrap() + ); } #[test] @@ -1121,16 +1814,19 @@ mod tests { borders: ratatui::widgets::Borders::NONE, is_focused: true, }; - let layer = crate::app::state::PaneGraphicsLayer::new( + let layer = crate::app::pane_graphics::Layer::inline( crate::api::schema::PaneGraphicsFormat::Rgba, 80, 30, vec![255; 80 * 30 * 4], crate::api::schema::PaneGraphicsPlacementParams::default(), + 0, ); let placement = pane_graphics_host_placement( &info, + "primary", + PANE_GRAPHICS_IMAGE_ID_BIT | 1, HostCellSize { width_px: 10, height_px: 10, @@ -1151,166 +1847,55 @@ mod tests { #[test] fn graphics_update_uploads_once_then_repositions_only() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); - - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut images, - &mut placements, - &mut sources, - ); - let first = String::from_utf8_lossy(&bytes); - assert!(first.contains("a=t")); - assert!(first.contains("a=p")); - - bytes.clear(); - let same = test_placement(0, 0); - encode_graphics_update( - &mut bytes, - &[same], - false, - &mut images, - &mut placements, - &mut sources, - ); - assert!(bytes.is_empty()); - - let mut z_changed = test_placement(0, 0); - z_changed.placement.z = 1; - encode_graphics_update( - &mut bytes, - &[z_changed], - false, - &mut images, - &mut placements, - &mut sources, - ); - let z_changed_bytes = String::from_utf8_lossy(&bytes); - assert!(!z_changed_bytes.contains("a=t")); - assert!(z_changed_bytes.contains("a=p")); + let mut cache = HostGraphicsCache::default(); + let first = update(&mut cache, &[test_placement(0, 0)], false); + assert!(String::from_utf8_lossy(&first).contains("a=t")); + assert!(String::from_utf8_lossy(&first).contains("a=p")); + assert!(update(&mut cache, &[test_placement(0, 0)], false).is_empty()); - bytes.clear(); - let moved = test_placement(0, 1); - encode_graphics_update( - &mut bytes, - &[moved], - false, - &mut images, - &mut placements, - &mut sources, - ); - let moved_bytes = String::from_utf8_lossy(&bytes); - assert!(!moved_bytes.contains("a=t")); - assert!(moved_bytes.contains("a=p")); + let mut changed = test_placement(0, 0); + changed.placement.z = 1; + for placement in [changed, test_placement(0, 1)] { + let bytes = update(&mut cache, &[placement], false); + assert!(!String::from_utf8_lossy(&bytes).contains("a=t")); + assert!(String::from_utf8_lossy(&bytes).contains("a=p")); + } } #[test] fn view_change_redisplays_unchanged_visible_placement() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); - - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut images, - &mut placements, - &mut sources, - ); - assert_eq!(placements.len(), 1); - - bytes.clear(); - let same = test_placement(0, 0); - encode_graphics_update( - &mut bytes, - &[same], - true, - &mut images, - &mut placements, - &mut sources, - ); - let redisplay = String::from_utf8_lossy(&bytes); - assert!(!redisplay.contains("a=t")); - assert!(redisplay.contains("a=p")); - assert_eq!(placements.len(), 1); + let mut cache = HostGraphicsCache::default(); + update(&mut cache, &[test_placement(0, 0)], false); + assert_eq!(cache.placements.len(), 1); + let bytes = update(&mut cache, &[test_placement(0, 0)], true); + assert!(!String::from_utf8_lossy(&bytes).contains("a=t")); + assert!(String::from_utf8_lossy(&bytes).contains("a=p")); + assert_eq!(cache.placements.len(), 1); } #[test] fn surface_reset_deletes_then_reuploads_and_redisplays_placement() { let mut cache = HostGraphicsCache::default(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); - - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut cache.images, - &mut cache.placements, - &mut cache.sources, - ); - assert_eq!(cache.images.len(), 1); - assert_eq!(cache.placements.len(), 1); - - bytes = cache.clear_bytes(); - let same = test_placement(0, 0); - encode_graphics_update( - &mut bytes, - &[same], - false, - &mut cache.images, - &mut cache.placements, - &mut cache.sources, - ); - + update(&mut cache, &[test_placement(0, 0)], false); + assert_eq!((cache.images.len(), cache.placements.len()), (1, 1)); + let mut bytes = cache.clear_bytes(); + bytes.extend(update(&mut cache, &[test_placement(0, 0)], false)); let redisplay = String::from_utf8_lossy(&bytes); assert!(redisplay.contains("a=d,d=I")); assert!(redisplay.contains("a=t")); assert!(redisplay.contains("a=p")); - assert_eq!(cache.images.len(), 1); - assert_eq!(cache.placements.len(), 1); + assert_eq!((cache.images.len(), cache.placements.len()), (1, 1)); } #[test] fn scrollback_offset_change_redisplays_placement() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); - - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut images, - &mut placements, - &mut sources, - ); - - bytes.clear(); + let mut cache = HostGraphicsCache::default(); + update(&mut cache, &[test_placement(0, 0)], false); let mut scrolled = test_placement(0, 0); scrolled.scrollback_offset = 3; - encode_graphics_update( - &mut bytes, - &[scrolled], - false, - &mut images, - &mut placements, - &mut sources, - ); - let redisplay = String::from_utf8_lossy(&bytes); - assert!(!redisplay.contains("a=t")); - assert!(redisplay.contains("a=p")); + let bytes = update(&mut cache, &[scrolled], false); + assert!(!String::from_utf8_lossy(&bytes).contains("a=t")); + assert!(String::from_utf8_lossy(&bytes).contains("a=p")); } #[test] @@ -1377,6 +1962,42 @@ mod tests { } #[test] + fn pane_layer_replacement_is_atomic_without_delete_to_blank() { + let mut images = HashMap::new(); + let mut placements = HashMap::new(); + let mut sources = HashMap::new(); + let mut bytes = Vec::new(); + let mut first = pane_layer_placement(0, 0); + first.host_image_id = Some((1 << 31) | 7); + encode_graphics_update( + &mut bytes, + &[first], + false, + &mut images, + &mut placements, + &mut sources, + ); + + bytes.clear(); + let mut changed = pane_layer_placement(0, 0); + changed.host_image_id = Some((1 << 31) | 7); + changed.placement.data_fingerprint += 1; + encode_graphics_update( + &mut bytes, + &[changed], + false, + &mut images, + &mut placements, + &mut sources, + ); + + let update = String::from_utf8_lossy(&bytes); + assert!(update.contains("a=T,t=d")); + assert!(update.contains(",p=") && update.contains(",C=1,q=2")); + assert!(!update.contains("a=d")); + } + + #[test] fn replaced_image_content_deletes_superseded_host_image() { let mut images = HashMap::new(); let mut placements = HashMap::new(); @@ -1555,40 +2176,143 @@ mod tests { } #[test] - fn removed_pane_layer_deletes_unreferenced_host_image() { + fn trusted_direct_image_uses_reserved_id_for_placement_without_upload() { + let key = (PaneId::from_raw(1), "primary".to_owned()); + let layer = crate::app::pane_graphics::Layer::inline( + crate::api::schema::PaneGraphicsFormat::Rgba, + 30, + 30, + vec![255; 30 * 30 * 4], + Default::default(), + 0, + ); + let reserved_id = (1 << 31) | 77; + let mut cache = HostGraphicsCache::default(); + cache.trust_pane_layer(&key, reserved_id, &layer); + let mut placement = pane_layer_placement(0, 0); + placement.host_image_id = Some(reserved_id); + placement.placement.data.clear(); + placement.placement.data_len = layer.data_len(); + placement.placement.data_fingerprint = layer.data_fingerprint; + let mut bytes = Vec::new(); + + encode_graphics_update( + &mut bytes, + &[placement], + false, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, + ); + + let update = String::from_utf8_lossy(&bytes); + assert!(update.contains(&format!("a=p,i={reserved_id}"))); + assert!(!update.contains("a=t")); + + let live = HashSet::from([HostSourceKey::PaneLayer { + pane_id: key.0, + layer_id: key.1.clone(), + }]); + let hidden = String::from_utf8(cache.hide_except_live_pane_layers(&live)).unwrap(); + assert!(hidden.contains("a=d,d=i")); + assert!(!hidden.contains("a=d,d=I")); + assert!(cache.images.contains_key(&reserved_id)); + assert!(cache.placements.is_empty()); + + let mut returning = pane_layer_placement(0, 0); + returning.host_image_id = Some(reserved_id); + returning.placement.data.clear(); + returning.placement.data_len = layer.data_len(); + returning.placement.data_fingerprint = layer.data_fingerprint; + let mut replay = Vec::new(); + encode_graphics_update( + &mut replay, + &[returning], + false, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, + ); + let replay = String::from_utf8(replay).unwrap(); + assert!(replay.contains(&format!("a=p,i={reserved_id}"))); + assert!(!replay.contains("a=t")); + + cache.forget_pane_layer(&key, reserved_id); + let mut fallback = pane_layer_placement(0, 0); + fallback.host_image_id = Some(reserved_id); + let mut retransmit = Vec::new(); + encode_graphics_update( + &mut retransmit, + &[fallback], + false, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, + ); + assert!(String::from_utf8_lossy(&retransmit).contains("a=t")); + } + + #[test] + fn hidden_layer_and_full_redraw_replay_placement_without_pixels() { let mut images = HashMap::new(); let mut placements = HashMap::new(); let mut sources = HashMap::new(); let mut bytes = Vec::new(); + let placement = pane_layer_placement(0, 0); encode_graphics_update( &mut bytes, - &[pane_layer_placement(0, 0)], + &[placement], false, &mut images, &mut placements, &mut sources, ); - let host_id = *images.keys().next().expect("uploaded pane layer"); - bytes.clear(); + for (visible, replay) in [(false, false), (true, false), (true, true)] { + bytes.clear(); + let current = visible.then(|| pane_layer_placement(0, 0)); + encode_graphics_update( + &mut bytes, + current.as_slice(), + replay, + &mut images, + &mut placements, + &mut sources, + ); + let update = String::from_utf8_lossy(&bytes); + assert!(!update.contains("a=t")); + assert!(!update.contains("a=d,d=I")); + assert_eq!(update.contains("a=p"), visible); + } + assert_eq!(images.len(), 1); + assert_eq!(sources.len(), 1); + } + + #[test] + fn removed_pane_layer_deletes_unreferenced_host_image() { + let mut cache = HostGraphicsCache::default(); + let mut bytes = Vec::new(); encode_graphics_update( &mut bytes, - &[], + &[pane_layer_placement(0, 0)], false, - &mut images, - &mut placements, - &mut sources, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, ); + let host_id = *cache.images.keys().next().expect("uploaded pane layer"); + + bytes = drain_graphics_updates(&mut cache, &[], &HashSet::new()); let delete = String::from_utf8_lossy(&bytes); assert!(delete.contains(&format!("a=d,d=I,i={host_id}"))); - assert!(images.is_empty()); - assert!(placements.is_empty()); - assert!(sources.is_empty()); + assert!(cache.images.is_empty()); + assert!(cache.placements.is_empty()); + assert!(cache.sources.is_empty()); } #[test] - fn clipped_pane_layer_deletes_unreferenced_host_image() { + fn hidden_pane_layer_retains_image_and_removes_only_placement() { let mut images = HashMap::new(); let mut placements = HashMap::new(); let mut sources = HashMap::new(); @@ -1613,11 +2337,12 @@ mod tests { &mut sources, ); - let delete = String::from_utf8_lossy(&bytes); - assert!(delete.contains(&format!("a=d,d=I,i={host_id}"))); - assert!(images.is_empty()); + let update = String::from_utf8_lossy(&bytes); + assert!(update.contains("a=d,d=i")); + assert!(!update.contains(&format!("a=d,d=I,i={host_id}"))); + assert_eq!(images.len(), 1); assert!(placements.is_empty()); - assert!(sources.is_empty()); + assert_eq!(sources.len(), 1); } #[test] @@ -1668,35 +2393,118 @@ mod tests { #[test] fn removed_pane_layer_preserves_image_shared_with_terminal_source() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); + let mut cache = HostGraphicsCache::default(); let mut bytes = Vec::new(); encode_graphics_update( &mut bytes, &[pane_layer_placement(0, 0), test_placement(4, 0)], false, - &mut images, - &mut placements, - &mut sources, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, ); - assert_eq!(images.len(), 1); + assert_eq!(cache.images.len(), 1); - bytes.clear(); + bytes = drain_graphics_updates(&mut cache, &[], &HashSet::new()); encode_graphics_update( &mut bytes, &[test_placement(4, 0)], false, - &mut images, - &mut placements, - &mut sources, + &mut cache.images, + &mut cache.placements, + &mut cache.sources, ); let update = String::from_utf8_lossy(&bytes); assert!(!update.contains("a=d,d=I")); - assert_eq!(images.len(), 1); - assert_eq!(placements.len(), 1); - assert_eq!(sources.len(), 1); + assert_eq!(cache.images.len(), 1); + assert_eq!(cache.placements.len(), 1); + assert_eq!(cache.sources.len(), 1); + } + + #[test] + fn changing_first_source_does_not_starve_second_source() { + let layers = |first| { + [(1, "a", first), (2, "b", 80)].map(|(id, name, fingerprint)| { + let mut placement = pane_layer_placement(0, 0); + placement.host_image_id = Some(PANE_GRAPHICS_IMAGE_ID_BIT | id); + placement.source_key = HostSourceKey::PaneLayer { + pane_id: placement.pane_id, + layer_id: name.into(), + }; + placement.placement.data_fingerprint = fingerprint; + placement + }) + }; + let initial = layers(42); + let live = initial.iter().map(|p| p.source_key.clone()).collect(); + let mut cache = HostGraphicsCache::default(); + assert!(encode_graphics_update_incremental(&mut cache, &initial, &live, None).incomplete); + assert!( + encode_graphics_update_incremental(&mut cache, &layers(43), &live, None).incomplete + ); + assert_eq!(cache.images.len(), 2, "second source uploaded next"); + + let terminal = |id| { + let mut placement = test_placement(0, 0); + placement.placement.image_id = id; + placement.placement.data_fingerprint = u64::from(id); + placement.source_key = HostSourceKey::Terminal { + pane_id: placement.pane_id, + image_id: id, + }; + placement + }; + let second = terminal(99).source_key; + let mut cache = HostGraphicsCache::default(); + for id in 1..=3 { + assert!( + encode_graphics_update_incremental( + &mut cache, + &[terminal(id), terminal(99)], + &HashSet::new(), + None, + ) + .incomplete + ); + } + assert!(cache.sources.contains_key(&second)); + } + + #[test] + fn large_terminal_image_is_local_but_quarantined_headless() { + let placements = || { + let mut large = test_placement(0, 0); + large.placement.data_len = 24 * 1024 * 1024; + let mut later = test_placement(4, 0); + later.placement.image_id = 8; + later.source_key = HostSourceKey::Terminal { + pane_id: later.pane_id, + image_id: 8, + }; + [large, later] + }; + for (budget, expected) in [ + (None, (true, 1, 0)), + (Some(HEADLESS_GRAPHICS_TRANSACTION_BUDGET), (false, 1, 1)), + ] { + let mut cache = HostGraphicsCache::default(); + let encoded = encode_graphics_update_incremental( + &mut cache, + &placements(), + &HashSet::new(), + budget, + ); + assert!(String::from_utf8_lossy(&encoded.bytes).contains("a=t")); + assert_eq!( + ( + encoded.incomplete, + cache.images.len(), + cache.oversized.len() + ), + expected + ); + } } #[test] @@ -1719,37 +2527,12 @@ mod tests { )); encode_display_placement(&mut encoded, clipped, host_id, 1, 0); - assert!(encoded.len() < crate::protocol::MAX_GRAPHICS_FRAME_SIZE); - } - - #[test] - fn view_change_deletes_stale_placement_immediately() { - let mut images = HashMap::new(); - let mut placements = HashMap::new(); - let mut sources = HashMap::new(); - let mut bytes = Vec::new(); - let placement = test_placement(0, 0); - - encode_graphics_update( - &mut bytes, - &[placement], - false, - &mut images, - &mut placements, - &mut sources, - ); - bytes.clear(); - encode_graphics_update( - &mut bytes, - &[], - true, - &mut images, - &mut placements, - &mut sources, - ); - - let delete = String::from_utf8_lossy(&bytes); - assert!(delete.contains("a=d,d=i")); - assert!(placements.is_empty()); + let mut framed = Vec::new(); + crate::protocol::write_message( + &mut framed, + &crate::protocol::ServerMessage::Graphics { bytes: encoded }, + ) + .unwrap(); + assert!(framed.len() <= crate::protocol::MAX_GRAPHICS_FRAME_SIZE + 4); } } |
