fix: keep offscreen kitty images loaded instead of resending them on scroll (#4686)

1 files changed, 211 insertions, 6 deletions

diff --git a/src/server/headless/tests/pane_graphics.rs b/src/server/headless/tests/pane_graphics.rs
index 190eaa80..1255f4eb 100644
--- a/src/server/headless/tests/pane_graphics.rs
+++ b/src/server/headless/tests/pane_graphics.rs
@@ -97,6 +97,73 @@ async fn first_kitty_image_updates_retained_surface_without_full_redraw() {
 }
 
 #[tokio::test]
+async fn offscreen_images_keep_text_updates_on_the_retained_path() {
+    let (mut server, _control_rx, client_rx, pane_id) = retained_test_server_with_control(
+        b"\x1b_Ga=T,f=32,t=d,i=7,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\",
+    );
+    let client = server.clients.get_mut(&1).unwrap();
+    client.mode = ClientConnectionMode::ClientShell;
+    client.render_state =
+        crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame);
+    client.cell_size = crate::kitty_graphics::HostCellSize {
+        width_px: 10,
+        height_px: 20,
+    };
+    let sources = HashSet::from([pane_id]);
+    server.render_and_stream();
+    let _ = receive_render(&client_rx, Duration::from_millis(100));
+
+    write_shared_test_pane(&mut server, pane_id, b"\x1b_Ga=d,d=a,q=2\x1b\\");
+    assert!(server.render_retained_pane_surface_and_stream(&sources));
+    let ServerMessage::PaneSurface(hidden) =
+        read_server_message(receive_render(&client_rx, Duration::from_millis(100)))
+    else {
+        panic!("expected hidden image scene");
+    };
+    assert!(hidden.graphics.placements.is_empty());
+    assert_eq!(hidden.graphics.retained_assets.len(), 1);
+
+    // Clients reject row patches while images are retained, so the text must
+    // arrive in a surface that keeps the hidden image without resending it.
+    write_shared_test_pane(&mut server, pane_id, b"\rtext while hidden");
+    assert!(server.render_retained_pane_surface_and_stream(&sources));
+    let ServerMessage::PaneSurface(text_update) =
+        read_server_message(receive_render(&client_rx, Duration::from_millis(100)))
+    else {
+        panic!("expected a surface the client accepts while images are retained");
+    };
+    assert!(frame_text(&text_update.frame).contains("text while hidden"));
+    assert!(text_update.graphics.placements.is_empty());
+    assert!(text_update.graphics.assets.is_empty());
+    assert_eq!(
+        text_update.graphics.retained_assets,
+        hidden.graphics.retained_assets
+    );
+
+    write_shared_test_pane(&mut server, pane_id, b"\x1b_Ga=p,i=7,c=1,r=1,q=2\x1b\\");
+    assert!(server.render_retained_pane_surface_and_stream(&sources));
+    let ServerMessage::PaneSurface(shown) =
+        read_server_message(receive_render(&client_rx, Duration::from_millis(100)))
+    else {
+        panic!("expected the hidden image to return");
+    };
+    assert_eq!(shown.graphics.placements.len(), 1);
+    assert!(shown.graphics.assets.is_empty());
+
+    write_shared_test_pane(&mut server, pane_id, b"\x1b_Ga=d,d=a,q=2\x1b\\");
+    assert!(server.render_retained_pane_surface_and_stream(&sources));
+    let _ = receive_render(&client_rx, Duration::from_millis(100));
+    write_shared_test_pane(&mut server, pane_id, b"\x1b_Ga=d,d=I,i=7,q=2\x1b\\");
+    assert!(server.render_retained_pane_surface_and_stream(&sources));
+    let ServerMessage::PaneSurface(deleted) =
+        read_server_message(receive_render(&client_rx, Duration::from_millis(100)))
+    else {
+        panic!("expected the deleted image to be released");
+    };
+    assert!(deleted.graphics.retained_assets.is_empty());
+}
+
+#[tokio::test]
 async fn retained_unicode_image_arrives_after_fragmented_upload_without_reupload() {
     let (mut server, _control_rx, client_rx, pane_id) =
         retained_test_server_with_control(b"\x1b[?1049h");
@@ -174,7 +241,7 @@ async fn retained_unicode_image_arrives_after_fragmented_upload_without_reupload
 async fn render_scale_profile_retained_graphics() {
     use ratatui::layout::Direction;
     for retained in [false, true] {
-        for with_image in [false, true] {
+        for image in ["none", "visible", "offscreen"] {
             for count in [1, 15] {
                 let (mut server, _control_rx, client_rx, root) =
                     retained_test_server_with_control(b"populated terminal\r\n");
@@ -205,7 +272,7 @@ async fn render_scale_profile_retained_graphics() {
                     width_px: 10,
                     height_px: 20,
                 };
-                if with_image {
+                if image != "none" {
                     write_shared_test_pane(
                         &mut server,
                         root,
@@ -214,6 +281,11 @@ async fn render_scale_profile_retained_graphics() {
                 }
                 server.render_and_stream();
                 let _ = receive_render(&client_rx, Duration::from_millis(100));
+                if image == "offscreen" {
+                    write_shared_test_pane(&mut server, root, b"\x1b_Ga=d,d=a,q=2\x1b\\");
+                    server.render_and_stream();
+                    let _ = receive_render(&client_rx, Duration::from_millis(100));
+                }
                 let sources = pane_ids.iter().copied().collect();
                 let mut samples = Vec::new();
                 for sample in 0..110 {
@@ -242,10 +314,10 @@ async fn render_scale_profile_retained_graphics() {
                 }
                 samples.sort_unstable();
                 println!(
-                "retained={retained} 80x24 panes={count} image={with_image} median_us={} p95_us={}",
-                samples[50].as_micros(),
-                samples[94].as_micros()
-            );
+                    "retained={retained} 80x24 panes={count} image={image} median_us={} p95_us={}",
+                    samples[50].as_micros(),
+                    samples[94].as_micros()
+                );
             }
         }
     }
@@ -322,6 +394,139 @@ async fn client_shell_delivers_equal_pixels_for_distinct_terminal_image_ids() {
     assert!(surface.graphics.assets.is_empty());
 }
 
+#[tokio::test]
+async fn client_shell_keeps_offscreen_terminal_image_loaded_until_the_image_is_deleted() {
+    let (mut server, _control_rx, client_rx, pane) = retained_test_server_with_control(
+        b"\x1b_Ga=t,f=32,t=d,i=7,s=1,v=1,q=2;/wAA/w==\x1b\\\x1b_Ga=p,i=7,c=1,r=1,q=2\x1b\\",
+    );
+    let client = server.clients.get_mut(&1).unwrap();
+    client.mode = ClientConnectionMode::ClientShell;
+    client.render_state =
+        crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame);
+    client.cell_size = crate::kitty_graphics::HostCellSize {
+        width_px: 10,
+        height_px: 20,
+    };
+    let render = |server: &mut HeadlessServer| {
+        server.render_and_stream();
+        let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
+        let ServerMessage::PaneSurface(surface) = message else {
+            panic!("expected client shell pane surface");
+        };
+        surface.graphics
+    };
+
+    let shown = render(&mut server);
+    assert_eq!(shown.placements.len(), 1);
+    assert_eq!(shown.assets.len(), 1);
+    let key = shown.assets[0].key.clone();
+
+    // Scrolling an image out of view removes its placement but keeps the image.
+    write_shared_test_pane(&mut server, pane, b"\x1b_Ga=d,d=a,q=2\x1b\\");
+    let hidden = render(&mut server);
+    assert!(hidden.placements.is_empty());
+    assert!(hidden.assets.is_empty());
+    assert_eq!(hidden.retained_assets, vec![key.clone()]);
+
+    write_shared_test_pane(&mut server, pane, b"\x1b_Ga=p,i=7,c=1,r=1,q=2\x1b\\");
+    let returned = render(&mut server);
+    assert_eq!(returned.placements.len(), 1);
+    assert_eq!(returned.placements[0].asset, key);
+    assert!(
+        returned.assets.is_empty(),
+        "an image the client still holds must not be sent again"
+    );
+
+    write_shared_test_pane(&mut server, pane, b"\x1b_Ga=d,d=a,q=2\x1b\\");
+    assert_eq!(render(&mut server).retained_assets, vec![key.clone()]);
+
+    // Replacing a hidden image with same-sized pixels releases the stale copy.
+    write_shared_test_pane(
+        &mut server,
+        pane,
+        b"\x1b_Ga=t,f=32,t=d,i=7,s=1,v=1,q=2;AAD//w==\x1b\\",
+    );
+    assert!(render(&mut server).retained_assets.is_empty());
+    write_shared_test_pane(&mut server, pane, b"\x1b_Ga=p,i=7,c=1,r=1,q=2\x1b\\");
+    let replaced_hidden = render(&mut server);
+    assert_eq!(replaced_hidden.assets.len(), 1);
+    assert_eq!(replaced_hidden.assets[0].data, vec![0, 0, 255, 255]);
+    write_shared_test_pane(&mut server, pane, b"\x1b_Ga=d,d=a,q=2\x1b\\");
+    assert_eq!(render(&mut server).retained_assets.len(), 1);
+
+    // Deleting a hidden image releases it on the host.
+    write_shared_test_pane(&mut server, pane, b"\x1b_Ga=d,d=I,i=7,q=2\x1b\\");
+    let deleted = render(&mut server);
+    assert!(deleted.placements.is_empty());
+    assert!(deleted.retained_assets.is_empty());
+
+    // Reusing the id for new pixels delivers them instead of the old image.
+    write_shared_test_pane(
+        &mut server,
+        pane,
+        b"\x1b_Ga=T,f=32,t=d,i=7,s=1,v=1,c=1,r=1,q=2;AP8A/w==\x1b\\",
+    );
+    let replaced = render(&mut server);
+    assert_eq!(replaced.assets.len(), 1);
+    assert_eq!(replaced.assets[0].data, vec![0, 255, 0, 255]);
+    assert_ne!(replaced.assets[0].key, key);
+}
+
+#[tokio::test]
+async fn client_shell_evicts_least_recently_visible_offscreen_terminal_images() {
+    let mut screen = Vec::new();
+    for image_id in 1..=17 {
+        screen.extend_from_slice(
+            format!("\x1b_Ga=T,f=32,t=d,i={image_id},s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\")
+                .as_bytes(),
+        );
+    }
+    let (mut server, _control_rx, client_rx, pane) = retained_test_server_with_control(&screen);
+    let client = server.clients.get_mut(&1).unwrap();
+    client.mode = ClientConnectionMode::ClientShell;
+    client.render_state =
+        crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame);
+    client.cell_size = crate::kitty_graphics::HostCellSize {
+        width_px: 10,
+        height_px: 20,
+    };
+    let render = |server: &mut HeadlessServer| {
+        server.render_and_stream();
+        let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100)));
+        let ServerMessage::PaneSurface(surface) = message else {
+            panic!("expected client shell pane surface");
+        };
+        surface.graphics
+    };
+    let retained_ids = |graphics: &crate::protocol::SurfaceGraphicsScene| {
+        graphics
+            .retained_assets
+            .iter()
+            .map(|key| match key.source {
+                crate::protocol::SurfaceGraphicsSource::Terminal { image_id, .. } => image_id,
+                _ => panic!("expected terminal image"),
+            })
+            .collect::<Vec<_>>()
+    };
+    assert_eq!(render(&mut server).assets.len(), 17);
+
+    write_shared_test_pane(&mut server, pane, b"\x1b_Ga=d,d=a,q=2\x1b\\");
+    let hidden = render(&mut server);
+    assert!(hidden.placements.is_empty());
+    assert_eq!(retained_ids(&hidden), (1..=16).collect::<Vec<_>>());
+
+    // Image 17 was evicted, so showing it sends its pixels again.
+    write_shared_test_pane(&mut server, pane, b"\x1b_Ga=p,i=17,c=1,r=1,q=2\x1b\\");
+    assert_eq!(render(&mut server).assets.len(), 1);
+
+    // The most recently visible image is kept first; the oldest is evicted.
+    write_shared_test_pane(&mut server, pane, b"\x1b_Ga=d,d=a,q=2\x1b\\");
+    let rehidden = render(&mut server);
+    let mut expected = vec![17];
+    expected.extend(1..=15);
+    assert_eq!(retained_ids(&rehidden), expected);
+}
+
 fn fill_render_lane(server: &HeadlessServer) {
     let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig)
         .expect("dummy frame");