From 4eceb75d8a899b35c28383501f4beca06c9f8a3d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 3 Jul 2026 20:49:15 -0300 Subject: fix: kitty graphics streams freeze on partial updates and leak host images (#948) * fix: request a render on kitty graphics writes pty output containing a kitty graphics sequence only scheduled the deferred settle render. that render's wake-up notifies the render loop only when the dirty flag was not already set, so under a continuous stream of image updates the notify can be dropped and the pane keeps showing an old frame until something else forces a full render. request an immediate render like any other output. the dirty flag and the minimum render interval still coalesce chunked writes, and the settle render stays as a backstop. refs #947 * fix: fingerprint kitty images exactly across retransmissions the image fingerprint hashed only the first, middle, and last 4096 bytes of pixel data. a program that streams updates by retransmitting one image id keeps the length and dimensions constant, so a frame whose changed pixels all fall outside those three windows gets the same fingerprint as the frame before it. the encoder then treats the image as already uploaded, emits no graphics bytes, and the frame is dropped as identical to the previous render. the pane stays stuck on the last uploaded image until a resize resets the graphics surface. hash the full payload instead. to keep that off the render hot path, cache the fingerprint per image id and recompute it only when the image's transmit time changes. libghostty-vt already refreshes the transmit time on every transmission; vendor patch 0002 exposes it through a new GHOSTTY_KITTY_IMAGE_DATA_TRANSMIT_TIME_NS accessor. a static image costs one map lookup per render and a streaming image one hash per transmitted frame. refs #947 * fix: delete superseded host kitty images host image ids derive from image content, so a pane image whose pixels change maps to a new host id on every update. nothing deleted the host image it replaced, and a streaming pane left one full-size image behind in the host terminal per changed frame until the graphics surface was reset. track which host image backs each pane image and delete the replaced one once no source references it. refs #947 * fix: prune stale kitty graphics source entries a source entry whose pane closed or whose placement disappeared stayed in the cache forever. through the shared-reference check it could keep an old host image alive that nothing visible used anymore, and the entries themselves accumulated without bound. prune sources against the visible set at the start of every encode pass. refs #947 * docs: link issue and upstream discussion for vendor patch 0002 the transmit time accessor now has a removal path: the ghostty discussion proposing it upstream is linked in the patch ledger. refs #947 --------- Co-authored-by: Can Celik --- vendor/libghostty-vt.patches.md | 42 ++++++++ .../include/ghostty/vt/kitty_graphics.h | 9 ++ .../src/terminal/c/kitty_graphics.zig | 55 ++++++++++ .../0002-expose-kitty-image-transmit-time-ns.patch | 112 +++++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 vendor/patches/libghostty-vt/0002-expose-kitty-image-transmit-time-ns.patch (limited to 'vendor') diff --git a/vendor/libghostty-vt.patches.md b/vendor/libghostty-vt.patches.md index 98b7569e..94bb998b 100644 --- a/vendor/libghostty-vt.patches.md +++ b/vendor/libghostty-vt.patches.md @@ -38,3 +38,45 @@ verification: zig build test-lib-vt -Demit-lib-vt -Doptimize=ReleaseSafe -Dtest-filter="resize shrinks both axes with cursor at bottom" zig build test-lib-vt -Demit-lib-vt -Doptimize=ReleaseSafe -Dtest-filter="PageList resize less rows and cols cursor at bottom" ``` + +## 0002 expose kitty image transmit time in the C API + +status: active + +patch: `vendor/patches/libghostty-vt/0002-expose-kitty-image-transmit-time-ns.patch` + +herdr issue: https://github.com/ogulcancelik/herdr/issues/947 + +upstream discussion: https://github.com/ghostty-org/ghostty/discussions/13177 +(proposes extending the kitty graphics inspection C API from +https://github.com/ghostty-org/ghostty/pull/12145, which has no transmit +time/serial accessor yet) + +introduced upstream: not yet + +vendored base: `0f7cd84b880b203c98683e520e84b9db0c5938d8` + +local files: + +- `vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h` +- `vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig` + +reason: herdr fingerprints kitty image data to decide when to re-encode an +image for render clients. Hashing the full payload on every render is too +expensive for multi-megabyte images, and sampling windows misses small +changes, freezing streaming sources. The image's transmit time already +refreshes on every (re)transmission, so exposing it as +`GHOSTTY_KITTY_IMAGE_DATA_TRANSMIT_TIME_NS` gives herdr an exact, O(1) change +serial to invalidate a cached full-data fingerprint. + +remove when: the vendored source commit exposes the image transmit time (or an +equivalent transmission serial) in the C API and +`ghostty::tests::kitty_image_fingerprint_refreshes_on_retransmission` passes +without this patch. + +verification: + +```sh +zig build test-lib-vt -Dtest-filter="image_get transmit_time_ns changes on retransmission" +cargo nextest run kitty_image_fingerprint +``` diff --git a/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h b/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h index 9bace3a3..d5b50067 100644 --- a/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h +++ b/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h @@ -343,6 +343,15 @@ typedef enum GHOSTTY_ENUM_TYPED { */ GHOSTTY_KITTY_IMAGE_DATA_DATA_LEN = 8, + /** + * Timestamp of the image's most recent transmission, in nanoseconds. + * The epoch is unspecified; only equality and ordering between values + * read from the same terminal are meaningful. + * + * Output type: uint64_t * + */ + GHOSTTY_KITTY_IMAGE_DATA_TRANSMIT_TIME_NS = 9, + GHOSTTY_KITTY_IMAGE_DATA_MAX_VALUE = GHOSTTY_ENUM_MAX_VALUE, } GhosttyKittyGraphicsImageData; diff --git a/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig b/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig index d50b8c4c..a00632ed 100644 --- a/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig +++ b/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig @@ -178,6 +178,7 @@ pub const ImageData = enum(c_int) { compression = 6, data_ptr = 7, data_len = 8, + transmit_time_ns = 9, pub fn OutType(comptime self: ImageData) type { return switch (self) { @@ -187,6 +188,7 @@ pub const ImageData = enum(c_int) { .compression => ImageCompression, .data_ptr => [*]const u8, .data_len => usize, + .transmit_time_ns => u64, }; } }; @@ -258,11 +260,22 @@ fn imageGetTyped( .compression => out.* = image.compression, .data_ptr => out.* = image.data.ptr, .data_len => out.* = image.data.len, + .transmit_time_ns => out.* = instantNanos(image.transmit_time), } return .success; } +/// Flattens an Instant's platform timestamp to nanoseconds for the C API. +/// The epoch is unspecified; only equality and ordering are meaningful. +fn instantNanos(instant: std.time.Instant) u64 { + if (@TypeOf(instant.timestamp) == u64) return instant.timestamp; + const ts = instant.timestamp; + const sec: u64 = @intCast(@max(ts.sec, 0)); + const nsec: u64 = @intCast(@max(ts.nsec, 0)); + return sec *| std.time.ns_per_s +| nsec; +} + pub fn placement_iterator_new( alloc_: ?*const CAllocator, out: *PlacementIterator, @@ -977,6 +990,48 @@ test "image_get_handle and image_get with transmitted image" { try testing.expect(data_len > 0); } +test "image_get transmit_time_ns changes on retransmission" { + if (comptime !build_options.kitty_graphics) return error.SkipZigTest; + + var t: terminal_c.Terminal = null; + try testing.expectEqual(Result.success, terminal_c.new( + &lib.alloc.test_allocator, + &t, + .{ .cols = 80, .rows = 24, .max_scrollback = 0 }, + )); + defer terminal_c.free(t); + + const cmd = "\x1b_Ga=T,t=d,f=24,i=1,p=1,s=1,v=2;////////\x1b\\"; + terminal_c.vt_write(t, cmd.ptr, cmd.len); + + var graphics: KittyGraphics = undefined; + try testing.expectEqual(Result.success, terminal_c.get( + t, + .kitty_graphics, + @ptrCast(&graphics), + )); + + var first: u64 = undefined; + try testing.expectEqual(Result.success, image_get( + image_get_handle(graphics, 1), + .transmit_time_ns, + @ptrCast(&first), + )); + try testing.expect(first > 0); + + std.Thread.sleep(1 * std.time.ns_per_ms); + const retransmit = "\x1b_Ga=t,t=d,f=24,i=1,s=1,v=2;AAAAAAAA\x1b\\"; + terminal_c.vt_write(t, retransmit.ptr, retransmit.len); + + var second: u64 = undefined; + try testing.expectEqual(Result.success, image_get( + image_get_handle(graphics, 1), + .transmit_time_ns, + @ptrCast(&second), + )); + try testing.expect(second > first); +} + test "placement_rect with transmit and display" { if (comptime !build_options.kitty_graphics) return error.SkipZigTest; diff --git a/vendor/patches/libghostty-vt/0002-expose-kitty-image-transmit-time-ns.patch b/vendor/patches/libghostty-vt/0002-expose-kitty-image-transmit-time-ns.patch new file mode 100644 index 00000000..4289740c --- /dev/null +++ b/vendor/patches/libghostty-vt/0002-expose-kitty-image-transmit-time-ns.patch @@ -0,0 +1,112 @@ +diff --git a/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h b/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h +index 9bace3a..d5b5006 100644 +--- a/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h ++++ b/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h +@@ -343,6 +343,15 @@ typedef enum GHOSTTY_ENUM_TYPED { + */ + GHOSTTY_KITTY_IMAGE_DATA_DATA_LEN = 8, + ++ /** ++ * Timestamp of the image's most recent transmission, in nanoseconds. ++ * The epoch is unspecified; only equality and ordering between values ++ * read from the same terminal are meaningful. ++ * ++ * Output type: uint64_t * ++ */ ++ GHOSTTY_KITTY_IMAGE_DATA_TRANSMIT_TIME_NS = 9, ++ + GHOSTTY_KITTY_IMAGE_DATA_MAX_VALUE = GHOSTTY_ENUM_MAX_VALUE, + } GhosttyKittyGraphicsImageData; + +diff --git a/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig b/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig +index d50b8c4..a00632e 100644 +--- a/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig ++++ b/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig +@@ -178,6 +178,7 @@ pub const ImageData = enum(c_int) { + compression = 6, + data_ptr = 7, + data_len = 8, ++ transmit_time_ns = 9, + + pub fn OutType(comptime self: ImageData) type { + return switch (self) { +@@ -187,6 +188,7 @@ pub const ImageData = enum(c_int) { + .compression => ImageCompression, + .data_ptr => [*]const u8, + .data_len => usize, ++ .transmit_time_ns => u64, + }; + } + }; +@@ -258,11 +260,22 @@ fn imageGetTyped( + .compression => out.* = image.compression, + .data_ptr => out.* = image.data.ptr, + .data_len => out.* = image.data.len, ++ .transmit_time_ns => out.* = instantNanos(image.transmit_time), + } + + return .success; + } + ++/// Flattens an Instant's platform timestamp to nanoseconds for the C API. ++/// The epoch is unspecified; only equality and ordering are meaningful. ++fn instantNanos(instant: std.time.Instant) u64 { ++ if (@TypeOf(instant.timestamp) == u64) return instant.timestamp; ++ const ts = instant.timestamp; ++ const sec: u64 = @intCast(@max(ts.sec, 0)); ++ const nsec: u64 = @intCast(@max(ts.nsec, 0)); ++ return sec *| std.time.ns_per_s +| nsec; ++} ++ + pub fn placement_iterator_new( + alloc_: ?*const CAllocator, + out: *PlacementIterator, +@@ -977,6 +990,48 @@ test "image_get_handle and image_get with transmitted image" { + try testing.expect(data_len > 0); + } + ++test "image_get transmit_time_ns changes on retransmission" { ++ if (comptime !build_options.kitty_graphics) return error.SkipZigTest; ++ ++ var t: terminal_c.Terminal = null; ++ try testing.expectEqual(Result.success, terminal_c.new( ++ &lib.alloc.test_allocator, ++ &t, ++ .{ .cols = 80, .rows = 24, .max_scrollback = 0 }, ++ )); ++ defer terminal_c.free(t); ++ ++ const cmd = "\x1b_Ga=T,t=d,f=24,i=1,p=1,s=1,v=2;////////\x1b\\"; ++ terminal_c.vt_write(t, cmd.ptr, cmd.len); ++ ++ var graphics: KittyGraphics = undefined; ++ try testing.expectEqual(Result.success, terminal_c.get( ++ t, ++ .kitty_graphics, ++ @ptrCast(&graphics), ++ )); ++ ++ var first: u64 = undefined; ++ try testing.expectEqual(Result.success, image_get( ++ image_get_handle(graphics, 1), ++ .transmit_time_ns, ++ @ptrCast(&first), ++ )); ++ try testing.expect(first > 0); ++ ++ std.Thread.sleep(1 * std.time.ns_per_ms); ++ const retransmit = "\x1b_Ga=t,t=d,f=24,i=1,s=1,v=2;AAAAAAAA\x1b\\"; ++ terminal_c.vt_write(t, retransmit.ptr, retransmit.len); ++ ++ var second: u64 = undefined; ++ try testing.expectEqual(Result.success, image_get( ++ image_get_handle(graphics, 1), ++ .transmit_time_ns, ++ @ptrCast(&second), ++ )); ++ try testing.expect(second > first); ++} ++ + test "placement_rect with transmit and display" { + if (comptime !build_options.kitty_graphics) return error.SkipZigTest; + -- cgit v1.2.3