diff options
77 files changed, 9152 insertions, 6239 deletions
diff --git a/docs/next/api/herdr-api.schema.json b/docs/next/api/herdr-api.schema.json index b4648b03..3151bff3 100644 --- a/docs/next/api/herdr-api.schema.json +++ b/docs/next/api/herdr-api.schema.json @@ -2598,110 +2598,6 @@ ], "type": "object" }, - "PaneGraphicsClearParams": { - "properties": { - "layer_id": { - "type": [ - "string", - "null" - ] - }, - "pane_id": { - "type": "string" - } - }, - "required": [ - "pane_id" - ], - "type": "object" - }, - "PaneGraphicsFormat": { - "enum": [ - "png", - "rgb", - "rgba", - "bgra" - ], - "type": "string" - }, - "PaneGraphicsPlacementParams": { - "properties": { - "grid_cols": { - "default": 0, - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "grid_rows": { - "default": 0, - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "viewport_col": { - "default": 0, - "format": "int32", - "type": "integer" - }, - "viewport_row": { - "default": 0, - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "PaneGraphicsSetParams": { - "properties": { - "data_base64": { - "default": "", - "type": "string" - }, - "format": { - "$ref": "#/schemas/request/$defs/PaneGraphicsFormat" - }, - "image_height": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "image_width": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "layer_id": { - "type": [ - "string", - "null" - ] - }, - "pane_id": { - "type": "string" - }, - "placement": { - "$ref": "#/schemas/request/$defs/PaneGraphicsPlacementParams", - "default": { - "grid_cols": 0, - "grid_rows": 0, - "viewport_col": 0, - "viewport_row": 0 - } - }, - "z_index": { - "default": 0, - "format": "int32", - "type": "integer" - } - }, - "required": [ - "pane_id", - "format", - "image_width", - "image_height" - ], - "type": "object" - }, "PaneInputSetParams": { "properties": { "pane_id": { @@ -6016,54 +5912,6 @@ { "properties": { "method": { - "const": "pane.graphics.set", - "type": "string" - }, - "params": { - "$ref": "#/schemas/request/$defs/PaneGraphicsSetParams" - } - }, - "required": [ - "method", - "params" - ], - "type": "object" - }, - { - "properties": { - "method": { - "const": "pane.graphics.clear", - "type": "string" - }, - "params": { - "$ref": "#/schemas/request/$defs/PaneGraphicsClearParams" - } - }, - "required": [ - "method", - "params" - ], - "type": "object" - }, - { - "properties": { - "method": { - "const": "pane.graphics.info", - "type": "string" - }, - "params": { - "$ref": "#/schemas/request/$defs/PaneTarget" - } - }, - "required": [ - "method", - "params" - ], - "type": "object" - }, - { - "properties": { - "method": { "const": "pane.report_agent", "type": "string" }, @@ -10029,108 +9877,6 @@ }, { "properties": { - "revision": { - "format": "uint64", - "minimum": 0, - "type": "integer" - }, - "sequence": { - "format": "uint64", - "minimum": 0, - "type": "integer" - }, - "type": { - "const": "pane_graphics_frame_ack", - "type": "string" - } - }, - "required": [ - "type", - "sequence", - "revision" - ], - "type": "object" - }, - { - "properties": { - "cell_height_px": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "cell_width_px": { - "format": "uint32", - "minimum": 0, - "type": "integer" - }, - "file_frame_damage": { - "default": false, - "description": "Accepts damage metadata while still consuming a complete canonical file.", - "type": "boolean" - }, - "file_frame_direct_max_bytes": { - "format": "uint", - "minimum": 0, - "type": [ - "integer", - "null" - ] - }, - "file_frame_directory": { - "type": [ - "string", - "null" - ] - }, - "file_frame_formats": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_frame_max_bytes": { - "format": "uint", - "minimum": 0, - "type": [ - "integer", - "null" - ] - }, - "file_frame_transport": { - "type": [ - "string", - "null" - ] - }, - "max_layers_per_pane": { - "default": 0, - "format": "uint", - "minimum": 0, - "type": "integer" - }, - "pane_visible": { - "description": "True only when this pane is on the currently rendered terminal surface.", - "type": "boolean" - }, - "pixel_mouse": { - "default": false, - "type": "boolean" - }, - "type": { - "const": "pane_graphics_info", - "type": "string" - } - }, - "required": [ - "type", - "cell_width_px", - "cell_height_px", - "pane_visible" - ], - "type": "object" - }, - { - "properties": { "explain": true, "type": { "const": "agent_explain", diff --git a/docs/next/website/src/content/docs/configuration.mdx b/docs/next/website/src/content/docs/configuration.mdx index 3dae7168..0d68a0f4 100644 --- a/docs/next/website/src/content/docs/configuration.mdx +++ b/docs/next/website/src/content/docs/configuration.mdx @@ -512,9 +512,19 @@ Search the [Config reference](/docs/config-reference/) for scrollback limits, ne ## Kitty graphics -Herdr renders pane images by default in compatible outer terminals. Popups, menus, and notifications temporarily hide only the image placements they overlap. Uncovered images stay visible, and hidden placements return when uncovered. Images do not dim with dialog backgrounds. +Applications and plugins integrate through the standard Kitty graphics protocol emitted in pane terminal output. Herdr renders these images by default in compatible outer terminals. The external socket pane-overlay API has been removed; there is no replacement socket image API. -To disable graphics rendering and the pane graphics API: +Popups, menus, and notifications temporarily hide only the image placements they overlap. Uncovered images stay visible, and hidden placements return when uncovered. Images do not dim with dialog backgrounds. + +Herdr automatically chooses eligible file transports, with no application changes or environment switches. On local Unix Ghostty clients, it can send image data through temporary files instead of encoding it into terminal output. A small file-consumption probe checks support first. Other terminals, SSH-launched clients, and nested multiplexers retain inline output for this last-hop optimization. + +For eligible local clients, native RGBA images travel from server to client as paths to Herdr-owned snapshots rather than pixel bytes. Unsupported clients and failed transfers fall back to inline delivery. Remote server paths are never forwarded to the local terminal. Herdr keeps image placement, clipping, and visibility under its control. + +On Linux, whole-file, uncompressed RGBA uploads can use copy-on-write snapshots before Herdr reads their pixels. This requires filesystem support for cloning into Herdr's private `/var/tmp` store. Unsupported filesystems, temporary-file uploads, shared memory, and other formats use the normal loader. Snapshots are limited to 16 MiB each and 64 MiB total; animation and unsupported clients read their pixels when needed. Herdr never relies on a mutable producer path after accepting an image. + +PNG uploads retain full decoding and validation. Herdr does not defer corrupt-image validation to the outer terminal. + +To disable Kitty graphics rendering: ```toml [terminal] @@ -523,7 +533,7 @@ kitty_graphics = false The legacy `experimental.kitty_graphics` setting remains accepted for existing configurations. `terminal.kitty_graphics` takes precedence when both are set. -Changing this setting requires restarting the affected Herdr server or reattaching the client. In remote sessions, the server's setting controls pane graphics parsing and API availability, while the local client's setting controls output to the outer terminal. +Changing this setting requires restarting the affected Herdr server or reattaching the client. In remote sessions, the server's setting controls Kitty graphics parsing, while the local client's setting controls output to the outer terminal. ## Agent session restore diff --git a/docs/next/website/src/content/docs/ja/socket-api.mdx b/docs/next/website/src/content/docs/ja/socket-api.mdx index 09532d5e..35a03625 100644 --- a/docs/next/website/src/content/docs/ja/socket-api.mdx +++ b/docs/next/website/src/content/docs/ja/socket-api.mdx @@ -99,7 +99,7 @@ herdr pane read w1:p2 --source recent --lines 50 | ワークスペース | `workspace.create`、`workspace.list`、`workspace.get`、`workspace.focus`、`workspace.rename`、`workspace.move`、`workspace.move_block`、`workspace.report_metadata`、`workspace.close` | | Worktree | `worktree.list`、`worktree.create`、`worktree.open`、`worktree.remove` | | タブ | `tab.create`、`tab.list`、`tab.get`、`tab.focus`、`tab.rename`、`tab.move`、`tab.close` | -| ペイン | `pane.split`、`pane.swap`、`pane.move`、`pane.zoom`、`pane.layout`、`pane.process_info`、`pane.neighbor`、`pane.edges`、`pane.focus_direction`、`pane.resize`、`pane.list`、`pane.current`、`pane.get`、`pane.rename`、`pane.send_text`、`pane.send_keys`、`pane.send_input`、`pane.read`、`pane.graphics.info`、`pane.graphics.set`、`pane.graphics.clear`、`pane.graphics.stream`、`pane.report_agent`、`pane.report_agent_session`、`pane.report_metadata`、`pane.clear_agent_authority`、`pane.release_agent`、`pane.close`、`pane.wait_for_output` | +| ペイン | `pane.split`、`pane.swap`、`pane.move`、`pane.zoom`、`pane.layout`、`pane.process_info`、`pane.neighbor`、`pane.edges`、`pane.focus_direction`、`pane.resize`、`pane.list`、`pane.current`、`pane.get`、`pane.rename`、`pane.send_text`、`pane.send_keys`、`pane.send_input`、`pane.read`、`pane.report_agent`、`pane.report_agent_session`、`pane.report_metadata`、`pane.clear_agent_authority`、`pane.release_agent`、`pane.close`、`pane.wait_for_output` | | ポップアップ | `popup.close` | | レイアウト | `layout.export`、`layout.apply`、`layout.set_split_ratio` | | エージェント | `agent.list`、`agent.get`、`agent.read`、`agent.explain`、`agent.send_keys`、`agent.prompt`、`agent.wait`、`agent.rename`、`agent.focus`、`agent.start` | @@ -147,20 +147,9 @@ CLI の `herdr api snapshot` は、クライアントやエージェントが簡 ### ペイングラフィックス -ペイングラフィックスを使うと、プラグインがペイン上に画像データを配置できます。デフォルトで利用でき、サーバーで `[terminal].kitty_graphics = false` が設定されている場合は、すべてのペイングラフィックスメソッドが `feature_disabled` を返します。この設定の変更にはサーバーの再起動が必要です。`pane.graphics.info` はグラフィックスレイヤーを作成せず、接続中クライアントのセルの幅と高さをピクセル単位で返します。`pane.graphics.set` は `data_base64` の `png`、`rgb`、`rgba` データを受け取り、`pane.graphics.clear` はレイヤーを削除します。 +アプリケーションとプラグインは、ペインのターミナル出力を通じて標準の Kitty グラフィックスプロトコルを送信してください。Herdr が対応する外側ターミナルで画像の配置、クリッピング、表示状態を管理します。 -```json -{"id":"graphics_info","method":"pane.graphics.info","params":{"pane_id":"w1:p1"}} -{"id":"graphics_set","method":"pane.graphics.set","params":{"pane_id":"w1:p1","format":"png","image_width":800,"image_height":600,"data_base64":"...","placement":{"viewport_col":0,"viewport_row":0,"grid_cols":80,"grid_rows":30}}} -{"id":"graphics_clear","method":"pane.graphics.clear","params":{"pane_id":"w1:p1"}} -``` - -フレームを繰り返し送る場合は、`pane.graphics.stream` で専用ソケットを開きます。Herdr が `ok` を返した後、フレームごとに 1 つの JSON ヘッダーと正確に `data_length` バイトの生データを送ります。ソケットが閉じるまでストリームがそのペインのグラフィックスレイヤーを所有し、同時に set、clear、stream を要求すると `stream_conflict` を返します。 - -```json -{"id":"graphics_stream","method":"pane.graphics.stream","params":{"pane_id":"w1:p1"}} -{"format":"png","image_width":800,"image_height":600,"data_length":12345,"placement":{"viewport_col":0,"viewport_row":0,"grid_cols":80,"grid_rows":30}} -``` +外部ソケットのペイン画像オーバーレイ API は削除されました。`pane.graphics.info`、`pane.graphics.set`、`pane.graphics.clear`、`pane.graphics.stream` は利用できず、代替のソケット画像 API もありません。旧サーバーとの内部通信プロトコルの互換性は、これらの公開メソッドを復活させるものではありません。 `pane.layout` は、`workspace_id`、`tab_id`、`zoomed`、外側の `area`、`focused_pane_id`、ペインの矩形、分割の矩形/比率を含むタブレイアウトのスナップショットを返します。`pane.neighbor` と `pane.edges` は同じレイアウトスナップショットを含むので、クライアントは非公開のレイアウト状態なしに次の判断を下せます。 diff --git a/docs/next/website/src/content/docs/socket-api.mdx b/docs/next/website/src/content/docs/socket-api.mdx index e6ae6a57..4572b5ab 100644 --- a/docs/next/website/src/content/docs/socket-api.mdx +++ b/docs/next/website/src/content/docs/socket-api.mdx @@ -103,7 +103,7 @@ Raw socket method names use dot notation: | Workspace | `workspace.create`, `workspace.list`, `workspace.get`, `workspace.focus`, `workspace.rename`, `workspace.move`, `workspace.move_block`, `workspace.report_metadata`, `workspace.close` | | Worktree | `worktree.list`, `worktree.create`, `worktree.open`, `worktree.remove` | | Tab | `tab.create`, `tab.list`, `tab.get`, `tab.focus`, `tab.rename`, `tab.move`, `tab.close` | -| Pane | `pane.split`, `pane.swap`, `pane.move`, `pane.zoom`, `pane.layout`, `pane.process_info`, `pane.neighbor`, `pane.edges`, `pane.focus_direction`, `pane.resize`, `pane.list`, `pane.current`, `pane.get`, `pane.rename`, `pane.send_text`, `pane.send_keys`, `pane.send_input`, `pane.read`, `pane.graphics.info`, `pane.graphics.set`, `pane.graphics.clear`, `pane.graphics.stream`, `pane.report_agent`, `pane.report_agent_session`, `pane.report_metadata`, `pane.clear_agent_authority`, `pane.release_agent`, `pane.close`, `pane.wait_for_output` | +| Pane | `pane.split`, `pane.swap`, `pane.move`, `pane.zoom`, `pane.layout`, `pane.process_info`, `pane.neighbor`, `pane.edges`, `pane.focus_direction`, `pane.resize`, `pane.list`, `pane.current`, `pane.get`, `pane.rename`, `pane.send_text`, `pane.send_keys`, `pane.send_input`, `pane.read`, `pane.report_agent`, `pane.report_agent_session`, `pane.report_metadata`, `pane.clear_agent_authority`, `pane.release_agent`, `pane.close`, `pane.wait_for_output` | | Popup | `popup.close` | | Layout | `layout.export`, `layout.apply`, `layout.set_split_ratio` | | Agent | `agent.list`, `agent.get`, `agent.read`, `agent.explain`, `agent.send_keys`, `agent.prompt`, `agent.wait`, `agent.rename`, `agent.focus`, `agent.start`, `agent.view.set`, `agent.view.clear` | @@ -176,63 +176,14 @@ Clients can treat `offset_from_bottom == 0` as at-bottom state. ### Pane graphics -Pane graphics let a plugin place image data over a pane. They are available by -default; when the server has `[terminal].kitty_graphics = false`, every pane -graphics method returns `feature_disabled`. Changing this setting requires a -server restart. Calling `pane.graphics.info` explicitly -activates capability discovery and returns the attached client's cell size, -file-frame options, pixel-mouse support, the 16-layer limit, and `pane_visible`. -`pane_visible` is true only when the target is in the active workspace and tab -and is not hidden by zoom. Short-lived UI modes do not change it. - -`pane.graphics.set`, `pane.graphics.clear`, and `pane.graphics.stream` accept an -optional `layer_id` (default `primary`). Set and stream also accept `z_index`; -layers are placed in stable `(z_index, layer_id)` order. Each stream exclusively owns its layer, and -closing it removes that layer. Inline frames accept `png`, `rgb`, `rgba`, or -`bgra`; BGRA is normalized once to owned RGBA. Herdr advances the host cache -one image transaction per render pass, so arbitrary layer sets progress without -an aggregate frame. Client transport keeps each transaction within its 32 MiB -wire limit. - -```json -{"id":"graphics_info","method":"pane.graphics.info","params":{"pane_id":"w1:p1"}} -{"id":"graphics_set","method":"pane.graphics.set","params":{"pane_id":"w1:p1","format":"png","image_width":800,"image_height":600,"data_base64":"...","placement":{"viewport_col":0,"viewport_row":0,"grid_cols":80,"grid_rows":30}}} -{"id":"graphics_clear","method":"pane.graphics.clear","params":{"pane_id":"w1:p1"}} -``` - -For repeated frames, open a dedicated socket with `pane.graphics.stream`. After -Herdr replies with `ok`, send one JSON header and then exactly `data_length` raw -bytes per inline frame. Concurrent operations on that layer return -`stream_conflict`. - -```json -{"id":"graphics_stream","method":"pane.graphics.stream","params":{"pane_id":"w1:p1","z_index":0}} -{"format":"png","image_width":800,"image_height":600,"data_length":12345,"placement":{"viewport_col":0,"viewport_row":0,"grid_cols":80,"grid_rows":30}} -``` - -When `pane.graphics.info` advertises `file_frame_transport: "direct-kitty"`, an -eligible local Ghostty, kitty, or WezTerm client may submit an immutable private -`rgba` or `bgra` file with `file.path`, `sequence`, and `revision`. Direct Kitty -file transport is reserved for the default `primary` page layer; named secondary -layers use owned inline RGBA. BGRA is always copied, swizzled, and rendered -inline. Herdr replies with a `pane_graphics_frame_ack` only after the terminal -accepts the file, or after a safe owned inline fallback is installed. Confirmed -file-transport failure disables direct files for that client connection without -disabling exact pixel mouse. A timeout or client loss closes the stream without -acknowledging source reuse. Clients that cannot negotiate direct file transport -remain on the owned inline fallback. - -Direct files are always complete canonical `width * height * 4` RGBA frames. -`file_frame_max_bytes` is the limit that remains eligible for owned inline fallback. -Primary-layer RGBA files may use the larger `file_frame_direct_max_bytes` limit when -`file_frame_transport` is available. Frames above the fallback limit are acknowledged -only when the terminal accepts the direct transfer; rejection closes the stream. If a -frame cannot use owned inline fallback while its pane is temporarily hidden or cannot be -placed during a redraw, Herdr uploads the image without displaying it and replays its -placement when the pane becomes visible again. -`file_frame_damage: true` means Herdr accepts optional damage metadata for -producer-side canonical-ring efficiency; it still copies or presents the full file. -Resize and full redraw replay placements without retransmitting pixels. +Applications and plugins should emit the standard Kitty graphics protocol through +their pane's terminal output. Herdr manages image placement, clipping, and +visibility in compatible outer terminals. + +The external socket pane-overlay API has been removed: `pane.graphics.info`, +`pane.graphics.set`, `pane.graphics.clear`, and `pane.graphics.stream` are no +longer available. There is no replacement socket image API. Internal wire +compatibility with older servers does not restore these public methods. `pane.layout` returns the tab layout snapshot with `workspace_id`, `tab_id`, `zoomed`, outer `area`, `focused_pane_id`, pane rects, and split rects/ratios. diff --git a/docs/next/website/src/content/docs/zh-cn/socket-api.mdx b/docs/next/website/src/content/docs/zh-cn/socket-api.mdx index 3aae7131..f947fc43 100644 --- a/docs/next/website/src/content/docs/zh-cn/socket-api.mdx +++ b/docs/next/website/src/content/docs/zh-cn/socket-api.mdx @@ -99,7 +99,7 @@ herdr pane read w1:p2 --source recent --lines 50 | 工作区 | `workspace.create`、`workspace.list`、`workspace.get`、`workspace.focus`、`workspace.rename`、`workspace.move`、`workspace.move_block`、`workspace.report_metadata`、`workspace.close` | | Worktree | `worktree.list`、`worktree.create`、`worktree.open`、`worktree.remove` | | 标签页 | `tab.create`、`tab.list`、`tab.get`、`tab.focus`、`tab.rename`、`tab.move`、`tab.close` | -| 窗格 | `pane.split`、`pane.swap`、`pane.move`、`pane.zoom`、`pane.layout`、`pane.process_info`、`pane.neighbor`、`pane.edges`、`pane.focus_direction`、`pane.resize`、`pane.list`、`pane.current`、`pane.get`、`pane.rename`、`pane.send_text`、`pane.send_keys`、`pane.send_input`、`pane.read`、`pane.graphics.info`、`pane.graphics.set`、`pane.graphics.clear`、`pane.graphics.stream`、`pane.report_agent`、`pane.report_agent_session`、`pane.report_metadata`、`pane.clear_agent_authority`、`pane.release_agent`、`pane.close`、`pane.wait_for_output` | +| 窗格 | `pane.split`、`pane.swap`、`pane.move`、`pane.zoom`、`pane.layout`、`pane.process_info`、`pane.neighbor`、`pane.edges`、`pane.focus_direction`、`pane.resize`、`pane.list`、`pane.current`、`pane.get`、`pane.rename`、`pane.send_text`、`pane.send_keys`、`pane.send_input`、`pane.read`、`pane.report_agent`、`pane.report_agent_session`、`pane.report_metadata`、`pane.clear_agent_authority`、`pane.release_agent`、`pane.close`、`pane.wait_for_output` | | 弹窗 | `popup.close` | | 布局 | `layout.export`、`layout.apply`、`layout.set_split_ratio` | | 智能体 | `agent.list`、`agent.get`、`agent.read`、`agent.explain`、`agent.send_keys`、`agent.prompt`、`agent.wait`、`agent.rename`、`agent.focus`、`agent.start` | @@ -147,20 +147,9 @@ CLI 的 `herdr api snapshot` 会把当前 `session.snapshot` 响应输出为 JSO ### 窗格图形 -窗格图形允许插件在窗格上放置图像数据。该功能默认可用；当服务器设置 `[terminal].kitty_graphics = false` 时，所有窗格图形方法都返回 `feature_disabled`。更改此设置需要重启服务器。调用 `pane.graphics.info` 会返回所连接客户端的单元格像素宽度和高度,但不会创建图形层。`pane.graphics.set` 接受 `data_base64` 中的 `png`、`rgb` 或 `rgba` 数据,`pane.graphics.clear` 删除该图层。 +应用和插件应通过窗格的终端输出发送标准 Kitty 图形协议。Herdr 在兼容的外层终端中管理图像的位置、裁剪和可见性。 -```json -{"id":"graphics_info","method":"pane.graphics.info","params":{"pane_id":"w1:p1"}} -{"id":"graphics_set","method":"pane.graphics.set","params":{"pane_id":"w1:p1","format":"png","image_width":800,"image_height":600,"data_base64":"...","placement":{"viewport_col":0,"viewport_row":0,"grid_cols":80,"grid_rows":30}}} -{"id":"graphics_clear","method":"pane.graphics.clear","params":{"pane_id":"w1:p1"}} -``` - -要重复发送帧,用 `pane.graphics.stream` 打开专用 socket。Herdr 回复 `ok` 后,每一帧发送一个 JSON 头,再发送恰好 `data_length` 字节的原始数据。socket 关闭前,该流拥有对应窗格的图形层;并发 set、clear 或 stream 请求返回 `stream_conflict`。 - -```json -{"id":"graphics_stream","method":"pane.graphics.stream","params":{"pane_id":"w1:p1"}} -{"format":"png","image_width":800,"image_height":600,"data_length":12345,"placement":{"viewport_col":0,"viewport_row":0,"grid_cols":80,"grid_rows":30}} -``` +外部 socket 窗格图像叠加 API 已移除:`pane.graphics.info`、`pane.graphics.set`、`pane.graphics.clear` 和 `pane.graphics.stream` 不再可用,也没有替代的 socket 图像 API。与旧服务器的内部传输协议兼容性不会恢复这些公开方法。 `pane.layout` 返回标签页布局快照,包含 `workspace_id`、`tab_id`、`zoomed`、外层 `area`、`focused_pane_id`、窗格矩形和分割矩形/比例。`pane.neighbor` 和 `pane.edges` 也包含同一份布局快照,让客户端不需要私有布局状态就能做出下一步决策。 diff --git a/docs/next/website/src/data/config-reference.json b/docs/next/website/src/data/config-reference.json index 76318e3a..299c2ced 100644 --- a/docs/next/website/src/data/config-reference.json +++ b/docs/next/website/src/data/config-reference.json @@ -439,7 +439,7 @@ "key": "terminal.kitty_graphics", "type": "boolean", "default": "true", - "description": "Render pane images in Kitty graphics-compatible outer terminals. Set false to disable graphics rendering and the pane graphics API. Restart the server or reattach the client after changing it. In remote sessions, server configuration controls parsing and API availability while local client configuration controls outer-terminal output." + "description": "Render pane images in Kitty graphics-compatible outer terminals. Applications use the standard Kitty graphics protocol through pane terminal output; the external socket pane-overlay API is no longer available. Set false to disable graphics rendering. Restart the server or reattach the client after changing it. In remote sessions, server configuration controls Kitty graphics parsing while local client configuration controls outer-terminal output." } ] }, diff --git a/src/api/mod.rs b/src/api/mod.rs index bd2983e4..86f90da8 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -64,13 +64,6 @@ pub(crate) fn request_changes_ui(request: &Request) -> bool { | Method::PaneFocus(_) | Method::PaneInputSet(_) | Method::PaneRename(_) - | Method::PaneGraphicsSet(_) - | Method::PaneGraphicsClear(_) - | Method::PaneGraphicsStream(_) - | Method::PaneGraphicsStreamSet(_) - | Method::PaneGraphicsStreamDirect(_) - | Method::PaneGraphicsStreamOpen(_) - | Method::PaneGraphicsStreamClose(_) | Method::PaneReportAgent(_) | Method::PaneReportAgentSession(_) | Method::PaneReportMetadata(_) @@ -91,7 +84,6 @@ pub struct ApiRequestMessage { pub request: Request, pub respond_to: std::sync::mpsc::Sender<String>, pub response_write_complete: Option<std::sync::mpsc::Receiver<()>>, - pub stream_active: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>, } pub type ApiRequestSender = mpsc::UnboundedSender<ApiRequestMessage>; diff --git a/src/api/schema.rs b/src/api/schema.rs index d7e1a950..439ed331 100644 --- a/src/api/schema.rs +++ b/src/api/schema.rs @@ -201,27 +201,6 @@ pub enum Method { PaneSendInput(PaneSendInputParams), #[serde(rename = "pane.read")] PaneRead(PaneReadParams), - #[serde(rename = "pane.graphics.set")] - PaneGraphicsSet(PaneGraphicsSetParams), - #[serde(rename = "pane.graphics.clear")] - PaneGraphicsClear(PaneGraphicsClearParams), - #[serde(rename = "pane.graphics.info")] - PaneGraphicsInfo(PaneTarget), - #[serde(rename = "pane.graphics.stream")] - #[schemars(skip)] - PaneGraphicsStream(PaneGraphicsStreamParams), - #[serde(skip)] - #[schemars(skip)] - PaneGraphicsStreamSet(PaneGraphicsSetParams), - #[serde(skip)] - #[schemars(skip)] - PaneGraphicsStreamDirect(PaneGraphicsDirectParams), - #[serde(skip)] - #[schemars(skip)] - PaneGraphicsStreamOpen(PaneGraphicsStreamParams), - #[serde(skip)] - #[schemars(skip)] - PaneGraphicsStreamClose(PaneGraphicsStreamParams), #[serde(rename = "pane.report_agent")] PaneReportAgent(PaneReportAgentParams), #[serde(rename = "pane.report_agent_session")] diff --git a/src/api/schema/panes.rs b/src/api/schema/panes.rs index 67d5185e..dfef31f8 100644 --- a/src/api/schema/panes.rs +++ b/src/api/schema/panes.rs @@ -2,14 +2,6 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -pub(crate) const PANE_GRAPHICS_SET_MAX_BYTES: usize = 512 * 1024; -pub(crate) const PANE_GRAPHICS_STREAM_MAX_BYTES: usize = 16 * 1024 * 1024; -pub(crate) const PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES: usize = 400 * 1024 * 1024; -pub(crate) const PANE_GRAPHICS_MAX_LAYERS_PER_PANE: usize = 16; -pub(crate) const PANE_GRAPHICS_MAX_LAYERS_TOTAL: usize = 64; -pub(crate) const PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL: usize = 64 * 1024 * 1024; -pub(crate) const PANE_GRAPHICS_PRIMARY_LAYER_ID: &str = "primary"; - use super::agents::AgentSessionInfo; use super::common::{AgentStatus, PaneAgentState, ReadFormat, ReadSource, SplitDirection}; @@ -366,84 +358,6 @@ pub struct PaneReadParams { pub(crate) intent: super::common::ReadIntent, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -#[serde(rename_all = "snake_case")] -pub enum PaneGraphicsFormat { - Png, - Rgb, - Rgba, - Bgra, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -pub struct PaneGraphicsSetParams { - pub pane_id: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub layer_id: Option<String>, - #[serde(default)] - pub z_index: i32, - #[serde(skip)] - #[schemars(skip)] - pub owner: String, - pub format: PaneGraphicsFormat, - pub image_width: u32, - pub image_height: u32, - #[serde(skip)] - pub data: Option<Vec<u8>>, - #[serde(default)] - pub data_base64: String, - #[serde(default)] - pub placement: PaneGraphicsPlacementParams, -} - -#[derive( - Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema, Default, -)] -pub struct PaneGraphicsPlacementParams { - #[serde(default)] - pub viewport_col: i32, - #[serde(default)] - pub viewport_row: i32, - #[serde(default)] - pub grid_cols: u32, - #[serde(default)] - pub grid_rows: u32, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -pub struct PaneGraphicsClearParams { - pub pane_id: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub layer_id: Option<String>, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PaneGraphicsDirectParams { - pub pane_id: String, - pub layer_id: Option<String>, - pub z_index: i32, - pub owner: String, - pub image_width: u32, - pub image_height: u32, - pub format: PaneGraphicsFormat, - pub path: String, - pub sequence: u64, - pub revision: u64, - pub placement: PaneGraphicsPlacementParams, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] -pub struct PaneGraphicsStreamParams { - pub pane_id: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub layer_id: Option<String>, - #[serde(default)] - pub z_index: i32, - #[serde(skip)] - #[schemars(skip)] - pub owner: String, -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)] pub struct PaneReportAgentParams { pub pane_id: String, diff --git a/src/api/schema/response.rs b/src/api/schema/response.rs index 5cc66bdc..e92e4952 100644 --- a/src/api/schema/response.rs +++ b/src/api/schema/response.rs @@ -181,33 +181,6 @@ pub enum ResponseResult { #[serde(default, skip_serializing_if = "Option::is_none")] current_global: Option<u64>, }, - PaneGraphicsFrameAck { - sequence: u64, - revision: u64, - }, - PaneGraphicsInfo { - cell_width_px: u32, - cell_height_px: u32, - /// True only when this pane is on the currently rendered terminal surface. - pane_visible: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - file_frame_directory: Option<String>, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - file_frame_formats: Vec<String>, - #[serde(default, skip_serializing_if = "Option::is_none")] - file_frame_max_bytes: Option<usize>, - #[serde(default, skip_serializing_if = "Option::is_none")] - file_frame_direct_max_bytes: Option<usize>, - /// Accepts damage metadata while still consuming a complete canonical file. - #[serde(default)] - file_frame_damage: bool, - #[serde(default)] - max_layers_per_pane: usize, - #[serde(default)] - pixel_mouse: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - file_frame_transport: Option<String>, - }, AgentExplain { explain: serde_json::Value, }, diff --git a/src/api/server.rs b/src/api/server.rs index edb778f7..30d92be4 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -22,8 +22,6 @@ use crate::ipc::{ socket_file_identity, LocalStream, LocalStreamRead, SocketFileIdentity, }; -mod pane_graphics_stream; - #[cfg(test)] mod subscription_socket_tests; @@ -162,6 +160,30 @@ fn start_server_inner( }) } +fn retired_pane_graphics_method_error(line: &str, id: &str) -> Option<ErrorResponse> { + #[derive(serde::Deserialize)] + struct RequestMethod { + method: String, + } + + let envelope = serde_json::from_str::<RequestMethod>(line).ok()?; + let method = envelope.method.as_str(); + if !matches!( + method, + "pane.graphics.info" | "pane.graphics.set" | "pane.graphics.clear" | "pane.graphics.stream" + ) { + return None; + } + + Some(ErrorResponse { + id: id.into(), + error: ErrorBody { + code: "unknown_method".into(), + message: format!("unknown method: {method}"), + }, + }) +} + fn prepare_socket_path(path: &Path) -> std::io::Result<()> { crate::ipc::prepare_socket_path(path, |path| { format!( @@ -233,16 +255,15 @@ fn handle_connection_with_stop( } else { String::new() }; - write_json_line_allow_disconnect( - &mut stream, - &ErrorResponse { + let response = + retired_pane_graphics_method_error(line, &id).unwrap_or_else(|| ErrorResponse { id, error: ErrorBody { code: "invalid_request".into(), message: format!("invalid request: {request_error}"), }, - }, - )?; + }); + write_json_line_allow_disconnect(&mut stream, &response)?; return Ok(()); } }; @@ -296,22 +317,6 @@ fn handle_connection_with_stop( } Ok(()) } - Method::PaneGraphicsStream(params) => { - let result = - pane_graphics_stream::serve(stream, request_id.clone(), params, api_tx, running); - match &result { - Ok(()) => crate::logging::api_request_completed( - &request_id, - method, - "stream_closed", - changes_ui, - ), - Err(err) => { - crate::logging::api_request_failed(&request_id, method, &err.to_string()) - } - } - result - } Method::EventsSubscribe(params) => { let result = stream_subscriptions( stream, @@ -478,7 +483,7 @@ fn handle_request( ); } - dispatch_to_app(request, api_tx, None, response_write_complete, None, None) + dispatch_to_app(request, api_tx, None, response_write_complete, None) } pub(crate) fn api_method_name(method: &Method) -> &'static str { @@ -561,14 +566,6 @@ pub(crate) fn api_method_name(method: &Method) -> &'static str { Method::PaneSendKeys(_) => "pane.send_keys", Method::PaneSendInput(_) => "pane.send_input", Method::PaneRead(_) => "pane.read", - Method::PaneGraphicsSet(_) => "pane.graphics.set", - Method::PaneGraphicsClear(_) => "pane.graphics.clear", - Method::PaneGraphicsInfo(_) => "pane.graphics.info", - Method::PaneGraphicsStream(_) => "pane.graphics.stream", - Method::PaneGraphicsStreamSet(_) => "pane.graphics.stream.set", - Method::PaneGraphicsStreamDirect(_) => "pane.graphics.stream.direct", - Method::PaneGraphicsStreamOpen(_) => "pane.graphics.stream.open", - Method::PaneGraphicsStreamClose(_) => "pane.graphics.stream.close", Method::PaneReportAgent(_) => "pane.report_agent", Method::PaneReportAgentSession(_) => "pane.report_agent_session", Method::PaneReportMetadata(_) => "pane.report_metadata", @@ -928,7 +925,7 @@ pub(super) fn dispatch_to_app_with_timeout( api_tx: &ApiRequestSender, timeout: Option<Duration>, ) -> String { - dispatch_to_app(request, api_tx, timeout, None, None, None) + dispatch_to_app(request, api_tx, timeout, None, None) } pub(super) fn dispatch_to_app_with_caller_timeout( @@ -941,55 +938,24 @@ pub(super) fn dispatch_to_app_with_caller_timeout( api_tx, timeout, None, - None, Some(("timeout", "timed out waiting for agent status")), ) } -pub(super) fn dispatch_stream_open( - request: Request, - api_tx: &ApiRequestSender, - timeout: Duration, - active: Arc<AtomicBool>, -) -> String { - dispatch_to_app(request, api_tx, Some(timeout), None, Some(active), None) -} - -pub(super) fn dispatch_stream_frame( - request: Request, - api_tx: &ApiRequestSender, - active: Arc<AtomicBool>, -) -> String { - dispatch_to_app( - request, - api_tx, - Some(crate::app::pane_graphics::DIRECT_OUTER_TIMEOUT), - None, - Some(active), - None, - ) -} - fn dispatch_to_app( request: Request, api_tx: &ApiRequestSender, timeout: Option<Duration>, response_write_complete: Option<std::sync::mpsc::Receiver<()>>, - stream_active: Option<Arc<AtomicBool>>, timeout_response: Option<(&str, &str)>, ) -> String { let request_id = request.id.clone(); - let request_active = stream_active.clone(); let (respond_to, response_rx) = std::sync::mpsc::channel(); if let Err(err) = api_tx.send(ApiRequestMessage { request, respond_to, response_write_complete, - stream_active, }) { - if let Some(active) = request_active { - active.store(false, Ordering::Release); - } return error_response_json( request_id, "server_unavailable", @@ -1019,9 +985,6 @@ fn dispatch_to_app( match response { Ok(response) => response, Err(err) => { - if let Some(active) = request_active { - active.store(false, Ordering::Release); - } if err.kind() == std::io::ErrorKind::TimedOut { if let Some((code, message)) = timeout_response { return error_response_json(request_id, code, message.into()); @@ -1300,6 +1263,93 @@ mod tests { } #[test] + fn removed_pane_graphics_methods_return_unknown_method_without_stream_upgrade() { + for method in [ + "pane.graphics.info", + "pane.graphics.set", + "pane.graphics.clear", + "pane.graphics.stream", + ] { + let (mut client, server, _path) = local_stream_pair("removed-pane-graphics"); + let (api_tx, mut api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>(); + writeln!( + client, + "{{\"id\":\"removed\",\"method\":\"{method}\",\"params\":{{}}}}" + ) + .unwrap(); + client.flush().unwrap(); + + handle_connection( + server, + &api_tx, + &EventHub::default(), + &Arc::new(AtomicBool::new(true)), + None, + ) + .unwrap(); + + let response = read_line(&mut client); + let response: serde_json::Value = serde_json::from_str(&response).unwrap(); + assert_eq!(response["id"], "removed", "{method}"); + assert_eq!(response["error"]["code"], "unknown_method", "{method}"); + assert!(response["error"]["message"] + .as_str() + .is_some_and(|message| message.contains(method))); + assert!(response.get("result").is_none(), "{method}"); + assert!(api_rx.try_recv().is_err(), "{method} reached the app"); + } + } + + #[test] + fn unrelated_unknown_method_retains_standard_invalid_request_response() { + let (mut client, server, _path) = local_stream_pair("unknown-api-request"); + let (api_tx, mut api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>(); + client + .write_all(b"{\"id\":\"unknown\",\"method\":\"nope\",\"params\":{}}\n") + .unwrap(); + client.flush().unwrap(); + + handle_connection( + server, + &api_tx, + &EventHub::default(), + &Arc::new(AtomicBool::new(true)), + None, + ) + .unwrap(); + + let response = read_line(&mut client); + let response: serde_json::Value = serde_json::from_str(&response).unwrap(); + assert_eq!(response["id"], "unknown"); + assert_eq!(response["error"]["code"], "invalid_request"); + assert!(api_rx.try_recv().is_err()); + } + + #[test] + fn ordinary_api_request_still_uses_normal_connection_path() { + let (mut client, server, _path) = local_stream_pair("ordinary-api-request"); + let (api_tx, _api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>(); + client + .write_all(b"{\"id\":\"ordinary\",\"method\":\"ping\",\"params\":{}}\n") + .unwrap(); + client.flush().unwrap(); + + handle_connection( + server, + &api_tx, + &EventHub::default(), + &Arc::new(AtomicBool::new(true)), + None, + ) + .unwrap(); + + let response = read_line(&mut client); + let response: serde_json::Value = serde_json::from_str(&response).unwrap(); + assert_eq!(response["id"], "ordinary"); + assert_eq!(response["result"]["type"], "pong"); + } + + #[test] fn ping_request_returns_pong() { let (tx, _rx) = mpsc::unbounded_channel(); let response = handle_request( @@ -1758,39 +1808,3 @@ mod tests { server_thread.join().unwrap(); } } - -#[cfg(test)] -mod pane_graphics_request_tests { - use super::*; - use base64::Engine as _; - - #[test] - fn maximum_public_graphics_request_fits_initial_json_line() { - let request = Request { - id: "graphics-max".into(), - method: Method::PaneGraphicsSet(crate::api::schema::PaneGraphicsSetParams { - pane_id: "pane_1".into(), - layer_id: None, - z_index: 0, - owner: String::new(), - format: crate::api::schema::PaneGraphicsFormat::Png, - image_width: 1, - image_height: 1, - data_base64: base64::engine::general_purpose::STANDARD - .encode(vec![1_u8; crate::api::schema::PANE_GRAPHICS_SET_MAX_BYTES]), - data: None, - placement: crate::api::schema::PaneGraphicsPlacementParams::default(), - }), - }; - let encoded = serde_json::to_vec(&request).unwrap(); - - assert!(encoded.len() < MAX_INITIAL_REQUEST_BYTES); - } - - #[test] - fn duplicate_method_cannot_be_reinterpreted_as_graphics_stream() { - let encoded = r#"{"id":"duplicate","method":"ping","method":"pane.graphics.stream","params":{"pane_id":"pane_1"}}"#; - - assert!(serde_json::from_str::<Request>(encoded).is_err()); - } -} diff --git a/src/api/server/pane_graphics_stream.rs b/src/api/server/pane_graphics_stream.rs deleted file mode 100644 index 55816b1d..00000000 --- a/src/api/server/pane_graphics_stream.rs +++ /dev/null @@ -1,1114 +0,0 @@ -use std::io::{self, Read}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use interprocess::local_socket::traits::Stream as _; - -use crate::api::schema::{ - ErrorBody, ErrorResponse, Method, PaneGraphicsSetParams, PaneGraphicsStreamParams, Request, - ResponseResult, SuccessResponse, -}; -use crate::api::ApiRequestSender; -use crate::ipc::{is_connection_closed_error, LocalStream}; - -use super::{ - api_response_outcome, dispatch_stream_frame, dispatch_stream_open, - dispatch_to_app_with_timeout, write_json_line, write_json_line_allow_disconnect, - write_text_line_allow_disconnect, APP_RESPONSE_TIMEOUT, CONNECTION_POLL_INTERVAL, -}; - -const MAX_STREAM_FRAME_HEADER_BYTES: usize = 64 * 1024; -const STREAM_FRAME_BODY_CHUNK_BYTES: usize = 64 * 1024; -const STREAM_FRAME_HEADER_IDLE_TIMEOUT: Duration = Duration::from_secs(5); -const STREAM_FRAME_HEADER_TIMEOUT: Duration = Duration::from_secs(30); -const STREAM_FRAME_BODY_IDLE_TIMEOUT: Duration = Duration::from_secs(5); -const STREAM_FRAME_BODY_TIMEOUT: Duration = Duration::from_secs(30); -const STREAM_FALLBACK_POLL_INTERVAL: Duration = Duration::from_millis(1); -const STREAM_FALLBACK_FAST_POLLS: u8 = 32; -static NEXT_PANE_GRAPHICS_STREAM_OWNER: AtomicU64 = AtomicU64::new(1); - -#[derive(serde::Deserialize)] -struct FrameFile { - path: String, -} - -#[derive(serde::Deserialize)] -struct FrameHeader { - format: crate::api::schema::PaneGraphicsFormat, - image_width: u32, - image_height: u32, - #[serde(default)] - data_length: Option<usize>, - #[serde(default)] - file: Option<FrameFile>, - #[serde(default)] - sequence: u64, - #[serde(default)] - revision: u64, - #[serde(default)] - placement: crate::api::schema::PaneGraphicsPlacementParams, -} - -#[derive(Clone, Copy)] -struct ReadTimeouts { - header_idle: Duration, - header_total: Duration, - body_idle: Duration, - body_total: Duration, -} - -const READ_TIMEOUTS: ReadTimeouts = ReadTimeouts { - header_idle: STREAM_FRAME_HEADER_IDLE_TIMEOUT, - header_total: STREAM_FRAME_HEADER_TIMEOUT, - body_idle: STREAM_FRAME_BODY_IDLE_TIMEOUT, - body_total: STREAM_FRAME_BODY_TIMEOUT, -}; - -pub(super) fn serve( - stream: LocalStream, - request_id: String, - params: PaneGraphicsStreamParams, - api_tx: &ApiRequestSender, - running: &Arc<AtomicBool>, -) -> std::io::Result<()> { - serve_with_open_timeout( - stream, - request_id, - params, - api_tx, - running, - APP_RESPONSE_TIMEOUT, - ) -} - -fn serve_with_open_timeout( - stream: LocalStream, - request_id: String, - params: PaneGraphicsStreamParams, - api_tx: &ApiRequestSender, - running: &Arc<AtomicBool>, - open_timeout: Duration, -) -> std::io::Result<()> { - serve_with_timeouts( - stream, - request_id, - params, - api_tx, - running, - open_timeout, - READ_TIMEOUTS, - ) -} - -fn serve_with_timeouts( - mut stream: LocalStream, - request_id: String, - mut params: PaneGraphicsStreamParams, - api_tx: &ApiRequestSender, - running: &Arc<AtomicBool>, - open_timeout: Duration, - read_timeouts: ReadTimeouts, -) -> std::io::Result<()> { - let pane_id = params.pane_id.clone(); - let layer_id = params.layer_id.clone(); - let z_index = params.z_index; - let owner = next_owner(); - params.owner = owner.clone(); - let stream_active = Arc::new(AtomicBool::new(true)); - let open_response = dispatch_stream_open( - Request { - id: request_id.clone(), - method: Method::PaneGraphicsStreamOpen(params), - }, - api_tx, - open_timeout, - Arc::clone(&stream_active), - ); - if api_response_outcome(&open_response) != "ok" { - stream_active.store(false, Ordering::Release); - let write_result = write_text_line_allow_disconnect(&mut stream, &open_response); - clear_layer(&pane_id, layer_id.as_deref(), z_index, &owner, api_tx); - write_result?; - return Ok(()); - } - - if let Err(err) = write_json_line( - &mut stream, - &SuccessResponse { - id: request_id.clone(), - result: ResponseResult::Ok {}, - }, - ) { - stream_active.store(false, Ordering::Release); - clear_layer(&pane_id, layer_id.as_deref(), z_index, &owner, api_tx); - if is_connection_closed_error(&err) { - return Ok(()); - } - return Err(err); - } - - let result = serve_frames( - &mut stream, - &request_id, - &owner, - &pane_id, - layer_id.as_deref(), - z_index, - api_tx, - running, - &stream_active, - read_timeouts, - ); - stream_active.store(false, Ordering::Release); - clear_layer(&pane_id, layer_id.as_deref(), z_index, &owner, api_tx); - result -} - -fn serve_frames( - stream: &mut LocalStream, - request_id: &str, - owner: &str, - pane_id: &str, - layer_id: Option<&str>, - z_index: i32, - api_tx: &ApiRequestSender, - running: &Arc<AtomicBool>, - stream_active: &Arc<AtomicBool>, - timeouts: ReadTimeouts, -) -> std::io::Result<()> { - let mut frame_seq = 0_u64; - while stream_is_running(running, stream_active) { - let Some(header_line) = read_line( - stream, - running, - stream_active, - MAX_STREAM_FRAME_HEADER_BYTES, - timeouts.header_idle, - timeouts.header_total, - )? - else { - return Ok(()); - }; - let header_line = header_line.trim(); - if header_line.is_empty() { - continue; - } - let header = match serde_json::from_str::<FrameHeader>(header_line) { - Ok(header) => header, - Err(err) => { - write_json_line_allow_disconnect( - stream, - &ErrorResponse { - id: request_id.to_string(), - error: ErrorBody { - code: "invalid_frame".into(), - message: format!("invalid frame header: {err}"), - }, - }, - )?; - return Ok(()); - } - }; - if let Some(file) = header.file { - if !matches!( - header.format, - crate::api::schema::PaneGraphicsFormat::Rgba - | crate::api::schema::PaneGraphicsFormat::Bgra - ) { - write_json_line_allow_disconnect( - stream, - &ErrorResponse { - id: request_id.to_string(), - error: ErrorBody { - code: "invalid_frame".into(), - message: "file frames require rgba or bgra".into(), - }, - }, - )?; - return Ok(()); - } - let response = dispatch_stream_frame( - Request { - id: format!("{request_id}:file:{}", header.sequence), - method: Method::PaneGraphicsStreamDirect( - crate::api::schema::PaneGraphicsDirectParams { - pane_id: pane_id.to_owned(), - layer_id: layer_id.map(str::to_owned), - z_index, - owner: owner.to_owned(), - image_width: header.image_width, - image_height: header.image_height, - format: header.format, - path: file.path, - sequence: header.sequence, - revision: header.revision, - placement: header.placement, - }, - ), - }, - api_tx, - Arc::clone(stream_active), - ); - write_text_line_allow_disconnect(stream, &response)?; - if api_response_outcome(&response) != "ok" { - return Ok(()); - } - continue; - } - - let Some(data_length) = header.data_length else { - write_json_line_allow_disconnect( - stream, - &ErrorResponse { - id: request_id.to_string(), - error: ErrorBody { - code: "invalid_frame".into(), - message: "frame requires data_length or file".into(), - }, - }, - )?; - return Ok(()); - }; - if data_length == 0 { - write_json_line_allow_disconnect( - stream, - &ErrorResponse { - id: request_id.to_string(), - error: ErrorBody { - code: "invalid_frame".into(), - message: "frame data_length must be greater than zero".into(), - }, - }, - )?; - return Ok(()); - } - if data_length > crate::api::schema::PANE_GRAPHICS_STREAM_MAX_BYTES { - write_json_line_allow_disconnect( - stream, - &ErrorResponse { - id: request_id.to_string(), - error: ErrorBody { - code: "image_too_large".into(), - message: "frame data is too large".into(), - }, - }, - )?; - return Ok(()); - } - - let Some(data) = read_exact( - stream, - data_length, - running, - stream_active, - timeouts.body_idle, - timeouts.body_total, - )? - else { - return Ok(()); - }; - - frame_seq = frame_seq.saturating_add(1); - let frame_id = format!("{request_id}:frame:{frame_seq}"); - let response = dispatch_to_app_with_timeout( - Request { - id: frame_id, - method: Method::PaneGraphicsStreamSet(PaneGraphicsSetParams { - pane_id: pane_id.to_string(), - layer_id: layer_id.map(str::to_owned), - z_index, - owner: owner.to_string(), - format: header.format, - image_width: header.image_width, - image_height: header.image_height, - data: Some(data), - data_base64: String::new(), - placement: header.placement, - }), - }, - api_tx, - Some(APP_RESPONSE_TIMEOUT), - ); - if api_response_outcome(&response) != "ok" { - write_text_line_allow_disconnect(stream, &response)?; - return Ok(()); - } - } - - Ok(()) -} - -fn stream_is_running(running: &AtomicBool, stream_active: &AtomicBool) -> bool { - running.load(Ordering::Relaxed) && stream_active.load(Ordering::Acquire) -} - -fn next_owner() -> String { - let id = NEXT_PANE_GRAPHICS_STREAM_OWNER.fetch_add(1, Ordering::Relaxed); - format!("pane.graphics.stream:{}:{id}", std::process::id()) -} - -fn clear_layer( - pane_id: &str, - layer_id: Option<&str>, - z_index: i32, - owner: &str, - api_tx: &ApiRequestSender, -) { - let _response = dispatch_to_app_with_timeout( - Request { - id: format!("pane.graphics.stream.clear:{pane_id}"), - method: Method::PaneGraphicsStreamClose(PaneGraphicsStreamParams { - pane_id: pane_id.to_string(), - layer_id: layer_id.map(str::to_owned), - z_index, - owner: owner.to_string(), - }), - }, - api_tx, - Some(APP_RESPONSE_TIMEOUT), - ); -} - -fn read_line( - stream: &mut LocalStream, - running: &Arc<AtomicBool>, - stream_active: &Arc<AtomicBool>, - max_bytes: usize, - idle_timeout: Duration, - total_timeout: Duration, -) -> std::io::Result<Option<String>> { - with_timed_reads(stream, |stream, mut wait| { - let mut bytes = Vec::new(); - let mut byte = [0_u8; 1]; - let mut total_deadline = None; - let mut idle_deadline = None; - - loop { - if !stream_is_running(running, stream_active) { - return Ok(None); - } - ensure_before_deadlines( - idle_deadline, - total_deadline, - "timed out reading stream frame header", - )?; - match wait.read(stream, &mut byte) { - Ok(0) => return Ok(None), - Ok(_) => { - wait.on_progress(); - let now = Instant::now(); - let total_deadline_at = - *total_deadline.get_or_insert_with(|| now + total_timeout); - idle_deadline = Some(now + idle_timeout); - if now >= total_deadline_at { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "timed out reading stream frame header", - )); - } - bytes.push(byte[0]); - if byte[0] == b'\n' { - return String::from_utf8(bytes) - .map(Some) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err)); - } - if bytes.len() > max_bytes { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "stream frame header is too large", - )); - } - } - Err(err) if read_should_retry(&err) => { - wait.after_retry(idle_deadline, total_deadline); - } - Err(err) if is_connection_closed_error(&err) => return Ok(None), - Err(err) => return Err(err), - } - } - }) -} - -fn read_exact( - stream: &mut LocalStream, - len: usize, - running: &Arc<AtomicBool>, - stream_active: &Arc<AtomicBool>, - idle_timeout: Duration, - total_timeout: Duration, -) -> std::io::Result<Option<Vec<u8>>> { - with_timed_reads(stream, |stream, mut wait| { - let mut data = Vec::new(); - let mut chunk = vec![0_u8; STREAM_FRAME_BODY_CHUNK_BYTES.min(len)]; - let total_deadline = Instant::now() + total_timeout; - let mut idle_deadline = Instant::now() + idle_timeout; - - while data.len() < len { - if !stream_is_running(running, stream_active) { - return Ok(None); - } - ensure_before_deadlines( - Some(idle_deadline), - Some(total_deadline), - "timed out reading stream frame body", - )?; - let remaining = len - data.len(); - let read_len = remaining.min(chunk.len()); - match wait.read(stream, &mut chunk[..read_len]) { - Ok(0) if data.is_empty() => return Ok(None), - Ok(0) => { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "stream ended mid-frame", - )); - } - Ok(n) => { - wait.on_progress(); - let now = Instant::now(); - if now >= total_deadline { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "timed out reading stream frame body", - )); - } - data.extend_from_slice(&chunk[..n]); - idle_deadline = now + idle_timeout; - } - Err(err) if read_should_retry(&err) => { - wait.after_retry(Some(idle_deadline), Some(total_deadline)); - } - Err(err) if is_connection_closed_error(&err) && data.is_empty() => return Ok(None), - Err(err) => return Err(err), - } - } - - Ok(Some(data)) - }) -} - -#[derive(Clone, Copy)] -enum ReadWait { - SocketTimeout, - Poll(PollBackoff), -} - -impl ReadWait { - fn read(&self, stream: &mut LocalStream, buffer: &mut [u8]) -> io::Result<usize> { - if matches!(self, Self::SocketTimeout) { - return stream.read(buffer); - } - match crate::ipc::poll_local_stream_read_count(stream, buffer)? { - crate::ipc::LocalStreamReadCount::Data(count) => Ok(count), - crate::ipc::LocalStreamReadCount::Pending => Err(io::ErrorKind::WouldBlock.into()), - crate::ipc::LocalStreamReadCount::Closed => Ok(0), - } - } - - fn after_retry(&mut self, idle_deadline: Option<Instant>, total_deadline: Option<Instant>) { - if let Self::Poll(backoff) = self { - sleep_until_poll(idle_deadline, total_deadline, backoff.interval); - backoff.advance(); - } - } - - fn on_progress(&mut self) { - if let Self::Poll(backoff) = self { - backoff.reset(); - } - } -} - -#[derive(Clone, Copy)] -struct PollBackoff { - interval: Duration, - fast_polls_remaining: u8, -} - -impl PollBackoff { - fn new() -> Self { - Self { - interval: STREAM_FALLBACK_POLL_INTERVAL, - fast_polls_remaining: STREAM_FALLBACK_FAST_POLLS, - } - } - - fn advance(&mut self) { - if self.fast_polls_remaining > 0 { - self.fast_polls_remaining -= 1; - return; - } - self.interval = (self.interval * 2).min(CONNECTION_POLL_INTERVAL); - } - - fn reset(&mut self) { - *self = Self::new(); - } -} - -fn with_timed_reads<T>( - stream: &mut LocalStream, - read: impl FnOnce(&mut LocalStream, ReadWait) -> std::io::Result<Option<T>>, -) -> std::io::Result<Option<T>> { - match stream.set_recv_timeout(Some(CONNECTION_POLL_INTERVAL)) { - Ok(()) => { - let result = read(stream, ReadWait::SocketTimeout); - finish_timed_read(result, || stream.set_recv_timeout(None)) - } - Err(err) if err.kind() == io::ErrorKind::Unsupported => { - crate::ipc::set_local_stream_polling(stream, true)?; - let result = read(stream, ReadWait::Poll(PollBackoff::new())); - finish_timed_read(result, || { - crate::ipc::set_local_stream_polling(stream, false) - }) - } - // A peer can disconnect after the caller's running check but before - // setsockopt. macOS reports that closed-socket race as EINVAL. - Err(err) if err.kind() == io::ErrorKind::InvalidInput => Ok(None), - Err(err) => Err(err), - } -} - -fn finish_timed_read<T>( - result: std::io::Result<Option<T>>, - reset: impl FnOnce() -> std::io::Result<()>, -) -> std::io::Result<Option<T>> { - match result { - // None is terminal for this dedicated stream. macOS returns EINVAL when - // socket options are restored after the peer has already disconnected. - Ok(None) => Ok(None), - Ok(value) => { - reset()?; - Ok(value) - } - Err(err) => { - let _ = reset(); - Err(err) - } - } -} - -fn ensure_before_deadlines( - idle_deadline: Option<Instant>, - total_deadline: Option<Instant>, - message: &str, -) -> std::io::Result<()> { - let now = Instant::now(); - if idle_deadline.is_some_and(|deadline| now >= deadline) - || total_deadline.is_some_and(|deadline| now >= deadline) - { - return Err(io::Error::new(io::ErrorKind::TimedOut, message)); - } - Ok(()) -} - -fn sleep_until_poll( - idle_deadline: Option<Instant>, - total_deadline: Option<Instant>, - poll_interval: Duration, -) { - let now = Instant::now(); - let until_deadline = [idle_deadline, total_deadline] - .into_iter() - .flatten() - .filter_map(|deadline| deadline.checked_duration_since(now)) - .min() - .unwrap_or(poll_interval); - std::thread::sleep(poll_interval.min(until_deadline)); -} - -fn read_should_retry(err: &io::Error) -> bool { - matches!( - err.kind(), - io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut | io::ErrorKind::Interrupted - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::api::schema::{ErrorResponse, Method, ResponseResult, SuccessResponse}; - use crate::api::ApiRequestMessage; - #[cfg(unix)] - use crate::api::EventHub; - use crate::ipc::LocalStream; - use interprocess::local_socket::traits::Listener as _; - use std::io::{BufRead, BufReader, Write}; - use std::path::PathBuf; - use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; - use std::sync::Arc; - use std::time::{Duration, Instant}; - use tokio::sync::mpsc; - - static NEXT_LOCAL_STREAM_ID: AtomicU64 = AtomicU64::new(1); - - fn local_stream_pair(_name: &str) -> (LocalStream, LocalStream, PathBuf) { - let unique = format!( - "hpg-{}-{}.sock", - std::process::id(), - NEXT_LOCAL_STREAM_ID.fetch_add(1, Ordering::Relaxed) - ); - let path = std::env::temp_dir().join(unique); - let listener = crate::ipc::bind_local_listener(&path).unwrap(); - let client = crate::ipc::connect_local_stream(&path).unwrap(); - let server = listener.accept().unwrap(); - (client, server, path) - } - - fn read_response_line(stream: &mut LocalStream) -> String { - let mut reader = BufReader::new(stream); - let mut line = String::new(); - reader.read_line(&mut line).unwrap(); - line - } - - #[cfg(unix)] - fn assert_server_stream_owner(owner: &str) { - assert!(owner.starts_with("pane.graphics.stream:")); - } - - fn respond_ok(message: ApiRequestMessage) { - let response = serde_json::to_string(&SuccessResponse { - id: message.request.id, - result: ResponseResult::Ok {}, - }) - .unwrap(); - message.respond_to.send(response).unwrap(); - } - - fn assert_close_and_respond(message: ApiRequestMessage, pane_id: &str, owner: &str) { - match &message.request.method { - Method::PaneGraphicsStreamClose(params) => { - assert_eq!(params.pane_id, pane_id); - assert_eq!(params.owner, owner); - } - other => panic!("unexpected close request: {other:?}"), - } - respond_ok(message); - } - - #[test] - fn browser_file_header_accepts_damage_but_keeps_full_canonical_frame() { - let header: FrameHeader = serde_json::from_str( - r#"{"format":"rgba","image_width":2,"image_height":3,"sequence":7,"revision":8,"file":{"path":"/private/frame"},"damage":{"x":1,"y":1,"width":1,"height":1},"transport":"direct-kitty","placement":{"grid_cols":2,"grid_rows":3}}"#, - ) - .unwrap(); - assert_eq!(header.data_length, None); - assert_eq!(header.file.unwrap().path, "/private/frame"); - assert_eq!((header.sequence, header.revision), (7, 8)); - assert_eq!((header.image_width, header.image_height), (2, 3)); - assert_eq!( - (header.placement.grid_cols, header.placement.grid_rows), - (2, 3) - ); - } - - #[test] - fn timed_read_skips_reset_after_stream_ends() { - let mut reset_called = false; - let result = finish_timed_read::<()>(Ok(None), || { - reset_called = true; - Ok(()) - }); - - assert!(result.unwrap().is_none()); - assert!(!reset_called); - } - - #[cfg(unix)] - #[test] - fn pane_graphics_stream_dispatches_binary_frames() { - let (api_tx, mut api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>(); - let (mut client, server, _path) = local_stream_pair("api-pane-graphics-stream"); - client - .write_all(br#"{"id":"stream_1","method":"pane.graphics.stream","params":{"pane_id":"pane_1"}}"#) - .unwrap(); - client.write_all(b"\n").unwrap(); - client.flush().unwrap(); - - let running = Arc::new(AtomicBool::new(true)); - let server_running = Arc::clone(&running); - let event_hub = EventHub::default(); - let server_thread = std::thread::spawn(move || { - super::super::handle_connection(server, &api_tx, &event_hub, &server_running, None) - }); - - let open = api_rx.blocking_recv().unwrap(); - let stream_owner = match &open.request.method { - Method::PaneGraphicsStreamOpen(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_server_stream_owner(¶ms.owner); - assert_ne!(params.owner, "stream_1"); - params.owner.clone() - } - other => panic!("unexpected open request: {other:?}"), - }; - respond_ok(open); - - let ack: SuccessResponse = serde_json::from_str(&read_response_line(&mut client)).unwrap(); - assert_eq!(ack.id, "stream_1"); - assert_eq!(ack.result, ResponseResult::Ok {}); - - client - .write_all( - br#"{"format":"png","image_width":2,"image_height":1,"data_length":4,"placement":{"grid_cols":10,"grid_rows":5}}"#, - ) - .unwrap(); - client.write_all(b"\n").unwrap(); - client.write_all(&[1_u8, 2, 3, 4]).unwrap(); - client.flush().unwrap(); - - let msg = api_rx.blocking_recv().unwrap(); - match &msg.request.method { - Method::PaneGraphicsStreamSet(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_eq!(params.owner, stream_owner); - assert_eq!(params.image_width, 2); - assert_eq!(params.image_height, 1); - assert_eq!(params.data.as_deref(), Some(&[1_u8, 2, 3, 4][..])); - assert!(params.data_base64.is_empty()); - assert_eq!(params.placement.grid_cols, 10); - assert_eq!(params.placement.grid_rows, 5); - } - other => panic!("unexpected request: {other:?}"), - } - respond_ok(msg); - - drop(client); - running.store(false, Ordering::Relaxed); - assert_close_and_respond(api_rx.blocking_recv().unwrap(), "pane_1", &stream_owner); - assert!(server_thread.join().unwrap().is_ok()); - } - - #[cfg(unix)] - #[test] - fn pane_graphics_stream_reports_open_errors_before_ack() { - let (api_tx, mut api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>(); - let (mut client, server, _path) = local_stream_pair("api-pane-graphics-stream-error"); - client - .write_all(br#"{"id":"stream_2","method":"pane.graphics.stream","params":{"pane_id":"pane_1"}}"#) - .unwrap(); - client.write_all(b"\n").unwrap(); - client.flush().unwrap(); - - let running = Arc::new(AtomicBool::new(true)); - let server_running = Arc::clone(&running); - let event_hub = EventHub::default(); - let server_thread = std::thread::spawn(move || { - super::super::handle_connection(server, &api_tx, &event_hub, &server_running, None) - }); - - let open = api_rx.blocking_recv().unwrap(); - let stream_owner = match &open.request.method { - Method::PaneGraphicsStreamOpen(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_server_stream_owner(¶ms.owner); - assert_ne!(params.owner, "stream_2"); - params.owner.clone() - } - other => panic!("unexpected open request: {other:?}"), - }; - open.respond_to - .send(super::super::error_response_json( - open.request.id, - "feature_disabled", - "pane graphics are disabled by terminal.kitty_graphics".into(), - )) - .unwrap(); - - let response: ErrorResponse = - serde_json::from_str(&read_response_line(&mut client)).unwrap(); - assert_eq!(response.id, "stream_2"); - assert_eq!(response.error.code, "feature_disabled"); - - assert_close_and_respond(api_rx.blocking_recv().unwrap(), "pane_1", &stream_owner); - - drop(client); - running.store(false, Ordering::Relaxed); - assert!(server_thread.join().unwrap().is_ok()); - } - - #[cfg(unix)] - #[test] - fn pane_graphics_stream_closes_claim_after_open_timeout() { - let (api_tx, mut api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>(); - let (mut client, server, _path) = local_stream_pair("api-pane-graphics-stream-timeout"); - let running = Arc::new(AtomicBool::new(true)); - let server_running = Arc::clone(&running); - let server_thread = std::thread::spawn(move || { - serve_with_open_timeout( - server, - "stream_timeout".into(), - PaneGraphicsStreamParams { - pane_id: "pane_1".into(), - layer_id: None, - z_index: 0, - owner: String::new(), - }, - &api_tx, - &server_running, - Duration::from_millis(10), - ) - }); - - let open = api_rx.blocking_recv().unwrap(); - let stream_owner = match &open.request.method { - Method::PaneGraphicsStreamOpen(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_server_stream_owner(¶ms.owner); - assert_ne!(params.owner, "stream_timeout"); - params.owner.clone() - } - other => panic!("unexpected open request: {other:?}"), - }; - - let response: ErrorResponse = - serde_json::from_str(&read_response_line(&mut client)).unwrap(); - assert_eq!(response.id, "stream_timeout"); - assert_eq!(response.error.code, "server_unavailable"); - assert!(response.error.message.contains("timed out")); - - assert_close_and_respond(api_rx.blocking_recv().unwrap(), "pane_1", &stream_owner); - - drop(open); - drop(client); - running.store(false, Ordering::Relaxed); - assert!(server_thread.join().unwrap().is_ok()); - } - - #[cfg(unix)] - #[test] - fn pane_graphics_stream_closes_claim_when_client_disconnects_before_ack() { - let (api_tx, mut api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>(); - let (mut client, server, _path) = - local_stream_pair("api-pane-graphics-stream-ack-disconnect"); - client - .write_all(br#"{"id":"stream_3","method":"pane.graphics.stream","params":{"pane_id":"pane_1"}}"#) - .unwrap(); - client.write_all(b"\n").unwrap(); - client.flush().unwrap(); - - let running = Arc::new(AtomicBool::new(true)); - let server_running = Arc::clone(&running); - let event_hub = EventHub::default(); - let server_thread = std::thread::spawn(move || { - super::super::handle_connection(server, &api_tx, &event_hub, &server_running, None) - }); - - let open = api_rx.blocking_recv().unwrap(); - let stream_owner = match &open.request.method { - Method::PaneGraphicsStreamOpen(params) => { - assert_eq!(params.pane_id, "pane_1"); - assert_server_stream_owner(¶ms.owner); - assert_ne!(params.owner, "stream_3"); - params.owner.clone() - } - other => panic!("unexpected open request: {other:?}"), - }; - - drop(client); - respond_ok(open); - - let (close_tx, close_rx) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - close_tx.send(api_rx.blocking_recv()).unwrap(); - }); - let close = close_rx - .recv_timeout(Duration::from_secs(2)) - .unwrap() - .unwrap(); - assert_close_and_respond(close, "pane_1", &stream_owner); - - running.store(false, Ordering::Relaxed); - assert!(server_thread.join().unwrap().is_ok()); - } - - #[test] - fn idle_graphics_stream_waits_for_header_without_timing_out() { - let (_client, mut server, _path) = local_stream_pair("graphics-idle-header"); - let running = Arc::new(AtomicBool::new(true)); - let active = Arc::new(AtomicBool::new(true)); - let stop = Arc::clone(&running); - let stopper = std::thread::spawn(move || { - std::thread::sleep(Duration::from_millis(30)); - stop.store(false, Ordering::Relaxed); - }); - - let result = read_line( - &mut server, - &running, - &active, - MAX_STREAM_FRAME_HEADER_BYTES, - Duration::from_millis(10), - Duration::from_millis(20), - ) - .unwrap(); - - assert!(result.is_none()); - stopper.join().unwrap(); - } - - #[test] - fn fallback_poll_backoff_preserves_fast_window_then_reaches_poll_ceiling() { - let mut backoff = PollBackoff::new(); - for _ in 0..STREAM_FALLBACK_FAST_POLLS { - backoff.advance(); - assert_eq!(backoff.interval, STREAM_FALLBACK_POLL_INTERVAL); - } - - backoff.advance(); - assert_eq!(backoff.interval, Duration::from_millis(2)); - for _ in 0..6 { - backoff.advance(); - } - assert_eq!(backoff.interval, CONNECTION_POLL_INTERVAL); - - backoff.reset(); - assert_eq!(backoff.interval, STREAM_FALLBACK_POLL_INTERVAL); - assert_eq!(backoff.fast_polls_remaining, STREAM_FALLBACK_FAST_POLLS); - } - - #[test] - fn partial_graphics_header_times_out_after_first_byte() { - let (mut client, mut server, _path) = local_stream_pair("graphics-partial-header"); - client.write_all(b"{").unwrap(); - client.flush().unwrap(); - let running = Arc::new(AtomicBool::new(true)); - let active = Arc::new(AtomicBool::new(true)); - - let error = read_line( - &mut server, - &running, - &active, - MAX_STREAM_FRAME_HEADER_BYTES, - Duration::from_millis(20), - Duration::from_millis(100), - ) - .unwrap_err(); - - assert_eq!(error.kind(), io::ErrorKind::TimedOut); - } - - #[test] - fn trickled_graphics_body_obeys_absolute_deadline() { - let (mut client, mut server, _path) = local_stream_pair("graphics-trickle-body"); - client.write_all(&[1_u8]).unwrap(); - client.flush().unwrap(); - let running = Arc::new(AtomicBool::new(true)); - let active = Arc::new(AtomicBool::new(true)); - let writer_running = Arc::clone(&running); - let writer = std::thread::spawn(move || { - while writer_running.load(Ordering::Relaxed) { - if client.write_all(&[1_u8]).is_err() { - break; - } - let _ = client.flush(); - std::thread::sleep(Duration::from_millis(5)); - } - }); - - let started = Instant::now(); - let error = read_exact( - &mut server, - 1024, - &running, - &active, - Duration::from_millis(20), - Duration::from_millis(60), - ) - .unwrap_err(); - - running.store(false, Ordering::Relaxed); - writer.join().unwrap(); - assert_eq!(error.kind(), io::ErrorKind::TimedOut); - assert!(started.elapsed() >= Duration::from_millis(50)); - } - - #[test] - fn timed_out_header_dispatches_owner_scoped_stream_close() { - let (mut client, server, _path) = local_stream_pair("graphics-timeout-close"); - let (api_tx, mut api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>(); - let running = Arc::new(AtomicBool::new(true)); - let server_running = Arc::clone(&running); - let server_thread = std::thread::spawn(move || { - serve_with_timeouts( - server, - "stream-timeout".into(), - PaneGraphicsStreamParams { - pane_id: "pane_1".into(), - layer_id: None, - z_index: 0, - owner: String::new(), - }, - &api_tx, - &server_running, - Duration::from_secs(1), - ReadTimeouts { - header_idle: Duration::from_millis(20), - header_total: Duration::from_millis(100), - body_idle: Duration::from_millis(20), - body_total: Duration::from_millis(100), - }, - ) - }); - - let open = api_rx.blocking_recv().unwrap(); - let owner = match &open.request.method { - Method::PaneGraphicsStreamOpen(params) => params.owner.clone(), - other => panic!("unexpected open request: {other:?}"), - }; - respond_ok(open); - let ack: SuccessResponse = serde_json::from_str(&read_response_line(&mut client)).unwrap(); - assert_eq!(ack.id, "stream-timeout"); - client.write_all(b"{").unwrap(); - client.flush().unwrap(); - - let (close_tx, close_rx) = std::sync::mpsc::channel(); - std::thread::spawn(move || close_tx.send(api_rx.blocking_recv()).unwrap()); - let close = close_rx - .recv_timeout(Duration::from_secs(1)) - .unwrap() - .unwrap(); - assert_close_and_respond(close, "pane_1", &owner); - - let error = server_thread.join().unwrap().unwrap_err(); - assert_eq!(error.kind(), io::ErrorKind::TimedOut); - } - - #[test] - fn oversized_stream_frame_is_rejected_before_body_or_app_dispatch() { - let (mut client, mut server, _path) = local_stream_pair("graphics-oversized-frame"); - let (api_tx, mut api_rx) = mpsc::unbounded_channel::<ApiRequestMessage>(); - let running = Arc::new(AtomicBool::new(true)); - let server_running = Arc::clone(&running); - let stream_active = Arc::new(AtomicBool::new(true)); - let server_thread = std::thread::spawn(move || { - serve_frames( - &mut server, - "stream-oversized", - "owner-1", - "pane_1", - None, - 0, - &api_tx, - &server_running, - &stream_active, - READ_TIMEOUTS, - ) - }); - let header = serde_json::json!({ - "format": "png", - "image_width": 1, - "image_height": 1, - "data_length": crate::api::schema::PANE_GRAPHICS_STREAM_MAX_BYTES + 1, - }); - client.write_all(format!("{header}\n").as_bytes()).unwrap(); - client.flush().unwrap(); - - let response: ErrorResponse = - serde_json::from_str(&read_response_line(&mut client)).unwrap(); - assert_eq!(response.error.code, "image_too_large"); - assert!(server_thread.join().unwrap().is_ok()); - assert!(api_rx.try_recv().is_err()); - } -} diff --git a/src/app/api.rs b/src/app/api.rs index c56a07a1..62c8a6b5 100644 --- a/src/app/api.rs +++ b/src/app/api.rs @@ -5,7 +5,6 @@ mod agents; mod env; mod integrations; mod layouts; -mod pane_graphics; mod panes; pub(crate) mod plugins; pub(super) mod responses; @@ -1152,34 +1151,6 @@ impl App { } Method::PaneRename(params) => return self.handle_pane_rename(request.id, params), Method::PaneRead(params) => return self.handle_pane_read(request.id, params), - Method::PaneGraphicsSet(params) => { - return self.handle_pane_graphics_set(request.id, params); - } - Method::PaneGraphicsClear(params) => { - return self.handle_pane_graphics_clear(request.id, params); - } - Method::PaneGraphicsInfo(params) => { - return self.handle_pane_graphics_info(request.id, params); - } - Method::PaneGraphicsStream(_) => { - return responses::encode_error( - request.id, - "stream_transport_required", - "pane.graphics.stream requires the streaming socket transport", - ); - } - Method::PaneGraphicsStreamSet(params) => { - return self.handle_pane_graphics_stream_set(request.id, params); - } - Method::PaneGraphicsStreamDirect(params) => { - return self.handle_pane_graphics_stream_direct(request.id, params); - } - Method::PaneGraphicsStreamOpen(params) => { - return self.handle_pane_graphics_stream_open(request.id, params); - } - Method::PaneGraphicsStreamClose(params) => { - return self.handle_pane_graphics_stream_close(request.id, params); - } Method::PaneReportAgent(params) => { return self.handle_pane_report_agent(request.id, params); } diff --git a/src/app/api/pane_graphics.rs b/src/app/api/pane_graphics.rs deleted file mode 100644 index 649282b5..00000000 --- a/src/app/api/pane_graphics.rs +++ /dev/null @@ -1,1284 +0,0 @@ -use base64::Engine; - -use crate::api::schema::{ - PaneGraphicsClearParams, PaneGraphicsSetParams, PaneGraphicsStreamParams, ResponseResult, - PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES, PANE_GRAPHICS_MAX_LAYERS_PER_PANE, - PANE_GRAPHICS_PRIMARY_LAYER_ID, PANE_GRAPHICS_SET_MAX_BYTES, PANE_GRAPHICS_STREAM_MAX_BYTES, -}; -use crate::app::pane_graphics::{Key as PaneGraphicsKey, Layer, Slot}; -use crate::app::App; -use crate::layout::PaneId; - -use super::responses::{encode_error, encode_success}; - -impl App { - /// Whether graphics for a pane can be placed on the surface rendered by this client. - /// - /// This deliberately follows the pane layout rather than pane existence: inactive - /// workspaces/tabs and panes hidden by zoom are not placeable. Short-lived UI modes do not - /// suspend the producer because the pane becomes visible again without a layout event. - fn pane_graphics_visible(&self, ws_idx: usize, pane_id: PaneId) -> bool { - self.state.pane_visible_on_active_surface(ws_idx, pane_id) - } - - pub(super) fn handle_pane_graphics_info( - &mut self, - id: String, - target: crate::api::schema::PaneTarget, - ) -> String { - if let Err(response) = require_enabled(self, &id) { - return response; - } - let Some((ws_idx, pane_id)) = self.parse_pane_id(&target.pane_id) else { - return pane_not_found(id, &target.pane_id); - }; - let pane_visible = self.pane_graphics_visible(ws_idx, pane_id); - if !self.state.host_cell_size.is_known() { - return encode_error(id, "cell_size_unavailable", "host cell size is unavailable"); - } - let file_frame_directory = self - .direct_graphics_available - .then(|| self.pane_graphics_files.source_directory().ok()) - .flatten(); - let direct = file_frame_directory.is_some(); - encode_success( - id, - ResponseResult::PaneGraphicsInfo { - cell_width_px: self.state.host_cell_size.width_px, - cell_height_px: self.state.host_cell_size.height_px, - pane_visible, - file_frame_directory: file_frame_directory - .map(|directory| directory.to_string_lossy().into_owned()), - file_frame_formats: if direct { - vec!["rgba".into(), "bgra".into()] - } else { - Vec::new() - }, - file_frame_max_bytes: direct.then_some(PANE_GRAPHICS_STREAM_MAX_BYTES), - file_frame_direct_max_bytes: direct.then_some(PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES), - // Damage metadata never changes the complete canonical frame contract. - file_frame_damage: true, - max_layers_per_pane: PANE_GRAPHICS_MAX_LAYERS_PER_PANE, - pixel_mouse: self.pixel_mouse_available, - file_frame_transport: direct.then(|| "direct-kitty".into()), - }, - ) - } - - pub(crate) fn attach_pane_graphics_stream_active( - &mut self, - params: &PaneGraphicsStreamParams, - active: std::sync::Arc<std::sync::atomic::AtomicBool>, - response: &str, - ) { - if serde_json::from_str::<crate::api::schema::SuccessResponse>(response).is_err() { - return; - } - let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { - return; - }; - let Ok(key) = graphics_key(pane_id, params.layer_id.as_deref()) else { - return; - }; - self.pane_graphics - .attach_stream_active(&key, ¶ms.owner, active); - } - - pub(super) fn handle_pane_graphics_set( - &mut self, - id: String, - params: PaneGraphicsSetParams, - ) -> String { - if let Err(response) = require_enabled(self, &id) { - return response; - } - let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { - return pane_not_found(id, ¶ms.pane_id); - }; - let key = match graphics_key(pane_id, params.layer_id.as_deref()) { - Ok(key) => key, - Err(message) => return encode_error(id, "invalid_layer_id", message), - }; - self.reclaim_inactive_stream(&key); - if self - .pane_graphics - .slots - .get(&key) - .is_some_and(|slot| slot.stream_owner.is_some()) - { - return encode_error( - id, - "stream_conflict", - "pane graphics layer has an active stream", - ); - } - self.set_layer(id, key, params) - } - - pub(super) fn handle_pane_graphics_clear( - &mut self, - id: String, - params: PaneGraphicsClearParams, - ) -> String { - if let Err(response) = require_enabled(self, &id) { - return response; - } - let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { - return pane_not_found(id, ¶ms.pane_id); - }; - let key = match graphics_key(pane_id, params.layer_id.as_deref()) { - Ok(key) => key, - Err(message) => return encode_error(id, "invalid_layer_id", message), - }; - self.reclaim_inactive_stream(&key); - if self - .pane_graphics - .slots - .get(&key) - .is_some_and(|slot| slot.stream_owner.is_some()) - { - return encode_error( - id, - "stream_conflict", - "pane graphics layer has an active stream", - ); - } - if self.pane_graphics.slots.remove(&key).is_some() { - self.pane_graphics.mark_changed(); - } - encode_success(id, ResponseResult::Ok {}) - } - - pub(super) fn handle_pane_graphics_stream_set( - &mut self, - id: String, - params: PaneGraphicsSetParams, - ) -> String { - if let Err(response) = require_enabled(self, &id) { - return response; - } - let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { - return pane_not_found(id, ¶ms.pane_id); - }; - let key = match graphics_key(pane_id, params.layer_id.as_deref()) { - Ok(key) => key, - Err(message) => return encode_error(id, "invalid_layer_id", message), - }; - match self.pane_graphics.slots.get(&key) { - Some(slot) - if slot.stream_owner.as_ref() == Some(¶ms.owner) && slot.stream_is_active() => - { - self.set_layer(id, key, params) - } - Some(slot) if slot.stream_owner.as_ref() == Some(¶ms.owner) => { - encode_error(id, "stream_closed", "pane graphics stream is not active") - } - Some(_) => encode_error( - id, - "stream_conflict", - "pane graphics stream owner does not match active stream", - ), - None => encode_error(id, "stream_closed", "pane graphics stream is not active"), - } - } - - pub(super) fn handle_pane_graphics_stream_open( - &mut self, - id: String, - params: PaneGraphicsStreamParams, - ) -> String { - if let Err(response) = require_enabled(self, &id) { - return response; - } - if params.owner.is_empty() { - return encode_error( - id, - "invalid_stream", - "pane graphics stream owner is required", - ); - } - let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { - return pane_not_found(id, ¶ms.pane_id); - }; - let key = match graphics_key(pane_id, params.layer_id.as_deref()) { - Ok(key) => key, - Err(message) => return encode_error(id, "invalid_layer_id", message), - }; - let stale = self - .pane_graphics - .slots - .get(&key) - .is_some_and(|slot| slot.stream_owner.is_some() && !slot.stream_is_active()); - if stale { - self.pane_graphics.slots.remove(&key); - } - if self - .pane_graphics - .slots - .get(&key) - .and_then(|slot| slot.stream_owner.as_ref()) - .is_some() - { - return encode_error( - id, - "stream_conflict", - "pane graphics layer has an active stream", - ); - } - let exists = self.pane_graphics.slots.contains_key(&key); - if !self.pane_graphics.can_add_slot(&key) - || (!exists - && self.pane_graphics.layer_count(pane_id) >= PANE_GRAPHICS_MAX_LAYERS_PER_PANE) - { - return encode_error(id, "layer_limit", "pane graphics layer limit reached"); - } - let Some(host_image_id) = self.pane_graphics.reserve_image_id(&key) else { - return encode_error(id, "layer_limit", "pane graphics image ids are exhausted"); - }; - self.pane_graphics.slots.insert( - key, - Slot { - host_image_id, - layer: None, - stream_owner: Some(params.owner), - stream_active: Some(std::sync::Arc::new(std::sync::atomic::AtomicBool::new( - true, - ))), - direct_gate: None, - }, - ); - self.pane_graphics.mark_changed(); - encode_success(id, ResponseResult::Ok {}) - } - - pub(super) fn handle_pane_graphics_stream_close( - &mut self, - id: String, - params: PaneGraphicsStreamParams, - ) -> String { - let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { - return pane_not_found(id, ¶ms.pane_id); - }; - if let Ok(key) = graphics_key(pane_id, params.layer_id.as_deref()) { - if self - .pane_graphics - .slots - .get(&key) - .and_then(|slot| slot.stream_owner.as_ref()) - .is_some_and(|owner| owner == ¶ms.owner) - { - self.pane_graphics.slots.remove(&key); - self.pane_graphics.mark_changed(); - } - } - encode_success(id, ResponseResult::Ok {}) - } - - pub(super) fn handle_pane_graphics_stream_direct( - &mut self, - id: String, - params: crate::api::schema::PaneGraphicsDirectParams, - ) -> String { - if let Err(response) = require_enabled(self, &id) { - return response; - } - let Some((_, pane_id)) = self.parse_pane_id(¶ms.pane_id) else { - return pane_not_found(id, ¶ms.pane_id); - }; - let key = match graphics_key(pane_id, params.layer_id.as_deref()) { - Ok(key) => key, - Err(message) => return encode_error(id, "invalid_layer_id", message), - }; - let owner_matches = self.pane_graphics.slots.get(&key).is_some_and(|slot| { - slot.stream_owner.as_deref() == Some(params.owner.as_str()) && slot.stream_is_active() - }); - if !owner_matches { - return encode_error(id, "stream_closed", "pane graphics stream is not active"); - } - if params.image_width == 0 || params.image_height == 0 { - return encode_error( - id, - "invalid_image", - "image_width and image_height must be greater than zero", - ); - } - if !matches!( - params.format, - crate::api::schema::PaneGraphicsFormat::Rgba - | crate::api::schema::PaneGraphicsFormat::Bgra - ) { - return encode_error(id, "invalid_image", "direct frames require rgba or bgra"); - } - let primary = key.1 == PANE_GRAPHICS_PRIMARY_LAYER_ID; - let direct = self.direct_graphics_available - && primary - && params.format == crate::api::schema::PaneGraphicsFormat::Rgba; - let max_bytes = if direct { - PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES - } else { - PANE_GRAPHICS_STREAM_MAX_BYTES - }; - let expected_len = - match expected_len(params.format, params.image_width, params.image_height) { - Ok(Some(len)) if len <= max_bytes => len, - _ => return encode_error(id, "invalid_image", "invalid direct RGBA dimensions"), - }; - let lease = match self - .pane_graphics_files - .lease(std::path::Path::new(¶ms.path), expected_len) - { - Ok(lease) => lease, - Err(err) => return encode_error(id, "invalid_frame_file", err.to_string()), - }; - if !direct && !self.pane_graphics.can_store_inline(&key, expected_len) { - return encode_error( - id, - "graphics_budget_exceeded", - "pane graphics inline memory limit reached", - ); - } - let layer = if direct { - Layer::direct( - params.image_width, - params.image_height, - lease, - params.placement, - params.z_index, - ) - } else { - let mut data = match lease.copy_rgba() { - Ok(data) => data, - Err(err) => return encode_error(id, "invalid_frame_file", err.to_string()), - }; - canonicalize_bgra(params.format, &mut data); - Layer::inline( - crate::api::schema::PaneGraphicsFormat::Rgba, - params.image_width, - params.image_height, - data, - params.placement, - params.z_index, - ) - }; - let Some(slot) = self.pane_graphics.slots.get_mut(&key) else { - return encode_error(id, "stream_closed", "pane graphics stream is not active"); - }; - if !slot.stream_is_active() { - return encode_error(id, "stream_closed", "pane graphics stream is not active"); - } - slot.layer = Some(layer); - slot.direct_gate = None; - self.pane_graphics.mark_changed(); - encode_success( - id, - ResponseResult::PaneGraphicsFrameAck { - sequence: params.sequence, - revision: params.revision, - }, - ) - } - - fn set_layer( - &mut self, - id: String, - key: PaneGraphicsKey, - params: PaneGraphicsSetParams, - ) -> String { - if !self.pane_graphics.can_add_slot(&key) - || (!self.pane_graphics.slots.contains_key(&key) - && self.pane_graphics.layer_count(key.0) >= PANE_GRAPHICS_MAX_LAYERS_PER_PANE) - { - return encode_error(id, "layer_limit", "pane graphics layer limit reached"); - } - if params.image_width == 0 || params.image_height == 0 { - return encode_error( - id, - "invalid_image", - "image_width and image_height must be greater than zero", - ); - } - let expected = match expected_len(params.format, params.image_width, params.image_height) { - Ok(expected) => expected, - Err(()) => return encode_error(id, "invalid_image", "image dimensions are too large"), - }; - let streamed = params.data.is_some(); - let mut data = match params.data { - Some(data) => data, - None => match base64::engine::general_purpose::STANDARD.decode(params.data_base64) { - Ok(data) => data, - Err(_) => { - return encode_error(id, "invalid_image", "data_base64 is not valid base64") - } - }, - }; - let limit = if streamed { - PANE_GRAPHICS_STREAM_MAX_BYTES - } else { - PANE_GRAPHICS_SET_MAX_BYTES - }; - if data.is_empty() { - return encode_error(id, "invalid_image", "image data must not be empty"); - } - if data.len() > limit { - return encode_error(id, "image_too_large", "image data is too large"); - } - if expected.is_some_and(|size| size != data.len()) { - return encode_error( - id, - "invalid_image", - "image data does not match the frame contract", - ); - } - let format = canonicalize_bgra(params.format, &mut data); - if !self.pane_graphics.can_store_inline(&key, data.len()) { - return encode_error( - id, - "graphics_budget_exceeded", - "pane graphics inline memory limit reached", - ); - } - let Some(host_image_id) = self.pane_graphics.reserve_image_id(&key) else { - return encode_error(id, "layer_limit", "pane graphics image ids are exhausted"); - }; - // Move the liveness handle out before replacing the slot so Slot::drop cannot - // deactivate the stream that the replacement still owns. - let (stream_owner, stream_active) = self - .pane_graphics - .slots - .get_mut(&key) - .map(|slot| (slot.stream_owner.take(), slot.stream_active.take())) - .unwrap_or_default(); - self.pane_graphics.slots.insert( - key, - Slot { - host_image_id, - layer: Some(Layer::inline( - format, - params.image_width, - params.image_height, - data, - params.placement, - params.z_index, - )), - stream_owner, - stream_active, - direct_gate: None, - }, - ); - self.pane_graphics.mark_changed(); - encode_success(id, ResponseResult::Ok {}) - } - - fn reclaim_inactive_stream(&mut self, key: &PaneGraphicsKey) { - let stale = self - .pane_graphics - .slots - .get(key) - .is_some_and(|slot| slot.stream_owner.is_some() && !slot.stream_is_active()); - if stale { - self.pane_graphics.slots.remove(key); - self.pane_graphics.mark_changed(); - } - } -} - -fn canonicalize_bgra( - format: crate::api::schema::PaneGraphicsFormat, - data: &mut [u8], -) -> crate::api::schema::PaneGraphicsFormat { - if format != crate::api::schema::PaneGraphicsFormat::Bgra { - return format; - } - for pixel in data.chunks_exact_mut(4) { - pixel.swap(0, 2); - } - crate::api::schema::PaneGraphicsFormat::Rgba -} - -fn graphics_key(pane_id: PaneId, layer_id: Option<&str>) -> Result<PaneGraphicsKey, &'static str> { - let layer_id = layer_id.unwrap_or(PANE_GRAPHICS_PRIMARY_LAYER_ID); - if layer_id.is_empty() || layer_id.len() > 64 { - return Err("layer_id must contain between 1 and 64 characters"); - } - if !layer_id - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')) - { - return Err("layer_id contains unsupported characters"); - } - Ok((pane_id, layer_id.to_owned())) -} - -fn expected_len( - format: crate::api::schema::PaneGraphicsFormat, - width: u32, - height: u32, -) -> Result<Option<usize>, ()> { - let bytes = match format { - crate::api::schema::PaneGraphicsFormat::Png => return Ok(None), - crate::api::schema::PaneGraphicsFormat::Rgb => 3_u64, - crate::api::schema::PaneGraphicsFormat::Rgba - | crate::api::schema::PaneGraphicsFormat::Bgra => 4_u64, - }; - u64::from(width) - .checked_mul(u64::from(height)) - .and_then(|pixels| pixels.checked_mul(bytes)) - .and_then(|size| usize::try_from(size).ok()) - .map(Some) - .ok_or(()) -} - -fn require_enabled(app: &App, id: &str) -> Result<(), String> { - app.state - .kitty_graphics_enabled - .then_some(()) - .ok_or_else(|| { - encode_error( - id.to_owned(), - "feature_disabled", - "pane graphics are disabled by terminal.kitty_graphics", - ) - }) -} - -fn pane_not_found(id: String, pane_id: &str) -> String { - encode_error(id, "pane_not_found", format!("pane {pane_id} not found")) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::api::schema::{ErrorResponse, SuccessResponse}; - use crate::config::Config; - use crate::workspace::Workspace; - - fn app() -> (App, String) { - let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let mut app = App::new( - &Config::default(), - crate::app::AppPolicy::TEST, - None, - rx, - crate::api::EventHub::default(), - ); - app.state.workspaces = vec![Workspace::test_new("graphics")]; - app.state.ensure_test_terminals(); - let pane = app.state.workspaces[0].tabs[0].root_pane; - let public = app.public_pane_id(0, pane).unwrap(); - app.state.kitty_graphics_enabled = true; - (app, public) - } - - fn frame( - pane_id: String, - layer_id: Option<&str>, - owner: &str, - value: u8, - ) -> PaneGraphicsSetParams { - PaneGraphicsSetParams { - pane_id, - layer_id: layer_id.map(str::to_owned), - z_index: 2, - owner: owner.to_owned(), - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - data: Some(vec![value; 4]), - data_base64: String::new(), - placement: Default::default(), - } - } - - fn pane_visible(response: &str) -> bool { - serde_json::from_str::<serde_json::Value>(response).unwrap()["result"]["pane_visible"] - .as_bool() - .unwrap() - } - - #[test] - fn info_reports_visibility_for_terminal_surface_workspace_tab_and_zoom() { - let (mut app, pane_id) = app(); - app.state.mode = crate::app::Mode::Terminal; - app.state.active = Some(0); - app.state.host_cell_size = crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }; - let visible = app.handle_pane_graphics_info( - "visible".into(), - crate::api::schema::PaneTarget { - pane_id: pane_id.clone(), - }, - ); - assert!(pane_visible(&visible)); - - let hidden_workspace = Workspace::test_new("hidden"); - let hidden_workspace_pane = hidden_workspace.tabs[0].root_pane; - app.state.workspaces.push(hidden_workspace); - let hidden_workspace_id = app.public_pane_id(1, hidden_workspace_pane).unwrap(); - let hidden = app.handle_pane_graphics_info( - "hidden-workspace".into(), - crate::api::schema::PaneTarget { - pane_id: hidden_workspace_id, - }, - ); - assert!(!pane_visible(&hidden)); - - let inactive_tab = app.state.workspaces[0].test_add_tab(Some("inactive")); - let inactive_pane = app.state.workspaces[0].tabs[inactive_tab].root_pane; - let inactive_id = app.public_pane_id(0, inactive_pane).unwrap(); - let hidden_tab = app.handle_pane_graphics_info( - "hidden-tab".into(), - crate::api::schema::PaneTarget { - pane_id: inactive_id, - }, - ); - assert!(!pane_visible(&hidden_tab)); - - app.state.workspaces[0].test_split(ratatui::layout::Direction::Horizontal); - app.state.workspaces[0].tabs[0].zoomed = true; - let zoomed_away = app.handle_pane_graphics_info( - "zoomed-away".into(), - crate::api::schema::PaneTarget { - pane_id: pane_id.clone(), - }, - ); - assert!(!pane_visible(&zoomed_away)); - - app.state.workspaces[0].tabs[0].zoomed = false; - app.state.mode = crate::app::Mode::Navigate; - let short_lived_mode = app.handle_pane_graphics_info( - "navigate".into(), - crate::api::schema::PaneTarget { pane_id }, - ); - assert!(pane_visible(&short_lived_mode)); - } - - #[test] - fn non_persistent_info_keeps_fast_transport_and_exact_pixels_disabled() { - let (mut app, pane_id) = app(); - app.state.host_cell_size = crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }; - assert!(!app.pane_graphics_files.is_initialized()); - let response = app - .handle_pane_graphics_info("info".into(), crate::api::schema::PaneTarget { pane_id }); - let success: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!( - success.result, - ResponseResult::PaneGraphicsInfo { - file_frame_damage: true, - max_layers_per_pane: 16, - pixel_mouse: false, - .. - } - )); - assert!(app.pane_graphics.slots.is_empty()); - assert!(!app.pane_graphics_files.is_initialized()); - } - - #[cfg(unix)] - #[test] - fn info_advertises_rgba_direct_and_bgra_fallback_file_formats_on_demand() { - let (mut app, pane_id) = app(); - app.state.host_cell_size = crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }; - app.direct_graphics_available = true; - let response = app - .handle_pane_graphics_info("info".into(), crate::api::schema::PaneTarget { pane_id }); - let value: serde_json::Value = serde_json::from_str(&response).unwrap(); - assert!(app.pane_graphics_files.is_initialized()); - assert_eq!( - value["result"]["file_frame_formats"], - serde_json::json!(["rgba", "bgra"]) - ); - assert_eq!( - value["result"]["file_frame_max_bytes"], - PANE_GRAPHICS_STREAM_MAX_BYTES - ); - assert_eq!( - value["result"]["file_frame_direct_max_bytes"], - PANE_GRAPHICS_DIRECT_FILE_MAX_BYTES - ); - assert_eq!(value["result"]["file_frame_damage"], true); - assert_eq!(value["result"]["file_frame_transport"], "direct-kitty"); - } - - #[test] - fn layers_are_isolated_and_stream_owned() { - let (mut app, pane_id) = app(); - for (layer, z, value) in [(None, 0, 1), (Some("chrome"), 10, 2)] { - let mut request = frame(pane_id.clone(), layer, "", value); - request.z_index = z; - let response = app.handle_pane_graphics_set(format!("set-{value}"), request); - assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok()); - } - let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); - assert_eq!( - app.pane_graphics - .slots - .get(&(pane, PANE_GRAPHICS_PRIMARY_LAYER_ID.into())) - .and_then(|slot| slot.layer.as_ref()) - .unwrap() - .z_index, - 0 - ); - assert_eq!( - app.pane_graphics - .slots - .get(&(pane, "chrome".into())) - .and_then(|slot| slot.layer.as_ref()) - .unwrap() - .z_index, - 10 - ); - let response = app.handle_pane_graphics_stream_open( - "open".into(), - PaneGraphicsStreamParams { - pane_id: pane_id.clone(), - layer_id: Some("chrome".into()), - z_index: 10, - owner: "stream".into(), - }, - ); - assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok()); - let response = - app.handle_pane_graphics_set("conflict".into(), frame(pane_id, Some("chrome"), "", 3)); - let error: ErrorResponse = serde_json::from_str(&response).unwrap(); - assert_eq!(error.error.code, "stream_conflict"); - } - - fn error_code(response: &str) -> String { - serde_json::from_str::<ErrorResponse>(response) - .unwrap() - .error - .code - } - - #[test] - fn validates_layer_and_frame_contracts() { - let cases = [ - (Some("bad space"), 1, 1, vec![1; 4], "invalid_layer_id"), - (None, 0, 1, vec![1; 4], "invalid_image"), - (None, 1, 1, Vec::new(), "invalid_image"), - (None, 1, 1, vec![1; 3], "invalid_image"), - ]; - for (layer, width, height, data, expected) in cases { - let (mut app, pane_id) = app(); - let mut params = frame(pane_id, layer, "", 1); - params.image_width = width; - params.image_height = height; - params.data = Some(data); - let response = app.handle_pane_graphics_set("invalid".into(), params); - assert_eq!(error_code(&response), expected); - assert!(app.pane_graphics.slots.is_empty()); - } - } - - #[test] - fn disabled_requests_have_no_graphics_side_effects() { - let (mut app, pane_id) = app(); - app.state.kitty_graphics_enabled = false; - let revision = app.pane_graphics.revision(); - - let response = app.handle_pane_graphics_set("disabled".into(), frame(pane_id, None, "", 1)); - - assert_eq!(error_code(&response), "feature_disabled"); - assert!(app.pane_graphics.slots.is_empty()); - assert_eq!(app.pane_graphics.revision(), revision); - } - - #[test] - fn stream_owner_controls_only_its_named_layer() { - let (mut app, pane_id) = app(); - let params = |owner: &str| PaneGraphicsStreamParams { - pane_id: pane_id.clone(), - layer_id: Some("chrome".into()), - z_index: 7, - owner: owner.into(), - }; - assert!(serde_json::from_str::<SuccessResponse>( - &app.handle_pane_graphics_stream_open("open".into(), params("owner")) - ) - .is_ok()); - assert_eq!( - error_code(&app.handle_pane_graphics_stream_set( - "wrong".into(), - frame(pane_id.clone(), Some("chrome"), "other", 2), - )), - "stream_conflict" - ); - assert!( - serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_stream_set( - "frame".into(), - frame(pane_id.clone(), Some("chrome"), "owner", 3), - )) - .is_ok() - ); - app.handle_pane_graphics_stream_close("stale".into(), params("other")); - assert_eq!(app.pane_graphics.slots.len(), 1); - app.handle_pane_graphics_stream_close("close".into(), params("owner")); - assert!(app.pane_graphics.slots.is_empty()); - } - - #[test] - fn static_set_and_clear_reclaim_inactive_stream_layers() { - let (mut app, pane_id) = app(); - let open = |layer: &str| PaneGraphicsStreamParams { - pane_id: pane_id.clone(), - layer_id: Some(layer.into()), - z_index: 0, - owner: format!("owner-{layer}"), - }; - for layer in ["set", "clear"] { - assert!(serde_json::from_str::<SuccessResponse>( - &app.handle_pane_graphics_stream_open(layer.into(), open(layer)) - ) - .is_ok()); - let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); - app.pane_graphics.slots[&(pane, layer.into())] - .stream_active - .as_ref() - .unwrap() - .store(false, std::sync::atomic::Ordering::Release); - } - - assert!( - serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_set( - "set-static".into(), - frame(pane_id.clone(), Some("set"), "", 4), - )) - .is_ok() - ); - assert!( - serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_clear( - "clear-static".into(), - PaneGraphicsClearParams { - pane_id, - layer_id: Some("clear".into()), - }, - )) - .is_ok() - ); - assert_eq!(app.pane_graphics.slots.len(), 1); - assert!(app - .pane_graphics - .slots - .values() - .all(|slot| slot.stream_owner.is_none())); - } - - #[test] - fn layer_limit_counts_empty_and_populated_stream_slots() { - let (mut app, pane_id) = app(); - for index in 0..PANE_GRAPHICS_MAX_LAYERS_PER_PANE { - let layer_id = format!("layer-{index}"); - let response = app.handle_pane_graphics_stream_open( - layer_id.clone(), - PaneGraphicsStreamParams { - pane_id: pane_id.clone(), - layer_id: Some(layer_id), - z_index: index as i32, - owner: format!("owner-{index}"), - }, - ); - assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok()); - } - let response = app.handle_pane_graphics_set( - "set-overflow".into(), - frame(pane_id.clone(), Some("set-overflow"), "", 1), - ); - assert_eq!(error_code(&response), "layer_limit"); - let response = app.handle_pane_graphics_stream_open( - "overflow".into(), - PaneGraphicsStreamParams { - pane_id, - layer_id: Some("overflow".into()), - z_index: 0, - owner: "overflow".into(), - }, - ); - assert_eq!(error_code(&response), "layer_limit"); - assert_eq!(app.pane_graphics.slots.len(), 16); - } - - #[test] - fn set_and_clear_preserve_runtime_only_layer_details() { - let (mut app, pane_id) = app(); - let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); - let mut params = frame(pane_id.clone(), None, "", 9); - params.placement.grid_cols = 10; - params.placement.grid_rows = 4; - assert!(serde_json::from_str::<SuccessResponse>( - &app.handle_pane_graphics_set("set".into(), params) - ) - .is_ok()); - let layer = app - .pane_graphics - .slots - .get(&(pane, PANE_GRAPHICS_PRIMARY_LAYER_ID.into())) - .and_then(|slot| slot.layer.as_ref()) - .unwrap(); - assert_eq!(layer.inline_data(), Some([9; 4].as_slice())); - assert_eq!((layer.render.grid_cols, layer.render.grid_rows), (10, 4)); - assert!(!app.state.session_dirty); - - assert!( - serde_json::from_str::<SuccessResponse>(&app.handle_pane_graphics_clear( - "clear".into(), - PaneGraphicsClearParams { - pane_id, - layer_id: None, - }, - )) - .is_ok() - ); - assert!(app.pane_graphics.slots.is_empty()); - } - - #[test] - fn raw_formats_require_exact_non_overflowing_lengths() { - for (format, exact) in [ - (crate::api::schema::PaneGraphicsFormat::Rgb, 6), - (crate::api::schema::PaneGraphicsFormat::Rgba, 8), - ] { - for (length, expected) in [(exact, "ok"), (exact - 1, "invalid_image")] { - let (mut app, pane_id) = app(); - let mut params = frame(pane_id, None, "", 1); - params.format = format; - params.image_width = 2; - params.data = Some(vec![1; length]); - let response = app.handle_pane_graphics_set("length".into(), params); - if expected == "ok" { - assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok()); - } else { - assert_eq!(error_code(&response), expected); - } - } - let (mut app, pane_id) = app(); - let mut params = frame(pane_id, None, "", 1); - params.format = format; - params.image_width = u32::MAX; - params.image_height = u32::MAX; - assert_eq!( - error_code(&app.handle_pane_graphics_set("overflow".into(), params)), - "invalid_image" - ); - } - } - - #[test] - fn public_frames_validate_base64_and_public_size_limit() { - let (mut app, pane_id) = app(); - let mut params = frame(pane_id.clone(), None, "", 1); - params.format = crate::api::schema::PaneGraphicsFormat::Png; - params.data = None; - params.data_base64 = "not base64".into(); - assert_eq!( - error_code(&app.handle_pane_graphics_set("base64".into(), params)), - "invalid_image" - ); - - let mut params = frame(pane_id, None, "", 1); - params.format = crate::api::schema::PaneGraphicsFormat::Png; - params.data = None; - params.data_base64 = - base64::engine::general_purpose::STANDARD - .encode(vec![1; PANE_GRAPHICS_SET_MAX_BYTES + 1]); - assert_eq!( - error_code(&app.handle_pane_graphics_set("large".into(), params)), - "image_too_large" - ); - } - - #[test] - fn aliases_resolve_to_the_same_layer_identity() { - let (mut app, pane_id) = app(); - let (_, pane) = app.parse_pane_id(&pane_id).unwrap(); - let alias = format!("{pane_id}:alias"); - app.state.public_pane_id_aliases.insert(alias.clone(), pane); - let params = |target: String, owner: &str| PaneGraphicsStreamParams { - pane_id: target, - layer_id: None, - z_index: 0, - owner: owner.into(), - }; - assert!(serde_json::from_str::<SuccessResponse>( - &app.handle_pane_graphics_stream_open("open".into(), params(pane_id.clone(), "owner"),) - ) - .is_ok()); - assert_eq!( - error_code(&app.handle_pane_graphics_stream_open( - "conflict".into(), - params(alias.clone(), "other"), - )), - "stream_conflict" - ); - app.handle_pane_graphics_stream_close("stale".into(), params(alias.clone(), "other")); - assert_eq!(app.pane_graphics.slots.len(), 1); - app.handle_pane_graphics_stream_close("close".into(), params(alias, "owner")); - assert!(app.pane_graphics.slots.is_empty()); - } - - #[cfg(unix)] - fn direct_file(app: &App, name: &str, data: &[u8]) -> String { - use std::io::Write as _; - use std::os::unix::fs::OpenOptionsExt as _; - let path = app - .pane_graphics_files - .source_directory() - .unwrap() - .join(name); - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&path) - .unwrap(); - file.write_all(data).unwrap(); - path.to_string_lossy().into_owned() - } - - #[cfg(unix)] - fn sparse_direct_file(app: &App, name: &str, len: usize) -> String { - use std::os::unix::fs::OpenOptionsExt as _; - let path = app - .pane_graphics_files - .source_directory() - .unwrap() - .join(name); - let file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&path) - .unwrap(); - file.set_len(len as u64).unwrap(); - path.to_string_lossy().into_owned() - } - - #[cfg(unix)] - fn direct_params( - pane_id: String, - owner: &str, - path: String, - ) -> crate::api::schema::PaneGraphicsDirectParams { - crate::api::schema::PaneGraphicsDirectParams { - pane_id, - layer_id: None, - z_index: 0, - owner: owner.into(), - image_width: 1, - image_height: 1, - format: crate::api::schema::PaneGraphicsFormat::Rgba, - path, - sequence: 1, - revision: 1, - placement: Default::default(), - } - } - - #[cfg(unix)] - #[test] - fn direct_file_is_leased_or_safely_copied_for_inline_fallback() { - let (mut app, pane_id) = app(); - let stream = PaneGraphicsStreamParams { - pane_id: pane_id.clone(), - layer_id: None, - z_index: 0, - owner: "owner".into(), - }; - app.handle_pane_graphics_stream_open("open".into(), stream.clone()); - for (width, height) in [(0, 1), (1, 0)] { - let mut params = direct_params(pane_id.clone(), "owner", "unused".into()); - (params.image_width, params.image_height) = (width, height); - assert_eq!( - error_code(&app.handle_pane_graphics_stream_direct("invalid".into(), params)), - "invalid_image" - ); - } - for format in [ - crate::api::schema::PaneGraphicsFormat::Rgb, - crate::api::schema::PaneGraphicsFormat::Png, - ] { - let mut params = direct_params(pane_id.clone(), "owner", "unused".into()); - params.format = format; - assert_eq!( - error_code(&app.handle_pane_graphics_stream_direct("invalid".into(), params)), - "invalid_image" - ); - } - app.direct_graphics_available = true; - let path = direct_file(&app, "direct", &[1, 2, 3, 4]); - let response = app.handle_pane_graphics_stream_direct( - "direct".into(), - crate::api::schema::PaneGraphicsDirectParams { - sequence: 7, - revision: 8, - ..direct_params(pane_id.clone(), "owner", path) - }, - ); - let ack: SuccessResponse = serde_json::from_str(&response).unwrap(); - assert!(matches!( - ack.result, - ResponseResult::PaneGraphicsFrameAck { - sequence: 7, - revision: 8 - } - )); - assert!(app - .pane_graphics - .slots - .values() - .next() - .unwrap() - .layer - .as_ref() - .unwrap() - .direct_lease() - .is_some()); - - app.handle_pane_graphics_stream_close("close".into(), stream.clone()); - app.handle_pane_graphics_stream_open("reopen".into(), stream); - app.direct_graphics_available = false; - let path = direct_file(&app, "fallback", &[5, 6, 7, 8]); - app.handle_pane_graphics_stream_direct( - "fallback".into(), - crate::api::schema::PaneGraphicsDirectParams { - sequence: 9, - revision: 10, - ..direct_params(pane_id, "owner", path) - }, - ); - assert_eq!( - app.pane_graphics - .slots - .values() - .next() - .unwrap() - .layer - .as_ref() - .unwrap() - .inline_data(), - Some([5, 6, 7, 8].as_slice()) - ); - } - - #[cfg(unix)] - #[test] - fn direct_primary_rgba_accepts_fullscreen_retina_frame() { - let (mut app, pane_id) = app(); - app.direct_graphics_available = true; - app.handle_pane_graphics_stream_open( - "open".into(), - PaneGraphicsStreamParams { - pane_id: pane_id.clone(), - layer_id: None, - z_index: 0, - owner: "owner".into(), - }, - ); - let (image_width, image_height) = (3456, 2234); - let len = image_width * image_height * 4; - let path = sparse_direct_file(&app, "retina-frame", len as usize); - let response = app.handle_pane_graphics_stream_direct( - "frame".into(), - crate::api::schema::PaneGraphicsDirectParams { - image_width, - image_height, - ..direct_params(pane_id, "owner", path) - }, - ); - - assert!(serde_json::from_str::<SuccessResponse>(&response).is_ok()); - assert!(app - .pane_graphics - .slots - .values() - .next() - .unwrap() - .layer - .as_ref() - .unwrap() - .direct_lease() - .is_some()); - } - - #[cfg(unix)] - #[test] - fn bgra_and_secondary_file_frames_are_canonical_owned_rgba() { - let (mut app, pane_id) = app(); - app.direct_graphics_available = true; - for (layer, format, source) in [ - ( - "browser-chrome", - crate::api::schema::PaneGraphicsFormat::Rgba, - [1, 2, 3, 4], - ), - ( - "primary", - crate::api::schema::PaneGraphicsFormat::Bgra, - [3, 2, 1, 4], - ), - ] { - app.handle_pane_graphics_stream_open( - format!("open-{layer}"), - PaneGraphicsStreamParams { - pane_id: pane_id.clone(), - layer_id: Some(layer.into()), - z_index: 0, - owner: layer.into(), - }, - ); - let path = direct_file(&app, layer, &source); - app.handle_pane_graphics_stream_direct( - format!("frame-{layer}"), - crate::api::schema::PaneGraphicsDirectParams { - layer_id: Some(layer.into()), - format, - ..direct_params(pane_id.clone(), layer, path) - }, - ); - let key = (app.parse_pane_id(&pane_id).unwrap().1, layer.into()); - assert_eq!( - app.pane_graphics.slots[&key] - .layer - .as_ref() - .unwrap() - .inline_data(), - Some([1, 2, 3, 4].as_slice()) - ); - } - } - - #[test] - fn disabled_info_and_stream_requests_are_rejected() { - let (mut app, pane_id) = app(); - app.state.kitty_graphics_enabled = false; - assert_eq!( - error_code(&app.handle_pane_graphics_info( - "info".into(), - crate::api::schema::PaneTarget { - pane_id: pane_id.clone(), - }, - )), - "feature_disabled" - ); - assert_eq!( - error_code(&app.handle_pane_graphics_stream_open( - "open".into(), - PaneGraphicsStreamParams { - pane_id, - layer_id: None, - z_index: 0, - owner: "owner".into(), - }, - )), - "feature_disabled" - ); - assert!(app.pane_graphics.slots.is_empty()); - } -} diff --git a/src/app/mod.rs b/src/app/mod.rs index 03e89026..e358a562 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -17,7 +17,6 @@ mod creation; mod custom_commands; mod git_refresh; mod ids; -pub(crate) mod pane_graphics; mod popup; mod runtime; mod session; @@ -103,9 +102,6 @@ impl AppPolicy { pub struct App { pub state: AppState, - pub(crate) pane_graphics: pane_graphics::Runtime, - pub(crate) pane_graphics_files: Arc<crate::pane_graphics_files::FileStore>, - pub(crate) direct_graphics_available: bool, pub(crate) pixel_mouse_available: bool, pub(crate) terminal_runtimes: crate::terminal::TerminalRuntimeRegistry, pub event_tx: mpsc::Sender<AppEvent>, @@ -575,9 +571,6 @@ impl App { toast_deadline: None, last_api_notification_at: None, state, - pane_graphics: pane_graphics::Runtime::default(), - pane_graphics_files: Arc::new(crate::pane_graphics_files::FileStore::default()), - direct_graphics_available: false, pixel_mouse_available: false, terminal_runtimes: restored_terminal_runtimes, event_tx, diff --git a/src/app/pane_graphics.rs b/src/app/pane_graphics.rs deleted file mode 100644 index 71f998d8..00000000 --- a/src/app/pane_graphics.rs +++ /dev/null @@ -1,350 +0,0 @@ -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; - -use crate::api::schema::{PaneGraphicsFormat, PaneGraphicsPlacementParams}; -use crate::layout::PaneId; - -pub(crate) type Key = (PaneId, String); - -pub(crate) const DIRECT_DELIVERY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); -pub(crate) const DIRECT_RESPONSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(3); -pub(crate) const DIRECT_OUTER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(9); - -#[derive(Debug)] -pub(crate) enum Backing { - Inline(Vec<u8>), - Direct(crate::pane_graphics_files::Lease), - Resident { len: usize, client_id: u64 }, -} - -#[derive(Debug)] -pub(crate) struct Layer { - pub(crate) format: PaneGraphicsFormat, - pub(crate) image_width: u32, - pub(crate) image_height: u32, - pub(crate) backing: Backing, - pub(crate) data_fingerprint: u64, - pub(crate) render: PaneGraphicsPlacementParams, - pub(crate) z_index: i32, -} - -#[derive(Debug)] -pub(crate) struct DirectGate { - pub(crate) transfer_id: u64, - pub(crate) image_id: u32, - pub(crate) client_id: u64, - pub(crate) deadline: std::time::Instant, - pub(crate) written: bool, - pub(crate) success_response: String, - pub(crate) respond_to: std::sync::mpsc::Sender<String>, -} - -#[derive(Debug)] -pub(crate) struct Slot { - pub(crate) host_image_id: u32, - pub(crate) layer: Option<Layer>, - pub(crate) stream_owner: Option<String>, - pub(crate) stream_active: Option<std::sync::Arc<std::sync::atomic::AtomicBool>>, - pub(crate) direct_gate: Option<DirectGate>, -} - -impl Slot { - #[cfg(test)] - pub(crate) fn test(host_image_id: u32, layer: Option<Layer>) -> Self { - Self { - host_image_id, - layer, - stream_owner: None, - stream_active: None, - direct_gate: None, - } - } - - pub(crate) fn stream_is_active(&self) -> bool { - self.stream_active - .as_ref() - .is_some_and(|active| active.load(std::sync::atomic::Ordering::Acquire)) - } - - pub(crate) fn direct_client(&self) -> Option<u64> { - self.direct_gate - .as_ref() - .map(|gate| gate.client_id) - .or_else(|| self.layer.as_ref().and_then(Layer::resident_client)) - } -} - -impl Drop for Slot { - fn drop(&mut self) { - if let Some(active) = &self.stream_active { - active.store(false, std::sync::atomic::Ordering::Release); - } - } -} - -#[derive(Debug)] -pub(crate) struct Runtime { - pub(crate) slots: HashMap<Key, Slot>, - revision: u64, - next_host_image_id: u32, -} - -impl Default for Runtime { - fn default() -> Self { - Self { - slots: HashMap::new(), - revision: 0, - next_host_image_id: 1 << 31, - } - } -} - -impl Runtime { - pub(crate) fn revision(&self) -> u64 { - self.revision - } - - pub(crate) fn mark_changed(&mut self) { - self.revision = self.revision.wrapping_add(1); - } - - #[cfg(test)] - pub(crate) fn clear(&mut self) { - self.slots.clear(); - self.mark_changed(); - } - - pub(crate) fn reserve_image_id(&mut self, key: &Key) -> Option<u32> { - if let Some(id) = self.slots.get(key).map(|slot| slot.host_image_id) { - return Some(id); - } - for _ in 0..=self.slots.len() { - let id = self.next_host_image_id | (1 << 31); - self.next_host_image_id = id.wrapping_add(1) | (1 << 31); - if self.slots.values().all(|slot| slot.host_image_id != id) { - return Some(id); - } - } - None - } - - pub(crate) fn layer_count(&self, pane_id: PaneId) -> usize { - self.slots.keys().filter(|(id, _)| *id == pane_id).count() - } - - pub(crate) fn can_add_slot(&self, key: &Key) -> bool { - self.slots.contains_key(key) - || self.slots.len() < crate::api::schema::panes::PANE_GRAPHICS_MAX_LAYERS_TOTAL - } - - pub(crate) fn can_store_inline(&self, key: &Key, len: usize) -> bool { - let replaced = self - .slots - .get(key) - .and_then(|slot| slot.layer.as_ref()) - .and_then(Layer::inline_data) - .map_or(0, <[u8]>::len); - self.inline_bytes() - .saturating_sub(replaced) - .saturating_add(len) - <= crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL - } - - fn inline_bytes(&self) -> usize { - self.slots - .values() - .filter_map(|slot| slot.layer.as_ref()) - .filter_map(Layer::inline_data) - .map(<[u8]>::len) - .sum() - } - - pub(crate) fn attach_stream_active( - &mut self, - key: &Key, - owner: &str, - active: std::sync::Arc<std::sync::atomic::AtomicBool>, - ) { - if let Some(slot) = self - .slots - .get_mut(key) - .filter(|slot| slot.stream_owner.as_deref() == Some(owner)) - { - slot.stream_active = Some(active); - } - } - - pub(crate) fn retain_live_panes(&mut self, state: &crate::app::state::AppState) -> bool { - if self.slots.is_empty() { - return false; - } - let before = self.slots.len(); - self.slots.retain(|(pane_id, _), _| { - state - .workspaces - .iter() - .any(|workspace| workspace.pane_state(*pane_id).is_some()) - }); - if self.slots.len() == before { - return false; - } - self.mark_changed(); - true - } -} - -impl Layer { - pub(crate) fn inline( - format: PaneGraphicsFormat, - image_width: u32, - image_height: u32, - data: Vec<u8>, - render: PaneGraphicsPlacementParams, - z_index: i32, - ) -> Self { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - data.hash(&mut hasher); - Self { - format, - image_width, - image_height, - backing: Backing::Inline(data), - data_fingerprint: hasher.finish(), - render, - z_index, - } - } - - pub(crate) fn direct( - image_width: u32, - image_height: u32, - lease: crate::pane_graphics_files::Lease, - render: PaneGraphicsPlacementParams, - z_index: i32, - ) -> Self { - let data_fingerprint = lease.fingerprint(); - Self { - format: PaneGraphicsFormat::Rgba, - image_width, - image_height, - backing: Backing::Direct(lease), - data_fingerprint, - render, - z_index, - } - } - - pub(crate) fn data_len(&self) -> usize { - match &self.backing { - Backing::Inline(data) => data.len(), - Backing::Direct(lease) => lease.len(), - Backing::Resident { len, .. } => *len, - } - } - - pub(crate) fn inline_data(&self) -> Option<&[u8]> { - match &self.backing { - Backing::Inline(data) => Some(data), - Backing::Direct(_) | Backing::Resident { .. } => None, - } - } - - pub(crate) fn direct_lease(&self) -> Option<&crate::pane_graphics_files::Lease> { - match &self.backing { - Backing::Direct(lease) => Some(lease), - Backing::Inline(_) | Backing::Resident { .. } => None, - } - } - - pub(crate) fn mark_resident(&mut self, client_id: u64) -> bool { - let Backing::Direct(lease) = &self.backing else { - return false; - }; - self.backing = Backing::Resident { - len: lease.len(), - client_id, - }; - true - } - - pub(crate) fn resident_client(&self) -> Option<u64> { - match self.backing { - Backing::Resident { client_id, .. } => Some(client_id), - Backing::Inline(_) | Backing::Direct(_) => None, - } - } - - pub(crate) fn terminal_only(&self) -> bool { - matches!(self.backing, Backing::Direct(_) | Backing::Resident { .. }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn stream_liveness_attaches_only_to_the_exact_owned_layer() { - let mut runtime = Runtime::default(); - let first = (PaneId::from_raw(1), "first".into()); - let second = (PaneId::from_raw(2), "second".into()); - for (key, id) in [(&first, 1), (&second, 2)] { - runtime.slots.insert( - key.clone(), - Slot { - host_image_id: (1 << 31) | id, - layer: None, - stream_owner: Some("shared-owner".into()), - stream_active: None, - direct_gate: None, - }, - ); - } - let active = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); - - runtime.attach_stream_active(&second, "shared-owner", active.clone()); - - assert!(runtime.slots[&first].stream_active.is_none()); - assert!(std::sync::Arc::ptr_eq( - runtime.slots[&second].stream_active.as_ref().unwrap(), - &active - )); - } - - #[test] - fn runtime_bounds_total_slots_and_inline_bytes() { - let mut runtime = Runtime::default(); - for index in 0..crate::api::schema::panes::PANE_GRAPHICS_MAX_LAYERS_TOTAL { - let key = (PaneId::from_raw(1), format!("layer-{index}")); - runtime - .slots - .insert(key, Slot::test((1 << 31) | index as u32, None)); - } - assert!(!runtime.can_add_slot(&(PaneId::from_raw(2), "extra".into()))); - - let mut runtime = Runtime::default(); - let key = (PaneId::from_raw(1), "primary".into()); - runtime.slots.insert( - key.clone(), - Slot::test( - 1 << 31, - Some(Layer::inline( - PaneGraphicsFormat::Rgba, - 1, - 1, - vec![0; 4], - PaneGraphicsPlacementParams::default(), - 0, - )), - ), - ); - assert!(runtime.can_store_inline( - &key, - crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL, - )); - assert!(!runtime.can_store_inline( - &(PaneId::from_raw(2), "primary".into()), - crate::api::schema::panes::PANE_GRAPHICS_MAX_INLINE_BYTES_TOTAL, - )); - } -} diff --git a/src/client/catalog_reload.rs b/src/client/catalog_reload.rs index a20b74dc..89715f7d 100644 --- a/src/client/catalog_reload.rs +++ b/src/client/catalog_reload.rs @@ -47,7 +47,7 @@ pub(super) fn apply_profiles( endpoints.disconnect(&endpoint_id); let cancelled = commands.disconnect(&endpoint_id); #[cfg(unix)] - state.retire_endpoint_graphics(&endpoint_id); + state.forget_endpoint_graphics(&endpoint_id); if let Some(shell) = state.shell.as_mut() { for request_id in cancelled { shell.cancel_endpoint_request(&request_id); diff --git a/src/client/endpoint/supervisor.rs b/src/client/endpoint/supervisor.rs index ff86ac77..5b4e7ad1 100644 --- a/src/client/endpoint/supervisor.rs +++ b/src/client/endpoint/supervisor.rs @@ -303,6 +303,7 @@ fn connect_once( options.endpoint_keybindings, options.mouse_capture, false, + matches!(target, ConnectTarget::Local(_)), ) .map_err(handshake_error)?; if handshake.encoding != RenderEncoding::SemanticFrame { diff --git a/src/client/frame_output.rs b/src/client/frame_output.rs index 5d12c78c..d9e03f14 100644 --- a/src/client/frame_output.rs +++ b/src/client/frame_output.rs @@ -1,11 +1,86 @@ use std::collections::HashSet; -use std::io; +use std::io::{self, Write as _}; use std::sync::{Mutex, OnceLock}; -use crate::protocol::render_ansi; +use crate::kitty_graphics::{GraphicsOperation, GraphicsOutput}; +use crate::protocol::{render_ansi, FrameData}; +use base64::Engine as _; + +/// Local output only; raw uploads are never added to a published frame codec. +#[derive(Debug)] +pub(crate) struct ComposedFrame { + pub(crate) frame: FrameData, + pub(crate) graphics: GraphicsOutput, +} + +impl From<FrameData> for ComposedFrame { + fn from(mut frame: FrameData) -> Self { + let graphics = GraphicsOutput::from_bytes(std::mem::take(&mut frame.graphics)); + Self { frame, graphics } + } +} + +impl std::ops::Deref for ComposedFrame { + type Target = FrameData; + + fn deref(&self) -> &Self::Target { + &self.frame + } +} static RECEIVED_KITTY_GRAPHICS_IDS: OnceLock<Mutex<HashSet<u32>>> = OnceLock::new(); +pub(super) fn write_composed_frame( + mut writer: impl io::Write, + encoded: &[u8], + graphics: &GraphicsOutput, + files: &mut super::image_files::FileTransport, +) -> io::Result<()> { + if graphics.is_empty() { + return writer.write_all(encoded); + } + let mut writer = io::BufWriter::with_capacity(64 * 1024, writer); + let insertion = render_ansi::final_sync_output_end(encoded).unwrap_or(encoded.len()); + writer.write_all(&encoded[..insertion])?; + writer.write_all(b"\x1b7")?; + for operation in &graphics.operations { + match operation { + GraphicsOperation::Bytes(bytes) => { + record_received_kitty_graphics(bytes); + writer.write_all(bytes)?; + } + GraphicsOperation::Upload { control, data } => { + let file_eligible = control + .split(',') + .any(|part| matches!(part, "f=24" | "f=32" | "f=100")); + if file_eligible { + if let Some(path) = files + .probe() + .and_then(|path| path.to_str().map(str::to_owned)) + { + let path = + base64::engine::general_purpose::STANDARD.encode(path.as_bytes()); + write!(writer, "\x1b_Ga=q,t=t,f=32,s=1,v=1,i=1,q=2;{path}\x1b\\")?; + } + } + let path = file_eligible.then(|| files.prepare(data)).flatten(); + let header = format!("\x1b_G{control};\x1b\\"); + record_received_kitty_graphics(header.as_bytes()); + if let Some(path) = path.as_ref().and_then(|path| path.to_str()) { + let control = control.replace(",t=d,", ",t=t,"); + let path = base64::engine::general_purpose::STANDARD.encode(path.as_bytes()); + write!(writer, "\x1b_G{control};{path}\x1b\\")?; + } else { + crate::kitty_graphics::write_kitty_data(&mut writer, control, data)?; + } + } + } + } + writer.write_all(b"\x1b8")?; + writer.write_all(&encoded[insertion..])?; + io::Write::flush(&mut writer) +} + pub(super) fn write_encoded_frame_with_graphics( mut writer: impl io::Write, encoded: &[u8], diff --git a/src/client/frame_output_tests.rs b/src/client/frame_output_tests.rs new file mode 100644 index 00000000..19c3717b --- /dev/null +++ b/src/client/frame_output_tests.rs @@ -0,0 +1,192 @@ +use super::frame_output::*; +use super::image_files::FileTransport; +use crate::kitty_graphics::{GraphicsOperation, GraphicsOutput}; +use std::sync::Arc; + +fn output(format: u32) -> GraphicsOutput { + GraphicsOutput { + operations: vec![ + GraphicsOperation::Bytes(b"before".to_vec()), + GraphicsOperation::Upload { + control: format!("a=t,t=d,f={format},s=1,v=1,i=42,q=2"), + data: Arc::from([1, 2, 3, 4]), + }, + GraphicsOperation::Bytes(b"after".to_vec()), + ], + } +} + +#[test] +fn deferred_inline_output_matches_existing_frame_envelope() { + let graphics = output(32); + let encoded = b"\x1b[?2026htext\x1b[?2026l"; + let mut expected = Vec::new(); + write_encoded_frame_with_graphics( + &mut expected, + encoded, + &graphics.clone().into_inline_bytes(), + ) + .unwrap(); + let mut actual = Vec::new(); + write_composed_frame( + &mut actual, + encoded, + &graphics, + &mut FileTransport::default(), + ) + .unwrap(); + assert_eq!(actual, expected); +} + +#[test] +fn native_swap_cleanup_and_placements_share_the_frame_sync_envelope() { + let graphics = GraphicsOutput::from_bytes( + concat!( + "\x1b_Ga=d,d=I,i=41,q=2;\x1b\\", + "\x1b_Ga=p,i=42,p=1,q=2;\x1b\\", + "\x1b_Ga=p,i=42,p=2,q=2;\x1b\\", + ) + .as_bytes() + .to_vec(), + ); + let mut actual = Vec::new(); + write_composed_frame( + &mut actual, + b"\x1b[?2026hframe\x1b[?2026l", + &graphics, + &mut FileTransport::default(), + ) + .unwrap(); + let text = std::str::from_utf8(&actual).unwrap(); + let sync_begin = text.find("\x1b[?2026h").unwrap(); + let sync_end = text.rfind("\x1b[?2026l").unwrap(); + let deletion = text.find("a=d,d=I,i=41").unwrap(); + let placements = text + .match_indices("a=p,i=42") + .map(|(at, _)| at) + .collect::<Vec<_>>(); + assert!(deletion > sync_begin && deletion < sync_end); + assert!(placements.len() >= 2); + assert!(placements + .iter() + .all(|placement| *placement > sync_begin && *placement < sync_end)); +} + +#[test] +fn deferred_output_propagates_write_failure() { + struct Broken; + impl std::io::Write for Broken { + fn write(&mut self, _: &[u8]) -> std::io::Result<usize> { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + assert_eq!( + write_composed_frame(Broken, b"text", &output(32), &mut FileTransport::default()) + .unwrap_err() + .kind(), + std::io::ErrorKind::BrokenPipe + ); +} + +#[cfg(unix)] +mod local_files { + use super::*; + use base64::Engine as _; + use std::path::PathBuf; + struct Scratch(PathBuf); + impl Scratch { + fn new() -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let id = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = PathBuf::from(format!( + "/var/tmp/herdr-output-test-{}-{id}", + std::process::id() + )); + std::fs::create_dir(&path).unwrap(); + Self(path) + } + } + impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + fn command_path(bytes: &[u8], header: &str) -> PathBuf { + let text = std::str::from_utf8(bytes).unwrap(); + let command = text + .split("\x1b_G") + .find(|part| part.starts_with(header)) + .unwrap(); + let payload = command + .split_once(';') + .unwrap() + .1 + .split("\x1b\\") + .next() + .unwrap(); + PathBuf::from( + String::from_utf8( + base64::engine::general_purpose::STANDARD + .decode(payload) + .unwrap(), + ) + .unwrap(), + ) + } + #[test] + fn files_begin_only_after_probe_consumption_and_keep_order() { + for format in [24, 32, 100] { + let root = Scratch::new(); + let mut files = FileTransport::for_test(root.0.clone()); + let graphics = output(format); + let mut first = Vec::new(); + write_composed_frame(&mut first, b"text", &graphics, &mut files).unwrap(); + let first_text = std::str::from_utf8(&first).unwrap(); + assert!(first_text.contains("a=q,t=t")); + assert!(first_text.contains("a=t,t=d")); + let probe = command_path(&first, "a=q,"); + assert_eq!(std::fs::read(&probe).unwrap().len(), 4); + let mut waiting = Vec::new(); + write_composed_frame(&mut waiting, b"text", &graphics, &mut files).unwrap(); + assert!(!std::str::from_utf8(&waiting).unwrap().contains("a=q,")); + assert!(std::str::from_utf8(&waiting).unwrap().contains("a=t,t=d")); + std::fs::remove_file(&probe).unwrap(); + let mut ready = Vec::new(); + write_composed_frame(&mut ready, b"text", &graphics, &mut files).unwrap(); + let text = std::str::from_utf8(&ready).unwrap(); + assert!(text.contains(&format!("before\x1b_Ga=t,t=t,f={format}"))); + assert!(text.contains("\x1b\\after")); + assert!(!text.contains("AQIDBA==")); + let image = command_path(&ready, "a=t,"); + assert_eq!(std::fs::read(&image).unwrap(), [1, 2, 3, 4]); + drop(files); + assert!(!image.exists()); + } + } + #[test] + fn unsupported_format_and_file_creation_failure_remain_inline() { + let root = Scratch::new(); + let mut files = FileTransport::for_test(root.0.clone()); + let mut unsupported = Vec::new(); + write_composed_frame(&mut unsupported, b"", &output(999), &mut files).unwrap(); + assert!(std::str::from_utf8(&unsupported) + .unwrap() + .contains("a=t,t=d,f=999")); + assert_eq!(std::fs::read_dir(&root.0).unwrap().count(), 0); + let mut broken = FileTransport::for_test(root.0.join("absent")); + let mut actual = Vec::new(); + write_composed_frame(&mut actual, b"", &output(32), &mut broken).unwrap(); + let mut expected = Vec::new(); + write_composed_frame( + &mut expected, + b"", + &output(32), + &mut FileTransport::default(), + ) + .unwrap(); + assert_eq!(actual, expected); + } +} diff --git a/src/client/handshake.rs b/src/client/handshake.rs index 69e2f52f..f25cc1b0 100644 --- a/src/client/handshake.rs +++ b/src/client/handshake.rs @@ -69,6 +69,17 @@ pub(super) fn direct_graphics_profile_values( supported && !blocked_transport && terminals } +/// Server-owned files require a shared filesystem; the host terminal profile alone +/// cannot establish that for a saved SSH endpoint. +fn direct_graphics_capability( + local_transport: bool, + exact_cell_size: bool, + cell_size: (u32, u32), + profile_allowed: bool, +) -> bool { + local_transport && exact_cell_size && cell_size.0 > 0 && cell_size.1 > 0 && profile_allowed +} + #[cfg(unix)] fn direct_graphics_profile_allowed() -> bool { let term_program = std::env::var("TERM_PROGRAM").unwrap_or_default(); @@ -138,6 +149,7 @@ pub(crate) fn probe_endpoint_negotiation( false, false, false, + false, // This probe uses an SSH bridge, not a shared filesystem. ) .map_err(io::Error::other)?; Ok(super::endpoint::EndpointNegotiation::new( @@ -150,7 +162,8 @@ pub(crate) fn probe_endpoint_negotiation( /// /// Direct terminal clients retain the same-install private protocol. Client-owned /// shells use the stable endpoint generation and negotiate whole codecs without -/// comparing Herdr build versions. +/// comparing Herdr build versions. `local_transport` means the endpoint shares the +/// client's filesystem, not merely that its bridge exposes a local socket. pub(super) fn do_handshake( stream: &mut LocalStream, cols: u16, @@ -162,6 +175,7 @@ pub(super) fn do_handshake( endpoint_keybindings: bool, mouse_capture: bool, surface_active: bool, + local_transport: bool, ) -> Result<HandshakeResult, ClientError> { stream .set_nonblocking(false) @@ -175,10 +189,12 @@ pub(super) fn do_handshake( cell_height_px, surface_size, pixel_mouse: exact_cell_size && cfg!(unix), - direct_graphics: exact_cell_size - && cell_width_px > 0 - && cell_height_px > 0 - && direct_graphics_profile_allowed(), + direct_graphics: direct_graphics_capability( + local_transport, + exact_cell_size, + (cell_width_px, cell_height_px), + direct_graphics_profile_allowed(), + ), endpoint_keybindings, mouse_capture, surface_active, @@ -295,3 +311,20 @@ pub(super) fn do_handshake( ))), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn direct_graphics_requires_local_transport_even_on_supported_host_terminal() { + let supported = direct_graphics_profile_values("ghostty", "", false, false, true); + assert!(supported); + assert!(direct_graphics_capability(true, true, (8, 16), supported)); + assert!(!direct_graphics_capability(false, true, (8, 16), supported)); + assert!(!direct_graphics_capability(true, false, (8, 16), supported)); + assert!(!direct_graphics_capability(true, true, (0, 16), supported)); + assert!(!direct_graphics_capability(true, true, (8, 0), supported)); + assert!(!direct_graphics_capability(true, true, (8, 16), false)); + } +} diff --git a/src/client/image_files.rs b/src/client/image_files.rs new file mode 100644 index 00000000..cc854c85 --- /dev/null +++ b/src/client/image_files.rs @@ -0,0 +1,138 @@ +//! Client-owned Kitty `t=t` files, prepared only at the final output boundary. +//! +//! A returned path is published: it must stay readable until the terminal unlinks +//! it, or the client shuts down. Never use `cleanup` to cancel a frame. This ledger +//! does not encode pixels or accept paths from a server. A silent raw-file query +//! gates conversion for the local Ghostty allowlist; an explicit kill switch +//! always wins. Local writes alone do not prove support. Non-consumption disables +//! new files for this client, without deleting files the terminal may still open. + +use std::path::PathBuf; + +#[derive(Default)] +pub(crate) struct FileTransport { + #[cfg(unix)] + inner: Option<crate::platform::unix_image_files::Ledger>, +} + +impl FileTransport { + #[cfg(all(test, unix))] + pub(crate) fn for_test(root: PathBuf) -> Self { + Self { + inner: Some(crate::platform::unix_image_files::Ledger::for_test(root)), + } + } + + pub(crate) fn from_environment() -> Self { + Self { + #[cfg(unix)] + inner: environment_allows().then(crate::platform::unix_image_files::Ledger::new), + } + } + + /// Once per client, return a private 1x1 RGBA query payload. The output + /// boundary must emit this as `a=q,t=t,f=32,s=1,v=1,q=2` before inline image + /// output. No real upload is converted until Ghostty consumes this file. + pub(crate) fn probe(&mut self) -> Option<PathBuf> { + #[cfg(unix)] + { + self.inner.as_mut()?.probe() + } + #[cfg(not(unix))] + { + None + } + } + + /// Returns a complete, closed file containing exactly `data`, or requests + /// inline fallback. The caller must preserve output ordering and encode the + /// returned local path as Kitty's base64 filename payload with `t=t`. + pub(crate) fn prepare(&mut self, data: &[u8]) -> Option<PathBuf> { + #[cfg(unix)] + { + self.inner.as_mut()?.prepare(data) + } + #[cfg(not(unix))] + { + let _ = data; + None + } + } +} + +#[cfg(unix)] +use std::ffi::OsString; +#[cfg(unix)] +use std::io::IsTerminal; + +#[cfg(unix)] +fn environment_allows() -> bool { + allowed( + std::io::stdin().is_terminal(), + std::io::stdout().is_terminal(), + |key| std::env::var_os(key), + ) +} + +#[cfg(unix)] +fn allowed(stdin_tty: bool, stdout_tty: bool, env: impl Fn(&str) -> Option<OsString>) -> bool { + stdin_tty + && stdout_tty + && env("TERM_PROGRAM").as_deref() == Some(std::ffi::OsStr::new("ghostty")) + && [ + "SSH_CONNECTION", + "SSH_TTY", + "SSH_CLIENT", + "TMUX", + "STY", + "HERDR_PANE_ID", + "HERDR_PANE_RUNTIME_ID", + ] + .iter() + .all(|key| env(key).is_none()) + // Named-session selection (HERDR_SESSION) and HERDR_REMOTE_KEYBINDINGS + // deliberately do not affect client locality. +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + #[test] + fn conservative_gate_without_process_environment_mutation() { + let base = |key: &str| match key { + "TERM_PROGRAM" => Some(OsString::from("ghostty")), + "HERDR_SESSION" => Some(OsString::from("named-session")), + "HERDR_REMOTE_KEYBINDINGS" => Some(OsString::from("1")), + _ => None, + }; + assert!(allowed(true, true, base)); + assert!(!allowed(false, true, base)); + assert!(!allowed(true, false, base)); + for key in [ + "SSH_CONNECTION", + "SSH_TTY", + "SSH_CLIENT", + "TMUX", + "STY", + "HERDR_PANE_ID", + "HERDR_PANE_RUNTIME_ID", + ] { + assert!(!allowed(true, true, |name| if name == key { + Some(OsString::new()) + } else { + base(name) + })); + } + assert!(!allowed(true, true, |key| if key == "TERM_PROGRAM" { + Some(OsString::from("kitty")) + } else { + base(key) + })); + } + #[test] + fn default_is_disabled() { + let mut transport = FileTransport::default(); + assert!(transport.probe().is_none()); + assert!(transport.prepare(b"inline").is_none()); + } +} diff --git a/src/client/mod.rs b/src/client/mod.rs index 9ad51a36..1a0f8dd5 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -24,7 +24,10 @@ mod endpoint_commands; mod errors; mod events; mod frame_output; +#[cfg(test)] +mod frame_output_tests; mod handshake; +mod image_files; mod input; mod loop_config; mod notifications; @@ -48,6 +51,8 @@ use events::ClientLoopEvent; use loop_config::ClientLoopConfig; use shell_runtime::*; use state::ClientState; +#[cfg(unix)] +use state::RetiredDirectGraphicsMatch; use transport::*; #[cfg(test)] @@ -135,7 +140,7 @@ use tracing::{debug, info, warn}; use crate::ipc::LocalStream; use crate::protocol::render_ansi; -use crate::protocol::{self, ClientMessage, FrameData, ServerMessage, MAX_GRAPHICS_FRAME_SIZE}; +use crate::protocol::{self, ClientMessage, ServerMessage, MAX_GRAPHICS_FRAME_SIZE}; #[cfg(test)] use crate::protocol::{AttachScrollDirection, AttachScrollSource, NotifyKind}; use crate::server::socket_paths::client_socket_path; @@ -244,6 +249,7 @@ fn run_client_with_mode( endpoint_keybindings, loop_config.mouse_capture_active, true, + !is_remote_client_process(), ) .map_err(|error| io::Error::other(error.to_string()))?; if federated @@ -364,6 +370,31 @@ fn run_client_with_mode( Ok(()) } +// This guards server-supplied paths only. Client-owned temporary files generated +// from received graphics bytes remain usable for remote endpoints. +fn server_graphics_files_allowed( + endpoint_id: &endpoint::ClientEndpointId, + remote_client_process: bool, +) -> bool { + endpoint_id.is_local() && !remote_client_process +} + +#[cfg(unix)] +fn graphics_owner_is_active( + state: &ClientState, + endpoints: &endpoint::EndpointRegistry, + owner: &endpoint::ClientEndpointId, +) -> bool { + endpoints.active_id() == owner + && endpoints + .connection(owner) + .is_some_and(|connection| connection.surface_active) + && state + .shell + .as_ref() + .is_some_and(|shell| shell.endpoint_is_active(owner)) +} + /// The main client event loop. /// /// Uses a threaded architecture: @@ -392,6 +423,7 @@ async fn run_client_loop( let mut state = ClientState { blit_encoder: render_ansi::BlitEncoder::new(), + image_files: image_files::FileTransport::from_environment(), mouse_capture_active: config.mouse_capture_active, endpoint_mouse_capture_requested: false, endpoint_sgr_pixels_requested: false, @@ -410,7 +442,10 @@ async fn run_client_loop( #[cfg(unix)] direct_graphics_response: Arc::new(Mutex::new(direct_graphics::ResponseMatcher::default())), #[cfg(unix)] - retired_direct_graphics: None, + retired_direct_graphics: HashMap::new(), + #[cfg(unix)] + disabled_native_graphics: Default::default(), + pending_native_cleanup: Vec::new(), #[cfg(unix)] pending_surface_graphics: HashMap::new(), attach_escape, @@ -690,10 +725,10 @@ async fn run_client_loop( let frozen = state.presentation_frozen; state.presentation_frozen = false; state.present_graphics(&cleanup); + state.presentation_frozen = frozen; if let Some(frame) = frame { - state.present_frame(frame); + state.present_frozen_chrome(frame); } - state.presentation_frozen = frozen; } } } @@ -937,42 +972,70 @@ async fn run_client_loop( let pending_key = state .pending_surface_graphics .keys() - .find(|(_, transfer_id, image_id)| { + .find(|(_, _, transfer_id, image_id)| { *transfer_id == response.transfer_id && *image_id == response.image_id }) .cloned(); - let owner = pending_key - .as_ref() - .map(|(endpoint_id, _, _)| endpoint_id.clone()); - let composed = pending_key - .and_then(|key| { - state - .pending_surface_graphics - .remove(&key) - .map(|asset| (key, asset)) - }) - .filter(|_| response.success) - .and_then(|((endpoint_id, _, _), asset)| { - if write_stream.active_id() != &endpoint_id { + let pending = pending_key.and_then(|key| { + state + .pending_surface_graphics + .remove(&key) + .map(|asset| (key, asset)) + }); + let Some(((owner, _, _, _), asset)) = pending else { + // Raw/non-surface transfers retain their existing response handling. + continue; + }; + let eligible = response.success + && graphics_owner_is_active(&state, &write_stream, &owner) + && !state.presentation_frozen + && state.shell.as_ref().is_some_and(|shell| { + shell.accepts_direct_graphics_asset(&asset, response.image_id) + }); + let size = state.reported_size; + let prepared = eligible + .then(|| { + let shell = state.shell.as_mut()?; + let checkpoint = shell.direct_graphics_checkpoint(); + if !shell.trust_direct_graphics_asset(&asset, response.image_id) { + shell.restore_direct_graphics_checkpoint(checkpoint); return None; } - let shell = state.shell.as_mut()?; - shell - .trust_direct_graphics_asset(&asset, response.image_id) - .then(|| shell.compose(state.reported_size.0, state.reported_size.1)) - .flatten() - }); + match shell.compose(size.0, size.1) { + Some(frame) => Some((frame, checkpoint)), + None => { + shell.restore_direct_graphics_checkpoint(checkpoint); + None + } + } + }) + .flatten(); + let accepted = if let Some((frame, checkpoint)) = prepared { + if state.try_present_frame(frame) { + true + } else { + state + .shell + .as_mut() + .expect("prepared direct graphics requires a shell") + .restore_direct_graphics_checkpoint(checkpoint); + false + } + } else { + false + }; + if !accepted { + // The upload may have reached the terminal even when its response, owner, or + // composed swap cannot be accepted. Never leave that unowned image resident. + // Restoring the checkpoint also restores old-bank stale-image bookkeeping. + state.queue_native_image_cleanup(response.image_id); + } let message = ClientMessage::GraphicsTransmissionResult { transfer_id: response.transfer_id, image_id: response.image_id, - success: response.success, + success: accepted, }; - if let Some(owner) = owner { - write_stream.send_to(&owner, &message); - } - if let Some(frame) = composed { - state.present_frame(frame); - } + write_stream.send_to(&owner, &message); } #[cfg(unix)] ClientLoopEvent::PixelMouse(data, geometry) => { @@ -1228,6 +1291,8 @@ async fn run_client_loop( shell.compose(state.reported_size.0, state.reported_size.1) }); let reader_quit = writer.stop_handle(); + #[cfg(unix)] + state.start_endpoint_graphics_generation(&endpoint_id, generation); write_stream.insert( endpoint_id.clone(), writer, @@ -1287,6 +1352,30 @@ async fn run_client_loop( continue; } write_stream.received(&endpoint_id, generation, now); + // Retirements belong to the exact live connection, even after deactivation. + // They must not be dropped with frozen/inactive presentation effects. + if matches!( + message.as_ref(), + ServerMessage::GraphicsTransmissionRetired { .. } + ) { + #[cfg(unix)] + if let ServerMessage::GraphicsTransmissionRetired { + transfer_id, + image_id, + } = message.as_ref() + { + let owner_active = + graphics_owner_is_active(&state, &write_stream, &endpoint_id); + state.receive_graphics_retirement( + &endpoint_id, + generation, + *transfer_id, + *image_id, + owner_active, + ); + } + continue; + } let endpoint_active = write_stream.active_id() == &endpoint_id && write_stream .connection(&endpoint_id) @@ -1463,32 +1552,73 @@ async fn run_client_loop( control, surface_asset, } => { + // Never interpret an SSH server's filesystem path on this host, + // including legacy peers that send files without negotiation. + if !server_graphics_files_allowed(&endpoint_id, is_remote_client_process()) + { + write_stream.send_to( + &endpoint_id, + &ClientMessage::GraphicsTransmissionResult { + transfer_id, + image_id, + success: false, + }, + ); + continue; + } #[cfg(unix)] { - if state.retired_direct_graphics.take() - == Some((endpoint_id.clone(), transfer_id, image_id)) - { + let retirement = state.match_retired_direct_graphics( + &endpoint_id, + generation, + transfer_id, + image_id, + ); + if retirement == RetiredDirectGraphicsMatch::Exact { continue; } let surface_asset_valid = match (state.shell.as_ref(), &surface_asset) { (Some(shell), Some(asset)) => { - crate::kitty_graphics::surface::host_image_id( - shell.graphics_scope(), - asset, - ) == image_id + shell.accepts_direct_graphics_asset(asset, image_id) } (None, None) => true, _ => false, }; - let valid = state.kitty_graphics_enabled + let native = surface_asset.as_ref().is_some_and(|asset| { + matches!( + asset.source, + crate::protocol::SurfaceGraphicsSource::Terminal { .. } + ) + }); + let native_valid = !native + || surface_asset.as_ref().is_some_and(|asset| { + !state.presentation_frozen + && state.disabled_native_graphics.get(&endpoint_id) + != Some(&generation) + && graphics_owner_is_active( + &state, + &write_stream, + &endpoint_id, + ) + && leading.is_empty() + && asset.format + == crate::protocol::SurfaceGraphicsFormat::Rgba + && expected_len == asset.data_len + && control + == format!( + "a=t,f=32,s={},v={},i={image_id},q=0", + asset.image_width, asset.image_height + ) + }); + let valid = retirement != RetiredDirectGraphicsMatch::Saturated + && state.kitty_graphics_enabled + && native_valid && surface_asset_valid && usize::try_from(expected_len).ok().is_some_and(|len| { - crate::pane_graphics_files::validate_direct_source( - std::path::Path::new(&path), - len, - ) - .is_ok() - && direct_graphics::valid_control(&control, image_id, len) + let path = std::path::Path::new(&path); + let valid_source = crate::pane_graphics_files::validate_direct_source(path, len).is_ok() + || (native && crate::pane_graphics_files::validate_native_source(path, len).is_ok()); + valid_source && direct_graphics::valid_control(&control, image_id, len) }) && state .direct_graphics_response @@ -1503,8 +1633,9 @@ async fn run_client_loop( &path, ); let mut stdout = io::stdout(); - let written = stdout - .write_all(&command) + let written = state + .flush_native_cleanup(&mut stdout) + .and_then(|()| stdout.write_all(&command)) .and_then(|()| stdout.flush()) .is_ok(); if written { @@ -1517,7 +1648,7 @@ async fn run_client_loop( if sent { if let Some(asset) = surface_asset { state.pending_surface_graphics.insert( - (endpoint_id.clone(), transfer_id, image_id), + (endpoint_id.clone(), generation, transfer_id, image_id), asset, ); } @@ -1528,12 +1659,11 @@ async fn run_client_loop( transfer_id, image_id, }; - if let Err(err) = write_to_server(&mut write_stream, &started) { - return Err(ClientError::ConnectionLost(err)); - } + write_stream.send_to(&endpoint_id, &started); } else { state.pending_surface_graphics.remove(&( endpoint_id.clone(), + generation, transfer_id, image_id, )); @@ -1549,9 +1679,7 @@ async fn run_client_loop( image_id, success: false, }; - if let Err(err) = write_to_server(&mut write_stream, &result) { - return Err(ClientError::ConnectionLost(err)); - } + write_stream.send_to(&endpoint_id, &result); } } #[cfg(not(unix))] @@ -1565,33 +1693,8 @@ async fn run_client_loop( surface_asset, ); } - ServerMessage::GraphicsTransmissionRetired { - transfer_id, - image_id, - } => { - #[cfg(unix)] - { - state.retired_direct_graphics = - Some((endpoint_id.clone(), transfer_id, image_id)); - state.pending_surface_graphics.remove(&( - endpoint_id.clone(), - transfer_id, - image_id, - )); - let cleanup = state.shell.as_mut().map_or_else(Vec::new, |shell| { - shell.retire_direct_graphics_image(image_id); - shell - .compose(state.reported_size.0, state.reported_size.1) - .map(|frame| frame.graphics) - .unwrap_or_else(|| shell.take_pending_graphics_cleanup()) - }); - state.present_graphics(&cleanup); - if let Ok(mut matcher) = state.direct_graphics_response.lock() { - matcher.retire(transfer_id); - } - } - #[cfg(not(unix))] - let _ = (transfer_id, image_id); + ServerMessage::GraphicsTransmissionRetired { .. } => { + unreachable!("retirements are handled before presentation gating") } ServerMessage::ServerShutdown { reason } => { if !federated && endpoint_id.is_local() { diff --git a/src/client/shell/composition.rs b/src/client/shell/composition.rs index 9b710dd5..b7ab4a96 100644 --- a/src/client/shell/composition.rs +++ b/src/client/shell/composition.rs @@ -147,7 +147,11 @@ impl ClientShellState { FrameData::from_ratatui_buffer_with_hyperlinks(&buffer, None, &[]) } - pub(crate) fn compose(&mut self, cols: u16, rows: u16) -> Option<FrameData> { + pub(crate) fn compose( + &mut self, + cols: u16, + rows: u16, + ) -> Option<crate::client::frame_output::ComposedFrame> { self.last_composed_at = Some(std::time::Instant::now()); self.selection_repaint_deadline = None; if self.last_composed_size != Some((cols, rows)) && self.mode == ClientShellMode::Navigate { @@ -167,7 +171,7 @@ impl ClientShellState { && self.navigation_target_valid(&pending.target) }); if self.snapshot.is_none() || self.pane_surface.is_none() { - return Some(self.compose_unavailable(cols, rows)); + return Some(self.compose_unavailable(cols, rows).into()); } let snapshot = self.snapshot.as_deref()?; // Do not compose a retained surface while waiting for its matching snapshot or @@ -716,8 +720,8 @@ impl ClientShellState { self.hits.pane_splits.clear(); self.hits.popup = None; } - self.compose_graphics(&mut frame, layout, &occlusion); - Some(frame) + let graphics = self.compose_graphics(layout, &occlusion); + Some(crate::client::frame_output::ComposedFrame { frame, graphics }) } } diff --git a/src/client/shell/graphics.rs b/src/client/shell/graphics.rs index a7ae81e3..35e61dbb 100644 --- a/src/client/shell/graphics.rs +++ b/src/client/shell/graphics.rs @@ -1,12 +1,37 @@ use super::*; +#[cfg(unix)] +pub(crate) struct ClientGraphicsCheckpoint(crate::kitty_graphics::surface::ClientState); + impl ClientShellState { - #[cfg(unix)] + #[cfg(all(test, unix))] pub(crate) fn graphics_scope(&self) -> &str { self.graphics.scope() } #[cfg(unix)] + pub(crate) fn accepts_direct_graphics_asset( + &self, + key: &crate::protocol::SurfaceGraphicsAssetKey, + image_id: u32, + ) -> bool { + self.graphics.accepts_direct_asset(key, image_id) + } + + #[cfg(unix)] + pub(crate) fn direct_graphics_checkpoint(&self) -> ClientGraphicsCheckpoint { + ClientGraphicsCheckpoint(self.graphics.clone()) + } + + #[cfg(unix)] + pub(crate) fn restore_direct_graphics_checkpoint( + &mut self, + checkpoint: ClientGraphicsCheckpoint, + ) { + self.graphics = checkpoint.0; + } + + #[cfg(unix)] pub(crate) fn trust_direct_graphics_asset( &mut self, key: &crate::protocol::SurfaceGraphicsAssetKey, @@ -33,10 +58,9 @@ impl ClientShellState { pub(super) fn compose_graphics( &mut self, - frame: &mut FrameData, layout: ClientShellLayout, occlusion: &crate::kitty_graphics::surface::Occlusion, - ) { + ) -> crate::kitty_graphics::GraphicsOutput { let visibility = if self.endpoint_error.is_some() { crate::kitty_graphics::surface::Visibility::Hidden } else if self.hits.popup.is_some() { @@ -49,12 +73,12 @@ impl ClientShellState { .popup .as_ref() .map(|popup| (popup.inner_rect.x, popup.inner_rect.y)); - frame.graphics = self.graphics.encode( + self.graphics.encode_output( visibility, (layout.pane_surface.x, layout.pane_surface.y), popup_origin, self.graphics_cell_size, occlusion, - ); + ) } } diff --git a/src/client/shell/state.rs b/src/client/shell/state.rs index 0d6937e0..c170c9b0 100644 --- a/src/client/shell/state.rs +++ b/src/client/shell/state.rs @@ -1015,7 +1015,7 @@ impl ClientShellState { pane_surface_generation: None, pane_surface: None, pending_pane_surface: None, - graphics: crate::kitty_graphics::surface::ClientState::default(), + graphics: crate::kitty_graphics::surface::ClientState::new(), graphics_cell_size: crate::kitty_graphics::HostCellSize { width_px: 1, height_px: 1, diff --git a/src/client/shell/tests/copy.rs b/src/client/shell/tests/copy.rs index 112417fc..67e36f23 100644 --- a/src/client/shell/tests/copy.rs +++ b/src/client/shell/tests/copy.rs @@ -1563,7 +1563,7 @@ fn navigator_narrow_layout_and_long_search_stay_inside_the_popup() { navigator.query = "界".repeat(100).as_str().into(); let frame = state.compose(width, height).expect("navigator frame"); let popup = state.hits.navigator_popup; - let cursor = frame.cursor.expect("search cursor"); + let cursor = frame.cursor.as_ref().expect("search cursor"); assert!(super::super::contains(popup, (cursor.x, cursor.y))); assert!(state.hits.navigator_rows.is_empty()); assert!(popup.right() <= width && popup.bottom() <= height); diff --git a/src/client/shell/tests/graphics.rs b/src/client/shell/tests/graphics.rs index 240e031c..e54301a4 100644 --- a/src/client/shell/tests/graphics.rs +++ b/src/client/shell/tests/graphics.rs @@ -63,6 +63,48 @@ fn add_main_image( surface.graphics.placements.push(placement); } +#[cfg(unix)] +#[test] +fn failed_direct_ack_composition_restores_graphics_for_inline_retry() { + use crate::kitty_graphics::surface::host_image_id; + + let mut state = ClientShellState::new(ClientShellConfig::from_config(&Config::default())); + state.set_snapshot(Box::new(snapshot())); + let mut pane = surface(); + let (mut asset, mut placement) = image( + SurfaceGraphicsTarget::Pane { + pane_id: "pane_1".into(), + }, + 0, + 0, + 71, + ); + asset.key.source = SurfaceGraphicsSource::PaneLayer { + pane_id: "pane_1".into(), + layer_id: "direct-rollback".into(), + }; + placement.asset = asset.key.clone(); + let key = asset.key.clone(); + pane.graphics.assets.push(asset); + pane.graphics.placements.push(placement); + state.set_pane_surface(pane); + + // Model an ACK arriving while projection pairing makes composition unavailable. + state.pending_pane_surface = Some(surface()); + let image_id = host_image_id(state.graphics_scope(), &key); + let graphics_before = format!("{:?}", state.graphics); + let checkpoint = state.direct_graphics_checkpoint(); + assert!(state.trust_direct_graphics_asset(&key, image_id)); + assert!(state.compose(106, 20).is_none()); + state.restore_direct_graphics_checkpoint(checkpoint); + assert_eq!(format!("{:?}", state.graphics), graphics_before); + + // Once pairing is available, fallback still owns the asset and uploads it normally. + state.pending_pane_surface = None; + let retry = state.compose(106, 20).expect("fallback frame"); + assert!(String::from_utf8_lossy(&retry.graphics.into_inline_bytes()).contains("a=t")); +} + fn is_placed(bytes: &[u8], point: (u16, u16)) -> bool { String::from_utf8_lossy(bytes).contains(&format!("\x1b[{};{}H", point.1 + 1, point.0 + 1)) } @@ -85,11 +127,11 @@ fn assert_graphics_cover(state: &mut ClientShellState, covered: Rect, cols: u16, state.set_pane_surface(surface); let frame = state.compose(cols, rows).unwrap(); assert!( - is_placed(&frame.graphics, outside), + is_placed(&frame.graphics.clone().into_inline_bytes(), outside), "outside={outside:?} cover={covered:?}" ); assert!( - !is_placed(&frame.graphics, inside), + !is_placed(&frame.graphics.clone().into_inline_bytes(), inside), "inside={inside:?} cover={covered:?}" ); } @@ -327,18 +369,38 @@ fn every_dialog_and_menu_occludes_its_panel_not_the_whole_screen() { state.compose(106, 40).unwrap(); state.overlay = Some(overlay); let frame = state.compose(106, 40).unwrap(); - assert!(is_placed(&frame.graphics, outside), "{:?}", state.overlay); + assert!( + is_placed(&frame.graphics.clone().into_inline_bytes(), outside), + "{:?}", + state.overlay + ); if let Some(border) = border { - assert!(!is_placed(&frame.graphics, border), "{:?}", state.overlay); - assert!(String::from_utf8_lossy(&frame.graphics).contains("a=d,d=i")); + assert!( + !is_placed(&frame.graphics.clone().into_inline_bytes(), border), + "{:?}", + state.overlay + ); + assert!( + String::from_utf8_lossy(&frame.graphics.clone().into_inline_bytes()) + .contains("a=d,d=i") + ); } state.overlay = None; let restored = state.compose(106, 40).unwrap(); - assert!(is_placed(&restored.graphics, outside)); + assert!(is_placed( + &restored.graphics.clone().into_inline_bytes(), + outside + )); if let Some(border) = border { - assert!(is_placed(&restored.graphics, border)); + assert!(is_placed( + &restored.graphics.clone().into_inline_bytes(), + border + )); } - assert!(!String::from_utf8_lossy(&restored.graphics).contains("a=t")); + assert!( + !String::from_utf8_lossy(&restored.graphics.clone().into_inline_bytes()) + .contains("a=t") + ); } } @@ -377,17 +439,26 @@ fn selection_copy_cursor_and_search_hide_only_the_highlighted_images() { }); let frame = state.compose(106, 20).unwrap(); for point in &points[..3] { - assert!(!is_placed(&frame.graphics, *point), "{point:?}"); + assert!( + !is_placed(&frame.graphics.clone().into_inline_bytes(), *point), + "{point:?}" + ); } - assert!(is_placed(&frame.graphics, points[3])); + assert!(is_placed( + &frame.graphics.clone().into_inline_bytes(), + points[3] + )); state.mode = ClientShellMode::Terminal; state.copy_mode = None; state.selection = None; let frame = state.compose(106, 20).unwrap(); for point in points { - assert!(is_placed(&frame.graphics, point)); + assert!(is_placed( + &frame.graphics.clone().into_inline_bytes(), + point + )); } - assert!(!String::from_utf8_lossy(&frame.graphics).contains("a=t")); + assert!(!String::from_utf8_lossy(&frame.graphics.clone().into_inline_bytes()).contains("a=t")); } #[test] @@ -400,15 +471,26 @@ fn mobile_switcher_still_hides_the_entire_underlying_surface() { add_main_image(&mut surface, layout, point, 1); state.set_pane_surface(surface); let frame = state.compose(40, 24).unwrap(); - assert!(is_placed(&frame.graphics, point)); + assert!(is_placed( + &frame.graphics.clone().into_inline_bytes(), + point + )); state.mode = ClientShellMode::Navigate; let frame = state.compose(40, 24).unwrap(); - assert!(!is_placed(&frame.graphics, point)); - assert!(String::from_utf8_lossy(&frame.graphics).contains("a=d,d=i")); + assert!(!is_placed( + &frame.graphics.clone().into_inline_bytes(), + point + )); + assert!( + String::from_utf8_lossy(&frame.graphics.clone().into_inline_bytes()).contains("a=d,d=i") + ); state.mode = ClientShellMode::Terminal; let frame = state.compose(40, 24).unwrap(); - assert!(is_placed(&frame.graphics, point)); - assert!(!String::from_utf8_lossy(&frame.graphics).contains("a=t")); + assert!(is_placed( + &frame.graphics.clone().into_inline_bytes(), + point + )); + assert!(!String::from_utf8_lossy(&frame.graphics.clone().into_inline_bytes()).contains("a=t")); } #[test] @@ -437,15 +519,33 @@ fn popup_terminal_keeps_own_graphics_and_hides_only_background_overlap() { surface.graphics.placements.push(placement); state.set_pane_surface(surface); let frame = state.compose(106, 20).unwrap(); - assert!(is_placed(&frame.graphics, outside)); - assert!(!is_placed(&frame.graphics, border)); - assert!(is_placed(&frame.graphics, inside)); + assert!(is_placed( + &frame.graphics.clone().into_inline_bytes(), + outside + )); + assert!(!is_placed( + &frame.graphics.clone().into_inline_bytes(), + border + )); + assert!(is_placed( + &frame.graphics.clone().into_inline_bytes(), + inside + )); state.overlay = Some(ClientShellOverlay::Onboarding); let frame = state.compose(106, 20).unwrap(); - assert!(is_placed(&frame.graphics, outside)); - assert!(!is_placed(&frame.graphics, inside)); + assert!(is_placed( + &frame.graphics.clone().into_inline_bytes(), + outside + )); + assert!(!is_placed( + &frame.graphics.clone().into_inline_bytes(), + inside + )); state.overlay = None; let frame = state.compose(106, 20).unwrap(); - assert!(is_placed(&frame.graphics, inside)); - assert!(!String::from_utf8_lossy(&frame.graphics).contains("a=t")); + assert!(is_placed( + &frame.graphics.clone().into_inline_bytes(), + inside + )); + assert!(!String::from_utf8_lossy(&frame.graphics.clone().into_inline_bytes()).contains("a=t")); } diff --git a/src/client/shell/tests/input.rs b/src/client/shell/tests/input.rs index b55cf9a2..b54548f2 100644 --- a/src/client/shell/tests/input.rs +++ b/src/client/shell/tests/input.rs @@ -216,18 +216,24 @@ fn client_shell_graphics_follow_final_shell_origin_and_local_overlay_visibility( state.set_pane_surface(pane_surface); let visible = state.compose(106, 20).expect("visible graphics frame"); - let visible = String::from_utf8_lossy(&visible.graphics); + let visible = visible.graphics.clone().into_inline_bytes(); + let visible = String::from_utf8_lossy(&visible); assert!(visible.contains("a=t,t=d")); assert!(visible.contains("\u{1b}[2;27H")); state.overlay = Some(ClientShellOverlay::Onboarding); let uncovered = state.compose(106, 20).expect("overlay frame"); - assert!(!String::from_utf8_lossy(&uncovered.graphics).contains("a=d")); - assert!(String::from_utf8_lossy(&uncovered.graphics).contains("a=p")); + assert!( + !String::from_utf8_lossy(&uncovered.graphics.clone().into_inline_bytes()).contains("a=d") + ); + assert!( + String::from_utf8_lossy(&uncovered.graphics.clone().into_inline_bytes()).contains("a=p") + ); state.overlay = None; let restored = state.compose(106, 20).expect("restored graphics frame"); - let restored = String::from_utf8_lossy(&restored.graphics); + let restored = restored.graphics.clone().into_inline_bytes(); + let restored = String::from_utf8_lossy(&restored); assert!(restored.contains("a=p")); assert!(!restored.contains("a=t,t=d")); } diff --git a/src/client/shell/tests/text_editing.rs b/src/client/shell/tests/text_editing.rs index b63ce9f6..118a606e 100644 --- a/src/client/shell/tests/text_editing.rs +++ b/src/client/shell/tests/text_editing.rs @@ -384,7 +384,7 @@ fn every_field_renders_long_unicode_across_resize_without_mutation() { for (width, height) in [(120, 40), (60, 20), (12, 6), (1, 1), (120, 40)] { let before = editor(&mut state).clone(); if let Some(frame) = state.compose(width, height) { - if let Some(cursor) = frame.cursor.filter(|cursor| cursor.visible) { + if let Some(cursor) = frame.cursor.as_ref().filter(|cursor| cursor.visible) { assert!(cursor.x < width && cursor.y < height, "field {field}"); } } diff --git a/src/client/shell_runtime.rs b/src/client/shell_runtime.rs index 3843654f..d0eafdf6 100644 --- a/src/client/shell_runtime.rs +++ b/src/client/shell_runtime.rs @@ -385,14 +385,6 @@ pub(super) fn complete_endpoint_activation( completion, endpoint::ActivationCompletion::AwaitingPresentationSync { .. } ) { - #[cfg(unix)] - if let endpoint::ActivationCompletion::AwaitingPresentationSync { previous, endpoint } = - &completion - { - if previous != endpoint { - state.retire_endpoint_graphics(previous); - } - } // The coherent target frame can replace the frozen source now, but the registry keeps // pane input disabled until a second projection epoch has replayed host modes/effects. state.unfreeze_presentation(); @@ -511,7 +503,7 @@ pub(super) fn handle_endpoint_disconnect( ) -> bool { supervisors.disconnected(endpoint_id, generation, now); #[cfg(unix)] - state.retire_endpoint_graphics(endpoint_id); + state.retire_endpoint_graphics(endpoint_id, generation); if pending_activation .as_ref() .is_some_and(|pending| pending.involves_endpoint(endpoint_id)) @@ -572,7 +564,7 @@ pub(super) fn handle_endpoint_attention( now, ); #[cfg(unix)] - state.retire_endpoint_graphics(endpoint_id); + state.retire_endpoint_graphics(endpoint_id, generation); if pending_activation .as_ref() .is_some_and(|pending| pending.involves_endpoint(endpoint_id)) @@ -681,7 +673,7 @@ pub(super) fn install_client_shell_snapshot( pub(super) fn finish_client_shell_input( state: &mut ClientState, outcome: shell::ClientShellInput, - frame: Option<FrameData>, + frame: Option<super::frame_output::ComposedFrame>, endpoints: &mut endpoint::EndpointRegistry, pending_activation: &mut Option<endpoint::PendingEndpointActivation>, endpoint_commands: &mut endpoint_commands::EndpointCommands, diff --git a/src/client/state.rs b/src/client/state.rs index 83b07429..e6b8abf3 100644 --- a/src/client/state.rs +++ b/src/client/state.rs @@ -1,9 +1,39 @@ use super::*; +#[cfg(unix)] +const MAX_RETIRED_DIRECT_GRAPHICS: usize = 64; + +#[cfg(unix)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum RetiredDirectGraphicsMatch { + None, + Exact, + Saturated, +} + +#[cfg(unix)] +pub(super) struct RetiredDirectGraphics { + generation: u64, + transfers: Vec<(u64, u32)>, + saturated: bool, +} + +#[cfg(unix)] +impl RetiredDirectGraphics { + fn new(generation: u64) -> Self { + Self { + generation, + transfers: Vec::new(), + saturated: false, + } + } +} + /// State tracking for the thin client. pub(super) struct ClientState { /// Stateful semantic-frame encoder used when the server sends FrameData. pub(super) blit_encoder: render_ansi::BlitEncoder, + pub(super) image_files: image_files::FileTransport, pub(super) mouse_capture_active: bool, pub(super) endpoint_mouse_capture_requested: bool, pub(super) endpoint_sgr_pixels_requested: bool, @@ -24,10 +54,15 @@ pub(super) struct ClientState { #[cfg(unix)] pub(super) direct_graphics_response: Arc<Mutex<direct_graphics::ResponseMatcher>>, #[cfg(unix)] - pub(super) retired_direct_graphics: Option<(endpoint::ClientEndpointId, u64, u32)>, + pub(super) retired_direct_graphics: HashMap<endpoint::ClientEndpointId, RetiredDirectGraphics>, #[cfg(unix)] - pub(super) pending_surface_graphics: - HashMap<(endpoint::ClientEndpointId, u64, u32), crate::protocol::SurfaceGraphicsAssetKey>, + pub(super) disabled_native_graphics: HashMap<endpoint::ClientEndpointId, u64>, + pub(super) pending_native_cleanup: Vec<u8>, + #[cfg(unix)] + pub(super) pending_surface_graphics: HashMap< + (endpoint::ClientEndpointId, u64, u64, u32), + crate::protocol::SurfaceGraphicsAssetKey, + >, pub(super) attach_escape: Option<AttachEscapeState>, #[cfg(unix)] pub(super) mouse_scroll_lines: usize, @@ -63,6 +98,7 @@ impl ClientState { pub(super) fn test_new() -> Self { Self { blit_encoder: render_ansi::BlitEncoder::new(), + image_files: image_files::FileTransport::default(), mouse_capture_active: false, endpoint_mouse_capture_requested: false, endpoint_sgr_pixels_requested: false, @@ -81,7 +117,10 @@ impl ClientState { #[cfg(unix)] direct_graphics_response: Default::default(), #[cfg(unix)] - retired_direct_graphics: None, + retired_direct_graphics: HashMap::new(), + #[cfg(unix)] + disabled_native_graphics: Default::default(), + pending_native_cleanup: Vec::new(), #[cfg(unix)] pending_surface_graphics: HashMap::new(), attach_escape: None, @@ -162,11 +201,109 @@ impl ClientState { /// Present a composed error/chrome frame while retaining the handoff input freeze. The pane /// cells are still the last coherent surface; only client chrome (including the error) moves. - pub(super) fn present_frozen_chrome(&mut self, frame_data: FrameData) { + pub(super) fn present_frozen_chrome( + &mut self, + frame_data: impl Into<frame_output::ComposedFrame>, + ) { let frozen = self.presentation_frozen; + // Chrome can repaint the frozen source, but cannot retire its staged images. + let deferred_cleanup = frozen.then(|| std::mem::take(&mut self.pending_native_cleanup)); self.presentation_frozen = false; self.present_frame(frame_data); self.presentation_frozen = frozen; + if let Some(cleanup) = deferred_cleanup { + self.pending_native_cleanup = cleanup; + } + } + + #[cfg(unix)] + pub(super) fn queue_native_image_cleanup(&mut self, image_id: u32) { + crate::kitty_graphics::encode_delete_image(&mut self.pending_native_cleanup, image_id); + } + + /// Retirement is lifecycle bookkeeping, not a presentation effect. The caller has + /// already checked the endpoint generation, even for inactive/frozen owners. + #[cfg(unix)] + pub(super) fn receive_graphics_retirement( + &mut self, + endpoint_id: &endpoint::ClientEndpointId, + generation: u64, + transfer_id: u64, + image_id: u32, + owner_active: bool, + ) { + if transfer_id & crate::kitty_graphics::surface::NATIVE_TRANSFER_BIT != 0 { + self.disabled_native_graphics + .insert(endpoint_id.clone(), generation); + } + self.record_retired_direct_graphics(endpoint_id.clone(), generation, transfer_id, image_id); + let upload_pending = self + .pending_surface_graphics + .remove(&(endpoint_id.clone(), generation, transfer_id, image_id)) + .is_some(); + // Retirement cleanup is terminal-owned. A native retirement owns the + // ID only while its exact upload is pending: collision rejection and a + // late post-ACK retirement must not delete the currently visible bank. + // API/non-native retirement retains its prior ownership behavior. + let owns_image = self.queue_retired_graphics_cleanup( + transfer_id, + image_id, + upload_pending, + owner_active, + ); + let frame = if owns_image && owner_active { + let frozen = self.presentation_frozen; + let size = self.reported_size; + self.shell.as_mut().and_then(|shell| { + shell.retire_direct_graphics_image(image_id); + (!frozen).then(|| shell.compose(size.0, size.1)).flatten() + }) + } else { + None + }; + // A retirement repaint is a normal synchronized frame. In particular, + // do not extract its graphics into a graphics-only swap. + if let Some(frame) = frame { + self.present_frame(frame); + } + if let Ok(mut matcher) = self.direct_graphics_response.lock() { + matcher.retire(transfer_id); + } + } + + /// Returns whether a retirement owns an image that may be removed from the terminal/cache. + /// Native rejections can retire a proposed colliding ID without ever uploading it; only an + /// exact still-pending native upload gives that retirement ownership of the image. + #[cfg(unix)] + pub(super) fn queue_retired_graphics_cleanup( + &mut self, + transfer_id: u64, + image_id: u32, + upload_pending: bool, + owner_active: bool, + ) -> bool { + let native = transfer_id & crate::kitty_graphics::surface::NATIVE_TRANSFER_BIT != 0; + if native && !upload_pending { + return false; + } + if native || !owner_active { + self.queue_native_image_cleanup(image_id); + } + true + } + + #[cfg(unix)] + pub(super) fn flush_native_cleanup( + &mut self, + writer: &mut impl std::io::Write, + ) -> io::Result<()> { + if self.presentation_frozen || self.pending_native_cleanup.is_empty() { + return Ok(()); + } + writer.write_all(&self.pending_native_cleanup)?; + writer.flush()?; + self.pending_native_cleanup.clear(); + Ok(()) } pub(super) fn present_graphics(&mut self, graphics: &[u8]) { @@ -182,7 +319,10 @@ impl ClientState { &mut self, patch: shell::ClientComposedSurfacePatch, ) -> io::Result<bool> { - if self.presentation_frozen || self.repaint_pending { + if self.presentation_frozen + || self.repaint_pending + || (self.kitty_graphics_enabled && !self.pending_native_cleanup.is_empty()) + { crate::render_prof::event("client_surface_patch.fallback.repaint"); return Ok(false); } @@ -222,21 +362,28 @@ impl ClientState { } #[cfg(unix)] - pub(super) fn retire_endpoint_graphics(&mut self, endpoint_id: &endpoint::ClientEndpointId) { - let transfer_ids = self + fn retire_pending_endpoint_graphics( + &mut self, + endpoint_id: &endpoint::ClientEndpointId, + generation: Option<u64>, + ) { + let retired = self .pending_surface_graphics .keys() - .filter(|(owner, _, _)| owner == endpoint_id) - .map(|(_, transfer_id, _)| *transfer_id) + .filter(|(owner, pending_generation, _, _)| { + owner == endpoint_id + && generation.is_none_or(|generation| *pending_generation == generation) + }) + .cloned() .collect::<Vec<_>>(); - self.pending_surface_graphics - .retain(|(owner, _, _), _| owner != endpoint_id); - if self - .retired_direct_graphics - .as_ref() - .is_some_and(|(owner, _, _)| owner == endpoint_id) - { - self.retired_direct_graphics = None; + let mut transfer_ids = Vec::with_capacity(retired.len()); + for key in retired { + let (_, _, transfer_id, image_id) = &key; + if transfer_id & crate::kitty_graphics::surface::NATIVE_TRANSFER_BIT != 0 { + self.queue_native_image_cleanup(*image_id); + } + transfer_ids.push(*transfer_id); + self.pending_surface_graphics.remove(&key); } if let Ok(mut matcher) = self.direct_graphics_response.lock() { for transfer_id in transfer_ids { @@ -245,10 +392,162 @@ impl ClientState { } } - pub(super) fn present_frame(&mut self, frame_data: FrameData) { - if self.presentation_frozen { + #[cfg(unix)] + pub(super) fn record_retired_direct_graphics( + &mut self, + endpoint_id: endpoint::ClientEndpointId, + generation: u64, + transfer_id: u64, + image_id: u32, + ) { + let retired = self + .retired_direct_graphics + .entry(endpoint_id) + .or_insert_with(|| RetiredDirectGraphics::new(generation)); + if retired.generation != generation { + *retired = RetiredDirectGraphics::new(generation); + } + if retired.saturated || retired.transfers.contains(&(transfer_id, image_id)) { + return; + } + if retired.transfers.len() == MAX_RETIRED_DIRECT_GRAPHICS { + // Never evict an older tombstone: a delayed file for an evicted tuple could otherwise + // overwrite a visible image. Reject every unrecognized file until generation reset. + retired.saturated = true; return; } + retired.transfers.push((transfer_id, image_id)); + } + + #[cfg(unix)] + pub(super) fn match_retired_direct_graphics( + &mut self, + endpoint_id: &endpoint::ClientEndpointId, + generation: u64, + transfer_id: u64, + image_id: u32, + ) -> RetiredDirectGraphicsMatch { + let Some(retired) = self.retired_direct_graphics.get_mut(endpoint_id) else { + return RetiredDirectGraphicsMatch::None; + }; + if retired.generation != generation { + return RetiredDirectGraphicsMatch::None; + } + if let Some(index) = retired + .transfers + .iter() + .position(|transfer| *transfer == (transfer_id, image_id)) + { + retired.transfers.swap_remove(index); + if retired.transfers.is_empty() && !retired.saturated { + self.retired_direct_graphics.remove(endpoint_id); + } + return RetiredDirectGraphicsMatch::Exact; + } + if retired.saturated { + RetiredDirectGraphicsMatch::Saturated + } else { + RetiredDirectGraphicsMatch::None + } + } + + /// Forget graphics guards only for the connection generation which actually disconnected. + /// A delayed disconnect from an older generation must not clear its replacement's guards. + #[cfg(unix)] + pub(super) fn retire_endpoint_graphics( + &mut self, + endpoint_id: &endpoint::ClientEndpointId, + generation: u64, + ) { + self.retire_pending_endpoint_graphics(endpoint_id, Some(generation)); + if self.disabled_native_graphics.get(endpoint_id) == Some(&generation) { + self.disabled_native_graphics.remove(endpoint_id); + } + if self + .retired_direct_graphics + .get(endpoint_id) + .is_some_and(|retired| retired.generation == generation) + { + self.retired_direct_graphics.remove(endpoint_id); + } + } + + /// Reset bounded per-endpoint state when a replacement connection becomes authoritative. + #[cfg(unix)] + pub(super) fn start_endpoint_graphics_generation( + &mut self, + endpoint_id: &endpoint::ClientEndpointId, + generation: u64, + ) { + self.retire_pending_endpoint_graphics(endpoint_id, None); + if self.disabled_native_graphics.get(endpoint_id) != Some(&generation) { + self.disabled_native_graphics.remove(endpoint_id); + } + if self + .retired_direct_graphics + .get(endpoint_id) + .is_some_and(|retired| retired.generation != generation) + { + self.retired_direct_graphics.remove(endpoint_id); + } + } + + #[cfg(unix)] + pub(super) fn forget_endpoint_graphics(&mut self, endpoint_id: &endpoint::ClientEndpointId) { + self.retire_pending_endpoint_graphics(endpoint_id, None); + self.disabled_native_graphics.remove(endpoint_id); + self.retired_direct_graphics.remove(endpoint_id); + } + + pub(super) fn present_frame(&mut self, frame_data: impl Into<frame_output::ComposedFrame>) { + let _ = self.try_present_frame(frame_data); + } + + fn write_composed_output( + &mut self, + writer: &mut impl io::Write, + encoded: &[u8], + mut graphics: crate::kitty_graphics::GraphicsOutput, + ) -> io::Result<()> { + if self.kitty_graphics_enabled { + if !self.pending_native_cleanup.is_empty() { + graphics.operations.insert( + 0, + crate::kitty_graphics::GraphicsOperation::Bytes( + self.pending_native_cleanup.clone(), + ), + ); + } + frame_output::write_composed_frame( + writer.by_ref(), + encoded, + &graphics, + &mut self.image_files, + )?; + } else { + writer.write_all(encoded)?; + } + writer.flush()?; + if self.kitty_graphics_enabled { + self.pending_native_cleanup.clear(); + } + Ok(()) + } + + /// Presents and commits a frame only after all terminal output has been written successfully. + /// Callers which acknowledge presentation-sensitive work use the return value rather than + /// treating composition as presentation. + pub(super) fn try_present_frame( + &mut self, + frame_data: impl Into<frame_output::ComposedFrame>, + ) -> bool { + if self.presentation_frozen { + return false; + } + let frame_output::ComposedFrame { + frame: frame_data, + graphics, + } = frame_data.into(); let frame_data = if self.draw_host_cursor { render_ansi::frame_with_drawn_cursor(frame_data) } else { @@ -261,14 +560,280 @@ impl ClientState { self.blit_encoder.encode(&frame_data, self.repaint_pending) }; let mut stdout = io::stdout(); - let graphics = if self.kitty_graphics_enabled { - frame_data.graphics.as_slice() - } else { - &[] - }; - let _ = write_encoded_frame_with_graphics(&mut stdout, &encoded.bytes, graphics); - let _ = stdout.flush(); + if let Err(error) = self.write_composed_output(&mut stdout, &encoded.bytes, graphics) { + tracing::warn!(%error, "failed to present client frame"); + self.repaint_pending = true; + return false; + } self.blit_encoder.commit(frame_data, encoded); self.repaint_pending = false; + true + } +} + +#[cfg(all(test, unix))] +mod native_cleanup_tests { + use super::*; + + #[test] + fn retired_graphics_tombstones_survive_handoff_and_only_exact_files_consume_them() { + let mut state = ClientState::test_new(); + let endpoint = endpoint::ClientEndpointId::Local; + state.record_retired_direct_graphics(endpoint.clone(), 7, 11, 1234); + state.record_retired_direct_graphics(endpoint.clone(), 7, 12, 5678); + state.disabled_native_graphics.insert(endpoint.clone(), 7); + + // Switching away and back does not reset guards, and an unrelated queued file cannot + // consume either of two sequential API retirement tombstones. + assert_eq!( + state.match_retired_direct_graphics(&endpoint, 7, 13, 9999), + RetiredDirectGraphicsMatch::None + ); + assert_eq!( + state + .retired_direct_graphics + .get(&endpoint) + .unwrap() + .transfers + .len(), + 2 + ); + assert_eq!(state.disabled_native_graphics.get(&endpoint), Some(&7)); + assert_eq!( + state.match_retired_direct_graphics(&endpoint, 7, 11, 1234), + RetiredDirectGraphicsMatch::Exact + ); + assert!(state + .retired_direct_graphics + .get(&endpoint) + .unwrap() + .transfers + .contains(&(12, 5678))); + assert_eq!( + state.match_retired_direct_graphics(&endpoint, 7, 12, 5678), + RetiredDirectGraphicsMatch::Exact + ); + assert!(!state.retired_direct_graphics.contains_key(&endpoint)); + } + + #[test] + fn replacement_generation_resets_bounded_graphics_guards() { + let mut state = ClientState::test_new(); + let endpoint = endpoint::ClientEndpointId::Local; + let native = crate::kitty_graphics::surface::NATIVE_TRANSFER_BIT | 11; + state.record_retired_direct_graphics(endpoint.clone(), 7, native, 1234); + state.disabled_native_graphics.insert(endpoint.clone(), 7); + + state.start_endpoint_graphics_generation(&endpoint, 8); + + assert_eq!( + state.match_retired_direct_graphics(&endpoint, 7, native, 1234), + RetiredDirectGraphicsMatch::None + ); + assert!(!state.disabled_native_graphics.contains_key(&endpoint)); + } + + #[test] + fn retired_graphics_tombstone_overflow_fails_closed_until_generation_reset() { + let mut state = ClientState::test_new(); + let endpoint = endpoint::ClientEndpointId::Local; + for transfer_id in 0..=MAX_RETIRED_DIRECT_GRAPHICS as u64 { + state.record_retired_direct_graphics( + endpoint.clone(), + 7, + transfer_id, + transfer_id as u32, + ); + } + let retired = state.retired_direct_graphics.get(&endpoint).unwrap(); + assert_eq!(retired.transfers.len(), MAX_RETIRED_DIRECT_GRAPHICS); + assert!(retired.saturated); + assert_eq!( + state.match_retired_direct_graphics(&endpoint, 7, 1, 1), + RetiredDirectGraphicsMatch::Exact + ); + assert_eq!( + state.match_retired_direct_graphics(&endpoint, 7, 10_000, 10_000), + RetiredDirectGraphicsMatch::Saturated + ); + assert!( + state + .retired_direct_graphics + .get(&endpoint) + .unwrap() + .saturated + ); + + state.start_endpoint_graphics_generation(&endpoint, 8); + assert!(!state.retired_direct_graphics.contains_key(&endpoint)); + state.record_retired_direct_graphics(endpoint.clone(), 8, 1, 2); + assert_eq!( + state.match_retired_direct_graphics(&endpoint, 8, 9, 9), + RetiredDirectGraphicsMatch::None + ); + assert_eq!( + state.match_retired_direct_graphics(&endpoint, 8, 1, 2), + RetiredDirectGraphicsMatch::Exact + ); + } + + #[test] + fn pending_none_native_retirement_does_not_touch_cleanup_or_shell_cache() { + let mut state = ClientState::test_new(); + state.pending_native_cleanup.extend_from_slice(b"existing"); + let transfer = crate::kitty_graphics::surface::NATIVE_TRANSFER_BIT | 7; + + assert!(!state.queue_retired_graphics_cleanup(transfer, 1234, false, true)); + assert_eq!(state.pending_native_cleanup, b"existing"); + assert!(state + .shell + .as_mut() + .expect("test shell") + .take_pending_graphics_cleanup() + .is_empty()); + } + + #[test] + fn queued_cleanup_is_synchronized_and_respects_graphics_capability() { + let mut state = ClientState::test_new(); + state.kitty_graphics_enabled = true; + state.queue_native_image_cleanup(42); + let encoded = b"\x1b[?2026htext\x1b[?2026l"; + let mut output = Vec::new(); + state + .write_composed_output(&mut output, encoded, Default::default()) + .unwrap(); + let text = String::from_utf8(output).unwrap(); + assert!(text.find("\x1b[?2026h").unwrap() < text.find("a=d,d=I,i=42").unwrap()); + assert!(text.find("a=d,d=I,i=42").unwrap() < text.find("\x1b[?2026l").unwrap()); + assert!(state.pending_native_cleanup.is_empty()); + state.kitty_graphics_enabled = false; + state.queue_native_image_cleanup(43); + let mut output = Vec::new(); + state + .write_composed_output(&mut output, encoded, Default::default()) + .unwrap(); + assert_eq!(output, encoded); + assert!(!state.pending_native_cleanup.is_empty()); + state.kitty_graphics_enabled = true; + let mut no_capacity = &mut [][..]; + assert!(state + .write_composed_output(&mut no_capacity, encoded, Default::default()) + .is_err()); + assert!(!state.pending_native_cleanup.is_empty()); + } + + #[test] + fn active_api_retirement_leaves_cleanup_to_composition() { + let mut state = ClientState::test_new(); + assert!(state.queue_retired_graphics_cleanup(7, 1234, true, true)); + assert!(state.pending_native_cleanup.is_empty()); + assert!(state.queue_retired_graphics_cleanup(8, 5678, true, false)); + assert!(!state.pending_native_cleanup.is_empty()); + } + + #[test] + fn unacknowledged_native_upload_cleanup_survives_frozen_disconnect() { + let mut state = ClientState::test_new(); + let endpoint = endpoint::ClientEndpointId::Local; + let key = crate::protocol::SurfaceGraphicsAssetKey { + source: crate::protocol::SurfaceGraphicsSource::Terminal { + target: crate::protocol::SurfaceGraphicsTarget::Pane { + pane_id: "p".into(), + }, + image_id: 1, + }, + image_width: 1, + image_height: 1, + format: crate::protocol::SurfaceGraphicsFormat::Rgba, + data_len: 4, + data_fingerprint: 1, + }; + let transfer = crate::kitty_graphics::surface::NATIVE_TRANSFER_BIT | 2; + state + .pending_surface_graphics + .insert((endpoint.clone(), 9, transfer, 1234), key); + state.presentation_frozen = true; + state.retire_endpoint_graphics(&endpoint, 9); + assert!(state.pending_surface_graphics.is_empty()); + let mut output = Vec::new(); + state.flush_native_cleanup(&mut output).unwrap(); + assert!(output.is_empty()); + assert!(!state.pending_native_cleanup.is_empty()); + state.unfreeze_presentation(); + state.flush_native_cleanup(&mut output).unwrap(); + assert!(String::from_utf8(output.clone()) + .unwrap() + .contains("a=d,d=I,i=1234")); + state.flush_native_cleanup(&mut output).unwrap(); + assert_eq!( + String::from_utf8(output).unwrap().matches("a=d,").count(), + 1 + ); + } + + #[test] + fn unacknowledged_native_upload_retirement_survives_frozen_handoff() { + let mut state = ClientState::test_new(); + let endpoint = endpoint::ClientEndpointId::Local; + let key = crate::protocol::SurfaceGraphicsAssetKey { + source: crate::protocol::SurfaceGraphicsSource::Terminal { + target: crate::protocol::SurfaceGraphicsTarget::Pane { + pane_id: "p".into(), + }, + image_id: 1, + }, + image_width: 1, + image_height: 1, + format: crate::protocol::SurfaceGraphicsFormat::Rgba, + data_len: 4, + data_fingerprint: 1, + }; + let transfer = crate::kitty_graphics::surface::NATIVE_TRANSFER_BIT | 2; + state + .pending_surface_graphics + .insert((endpoint.clone(), 9, transfer, 1234), key); + state.presentation_frozen = true; + // The source connection is still live, but no longer owns the active shell. + state.receive_graphics_retirement(&endpoint, 8, transfer, 1234, false); + assert_eq!(state.pending_surface_graphics.len(), 1); + assert!(state.pending_native_cleanup.is_empty()); + state.receive_graphics_retirement(&endpoint, 9, transfer, 1234, false); + assert_eq!( + state.match_retired_direct_graphics(&endpoint, 9, transfer, 1234), + RetiredDirectGraphicsMatch::Exact + ); + assert!(state + .shell + .as_mut() + .unwrap() + .take_pending_graphics_cleanup() + .is_empty()); + assert!(state.pending_surface_graphics.is_empty()); + let mut output = Vec::new(); + state.flush_native_cleanup(&mut output).unwrap(); + assert!(output.is_empty()); + assert!(!state.pending_native_cleanup.is_empty()); + let cleanup = state.pending_native_cleanup.clone(); + let frame = crate::protocol::FrameData::from_ratatui_buffer( + &ratatui::buffer::Buffer::empty(ratatui::layout::Rect::new(0, 0, 1, 1)), + None, + ); + state.kitty_graphics_enabled = true; + assert!(!state.try_present_frame(frame.clone())); + assert_eq!(state.pending_native_cleanup, cleanup); + state.present_frozen_chrome(frame); + assert!(state.presentation_frozen); + assert_eq!(state.pending_native_cleanup, cleanup); + state.unfreeze_presentation(); + state.flush_native_cleanup(&mut output).unwrap(); + assert!(String::from_utf8(output.clone()) + .unwrap() + .contains("a=d,d=I,i=1234")); + state.flush_native_cleanup(&mut output).unwrap(); + assert_eq!( + String::from_utf8(output).unwrap().matches("a=d,").count(), + 1 + ); } } diff --git a/src/client/terminal_sessions.rs b/src/client/terminal_sessions.rs index c7328efd..6d3160c8 100644 --- a/src/client/terminal_sessions.rs +++ b/src/client/terminal_sessions.rs @@ -100,6 +100,7 @@ fn connect_terminal_session_stream( false, false, true, + true, ) { Ok(handshake) if handshake.encoding == RenderEncoding::TerminalAnsi => {} Ok(handshake) => { diff --git a/src/client/tests/mod.rs b/src/client/tests/mod.rs index 2665e7fd..8519ea07 100644 --- a/src/client/tests/mod.rs +++ b/src/client/tests/mod.rs @@ -69,6 +69,21 @@ fn direct_graphics_profile_is_narrow_and_transport_safe() { )); } +#[test] +fn server_graphics_files_require_local_filesystem_not_just_local_terminal() { + let local = endpoint::ClientEndpointId::Local; + let ssh = endpoint::ClientEndpointId::Ssh( + endpoint::ProfileId::parse("0123456789abcdef0123456789abcdef").unwrap(), + ); + // Keep old local peers working without requiring a negotiated capability. + assert!(server_graphics_files_allowed(&local, false)); + // Saved SSH endpoints run in the normal local client process. + assert!(!server_graphics_files_allowed(&ssh, false)); + // A standalone --remote bridge can retain the Local endpoint identity. + assert!(!server_graphics_files_allowed(&local, true)); + assert!(!server_graphics_files_allowed(&ssh, true)); +} + fn restore_env_var(key: &str, value: Option<OsString>) { if let Some(value) = value { std::env::set_var(key, value); diff --git a/src/ghostty/bindings.rs b/src/ghostty/bindings.rs index ceb2d699..33443aca 100644 --- a/src/ghostty/bindings.rs +++ b/src/ghostty/bindings.rs @@ -2069,16 +2069,28 @@ pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_FORMAT: #[doc = " Compression of the image. Always\n GHOSTTY_KITTY_IMAGE_COMPRESSION_NONE; compressed payloads are\n inflated before storage.\n\n Output type: GhosttyKittyImageCompression *"] pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_COMPRESSION: GhosttyKittyGraphicsImageData = 6; -#[doc = " Borrowed pointer to the raw pixel data. Valid as long as the\n underlying terminal is not mutated. Returns GHOSTTY_NO_VALUE when\n the image metadata is resident but its pixel payload is pending.\n\n The data is always fully decoded, uncompressed pixels in the\n format reported by GHOSTTY_KITTY_IMAGE_DATA_FORMAT: zlib payloads\n are inflated and PNG payloads are decoded to RGBA at transmission\n time, before the image is stored. Consumers can upload this\n directly to the GPU without any decode step.\n\n For an animated image (Kitty graphics animation, actions a=f/a=a)\n this is the pixel data of the current animation frame. The\n image's GHOSTTY_KITTY_IMAGE_DATA_GENERATION changes whenever the\n current frame changes, so generation-keyed caches remain\n coherent.\n\n Output type: const uint8_t **"] +#[doc = " Borrowed pointer to the raw pixel data. Valid as long as the\n underlying terminal is not mutated. Returns GHOSTTY_NO_VALUE when\n the image metadata is resident but its pixel payload is pending, or\n when the experimental PRESERVE_PNG option retained encoded PNG data.\n\n The data is always fully decoded, uncompressed pixels in the\n format reported by GHOSTTY_KITTY_IMAGE_DATA_FORMAT: zlib payloads\n are inflated and PNG payloads are normally decoded to RGBA at transmission\n time, before the image is stored. Consumers can upload returned pixels\n directly to the GPU without any decode step.\n\n For an animated image (Kitty graphics animation, actions a=f/a=a)\n this is the pixel data of the current animation frame. The\n image's GHOSTTY_KITTY_IMAGE_DATA_GENERATION changes whenever the\n current frame changes, so generation-keyed caches remain\n coherent.\n\n Output type: const uint8_t **"] pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_DATA_PTR: GhosttyKittyGraphicsImageData = 7; -#[doc = " Length of the raw pixel data in bytes. Always equal to\n width * height * bytes-per-pixel for the reported format. For a\n pending image, this is the expected length reserved against the\n storage limit even though DATA_PTR is not available yet.\n\n Output type: size_t *"] +#[doc = " Length of the raw pixel data in bytes. Always equal to\n width * height * bytes-per-pixel for the reported format. For a\n pending or encoded PNG image, this is the expected decoded length reserved against the\n storage limit even though DATA_PTR is not available yet.\n\n Output type: size_t *"] pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_DATA_LEN: GhosttyKittyGraphicsImageData = 8; #[doc = " Generation stamp assigned when this image was added to (or\n replaced in) the storage. A changed generation for a given image\n ID means the pixel contents may have changed even when the\n dimensions, format, and data length are identical (e.g. a\n retransmission of the same image ID), so texture caches must key\n staleness on this value rather than on size heuristics.\n\n Stamps are unique and monotonically increasing process-wide and\n are drawn from the same sequence as\n GHOSTTY_KITTY_GRAPHICS_DATA_GENERATION. Never zero for a stored\n image, so zero can be used as an \"empty\" sentinel by callers. Pending\n payload completion preserves this value to retain image age; consumers\n detect that completion through GHOSTTY_KITTY_GRAPHICS_DATA_GENERATION\n and retry DATA_PTR.\n\n Output type: uint64_t *"] pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_GENERATION: GhosttyKittyGraphicsImageData = 9; -#[doc = " Generation stamp assigned when this image was added to (or\n replaced in) the storage. A changed generation for a given image\n ID means the pixel contents may have changed even when the\n dimensions, format, and data length are identical (e.g. a\n retransmission of the same image ID), so texture caches must key\n staleness on this value rather than on size heuristics.\n\n Stamps are unique and monotonically increasing process-wide and\n are drawn from the same sequence as\n GHOSTTY_KITTY_GRAPHICS_DATA_GENERATION. Never zero for a stored\n image, so zero can be used as an \"empty\" sentinel by callers. Pending\n payload completion preserves this value to retain image age; consumers\n detect that completion through GHOSTTY_KITTY_GRAPHICS_DATA_GENERATION\n and retry DATA_PTR.\n\n Output type: uint64_t *"] +#[doc = " Owned encoded PNG bytes, borrowed until the next mutation. NO_VALUE\n unless experimentally retained. Output type: const uint8_t **."] +pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_ENCODED_PNG_PTR: + GhosttyKittyGraphicsImageData = 10; +#[doc = " Encoded PNG length; NO_VALUE unless retained. Output type: size_t *."] +pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_ENCODED_PNG_LEN: + GhosttyKittyGraphicsImageData = 11; +#[doc = " Borrowed void* backing context; valid only while image is unchanged."] +pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_FILE_CONTEXT: + GhosttyKittyGraphicsImageData = 12; +#[doc = " uint64_t immutable revision fingerprint; never reads image pixels."] +pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_FILE_IDENTITY: + GhosttyKittyGraphicsImageData = 13; +#[doc = " uint64_t immutable revision fingerprint; never reads image pixels."] pub const GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_MAX_VALUE: GhosttyKittyGraphicsImageData = 2147483647; #[doc = " Queryable data kinds for ghostty_kitty_graphics_image_get().\n\n @ingroup kitty_graphics"] @@ -2282,6 +2294,76 @@ unsafe extern "C" { out_info: *mut GhosttyKittyGraphicsPlacementRenderInfo, ) -> GhosttyResult; } +#[doc = " Borrowed opened descriptor; size is sizeof this request. No path is supplied."] +#[repr(C)] +#[derive(Debug, Default, Copy, Clone)] +pub struct GhosttyKittyImageSnapshotFileRequest { + pub size: usize, + pub fd: i64, + pub expected_len: u64, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of GhosttyKittyImageSnapshotFileRequest"] + [::std::mem::size_of::<GhosttyKittyImageSnapshotFileRequest>() - 24usize]; + ["Alignment of GhosttyKittyImageSnapshotFileRequest"] + [::std::mem::align_of::<GhosttyKittyImageSnapshotFileRequest>() - 8usize]; + ["Offset of field: GhosttyKittyImageSnapshotFileRequest::size"] + [::std::mem::offset_of!(GhosttyKittyImageSnapshotFileRequest, size) - 0usize]; + ["Offset of field: GhosttyKittyImageSnapshotFileRequest::fd"] + [::std::mem::offset_of!(GhosttyKittyImageSnapshotFileRequest, fd) - 8usize]; + ["Offset of field: GhosttyKittyImageSnapshotFileRequest::expected_len"] + [::std::mem::offset_of!(GhosttyKittyImageSnapshotFileRequest, expected_len) - 16usize]; +}; +pub type GhosttyKittyImageFileReadFn = ::std::option::Option< + unsafe extern "C" fn(context: *mut ::std::os::raw::c_void, dest: *mut u8, len: usize) -> bool, +>; +pub type GhosttyKittyImageFileReleaseFn = + ::std::option::Option<unsafe extern "C" fn(context: *mut ::std::os::raw::c_void)>; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct GhosttyKittyImageFileBacking { + pub context: *mut ::std::os::raw::c_void, + pub identity: u64, + pub len: usize, + pub read: GhosttyKittyImageFileReadFn, + pub release: GhosttyKittyImageFileReleaseFn, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of GhosttyKittyImageFileBacking"] + [::std::mem::size_of::<GhosttyKittyImageFileBacking>() - 40usize]; + ["Alignment of GhosttyKittyImageFileBacking"] + [::std::mem::align_of::<GhosttyKittyImageFileBacking>() - 8usize]; + ["Offset of field: GhosttyKittyImageFileBacking::context"] + [::std::mem::offset_of!(GhosttyKittyImageFileBacking, context) - 0usize]; + ["Offset of field: GhosttyKittyImageFileBacking::identity"] + [::std::mem::offset_of!(GhosttyKittyImageFileBacking, identity) - 8usize]; + ["Offset of field: GhosttyKittyImageFileBacking::len"] + [::std::mem::offset_of!(GhosttyKittyImageFileBacking, len) - 16usize]; + ["Offset of field: GhosttyKittyImageFileBacking::read"] + [::std::mem::offset_of!(GhosttyKittyImageFileBacking, read) - 24usize]; + ["Offset of field: GhosttyKittyImageFileBacking::release"] + [::std::mem::offset_of!(GhosttyKittyImageFileBacking, release) - 32usize]; +}; +impl Default for GhosttyKittyImageFileBacking { + fn default() -> Self { + let mut s = ::std::mem::MaybeUninit::<Self>::uninit(); + unsafe { + ::std::ptr::write_bytes(s.as_mut_ptr(), 0, 1); + s.assume_init() + } + } +} +#[doc = " Optional synchronous callback, default NULL. Verify exact descriptor size\n and snapshot immutable bytes before returning true. True transfers one context\n reference; false transfers nothing. Do not close the borrowed descriptor or\n reenter the terminal. Read must fill exactly len bytes; release is called once.\n identity identifies the immutable revision, not a pathname or source inode."] +pub type GhosttyKittyImageSnapshotFileFn = ::std::option::Option< + unsafe extern "C" fn( + terminal: GhosttyTerminal, + userdata: *mut ::std::os::raw::c_void, + request: *const GhosttyKittyImageSnapshotFileRequest, + out_backing: *mut GhosttyKittyImageFileBacking, + ) -> bool, +>; #[doc = " Perform one bounded compression step suitable for idle scheduling."] pub const GhosttyTerminalCompressionMode_GHOSTTY_TERMINAL_COMPRESSION_MODE_INCREMENTAL: GhosttyTerminalCompressionMode = 0; @@ -3119,7 +3201,13 @@ pub const GhosttyTerminalOption_GHOSTTY_TERMINAL_OPT_CLIPBOARD_READ: GhosttyTerm #[doc = " Set the maximum total decoded bytes a single Kitty clipboard protocol\n (OSC 5522) write transaction may accumulate. The limit is captured\n when a transaction begins; an in-flight transaction keeps the limit\n it started with.\n\n Data beyond the limit fails the whole transaction with EFBIG. The\n transaction is discarded, later write-related packets are ignored\n until a new write begins, and nothing reaches the clipboard write\n callback.\n\n Transactions are buffered in memory, so this limit bounds how much\n memory a single write can make the terminal allocate. Pass SIZE_MAX\n to remove the limit. A NULL value pointer reverts to the built-in\n default of 64MiB, the minimum required by the protocol.\n\n This limit doesn't apply to OSC 52 writes, which are bounded by the\n maximum length of an escape sequence instead.\n\n Input type: size_t*"] pub const GhosttyTerminalOption_GHOSTTY_TERMINAL_OPT_CLIPBOARD_WRITE_MAX_BYTES: GhosttyTerminalOption = 39; -#[doc = " Set the maximum total decoded bytes a single Kitty clipboard protocol\n (OSC 5522) write transaction may accumulate. The limit is captured\n when a transaction begins; an in-flight transaction keeps the limit\n it started with.\n\n Data beyond the limit fails the whole transaction with EFBIG. The\n transaction is discarded, later write-related packets are ignored\n until a new write begins, and nothing reaches the clipboard write\n callback.\n\n Transactions are buffered in memory, so this limit bounds how much\n memory a single write can make the terminal allocate. Pass SIZE_MAX\n to remove the limit. A NULL value pointer reverts to the built-in\n default of 64MiB, the minimum required by the protocol.\n\n This limit doesn't apply to OSC 52 writes, which are bounded by the\n maximum length of an escape sequence instead.\n\n Input type: size_t*"] +#[doc = " Experimental, default-off PNG retention for quiet base transmissions.\n Input type: bool*. NULL is a no-op. Corrupt compressed pixels may only\n be detected when an animation operation materializes the image."] +pub const GhosttyTerminalOption_GHOSTTY_TERMINAL_OPT_KITTY_IMAGE_PRESERVE_PNG: + GhosttyTerminalOption = 40; +#[doc = " Optional synchronous GhosttyKittyImageSnapshotFileFn; NULL disables."] +pub const GhosttyTerminalOption_GHOSTTY_TERMINAL_OPT_KITTY_IMAGE_SNAPSHOT_FILE: + GhosttyTerminalOption = 41; +#[doc = " Optional synchronous GhosttyKittyImageSnapshotFileFn; NULL disables."] pub const GhosttyTerminalOption_GHOSTTY_TERMINAL_OPT_MAX_VALUE: GhosttyTerminalOption = 2147483647; #[doc = " Terminal option identifiers.\n\n These values are used with ghostty_terminal_set() to configure\n terminal callbacks and associated state.\n\n @ingroup terminal"] pub type GhosttyTerminalOption = ::std::os::raw::c_int; diff --git a/src/ghostty/mod.rs b/src/ghostty/mod.rs index 82714118..43a70fb1 100644 --- a/src/ghostty/mod.rs +++ b/src/ghostty/mod.rs @@ -22,7 +22,10 @@ use std::ops::RangeInclusive; use std::os::raw::c_char; use std::ptr; use std::slice; -use std::sync::{Mutex, Once, OnceLock}; +use std::sync::{Arc, Mutex, Once, OnceLock}; + +use crate::pane_graphics_files::OwnedExport; +mod native_source; pub use bindings as ffi; @@ -227,6 +230,7 @@ pub struct KittyImagePlacement { pub data_len: usize, pub data_fingerprint: u64, pub data: Vec<u8>, + pub(crate) source_file: Option<Arc<OwnedExport>>, pub render: KittyPlacementRenderInfo, } @@ -239,8 +243,11 @@ pub struct KittyImageDescriptor { pub format: KittyImageFormat, pub data_len: usize, pub data_fingerprint: u64, + pub source_file: bool, } +type KittyPlacementPayload = (KittyImageDescriptor, Vec<u8>, Option<Arc<OwnedExport>>); + #[derive(Debug, Clone, Copy)] struct KittyImageFingerprintEntry { generation: u64, @@ -690,6 +697,16 @@ fn install_png_decoder_once() { }); } +#[cfg(test)] +thread_local! { + static PNG_DECODE_CALLS: Cell<usize> = const { Cell::new(0) }; +} + +#[cfg(test)] +mod native_source_tests; +#[cfg(test)] +mod png_forward_tests; + unsafe extern "C" fn decode_png_trampoline( _userdata: *mut c_void, allocator: *const ffi::GhosttyAllocator, @@ -701,6 +718,8 @@ unsafe extern "C" fn decode_png_trampoline( return false; } let bytes = unsafe { slice::from_raw_parts(data, data_len) }; + #[cfg(test)] + PNG_DECODE_CALLS.with(|calls| calls.set(calls.get() + 1)); let Some(rgba) = decode_png_rgba(bytes) else { return false; }; @@ -820,6 +839,7 @@ pub struct Terminal { callback_state: Box<TerminalCallbackState>, kitty_fingerprints: Mutex<HashMap<u32, KittyImageFingerprintEntry>>, kitty_empty_generation: Cell<Option<u64>>, + kitty_png_forwarding: bool, } impl Terminal { @@ -845,6 +865,7 @@ impl Terminal { }), kitty_fingerprints: Mutex::new(HashMap::new()), kitty_empty_generation: Cell::new(None), + kitty_png_forwarding: false, }; let userdata = (&mut *terminal.callback_state as *mut TerminalCallbackState).cast(); let glyph_protocol = false; @@ -1072,6 +1093,25 @@ impl Terminal { ) .into_result()?; } + self.set_kitty_source_forwarding(true)?; + Ok(()) + } + + pub fn set_kitty_source_forwarding(&mut self, enabled: bool) -> Result<(), Error> { + native_source::set_forwarding(self.raw, enabled) + } + + #[cfg(test)] + fn set_kitty_png_forwarding(&mut self, enabled: bool) -> Result<(), Error> { + unsafe { + ffi::ghostty_terminal_set( + self.raw, + ffi::GhosttyTerminalOption_GHOSTTY_TERMINAL_OPT_KITTY_IMAGE_PRESERVE_PNG, + (&enabled as *const bool).cast(), + ) + .into_result()?; + } + self.kitty_png_forwarding = enabled; Ok(()) } @@ -1882,6 +1922,65 @@ impl Terminal { } } + // Retained files are authoritative: never ask the native lazy decoder for + // pixels just to describe a placement or calculate its cache identity. + fn kitty_placement_payload<F>( + &self, + image: ffi::GhosttyKittyGraphicsImage, + image_id: u32, + placement_id: u32, + image_width: u32, + image_height: u32, + decoded_format: KittyImageFormat, + needs_data: &mut F, + ) -> Result<KittyPlacementPayload, Error> + where + F: FnMut(KittyImageDescriptor) -> bool, + { + let source_file = native_source::image_source(image)?; + let (format, data_ptr, data_len, data_fingerprint) = if let Some(source) = &source_file { + ( + KittyImageFormat::Rgba, + ptr::null(), + source.len(), + source.fingerprint(), + ) + } else { + let (format, data_ptr, data_len) = + kitty_image_transmission_payload(image, decoded_format, self.kitty_png_forwarding)?; + let fingerprint = self.kitty_image_fingerprint_cached( + image, + image_id, + (data_ptr, data_len), + image_width, + image_height, + format, + ); + (format, data_ptr, data_len, fingerprint) + }; + let descriptor = KittyImageDescriptor { + image_id, + placement_id, + image_width, + image_height, + format, + data_len, + data_fingerprint, + source_file: source_file.is_some(), + }; + let data = if needs_data(descriptor) { + match &source_file { + Some(source) => source + .copy_rgba() + .map_err(|_| Error(ffi::GhosttyResult_GHOSTTY_INVALID_VALUE))?, + None => kitty_image_data_from_ptr(data_ptr, data_len), + } + } else { + Vec::new() + }; + Ok((descriptor, data, source_file)) + } + fn kitty_image_placement<F>( &self, graphics: ffi::GhosttyKittyGraphics, @@ -1937,29 +2036,21 @@ impl Terminal { iterator, ffi::GhosttyKittyGraphicsPlacementData_GHOSTTY_KITTY_GRAPHICS_PLACEMENT_DATA_PLACEMENT_ID, )?; - let (data_ptr, data_len) = kitty_image_data_ptr_len(image)?; - let data_fingerprint = self.kitty_image_fingerprint_cached( + let (descriptor, data, source_file) = self.kitty_placement_payload( image, image_id, - (data_ptr, data_len), - image_width, - image_height, - format, - ); - let descriptor = KittyImageDescriptor { - image_id, placement_id, image_width, image_height, format, + needs_data, + )?; + let KittyImageDescriptor { + format, data_len, data_fingerprint, - }; - let data = if needs_data(descriptor) { - kitty_image_data_from_ptr(data_ptr, data_len) - } else { - Vec::new() - }; + .. + } = descriptor; let x_offset = kitty_placement_u32( iterator, ffi::GhosttyKittyGraphicsPlacementData_GHOSTTY_KITTY_GRAPHICS_PLACEMENT_DATA_X_OFFSET, @@ -1985,6 +2076,7 @@ impl Terminal { data_len, data_fingerprint, data, + source_file, render: KittyPlacementRenderInfo { pixel_width: raw_info.pixel_width, pixel_height: raw_info.pixel_height, @@ -2081,29 +2173,21 @@ impl Terminal { continue; }; let placement_id = run.synthetic_placement_id(); - let (data_ptr, data_len) = kitty_image_data_ptr_len(image)?; - let data_fingerprint = self.kitty_image_fingerprint_cached( + let (descriptor, data, source_file) = self.kitty_placement_payload( image, image_id, - (data_ptr, data_len), - image_width, - image_height, - format, - ); - let descriptor = KittyImageDescriptor { - image_id, placement_id, image_width, image_height, format, + needs_data, + )?; + let KittyImageDescriptor { + format, data_len, data_fingerprint, - }; - let data = if needs_data(descriptor) { - kitty_image_data_from_ptr(data_ptr, data_len) - } else { - Vec::new() - }; + .. + } = descriptor; placements.push(KittyImagePlacement { image_id, placement_id, @@ -2116,6 +2200,7 @@ impl Terminal { data_len, data_fingerprint, data, + source_file, render: geometry.render, }); } @@ -2585,6 +2670,40 @@ fn kitty_image_compression( Ok(out) } +fn kitty_image_transmission_payload( + image: ffi::GhosttyKittyGraphicsImage, + decoded_format: KittyImageFormat, + forwarding: bool, +) -> Result<(KittyImageFormat, *const u8, usize), Error> { + if !forwarding { + let (ptr, len) = kitty_image_data_ptr_len(image)?; + return Ok((decoded_format, ptr, len)); + } + let mut ptr_out: *const u8 = ptr::null(); + let result = unsafe { + ffi::ghostty_kitty_graphics_image_get( + image, + ffi::GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_ENCODED_PNG_PTR, + (&mut ptr_out as *mut *const u8).cast(), + ) + }; + if result == ffi::GhosttyResult_GHOSTTY_NO_VALUE { + let (ptr, len) = kitty_image_data_ptr_len(image)?; + return Ok((decoded_format, ptr, len)); + } + result.into_result()?; + let mut len = 0usize; + unsafe { + ffi::ghostty_kitty_graphics_image_get( + image, + ffi::GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_ENCODED_PNG_LEN, + (&mut len as *mut usize).cast(), + ) + .into_result()?; + } + Ok((KittyImageFormat::Png, ptr_out, len)) +} + fn kitty_image_data_ptr_len( image: ffi::GhosttyKittyGraphicsImage, ) -> Result<(*const u8, usize), Error> { @@ -3593,6 +3712,66 @@ mod tests { } #[test] + fn kitty_png_replacement_rejects_invalid_payload_without_placing_it() { + use base64::Engine as _; + let mut png = Vec::new(); + { + let mut encoder = png::Encoder::new(&mut png, 2, 1); + encoder.set_color(png::ColorType::Rgba); + encoder.set_depth(png::BitDepth::Eight); + let mut writer = encoder.write_header().unwrap(); + writer + .write_image_data(&[255, 0, 0, 255, 0, 0, 255, 255]) + .unwrap(); + } + let mut terminal = Terminal::new(10, 5, 0).unwrap(); + terminal.enable_kitty_graphics().unwrap(); + let upload = |payload: &[u8]| { + format!( + "\x1b[H\x1b_Ga=T,f=100,i=7,p=3,c=2,r=1,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(payload) + ) + }; + terminal.write(upload(&png).as_bytes()); + let before = terminal.kitty_image_placements().unwrap(); + assert_eq!(before.len(), 1); + assert_eq!((before[0].image_width, before[0].image_height), (2, 1)); + assert_eq!(before[0].data, [255, 0, 0, 255, 0, 0, 255, 255]); + // The parser retires the old placement, but never places corrupt data. + terminal.write(upload(&png[..png.len() / 2]).as_bytes()); + assert!(terminal.kitty_image_placements().unwrap().is_empty()); + } + + #[cfg(unix)] + #[test] + fn kitty_file_image_survives_source_mutation_and_unlink() { + use base64::Engine as _; + let path = std::path::PathBuf::from(format!( + "/var/tmp/herdr-kitty-source-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&path, [255, 0, 0, 255]).unwrap(); + let mut terminal = Terminal::new(10, 5, 0).unwrap(); + terminal.enable_kitty_graphics().unwrap(); + terminal.write( + format!( + "\x1b_Ga=T,f=32,t=f,i=7,p=3,s=1,v=1,c=1,r=1,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(path.to_str().unwrap()) + ) + .as_bytes(), + ); + std::fs::write(&path, [0, 0, 255, 255]).unwrap(); + std::fs::remove_file(&path).unwrap(); + let placements = terminal.kitty_image_placements().unwrap(); + assert_eq!(placements.len(), 1); + assert_eq!(placements[0].data, [255, 0, 0, 255]); + } + + #[test] fn kitty_image_fingerprint_covers_full_payload() { let mut data = vec![1u8; 4096 * 4]; let original = diff --git a/src/ghostty/native_source.rs b/src/ghostty/native_source.rs new file mode 100644 index 00000000..3e4e8a3f --- /dev/null +++ b/src/ghostty/native_source.rs @@ -0,0 +1,165 @@ +//! Host-owned immutable file snapshots retained by the native image lifetime. +use super::{ffi, Error, GhosttyResultExt}; +use crate::pane_graphics_files::{FileStore, OwnedExport}; +use std::ffi::c_void; +use std::ptr; +use std::sync::{Arc, OnceLock}; + +static SOURCES: OnceLock<FileStore> = OnceLock::new(); + +// These getters never materialize native pixels. The image borrow protects the +// host's strong reference until we have acquired our own. +pub(super) fn image_source( + image: ffi::GhosttyKittyGraphicsImage, +) -> Result<Option<Arc<OwnedExport>>, Error> { + let mut context: *mut c_void = ptr::null_mut(); + let result = unsafe { + ffi::ghostty_kitty_graphics_image_get( + image, + ffi::GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_FILE_CONTEXT, + (&mut context as *mut *mut c_void).cast(), + ) + }; + if result == ffi::GhosttyResult_GHOSTTY_NO_VALUE { + return Ok(None); + } + result.into_result()?; + if context.is_null() { + return Ok(None); + } + let mut identity = 0u64; + unsafe { + ffi::ghostty_kitty_graphics_image_get( + image, + ffi::GhosttyKittyGraphicsImageData_GHOSTTY_KITTY_IMAGE_DATA_FILE_IDENTITY, + (&mut identity as *mut u64).cast(), + ) + .into_result()?; + let source = context.cast::<OwnedExport>(); + // All terminal snapshot callbacks in this wrapper produce this type. + // Identity is immutable and does not depend on decoded pixel storage. + if (*source).fingerprint() != identity { + return Err(Error(ffi::GhosttyResult_GHOSTTY_INVALID_VALUE)); + } + Arc::increment_strong_count(source); + Ok(Some(Arc::from_raw(source))) + } +} + +pub(super) fn set_forwarding(terminal: ffi::GhosttyTerminal, enabled: bool) -> Result<(), Error> { + let callback = if enabled && cfg!(target_os = "linux") { + snapshot as *const () as *const c_void + } else { + ptr::null() + }; + unsafe { + ffi::ghostty_terminal_set( + terminal, + ffi::GhosttyTerminalOption_GHOSTTY_TERMINAL_OPT_KITTY_IMAGE_SNAPSHOT_FILE, + callback, + ) + .into_result() + } +} + +unsafe extern "C" fn snapshot( + _terminal: ffi::GhosttyTerminal, + _userdata: *mut c_void, + request: *const ffi::GhosttyKittyImageSnapshotFileRequest, + out: *mut ffi::GhosttyKittyImageFileBacking, +) -> bool { + std::panic::catch_unwind(|| { + if request.is_null() || out.is_null() || !cfg!(target_os = "linux") { + return false; + } + let request = unsafe { &*request }; + if request.size != std::mem::size_of::<ffi::GhosttyKittyImageSnapshotFileRequest>() { + return false; + } + let Ok(len) = usize::try_from(request.expected_len) else { + return false; + }; + let Ok(source) = SOURCES + .get_or_init(FileStore::native_sources) + .snapshot(request.fd, len) + else { + return false; + }; + let source = Arc::new(source); + let backing = ffi::GhosttyKittyImageFileBacking { + identity: source.fingerprint(), + len: source.len(), + context: Arc::into_raw(source).cast_mut().cast(), + read: Some(read), + release: Some(release), + }; + unsafe { + out.write(backing); + } + true + }) + .unwrap_or(false) +} + +unsafe extern "C" fn read(context: *mut c_void, dest: *mut u8, len: usize) -> bool { + std::panic::catch_unwind(|| { + if context.is_null() || dest.is_null() { + return false; + } + let source = unsafe { &*context.cast::<OwnedExport>() }; + if len != source.len() || len > isize::MAX as usize { + return false; + } + source + .read_into(unsafe { std::slice::from_raw_parts_mut(dest, len) }) + .is_ok() + }) + .unwrap_or(false) +} + +unsafe extern "C" fn release(context: *mut c_void) { + let _ = std::panic::catch_unwind(|| { + if !context.is_null() { + drop(unsafe { Arc::from_raw(context.cast::<OwnedExport>()) }); + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn invalid_requests_decline_without_transferring_ownership() { + let mut out = ffi::GhosttyKittyImageFileBacking::default(); + let mut request = ffi::GhosttyKittyImageSnapshotFileRequest { + size: std::mem::size_of::<ffi::GhosttyKittyImageSnapshotFileRequest>(), + fd: -1, + expected_len: 4, + }; + unsafe { + assert!(!snapshot( + ptr::null_mut(), + ptr::null_mut(), + &request, + &mut out + )); + assert!(out.context.is_null()); + request.size = 0; + assert!(!snapshot( + ptr::null_mut(), + ptr::null_mut(), + &request, + &mut out + )); + assert!(!snapshot( + ptr::null_mut(), + ptr::null_mut(), + ptr::null(), + &mut out + )); + assert!(out.context.is_null()); + release(ptr::null_mut()); + } + } +} diff --git a/src/ghostty/native_source_tests.rs b/src/ghostty/native_source_tests.rs new file mode 100644 index 00000000..bb4ded46 --- /dev/null +++ b/src/ghostty/native_source_tests.rs @@ -0,0 +1,146 @@ +use super::*; +use base64::Engine as _; + +fn terminal() -> Terminal { + let mut terminal = Terminal::new(20, 10, 0).unwrap(); + terminal.enable_kitty_graphics().unwrap(); + terminal.resize(20, 10, 8, 16).unwrap(); + terminal +} + +#[test] +fn decoded_upload_remains_bytes_backed() { + let mut terminal = terminal(); + let bytes = [17u8; 16]; + terminal.write( + format!( + "\x1b_Ga=T,f=32,s=2,v=2,i=7,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(bytes) + ) + .as_bytes(), + ); + let metadata = terminal + .kitty_image_placements_with_data_filter(|descriptor| { + assert!(!descriptor.source_file); + assert_eq!(descriptor.data_len, bytes.len()); + false + }) + .unwrap(); + assert_eq!(metadata.len(), 1); + assert!(metadata[0].data.is_empty()); + assert!(metadata[0].source_file.is_none()); + let compatible = terminal.kitty_image_placements().unwrap(); + assert_eq!(compatible[0].data, bytes); + assert_eq!(compatible[0].data_fingerprint, metadata[0].data_fingerprint); +} + +#[cfg(unix)] +#[test] +fn regular_file_placement_survives_alternate_screen_replacement() { + let store = crate::pane_graphics_files::FileStore::default(); + let mut terminal = terminal(); + terminal.set_kitty_source_forwarding(false).unwrap(); + terminal.write(b"\x1b[?1049h\x1b[2J\x1b[H"); + for value in [17u8, 99] { + let bytes = [value; 16]; + let source = store.export(&bytes).unwrap(); + terminal.write( + format!( + "\x1b[H\x1b_Ga=T,t=f,f=32,s=2,v=2,i=77,p=1,c=2,r=1,C=1,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD + .encode(source.path().as_os_str().as_encoded_bytes()) + ) + .as_bytes(), + ); + let placements = terminal.kitty_image_placements().unwrap(); + assert_eq!(placements.len(), 1); + assert_eq!(placements[0].data, bytes); + assert_eq!(placements[0].render.viewport_row, 0); + assert_eq!(placements[0].render.grid_rows, 1); + assert_eq!(placements[0].render.grid_cols, 2); + } +} + +#[cfg(target_os = "linux")] +#[test] +fn file_upload_automatically_retains_snapshot_when_platform_supports_clone() { + use std::os::fd::AsRawFd; + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(0); + let base = std::env::var_os("TMPDIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| "/var/tmp".into()); + let path = base.join(format!( + "herdr-native-source-test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let bytes = [42u8; 16]; + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + use std::io::Write; + (&file).write_all(&bytes).unwrap(); + // Probe the actual platform, not filesystem names: overlayfs/tmpfs must + // decline rather than silently produce a userspace copy. + let probe = crate::pane_graphics_files::FileStore::native_sources() + .snapshot(i64::from(file.as_raw_fd()), bytes.len()); + let supported = match probe { + Ok(snapshot) => { + drop(snapshot); + true + } + Err(error) => { + eprintln!("native snapshot unsupported on test filesystem: {error}"); + false + } + }; + let mut terminal = terminal(); + terminal.write( + format!( + "\x1b_Ga=T,t=f,f=32,s=2,v=2,i=7,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(path.as_os_str().as_encoded_bytes()) + ) + .as_bytes(), + ); + let metadata = terminal + .kitty_image_placements_with_data_filter(|descriptor| { + assert_eq!(descriptor.source_file, supported); + false + }) + .unwrap(); + assert_eq!(metadata.len(), 1); + assert!(metadata[0].data.is_empty()); + assert_eq!(metadata[0].source_file.is_some(), supported); + let mut unicode = self::terminal(); + unicode.write( + format!( + "\x1b_Ga=T,t=f,f=32,s=2,v=2,U=1,c=1,r=1,i=8,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(path.as_os_str().as_encoded_bytes()) + ) + .as_bytes(), + ); + unicode.write("\x1b[H\x1b[38;2;0;0;8m\u{10eeee}\u{0305}\u{0305}\x1b[0m".as_bytes()); + let unicode_metadata = unicode + .kitty_image_placements_with_data_filter(|descriptor| { + assert_eq!(descriptor.source_file, supported); + false + }) + .unwrap(); + assert_eq!(unicode_metadata.len(), 1); + assert!(unicode_metadata[0].data.is_empty()); + assert_eq!(unicode_metadata[0].source_file.is_some(), supported); + std::fs::write(&path, [99u8; 16]).unwrap(); + assert_eq!(unicode.kitty_image_placements().unwrap()[0].data, bytes); + let compatible = terminal.kitty_image_placements().unwrap(); + assert_eq!(compatible[0].data, bytes); + assert_eq!(compatible[0].data_fingerprint, metadata[0].data_fingerprint); + drop(terminal); + if let Some(source) = &metadata[0].source_file { + assert_eq!(source.copy_rgba().unwrap(), bytes); + } + std::fs::remove_file(path).unwrap(); +} diff --git a/src/ghostty/png_forward_tests.rs b/src/ghostty/png_forward_tests.rs new file mode 100644 index 00000000..6d3904b2 --- /dev/null +++ b/src/ghostty/png_forward_tests.rs @@ -0,0 +1,232 @@ +use super::*; +use base64::Engine as _; + +fn fixture() -> Vec<u8> { + fixture_size(32, 16) +} + +fn fixture_size(width: u32, height: u32) -> Vec<u8> { + let mut bytes = Vec::new(); + { + let mut encoder = png::Encoder::new(&mut bytes, width, height); + encoder.set_color(png::ColorType::Rgba); + encoder.set_depth(png::BitDepth::Eight); + encoder + .write_header() + .unwrap() + .write_image_data(&vec![42; (width * height * 4) as usize]) + .unwrap(); + } + bytes +} + +fn terminal() -> Terminal { + let mut terminal = Terminal::new(20, 10, 0).unwrap(); + terminal.enable_kitty_graphics().unwrap(); + terminal.set_kitty_png_forwarding(true).unwrap(); + terminal.resize(20, 10, 8, 16).unwrap(); + terminal +} + +#[test] +#[ignore = "manual fixed-geometry native PNG ingestion/extraction scaling profile"] +fn native_png_render_scale_profile() { + let png = fixture_size(800, 480); + let upload = format!( + "\x1b[H\x1b_Ga=T,f=100,i=7,c=60,r=20,C=1,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(&png) + ); + for count in [1, 15] { + for forwarding in [false, true] { + let mut panes = (0..count) + .map(|_| { + let mut pane = Terminal::new(120, 40, 0).unwrap(); + pane.enable_kitty_graphics().unwrap(); + pane.set_kitty_png_forwarding(forwarding).unwrap(); + pane.resize(120, 40, 8, 16).unwrap(); + pane + }) + .collect::<Vec<_>>(); + let mut samples = Vec::new(); + for iteration in 0..65 { + let begin = std::time::Instant::now(); + for pane in &mut panes { + pane.write(upload.as_bytes()); + let placements = pane.kitty_image_placements().unwrap(); + assert_eq!(placements.len(), 1); + assert_eq!( + placements[0].data.len(), + if forwarding { png.len() } else { 800 * 480 * 4 } + ); + std::hint::black_box(placements); + } + if iteration >= 5 { + samples.push(begin.elapsed().as_micros()); + } + } + samples.sort_unstable(); + eprintln!("native PNG panes={count} forwarding={forwarding} median_us={} p95_us={} png_bytes={}", samples[30], samples[57], png.len()); + } + } +} + +#[test] +fn normal_kitty_graphics_fully_decodes_quiet_png_uploads() { + let mut terminal = Terminal::new(20, 10, 0).unwrap(); + terminal.enable_kitty_graphics().unwrap(); + terminal.resize(20, 10, 8, 16).unwrap(); + let before = PNG_DECODE_CALLS.get(); + terminal.write( + format!( + "\x1b_Ga=T,f=100,i=7,c=4,r=1,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(fixture()) + ) + .as_bytes(), + ); + let placements = terminal.kitty_image_placements().unwrap(); + assert_eq!(placements.len(), 1); + assert_eq!(placements[0].format, KittyImageFormat::Rgba); + assert_eq!(placements[0].data, vec![42; 32 * 16 * 4]); + assert_eq!(PNG_DECODE_CALLS.get(), before + 1); +} + +#[test] +fn quiet_png_forwarding_retains_exact_payload_without_pixel_decode() { + let bytes = fixture(); + let mut terminal = terminal(); + let before = PNG_DECODE_CALLS.get(); + terminal.write( + format!( + "\x1b_Ga=T,f=100,i=7,c=4,r=1,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(&bytes) + ) + .as_bytes(), + ); + let placements = terminal.kitty_image_placements().unwrap(); + assert_eq!(placements.len(), 1); + assert_eq!(placements[0].format, KittyImageFormat::Png); + assert_eq!(placements[0].data, bytes); + assert_eq!( + (placements[0].image_width, placements[0].image_height), + (32, 16) + ); + assert_eq!(PNG_DECODE_CALLS.get(), before); +} + +#[test] +fn placeholder_png_forwarding_and_lazy_animation_decode() { + let bytes = fixture(); + let mut terminal = terminal(); + let before = PNG_DECODE_CALLS.get(); + terminal.write( + format!( + "\x1b_Ga=T,f=100,U=1,i=7,c=4,r=1,q=2;{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(&bytes) + ) + .as_bytes(), + ); + terminal.write("\x1b[H\x1b[38;2;0;0;7m\u{10eeee}\u{0305}\u{0305}\x1b[0m".as_bytes()); + let placements = terminal.kitty_image_placements().unwrap(); + assert_eq!(placements.len(), 1); + assert_eq!(placements[0].format, KittyImageFormat::Png); + assert_eq!(placements[0].data, bytes); + assert_eq!(PNG_DECODE_CALLS.get(), before); + let fingerprint = placements[0].data_fingerprint; + terminal.write(b"\x1b_Ga=a,i=7,q=2;\x1b\\"); + let materialized = terminal.kitty_image_placements().unwrap(); + assert_eq!(PNG_DECODE_CALLS.get(), before + 1); + assert_eq!(materialized[0].format, KittyImageFormat::Rgba); + assert_eq!(materialized[0].data, [42; 32 * 16 * 4]); + assert_ne!(materialized[0].data_fingerprint, fingerprint); +} + +#[test] +fn experimental_quiet_png_defers_crc_valid_compressed_data_errors() { + let mut bytes = fixture(); + let mut offset = 8; + loop { + let len = u32::from_be_bytes(bytes[offset..offset + 4].try_into().unwrap()) as usize; + if &bytes[offset + 4..offset + 8] == b"IDAT" { + bytes[offset + 8] = 0; // Invalid zlib header, but structurally valid PNG chunks. + let mut crc = !0u32; + for &byte in &bytes[offset + 4..offset + 8 + len] { + crc ^= u32::from(byte); + for _ in 0..8 { + crc = (crc >> 1) ^ (0xedb88320u32 & 0u32.wrapping_sub(crc & 1)); + } + } + bytes[offset + 8 + len..offset + 12 + len].copy_from_slice(&(!crc).to_be_bytes()); + break; + } + offset += len + 12; + } + for quiet in [0, 2] { + let mut terminal = terminal(); + let before = PNG_DECODE_CALLS.get(); + terminal.write( + format!( + "\x1b_Ga=T,f=100,i=7,c=4,r=1,q={quiet};{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(&bytes) + ) + .as_bytes(), + ); + let placements = terminal.kitty_image_placements().unwrap(); + if quiet == 0 { + assert!(placements.is_empty()); + assert_eq!(PNG_DECODE_CALLS.get(), before + 1); + } else { + assert_eq!(placements[0].data, bytes); + assert_eq!(PNG_DECODE_CALLS.get(), before); + terminal.write(b"\x1b_Ga=a,i=7,q=2;\x1b\\"); + assert_eq!(PNG_DECODE_CALLS.get(), before + 1); + assert_eq!(terminal.kitty_image_placements().unwrap()[0].data, bytes); + } + } +} + +#[test] +fn response_bearing_png_and_queries_keep_full_validation() { + let bytes = fixture(); + for (action, quiet) in [('T', 0), ('T', 1), ('q', 2)] { + let mut terminal = terminal(); + let before = PNG_DECODE_CALLS.get(); + terminal.write( + format!( + "\x1b_Ga={action},f=100,i=7,c=4,r=1,q={quiet};{}\x1b\\", + base64::engine::general_purpose::STANDARD.encode(&bytes) + ) + .as_bytes(), + ); + assert_eq!( + PNG_DECODE_CALLS.get(), + before + 1, + "action {action}, quiet {quiet}" + ); + let placements = terminal.kitty_image_placements().unwrap(); + if action == 'q' { + assert!(placements.is_empty()); + } else { + assert_eq!(placements[0].format, KittyImageFormat::Rgba); + } + } +} + +#[test] +fn chunked_png_forwarding_inherits_quiet_mode() { + let bytes = fixture(); + let mut terminal = terminal(); + let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes); + let split = (encoded.len() / 8) * 4; + let before = PNG_DECODE_CALLS.get(); + terminal.write( + format!( + "\x1b_Ga=T,f=100,i=7,c=4,r=1,q=2,m=1;{}\x1b\\", + &encoded[..split] + ) + .as_bytes(), + ); + assert!(terminal.kitty_image_placements().unwrap().is_empty()); + terminal.write(format!("\x1b_Gm=0;{}\x1b\\", &encoded[split..]).as_bytes()); + assert_eq!(terminal.kitty_image_placements().unwrap()[0].data, bytes); + assert_eq!(PNG_DECODE_CALLS.get(), before); +} diff --git a/src/kitty_graphics.rs b/src/kitty_graphics.rs index 2eca21f5..6bca9443 100644 --- a/src/kitty_graphics.rs +++ b/src/kitty_graphics.rs @@ -9,21 +9,20 @@ use base64::Engine; use ratatui::layout::Rect; use crate::app::state::AppState; -use crate::ghostty::{ - KittyImageDescriptor, KittyImageFormat, KittyImagePlacement, KittyPlacementRenderInfo, -}; -use crate::layout::{PaneId, PaneInfo}; +use crate::ghostty::{KittyImageDescriptor, KittyImageFormat, KittyImagePlacement}; +use crate::layout::PaneId; use crate::terminal::TerminalRuntimeRegistry; +pub(crate) mod output; pub(crate) mod surface; +pub(crate) use output::{GraphicsOperation, GraphicsOutput}; +use std::sync::Arc; + const KITTY_CHUNK_BYTES: usize = 3072; -const MAX_OVERSIZED_SOURCES: usize = 256; pub(crate) const HEADLESS_GRAPHICS_TRANSACTION_BUDGET: usize = crate::protocol::MAX_GRAPHICS_FRAME_SIZE - crate::protocol::MAX_FRAME_SIZE; const HOST_IMAGE_ID_BASE: u32 = 10_000; -#[cfg(test)] -const PANE_GRAPHICS_IMAGE_ID_BIT: u32 = 1 << 31; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub(crate) struct HostCellSize { @@ -39,6 +38,8 @@ impl HostCellSize { #[derive(Debug)] struct HostPlacement { + /// Client-local shared pixels; headless placements retain their inline data. + raw_data: Option<Arc<[u8]>>, pane_id: PaneId, host_image_id: Option<u32>, area: Rect, @@ -54,10 +55,6 @@ enum HostSourceKey { pane_id: PaneId, image_id: u32, }, - PaneLayer { - pane_id: PaneId, - layer_id: String, - }, ClientSurface { scope: String, source: crate::protocol::SurfaceGraphicsSource, @@ -109,7 +106,6 @@ pub(crate) struct HostGraphicsCache { placements: HashMap<(u32, u32), PlacementSignature>, /// Host image currently backing each (pane, source image id) pair. sources: HashMap<HostSourceKey, u32>, - oversized: HashMap<HostSourceKey, ImageSignature>, continuation: Option<(HostSourceKey, u32, usize)>, replay_placements: bool, replayed_placements: HashSet<(u32, u32)>, @@ -125,32 +121,12 @@ pub(crate) fn is_enabled() -> bool { KITTY_GRAPHICS_ENABLED.load(Ordering::Acquire) } +#[cfg(test)] pub(crate) struct EncodedGraphics { pub(crate) bytes: Vec<u8>, pub(crate) incomplete: bool, } -/// Whether appending `additional` bytes to the `current_len` bytes already -/// assembled keeps the transaction inside the caller's budget. Without a -/// budget the incremental path intentionally stays one transaction per call. -fn coalesced_transaction_fits( - current_len: usize, - additional: usize, - transaction_budget: Option<usize>, -) -> bool { - let Some(budget) = transaction_budget else { - return false; - }; - current_len.saturating_add(additional) <= budget -} - -fn image_transaction_fits(placement: &HostPlacement, budget: Option<usize>) -> bool { - let Some(budget) = budget else { - return true; - }; - image_transfer_estimated_size(placement.placement.data_len) <= budget -} - pub(crate) fn image_transfer_estimated_size(data_len: usize) -> usize { let encoded = data_len.div_ceil(3).saturating_mul(4); let command_overhead = data_len.div_ceil(KITTY_CHUNK_BYTES).saturating_mul(16) + 1024; @@ -164,23 +140,10 @@ fn placement_identity(placement: &HostPlacement) -> (HostSourceKey, u32) { ) } -fn source_order(source: &HostSourceKey) -> (u32, String) { - match source { - HostSourceKey::Terminal { pane_id, .. } => (pane_id.raw(), String::new()), - HostSourceKey::PaneLayer { pane_id, layer_id } => (pane_id.raw(), layer_id.clone()), - HostSourceKey::ClientSurface { scope, source } => { - let mut hasher = DefaultHasher::new(); - scope.hash(&mut hasher); - source.hash(&mut hasher); - (hasher.finish() as u32, format!("{source:?}")) - } - } -} - fn encode_placement_update( cache: &mut HostGraphicsCache, placement: &HostPlacement, -) -> Option<Vec<u8>> { +) -> Option<GraphicsOutput> { let (clipped, format_code) = clipped_placement(placement)?; let host_id = placement .host_image_id @@ -201,37 +164,30 @@ fn encode_placement_update( } let mut bytes = Vec::new(); - let mut displayed = false; + let mut output = GraphicsOutput::default(); if !image_current { - if cache.images.contains_key(&host_id) - && matches!(placement.source_key, HostSourceKey::PaneLayer { .. }) - { - if !encode_transmit_and_display( - &mut bytes, - placement, - clipped, - format_code, - host_id, - placement_id, - ) { - return None; - } - displayed = true; - } else { - if cache.images.contains_key(&host_id) { - encode_delete_image(&mut bytes, host_id); - cache.placements.retain(|(id, _), _| *id != host_id); - cache.replayed_placements.retain(|(id, _)| *id != host_id); - } - if !encode_upload_image(&mut bytes, placement, format_code, host_id) { + if cache.images.contains_key(&host_id) { + encode_delete_image(&mut bytes, host_id); + cache.placements.retain(|(id, _), _| *id != host_id); + cache.replayed_placements.retain(|(id, _)| *id != host_id); + } + if let Some(data) = &placement.raw_data { + if data.is_empty() { return None; } + output.push_bytes(std::mem::take(&mut bytes)); + output.operations.push(GraphicsOperation::Upload { + control: upload_control(placement, format_code, host_id), + data: Arc::clone(data), + }); + } else if !encode_upload_image(&mut bytes, placement, format_code, host_id) { + return None; } cache.images.insert(host_id, image_signature); } release_superseded_source_image(&mut bytes, cache, placement.source_key.clone(), host_id); - if !displayed && !placement_current { + if !placement_current { encode_display_placement( &mut bytes, clipped, @@ -244,7 +200,8 @@ fn encode_placement_update( if cache.replay_placements { cache.replayed_placements.insert(key); } - Some(bytes) + output.push_bytes(bytes); + Some(output) } fn release_superseded_source_image( @@ -265,13 +222,22 @@ fn release_superseded_source_image( cache.replayed_placements.retain(|(id, _)| *id != previous); } +#[cfg(test)] fn encode_graphics_update_incremental( cache: &mut HostGraphicsCache, placements: &[HostPlacement], - live_pane_sources: &HashSet<HostSourceKey>, - transaction_budget: Option<usize>, - coalesce_placements: bool, ) -> EncodedGraphics { + let (output, incomplete) = encode_graphics_output_incremental(cache, placements); + EncodedGraphics { + bytes: output.into_inline_bytes(), + incomplete, + } +} + +fn encode_graphics_output_incremental( + cache: &mut HostGraphicsCache, + placements: &[HostPlacement], +) -> (GraphicsOutput, bool) { let desired_sources = placements .iter() .map(|placement| placement.source_key.clone()) @@ -301,48 +267,12 @@ fn encode_graphics_update_incremental( .map(|index| index + 1) .or_else(|| cache.continuation.as_ref().map(|cursor| cursor.2)) .map_or(0, |index| index % placements.len().max(1)); - let mut bytes = Vec::new(); + let mut bytes = GraphicsOutput::default(); let mut emitted = false; - let mut dead_sources = cache + cache .sources - .keys() - .filter(|source| { - matches!(source, HostSourceKey::PaneLayer { .. }) - && !live_pane_sources.contains(*source) - }) - .cloned() - .collect::<Vec<_>>(); - dead_sources.sort_by_key(source_order); - for source in dead_sources { - let host_id = cache.sources[&source]; - let last_reference = !cache - .sources - .iter() - .any(|(other, id)| *other != source && *id == host_id); - if emitted && last_reference { - return EncodedGraphics { - bytes, - incomplete: true, - }; - } - cache.sources.remove(&source); - if last_reference { - encode_delete_image(&mut bytes, host_id); - cache.images.remove(&host_id); - cache.placements.retain(|(id, _), _| *id != host_id); - cache.replayed_placements.retain(|(id, _)| *id != host_id); - emitted = true; - } - } - cache.sources.retain(|source, _| { - matches!(source, HostSourceKey::PaneLayer { .. }) || desired_sources.contains(source) - }); - cache.oversized.retain(|source, _| { - matches!(source, HostSourceKey::Terminal { .. }) - || live_pane_sources.contains(source) - || desired_sources.contains(source) - }); + .retain(|source, _| desired_sources.contains(source)); let mut stale = cache .placements @@ -351,56 +281,22 @@ fn encode_graphics_update_incremental( .copied() .collect::<Vec<_>>(); stale.sort_unstable(); - let mut stale_image = None; for key @ (host_id, placement_id) in stale { let mut transaction = Vec::new(); encode_delete_placement(&mut transaction, host_id, placement_id); - let same_image = stale_image == Some(host_id); - if emitted - && !(coalesce_placements - && same_image - && coalesced_transaction_fits(bytes.len(), transaction.len(), transaction_budget)) - { - return EncodedGraphics { - bytes, - incomplete: true, - }; + if emitted { + return (bytes, true); } - bytes.extend(transaction); + bytes.push_bytes(transaction); cache.placements.remove(&key); cache.replayed_placements.remove(&key); emitted = true; - stale_image = Some(host_id); } - // Keep unrelated images isolated, but treat every row of one logical image - // as part of its upload or replacement transaction. Sending only the first - // row exposes the blank placeholder cells until later frames catch up. - let coalesce_pass = coalesce_placements && !emitted; - let mut coalesce_target = None; + // Preserve one cache mutation transaction per incremental call. for offset in 0..placements.len() { let index = (start + offset) % placements.len(); let placement = &placements[index]; - let signature = image_signature(placement, kitty_format_code(placement.placement.format)); - if transaction_budget.is_some() - && cache.oversized.get(&placement.source_key) == Some(&signature) - { - continue; - } - cache.oversized.remove(&placement.source_key); - let host_id = placement - .host_image_id - .unwrap_or_else(|| host_image_id(placement.pane_id, &placement.placement)); - let image_cached = cache.images.get(&host_id) == Some(&signature); - // With the image uploaded and the source already bound to it, the - // transaction is a re-display only: no upload and no superseded-image - // delete from `release_superseded_source_image`. - let pure_redisplay = - image_cached && cache.sources.get(&placement.source_key) == Some(&host_id); - if !image_cached && !image_transaction_fits(placement, transaction_budget) { - cache.quarantine_oversized(placement.source_key.clone(), signature); - continue; - } let mut candidate = cache.clone(); let Some(transaction) = encode_placement_update(&mut candidate, placement) else { continue; @@ -409,47 +305,26 @@ fn encode_graphics_update_incremental( *cache = candidate; continue; } - let same_logical_image = coalesce_target.as_ref().is_none_or(|(source, target_id)| { - source == &placement.source_key && *target_id == host_id - }); - if emitted - && !(coalesce_pass - && pure_redisplay - && same_logical_image - && coalesced_transaction_fits(bytes.len(), transaction.len(), transaction_budget)) - { - return EncodedGraphics { - bytes, - incomplete: true, - }; + if emitted { + return (bytes, true); } *cache = candidate; let (source, id) = placement_identity(placement); cache.continuation = Some((source, id, (index + 1) % placements.len())); bytes.extend(transaction); emitted = true; - if coalesce_pass && !pure_redisplay { - coalesce_target = Some((placement.source_key.clone(), host_id)); - } } cache.replay_placements = false; cache.replayed_placements.clear(); - EncodedGraphics { - bytes, - incomplete: false, - } + (bytes, false) } #[cfg(test)] -fn drain_graphics_updates( - cache: &mut HostGraphicsCache, - placements: &[HostPlacement], - live: &HashSet<HostSourceKey>, -) -> Vec<u8> { +fn drain_graphics_updates(cache: &mut HostGraphicsCache, placements: &[HostPlacement]) -> Vec<u8> { let mut bytes = Vec::new(); loop { - let encoded = encode_graphics_update_incremental(cache, placements, live, None, false); + let encoded = encode_graphics_update_incremental(cache, placements); bytes.extend(encoded.bytes); if !encoded.incomplete { return bytes; @@ -464,15 +339,6 @@ impl HostGraphicsCache { self.replayed_placements.clear(); } - fn quarantine_oversized(&mut self, source: HostSourceKey, signature: ImageSignature) { - if !self.oversized.contains_key(&source) && self.oversized.len() >= MAX_OVERSIZED_SOURCES { - if let Some(evicted) = self.oversized.keys().next().cloned() { - self.oversized.remove(&evicted); - } - } - self.oversized.insert(source, signature); - } - pub(crate) fn request_placement_replay(&mut self) { if !self.replay_placements { self.replay_placements = true; @@ -488,7 +354,6 @@ impl HostGraphicsCache { self.images.clear(); self.placements.clear(); self.sources.clear(); - self.oversized.clear(); self.reset_incremental_progress(); bytes } @@ -496,13 +361,10 @@ impl HostGraphicsCache { fn collect_visible_placements( app: &AppState, - graphics: &crate::app::pane_graphics::Runtime, terminal_runtimes: &TerminalRuntimeRegistry, surface: crate::ui::TabSurfaceView<'_>, cell_size: HostCellSize, - uploaded_images: &HashMap<u32, ImageSignature>, - oversized_images: &HashMap<HostSourceKey, ImageSignature>, - client_id: u64, + delivered_images: &HashMap<HostSourceKey, ImageSignature>, ) -> Vec<HostPlacement> { let Some(target) = surface.target else { tracing::debug!("collect_visible_placements: no tab surface target"); @@ -531,33 +393,6 @@ fn collect_visible_placements( ); let mut placements = Vec::new(); for info in surface.pane_infos { - let mut pane_layers = graphics - .slots - .iter() - .filter_map(|((pane_id, layer_id), slot)| { - (*pane_id == info.id) - .then(|| { - slot.layer.as_ref().and_then(|layer| { - (!layer.terminal_only() || slot.direct_client() == Some(client_id)) - .then_some((layer_id, slot.host_image_id, layer)) - }) - }) - .flatten() - }) - .collect::<Vec<_>>(); - pane_layers.sort_by_key(|(layer_id, _, layer)| (layer.z_index, layer_id.as_str())); - for (layer_id, host_image_id, layer) in pane_layers { - placements.push(pane_graphics_host_placement( - info, - layer_id, - host_image_id, - cell_size, - layer, - uploaded_images, - true, - )); - } - let runtime = match app.runtime_for_pane_in_workspace(terminal_runtimes, ws_idx, info.id) { Some(rt) => rt, None => { @@ -567,19 +402,17 @@ fn collect_visible_placements( }; let mut requested_images = HashSet::new(); for placement in runtime.kitty_image_placements_with_data_filter(|descriptor| { - terminal_image_needs_data( - info.id, - descriptor, - uploaded_images, - oversized_images, - &mut requested_images, - ) + if descriptor.source_file { + return false; + } + terminal_image_needs_data(info.id, descriptor, delivered_images, &mut requested_images) }) { let scrollback_offset = runtime .scroll_metrics() .map(|m| m.offset_from_bottom as u32) .unwrap_or(0); placements.push(HostPlacement { + raw_data: None, pane_id: info.id, host_image_id: None, area: info.inner_rect, @@ -603,97 +436,19 @@ fn collect_visible_placements( fn terminal_image_needs_data( pane_id: PaneId, descriptor: KittyImageDescriptor, - uploaded_images: &HashMap<u32, ImageSignature>, - oversized_images: &HashMap<HostSourceKey, ImageSignature>, + delivered_images: &HashMap<HostSourceKey, ImageSignature>, requested_images: &mut HashSet<(HostSourceKey, ImageSignature)>, ) -> bool { let format_code = kitty_format_code(descriptor.format); let signature = image_signature_from_descriptor(descriptor, format_code); - let host_id = host_image_id_for_signature(pane_id, signature); let source = HostSourceKey::Terminal { pane_id, image_id: descriptor.image_id, }; - uploaded_images.get(&host_id).copied() != Some(signature) - && oversized_images.get(&source).copied() != Some(signature) + delivered_images.get(&source).copied() != Some(signature) && requested_images.insert((source, signature)) } -fn pane_graphics_host_placement( - info: &PaneInfo, - layer_id: &str, - host_id: u32, - cell_size: HostCellSize, - layer: &crate::app::pane_graphics::Layer, - uploaded_images: &HashMap<u32, ImageSignature>, - include_data: bool, -) -> HostPlacement { - let format = pane_graphics_kitty_format(layer.format); - let signature = pane_layer_image_signature(layer); - let data = if !include_data || uploaded_images.get(&host_id).copied() == Some(signature) { - Vec::new() - } else { - layer.inline_data().map(<[u8]>::to_vec).unwrap_or_default() - }; - let render = layer.render; - let grid_cols = if render.grid_cols == 0 { - u32::from(info.inner_rect.width) - } else { - render.grid_cols - }; - let grid_rows = if render.grid_rows == 0 { - u32::from(info.inner_rect.height) - } else { - render.grid_rows - }; - - HostPlacement { - pane_id: info.id, - host_image_id: Some(host_id), - area: info.inner_rect, - cell_size, - source_key: HostSourceKey::PaneLayer { - pane_id: info.id, - layer_id: layer_id.to_owned(), - }, - scrollback_offset: 0, - placement: KittyImagePlacement { - image_id: 1, - placement_id: 1, - z: layer.z_index, - x_offset: 0, - y_offset: 0, - image_width: layer.image_width, - image_height: layer.image_height, - format, - data_len: layer.data_len(), - data_fingerprint: layer.data_fingerprint, - data, - render: KittyPlacementRenderInfo { - pixel_width: layer.image_width, - pixel_height: layer.image_height, - grid_cols, - grid_rows, - viewport_col: render.viewport_col, - viewport_row: render.viewport_row, - source_x: 0, - source_y: 0, - source_width: 0, - source_height: 0, - }, - }, - } -} - -fn pane_graphics_kitty_format(format: crate::api::schema::PaneGraphicsFormat) -> KittyImageFormat { - match format { - crate::api::schema::PaneGraphicsFormat::Png => KittyImageFormat::Png, - crate::api::schema::PaneGraphicsFormat::Rgb => KittyImageFormat::Rgb, - crate::api::schema::PaneGraphicsFormat::Rgba - | crate::api::schema::PaneGraphicsFormat::Bgra => KittyImageFormat::Rgba, - } -} - fn host_image_id(pane_id: PaneId, placement: &KittyImagePlacement) -> u32 { let format_code = kitty_format_code(placement.format); host_image_id_for_signature( @@ -719,11 +474,6 @@ fn host_placement_id(source_key: &HostSourceKey, placement: &KittyImagePlacement let mut hasher = DefaultHasher::new(); match source_key { HostSourceKey::Terminal { pane_id, .. } => pane_id.raw().hash(&mut hasher), - HostSourceKey::PaneLayer { pane_id, layer_id } => { - "pane.graphics".hash(&mut hasher); - pane_id.raw().hash(&mut hasher); - layer_id.hash(&mut hasher); - } HostSourceKey::ClientSurface { scope, source } => { "client.surface".hash(&mut hasher); scope.hash(&mut hasher); @@ -735,11 +485,6 @@ fn host_placement_id(source_key: &HostSourceKey, placement: &KittyImagePlacement 1 + ((hasher.finish() as u32) % 900_000) } -pub(crate) struct DirectFileCommand { - pub(crate) leading: Vec<u8>, - pub(crate) control: String, -} - #[cfg(unix)] pub(crate) fn encode_kitty_regular_file( out: &mut Vec<u8>, @@ -754,7 +499,7 @@ pub(crate) fn encode_kitty_regular_file( out.extend_from_slice(b"\x1b8"); } -fn encode_delete_image(out: &mut Vec<u8>, id: u32) { +pub(crate) fn encode_delete_image(out: &mut Vec<u8>, id: u32) { let _ = write!(out, "\x1b_Ga=d,d=I,i={id},q=2;\x1b\\"); } @@ -765,45 +510,24 @@ fn encode_delete_placement(out: &mut Vec<u8>, host_id: u32, host_placement_id: u ); } -fn encode_upload_image( - out: &mut Vec<u8>, - placement: &HostPlacement, - format_code: u32, - host_id: u32, -) -> bool { - if placement.placement.data.is_empty() { - return false; - } - - let control = format!( +fn upload_control(placement: &HostPlacement, format_code: u32, host_id: u32) -> String { + format!( "a=t,t=d,f={format_code},s={},v={},i={host_id},q=2", placement.placement.image_width, placement.placement.image_height, - ); - encode_kitty_data(out, &control, &placement.placement.data); - true + ) } -fn encode_transmit_and_display( +fn encode_upload_image( out: &mut Vec<u8>, placement: &HostPlacement, - clipped: ClippedPlacement, format_code: u32, host_id: u32, - host_placement_id: u32, ) -> bool { if placement.placement.data.is_empty() { return false; } - let _ = write!(out, "\x1b[{};{}H", clipped.y + 1, clipped.x + 1); - let mut control = format!( - "a=T,t=d,f={format_code},s={},v={},i={host_id},p={host_placement_id},c={},r={},z={},C=1,q=2", - placement.placement.image_width, - placement.placement.image_height, - clipped.cols, - clipped.rows, - placement.placement.z, - ); - append_placement_controls(&mut control, clipped); + + let control = upload_control(placement, format_code, host_id); encode_kitty_data(out, &control, &placement.placement.data); true } @@ -993,16 +717,6 @@ fn scale_pixels(value: u32, source: u32, dest: u32) -> u32 { ((value as u64).saturating_mul(source as u64) / dest.max(1) as u64).min(u32::MAX as u64) as u32 } -fn pane_layer_image_signature(layer: &crate::app::pane_graphics::Layer) -> ImageSignature { - ImageSignature { - image_width: layer.image_width, - image_height: layer.image_height, - format_code: kitty_format_code(pane_graphics_kitty_format(layer.format)), - data_len: layer.data_len(), - data_fingerprint: layer.data_fingerprint, - } -} - fn image_signature(placement: &HostPlacement, format_code: u32) -> ImageSignature { ImageSignature { image_width: placement.placement.image_width, @@ -1056,27 +770,80 @@ fn kitty_format_code(format: KittyImageFormat) -> u32 { } fn encode_kitty_data(out: &mut Vec<u8>, control: &str, data: &[u8]) { + write_kitty_data(out, control, data).expect("writing to Vec cannot fail"); +} + +pub(crate) fn write_kitty_data( + out: &mut impl Write, + control: &str, + data: &[u8], +) -> std::io::Result<()> { let mut chunks = data.chunks(KITTY_CHUNK_BYTES).peekable(); let Some(first) = chunks.next() else { - return; + return Ok(()); }; let more = if chunks.peek().is_some() { 1 } else { 0 }; let encoded = base64::engine::general_purpose::STANDARD.encode(first); - let _ = write!(out, "\x1b_G{control},m={more};{encoded}\x1b\\"); + write!(out, "\x1b_G{control},m={more};{encoded}\x1b\\")?; while let Some(chunk) = chunks.next() { let more = if chunks.peek().is_some() { 1 } else { 0 }; let encoded = base64::engine::general_purpose::STANDARD.encode(chunk); - let _ = write!(out, "\x1b_Gm={more};{encoded}\x1b\\"); + write!(out, "\x1b_Gm={more};{encoded}\x1b\\")?; } + Ok(()) } #[cfg(test)] mod tests { use super::*; + #[test] + fn deferred_updates_match_inline_for_rgb_and_rgba() { + for format in [KittyImageFormat::Rgb, KittyImageFormat::Rgba] { + let mut first = test_placement(0, 0); + first.placement.format = format; + first.placement.data = vec![37; KITTY_CHUNK_BYTES * 2 + 7]; + first.placement.data_len = first.placement.data.len(); + let data: Arc<[u8]> = Arc::from(first.placement.data.clone()); + let mut second = test_placement(3, 0); + second.placement = first.placement.clone(); + second.placement.placement_id += 1; + let mut placements = [first, second]; + let mut inline_cache = HostGraphicsCache::default(); + let expected = drain_graphics_updates(&mut inline_cache, &placements); + for placement in &mut placements { + placement.placement.data.clear(); + placement.raw_data = Some(Arc::clone(&data)); + } + let mut cache = HostGraphicsCache::default(); + let mut output = GraphicsOutput::default(); + loop { + let (next, incomplete) = + encode_graphics_output_incremental(&mut cache, &placements); + output.extend(next); + if !incomplete { + break; + } + } + assert_eq!(output.clone().into_inline_bytes(), expected); + let uploads = output + .operations + .iter() + .filter_map(|op| match op { + GraphicsOperation::Upload { data, .. } => Some(data), + _ => None, + }) + .collect::<Vec<_>>(); + assert_eq!(uploads.len(), 1); + assert!(Arc::ptr_eq(uploads[0], &data)); + assert_eq!(cache.placements.len(), 2); + } + } + fn test_placement(viewport_col: i32, viewport_row: i32) -> HostPlacement { HostPlacement { + raw_data: None, pane_id: PaneId::from_raw(1), host_image_id: None, area: Rect::new(0, 0, 20, 10), @@ -1101,7 +868,8 @@ mod tests { data_len: 30 * 30 * 4, data_fingerprint: 42, data: vec![255; 30 * 30 * 4], - render: KittyPlacementRenderInfo { + source_file: None, + render: crate::ghostty::KittyPlacementRenderInfo { pixel_width: 0, pixel_height: 0, grid_cols: 3, @@ -1117,15 +885,6 @@ mod tests { } } - fn pane_layer_placement(viewport_col: i32, viewport_row: i32) -> HostPlacement { - let mut placement = test_placement(viewport_col, viewport_row); - placement.source_key = HostSourceKey::PaneLayer { - pane_id: placement.pane_id, - layer_id: "primary".into(), - }; - placement - } - fn update( cache: &mut HostGraphicsCache, placements: &[HostPlacement], @@ -1135,13 +894,7 @@ mod tests { if replay { cache.request_placement_replay(); } - let live = cache - .sources - .keys() - .filter(|source| matches!(source, HostSourceKey::PaneLayer { .. })) - .cloned() - .collect::<HashSet<_>>(); - bytes.extend(drain_graphics_updates(cache, placements, &live)); + bytes.extend(drain_graphics_updates(cache, placements)); bytes } @@ -1158,16 +911,6 @@ mod tests { host_placement_id(&placement.source_key, &placement.placement), expected ); - assert_ne!( - host_placement_id( - &HostSourceKey::PaneLayer { - pane_id: placement.pane_id, - layer_id: "primary".into(), - }, - &placement.placement, - ), - expected - ); } #[cfg(unix)] @@ -1187,33 +930,6 @@ mod tests { } #[test] - fn pane_graphics_image_ids_are_disjoint_from_terminal_image_ids() { - let placement = test_placement(0, 0); - let signature = image_signature(&placement, kitty_format_code(placement.placement.format)); - let terminal_id = host_image_id_for_signature(placement.pane_id, signature); - let mut graphics = crate::app::pane_graphics::Runtime::default(); - let primary = (placement.pane_id, "primary".into()); - let pane_graphics_id = graphics.reserve_image_id(&primary).unwrap(); - graphics.slots.insert( - primary.clone(), - crate::app::pane_graphics::Slot::test(pane_graphics_id, None), - ); - - assert_eq!(terminal_id & PANE_GRAPHICS_IMAGE_ID_BIT, 0); - assert_ne!(pane_graphics_id & PANE_GRAPHICS_IMAGE_ID_BIT, 0); - assert_eq!( - pane_graphics_id, - graphics.reserve_image_id(&primary).unwrap() - ); - assert_ne!( - pane_graphics_id, - graphics - .reserve_image_id(&(placement.pane_id, "toolbar".into())) - .unwrap() - ); - } - - #[test] fn clipped_placement_handles_positive_viewport_without_wrapping() { let placement = test_placement(2, 2); let (clipped, _) = clipped_placement(&placement).expect("visible placement"); @@ -1240,47 +956,6 @@ mod tests { } #[test] - fn pane_graphics_layer_defaults_to_full_pane_grid() { - let info = PaneInfo { - id: PaneId::from_raw(9), - rect: Rect::new(0, 0, 12, 5), - inner_rect: Rect::new(2, 1, 8, 3), - scrollbar_rect: None, - borders: ratatui::widgets::Borders::NONE, - is_focused: true, - }; - let layer = crate::app::pane_graphics::Layer::inline( - crate::api::schema::PaneGraphicsFormat::Rgba, - 80, - 30, - vec![255; 80 * 30 * 4], - crate::api::schema::PaneGraphicsPlacementParams::default(), - 0, - ); - - let placement = pane_graphics_host_placement( - &info, - "primary", - PANE_GRAPHICS_IMAGE_ID_BIT | 1, - HostCellSize { - width_px: 10, - height_px: 10, - }, - &layer, - &HashMap::new(), - true, - ); - let (clipped, format_code) = clipped_placement(&placement).expect("visible layer"); - - assert_eq!(format_code, 32); - assert_eq!(clipped.x, 2); - assert_eq!(clipped.y, 1); - assert_eq!(clipped.cols, 8); - assert_eq!(clipped.rows, 3); - assert_eq!(placement.placement.data.len(), 80 * 30 * 4); - } - - #[test] fn graphics_update_uploads_once_then_repositions_only() { let mut cache = HostGraphicsCache::default(); let first = update(&mut cache, &[test_placement(0, 0)], false); @@ -1335,30 +1010,6 @@ mod tests { #[test] fn changing_first_source_does_not_starve_second_source() { - let layers = |first| { - [(1, "a", first), (2, "b", 80)].map(|(id, name, fingerprint)| { - let mut placement = pane_layer_placement(0, 0); - placement.host_image_id = Some(PANE_GRAPHICS_IMAGE_ID_BIT | id); - placement.source_key = HostSourceKey::PaneLayer { - pane_id: placement.pane_id, - layer_id: name.into(), - }; - placement.placement.data_fingerprint = fingerprint; - placement - }) - }; - let initial = layers(42); - let live = initial.iter().map(|p| p.source_key.clone()).collect(); - let mut cache = HostGraphicsCache::default(); - assert!( - encode_graphics_update_incremental(&mut cache, &initial, &live, None, false).incomplete - ); - assert!( - encode_graphics_update_incremental(&mut cache, &layers(43), &live, None, false) - .incomplete - ); - assert_eq!(cache.images.len(), 2, "second source uploaded next"); - let terminal = |id| { let mut placement = test_placement(0, 0); placement.placement.image_id = id; @@ -1373,57 +1024,14 @@ mod tests { let mut cache = HostGraphicsCache::default(); for id in 1..=3 { assert!( - encode_graphics_update_incremental( - &mut cache, - &[terminal(id), terminal(99)], - &HashSet::new(), - None, - false, - ) - .incomplete + encode_graphics_update_incremental(&mut cache, &[terminal(id), terminal(99)]) + .incomplete ); } assert!(cache.sources.contains_key(&second)); } #[test] - fn large_terminal_image_is_local_but_quarantined_headless() { - let placements = || { - let mut large = test_placement(0, 0); - large.placement.data_len = 24 * 1024 * 1024; - let mut later = test_placement(4, 0); - later.placement.image_id = 8; - later.source_key = HostSourceKey::Terminal { - pane_id: later.pane_id, - image_id: 8, - }; - [large, later] - }; - for (budget, expected) in [ - (None, (true, 1, 0)), - (Some(HEADLESS_GRAPHICS_TRANSACTION_BUDGET), (false, 1, 1)), - ] { - let mut cache = HostGraphicsCache::default(); - let encoded = encode_graphics_update_incremental( - &mut cache, - &placements(), - &HashSet::new(), - budget, - false, - ); - assert!(String::from_utf8_lossy(&encoded.bytes).contains("a=t")); - assert_eq!( - ( - encoded.incomplete, - cache.images.len(), - cache.oversized.len() - ), - expected - ); - } - } - - #[test] fn terminal_image_data_requests_deduplicate_and_reconsider_changed_signatures() { let pane_id = PaneId::from_raw(1); let descriptor = KittyImageDescriptor { @@ -1434,13 +1042,13 @@ mod tests { format: KittyImageFormat::Rgba, data_len: 3456 * 2234 * 4, data_fingerprint: 42, + source_file: false, }; let mut requested = HashSet::new(); assert!(terminal_image_needs_data( pane_id, descriptor, &HashMap::new(), - &HashMap::new(), &mut requested, )); let mut second_placement = descriptor; @@ -1449,7 +1057,6 @@ mod tests { pane_id, second_placement, &HashMap::new(), - &HashMap::new(), &mut requested, )); @@ -1458,13 +1065,12 @@ mod tests { pane_id, image_id: descriptor.image_id, }; - let oversized = HashMap::from([(source, signature)]); + let delivered = HashMap::from([(source, signature)]); let mut requested = HashSet::new(); assert!(!terminal_image_needs_data( pane_id, descriptor, - &HashMap::new(), - &oversized, + &delivered, &mut requested, )); let mut changed = descriptor; @@ -1472,38 +1078,8 @@ mod tests { assert!(terminal_image_needs_data( pane_id, changed, - &HashMap::new(), - &oversized, + &delivered, &mut requested, )); } - - #[test] - fn maximum_pane_graphics_stream_payload_fits_client_graphics_frame() { - let mut placement = pane_layer_placement(0, 0); - placement.placement.format = KittyImageFormat::Png; - placement.placement.image_width = 1; - placement.placement.image_height = 1; - placement.placement.data = vec![1_u8; crate::api::schema::PANE_GRAPHICS_STREAM_MAX_BYTES]; - placement.placement.data_len = placement.placement.data.len(); - let (clipped, format_code) = clipped_placement(&placement).expect("visible placement"); - let host_id = host_image_id(placement.pane_id, &placement.placement); - let mut encoded = Vec::new(); - - assert!(encode_upload_image( - &mut encoded, - &placement, - format_code, - host_id, - )); - encode_display_placement(&mut encoded, clipped, host_id, 1, 0); - - let mut framed = Vec::new(); - crate::protocol::write_message( - &mut framed, - &crate::protocol::ServerMessage::Graphics { bytes: encoded }, - ) - .unwrap(); - assert!(framed.len() <= crate::protocol::MAX_GRAPHICS_FRAME_SIZE + 4); - } } diff --git a/src/kitty_graphics/output.rs b/src/kitty_graphics/output.rs new file mode 100644 index 00000000..134a1f08 --- /dev/null +++ b/src/kitty_graphics/output.rs @@ -0,0 +1,94 @@ +//! Ordered, local-only graphics output. Upload pixels stay raw until output I/O. +#[cfg(test)] +use std::io::{self, Write}; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub(crate) enum GraphicsOperation { + Bytes(Vec<u8>), + Upload { control: String, data: Arc<[u8]> }, +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct GraphicsOutput { + pub(crate) operations: Vec<GraphicsOperation>, +} + +impl GraphicsOutput { + pub(crate) fn from_bytes(bytes: Vec<u8>) -> Self { + let mut output = Self::default(); + output.push_bytes(bytes); + output + } + + pub(crate) fn push_bytes(&mut self, bytes: Vec<u8>) { + if !bytes.is_empty() { + self.operations.push(GraphicsOperation::Bytes(bytes)); + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.operations.is_empty() + } + + pub(crate) fn extend(&mut self, other: Self) { + self.operations.extend(other.operations); + } + + #[cfg(test)] + pub(crate) fn into_inline_bytes(self) -> Vec<u8> { + let mut bytes = Vec::new(); + for operation in self.operations { + match operation { + GraphicsOperation::Bytes(next) if bytes.is_empty() => bytes = next, + GraphicsOperation::Bytes(next) => bytes.extend(next), + GraphicsOperation::Upload { control, data } => { + super::encode_kitty_data(&mut bytes, &control, &data); + } + } + } + bytes + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn inline_fallback_preserves_order_and_chunk_encoding() { + let data: Arc<[u8]> = Arc::from(vec![42; super::super::KITTY_CHUNK_BYTES + 1]); + let control = "a=t,t=d,f=24,s=1,v=1,i=7,q=2"; + let mut output = GraphicsOutput::default(); + assert!(output.is_empty()); + output.extend(GraphicsOutput::from_bytes(b"before".to_vec())); + output.operations.push(GraphicsOperation::Upload { + control: control.into(), + data: Arc::clone(&data), + }); + output.extend(GraphicsOutput::from_bytes(b"after".to_vec())); + let mut expected = b"before".to_vec(); + super::super::encode_kitty_data(&mut expected, control, &data); + expected.extend_from_slice(b"after"); + assert_eq!(output.into_inline_bytes(), expected); + } + + #[test] + fn inline_write_propagates_io_errors() { + struct Broken; + impl Write for Broken { + fn write(&mut self, _: &[u8]) -> io::Result<usize> { + Err(io::Error::from(io::ErrorKind::BrokenPipe)) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + assert_eq!( + super::super::write_kitty_data(&mut Broken, "a=t,t=d,f=32", &[1, 2, 3, 4]) + .unwrap_err() + .kind(), + io::ErrorKind::BrokenPipe + ); + } +} diff --git a/src/kitty_graphics/surface.rs b/src/kitty_graphics/surface.rs index 4df05ffb..9e03d642 100644 --- a/src/kitty_graphics/surface.rs +++ b/src/kitty_graphics/surface.rs @@ -1,10 +1,13 @@ use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use super::output::GraphicsOutput; use ratatui::layout::Rect; use super::{ - clipped_placement, collect_visible_placements, encode_graphics_update_incremental, + clipped_placement, collect_visible_placements, encode_graphics_output_incremental, HostCellSize, HostGraphicsCache, HostPlacement, HostSourceKey, ImageSignature, }; use crate::ghostty::{ @@ -24,7 +27,15 @@ pub(crate) struct DeliveryCache { pending: bool, } +pub(crate) type SourceFiles = + HashMap<SurfaceGraphicsAssetKey, Arc<crate::pane_graphics_files::OwnedExport>>; + impl DeliveryCache { + pub(crate) fn forget_asset(&mut self, key: &SurfaceGraphicsAssetKey) { + self.assets.remove(key); + self.pending = true; + } + pub(crate) fn has_pending(&self) -> bool { self.pending } @@ -92,11 +103,13 @@ impl Occlusion { } } -#[derive(Debug, Default)] +#[derive(Clone, Debug, Default)] pub(crate) struct ClientState { scope: String, scene: SurfaceGraphicsScene, - assets: HashMap<SurfaceGraphicsAssetKey, Vec<u8>>, + display_scene: SurfaceGraphicsScene, + native_image_ids: HashMap<SurfaceGraphicsAssetKey, u32>, + assets: HashMap<SurfaceGraphicsAssetKey, Arc<[u8]>>, host: HostGraphicsCache, trusted_direct: HashMap<SurfaceGraphicsAssetKey, u32>, reset_pending: bool, @@ -105,6 +118,21 @@ pub(crate) struct ClientState { } impl ClientState { + pub(crate) fn new() -> Self { + Self::default() + } + + fn image_id(&self, key: &SurfaceGraphicsAssetKey) -> u32 { + if matches!(key.source, SurfaceGraphicsSource::Terminal { .. }) { + return self + .native_image_ids + .get(key) + .copied() + .unwrap_or_else(|| native_host_image_id(&self.scope, key)); + } + host_image_id(&self.scope, key) + } + pub(crate) fn scope(&self) -> &str { &self.scope } @@ -115,6 +143,8 @@ impl ClientState { } self.scope = scope.to_owned(); self.scene = SurfaceGraphicsScene::default(); + self.display_scene = SurfaceGraphicsScene::default(); + self.native_image_ids.clear(); self.assets.clear(); self.trusted_direct.clear(); self.stale_images.clear(); @@ -123,17 +153,53 @@ impl ClientState { } #[cfg(unix)] + pub(crate) fn accepts_direct_asset( + &self, + key: &SurfaceGraphicsAssetKey, + image_id: u32, + ) -> bool { + if matches!(key.source, SurfaceGraphicsSource::Terminal { .. }) { + return !self.stale_images.contains(&image_id) + && !self.forced_delete_images.contains(&image_id) + && (image_id == native_host_image_id(&self.scope, key) + || image_id == (native_host_image_id(&self.scope, key) ^ NATIVE_SLOT_BIT)) + && !self + .display_scene + .placements + .iter() + .map(|p| &p.asset) + .chain(self.display_scene.retained_assets.iter()) + .any(|visible| { + self.image_id(visible) == image_id + && self.host.images.get(&image_id) + == Some(&image_signature_from_asset(visible)) + }) + && (self + .scene + .placements + .iter() + .any(|placement| &placement.asset == key) + || self.scene.retained_assets.contains(key)); + } + image_id == host_image_id(&self.scope, key) + } + + #[cfg(unix)] pub(crate) fn trust_direct_asset( &mut self, key: &SurfaceGraphicsAssetKey, image_id: u32, ) -> bool { - if image_id != host_image_id(&self.scope, key) { + if !self.accepts_direct_asset(key, image_id) { return false; } + if matches!(key.source, SurfaceGraphicsSource::Terminal { .. }) { + self.native_image_ids.insert(key.clone(), image_id); + } self.host .images .insert(image_id, image_signature_from_asset(key)); + self.refresh_display_scene(); if self .scene .placements @@ -182,18 +248,6 @@ impl ClientState { .map(|placement| placement.asset.clone()) .chain(scene.retained_assets.iter().cloned()) .collect::<HashSet<_>>(); - let previous = self - .scene - .placements - .iter() - .map(|placement| placement.asset.clone()) - .chain(self.scene.retained_assets.iter().cloned()) - .collect::<HashSet<_>>(); - self.stale_images.extend( - previous - .difference(&desired) - .map(|key| host_image_id(&self.scope, key)), - ); let unclaimed = self .trusted_direct .iter() @@ -206,15 +260,107 @@ impl ClientState { .iter() .map(|placement| placement.asset.clone()) .collect::<HashSet<_>>(); - self.assets.retain(|key, _| placed.contains(key)); for asset in std::mem::take(&mut scene.assets) { if asset.data.len() as u64 == asset.key.data_len && placed.contains(&asset.key) { - self.assets.insert(asset.key, asset.data); + self.assets.insert(asset.key, Arc::from(asset.data)); } } self.scene = scene; + self.refresh_display_scene(); } + fn refresh_display_scene(&mut self) { + let mut next = self.scene.clone(); + let previous_placements = self + .display_scene + .placements + .iter() + .map(|p| ((&p.asset.source, p.logical_placement_id, p.x, p.y), p)) + .collect::<HashMap<_, _>>(); + let previous_assets = self + .display_scene + .retained_assets + .iter() + .chain(self.display_scene.placements.iter().map(|p| &p.asset)) + .filter(|key| { + self.host.images.get(&self.image_id(key)) == Some(&image_signature_from_asset(key)) + }) + .map(|key| (&key.source, key)) + .collect::<HashMap<_, _>>(); + for placement in &mut next.placements { + let key = &placement.asset; + if !matches!(key.source, SurfaceGraphicsSource::Terminal { .. }) + || self.assets.contains_key(key) + || self.host.images.get(&self.image_id(key)) + == Some(&image_signature_from_asset(key)) + { + continue; + } + // A metadata-only replacement is not visible until its upload is ACKed. + // Reuse only an identical placement, never stale resize/scroll geometry. + if let Some(previous) = previous_placements + .get(&( + &key.source, + placement.logical_placement_id, + placement.x, + placement.y, + )) + .copied() + .filter(|previous| { + if self.host.images.get(&self.image_id(&previous.asset)) + != Some(&image_signature_from_asset(&previous.asset)) + { + return false; + } + let mut candidate = (*previous).clone(); + candidate.asset = key.clone(); + candidate == *placement + }) + { + placement.asset = previous.asset.clone(); + } + } + for key in &mut next.retained_assets { + if matches!(key.source, SurfaceGraphicsSource::Terminal { .. }) + && !self.assets.contains_key(key) + && self.host.images.get(&self.image_id(key)) + != Some(&image_signature_from_asset(key)) + { + if let Some(previous) = previous_assets.get(&key.source) { + *key = (*previous).clone(); + } + } + } + let desired = next + .placements + .iter() + .map(|p| &p.asset) + .chain(next.retained_assets.iter()) + .collect::<HashSet<_>>(); + let stale = self + .display_scene + .placements + .iter() + .map(|p| &p.asset) + .chain(self.display_scene.retained_assets.iter()) + .filter(|key| !desired.contains(key)) + .map(|key| self.image_id(key)) + .collect::<Vec<_>>(); + self.stale_images.extend(stale); + let owned = self + .scene + .placements + .iter() + .map(|p| &p.asset) + .chain(self.scene.retained_assets.iter()) + .chain(desired.iter().copied()) + .collect::<HashSet<_>>(); + self.assets.retain(|key, _| owned.contains(key)); + self.native_image_ids.retain(|key, _| owned.contains(key)); + self.display_scene = next; + } + + #[cfg(test)] pub(crate) fn encode( &mut self, visibility: Visibility, @@ -223,6 +369,18 @@ impl ClientState { cell_size: HostCellSize, occlusion: &Occlusion, ) -> Vec<u8> { + self.encode_output(visibility, main_origin, popup_origin, cell_size, occlusion) + .into_inline_bytes() + } + + pub(crate) fn encode_output( + &mut self, + visibility: Visibility, + main_origin: (u16, u16), + popup_origin: Option<(u16, u16)>, + cell_size: HostCellSize, + occlusion: &Occlusion, + ) -> GraphicsOutput { let mut bytes = self.take_pending_cleanup(); self.stale_images.sort_unstable(); self.stale_images.dedup(); @@ -238,69 +396,52 @@ impl ClientState { } if !cell_size.is_known() || self.scope.is_empty() { bytes.extend(self.host.clear_bytes()); - return bytes; + return GraphicsOutput::from_bytes(bytes); } let placements = self - .scene + .display_scene .placements .iter() .filter_map(|placement| { client_host_placement( &self.scope, placement, - self.assets.get(&placement.asset).map(Vec::as_slice), visibility, main_origin, popup_origin, cell_size, occlusion, ) + .map(|mut host| { + host.host_image_id = Some(self.image_id(&placement.asset)); + host.raw_data = self.assets.get(&placement.asset).map(Arc::clone); + host + }) }) .collect::<Vec<_>>(); + let mut output = GraphicsOutput::from_bytes(bytes); self.host.request_placement_replay(); loop { - let encoded = encode_graphics_update_incremental( - &mut self.host, - &placements, - &HashSet::new(), - None, - false, - ); - bytes.extend(encoded.bytes); - if !encoded.incomplete { - return bytes; + let (encoded, incomplete) = + encode_graphics_output_incremental(&mut self.host, &placements); + output.extend(encoded); + if !incomplete { + return output; } } } } -pub(crate) fn pane_layer_asset_key( - app: &crate::app::App, - key: &crate::app::pane_graphics::Key, - layer: &crate::app::pane_graphics::Layer, -) -> Option<SurfaceGraphicsAssetKey> { - let workspace_index = app - .state - .workspaces - .iter() - .position(|workspace| workspace.pane_state(key.0).is_some())?; - Some(SurfaceGraphicsAssetKey { - source: SurfaceGraphicsSource::PaneLayer { - pane_id: app.public_pane_id(workspace_index, key.0)?, - layer_id: key.1.clone(), - }, - image_width: layer.image_width, - image_height: layer.image_height, - format: match layer.format { - crate::api::schema::PaneGraphicsFormat::Rgb => SurfaceGraphicsFormat::Rgb, - crate::api::schema::PaneGraphicsFormat::Rgba - | crate::api::schema::PaneGraphicsFormat::Bgra => SurfaceGraphicsFormat::Rgba, - crate::api::schema::PaneGraphicsFormat::Png => SurfaceGraphicsFormat::Png, - }, - data_len: layer.data_len() as u64, - data_fingerprint: layer.data_fingerprint, - }) +pub(crate) const NATIVE_TRANSFER_BIT: u64 = 1 << 63; +pub(crate) const NATIVE_SLOT_BIT: u32 = 1 << 29; + +pub(crate) fn native_host_image_id(scope: &str, key: &SurfaceGraphicsAssetKey) -> u32 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + scope.hash(&mut hasher); + key.source.hash(&mut hasher); + // Keep native IDs separate from the direct API namespace. + 0x4000_0000 | (hasher.finish() as u32 & 0x1fff_ffff) } pub(crate) fn host_image_id(scope: &str, key: &SurfaceGraphicsAssetKey) -> u32 { @@ -310,6 +451,7 @@ pub(crate) fn host_image_id(scope: &str, key: &SurfaceGraphicsAssetKey) -> u32 { 10_000 + ((hasher.finish() as u32) % 900_000) } +#[cfg(all(test, unix))] pub(crate) fn direct_upload_control(scope: &str, key: &SurfaceGraphicsAssetKey) -> (u32, String) { let image_id = host_image_id(scope, key); ( @@ -329,10 +471,14 @@ pub(crate) fn collect_scene( popup_content_size: Option<(u16, u16)>, cell_size: HostCellSize, delivered: &DeliveryCache, - client_id: u64, -) -> (SurfaceGraphicsScene, DeliveryCache) { + _client_id: u64, +) -> (SurfaceGraphicsScene, DeliveryCache, SourceFiles) { if !cell_size.is_known() { - return (SurfaceGraphicsScene::default(), DeliveryCache::default()); + return ( + SurfaceGraphicsScene::default(), + DeliveryCache::default(), + SourceFiles::default(), + ); } let workspace_index = surface.target.map(|target| target.workspace_index); let mut targets = HashMap::new(); @@ -357,10 +503,9 @@ pub(crate) fn collect_scene( targets.insert(*pane_id, target.clone()); } - // Reconstruct only the small image-signature index expected by the existing - // collector. This prevents copying already-delivered image payloads on each - // pane-scaled render while keeping Ghostty as the authoritative image store. - let mut uploaded_images = HashMap::new(); + // Reconstruct the small source-signature index expected by the collector. + // This avoids copying already-delivered payloads while Ghostty remains the + // authoritative image store. let mut delivered_terminal_images = HashMap::new(); for key in &delivered.assets { let signature = image_signature_from_asset(key); @@ -379,13 +524,8 @@ pub(crate) fn collect_scene( ); } } - SurfaceGraphicsSource::PaneLayer { pane_id, layer_id } => { - if let Some(pane_id) = public_panes.get(pane_id) { - if let Some(slot) = app.pane_graphics.slots.get(&(*pane_id, layer_id.clone())) { - uploaded_images.insert(slot.host_image_id, signature); - } - } - } + // Frozen for old-server compatibility; current servers no longer collect layers. + SurfaceGraphicsSource::PaneLayer { .. } => {} SurfaceGraphicsSource::Terminal { target: SurfaceGraphicsTarget::Popup { .. }, .. @@ -394,13 +534,10 @@ pub(crate) fn collect_scene( } let mut host_placements = collect_visible_placements( &app.state, - &app.pane_graphics, &app.terminal_runtimes, surface, cell_size, - &uploaded_images, &delivered_terminal_images, - client_id, ); if let (Some(popup), Some((width, height)), Some((_, target))) = ( @@ -418,9 +555,10 @@ pub(crate) fn collect_scene( }, descriptor, ); - !delivered.assets.contains(&key) && requested.insert(key) + !descriptor.source_file && !delivered.assets.contains(&key) && requested.insert(key) }) { host_placements.push(HostPlacement { + raw_data: None, pane_id: popup.pane_id, host_image_id: None, area: Rect::new(0, 0, width, height), @@ -441,6 +579,7 @@ pub(crate) fn collect_scene( let mut placements = Vec::new(); let mut asset_data = HashMap::<SurfaceGraphicsAssetKey, Vec<u8>>::new(); + let mut sources = SourceFiles::default(); for mut placement in host_placements { if placements.len() == MAX_SURFACE_GRAPHICS_PLACEMENTS { break; @@ -453,21 +592,18 @@ pub(crate) fn collect_scene( target, image_id: *image_id, }, - HostSourceKey::PaneLayer { layer_id, .. } => { - let SurfaceGraphicsTarget::Pane { pane_id } = target else { - continue; - }; - SurfaceGraphicsSource::PaneLayer { - pane_id, - layer_id: layer_id.clone(), - } - } HostSourceKey::ClientSurface { .. } => continue, }; let Some((clipped, _)) = clipped_placement(&placement) else { continue; }; let asset = asset_key(source, &placement.placement); + if !delivered.assets.contains(&asset) { + if let Some(file) = placement.placement.source_file.take() { + sources.entry(asset.clone()).or_insert(file); + asset_data.entry(asset.clone()).or_default(); + } + } if !placement.placement.data.is_empty() { asset_data .entry(asset.clone()) @@ -507,7 +643,9 @@ pub(crate) fn collect_scene( if next.assets.contains(&key) { continue; } - let encoded_size = super::image_transfer_estimated_size(data.len()); + let encoded_size = super::image_transfer_estimated_size( + usize::try_from(key.data_len).unwrap_or(usize::MAX), + ); if encoded_size > super::HEADLESS_GRAPHICS_TRANSACTION_BUDGET { continue; } @@ -532,17 +670,8 @@ pub(crate) fn collect_scene( placement.x, ) }); - let mut retained_assets = app - .pane_graphics - .slots - .iter() - .filter_map(|(key, slot)| { - let layer = slot.layer.as_ref()?; - (slot.direct_client() == Some(client_id)) - .then(|| pane_layer_asset_key(app, key, layer))? - }) - .collect::<Vec<_>>(); - retained_assets.sort_by_key(|key| format!("{:?}", key.source)); + let retained_assets = Vec::new(); + sources.retain(|key, _| next.assets.contains(key)); ( SurfaceGraphicsScene { assets, @@ -550,6 +679,7 @@ pub(crate) fn collect_scene( retained_assets, }, next, + sources, ) } @@ -602,7 +732,6 @@ fn asset_key( fn client_host_placement( scope: &str, placement: &SurfaceGraphicsPlacement, - data: Option<&[u8]>, visibility: Visibility, main_origin: (u16, u16), popup_origin: Option<(u16, u16)>, @@ -653,6 +782,7 @@ fn client_host_placement( let cols = placement.cols.min(u32::from(u16::MAX)) as u16; let rows = placement.rows.min(u32::from(u16::MAX)) as u16; Some(HostPlacement { + raw_data: None, pane_id, host_image_id: Some(host_image_id), area: Rect::new( @@ -678,7 +808,8 @@ fn client_host_placement( }, data_len: usize::try_from(placement.asset.data_len).unwrap_or(usize::MAX), data_fingerprint: placement.asset.data_fingerprint, - data: data.unwrap_or_default().to_vec(), + source_file: None, + data: Vec::new(), render: KittyPlacementRenderInfo { pixel_width: placement.cols.saturating_mul(cell_size.width_px), pixel_height: placement.rows.saturating_mul(cell_size.height_px), @@ -753,6 +884,387 @@ mod tests { } #[test] + fn legacy_pane_layer_scene_still_uploads_and_places() { + let mut state = ClientState::default(); + state.set_scope("old-server-pane-layer"); + let mut image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "pane".into(), + }, + 1, + vec![1, 2, 3, 255], + ); + image.key.source = SurfaceGraphicsSource::PaneLayer { + pane_id: "pane".into(), + layer_id: "primary".into(), + }; + state.set_scene(scene(image, 2, 1)); + + let bytes = String::from_utf8(state.encode( + Visibility::Main, + (10, 5), + None, + HostCellSize { + width_px: 8, + height_px: 16, + }, + &Occlusion::default(), + )) + .unwrap(); + + assert!(bytes.contains("a=t,t=d"), "{bytes:?}"); + assert!(bytes.contains("a=p,"), "{bytes:?}"); + assert!(bytes.contains("\u{1b}[7;13H"), "{bytes:?}"); + } + + #[test] + fn stable_ids_replace_native_pixels_and_delete_on_removal() { + use crate::kitty_graphics::GraphicsOperation; + let mut state = ClientState::default(); + state.set_scope("stable-native"); + let target = SurfaceGraphicsTarget::Pane { + pane_id: "pane".into(), + }; + let first = asset(target.clone(), 1, vec![1, 2, 3, 255]); + let id = state.image_id(&first.key); + let cell = HostCellSize { + width_px: 8, + height_px: 16, + }; + for revision in 1..=8 { + let next = asset(target.clone(), revision, vec![revision as u8, 2, 3, 255]); + assert_eq!(state.image_id(&next.key), id); + state.set_scene(scene(next, 0, 0)); + let output = + state.encode_output(Visibility::Main, (0, 0), None, cell, &Occlusion::default()); + let uploads = output + .operations + .iter() + .filter_map(|op| match op { + GraphicsOperation::Upload { control, data } => Some((control, data)), + _ => None, + }) + .collect::<Vec<_>>(); + assert_eq!(uploads.len(), 1); + assert!(uploads[0].0.contains(&format!("i={id},"))); + assert_eq!(uploads[0].1.as_ref(), &[revision as u8, 2, 3, 255]); + let bytes = String::from_utf8(output.into_inline_bytes()).unwrap(); + let upload = bytes.find("a=t,").unwrap(); + let display = bytes.find("a=p,").unwrap(); + assert!(upload < display); + if revision > 1 { + assert!(bytes.find("a=d,d=I,").unwrap() < upload); + } + assert_eq!(state.host.images.len(), 1); + } + let mut other = first.key.clone(); + other.source = SurfaceGraphicsSource::Terminal { + target: SurfaceGraphicsTarget::Pane { + pane_id: "other-pane".into(), + }, + image_id: 7, + }; + assert_ne!(state.image_id(&other), id); + other.source = SurfaceGraphicsSource::PaneLayer { + pane_id: "pane".into(), + layer_id: "layer".into(), + }; + assert_eq!(state.image_id(&other), host_image_id(state.scope(), &other)); + let legacy_id = state.image_id(&other); + other.data_fingerprint += 1; + assert_ne!(state.image_id(&other), legacy_id); + state.set_scene(SurfaceGraphicsScene::default()); + let bytes = state.encode(Visibility::Main, (0, 0), None, cell, &Occlusion::default()); + assert!(String::from_utf8(bytes) + .unwrap() + .contains(&format!("a=d,d=I,i={id}"))); + assert!(state.host.images.is_empty()); + } + + #[cfg(unix)] + #[test] + fn native_file_adoption_keeps_previous_frame_until_replacement_ack() { + let mut state = ClientState::default(); + state.set_scope("native-files"); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "p1".into(), + }, + 1, + vec![1, 2, 3, 255], + ); + let id = native_host_image_id(state.scope(), &image.key); + assert!(!state.accepts_direct_asset(&image.key, id)); + let mut metadata = scene(image.clone(), 0, 0); + metadata.assets.clear(); + state.set_scene(metadata); + let cell = HostCellSize { + width_px: 8, + height_px: 16, + }; + state.encode(Visibility::Main, (0, 0), None, cell, &Occlusion::default()); + assert!(state.trust_direct_asset(&image.key, id)); + let first = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )) + .unwrap(); + assert!(first.contains("a=p,")); + assert!(!first.contains("a=t,")); + let mut next = image.clone(); + next.key.data_fingerprint += 1; + let mut metadata = scene(next.clone(), 0, 0); + metadata.assets.clear(); + state.set_scene(metadata); + assert!(!state.trust_direct_asset(&image.key, id)); + let cleanup = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )) + .unwrap(); + assert!(!cleanup.contains("a=d,")); + assert!(cleanup.contains(&format!("a=p,i={id},"))); + assert!(!state.accepts_direct_asset(&next.key, id)); + let replacement_id = id ^ NATIVE_SLOT_BIT; + assert!(state.trust_direct_asset(&next.key, replacement_id)); + let placed = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )) + .unwrap(); + assert!(placed.contains(&format!("a=p,i={replacement_id},"))); + assert!(placed.contains(&format!("a=d,d=I,i={id},"))); + assert!(!placed.contains("a=t,")); + assert_eq!(state.host.images.len(), 1); + assert_eq!(state.native_image_ids.len(), 1); + // The vacated bank is reusable, but the visible bank cannot be overwritten. + assert!(state.accepts_direct_asset(&next.key, id)); + assert!(!state.accepts_direct_asset(&next.key, replacement_id)); + assert!(!state.accepts_direct_asset(&next.key, host_image_id(state.scope(), &next.key))); + } + + #[cfg(unix)] + #[test] + fn native_replacements_use_two_ids_without_blank_pending_frames() { + let mut state = ClientState::default(); + state.set_scope("double-buffer"); + let cell = HostCellSize { + width_px: 8, + height_px: 16, + }; + let mut previous = None; + let mut ids = HashSet::new(); + for revision in 0..32 { + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "pane".into(), + }, + revision, + vec![revision as u8, 2, 3, 255], + ); + let base = native_host_image_id(state.scope(), &image.key); + let id = if revision % 2 == 0 { + base ^ NATIVE_SLOT_BIT + } else { + base + }; + ids.insert(id); + let mut metadata = scene(image.clone(), 0, 0); + metadata.assets.clear(); + state.set_scene(metadata.clone()); + for _ in 0..3 { + state.set_scene(metadata.clone()); + let pending = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )) + .unwrap(); + assert!(!pending.contains("a=d,"), "{pending:?}"); + if let Some(previous) = previous { + assert!(pending.contains(&format!("a=p,i={previous},"))); + assert!(!state.accepts_direct_asset(&image.key, previous)); + } + } + assert!(state.trust_direct_asset(&image.key, id)); + let swapped = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )) + .unwrap(); + assert!(swapped.contains(&format!("a=p,i={id},"))); + assert!(!swapped.contains(&format!("a=d,d=I,i={id},"))); + if let Some(previous) = previous { + assert!(swapped.contains(&format!("a=d,d=I,i={previous},"))); + } + assert_eq!(state.host.images.len(), 1); + assert_eq!(state.native_image_ids.len(), 1); + assert!(!state.accepts_direct_asset(&image.key, id)); + assert_eq!(id & 0x8000_0000, 0, "native IDs must not alias API IDs"); + previous = Some(id); + } + assert_eq!(ids.len(), 2); + state.set_scene(SurfaceGraphicsScene::default()); + let cleanup = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )) + .unwrap(); + assert!(cleanup.contains(&format!("a=d,d=I,i={},", previous.unwrap()))); + assert!(state.host.images.is_empty()); + assert!(state.native_image_ids.is_empty()); + } + + #[cfg(unix)] + #[test] + fn native_pending_replacement_failure_and_geometry_do_not_preserve_stale_frames() { + let mut state = ClientState::default(); + state.set_scope("replacement-fallback"); + let cell = HostCellSize { + width_px: 8, + height_px: 16, + }; + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "pane".into(), + }, + 1, + vec![1, 2, 3, 255], + ); + let base = native_host_image_id(state.scope(), &image.key); + let active = base ^ NATIVE_SLOT_BIT; + let mut metadata = scene(image.clone(), 0, 0); + metadata.assets.clear(); + state.set_scene(metadata); + state.encode(Visibility::Main, (0, 0), None, cell, &Occlusion::default()); + assert!(state.trust_direct_asset(&image.key, active)); + state.encode(Visibility::Main, (0, 0), None, cell, &Occlusion::default()); + let mut next = image.clone(); + next.key.data_fingerprint += 1; + let mut metadata = scene(next.clone(), 0, 0); + metadata.assets.clear(); + state.set_scene(metadata); + state.retire_direct_image(base); + let rejected = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )) + .unwrap(); + assert!(!rejected.contains(&format!("a=d,d=I,i={active},"))); + assert!(rejected.contains(&format!("a=p,i={active},"))); + state.set_scene(scene(next.clone(), 0, 0)); + let fallback = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )) + .unwrap(); + assert!(fallback.contains("a=t,")); + assert!(fallback.contains(&format!("a=p,i={base},"))); + assert!(fallback.contains(&format!("a=d,d=I,i={active},"))); + assert!(!state.accepts_direct_asset(&next.key, base)); + next.key.data_fingerprint += 1; + assert!(!state.accepts_direct_asset(&next.key, base)); + let mut moved = scene(next.clone(), 1, 0); + moved.assets.clear(); + state.set_scene(moved); + let geometry = String::from_utf8(state.encode( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )) + .unwrap(); + assert!(geometry.contains(&format!("a=d,d=I,i={base},"))); + assert!(!geometry.contains("a=p,")); + state.set_scene(SurfaceGraphicsScene::default()); + assert!(!state.trust_direct_asset(&next.key, active)); + } + + #[test] + fn deferred_client_reuses_resident_arc_and_orders_cleanup_before_upload() { + use super::super::output::GraphicsOperation; + let mut state = ClientState::default(); + state.set_scope("raw"); + let image = asset( + SurfaceGraphicsTarget::Pane { + pane_id: "pane".into(), + }, + 1, + vec![1, 2, 3, 4], + ); + let key = image.key.clone(); + let mut graphics = scene(image, 0, 0); + let mut second = graphics.placements[0].clone(); + second.logical_placement_id += 1; + second.x = 2; + graphics.placements.push(second); + let mut replacement = graphics.clone(); + replacement.assets.clear(); + state.set_scene(graphics); + let resident = Arc::clone(&state.assets[&key]); + state.set_scene(replacement); + assert!(Arc::ptr_eq(&resident, &state.assets[&key])); + state.forced_delete_images.push(4242); + let cell = HostCellSize { + width_px: 8, + height_px: 16, + }; + let output = + state.encode_output(Visibility::Main, (0, 0), None, cell, &Occlusion::default()); + assert!( + matches!(&output.operations[0], GraphicsOperation::Bytes(bytes) + if String::from_utf8_lossy(bytes).contains("a=d,d=I,i=4242")) + ); + let uploads = output + .operations + .iter() + .filter_map(|op| match op { + GraphicsOperation::Upload { data, .. } => Some(data), + _ => None, + }) + .collect::<Vec<_>>(); + assert_eq!(uploads.len(), 1); + assert!(Arc::ptr_eq(uploads[0], &resident)); + assert_eq!( + String::from_utf8(output.clone().into_inline_bytes()) + .unwrap() + .matches("a=p") + .count(), + 2 + ); + let replay = + state.encode_output(Visibility::Main, (0, 0), None, cell, &Occlusion::default()); + assert!(replay + .operations + .iter() + .all(|op| matches!(op, GraphicsOperation::Bytes(_)))); + } + + #[test] fn occlusion_uses_final_pixel_bounds_and_strict_overlap() { let cell = HostCellSize { width_px: 8, @@ -797,7 +1309,7 @@ mod tests { 1, vec![1, 2, 3, 4], ); - let id = host_image_id("occlusion", &image.key); + let id = state.image_id(&image.key); let mut graphics = scene(image.clone(), 0, 0); let mut second = graphics.placements[0].clone(); second.logical_placement_id = 4; @@ -907,12 +1419,12 @@ mod tests { cover.cover(Rect::new(0, 10, 80, 3)); cover.cover(Rect::new(60, 0, 20, 5)); } - let _ = state.encode(Visibility::Main, (0, 0), None, cell, &cover); + let _ = state.encode_output(Visibility::Main, (0, 0), None, cell, &cover); let mut samples = Vec::new(); for _ in 0..101 { let start = std::time::Instant::now(); for _ in 0..20 { - std::hint::black_box(state.encode( + std::hint::black_box(state.encode_output( Visibility::Main, (0, 0), None, @@ -923,7 +1435,57 @@ mod tests { samples.push(start.elapsed().as_nanos() / 20); } samples.sort_unstable(); - eprintln!("graphics occlusion panes={count} disjoint_overlays={covered} median_ns={} p95_ns={}", samples[50], samples[95]); + eprintln!( + "graphics occlusion panes={count} disjoint_overlays={covered} median_ns={} p95_ns={}", + samples[50], samples[95] + ); + } + #[cfg(unix)] + { + let mut state = ClientState::default(); + state.set_scope("swap-profile"); + state.set_scene(graphics.clone()); + state.encode_output(Visibility::Main, (0, 0), None, cell, &Occlusion::default()); + graphics.assets.clear(); + let mut samples = Vec::new(); + for revision in 1..=105 { + graphics.placements[0].asset.data_fingerprint = revision; + let key = graphics.placements[0].asset.clone(); + let base = native_host_image_id(state.scope(), &key); + let id = if revision % 2 == 1 { + base ^ NATIVE_SLOT_BIT + } else { + base + }; + let scene = graphics.clone(); + let start = std::time::Instant::now(); + state.set_scene(scene); + std::hint::black_box(state.encode_output( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )); + let checkpoint = state.clone(); + assert!(state.trust_direct_asset(&key, id)); + std::hint::black_box(state.encode_output( + Visibility::Main, + (0, 0), + None, + cell, + &Occlusion::default(), + )); + drop(checkpoint); + if revision > 5 { + samples.push(start.elapsed().as_nanos()); + } + } + samples.sort_unstable(); + eprintln!( + "graphics replacement panes={count} median_ns={} p95_ns={}", + samples[50], samples[95] + ); } } } @@ -1031,7 +1593,7 @@ mod tests { #[cfg(unix)] #[test] - fn trusted_direct_asset_is_placed_without_inline_reupload() { + fn legacy_direct_asset_is_placed_without_inline_reupload() { let mut state = ClientState::default(); state.set_scope("endpoint-a:boot-1"); let _ = state.encode( @@ -1044,14 +1606,18 @@ mod tests { }, &Occlusion::default(), ); - let image = asset( + let mut image = asset( SurfaceGraphicsTarget::Pane { pane_id: "w1:p1".into(), }, 15, vec![1, 2, 3, 4], ); - let image_id = host_image_id("endpoint-a:boot-1", &image.key); + image.key.source = SurfaceGraphicsSource::PaneLayer { + pane_id: "w1:p1".into(), + layer_id: "primary".into(), + }; + let image_id = state.image_id(&image.key); assert!(state.trust_direct_asset(&image.key, image_id)); let mut direct_scene = scene(image, 0, 0); direct_scene.assets.clear(); @@ -1085,7 +1651,7 @@ mod tests { 18, vec![1, 2, 3, 4], ); - let image_id = host_image_id("endpoint-a:boot-1", &image.key); + let image_id = state.image_id(&image.key); let mut direct_scene = scene(image.clone(), 0, 0); direct_scene.assets.clear(); state.set_scene(direct_scene); @@ -1119,7 +1685,7 @@ mod tests { 19, vec![1, 2, 3, 4], ); - let image_id = host_image_id("endpoint-a:boot-1", &image.key); + let image_id = state.image_id(&image.key); let mut active = scene(image.clone(), 0, 0); active.assets.clear(); state.set_scene(active.clone()); @@ -1235,18 +1801,22 @@ mod tests { #[cfg(unix)] #[test] - fn unclaimed_direct_asset_is_deleted_by_the_next_authoritative_scene() { + fn unclaimed_legacy_direct_asset_is_deleted_by_the_next_authoritative_scene() { let mut state = ClientState::default(); state.set_scope("endpoint-a:boot-1"); let _ = state.take_pending_cleanup(); - let image = asset( + let mut image = asset( SurfaceGraphicsTarget::Pane { pane_id: "w1:p1".into(), }, 16, vec![1, 2, 3, 4], ); - let image_id = host_image_id("endpoint-a:boot-1", &image.key); + image.key.source = SurfaceGraphicsSource::PaneLayer { + pane_id: "w1:p1".into(), + layer_id: "primary".into(), + }; + let image_id = state.image_id(&image.key); assert!(state.trust_direct_asset(&image.key, image_id)); state.set_scene(SurfaceGraphicsScene::default()); diff --git a/src/pane.rs b/src/pane.rs index bfeec638..7d42e9c0 100644 --- a/src/pane.rs +++ b/src/pane.rs @@ -3654,6 +3654,13 @@ impl PaneRuntime { #[cfg(test)] impl PaneRuntime { + #[cfg(unix)] + pub(crate) fn test_enable_kitty_source_forwarding(&self) { + let mut core = self.terminal.ghostty.core.lock().unwrap(); + core.terminal.enable_kitty_graphics().unwrap(); + core.terminal.set_kitty_source_forwarding(true).unwrap(); + } + pub(crate) fn test_with_channel(cols: u16, rows: u16) -> (Self, mpsc::Receiver<Bytes>) { Self::test_with_channel_and_scrollback_bytes(cols, rows, 0, &[], 4) } diff --git a/src/pane/cursor.rs b/src/pane/cursor.rs index 2ba94312..30137730 100644 --- a/src/pane/cursor.rs +++ b/src/pane/cursor.rs @@ -24,9 +24,16 @@ enum DecscusrParseState { } impl DecscusrTracker { - pub(crate) fn observe(&mut self, bytes: &[u8]) { - for &byte in bytes { - self.observe_byte(byte); + pub(crate) fn observe(&mut self, mut bytes: &[u8]) { + while !bytes.is_empty() { + if matches!(self.state, DecscusrParseState::Ground) { + let Some(escape) = bytes.iter().position(|&byte| byte == 0x1b) else { + return; + }; + bytes = &bytes[escape..]; + } + self.observe_byte(bytes[0]); + bytes = &bytes[1..]; } } @@ -256,6 +263,72 @@ fn is_jump(settled: TerminalCursorState, current: TerminalCursorState) -> bool { mod tests { use super::*; + fn assert_decscusr_matches_bytewise(chunks: &[&[u8]]) { + let mut optimized = DecscusrTracker::default(); + let mut bytewise = DecscusrTracker::default(); + for chunk in chunks { + optimized.observe(chunk); + for &byte in *chunk { + bytewise.observe_byte(byte); + } + assert_eq!( + optimized.cursor_shape_overridden(), + bytewise.cursor_shape_overridden(), + "chunks: {chunks:?}" + ); + // Compare the complete parser state, including incomplete CSI parameters. + assert_eq!( + format!("{:?}", optimized.state), + format!("{:?}", bytewise.state), + "chunks: {chunks:?}" + ); + } + } + + #[test] + fn decscusr_bulk_search_matches_bytewise_control_sequences_and_splits() { + let cases: &[&[u8]] = &[ + b"", + b"plain text\n\twithout escapes", + b"text\x1b[1 qmore\x1b[0 qend", + b"\x1b[ q\x1b[2 q\x1b[3 q\x1b[4 q\x1b[5 q\x1b[6 q\x1b[7 q", + b"\x1b\x1b[1 q\x1b[2;9 q\x1b[0:4 q", + b"\x1b[1q\x1b[? q\x1b[12$ q\x1b[1\x00 q\x1b[2\xff q", + b"\x1b[12\x1b[5 q\x1b]0;title\x07\x1bPdata\x1b\\", + b"\x1b[1 q\x1b", + b"\x1b[1 q\x1b[", + b"\x1b[1 q\x1b[0 ", + ]; + for &bytes in cases { + for split in 0..=bytes.len() { + assert_decscusr_matches_bytewise(&[&bytes[..split], &[], &bytes[split..]]); + } + let chunks: Vec<_> = bytes.chunks(1).collect(); + assert_decscusr_matches_bytewise(&chunks); + } + } + + #[test] + fn decscusr_bulk_search_matches_bytewise_all_byte_values() { + let all_bytes: Vec<u8> = (0..=255).collect(); + for split in 0..=all_bytes.len() { + assert_decscusr_matches_bytewise(&[&all_bytes[..split], &all_bytes[split..]]); + } + // Exercise every possible byte in ground, escape, and partial CSI states. + let prefixes: &[&[u8]] = &[b"", b"\x1b", b"\x1b[", b"\x1b[2", b"\x1b[0 ", b"\x1b[2;"]; + for &prefix in prefixes { + for byte in 0..=255u8 { + let mut bytes = b"\x1b[1 q".to_vec(); + bytes.extend_from_slice(prefix); + bytes.push(byte); + bytes.extend_from_slice(b" qtext\x1b[0 q\x1b[6 q"); + for split in 0..=bytes.len() { + assert_decscusr_matches_bytewise(&[&bytes[..split], &bytes[split..]]); + } + } + } + } + fn cursor(x: u16, y: u16, visible: bool, shape: u8) -> TerminalCursorState { TerminalCursorState { x, diff --git a/src/pane/osc.rs b/src/pane/osc.rs index 472fa7aa..85b8e780 100644 --- a/src/pane/osc.rs +++ b/src/pane/osc.rs @@ -63,8 +63,19 @@ fn is_ignored_string_intro(byte: u8) -> bool { impl DefaultColorOscTracker { pub(super) fn observe(&mut self, bytes: &[u8]) -> bool { let mut saw_default_color_set = false; - - for &byte in bytes { + let mut cursor = 0; + while cursor < bytes.len() { + if matches!( + self.state, + DefaultColorOscTrackerState::Ground | DefaultColorOscTrackerState::IgnoreString + ) { + let Some(offset) = bytes[cursor..].iter().position(|&byte| byte == 0x1b) else { + break; + }; + cursor += offset; + } + let byte = bytes[cursor]; + cursor += 1; match self.state { DefaultColorOscTrackerState::Ground => { if byte == 0x1b { @@ -157,7 +168,20 @@ pub(super) struct DefaultColorEventTracker { impl DefaultColorEventTracker { pub(super) fn observe(&mut self, bytes: &[u8]) { - for (index, &byte) in bytes.iter().enumerate() { + let mut cursor = 0; + while cursor < bytes.len() { + if matches!( + self.state, + DefaultColorOscTrackerState::Ground | DefaultColorOscTrackerState::IgnoreString + ) { + let Some(offset) = bytes[cursor..].iter().position(|&byte| byte == 0x1b) else { + break; + }; + cursor += offset; + } + let index = cursor; + let byte = bytes[cursor]; + cursor += 1; match self.state { DefaultColorOscTrackerState::Ground => { if byte == 0x1b { @@ -347,7 +371,19 @@ impl OscStreamCollector { const MAX_BODY_BYTES: usize = 4096; fn observe(&mut self, bytes: &[u8], mut receive: impl FnMut(&[u8])) { - for &byte in bytes { + let mut cursor = 0; + while cursor < bytes.len() { + if matches!( + self.state, + OscStreamState::Ground | OscStreamState::IgnoringString + ) { + let Some(offset) = bytes[cursor..].iter().position(|&byte| byte == 0x1b) else { + break; + }; + cursor += offset; + } + let byte = bytes[cursor]; + cursor += 1; match self.state { OscStreamState::Ground => { if byte == 0x1b { @@ -805,6 +841,59 @@ pub(super) fn restore_host_terminal_theme_if_needed( #[cfg(test)] mod tests { + #[test] + fn bulk_osc_scans_match_bytewise_state_and_response_offsets() { + use super::*; + let mut input = b"text\x1b_Gm=1;".to_vec(); + input.extend(std::iter::repeat_n(b'A', 8192)); + input.extend_from_slice(b"\x1b\\\x1b]10;?\x07\x1b]11;red\x1b\\\x1bPignored"); + input.extend(0u8..=255); + input.extend_from_slice(b"\x1b\\\x1b]12;"); + input.extend(std::iter::repeat_n(b'B', 4200)); + input.extend_from_slice(b"\x07\x1b]10;?\x1b\\\x1b]11;?\x07\x1b"); + for chunk_size in [1, 2, 3, 17, 4096, input.len()] { + let mut bulk = DefaultColorOscTracker::default(); + let mut scalar = DefaultColorOscTracker::default(); + let mut bulk_events = DefaultColorEventTracker::default(); + let mut scalar_events = DefaultColorEventTracker::default(); + let mut bulk_stream = OscStreamCollector::default(); + let mut scalar_stream = OscStreamCollector::default(); + for chunk in input.chunks(chunk_size) { + let changed = bulk.observe(chunk); + let mut scalar_changed = false; + let mut expected_events = Vec::new(); + let mut expected_bodies = Vec::new(); + for (offset, byte) in chunk.iter().enumerate() { + let byte = std::slice::from_ref(byte); + scalar_changed |= scalar.observe(byte); + scalar_events.observe(byte); + expected_events.extend(scalar_events.drain_pending().into_iter().map( + |mut event| { + event.end_offset += offset; + event + }, + )); + scalar_stream.observe(byte, |body| expected_bodies.push(body.to_vec())); + } + bulk_events.observe(chunk); + let mut bodies = Vec::new(); + bulk_stream.observe(chunk, |body| bodies.push(body.to_vec())); + assert_eq!(changed, scalar_changed); + assert_eq!((bulk.state, &bulk.body), (scalar.state, &scalar.body)); + assert_eq!(bulk_events.drain_pending(), expected_events); + assert_eq!( + (bulk_events.state, &bulk_events.body), + (scalar_events.state, &scalar_events.body) + ); + assert_eq!(bodies, expected_bodies); + assert_eq!( + (bulk_stream.state, &bulk_stream.body), + (scalar_stream.state, &scalar_stream.body) + ); + } + } + } + use tokio::sync::mpsc; use super::*; diff --git a/src/pane/xtgettcap.rs b/src/pane/xtgettcap.rs index 4d8fa572..ea6f3a09 100644 --- a/src/pane/xtgettcap.rs +++ b/src/pane/xtgettcap.rs @@ -38,7 +38,21 @@ enum C1XtgettcapTrackerState { impl C1XtgettcapQueryTracker { pub(super) fn observe(&mut self, bytes: &[u8]) { - for (index, &byte) in bytes.iter().enumerate() { + let mut index = 0; + while index < bytes.len() { + if self.state == C1XtgettcapTrackerState::IgnoreString { + // Payload cannot affect this state. CAN/SUB still need the + // global native-DCS bookkeeping when a dispatch is pending. + let next = bytes[index..].iter().position(|&byte| { + matches!(byte, 0x1b | 0x9c) + || (self.native_dcs_pending && matches!(byte, 0x18 | 0x1a)) + }); + match next { + Some(offset) => index += offset, + None => break, + } + } + let byte = bytes[index]; // With a 7-bit intro and raw ST, the native parser still holds the // DCS open. At its eventual unhook it can answer earlier keys in a // multi-key request again. Discard only that dispatch's XTGETTCAP @@ -164,6 +178,7 @@ impl C1XtgettcapQueryTracker { self.body.clear(); self.state = C1XtgettcapTrackerState::OversizedDcs; } + index += 1; } } @@ -239,3 +254,82 @@ fn append_upper_hex(bytes: &[u8], output: &mut Vec<u8>) { output.push(HEX[usize::from(byte & 0x0f)]); } } + +#[cfg(test)] +mod tests { + use super::*; + + fn observe_chunks(chunks: &[&[u8]]) -> (C1XtgettcapQueryTracker, Vec<C1XtgettcapResponse>) { + let mut tracker = C1XtgettcapQueryTracker::default(); + let mut pending = Vec::new(); + let mut offset = 0; + for chunk in chunks { + tracker.observe(chunk); + pending.extend(tracker.drain_pending().into_iter().map(|mut response| { + response.end_offset += offset; + response + })); + offset += chunk.len(); + } + (tracker, pending) + } + + fn assert_chunk_equivalence(bytes: &[u8]) { + let one_byte_chunks: Vec<_> = bytes.chunks(1).collect(); + let (expected, expected_pending) = observe_chunks(&one_byte_chunks); + // Includes a single bulk call and every split, especially ESC / ST. + for split in 0..=bytes.len() { + let (actual, actual_pending) = observe_chunks(&[&bytes[..split], &bytes[split..]]); + assert_eq!(actual_pending, expected_pending, "split {split}: {bytes:?}"); + assert_eq!(actual.state, expected.state); + assert_eq!(actual.raw_c1_intro, expected.raw_c1_intro); + assert_eq!(actual.native_dcs_pending, expected.native_dcs_pending); + assert_eq!(actual.body, expected.body); + assert_eq!(actual.pending, expected.pending); + } + } + + #[test] + fn ignored_string_bulk_matches_single_bytes() { + let mut all_bytes = b"\x1b_Gordinary kitty payload".to_vec(); + all_bytes.extend(0..=255u8); + all_bytes.extend_from_slice(b"\x1b\\\x90+q5463\x9c\x90+q5247"); + assert_chunk_equivalence(&all_bytes); + + // Exercise every byte while actually inside IgnoreString, both with + // and without a native dispatch waiting for ESC, CAN, or SUB. + for native_pending in [false, true] { + for intro in [b"\x1b_".as_slice(), b"\x98", b"\x9e", b"\x9f"] { + for byte in 0..=255u8 { + let mut bytes = Vec::new(); + if native_pending { + bytes.extend_from_slice(b"\x1bP+q5463;524742\x9c"); + } + bytes.extend_from_slice(intro); + bytes.extend_from_slice(b"Gordinary payload"); + bytes.push(byte); + assert_chunk_equivalence(&bytes); + bytes.extend_from_slice(b"more payload\x1b\\\x90+q5463\x9c\x90+q5247"); + assert_chunk_equivalence(&bytes); + } + } + } + } + + #[test] + fn ignored_string_preserves_native_dispatch_offsets() { + for cancel in [0x18, 0x1a, 0x1b] { + let mut bytes = b"\x1bP+q5463\x9c\x9fignored payload".to_vec(); + bytes.push(cancel); + let end_offset = bytes.len(); + bytes.extend_from_slice(b"\x1b\\\x90+q524742\x9c"); + let (_, pending) = observe_chunks(&[&bytes]); + assert_eq!(pending.len(), 3); + assert!(pending[1].suppress_native); + assert!(pending[1].bytes.is_empty()); + assert_eq!(pending[1].end_offset, end_offset); + assert_eq!(pending[2].end_offset, bytes.len()); + assert_chunk_equivalence(&bytes); + } + } +} diff --git a/src/pane_graphics_files.rs b/src/pane_graphics_files.rs index 527be1d5..5c272bbe 100644 --- a/src/pane_graphics_files.rs +++ b/src/pane_graphics_files.rs @@ -1,13 +1,13 @@ #[cfg(unix)] use std::fs::OpenOptions; use std::fs::{self, File}; -use std::io; +use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, Mutex, OnceLock}; #[cfg(unix)] -use std::os::unix::fs::{FileExt, MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::os::unix::fs::{DirBuilderExt, FileExt, MetadataExt, OpenOptionsExt, PermissionsExt}; #[cfg(unix)] const DIRECTORY_MODE: u32 = 0o700; @@ -19,6 +19,7 @@ pub(crate) struct FileStore { base: PathBuf, generation: OnceLock<Arc<Generation>>, next_fingerprint: AtomicU64, + native_budget: Option<Arc<Mutex<NativeBudget>>>, } #[derive(Debug)] @@ -27,13 +28,60 @@ struct Generation { source: PathBuf, } -#[derive(Clone, Debug)] -pub(crate) struct Lease { - inner: Arc<LeaseInner>, +/// Keep this owned snapshot alive until the terminal has consumed its path. +#[derive(Debug)] +pub(crate) struct OwnedExport { + lease: Lease, + _file: ExportFile, + _reservation: Option<Reservation>, +} + +// Also owns cleanup during writing/validation, before an export is published. +#[derive(Debug)] +struct ExportFile { + path: PathBuf, + _generation: Arc<Generation>, +} + +#[derive(Debug, Default)] +struct NativeBudget { + bytes: usize, + objects: usize, +} + +#[derive(Debug)] +struct Reservation { + budget: Arc<Mutex<NativeBudget>>, + len: usize, } +impl Reservation { + fn acquire(budget: &Arc<Mutex<NativeBudget>>, len: usize) -> io::Result<Self> { + let mut state = budget.lock().unwrap_or_else(|err| err.into_inner()); + if state.objects >= 64 || len > 64 * 1024 * 1024 - state.bytes { + return Err(io::Error::other("native snapshot budget exhausted")); + } + state.objects += 1; + state.bytes += len; + Ok(Self { + budget: Arc::clone(budget), + len, + }) + } +} + +impl Drop for Reservation { + fn drop(&mut self) { + let mut state = self.budget.lock().unwrap_or_else(|err| err.into_inner()); + state.objects -= 1; + state.bytes -= self.len; + } +} + +const MAX_EXPORT_BYTES: usize = 16 * 1024 * 1024; + #[derive(Debug)] -struct LeaseInner { +struct Lease { path: PathBuf, file: File, generation: Arc<Generation>, @@ -54,14 +102,53 @@ impl FileStore { base, generation: OnceLock::new(), next_fingerprint: AtomicU64::new(1), + native_budget: None, + } + } + + pub(crate) fn native_sources() -> Self { + let mut store = Self::new(native_base()); + store.native_budget = Some(Arc::new(Mutex::new(NativeBudget::default()))); + store + } + + /// Kernel-only bounded snapshot; any unsupported clone returns an error. + /// The caller keeps the borrowed descriptor alive throughout this call. + pub(crate) fn snapshot(&self, source_fd: i64, expected_len: usize) -> io::Result<OwnedExport> { + if expected_len == 0 || expected_len > MAX_EXPORT_BYTES || !expected_len.is_multiple_of(4) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid native RGBA length", + )); } + let budget = self.native_budget.as_ref().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "not a native source store") + })?; + let reservation = Reservation::acquire(budget, expected_len)?; + let generation = self.generation()?; + let id = self.next_fingerprint.fetch_add(1, Ordering::Relaxed); + let path = generation.source.join(format!("native-{id}.rgba")); + let destination = create_export(&path)?; + let file = ExportFile { + path, + _generation: generation, + }; + crate::platform::clone_native_image_source(source_fd, &destination, expected_len)?; + drop(destination); + let lease = self.lease(&file.path, expected_len)?; + Ok(OwnedExport { + lease, + _file: file, + _reservation: Some(reservation), + }) } + #[cfg(all(test, unix))] pub(crate) fn source_directory(&self) -> io::Result<PathBuf> { Ok(self.generation()?.source.clone()) } - pub(crate) fn lease(&self, path: &Path, expected_len: usize) -> io::Result<Lease> { + fn lease(&self, path: &Path, expected_len: usize) -> io::Result<Lease> { let generation = self.generation()?; validate_child(path, &generation.source)?; let file = open_no_follow(path)?; @@ -70,14 +157,46 @@ impl FileStore { validate_path_identity(path, &metadata)?; let fingerprint = self.next_fingerprint.fetch_add(1, Ordering::Relaxed); Ok(Lease { - inner: Arc::new(LeaseInner { - path: path.to_owned(), - file, - generation, - metadata, - len: expected_len, - fingerprint, - }), + path: path.to_owned(), + file, + generation, + metadata, + len: expected_len, + fingerprint, + }) + } + + /// Snapshot decoded bytes into a private, uniquely named file. Aggregate + /// outstanding-export limits are the caller's responsibility. + pub(crate) fn export(&self, data: &[u8]) -> io::Result<OwnedExport> { + if data.len() > MAX_EXPORT_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "decoded export exceeds 16 MiB", + )); + } + let reservation = self + .native_budget + .as_ref() + .map(|budget| Reservation::acquire(budget, data.len())) + .transpose()?; + let generation = self.generation()?; + // Share the lease identity allocator; create_new also protects against + // collisions with producer files. Never overwrite or remove a collision. + let id = self.next_fingerprint.fetch_add(1, Ordering::Relaxed); + let path = generation.source.join(format!("decoded-{id}.rgba")); + let mut writer = create_export(&path)?; + let file = ExportFile { + path, + _generation: generation, + }; + writer.write_all(data)?; + drop(writer); + let lease = self.lease(&file.path, data.len())?; + Ok(OwnedExport { + lease, + _file: file, + _reservation: reservation, }) } @@ -96,39 +215,90 @@ impl FileStore { } } - #[cfg(test)] + #[cfg(all(test, unix))] pub(crate) fn is_initialized(&self) -> bool { self.generation.get().is_some() } } -impl Lease { +impl PartialEq for OwnedExport { + fn eq(&self, other: &Self) -> bool { + self.path() == other.path() && self.fingerprint() == other.fingerprint() + } +} + +impl Eq for OwnedExport {} + +impl OwnedExport { + pub(crate) fn copy_rgba(&self) -> io::Result<Vec<u8>> { + self.lease.copy_rgba() + } + + pub(crate) fn read_into(&self, data: &mut [u8]) -> io::Result<()> { + self.lease.read_into(data) + } + pub(crate) fn path(&self) -> &Path { - &self.inner.path + self.lease.path() } pub(crate) fn len(&self) -> usize { - self.inner.len + self.lease.len() } pub(crate) fn fingerprint(&self) -> u64 { - self.inner.fingerprint + self.lease.fingerprint() } +} - pub(crate) fn copy_rgba(&self) -> io::Result<Vec<u8>> { - let _keep_generation_alive = &self.inner.generation; - validate_path_identity(&self.inner.path, &self.inner.metadata)?; - let mut data = vec![0; self.inner.len]; - read_exact_at(&self.inner.file, &mut data)?; - if has_byte_at(&self.inner.file, self.inner.len as u64)? { +impl Drop for ExportFile { + fn drop(&mut self) { + if let Err(err) = fs::remove_file(&self.path) { + if err.kind() != io::ErrorKind::NotFound { + tracing::warn!(path = %self.path.display(), err = %err, "failed to remove decoded graphics export"); + } + } + } +} + +impl Lease { + fn path(&self) -> &Path { + &self.path + } + + fn len(&self) -> usize { + self.len + } + + fn fingerprint(&self) -> u64 { + self.fingerprint + } + + fn copy_rgba(&self) -> io::Result<Vec<u8>> { + let mut data = vec![0; self.len]; + self.read_into(&mut data)?; + Ok(data) + } + + fn read_into(&self, data: &mut [u8]) -> io::Result<()> { + if data.len() != self.len { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "incorrect output length", + )); + } + let _keep_generation_alive = &self.generation; + validate_path_identity(&self.path, &self.metadata)?; + read_exact_at(&self.file, data)?; + if has_byte_at(&self.file, self.len as u64)? { return Err(io::Error::new( io::ErrorKind::InvalidData, "frame length changed while leased", )); } - validate_metadata(&self.inner.file.metadata()?, self.inner.len)?; - validate_path_identity(&self.inner.path, &self.inner.metadata)?; - Ok(data) + validate_metadata(&self.file.metadata()?, self.len)?; + validate_path_identity(&self.path, &self.metadata)?; + Ok(()) } } @@ -144,12 +314,17 @@ impl Drop for Generation { #[cfg(unix)] pub(crate) fn validate_direct_source(path: &Path, expected_len: usize) -> io::Result<()> { + validate_source_under(path, expected_len, runtime_base()) +} + +#[cfg(unix)] +fn validate_source_under(path: &Path, expected_len: usize, base: PathBuf) -> io::Result<()> { let source = path.parent().ok_or_else(invalid_path)?; let generation = source.parent().ok_or_else(invalid_path)?; if !path.is_absolute() || path.file_name().is_none() || source.file_name().and_then(|name| name.to_str()) != Some("source") - || generation.parent() != Some(runtime_base().as_path()) + || generation.parent() != Some(base.as_path()) || !generation .file_name() .and_then(|name| name.to_str()) @@ -157,7 +332,7 @@ pub(crate) fn validate_direct_source(path: &Path, expected_len: usize) -> io::Re { return Err(invalid_path()); } - for directory in [runtime_base(), generation.to_owned(), source.to_owned()] { + for directory in [base, generation.to_owned(), source.to_owned()] { validate_directory(&directory)?; } let file = open_no_follow(path)?; @@ -166,6 +341,26 @@ pub(crate) fn validate_direct_source(path: &Path, expected_len: usize) -> io::Re validate_path_identity(path, &metadata) } +/// Unlike the API validator, accepts only the dedicated native hierarchy. +#[cfg(unix)] +pub(crate) fn validate_native_source(path: &Path, expected_len: usize) -> io::Result<()> { + if expected_len == 0 || expected_len > MAX_EXPORT_BYTES || !expected_len.is_multiple_of(4) { + return Err(invalid_path()); + } + validate_source_under(path, expected_len, native_base()) +} + +fn native_base() -> PathBuf { + #[cfg(unix)] + { + PathBuf::from(format!("/var/tmp/herdr-native-sources-{}", effective_uid())) + } + #[cfg(not(unix))] + { + PathBuf::from("native-sources-unavailable") + } +} + fn create_generation(base: &Path) -> io::Result<Generation> { #[cfg(not(unix))] { @@ -177,8 +372,16 @@ fn create_generation(base: &Path) -> io::Result<Generation> { } #[cfg(unix)] { - fs::create_dir_all(base)?; - fs::set_permissions(base, fs::Permissions::from_mode(DIRECTORY_MODE))?; + if base == native_base() { + match fs::DirBuilder::new().mode(DIRECTORY_MODE).create(base) { + Ok(()) => {} + Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {} + Err(err) => return Err(err), + } + } else { + fs::create_dir_all(base)?; + fs::set_permissions(base, fs::Permissions::from_mode(DIRECTORY_MODE))?; + } validate_directory(base)?; remove_stale_generations(base); let nonce = std::time::SystemTime::now() @@ -275,6 +478,23 @@ fn validate_child(path: &Path, directory: &Path) -> io::Result<()> { Ok(()) } +fn create_export(path: &Path) -> io::Result<File> { + #[cfg(unix)] + { + OpenOptions::new() + .write(true) + .create_new(true) + .mode(FILE_MODE) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path) + } + #[cfg(not(unix))] + { + let _ = path; + Err(io::Error::new(io::ErrorKind::Unsupported, "Unix only")) + } +} + fn open_no_follow(path: &Path) -> io::Result<File> { #[cfg(unix)] { @@ -390,6 +610,277 @@ mod tests { } #[test] + fn native_budget_is_per_store_and_released_only_at_final_owner_drop() { + let store = FileStore::native_sources(); + let other = FileStore::native_sources(); + let budget = store.native_budget.as_ref().unwrap(); + let mut holds = Vec::new(); + for _ in 0..64 { + holds.push(Reservation::acquire(budget, 4).unwrap()); + } + assert!(Reservation::acquire(budget, 4).is_err()); + assert!(Reservation::acquire(other.native_budget.as_ref().unwrap(), 4).is_ok()); + holds.clear(); + for _ in 0..4 { + holds.push(Reservation::acquire(budget, MAX_EXPORT_BYTES).unwrap()); + } + assert!(Reservation::acquire(budget, 4).is_err()); + holds.clear(); + // Exercise the exact OwnedExport lifetime independently of FS reflink support. + let export = store.export(&[1, 2, 3, 4]).unwrap(); + let first = Arc::new(export); + let last = Arc::clone(&first); + let path = first.path().to_owned(); + drop(first); + assert_eq!(budget.lock().unwrap().objects, 1); + assert!(path.exists()); + drop(last); + assert_eq!(budget.lock().unwrap().objects, 0); + assert!(!path.exists()); + } + + #[test] + fn native_validation_is_separate_and_private() { + let store = FileStore::native_sources(); + let export = store.export(&[1, 2, 3, 4]).unwrap(); + validate_native_source(export.path(), 4).unwrap(); + assert!(validate_direct_source(export.path(), 4).is_err()); + assert!(validate_native_source(export.path(), 8).is_err()); + let decoded = FileStore::default().export(&[1, 2, 3, 4]).unwrap(); + assert!(validate_native_source(decoded.path(), 4).is_err()); + let link = export.path().with_extension("link"); + symlink(export.path(), &link).unwrap(); + assert!(validate_native_source(&link, 4).is_err()); + fs::remove_file(&link).unwrap(); + fs::hard_link(export.path(), &link).unwrap(); + assert!(validate_native_source(export.path(), 4).is_err()); + fs::remove_file(link).unwrap(); + fs::set_permissions(export.path(), fs::Permissions::from_mode(0o644)).unwrap(); + assert!(validate_native_source(export.path(), 4).is_err()); + } + + #[test] + fn snapshot_validation_releases_budget_and_cleans_staging() { + let store = FileStore::native_sources(); + for len in [0, 3, MAX_EXPORT_BYTES + 4] { + assert!(store.snapshot(-1, len).is_err()); + } + assert!(!store.is_initialized()); + assert!(store.snapshot(-1, 4).is_err()); + assert_eq!( + store + .native_budget + .as_ref() + .unwrap() + .lock() + .unwrap() + .objects, + 0 + ); + assert_eq!( + fs::read_dir(store.source_directory().unwrap()) + .unwrap() + .count(), + 0 + ); + } + + #[test] + fn snapshot_is_independent_and_preserves_borrowed_fd_offset_when_supported() { + use std::io::{Seek, SeekFrom}; + use std::os::fd::AsRawFd; + let store = FileStore::native_sources(); + let path = frame(&store, "producer", &vec![7; 4096]); + let mut producer = OpenOptions::new() + .read(true) + .write(true) + .open(&path) + .unwrap(); + producer.seek(SeekFrom::Start(13)).unwrap(); + assert!(store.snapshot(producer.as_raw_fd() as i64, 4).is_err()); + let result = store.snapshot(producer.as_raw_fd() as i64, 4096); + assert_eq!(producer.stream_position().unwrap(), 13); + let snapshot = match result { + Ok(snapshot) => snapshot, + Err(err) => { + // Unsupported FS/range/cross-device is a normal parent-reader fallback. + assert!( + err.kind() == io::ErrorKind::Unsupported + || matches!( + err.raw_os_error(), + Some( + libc::EOPNOTSUPP + | libc::ENOTTY + | libc::EXDEV + | libc::EINVAL + | libc::ENOSYS + ) + ), + "{err}" + ); + assert_eq!( + fs::read_dir(store.source_directory().unwrap()) + .unwrap() + .count(), + 1 + ); + assert_eq!( + store + .native_budget + .as_ref() + .unwrap() + .lock() + .unwrap() + .objects, + 0 + ); + return; + } + }; + producer.write_at(&vec![9; 4096], 0).unwrap(); + assert_eq!(snapshot.copy_rgba().unwrap(), vec![7; 4096]); + producer.set_len(0).unwrap(); + fs::remove_file(path).unwrap(); + let mut bytes = vec![0; 4096]; + snapshot.read_into(&mut bytes).unwrap(); + assert_eq!(bytes, vec![7; 4096]); + assert!(snapshot.read_into(&mut [0; 4]).is_err()); + assert_eq!(producer.stream_position().unwrap(), 13); + } + + #[test] + fn tmpfs_source_clone_failure_leaves_no_artifact() { + use std::os::fd::AsRawFd; + let path = PathBuf::from(format!( + "/dev/shm/herdr-snapshot-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let Ok(mut producer) = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .mode(FILE_MODE) + .open(&path) + else { + return; + }; + producer.write_all(&[1, 2, 3, 4]).unwrap(); + fs::remove_file(path).unwrap(); + let store = FileStore::native_sources(); + assert!(store.snapshot(producer.as_raw_fd() as i64, 4).is_err()); + assert_eq!( + fs::read_dir(store.source_directory().unwrap()) + .unwrap() + .count(), + 0 + ); + assert_eq!( + store + .native_budget + .as_ref() + .unwrap() + .lock() + .unwrap() + .objects, + 0 + ); + } + + #[test] + fn decoded_exports_snapshot_bytes_and_have_distinct_identities() { + let (store, base) = store(); + let mut data = vec![1, 2, 3, 4]; + let first = store.export(&data).unwrap(); + data.fill(9); + let second = store.export(&data).unwrap(); + assert_eq!(first.len(), 4); + assert_eq!(fs::read(first.path()).unwrap(), [1, 2, 3, 4]); + assert_eq!(first.lease.copy_rgba().unwrap(), [1, 2, 3, 4]); + assert_eq!(second.lease.copy_rgba().unwrap(), data); + assert_ne!(first.path(), second.path()); + assert_ne!(first.fingerprint(), second.fingerprint()); + assert_eq!( + first.path().parent().unwrap(), + store.source_directory().unwrap() + ); + validate_metadata(&fs::metadata(first.path()).unwrap(), 4).unwrap(); + let path = first.path().to_owned(); + drop(first); + assert!(!path.exists()); + assert!(second.path().exists()); + drop(second); + drop(store); + let _ = fs::remove_dir(base); + } + + #[test] + fn decoded_export_guard_retains_generation_until_consumed() { + let (store, base) = store(); + let export = store.export(&[1, 2, 3, 4]).unwrap(); + let path = export.path().to_owned(); + let source = store.source_directory().unwrap(); + drop(store); + assert_eq!(fs::read(&path).unwrap(), [1, 2, 3, 4]); + assert!(source.exists()); + drop(export); + assert!(!path.exists()); + assert!(!source.exists()); + let _ = fs::remove_dir(base); + } + + #[test] + fn decoded_export_rejects_oversize_and_never_overwrites_collisions() { + let (store, base) = store(); + assert!(store.export(&vec![0; MAX_EXPORT_BYTES + 1]).is_err()); + assert!(!store.is_initialized()); + let collision = frame(&store, "decoded-1.rgba", &[7, 8]); + assert!(store.export(&[1, 2, 3, 4]).is_err()); + assert_eq!(fs::read(&collision).unwrap(), [7, 8]); + assert_eq!( + fs::read_dir(store.source_directory().unwrap()) + .unwrap() + .count(), + 1 + ); + let export = store.export(&[1, 2, 3, 4]).unwrap(); + assert_ne!(export.path(), collision); + drop(export); + drop(store); + let _ = fs::remove_dir(base); + } + + #[test] + fn unpublished_export_guard_removes_partial_file_on_error() { + let (store, base) = store(); + let generation = store.generation().unwrap(); + let path = generation.source.join("partial.rgba"); + let result = (|| -> io::Result<()> { + let mut writer = create_export(&path)?; + let _cleanup = ExportFile { + path: path.clone(), + _generation: generation, + }; + writer.write_all(&[1, 2])?; + drop(writer); + store.lease(&path, 4)?; + Ok(()) + })(); + assert!(result.is_err()); + assert!(!path.exists()); + assert!(store.source_directory().unwrap().exists()); + drop(store); + let _ = fs::remove_dir(base); + } + + #[test] + fn decoded_export_passes_direct_source_validation() { + let store = FileStore::default(); + let export = store.export(&[1, 2, 3, 4]).unwrap(); + validate_direct_source(export.path(), export.len()).unwrap(); + } + + #[test] fn source_directory_is_lazy_private_and_removed_with_store() { let (store, base) = store(); assert!(!store.is_initialized()); diff --git a/src/platform/linux.rs b/src/platform/linux.rs index c73584e5..9be40488 100644 --- a/src/platform/linux.rs +++ b/src/platform/linux.rs @@ -1,3 +1,6 @@ +mod native_image_sources; +pub(crate) use native_image_sources::clone_native_image_source; + use std::{ collections::{HashSet, VecDeque}, io::{Read, Write}, diff --git a/src/platform/linux/native_image_sources.rs b/src/platform/linux/native_image_sources.rs new file mode 100644 index 00000000..b4d3b073 --- /dev/null +++ b/src/platform/linux/native_image_sources.rs @@ -0,0 +1,96 @@ +//! Bounded kernel CoW snapshots. No read/write or whole-file clone fallback. +use std::fs::File; +use std::io; +use std::os::fd::AsRawFd; + +#[repr(C)] +struct CloneRange { + src_fd: i64, + src_offset: u64, + src_length: u64, + dest_offset: u64, +} + +fn stat(fd: libc::c_int) -> io::Result<libc::stat> { + let mut result = std::mem::MaybeUninit::<libc::stat>::uninit(); + // SAFETY: fstat initializes the supplied stat on success; fd remains borrowed. + if unsafe { libc::fstat(fd, result.as_mut_ptr()) } != 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful fstat initialized the value. + Ok(unsafe { result.assume_init() }) +} + +fn invalid() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidInput, + "invalid native snapshot source or destination", + ) +} + +pub(crate) fn clone_native_image_source( + source_fd: i64, + destination: &File, + expected_len: usize, +) -> io::Result<()> { + let fd = libc::c_int::try_from(source_fd).map_err(|_| invalid())?; + if fd < 0 + || expected_len == 0 + || expected_len > 16 * 1024 * 1024 + || !expected_len.is_multiple_of(4) + { + return Err(invalid()); + } + let source = stat(fd)?; + let dest = stat(destination.as_raw_fd())?; + // SAFETY: geteuid has no preconditions. + let uid = unsafe { libc::geteuid() }; + let valid_source = |value: &libc::stat| { + value.st_mode & libc::S_IFMT == libc::S_IFREG + && value.st_uid == uid + && value.st_size == expected_len as i64 + && value.st_dev == source.st_dev + && value.st_ino == source.st_ino + }; + if !valid_source(&source) + || dest.st_mode & libc::S_IFMT != libc::S_IFREG + || dest.st_mode & 0o777 != 0o600 + || dest.st_uid != uid + || dest.st_nlink != 1 + || dest.st_size != 0 + || (dest.st_dev == source.st_dev && dest.st_ino == source.st_ino) + { + return Err(invalid()); + } + let range = CloneRange { + src_fd: source_fd, + src_offset: 0, + src_length: expected_len as u64, + dest_offset: 0, + }; + // Explicit nonzero length is essential: FICLONE (or a zero range length) + // clones through EOF and source growth could exceed the reservation before + // post-validation. This request cannot extend the destination past the + // reservation even if the producer changes size concurrently. Filesystems + // requiring block alignment may reject an EOF range after concurrent growth; + // that is a normal fallback, never a reason to try an unbounded clone. + // SAFETY: the request consumes a valid repr(C) range, both descriptors stay + // open throughout the call, and the source descriptor is never owned here. + if unsafe { libc::ioctl(destination.as_raw_fd(), libc::FICLONERANGE, &range) } != 0 { + return Err(io::Error::last_os_error()); + } + let after = stat(destination.as_raw_fd())?; + if !valid_source(&stat(fd)?) + || after.st_dev != dest.st_dev + || after.st_ino != dest.st_ino + || after.st_size != expected_len as i64 + || after.st_nlink != 1 + || after.st_uid != uid + || after.st_mode & 0o777 != 0o600 + { + return Err(invalid()); + } + // The kernel serializes clone against writes. These stats establish exact + // length at validation points, not producer quiescence or a content hash. + Ok(()) +} diff --git a/src/platform/mod.rs b/src/platform/mod.rs index 88f433bf..65717228 100644 --- a/src/platform/mod.rs +++ b/src/platform/mod.rs @@ -328,6 +328,8 @@ mod remote_bridge_tests; #[cfg(unix)] mod unix_common; #[cfg(unix)] +pub(crate) mod unix_image_files; +#[cfg(unix)] pub(crate) use unix_common::{ begin_cli_output, end_cli_output, forward_remote_bridge_stdio, RemoteBridgeWake, }; @@ -725,3 +727,16 @@ pub(crate) fn shared_ssh_control_path( "interactive SSH recovery requires Unix OpenSSH multiplexing", )) } + +/// Kernel CoW snapshots are deliberately unsupported outside Linux. +#[cfg(not(target_os = "linux"))] +pub(crate) fn clone_native_image_source( + _source_fd: i64, + _destination: &std::fs::File, + _expected_len: usize, +) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "native source cloning requires Linux", + )) +} diff --git a/src/platform/unix_image_files.rs b/src/platform/unix_image_files.rs new file mode 100644 index 00000000..795d4b10 --- /dev/null +++ b/src/platform/unix_image_files.rs @@ -0,0 +1,602 @@ +//! Unix-owned Kitty temporary-file ledger and bounded crash recovery. + +use std::fs::{self, DirBuilder, OpenOptions}; +use std::io::{self, Read, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime}; + +const MAX_FILES: usize = 8; +const MAX_BYTES: usize = 64 * 1024 * 1024; +const MAX_IMAGE_BYTES: usize = 16 * 1024 * 1024; +// Bound even abandoned generations: at most 512 MiB per Unix user. When all +// slots are live/recent, a new client uses inline rather than allocating more. +const MAX_GENERATIONS: usize = 8; +const STALE_GRACE: Duration = Duration::from_secs(60); +const GENERATION_PREFIX: &str = "herdr-tty-graphics-protocol-"; +const CONSUMPTION_TIMEOUT: Duration = Duration::from_secs(10); + +struct Pending { + path: PathBuf, + bytes: usize, + published: Instant, +} + +enum ProbeState { + Unsent, + Pending(PathBuf), + Consumed, +} + +pub(crate) struct Ledger { + root: PathBuf, + directory: Option<Generation>, + pending: Vec<Pending>, + next: u64, + disabled: bool, + probe: ProbeState, +} + +impl Ledger { + #[cfg(test)] + pub(crate) fn for_test(root: PathBuf) -> Self { + let mut ledger = Self::new(); + ledger.root = root; + ledger + } + + pub(crate) fn new() -> Self { + Self { + // Do not use TMPDIR/XDG_RUNTIME_DIR: a client override need not + // be in the terminal's accepted temporary-file roots. /tmp is + // explicitly accepted by Ghostty graphics_image.zig in v1.1.3, + // v1.2.3 and main, independent of its TMPDIR. /var/tmp is NOT + // generally accepted, so this bounded ledger must use /tmp. + root: PathBuf::from("/tmp"), + directory: None, + pending: Vec::new(), + next: 0, + disabled: false, + probe: ProbeState::Unsent, + } + } + + pub(crate) fn probe(&mut self) -> Option<PathBuf> { + if self.disabled || !matches!(self.probe, ProbeState::Unsent) { + return None; + } + let path = self.prepare_file_at(&[0, 0, 0, 0], Instant::now())?; + self.probe = ProbeState::Pending(path.clone()); + Some(path) + } + + pub(crate) fn prepare(&mut self, data: &[u8]) -> Option<PathBuf> { + self.prepare_at(data, Instant::now()) + } + + fn prepare_at(&mut self, data: &[u8], now: Instant) -> Option<PathBuf> { + self.reap(now); + if !matches!(self.probe, ProbeState::Consumed) { + return None; + } + self.prepare_file_at(data, now) + } + + fn prepare_file_at(&mut self, data: &[u8], now: Instant) -> Option<PathBuf> { + let bytes: usize = self.pending.iter().map(|entry| entry.bytes).sum(); + if self.disabled + || data.is_empty() + || data.len() > MAX_IMAGE_BYTES + || self.pending.len() >= MAX_FILES + || data.len() > MAX_BYTES - bytes + { + return None; + } + match self.write_file(data) { + Ok(path) => { + self.pending.push(Pending { + path: path.clone(), + bytes: data.len(), + published: now, + }); + Some(path) + } + Err(_) => { + // Do not repeatedly perform failing I/O on every frame. + self.disabled = true; + None + } + } + } + + fn reap(&mut self, now: Instant) { + if let ProbeState::Pending(path) = &self.probe { + if matches!(fs::symlink_metadata(path), Err(error) if error.kind() == io::ErrorKind::NotFound) + { + // With the allowlisted raw query this demonstrates local medium + // consumption, not arbitrary image acceptance or presentation. + self.probe = ProbeState::Consumed; + } + } + self.pending + .retain(|entry| match fs::symlink_metadata(&entry.path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => false, + _ => true, // Permission/I/O errors are NOT evidence of consumption. + }); + if self + .pending + .iter() + .any(|entry| now.saturating_duration_since(entry.published) >= CONSUMPTION_TIMEOUT) + { + self.disabled = true; + } + } + + fn write_file(&mut self, data: &[u8]) -> io::Result<PathBuf> { + if self.directory.is_none() { + self.directory = Some(create_generation(&self.root, SystemTime::now())?); + } + let number = self.next; + self.next = self + .next + .checked_add(1) + .ok_or_else(|| io::Error::other("file sequence exhausted"))?; + let path = self + .directory + .as_ref() + .ok_or_else(|| io::Error::other("missing file generation"))? + .path + .join(format!("tty-graphics-protocol-{number}")); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&path)?; + let result = file + .set_permissions(fs::Permissions::from_mode(0o600)) + .and_then(|()| file.write_all(data)); + // Close before publishing, including on write errors. No fsync is + // needed: the terminal reads the same filesystem/page cache. + drop(file); + if let Err(error) = result { + if fs::remove_file(&path).is_err() { + // Never published, but retain ownership if immediate + // cleanup failed so shutdown can retry the exact path. + self.pending.push(Pending { + path, + bytes: data.len(), + published: Instant::now(), + }); + } + return Err(error); + } + Ok(path) + } + + pub(crate) fn cleanup(&mut self) { + self.disabled = true; + for entry in self.pending.drain(..) { + let _ = fs::remove_file(entry.path); + } + if let Some(directory) = self.directory.take() { + // Never recursively delete: remove only paths we created. + let _ = fs::remove_dir(&directory.path); + } + } +} + +impl Drop for Ledger { + fn drop(&mut self) { + self.cleanup(); + } +} + +/// Keeping this directory descriptor locked prevents recovery by another client. +struct Generation { + path: PathBuf, + _lock: fs::File, +} + +fn effective_uid() -> u32 { + // SAFETY: geteuid takes no pointers and has no preconditions. + unsafe { libc::geteuid() } +} + +fn open_private_directory(path: &Path) -> io::Result<fs::File> { + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(path)?; + let metadata = file.metadata()?; + if metadata.uid() != effective_uid() || metadata.mode() & 0o7777 != 0o700 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "untrusted image directory", + )); + } + Ok(file) +} + +fn lock_directory(file: &fs::File) -> io::Result<()> { + // SAFETY: this live File owns the descriptor; flock neither consumes it nor + // dereferences pointers. Drop releases the advisory lock automatically. + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +fn create_generation(root: &Path, now: SystemTime) -> io::Result<Generation> { + let parent = root.join(format!("herdr-kitty-files-{}", effective_uid())); + match DirBuilder::new().mode(0o700).create(&parent) { + Ok(()) => fs::set_permissions(&parent, fs::Permissions::from_mode(0o700))?, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + // The persistent per-user parent is never removed: replacing its inode + // would split its lock domain. Other users cannot populate or rename it. + let parent_lock = open_private_directory(&parent)?; + lock_directory(&parent_lock)?; + if recover_generations(&parent, now)? >= MAX_GENERATIONS { + return Err(io::Error::other("image generation budget exhausted")); + } + let path = private_directory(&parent)?; + let result = open_private_directory(&path).and_then(|file| { + lock_directory(&file)?; + Ok(file) + }); + match result { + Ok(file) => Ok(Generation { path, _lock: file }), + Err(error) => { + let _ = fs::remove_dir(path); + Err(error) + } + } +} + +/// Caller holds the parent lock. Only unlocked, old generations can be removed; +/// terminal reads belonging to live clients are never invalidated by recovery. +/// Crash recovery is opportunistic at the next client's first file preparation. +fn recover_generations(parent: &Path, now: SystemTime) -> io::Result<usize> { + let entries = fs::read_dir(parent)? + .take(MAX_GENERATIONS + 1) + .collect::<io::Result<Vec<_>>>()?; + if entries.len() > MAX_GENERATIONS { + return Err(io::Error::other("unexpected image generation count")); + } + let mut remaining = entries.len(); + for entry in entries { + let name = entry.file_name(); + let Some(suffix) = name + .to_str() + .and_then(|name| name.strip_prefix(GENERATION_PREFIX)) + else { + continue; + }; + if suffix.len() != 32 || !suffix.bytes().all(|b| b.is_ascii_hexdigit()) { + continue; + } + let path = entry.path(); + let Ok(lock) = open_private_directory(&path) else { + continue; + }; + if lock_directory(&lock).is_err() { + continue; + } + let metadata = lock.metadata()?; + let old = metadata + .modified() + .ok() + .and_then(|modified| now.duration_since(modified).ok()) + .is_some_and(|age| age >= STALE_GRACE); + if old && remove_stale_generation(&path)? { + remaining -= 1; + } + } + Ok(remaining) +} + +/// Inspect the entire bounded set before deleting anything. Never follow links, +/// recurse, or remove unknown content, even inside our own namespace. +fn remove_stale_generation(path: &Path) -> io::Result<bool> { + let entries = fs::read_dir(path)? + .take(MAX_FILES + 1) + .collect::<io::Result<Vec<_>>>()?; + if entries.len() > MAX_FILES { + return Ok(false); + } + for entry in &entries { + let name = entry.file_name(); + let valid_name = name + .to_str() + .and_then(|name| name.strip_prefix("tty-graphics-protocol-")) + .is_some_and(|suffix| !suffix.is_empty() && suffix.bytes().all(|b| b.is_ascii_digit())); + let metadata = fs::symlink_metadata(entry.path())?; + if !valid_name + || !metadata.is_file() + || metadata.uid() != effective_uid() + || metadata.mode() & 0o7777 != 0o600 + || metadata.nlink() != 1 + { + return Ok(false); + } + } + for entry in entries { + fs::remove_file(entry.path())?; + } + fs::remove_dir(path)?; + Ok(true) +} + +fn private_directory(root: &Path) -> io::Result<PathBuf> { + for _ in 0..8 { + let mut random = [0u8; 16]; + fs::File::open("/dev/urandom")?.read_exact(&mut random)?; + let name = u128::from_ne_bytes(random); + let path = root.join(format!("{GENERATION_PREFIX}{name:032x}")); + match DirBuilder::new().mode(0o700).create(&path) { + Ok(()) => { + if let Err(error) = fs::set_permissions(&path, fs::Permissions::from_mode(0o700)) { + let _ = fs::remove_dir(&path); + return Err(error); + } + return Ok(path); + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + } + Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "temporary directory collisions", + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct TestLedger { + ledger: Ledger, + scratch: PathBuf, + } + + impl std::ops::Deref for TestLedger { + type Target = Ledger; + fn deref(&self) -> &Ledger { + &self.ledger + } + } + + impl std::ops::DerefMut for TestLedger { + fn deref_mut(&mut self) -> &mut Ledger { + &mut self.ledger + } + } + + impl Drop for TestLedger { + fn drop(&mut self) { + self.ledger.cleanup(); + // Only this test's uniquely allocated private scratch tree. + let _ = fs::remove_dir_all(&self.scratch); + } + } + + fn test_ledger() -> TestLedger { + let scratch = private_directory(Path::new("/var/tmp")).unwrap(); + let mut ledger = Ledger::new(); + ledger.root = scratch.clone(); + // Most ledger tests start after successful query consumption. + ledger.probe = ProbeState::Consumed; + TestLedger { ledger, scratch } + } + + #[test] + fn private_complete_unique_files_and_shutdown() { + let mut ledger = test_ledger(); + let first = ledger.prepare(b"raw\0pixels").unwrap(); + let second = ledger.prepare(b"other").unwrap(); + let dir = first.parent().unwrap().to_owned(); + assert_ne!(first, second); + assert_eq!(fs::read(&first).unwrap(), b"raw\0pixels"); + assert_eq!( + fs::metadata(&first).unwrap().permissions().mode() & 0o777, + 0o600 + ); + assert_eq!( + fs::metadata(&dir).unwrap().permissions().mode() & 0o777, + 0o700 + ); + assert!(first + .file_name() + .unwrap() + .to_str() + .unwrap() + .contains("tty-graphics-protocol")); + let mut other = test_ledger(); + assert_ne!( + other.prepare(b"other client").unwrap().parent().unwrap(), + dir + ); + drop(ledger); + assert!(!first.exists()); + assert!(!second.exists()); + assert!(!dir.exists()); + } + + #[test] + fn count_cap_and_terminal_consumption() { + let mut ledger = test_ledger(); + for _ in 0..MAX_FILES { + assert!(ledger.prepare(b"x").is_some()); + } + assert!(ledger.prepare(b"inline").is_none()); + let consumed = ledger.pending[0].path.clone(); + fs::remove_file(consumed).unwrap(); + assert!(ledger.prepare(b"replacement").is_some()); + assert_eq!(ledger.pending.len(), MAX_FILES); + } + + #[test] + fn byte_and_image_caps() { + let mut ledger = test_ledger(); + assert!(ledger.prepare(&[]).is_none()); + let mut data = vec![0; MAX_IMAGE_BYTES + 1]; + assert!(ledger.prepare(&data).is_none()); + assert!(ledger.directory.is_none()); + data.pop(); + for _ in 0..MAX_BYTES / MAX_IMAGE_BYTES { + assert!(ledger.prepare(&data).is_some()); + } + assert!(ledger.prepare(b"x").is_none()); + fs::remove_file(&ledger.pending[0].path).unwrap(); + assert!(ledger.prepare(&data).is_some()); + } + + #[test] + fn timeout_disables_without_unlinking_published_files() { + let mut ledger = test_ledger(); + let start = Instant::now(); + let path = ledger.prepare_at(b"pending", start).unwrap(); + assert!(ledger + .prepare_at(b"inline", start + CONSUMPTION_TIMEOUT) + .is_none()); + assert_eq!(fs::read(&path).unwrap(), b"pending"); + fs::remove_file(path).unwrap(); + assert!(ledger + .prepare_at(b"still inline", start + CONSUMPTION_TIMEOUT) + .is_none()); + } + + #[test] + fn local_creation_errors_fall_back_and_never_overwrite() { + let mut ledger = test_ledger(); + let path = ledger.prepare(b"published").unwrap(); + ledger.next = 0; // Force an exclusive-create collision. + assert!(ledger.prepare(b"must not overwrite").is_none()); + assert_eq!(fs::read(path).unwrap(), b"published"); + let mut broken = test_ledger(); + broken.root = ledger.pending[0].path.clone(); // Not a directory. + assert!(broken.prepare(b"inline").is_none()); + assert!(broken.disabled); + } + + #[test] + fn probe_is_once_bounded_and_required_before_real_uploads() { + let mut ledger = test_ledger(); + ledger.probe = ProbeState::Unsent; + assert!(ledger.prepare(b"real").is_none()); + assert!(ledger.directory.is_none()); + let query = ledger.probe().unwrap(); + assert_eq!(fs::read(&query).unwrap(), [0, 0, 0, 0]); + assert_eq!(ledger.pending.len(), 1); + assert_eq!(ledger.pending[0].bytes, 4); + assert!(ledger.probe().is_none()); + assert!(ledger.prepare(b"real").is_none()); + fs::remove_file(query).unwrap(); + assert!(ledger.prepare(b"real").is_some()); + assert!(ledger.probe().is_none()); + } + + #[test] + fn probe_timeout_preserves_file_and_permanently_disables_conversion() { + let mut ledger = test_ledger(); + ledger.probe = ProbeState::Unsent; + let query = ledger.probe().unwrap(); + let late = ledger.pending[0].published + CONSUMPTION_TIMEOUT; + assert!(ledger.prepare_at(b"real", late).is_none()); + assert!(ledger.disabled); + assert_eq!(fs::read(&query).unwrap(), [0, 0, 0, 0]); + fs::remove_file(query).unwrap(); + assert!(ledger.prepare_at(b"real", late).is_none()); + assert!(ledger.probe().is_none()); + } + + fn recover_for_test(parent: &Path, now: SystemTime) -> usize { + let lock = open_private_directory(parent).unwrap(); + lock_directory(&lock).unwrap(); + recover_generations(parent, now).unwrap() + } + + #[test] + fn recovery_preserves_live_and_recent_but_removes_abandoned_generations() { + let mut ledger = test_ledger(); + let image = ledger.prepare(b"live").unwrap(); + let generation_path = image.parent().unwrap().to_owned(); + let parent = generation_path.parent().unwrap(); + let later = SystemTime::now() + STALE_GRACE + Duration::from_secs(1); + assert_eq!(recover_for_test(parent, later), 1); + assert!(image.exists()); // Advisory lock protects live clients even if old. + let generation = ledger.directory.take().unwrap(); + ledger.pending.clear(); // Simulate process death without Drop cleanup. + drop(generation); + assert_eq!(recover_for_test(parent, SystemTime::now()), 1); + assert!(image.exists()); // Grace protects terminal work after a crash. + assert_eq!(recover_for_test(parent, later), 0); + assert!(!generation_path.exists()); + } + + #[test] + fn generation_cap_bounds_crash_leftovers_and_parent_lock_is_nonblocking() { + let fixture = test_ledger(); + let root = &fixture.scratch; + let now = SystemTime::now(); + let mut generations = Vec::new(); + for _ in 0..MAX_GENERATIONS { + generations.push(create_generation(root, now).unwrap()); + } + assert!(create_generation(root, now).is_err()); + let parent = generations[0].path.parent().unwrap().to_owned(); + let lock = open_private_directory(&parent).unwrap(); + lock_directory(&lock).unwrap(); + assert!(create_generation(root, now).is_err()); + drop(lock); + drop(generations); // Empty abandoned generations, as after an early crash. + assert!(create_generation(root, now).is_err()); + let recovered = + create_generation(root, now + STALE_GRACE + Duration::from_secs(1)).unwrap(); + assert_eq!(fs::read_dir(parent).unwrap().count(), 1); + drop(recovered); + } + + #[test] + fn recovery_rejects_symlinks_and_unknown_content() { + use std::os::unix::fs::symlink; + let mut ledger = test_ledger(); + let image = ledger.prepare(b"keep").unwrap(); + let generation_path = image.parent().unwrap().to_owned(); + let parent = generation_path.parent().unwrap(); + fs::remove_file(&image).unwrap(); + let outside = ledger.scratch.join("outside"); + fs::write(&outside, b"outside").unwrap(); + symlink(&outside, &image).unwrap(); + ledger.pending.clear(); + drop(ledger.directory.take()); + let later = SystemTime::now() + STALE_GRACE + Duration::from_secs(1); + assert_eq!(recover_for_test(parent, later), 1); + assert_eq!(fs::read(&outside).unwrap(), b"outside"); + fs::remove_file(&image).unwrap(); + fs::write(generation_path.join("unknown"), b"keep").unwrap(); + assert_eq!(recover_for_test(parent, later), 1); + assert!(generation_path.exists()); + } + + #[test] + fn existing_parent_must_be_owned_private_directory_not_symlink() { + use std::os::unix::fs::symlink; + let fixture = test_ledger(); + let parent = fixture + .scratch + .join(format!("herdr-kitty-files-{}", effective_uid())); + let destination = fixture.scratch.join("destination"); + fs::create_dir(&destination).unwrap(); + symlink(&destination, &parent).unwrap(); + assert!(create_generation(&fixture.scratch, SystemTime::now()).is_err()); + fs::remove_file(&parent).unwrap(); + fs::create_dir(&parent).unwrap(); + fs::set_permissions(&parent, fs::Permissions::from_mode(0o755)).unwrap(); + assert!(create_generation(&fixture.scratch, SystemTime::now()).is_err()); + } +} diff --git a/src/protocol/wire.rs b/src/protocol/wire.rs index aba166d6..e7fb2743 100644 --- a/src/protocol/wire.rs +++ b/src/protocol/wire.rs @@ -1190,9 +1190,55 @@ pub struct SurfaceGraphicsAssetKey { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SurfaceGraphicsAsset { pub key: SurfaceGraphicsAssetKey, + #[serde( + serialize_with = "serialize_graphics_bytes", + deserialize_with = "deserialize_graphics_bytes" + )] pub data: Vec<u8>, } +fn serialize_graphics_bytes<S: serde::Serializer>( + data: &[u8], + serializer: S, +) -> Result<S::Ok, S::Error> { + // Bincode's byte slice has the same length+bytes layout as Vec<u8>, + // but avoids per-byte serialization. Keep human-readable codecs unchanged. + if serializer.is_human_readable() { + data.serialize(serializer) + } else { + serializer.serialize_bytes(data) + } +} + +fn deserialize_graphics_bytes<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result<Vec<u8>, D::Error> { + if deserializer.is_human_readable() { + return Vec::<u8>::deserialize(deserializer); + } + + struct BytesVisitor; + impl<'de> serde::de::Visitor<'de> for BytesVisitor { + type Value = Vec<u8>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("image bytes") + } + + fn visit_bytes<E: serde::de::Error>(self, bytes: &[u8]) -> Result<Self::Value, E> { + Ok(bytes.to_vec()) + } + + fn visit_byte_buf<E: serde::de::Error>(self, bytes: Vec<u8>) -> Result<Self::Value, E> { + Ok(bytes) + } + } + + // The framed slice decoder checks the length against available input before + // handing bytes to the visitor, so a forged length cannot cause an allocation. + deserializer.deserialize_bytes(BytesVisitor) +} + /// One already-clipped desired placement relative to its target surface. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, bincode::Decode)] pub struct SurfaceGraphicsPlacement { @@ -1733,6 +1779,85 @@ pub fn check_client_version(client_version: u32) -> VersionCheck { #[cfg(test)] mod tests { use super::*; + + #[test] + fn graphics_bulk_codec_preserves_legacy_wire_and_json() { + #[derive(Serialize, Deserialize)] + struct LegacyAsset { + key: SurfaceGraphicsAssetKey, + data: Vec<u8>, + } + for len in [0, 1, 250, 251, 65535, 65536, 800 * 480 * 4] { + let asset = SurfaceGraphicsAsset { + key: SurfaceGraphicsAssetKey { + source: SurfaceGraphicsSource::Terminal { + target: SurfaceGraphicsTarget::Pane { + pane_id: "pane-1".into(), + }, + image_id: 7, + }, + image_width: 800, + image_height: 480, + format: SurfaceGraphicsFormat::Rgba, + data_len: len as u64, + data_fingerprint: 42, + }, + data: (0..len).map(|i| (i % 256) as u8).collect(), + }; + let legacy = LegacyAsset { + key: asset.key.clone(), + data: asset.data.clone(), + }; + let config = bincode::config::standard(); + let before = bincode::serde::encode_to_vec(&legacy, config).unwrap(); + let after = bincode::serde::encode_to_vec(&asset, config).unwrap(); + assert_eq!(after, before); + let (decoded, used): (SurfaceGraphicsAsset, _) = + bincode::serde::decode_from_slice(&before, config).unwrap(); + assert_eq!(decoded, asset); + assert_eq!(used, before.len()); + let (decoded, used): (LegacyAsset, _) = + bincode::serde::decode_from_slice(&after, config).unwrap(); + assert_eq!(decoded.data, asset.data); + assert_eq!(decoded.key, asset.key); + assert_eq!(used, after.len()); + let json = serde_json::to_value(&legacy).unwrap(); + assert_eq!(serde_json::to_value(&asset).unwrap(), json); + assert_eq!( + serde_json::from_value::<SurfaceGraphicsAsset>(json).unwrap(), + asset + ); + assert!( + bincode::serde::decode_from_slice::<SurfaceGraphicsAsset, _>( + &before[..before.len() - 1], + config + ) + .is_err() + ); + } + } + #[test] + fn graphics_bulk_decode_rejects_truncated_and_forged_lengths() { + #[derive(Debug, Deserialize)] + struct Bytes(#[serde(deserialize_with = "deserialize_graphics_bytes")] Vec<u8>); + let config = bincode::config::standard(); + let original: Vec<u8> = (0..=255).collect(); + let encoded = bincode::serde::encode_to_vec(&original, config).unwrap(); + for end in 0..encoded.len() { + assert!( + bincode::serde::decode_from_slice::<Bytes, _>(&encoded[..end], config).is_err() + ); + } + let (decoded, consumed): (Bytes, _) = + bincode::serde::decode_from_slice(&encoded, config).unwrap(); + assert_eq!(decoded.0, original); + assert_eq!(consumed, encoded.len()); + for length in [MAX_GRAPHICS_FRAME_SIZE as u64 + 1, u64::MAX] { + let forged = bincode::serde::encode_to_vec(length, config).unwrap(); + assert!(bincode::serde::decode_from_slice::<Bytes, _>(&forged, config).is_err()); + } + } + use ratatui::style::{Color, Modifier}; use sha2::{Digest, Sha256}; diff --git a/src/server/client_shell.rs b/src/server/client_shell.rs index d542896a..d79c89cf 100644 --- a/src/server/client_shell.rs +++ b/src/server/client_shell.rs @@ -274,6 +274,7 @@ pub(super) struct RenderedPaneSurface { pub(super) popup: Option<Box<protocol::ClientShellPopupSurface>>, pub(super) graphics: protocol::SurfaceGraphicsScene, pub(super) graphics_delivery: crate::kitty_graphics::surface::DeliveryCache, + pub(super) graphics_sources: crate::kitty_graphics::surface::SourceFiles, } #[derive(Debug)] @@ -438,16 +439,17 @@ pub(super) fn render_pane_surface( let popup = show_popup .then(|| render_popup_surface(app, area, resize_panes, cell_size)) .flatten(); - let (graphics, next_graphics_delivery) = crate::server::client_shell_graphics::collect( - app, - &layout.pane_infos, - &layout.split_borders, - popup.as_deref(), - target, - cell_size, - graphics_delivery, - client_id, - ); + let (graphics, next_graphics_delivery, graphics_sources) = + crate::server::client_shell_graphics::collect( + app, + &layout.pane_infos, + &layout.split_borders, + popup.as_deref(), + target, + cell_size, + graphics_delivery, + client_id, + ); if let Some(target) = target { for (&pane_id, &(epoch, _)) in &content_revisions_before { if let Some(runtime) = app.state.runtime_for_pane_in_workspace( @@ -488,6 +490,7 @@ pub(super) fn render_pane_surface( popup, graphics, graphics_delivery: next_graphics_delivery, + graphics_sources, }) } diff --git a/src/server/client_shell_graphics.rs b/src/server/client_shell_graphics.rs index 00355230..d0af03a9 100644 --- a/src/server/client_shell_graphics.rs +++ b/src/server/client_shell_graphics.rs @@ -1,4 +1,4 @@ -use crate::kitty_graphics::surface::DeliveryCache; +use crate::kitty_graphics::surface::{DeliveryCache, SourceFiles}; use crate::protocol::{ClientShellPopupSurface, SurfaceGraphicsScene}; pub(crate) fn collect_retained( @@ -8,7 +8,7 @@ pub(crate) fn collect_retained( cell_size: crate::kitty_graphics::HostCellSize, delivered: &DeliveryCache, client_id: u64, -) -> Option<(SurfaceGraphicsScene, DeliveryCache)> { +) -> Option<(SurfaceGraphicsScene, DeliveryCache, SourceFiles)> { let rect = |rect: crate::protocol::SurfaceRect| { ratatui::layout::Rect::new(rect.x, rect.y, rect.width, rect.height) }; @@ -52,7 +52,7 @@ pub(crate) fn collect( cell_size: crate::kitty_graphics::HostCellSize, delivered: &DeliveryCache, client_id: u64, -) -> (SurfaceGraphicsScene, DeliveryCache) { +) -> (SurfaceGraphicsScene, DeliveryCache, SourceFiles) { let popup_content_size = popup.map(|popup| (popup.frame.width, popup.frame.height)); crate::kitty_graphics::surface::collect_scene( app, diff --git a/src/server/client_transport.rs b/src/server/client_transport.rs index 0d297fb9..465408f5 100644 --- a/src/server/client_transport.rs +++ b/src/server/client_transport.rs @@ -132,6 +132,24 @@ impl ClientWriter { self.render.queue.discard_pending_render(); } + #[cfg(all(test, unix))] + pub(crate) fn test_paused() -> Self { + let queue = ClientWriterQueue::new(); + Self { + control: ClientControlWriter::queue(queue.clone()), + render: ClientRenderWriter::queue(queue), + } + } + + #[cfg(all(test, unix))] + pub(crate) fn test_drain(&self) -> Vec<Vec<u8>> { + let mut state = self.render.queue.lock_state(); + let mut frames = state.control.drain(..).collect::<Vec<_>>(); + frames.extend(state.ordered.drain(..)); + frames.extend(state.render.take()); + frames + } + #[cfg(test)] pub(crate) fn test_fill_render(&self, data: Vec<u8>) { self.render.try_send(data).unwrap(); diff --git a/src/server/headless.rs b/src/server/headless.rs index bba59498..fe8a67e8 100644 --- a/src/server/headless.rs +++ b/src/server/headless.rs @@ -76,8 +76,8 @@ mod bootstrap; mod client_views; mod endpoint_requests; mod lifecycle; +mod native_graphics; mod notifications; -mod pane_graphics; mod render; mod retained_surface; mod surface_interest; @@ -132,31 +132,16 @@ enum LoopEvent { RenderRequested, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +/// Presentation work caused by a server event. +/// +/// Keeping this classification separate from event delivery makes the input +/// contract explicit: events may reach a PTY without necessarily repainting. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] enum RenderImpact { - #[default] None, - Graphics, Full, } -impl RenderImpact { - fn merge(&mut self, other: Self) { - *self = (*self).max(other); - } -} - -fn record_render_impact(source: &'static str, impact: RenderImpact) { - let event = match (source, impact) { - ("api_requests", RenderImpact::Graphics) => "graphics_render_cause.api_requests", - ("api_requests", RenderImpact::Full) => "full_render_cause.api_requests", - ("server_events", RenderImpact::Graphics) => "graphics_render_cause.server_events", - ("server_events", RenderImpact::Full) => "full_render_cause.server_events", - _ => return, - }; - crate::render_prof::event(event); -} - // --------------------------------------------------------------------------- // Constants // --------------------------------------------------------------------------- @@ -205,6 +190,7 @@ pub struct HeadlessServer { client_socket_path: PathBuf, client_socket_identity: SocketFileIdentity, clients: HashMap<u64, ClientConnection>, + native_graphics: native_graphics::NativeGraphics, #[cfg(unix)] next_client_id: u64, /// The client currently driving session-wide host presentation and side effects. @@ -350,6 +336,7 @@ impl HeadlessServer { client_socket_path: client_path, client_socket_identity, clients: HashMap::new(), + native_graphics: Default::default(), #[cfg(unix)] next_client_id: 1, foreground_client_id: None, @@ -463,22 +450,7 @@ impl HeadlessServer { } // 3. Drain API requests. - if self.pane_graphics_runtime_active() { - let api_impact = self.drain_api_requests_with_render_impact(); - record_render_impact("api_requests", api_impact); - match api_impact { - RenderImpact::None => {} - RenderImpact::Graphics => { - needs_render = true; - needs_graphics_render = true; - } - RenderImpact::Full => { - needs_render = true; - needs_full_render = true; - needs_graphics_render = false; - } - } - } else if self.drain_api_requests_with_shutdown_check() { + if self.drain_api_requests_with_shutdown_check() { needs_render = true; needs_full_render = true; crate::render_prof::event("full_render_cause.api_requests"); @@ -494,22 +466,7 @@ impl HeadlessServer { self.accept_client_connections()?; // 5. Drain server events from client threads. - if self.pane_graphics_runtime_active() { - let server_impact = self.drain_server_events_with_render_impact(); - record_render_impact("server_events", server_impact); - match server_impact { - RenderImpact::None => {} - RenderImpact::Graphics => { - needs_render = true; - needs_graphics_render = true; - } - RenderImpact::Full => { - needs_render = true; - needs_full_render = true; - needs_graphics_render = false; - } - } - } else if self.drain_server_events() { + if self.drain_server_events() { needs_render = true; needs_full_render = true; crate::render_prof::event("full_render_cause.server_events"); @@ -541,13 +498,10 @@ impl HeadlessServer { crate::render_prof::event("full_render_cause.default_workspace"); } - if self.retain_live_pane_graphics() { - needs_render = true; - needs_graphics_render = true; - } - if self.expire_direct_graphics(now) { + if self.expire_native_graphics(now) { needs_render = true; - needs_graphics_render = true; + needs_full_render = true; + needs_graphics_render = false; } self.drain_client_config_reload_request(); @@ -689,43 +643,13 @@ impl HeadlessServer { } } LoopEvent::Api(msg) => { - if self.pane_graphics_runtime_active() { - let impact = self.handle_api_request_with_render_impact(*msg); - record_render_impact("api_requests", impact); - match impact { - RenderImpact::None => {} - RenderImpact::Graphics => { - needs_render = true; - needs_graphics_render = true; - } - RenderImpact::Full => { - needs_render = true; - needs_full_render = true; - needs_graphics_render = false; - } - } - } else if self.handle_api_request_with_shutdown_check(*msg) { + if self.handle_api_request_with_shutdown_check(*msg) { needs_render = true; needs_full_render = true; } } LoopEvent::ServerEvent(ev) => { - if self.pane_graphics_runtime_active() { - let impact = self.handle_server_event_with_render_impact(ev); - record_render_impact("server_events", impact); - match impact { - RenderImpact::None => {} - RenderImpact::Graphics => { - needs_render = true; - needs_graphics_render = true; - } - RenderImpact::Full => { - needs_render = true; - needs_full_render = true; - needs_graphics_render = false; - } - } - } else if self.handle_server_event(ev) { + if self.handle_server_event_with_render_impact(ev) == RenderImpact::Full { needs_render = true; needs_full_render = true; } @@ -825,15 +749,11 @@ impl HeadlessServer { } fn sync_foreground_client_state(&mut self) { - self.app.direct_graphics_available = self.direct_graphics_available(); self.app.pixel_mouse_available = self.foreground_client_id.is_some_and(|id| { self.clients .get(&id) .is_some_and(|client| client.pixel_mouse) }); - if !self.app.direct_graphics_available { - self.retire_all_direct_graphics(); - } let Some(client_id) = self.foreground_client_id else { self.effective_size = self.headless_size; self.app.state.outer_terminal_focus = None; @@ -966,53 +886,12 @@ impl HeadlessServer { }) } - fn existing_direct_graphics_client(&self) -> Option<u64> { - self.app - .pane_graphics - .slots - .values() - .filter_map(crate::app::pane_graphics::Slot::direct_client) - .filter_map(|client_id| { - let client = self.clients.get(&client_id)?; - self.client_supports_direct_graphics(client_id) - .then_some((client.last_activity, client_id)) - }) - .max() - .map(|(_, client_id)| client_id) - } - - fn direct_graphics_client(&self) -> Option<u64> { - self.existing_direct_graphics_client().or_else(|| { - self.foreground_client_id - .filter(|client_id| self.client_supports_direct_graphics(*client_id)) - }) - } - - fn direct_graphics_client_for_key(&self, key: &crate::app::pane_graphics::Key) -> Option<u64> { - match self - .app - .pane_graphics - .slots - .get(key) - .and_then(crate::app::pane_graphics::Slot::direct_client) - { - Some(client_id) => self - .client_supports_direct_graphics(client_id) - .then_some(client_id), - None => self.direct_graphics_client(), - } - } - - fn direct_graphics_available(&self) -> bool { - self.direct_graphics_client().is_some() - } - fn has_app_client(&self) -> bool { self.app_client_count() > 0 } fn remove_client(&mut self, client_id: u64) -> bool { - self.retire_direct_graphics_for_client(client_id); + self.disconnect_native_graphics(client_id); let disconnected_focus = self .clients .get(&client_id) @@ -1052,7 +931,6 @@ impl HeadlessServer { self.send_shell_focus_target(target, crate::ghostty::FocusEvent::Lost); } } - self.app.direct_graphics_available = self.direct_graphics_available(); if was_foreground { self.promote_latest_remaining_client() } else { @@ -1131,31 +1009,17 @@ impl HeadlessServer { } /// Drains server events from the dedicated channel. - /// - /// Uses the original full-render semantics when pane graphics are dormant. fn drain_server_events(&mut self) -> bool { let mut changed = false; while !self.should_quit.load(Ordering::Acquire) { let Ok(ev) = self.server_event_rx.try_recv() else { break; }; - changed |= self.handle_server_event(ev); + changed |= self.handle_server_event_with_render_impact(ev) == RenderImpact::Full; } changed } - /// Returns the strongest render impact from the drained event batch. - fn drain_server_events_with_render_impact(&mut self) -> RenderImpact { - let mut impact = RenderImpact::None; - while !self.should_quit.load(Ordering::Acquire) { - let Ok(ev) = self.server_event_rx.try_recv() else { - break; - }; - impact.merge(self.handle_server_event_with_render_impact(ev)); - } - impact - } - async fn reject_late_client_connections(&mut self) { self.server_event_rx.close(); while let Some(event) = self.server_event_rx.recv().await { @@ -2112,12 +1976,17 @@ impl HeadlessServer { transfer_id, image_id, success, - } => self.complete_direct_graphics(client_id, transfer_id, image_id, success), + } => self + .native_result(client_id, transfer_id, image_id, success) + .unwrap_or(false), ServerEvent::GraphicsTransmissionStarted { client_id, transfer_id, image_id, - } => self.start_direct_graphics_response(client_id, transfer_id, image_id), + } => { + self.native_started(client_id, transfer_id, image_id); + false + } ServerEvent::ClientAttachTerminal { client_id, terminal_id, @@ -2937,35 +2806,6 @@ impl HeadlessServer { } } - fn drain_api_requests_with_render_impact(&mut self) -> RenderImpact { - let mut impact = RenderImpact::None; - while !self.should_quit.load(Ordering::Acquire) { - let Ok(msg) = self.app.api_rx.try_recv() else { - break; - }; - impact.merge(self.handle_api_request_with_render_impact(msg)); - } - impact - } - - fn handle_api_request_with_render_impact( - &mut self, - msg: api::ApiRequestMessage, - ) -> RenderImpact { - if matches!( - &msg.request.method, - api::schema::Method::PaneGraphicsStreamSet(_) - | api::schema::Method::PaneGraphicsStreamDirect(_) - ) { - return self.handle_pane_graphics_stream_frame(msg); - } - if self.handle_api_request_with_shutdown_check(msg) { - RenderImpact::Full - } else { - RenderImpact::None - } - } - fn handle_api_request_with_shutdown_check_inner( &mut self, msg: api::ApiRequestMessage, @@ -2999,12 +2839,6 @@ impl HeadlessServer { }; let metadata_expired = self.app.expire_due_metadata(Instant::now()); - let stream_open = match &msg.request.method { - api::schema::Method::PaneGraphicsStreamOpen(params) => Some(params.clone()), - _ => None, - }; - let stream_active = msg.stream_active.clone(); - if let api::schema::Method::ServerLiveHandoff(params) = &msg.request.method { let handoff_result = self.perform_live_handoff(params.clone()); let handoff_succeeded = handoff_result.is_ok(); @@ -3054,16 +2888,7 @@ impl HeadlessServer { _ => {} } - let pane_graphics_revision_before = matches!( - &msg.request.method, - api::schema::Method::PaneGraphicsSet(_) - | api::schema::Method::PaneGraphicsClear(_) - | api::schema::Method::PaneGraphicsStreamOpen(_) - | api::schema::Method::PaneGraphicsStreamClose(_) - ) - .then_some(self.app.pane_graphics.revision()); - let mut changed = metadata_expired - | (pane_graphics_revision_before.is_none() && api::request_changes_ui(&msg.request)); + let mut changed = metadata_expired | api::request_changes_ui(&msg.request); let skip_default_workspace = skip_default_workspace_for_request || matches!( &msg.request.method, @@ -3184,10 +3009,6 @@ impl HeadlessServer { } } } - if let (Some(params), Some(active)) = (stream_open.as_ref(), stream_active) { - self.app - .attach_pane_graphics_stream_active(params, active, &response); - } if let Some(spec) = alt_screen_read_spec { if let Ok(success) = serde_json::from_str::<api::schema::SuccessResponse>(&response) { if let api::schema::ResponseResult::PaneRead { read } = success.result { @@ -3210,10 +3031,6 @@ impl HeadlessServer { } let _ = msg.respond_to.send(response); - if let Some(revision_before) = pane_graphics_revision_before { - changed |= revision_before != self.app.pane_graphics.revision(); - } - // Forward new toast state only when a client-local delivery mode is selected. // Herdr delivery renders the toast in-frame and must not ask clients to // show a terminal or system notification. diff --git a/src/server/headless/endpoint_requests.rs b/src/server/headless/endpoint_requests.rs index afdcc80a..6d2dd6ba 100644 --- a/src/server/headless/endpoint_requests.rs +++ b/src/server/headless/endpoint_requests.rs @@ -127,7 +127,6 @@ impl HeadlessServer { request: *request, respond_to, response_write_complete: None, - stream_active: None, }, ) } diff --git a/src/server/headless/native_graphics.rs b/src/server/headless/native_graphics.rs new file mode 100644 index 00000000..67417d38 --- /dev/null +++ b/src/server/headless/native_graphics.rs @@ -0,0 +1,599 @@ +//! Decoded native exports. Deliberately independent of API slots/leases. +use super::HeadlessServer; +use crate::kitty_graphics::surface::{DeliveryCache, SourceFiles, NATIVE_SLOT_BIT}; +use crate::pane_graphics_files::{FileStore, OwnedExport}; +use crate::protocol::{ + ServerMessage, SurfaceGraphicsAsset, SurfaceGraphicsAssetKey, SurfaceGraphicsFormat, + SurfaceGraphicsScene, SurfaceGraphicsSource, +}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::kitty_graphics::surface::NATIVE_TRANSFER_BIT as NATIVE_BIT; +const MAX_FILE: usize = 16 * 1024 * 1024; +const MAX_BYTES: usize = 64 * 1024 * 1024; +const DELIVERY_TIMEOUT: Duration = Duration::from_secs(5); +const RESPONSE_TIMEOUT: Duration = Duration::from_secs(3); + +pub(super) struct Pending { + export: Arc<OwnedExport>, + transfer_id: u64, + image_id: u32, + source: SurfaceGraphicsSource, + asset: SurfaceGraphicsAssetKey, + slot: bool, + deadline: Instant, + written: bool, + refresh_needed: bool, + scene: SurfaceGraphicsScene, + delivery: DeliveryCache, +} + +struct AcknowledgedSlot { + asset: SurfaceGraphicsAssetKey, + slot: bool, +} + +impl Pending { + pub(super) fn defer_inline_delivery(&mut self, delivery: &DeliveryCache) { + self.delivery = delivery.clone(); + self.refresh_needed = true; + } +} + +pub(super) struct NativeGraphics { + store: FileStore, + pending: HashMap<u64, Pending>, + disabled: HashSet<u64>, + next_transfer: u64, + source_retries: HashMap<u64, Instant>, + // The key and bank currently addressed for each logical source. Native state + // advances on ACK; inline state and source pruning advance on queued scenes. + acknowledged_slots: HashMap<u64, HashMap<SurfaceGraphicsSource, AcknowledgedSlot>>, +} +impl Default for NativeGraphics { + fn default() -> Self { + Self { + store: FileStore::default(), + pending: HashMap::new(), + disabled: HashSet::new(), + next_transfer: 1, + source_retries: HashMap::new(), + acknowledged_slots: HashMap::new(), + } + } +} +impl NativeGraphics { + #[cfg(all(test, unix))] + pub(super) fn busy(&self) -> bool { + !self.pending.is_empty() + } + pub(super) fn is_pending(&self, client: u64) -> bool { + self.pending.contains_key(&client) + } + pub(super) fn can_hold(&self, client: u64, scene: &SurfaceGraphicsScene) -> bool { + let Some(pending) = self.pending.get(&client) else { + return true; + }; + let same_image = |a: &crate::protocol::SurfaceGraphicsAssetKey, + b: &crate::protocol::SurfaceGraphicsAssetKey| { + a.source == b.source + && a.image_width == b.image_width + && a.image_height == b.image_height + && a.format == b.format + && a.data_len == b.data_len + }; + scene.placements.len() == pending.scene.placements.len() + && scene + .placements + .iter() + .zip(&pending.scene.placements) + .all(|(current, held)| { + if !same_image(¤t.asset, &held.asset) { + return false; + } + let mut geometry = current.clone(); + geometry.asset = held.asset.clone(); + geometry == *held + }) + && scene.retained_assets.len() == pending.scene.retained_assets.len() + && scene + .retained_assets + .iter() + .zip(&pending.scene.retained_assets) + .all(|(current, held)| same_image(current, held)) + } + + pub(super) fn hold(&self, client: u64) -> Option<(SurfaceGraphicsScene, DeliveryCache)> { + self.pending + .get(&client) + .map(|p| (p.scene.clone(), p.delivery.clone())) + } + /// Records bank state only after the scene metadata has been queued. + /// + /// `inline_assets` contains only payloads actually queued on the wire; a + /// selected native upload has already been removed and changes bank only + /// after its ACK. A new inline key addresses bank 0, while replaying the + /// exact native key keeps the client's existing key-to-bank mapping. + /// Pruning is committed here as well: a scene rejected by the writer must + /// not mutate bank history for pixels the client is still displaying. + pub(super) fn commit_scene( + &mut self, + client: u64, + scene: &SurfaceGraphicsScene, + inline_assets: &[SurfaceGraphicsAssetKey], + ) { + let present_sources: HashSet<_> = scene + .placements + .iter() + .map(|placement| &placement.asset.source) + .chain(scene.retained_assets.iter().map(|asset| &asset.source)) + .cloned() + .collect(); + let slots = self.acknowledged_slots.entry(client).or_default(); + slots.retain(|source, _| present_sources.contains(source)); + for asset in inline_assets + .iter() + .filter(|asset| matches!(asset.source, SurfaceGraphicsSource::Terminal { .. })) + { + let same_resident_key = slots + .get(&asset.source) + .is_some_and(|resident| resident.asset == *asset); + if !same_resident_key { + slots.insert( + asset.source.clone(), + AcknowledgedSlot { + asset: asset.clone(), + slot: false, + }, + ); + } + } + if slots.is_empty() { + self.acknowledged_slots.remove(&client); + } + } + #[cfg(all(test, unix))] + pub(super) fn prepare( + &mut self, + client: u64, + scope: &str, + scene: &mut SurfaceGraphicsScene, + delivery: &DeliveryCache, + ) -> Option<(Pending, ServerMessage)> { + self.prepare_with_sources(client, scope, scene, delivery, &mut SourceFiles::default()) + } + + pub(super) fn prepare_with_sources( + &mut self, + client: u64, + scope: &str, + scene: &mut SurfaceGraphicsScene, + delivery: &DeliveryCache, + sources: &mut SourceFiles, + ) -> Option<(Pending, ServerMessage)> { + if self.disabled.contains(&client) + || self.pending.contains_key(&client) + || self.pending.len() >= 8 + { + return None; + } + let index = scene + .assets + .iter() + .position(|asset| eligible(asset, sources))?; + let asset = &scene.assets[index]; + if self.pending.values().map(|p| p.export.len()).sum::<usize>() + + asset.key.data_len as usize + > MAX_BYTES + || self.next_transfer >= NATIVE_BIT + { + return None; + } + let export = if let Some(source) = sources.get(&asset.key) { + Arc::clone(source) + } else { + Arc::new(self.store.export(&asset.data).ok()?) + }; + let transfer_id = NATIVE_BIT | self.next_transfer; + self.next_transfer += 1; + let asset_key = asset.key.clone(); + let source = asset_key.source.clone(); + // Start in bank 1: an older client may still be displaying an inline image + // under the unchanged logical base ID. Thereafter always stage into the + // bank opposite the one this client currently addresses. + let slot = !self + .acknowledged_slots + .get(&client) + .and_then(|slots| slots.get(&source)) + .map(|resident| resident.slot) + .unwrap_or(false); + let base = crate::kitty_graphics::surface::native_host_image_id(scope, &asset.key); + let image_id = base ^ if slot { NATIVE_SLOT_BIT } else { 0 }; + let message = ServerMessage::GraphicsFile { + path: export.path().to_string_lossy().into_owned(), + expected_len: export.len() as u64, + image_id, + transfer_id, + leading: Vec::new(), + control: format!( + "a=t,f=32,s={},v={},i={image_id},q=0", + asset.key.image_width, asset.key.image_height + ), + surface_asset: Some(asset_key.clone()), + }; + sources.remove(&asset_key); + scene.assets.remove(index); + let mut held = scene.clone(); + held.assets.clear(); + Some(( + Pending { + export, + transfer_id, + image_id, + source, + asset: asset_key, + slot, + deadline: Instant::now() + DELIVERY_TIMEOUT, + written: false, + refresh_needed: delivery.has_pending(), + scene: held, + delivery: delivery.clone(), + }, + message, + )) + } + pub(super) fn commit(&mut self, client: u64, pending: Pending) { + self.pending.insert(client, pending); + } + fn matches(&self, client: u64, transfer: u64, image: u32) -> bool { + self.pending + .get(&client) + .is_some_and(|p| p.transfer_id == transfer && p.image_id == image) + } +} +fn materialize_sources( + scene: &mut SurfaceGraphicsScene, + delivery: &mut DeliveryCache, + sources: &mut SourceFiles, +) -> bool { + let mut failed = false; + scene.assets.retain_mut(|asset| { + let Some(source) = sources.remove(&asset.key) else { + return true; + }; + match source.copy_rgba() { + Ok(data) => { + asset.data = data; + true + } + Err(error) => { + tracing::debug!(%error, "native source fallback could not be read"); + failed = true; + delivery.forget_asset(&asset.key); + false + } + } + }); + sources.clear(); + failed +} + +fn eligible(asset: &SurfaceGraphicsAsset, sources: &SourceFiles) -> bool { + matches!(asset.key.source, SurfaceGraphicsSource::Terminal { .. }) + && asset.key.format == SurfaceGraphicsFormat::Rgba + && asset.key.data_len > 0 + && asset.key.data_len <= MAX_FILE as u64 + && sources.get(&asset.key).map_or_else( + || asset.key.data_len == asset.data.len() as u64, + |source| asset.key.data_len == source.len() as u64, + ) + && u64::from(asset.key.image_width) + .checked_mul(u64::from(asset.key.image_height)) + .and_then(|pixels| pixels.checked_mul(4)) + == Some(asset.key.data_len) +} +impl HeadlessServer { + pub(super) fn defer_changed_native_geometry( + &mut self, + client: u64, + scene: &SurfaceGraphicsScene, + ) -> bool { + if self.native_graphics.can_hold(client, scene) { + return false; + } + self.retire_native_graphics_for_client(client); + self.app.render_dirty.request_generic(); + if let Some(client) = self.clients.get_mut(&client) { + client.defer_full_render(); + } + true + } + + fn materialize_native_sources( + &mut self, + client: u64, + scene: &mut SurfaceGraphicsScene, + delivery: &mut DeliveryCache, + sources: &mut SourceFiles, + ) { + if materialize_sources(scene, delivery, sources) { + self.native_graphics + .source_retries + .entry(client) + .or_insert_with(|| Instant::now() + std::time::Duration::from_secs(1)); + } else { + self.native_graphics.source_retries.remove(&client); + } + } + + pub(super) fn prepare_native_scene( + &mut self, + client: u64, + scene: &mut SurfaceGraphicsScene, + delivery: &mut DeliveryCache, + sources: &mut SourceFiles, + ) -> Option<(Pending, ServerMessage)> { + if let Some(pending) = self.native_graphics.pending.get_mut(&client) { + pending.refresh_needed |= !scene.assets.is_empty() + || scene.placements != pending.scene.placements + || scene.retained_assets != pending.scene.retained_assets; + } + if let Some((held, held_delivery)) = self.native_graphics.hold(client) { + *scene = held; + *delivery = held_delivery; + sources.clear(); + return None; + } + if !self.client_supports_direct_graphics(client) { + self.materialize_native_sources(client, scene, delivery, sources); + return None; + } + let mut prepared = self.native_graphics.prepare_with_sources( + client, + &self.client_shell_boot_id, + scene, + delivery, + sources, + ); + self.materialize_native_sources(client, scene, delivery, sources); + if let Some((pending, _)) = &mut prepared { + pending.delivery = delivery.clone(); + pending.refresh_needed |= delivery.has_pending(); + } + prepared + } + + pub(super) fn native_started(&mut self, client: u64, transfer: u64, image: u32) -> bool { + if transfer & NATIVE_BIT == 0 { + return false; + } + if !self.native_graphics.disabled.contains(&client) + && self.native_graphics.matches(client, transfer, image) + { + let p = self + .native_graphics + .pending + .get_mut(&client) + .expect("matched"); + p.written = true; + p.deadline = Instant::now() + RESPONSE_TIMEOUT; + } + true + } + pub(super) fn native_result( + &mut self, + client: u64, + transfer: u64, + image: u32, + success: bool, + ) -> Option<bool> { + if transfer & NATIVE_BIT == 0 { + return None; + } + if !self.native_graphics.matches(client, transfer, image) { + return Some(false); + } + if self.native_graphics.disabled.contains(&client) { + return Some(self.expire_native_graphics(Instant::now())); + } + if success { + if !self.native_graphics.pending[&client].written { + return Some(false); + } + let pending = self + .native_graphics + .pending + .remove(&client) + .expect("matched"); + self.native_graphics + .acknowledged_slots + .entry(client) + .or_default() + .insert( + pending.source, + AcknowledgedSlot { + asset: pending.asset, + slot: pending.slot, + }, + ); + let refresh = pending.refresh_needed; + if refresh { + if let Some(c) = self.clients.get_mut(&client) { + c.defer_full_render(); + } + } + return Some(refresh); + } + self.native_graphics.disabled.insert(client); + self.native_graphics + .pending + .get_mut(&client) + .expect("matched") + .deadline = Instant::now(); + Some(self.expire_native_graphics(Instant::now())) + } + pub(super) fn expire_native_graphics(&mut self, now: Instant) -> bool { + let expired: Vec<_> = self + .native_graphics + .pending + .iter() + .filter(|(_, p)| p.deadline <= now) + .map(|(id, p)| (*id, p.transfer_id, p.image_id)) + .collect(); + let mut changed = false; + self.native_graphics.source_retries.retain(|id, deadline| { + if *deadline > now { + return true; + } + if let Some(client) = self.clients.get_mut(id) { + client.defer_full_render(); + changed = true; + } + false + }); + for (id, transfer_id, image_id) in expired { + self.native_graphics.disabled.insert(id); + let message = ServerMessage::GraphicsTransmissionRetired { + transfer_id, + image_id, + }; + // Control messages precede queued renders. The client permanently rejects + // retired native transfers, including a file still in the render queue. + let sent = self.send_to_client(id, message); + if !sent { + continue; + } // Keep the export alive until retirement is queued or disconnect cleans it. + self.native_graphics.pending.remove(&id); + self.native_graphics.disabled.insert(id); + if let Some(c) = self.clients.get_mut(&id) { + c.shell_graphics_delivery = DeliveryCache::default(); + c.defer_full_render(); + } + changed = true; + } + changed + } + pub(super) fn retire_native_graphics_for_client(&mut self, id: u64) { + if let Some(p) = self.native_graphics.pending.get_mut(&id) { + p.deadline = Instant::now(); + self.native_graphics.disabled.insert(id); + if self.expire_native_graphics(Instant::now()) { + self.app.render_dirty.request_generic(); + } + } + } + pub(super) fn disconnect_native_graphics(&mut self, id: u64) { + self.native_graphics.pending.remove(&id); + self.native_graphics.disabled.remove(&id); + self.native_graphics.source_retries.remove(&id); + self.native_graphics.acknowledged_slots.remove(&id); + } +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use crate::protocol::{SurfaceGraphicsAssetKey, SurfaceGraphicsTarget}; + + fn asset() -> SurfaceGraphicsAsset { + SurfaceGraphicsAsset { + key: SurfaceGraphicsAssetKey { + source: SurfaceGraphicsSource::Terminal { + target: SurfaceGraphicsTarget::Pane { + pane_id: "native-test".into(), + }, + image_id: 1, + }, + image_width: 1, + image_height: 1, + format: SurfaceGraphicsFormat::Rgba, + data_len: 4, + data_fingerprint: 123, + }, + data: vec![1, 2, 3, 255], + } + } + + #[test] + fn eligibility_is_decoded_rgba_terminal_only() { + let mut a = asset(); + assert!(eligible(&a, &SourceFiles::default())); + a.key.format = SurfaceGraphicsFormat::Png; + assert!(!eligible(&a, &SourceFiles::default())); + a.key.format = SurfaceGraphicsFormat::Rgba; + a.key.data_len = 5; + assert!(!eligible(&a, &SourceFiles::default())); + a.key.data_len = 4; + a.key.source = SurfaceGraphicsSource::PaneLayer { + pane_id: "p".into(), + layer_id: "l".into(), + }; + assert!(!eligible(&a, &SourceFiles::default())); + let mut a = asset(); + a.data.resize(MAX_FILE + 1, 0); + a.key.data_len = a.data.len() as u64; + assert!(!eligible(&a, &SourceFiles::default())); + } + + #[test] + fn pending_guard_and_client_identity_are_independent() { + let mut state = NativeGraphics::default(); + let mut scene = SurfaceGraphicsScene { + assets: vec![asset()], + ..Default::default() + }; + let (pending, _) = state + .prepare(7, "scope", &mut scene, &DeliveryCache::default()) + .unwrap(); + let path = pending.export.path().to_owned(); + let transfer = pending.transfer_id; + let image = pending.image_id; + assert_ne!(transfer & NATIVE_BIT, 0); + assert!(scene.assets.is_empty()); + assert!(!state.busy()); // prepare is not a delivery commit + state.commit(7, pending); + assert!(state.matches(7, transfer, image)); + assert!(!state.matches(8, transfer, image)); + assert!(!state.matches(7, transfer + 1, image)); + assert!(!state.matches(7, transfer, image + 1)); + assert!(path.exists()); + assert!(state.hold(7).unwrap().0.assets.is_empty()); + state.pending.remove(&7); + assert!(!path.exists()); + state.disabled.insert(7); + scene.assets.push(asset()); + assert!(state + .prepare(7, "scope", &mut scene, &DeliveryCache::default()) + .is_none()); + assert_eq!(scene.assets.len(), 1); // native fallback keeps authoritative inline bytes + } + + #[test] + fn one_pending_per_client_and_global_count_limit() { + let mut state = NativeGraphics::default(); + for id in 0..8 { + let mut scene = SurfaceGraphicsScene { + assets: vec![asset()], + ..Default::default() + }; + let (pending, _) = state + .prepare(id, "scope", &mut scene, &DeliveryCache::default()) + .unwrap(); + state.commit(id, pending); + scene.assets.push(asset()); + assert!(state + .prepare(id, "scope", &mut scene, &DeliveryCache::default()) + .is_none()); + } + let mut scene = SurfaceGraphicsScene { + assets: vec![asset()], + ..Default::default() + }; + assert!(state + .prepare(9, "scope", &mut scene, &DeliveryCache::default()) + .is_none()); + assert_eq!(scene.assets.len(), 1); + } +} diff --git a/src/server/headless/pane_graphics.rs b/src/server/headless/pane_graphics.rs deleted file mode 100644 index 947d2cb8..00000000 --- a/src/server/headless/pane_graphics.rs +++ /dev/null @@ -1,513 +0,0 @@ -use super::{HeadlessServer, RenderImpact}; -use crate::api; -use crate::protocol::{ServerMessage, MAX_GRAPHICS_FRAME_SIZE}; -use crate::server::clients::ClientConnectionMode; - -impl HeadlessServer { - pub(super) fn pane_graphics_runtime_active(&self) -> bool { - !self.app.pane_graphics.slots.is_empty() - } - - pub(super) fn handle_pane_graphics_stream_frame( - &mut self, - msg: api::ApiRequestMessage, - ) -> RenderImpact { - if self.shutting_down { - let _ = self.handle_api_request_with_shutdown_check_inner(msg, false, false); - return RenderImpact::None; - } - - let internal_changed = self.drain_all_internal_events_with_forwarding(); - let direct = match &msg.request.method { - api::schema::Method::PaneGraphicsStreamDirect(params) => Some(params.clone()), - _ => None, - }; - let direct_key = direct.as_ref().and_then(|params| { - self.app.parse_pane_id(¶ms.pane_id).map(|(_, pane_id)| { - ( - pane_id, - params - .layer_id - .clone() - .unwrap_or_else(|| api::schema::PANE_GRAPHICS_PRIMARY_LAYER_ID.into()), - ) - }) - }); - let direct_client = direct_key - .as_ref() - .and_then(|key| self.direct_graphics_client_for_key(key)); - let gate_busy = self - .app - .pane_graphics - .slots - .values() - .any(|slot| slot.direct_gate.is_some()); - self.app.direct_graphics_available = direct_client.is_some() && !gate_busy; - let response = self - .app - .handle_api_request_after_internal_events_drained(msg.request); - self.app.direct_graphics_available = self.direct_graphics_available(); - let succeeded = serde_json::from_str::<api::schema::SuccessResponse>(&response).is_ok(); - - if succeeded { - if let (Some(params), Some(client_id)) = (direct, direct_client) { - let direct_frame = direct_key.as_ref().and_then(|key| { - self.app.pane_graphics.slots.get(key).and_then(|slot| { - let active_owner = slot.stream_owner.as_deref() == Some(¶ms.owner) - && slot.stream_is_active(); - active_owner.then_some(())?; - slot.layer - .as_ref() - .and_then(crate::app::pane_graphics::Layer::direct_lease) - .map(|lease| { - ( - lease.path().to_string_lossy().into_owned(), - lease.len() as u64, - lease.fingerprint(), - ) - }) - }) - }); - if let (Some(key), Some((path, expected_len, transfer_id))) = - (direct_key.clone(), direct_frame) - { - let prepared = self.clients.get(&client_id).and_then(|client| { - matches!(client.mode, ClientConnectionMode::ClientShell).then_some(())?; - let layer = self - .app - .pane_graphics - .slots - .get(&key) - .and_then(|slot| slot.layer.as_ref())?; - let asset = crate::kitty_graphics::surface::pane_layer_asset_key( - &self.app, &key, layer, - )?; - let (image_id, control) = - crate::kitty_graphics::surface::direct_upload_control( - &self.client_shell_boot_id, - &asset, - ); - Some(( - crate::kitty_graphics::DirectFileCommand { - leading: Vec::new(), - control, - }, - Some(asset), - image_id, - )) - }); - let Some((command, surface_asset, image_id)) = prepared else { - if self.install_inline_fallback(&key) { - if msg.respond_to.send(response).is_err() { - self.retire_direct_gate(&key); - } - return if internal_changed { - RenderImpact::Full - } else { - RenderImpact::Graphics - }; - } - self.retire_direct_gate(&key); - return if internal_changed { - RenderImpact::Full - } else { - RenderImpact::Graphics - }; - }; - let message = ServerMessage::GraphicsFile { - path, - expected_len, - image_id, - transfer_id, - leading: command.leading, - control: command.control, - surface_asset, - }; - let send = - Self::frame_server_message_with_max(&message, MAX_GRAPHICS_FRAME_SIZE) - .map_err(|_| std::sync::mpsc::TrySendError::Full(Vec::new())) - .and_then(|framed| { - let Some(writer) = self - .clients - .get(&client_id) - .and_then(|client| client.writer.as_ref()) - else { - return Err(std::sync::mpsc::TrySendError::Disconnected( - framed, - )); - }; - writer.render.send_ordered(framed) - }); - match send { - Ok(()) => { - if let Some(slot) = self.app.pane_graphics.slots.get_mut(&key) { - slot.direct_gate = Some(crate::app::pane_graphics::DirectGate { - transfer_id, - image_id, - client_id, - deadline: std::time::Instant::now() - + crate::app::pane_graphics::DIRECT_DELIVERY_TIMEOUT, - written: false, - success_response: response, - respond_to: msg.respond_to, - }); - } - return if internal_changed { - RenderImpact::Full - } else { - RenderImpact::None - }; - } - Err(error) => { - self.handle_unwritten_direct_failure( - &key, - response, - msg.respond_to, - error, - ); - return RenderImpact::Graphics; - } - } - } - } - } - - let response_failed = msg.respond_to.send(response).is_err(); - if succeeded && response_failed { - if let Some(key) = direct_key { - self.retire_direct_gate(&key); - } - } - if internal_changed { - RenderImpact::Full - } else if succeeded { - RenderImpact::Graphics - } else { - RenderImpact::None - } - } - - fn install_inline_fallback(&mut self, key: &crate::app::pane_graphics::Key) -> bool { - let len = self - .app - .pane_graphics - .slots - .get(key) - .and_then(|slot| slot.layer.as_ref()) - .and_then(crate::app::pane_graphics::Layer::direct_lease) - .map(crate::pane_graphics_files::Lease::len); - let Some(len) = len else { - return false; - }; - if len > crate::api::schema::PANE_GRAPHICS_STREAM_MAX_BYTES - || !self.app.pane_graphics.can_store_inline(key, len) - { - return false; - } - let data = self - .app - .pane_graphics - .slots - .get(key) - .and_then(|slot| slot.layer.as_ref()) - .and_then(crate::app::pane_graphics::Layer::direct_lease) - .and_then(|lease| lease.copy_rgba().ok()); - let Some(data) = data else { - return false; - }; - let slot = self - .app - .pane_graphics - .slots - .get_mut(key) - .expect("matched slot"); - if !slot.stream_is_active() { - return false; - } - let layer = slot.layer.take().expect("direct layer"); - slot.layer = Some(crate::app::pane_graphics::Layer::inline( - layer.format, - layer.image_width, - layer.image_height, - data, - layer.render, - layer.z_index, - )); - self.app.pane_graphics.mark_changed(); - true - } - - pub(super) fn handle_unwritten_direct_failure( - &mut self, - key: &crate::app::pane_graphics::Key, - response: String, - respond_to: std::sync::mpsc::Sender<String>, - error: std::sync::mpsc::TrySendError<Vec<u8>>, - ) -> bool { - if matches!(error, std::sync::mpsc::TrySendError::Full(_)) - && self.install_inline_fallback(key) - && respond_to.send(response).is_ok() - { - return true; - } - self.retire_direct_gate(key); - false - } - - pub(super) fn retire_direct_gate(&mut self, key: &crate::app::pane_graphics::Key) { - if self.app.pane_graphics.slots.remove(key).is_some() { - self.app.pane_graphics.mark_changed(); - } - self.app.direct_graphics_available = self.direct_graphics_available(); - } - - pub(super) fn start_direct_graphics_response( - &mut self, - client_id: u64, - transfer_id: u64, - image_id: u32, - ) -> bool { - if let Some(gate) = self.app.pane_graphics.slots.values_mut().find_map(|slot| { - slot.stream_is_active() - .then_some(slot.direct_gate.as_mut()) - .flatten() - .filter(|gate| { - gate.client_id == client_id - && gate.transfer_id == transfer_id - && gate.image_id == image_id - }) - }) { - gate.written = true; - gate.deadline = - std::time::Instant::now() + crate::app::pane_graphics::DIRECT_RESPONSE_TIMEOUT; - } - false - } - - pub(super) fn complete_direct_graphics( - &mut self, - client_id: u64, - transfer_id: u64, - image_id: u32, - success: bool, - ) -> bool { - if self.shutting_down { - self.retire_all_direct_graphics(); - return false; - } - let key = self.app.pane_graphics.slots.iter().find_map(|(key, slot)| { - slot.direct_gate - .as_ref() - .is_some_and(|gate| { - slot.stream_is_active() - && gate.client_id == client_id - && gate.transfer_id == transfer_id - && gate.image_id == image_id - && (!success || gate.written) - }) - .then(|| key.clone()) - }); - let Some(key) = key else { - return false; - }; - let pane_is_live = self - .app - .state - .workspaces - .iter() - .any(|workspace| workspace.pane_state(key.0).is_some()); - if !pane_is_live { - self.retire_direct_gate_with_client_notice(&key); - return false; - } - if success { - let gate = { - let slot = self - .app - .pane_graphics - .slots - .get_mut(&key) - .expect("matched slot"); - if !slot.stream_is_active() { - return false; - } - if let Some(layer) = slot.layer.as_mut() { - layer.mark_resident(client_id); - } - slot.direct_gate.take().expect("matched gate") - }; - if gate.respond_to.send(gate.success_response).is_err() { - self.retire_direct_gate(&key); - return true; - } - self.app.direct_graphics_available = self.direct_graphics_available(); - return true; - } - - if let Some(client) = self.clients.get_mut(&client_id) { - client.direct_graphics = false; - } - self.app.direct_graphics_available = false; - if !self.install_inline_fallback(&key) { - self.retire_direct_gate(&key); - self.retire_all_direct_graphics(); - return true; - } - let gate = self - .app - .pane_graphics - .slots - .get_mut(&key) - .filter(|slot| slot.stream_is_active()) - .and_then(|slot| slot.direct_gate.take()); - let Some(gate) = gate else { - self.retire_direct_gate(&key); - return true; - }; - if gate.respond_to.send(gate.success_response).is_err() { - self.retire_direct_gate(&key); - return true; - } - self.retire_all_direct_graphics(); - true - } - - pub(super) fn expire_direct_graphics(&mut self, now: std::time::Instant) -> bool { - let expired = self - .app - .pane_graphics - .slots - .iter() - .filter_map(|(key, slot)| { - slot.direct_gate - .as_ref() - .filter(|gate| gate.deadline <= now) - .map(|gate| (key.clone(), gate.client_id)) - }) - .collect::<Vec<_>>(); - for (key, client_id) in &expired { - if let Some(slot) = self.app.pane_graphics.slots.get(key) { - if let Some(gate) = &slot.direct_gate { - self.send_to_client( - *client_id, - ServerMessage::GraphicsTransmissionRetired { - transfer_id: gate.transfer_id, - image_id: gate.image_id, - }, - ); - } - } - if let Some(client) = self.clients.get_mut(client_id) { - client.direct_graphics = false; - } - self.retire_direct_gate(key); - } - if !expired.is_empty() { - self.app.direct_graphics_available = false; - self.retire_all_direct_graphics(); - } - !expired.is_empty() - } - - fn retire_direct_gate_with_client_notice(&mut self, key: &crate::app::pane_graphics::Key) { - if let Some((client_id, transfer_id, image_id)) = self - .app - .pane_graphics - .slots - .get(key) - .and_then(|slot| slot.direct_gate.as_ref()) - .map(|gate| (gate.client_id, gate.transfer_id, gate.image_id)) - { - self.send_to_client( - client_id, - ServerMessage::GraphicsTransmissionRetired { - transfer_id, - image_id, - }, - ); - } - self.retire_direct_gate(key); - } - - pub(super) fn retain_live_pane_graphics(&mut self) -> bool { - let dead_direct = self - .app - .pane_graphics - .slots - .iter() - .filter(|((pane_id, _), slot)| { - slot.direct_gate.is_some() - && !self - .app - .state - .workspaces - .iter() - .any(|workspace| workspace.pane_state(*pane_id).is_some()) - }) - .map(|(key, _)| key.clone()) - .collect::<Vec<_>>(); - let changed = !dead_direct.is_empty(); - for key in dead_direct { - self.retire_direct_gate_with_client_notice(&key); - } - let retained = self.app.pane_graphics.retain_live_panes(&self.app.state); - changed || retained - } - - pub(super) fn retire_all_direct_graphics(&mut self) { - let retired = self - .app - .pane_graphics - .slots - .iter() - .filter(|(_, slot)| { - slot.layer - .as_ref() - .is_some_and(crate::app::pane_graphics::Layer::terminal_only) - }) - .map(|(key, slot)| { - let notification = slot - .direct_gate - .as_ref() - .map(|gate| (gate.client_id, gate.transfer_id, gate.image_id)); - (key.clone(), notification) - }) - .collect::<Vec<_>>(); - for (_, notification) in &retired { - if let Some((client_id, transfer_id, image_id)) = notification { - self.send_to_client( - *client_id, - ServerMessage::GraphicsTransmissionRetired { - transfer_id: *transfer_id, - image_id: *image_id, - }, - ); - } - } - for (key, _) in retired { - self.retire_direct_gate(&key); - } - } - - pub(super) fn retire_direct_graphics_for_client(&mut self, client_id: u64) { - let keys = self - .app - .pane_graphics - .slots - .iter() - .filter_map(|(key, slot)| { - let layer = slot.layer.as_ref()?; - let owned = layer.resident_client() == Some(client_id) - || (layer.direct_lease().is_some() - && slot - .direct_gate - .as_ref() - .is_some_and(|gate| gate.client_id == client_id)); - (layer.terminal_only() && owned).then(|| key.clone()) - }) - .collect::<Vec<_>>(); - for key in keys { - self.retire_direct_gate(&key); - } - } -} diff --git a/src/server/headless/render.rs b/src/server/headless/render.rs index a3bbedfa..5bc17c10 100644 --- a/src/server/headless/render.rs +++ b/src/server/headless/render.rs @@ -373,6 +373,15 @@ impl HeadlessServer { } pub(super) fn render_and_stream(&mut self) { + self.render_and_stream_with_graphics_limit(MAX_GRAPHICS_FRAME_SIZE); + } + + #[cfg(all(test, unix))] + pub(super) fn render_and_stream_with_test_graphics_limit(&mut self, max: usize) { + self.render_and_stream_with_graphics_limit(max); + } + + fn render_and_stream_with_graphics_limit(&mut self, graphics_frame_limit: usize) { let full_started = crate::render_prof::timer(); let render_targets = render_targets(&self.clients, self.foreground_client_id); @@ -665,8 +674,16 @@ impl HeadlessServer { popup, graphics, graphics_delivery: next_graphics_delivery, + graphics_sources, } = shell_render.expect("active shell surface"); - surface_parts = Some((panes, splits, popup, graphics, next_graphics_delivery)); + surface_parts = Some(( + panes, + splits, + popup, + graphics, + next_graphics_delivery, + graphics_sources, + )); frame } ClientConnectionMode::TerminalPending => continue, @@ -725,6 +742,20 @@ impl HeadlessServer { } }; + if surface_parts + .as_ref() + .is_some_and(|(_, _, _, graphics, _, _)| { + self.defer_changed_native_geometry(client_id, graphics) + }) + { + continue; + } + let mut native_upload = + surface_parts + .as_mut() + .and_then(|(_, _, _, graphics, delivery, sources)| { + self.prepare_native_scene(client_id, graphics, delivery, sources) + }); let Some(client) = self.clients.get_mut(&client_id) else { continue; }; @@ -734,62 +765,102 @@ impl HeadlessServer { }; let has_graphics = surface_parts .as_ref() - .is_some_and(|(_, _, _, graphics, _)| { + .is_some_and(|(_, _, _, graphics, _, _)| { !graphics.assets.is_empty() || !graphics.placements.is_empty() || !graphics.retained_assets.is_empty() }); let mut next_shell_graphics_delivery = None; - let prepared = if let Some((panes, splits, popup, graphics, delivery)) = surface_parts { - next_shell_graphics_delivery = Some(delivery); - client - .render_state - .prepare_pane_surface(protocol::PaneSurfaceFrame { - boot_id: self.client_shell_boot_id.clone(), - projection_revision: shell_projection_revision, - surface_revision: 0, - frame, - panes, - splits, - popup, - graphics, - }) - } else { - client.render_state.prepare_frame(frame) - }; + let prepared = + if let Some((panes, splits, popup, graphics, delivery, _)) = surface_parts { + next_shell_graphics_delivery = Some(delivery); + client.render_state.prepare_pane_surface_with_file( + protocol::PaneSurfaceFrame { + boot_id: self.client_shell_boot_id.clone(), + projection_revision: shell_projection_revision, + surface_revision: 0, + frame, + panes, + splits, + popup, + graphics, + }, + native_upload.is_some(), + ) + } else { + client.render_state.prepare_frame(frame) + }; let Some(mut prepared) = prepared else { client.clear_deferred_render(); crate::render_prof::event("full_render.skip_identical"); continue; }; let max = if has_graphics { - MAX_GRAPHICS_FRAME_SIZE + graphics_frame_limit } else { crate::protocol::MAX_FRAME_SIZE }; let mut shell_assets_deferred = false; - let serialized = match Self::frame_server_message_with_max(prepared.message(), max) { + let mut suppress_impossible_asset_retry = false; + let mut stripped_assets = Vec::new(); + let mut serialized = match Self::frame_server_message_with_max(prepared.message(), max) + { Ok(frame) => frame, Err(protocol::FramingError::Oversized { claimed, max }) if has_graphics => { warn!( client_id, - claimed, max, "dropping graphics assets from oversized pane surface" + claimed, max, "trimming inline graphics assets from oversized pane surface" ); - if !prepared.strip_pane_surface_assets() { - crate::render_prof::event("full_render.serialize_oversized"); + let framed = loop { + let Some(key) = prepared.pop_pane_surface_asset() else { + break None; + }; + stripped_assets.push(key); + match Self::frame_server_message_with_max(prepared.message(), max) { + Ok(framed) => break Some(framed), + Err(protocol::FramingError::Oversized { .. }) => {} + Err(err) => { + warn!(client_id, err = %err, "failed to serialize trimmed pane surface"); + broken_clients.push(client_id); + break None; + } + } + }; + let Some(framed) = framed else { + if stripped_assets.is_empty() && prepared.has_queued_surface_assets() { + // Delta/reuse owns an encoded payload that cannot be trimmed in + // place. Drop its baseline so the bounded full-surface path runs next. + client.render_state.request_repaint(); + client.defer_full_render(); + } else { + crate::render_prof::event("full_render.serialize_oversized"); + } continue; - } - next_shell_graphics_delivery = None; - shell_assets_deferred = true; - match Self::frame_server_message(prepared.message()) { - Ok(framed) => framed, - Err(err) => { - warn!(client_id, err = %err, "failed to serialize pane surface without assets"); - broken_clients.push(client_id); - crate::render_prof::event("full_render.serialize_error"); - continue; + }; + let made_progress = + native_upload.is_some() || prepared.has_queued_surface_assets(); + if let Some(delivery) = next_shell_graphics_delivery.as_mut() { + for key in &stripped_assets { + delivery.forget_asset(key); } + shell_assets_deferred = made_progress && !stripped_assets.is_empty(); + } + if let (Some((pending, _)), Some(delivery)) = ( + native_upload.as_mut(), + next_shell_graphics_delivery.as_ref(), + ) { + pending.defer_inline_delivery(delivery); } + if !made_progress { + // Keep the asset absent from the delivery cache, but do not spin on + // an identical frame when no payload can fit beside its metadata. + suppress_impossible_asset_retry = true; + warn!( + client_id, + "inline graphics asset cannot fit in a graphics frame; waiting for a later scene change" + ); + } + framed } Err(protocol::FramingError::Oversized { claimed, max }) => { warn!( @@ -806,11 +877,33 @@ impl HeadlessServer { continue; } }; - let shell_graphics_pending = next_shell_graphics_delivery - .as_ref() - .is_some_and(crate::kitty_graphics::surface::DeliveryCache::has_pending); - match writer.render.try_send(serialized) { + let shell_graphics_pending = !suppress_impossible_asset_retry + && next_shell_graphics_delivery + .as_ref() + .is_some_and(crate::kitty_graphics::surface::DeliveryCache::has_pending); + if let Some((_, message)) = &native_upload { + let Ok(file_frame) = + Self::frame_server_message_with_max(message, MAX_GRAPHICS_FRAME_SIZE) + else { + client.defer_full_render(); + continue; + }; + serialized.extend_from_slice(&file_frame); + } + let send = if native_upload.is_some() || self.native_graphics.is_pending(client_id) { + writer.render.send_ordered(serialized) + } else { + writer.render.try_send(serialized) + }; + match send { Ok(()) => { + if let Some((graphics, inline_assets)) = prepared.queued_surface_graphics() { + self.native_graphics + .commit_scene(client_id, graphics, inline_assets); + } + if let Some((pending, _)) = native_upload { + self.native_graphics.commit(client_id, pending); + } if let Some(delivery) = next_shell_graphics_delivery { client.shell_graphics_delivery = delivery; } diff --git a/src/server/headless/retained_surface.rs b/src/server/headless/retained_surface.rs index 07d33ef5..bf98bc3e 100644 --- a/src/server/headless/retained_surface.rs +++ b/src/server/headless/retained_surface.rs @@ -203,6 +203,7 @@ struct RetainedRecipientUpdate { graphics: Option<( protocol::PaneSurfaceFrame, crate::kitty_graphics::surface::DeliveryCache, + crate::kitty_graphics::surface::SourceFiles, )>, } @@ -446,7 +447,7 @@ impl HeadlessServer { let client = &self.clients[&client_id]; let mut next_surface = surface.clone(); crate::server::render_stream::apply_pane_surface_patch(&mut next_surface, &patch); - let Some((graphics, delivery)) = + let Some((graphics, delivery, sources)) = crate::server::client_shell_graphics::collect_retained( &self.app, &next_surface, @@ -460,7 +461,7 @@ impl HeadlessServer { }; graphics_changed = graphics != surface.graphics; next_surface.graphics = graphics; - Some((next_surface, delivery)) + Some((next_surface, delivery, sources)) } else { None }; @@ -490,8 +491,17 @@ impl HeadlessServer { let RetainedRecipientUpdate { client_id, patch, - graphics, + mut graphics, } = update; + if graphics.as_ref().is_some_and(|(surface, _, _)| { + self.defer_changed_native_geometry(client_id, &surface.graphics) + }) { + deferred += 1; + continue; + } + let native_upload = graphics.as_mut().and_then(|(surface, delivery, sources)| { + self.prepare_native_scene(client_id, &mut surface.graphics, delivery, sources) + }); let Some(client) = self.clients.get_mut(&client_id) else { continue; }; @@ -502,9 +512,11 @@ impl HeadlessServer { }; // The published row patch cannot carry images. Reuse the retained text/layout // in a graphics-capable surface message rather than invoking the full renderer. - let (prepared, graphics_delivery) = if let Some((surface, delivery)) = graphics { + let (prepared, graphics_delivery) = if let Some((surface, delivery, _)) = graphics { ( - client.render_state.prepare_pane_surface(surface), + client + .render_state + .prepare_pane_surface_with_file(surface, native_upload.is_some()), Some(delivery), ) } else { @@ -520,7 +532,7 @@ impl HeadlessServer { } else { protocol::MAX_FRAME_SIZE }; - let serialized = + let mut serialized = match Self::frame_server_message_with_max(prepared.message(), max_frame_size) { Ok(serialized) => serialized, Err(error) => { @@ -529,14 +541,39 @@ impl HeadlessServer { %error, "failed to serialize retained pane surface patch" ); + // A delta may own an encoded graphics payload that cannot be + // trimmed in place. Force the bounded full-surface recovery path. + client.render_state.request_repaint(); client.defer_full_render(); deferred += 1; continue; } }; + if let Some((_, message)) = &native_upload { + let Ok(file_frame) = + Self::frame_server_message_with_max(message, MAX_GRAPHICS_FRAME_SIZE) + else { + client.defer_full_render(); + deferred += 1; + continue; + }; + serialized.extend_from_slice(&file_frame); + } crate::render_prof::counter("retained_surface.bytes", serialized.len() as u64); - match writer.render.try_send(serialized) { + let send = if native_upload.is_some() || self.native_graphics.is_pending(client_id) { + writer.render.send_ordered(serialized) + } else { + writer.render.try_send(serialized) + }; + match send { Ok(()) => { + if let Some((graphics, inline_assets)) = prepared.queued_surface_graphics() { + self.native_graphics + .commit_scene(client_id, graphics, inline_assets); + } + if let Some((pending, _)) = native_upload { + self.native_graphics.commit(client_id, pending); + } let graphics_pending = graphics_delivery .as_ref() .is_some_and(crate::kitty_graphics::surface::DeliveryCache::has_pending); diff --git a/src/server/headless/surface_interest.rs b/src/server/headless/surface_interest.rs index 9772fc03..44a68b04 100644 --- a/src/server/headless/surface_interest.rs +++ b/src/server/headless/surface_interest.rs @@ -57,7 +57,7 @@ impl HeadlessServer { if let Some(held_inputs) = held_inputs { self.release_client_shell_inputs(client_id, held_inputs); - self.retire_direct_graphics_for_client(client_id); + self.retire_native_graphics_for_client(client_id); } if changed { self.finish_shell_location_reconciliation(focus_before, &focused_tabs_before); diff --git a/src/server/headless/tests/mod.rs b/src/server/headless/tests/mod.rs index 84031ce4..34258dd3 100644 --- a/src/server/headless/tests/mod.rs +++ b/src/server/headless/tests/mod.rs @@ -1,9 +1,10 @@ use super::*; -#[path = "pane_graphics.rs"] -mod pane_graphics_tests; +mod native_graphics; #[path = "pane_move.rs"] mod pane_move_tests; +#[path = "pane_graphics.rs"] +mod retained_graphics_tests; #[path = "surface_delta.rs"] mod surface_delta_tests; #[path = "surface_interest.rs"] @@ -102,6 +103,7 @@ fn test_headless_server_with_event_hub(event_hub: api::EventHub) -> HeadlessServ client_socket_path: socket_path, client_socket_identity, clients: HashMap::new(), + native_graphics: Default::default(), #[cfg(unix)] next_client_id: 1, foreground_client_id: None, @@ -245,7 +247,6 @@ fn headless_pane_list(server: &mut HeadlessServer) -> Vec<api::schema::PaneInfo> }, respond_to, response_write_complete: None, - stream_active: None, }); let response: api::schema::SuccessResponse = serde_json::from_str(&response_rx.recv().unwrap()).unwrap(); @@ -303,7 +304,6 @@ fn headless_api_request_drains_all_pending_internal_events_before_reading_state( }, respond_to, response_write_complete: None, - stream_active: None, }) ); let response = response_rx @@ -2100,7 +2100,6 @@ async fn client_local_navigation_does_not_emit_global_focus_transitions() { }, respond_to, response_write_complete: None, - stream_active: None, }, ); server.app.sync_focus_events(); @@ -2191,7 +2190,6 @@ async fn client_local_navigation_emits_pane_focused_only_when_that_client_moves( }, respond_to, response_write_complete: None, - stream_active: None, }, ); let response = response_rx.recv().expect("navigation response"); @@ -2323,7 +2321,6 @@ async fn repeated_layout_action_reapplies_controller_geometry() { }, respond_to, response_write_complete: None, - stream_active: None, }, )); @@ -2368,7 +2365,6 @@ async fn public_close_reapplies_controller_geometry() { }, respond_to, response_write_complete: None, - stream_active: None, }) ); @@ -2572,7 +2568,6 @@ async fn public_background_tab_create_preserves_client_locations() { }, respond_to, response_write_complete: None, - stream_active: None, }); assert_eq!( @@ -2620,7 +2615,6 @@ async fn public_workspace_focus_preserves_each_clients_remembered_tabs() { }, respond_to, response_write_complete: None, - stream_active: None, }); let first_location = server.clients[&41].shell_location.as_ref().unwrap(); @@ -2701,7 +2695,6 @@ async fn public_agent_focus_replaces_a_diverged_client_shell_projection() { }, respond_to, response_write_complete: None, - stream_active: None, }); let response: crate::api::schema::SuccessResponse = serde_json::from_str(&response_rx.recv().expect("agent focus response")).unwrap(); @@ -2776,7 +2769,6 @@ async fn public_api_focus_replaces_every_client_shell_projection() { }, respond_to, response_write_complete: None, - stream_active: None, }); assert_eq!(server.app.state.active, Some(1)); server.render_and_stream(); @@ -3393,7 +3385,6 @@ async fn worktree_discovery_does_not_block_client_typing() { }, respond_to, response_write_complete: None, - stream_active: None, }); entered .recv_timeout(Duration::from_secs(5)) @@ -6997,7 +6988,6 @@ fn notification_show_api_forwards_one_semantic_client_notification() { }, respond_to, response_write_complete: None, - stream_active: None, }); assert!(changed); @@ -7060,7 +7050,6 @@ fn notification_show_api_preserves_colon_in_forwarded_title() { }, respond_to, response_write_complete: None, - stream_active: None, }); assert!(changed); @@ -7106,7 +7095,6 @@ fn notification_show_api_validates_empty_title_before_disabled_delivery() { }, respond_to, response_write_complete: None, - stream_active: None, }); assert!(changed); @@ -7137,7 +7125,6 @@ fn notification_show_api_reports_no_foreground_client() { }, respond_to, response_write_complete: None, - stream_active: None, }); assert!(changed); @@ -7188,7 +7175,6 @@ fn notification_show_api_includes_sound_in_semantic_event() { }, respond_to, response_write_complete: None, - stream_active: None, }) ); @@ -7250,7 +7236,6 @@ fn completion_guard_api_report(server: &mut HeadlessServer, method: api::schema: }, respond_to, response_write_complete: None, - stream_active: None, }); let response = response_rx .recv_timeout(Duration::from_millis(100)) @@ -7585,7 +7570,6 @@ fn stale_api_agent_report_does_not_forward_done_sound() { }, respond_to, response_write_complete: None, - stream_active: None, }); assert!(changed); diff --git a/src/server/headless/tests/native_graphics.rs b/src/server/headless/tests/native_graphics.rs new file mode 100644 index 00000000..bde7cf08 --- /dev/null +++ b/src/server/headless/tests/native_graphics.rs @@ -0,0 +1,1411 @@ +//! Native export integration tests for file delivery, retirement, and fallback. +#![cfg(unix)] + +use super::*; +use crate::kitty_graphics::surface::{DeliveryCache, NATIVE_SLOT_BIT, NATIVE_TRANSFER_BIT}; +use crate::protocol::{ + SurfaceGraphicsAsset, SurfaceGraphicsAssetKey, SurfaceGraphicsFormat, SurfaceGraphicsPlacement, + SurfaceGraphicsScene, SurfaceGraphicsSource, SurfaceGraphicsTarget, +}; +use std::sync::mpsc::Receiver; +use std::time::{Duration, Instant}; + +fn native_started(server: &mut HeadlessServer, client: u64, transfer: u64, image: u32) -> bool { + server.native_started(client, transfer, image) +} + +fn native_result( + server: &mut HeadlessServer, + client: u64, + transfer: u64, + image: u32, + success: bool, +) -> bool { + server + .native_result(client, transfer, image, success) + .unwrap_or(false) +} + +fn commit_scene(server: &mut HeadlessServer, client: u64, scene: &SurfaceGraphicsScene) { + let inline_assets = scene + .assets + .iter() + .map(|asset| asset.key.clone()) + .collect::<Vec<_>>(); + server + .native_graphics + .commit_scene(client, scene, &inline_assets); +} + +fn scene() -> SurfaceGraphicsScene { + scene_for(1, 123) +} + +fn scene_for(image_id: u32, data_fingerprint: u64) -> SurfaceGraphicsScene { + let key = SurfaceGraphicsAssetKey { + source: SurfaceGraphicsSource::Terminal { + target: SurfaceGraphicsTarget::Pane { + pane_id: "native-test".into(), + }, + image_id, + }, + image_width: 1, + image_height: 1, + format: SurfaceGraphicsFormat::Rgba, + data_len: 4, + data_fingerprint, + }; + SurfaceGraphicsScene { + assets: vec![SurfaceGraphicsAsset { + key: key.clone(), + data: vec![1, 2, 3, 255], + }], + placements: vec![SurfaceGraphicsPlacement { + asset: key, + logical_placement_id: 1, + x: 2, + y: 3, + cols: 1, + rows: 1, + source_x: 0, + source_y: 0, + source_width: 1, + source_height: 1, + x_offset: 0, + y_offset: 0, + z: 0, + scrollback_offset: 0, + }], + retained_assets: Vec::new(), + } +} + +fn add_client(server: &mut HeadlessServer, id: u64) -> (Receiver<Vec<u8>>, Receiver<Vec<u8>>) { + let (writer, control, render) = test_client_writer(); + let mut client = ClientConnection::new( + (80, 24), + crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }, + id, + RenderEncoding::SemanticFrame, + Some(writer), + ); + client.mode = ClientConnectionMode::ClientShell; + client.direct_graphics = true; + client.pixel_mouse = true; + server.clients.insert(id, client); + server.foreground_client_id = Some(id); + (control, render) +} + +fn prepare_and_commit(server: &mut HeadlessServer, id: u64) -> (PathBuf, u64, u32) { + let mut desired = scene(); + let (pending, message) = server + .native_graphics + .prepare( + id, + &server.client_shell_boot_id, + &mut desired, + &DeliveryCache::default(), + ) + .expect("eligible native export"); + let ServerMessage::GraphicsFile { + path, + transfer_id, + image_id, + expected_len, + leading, + control, + surface_asset, + } = message + else { + panic!("expected file upload"); + }; + assert_ne!(transfer_id & NATIVE_TRANSFER_BIT, 0); + assert_eq!(expected_len, 4); + assert!(leading.is_empty()); + assert_eq!(control, format!("a=t,f=32,s=1,v=1,i={image_id},q=0")); + assert_eq!(surface_asset.as_ref(), Some(&desired.placements[0].asset)); + assert!(desired.assets.is_empty()); + assert_eq!(std::fs::read(&path).unwrap(), [1, 2, 3, 255]); + commit_scene(server, id, &desired); + server.native_graphics.commit(id, pending); + (PathBuf::from(path), transfer_id, image_id) +} + +fn prepare_scene_and_commit( + server: &mut HeadlessServer, + id: u64, + mut desired: SurfaceGraphicsScene, +) -> (u64, u32) { + let (pending, message) = server + .native_graphics + .prepare( + id, + &server.client_shell_boot_id, + &mut desired, + &DeliveryCache::default(), + ) + .expect("eligible native export"); + let ServerMessage::GraphicsFile { + transfer_id, + image_id, + .. + } = message + else { + panic!("expected file upload"); + }; + commit_scene(server, id, &desired); + server.native_graphics.commit(id, pending); + (transfer_id, image_id) +} + +fn ack_native(server: &mut HeadlessServer, id: u64, transfer: u64, image: u32) { + assert!(native_started(server, id, transfer, image)); + native_result(server, id, transfer, image, true); + assert!(!server.native_graphics.is_pending(id)); +} + +fn assert_retirement(control: &Receiver<Vec<u8>>, transfer: u64, image: u32) { + assert!(matches!( + read_server_message(control.recv_timeout(Duration::from_millis(100)).expect("control retirement")), + ServerMessage::GraphicsTransmissionRetired { transfer_id, image_id } + if transfer_id == transfer && image_id == image + )); +} + +fn assert_native_disabled(server: &mut HeadlessServer, id: u64) { + let mut desired = scene(); + assert!(server + .native_graphics + .prepare( + id, + &server.client_shell_boot_id, + &mut desired, + &DeliveryCache::default(), + ) + .is_none()); + assert_eq!(desired.assets[0].data, [1, 2, 3, 255]); + assert!(server.clients[&id].direct_graphics); +} + +#[test] +fn native_tokens_require_started_and_old_api_tokens_are_ignored() { + let mut server = test_headless_server(); + let (_control, _render) = add_client(&mut server, 7); + let (path, token, image) = prepare_and_commit(&mut server, 7); + + assert!(!native_result(&mut server, 7, token, image, true)); + assert!(server.native_graphics.is_pending(7)); + assert!(path.exists()); + assert!(!native_started( + &mut server, + 7, + token & !NATIVE_TRANSFER_BIT, + image + )); + assert!(!native_result( + &mut server, + 7, + token & !NATIVE_TRANSFER_BIT, + image, + true, + )); + assert!(native_started(&mut server, 7, token, image)); + assert!(!native_result(&mut server, 7, token, image, true)); + assert!(!server.native_graphics.is_pending(7)); + assert!(!path.exists()); +} + +#[test] +fn native_image_banks_advance_only_on_started_ack_and_are_scoped_and_pruned() { + let mut server = test_headless_server(); + let (_control, _render) = add_client(&mut server, 7); + let (_other_control, _other_render) = add_client(&mut server, 8); + let base = crate::kitty_graphics::surface::native_host_image_id( + &server.client_shell_boot_id, + &scene().placements[0].asset, + ); + + // Unknown sources start in bank 1, avoiding an inline image at the base ID. + let (first_transfer, first_image) = prepare_scene_and_commit(&mut server, 7, scene()); + assert_eq!(first_image, base ^ NATIVE_SLOT_BIT); + // A success before Started is not an ACK and neither completes nor advances. + assert!(!native_result( + &mut server, + 7, + first_transfer, + first_image, + true + )); + assert!(server.native_graphics.is_pending(7)); + ack_native(&mut server, 7, first_transfer, first_image); + + let (second_transfer, second_image) = + prepare_scene_and_commit(&mut server, 7, scene_for(1, 124)); + assert_eq!(second_image, base); + // A late ACK for the preceding transfer cannot advance the pending revision. + assert!(!native_result( + &mut server, + 7, + first_transfer, + first_image, + true + )); + assert!(server.native_graphics.is_pending(7)); + ack_native(&mut server, 7, second_transfer, second_image); + let (third_transfer, third_image) = prepare_scene_and_commit(&mut server, 7, scene_for(1, 125)); + assert_eq!(third_image, base ^ NATIVE_SLOT_BIT); + ack_native(&mut server, 7, third_transfer, third_image); + + // Selection is independent per client. + let (other_transfer, other_image) = prepare_scene_and_commit(&mut server, 8, scene_for(1, 126)); + assert_eq!(other_image, base ^ NATIVE_SLOT_BIT); + ack_native(&mut server, 8, other_transfer, other_image); + + // A different logical source also starts in bank 1. Retention keeps source + // 1's bank history even though it has no placement in this scene. + let mut source_two = scene_for(2, 127); + source_two + .retained_assets + .push(scene_for(1, 127).placements[0].asset.clone()); + let source_two_base = crate::kitty_graphics::surface::native_host_image_id( + &server.client_shell_boot_id, + &source_two.placements[0].asset, + ); + let (source_two_transfer, source_two_image) = + prepare_scene_and_commit(&mut server, 7, source_two); + assert_eq!(source_two_image, source_two_base ^ NATIVE_SLOT_BIT); + ack_native(&mut server, 7, source_two_transfer, source_two_image); + let (retained_transfer, retained_image) = + prepare_scene_and_commit(&mut server, 7, scene_for(1, 128)); + assert_eq!(retained_image, base); + ack_native(&mut server, 7, retained_transfer, retained_image); + + // Move source 1 back to bank 1, then omit it entirely. Returning after an + // unrelated upload starts at bank 1 rather than retaining stale history. + let (before_prune_transfer, before_prune_image) = + prepare_scene_and_commit(&mut server, 7, scene_for(1, 129)); + assert_eq!(before_prune_image, base ^ NATIVE_SLOT_BIT); + ack_native(&mut server, 7, before_prune_transfer, before_prune_image); + let (pruning_transfer, pruning_image) = + prepare_scene_and_commit(&mut server, 7, scene_for(2, 130)); + assert_eq!(pruning_image, source_two_base ^ NATIVE_SLOT_BIT); + ack_native(&mut server, 7, pruning_transfer, pruning_image); + let (restored_transfer, restored_image) = + prepare_scene_and_commit(&mut server, 7, scene_for(1, 131)); + assert_eq!(restored_image, base ^ NATIVE_SLOT_BIT); + ack_native(&mut server, 7, restored_transfer, restored_image); + + // Reusing a disconnected client ID must not inherit its source-bank history. + server.remove_client(7); + let (_control, _render) = add_client(&mut server, 7); + let (_reconnected_transfer, reconnected_image) = + prepare_scene_and_commit(&mut server, 7, scene_for(1, 132)); + assert_eq!(reconnected_image, base ^ NATIVE_SLOT_BIT); +} + +#[test] +fn inline_commit_updates_bank_only_for_a_new_asset_revision() { + // Replaying inline bytes for the exact ACKed key still addresses the native + // bank retained by the client, so it must not be recorded as bank 0. + let mut exact = test_headless_server(); + let (_control, _render) = add_client(&mut exact, 7); + let base = crate::kitty_graphics::surface::native_host_image_id( + &exact.client_shell_boot_id, + &scene().placements[0].asset, + ); + let (first_transfer, first_image) = prepare_scene_and_commit(&mut exact, 7, scene()); + assert_eq!(first_image, base ^ NATIVE_SLOT_BIT); + ack_native(&mut exact, 7, first_transfer, first_image); + commit_scene(&mut exact, 7, &scene()); + let (_next_transfer, next_image) = prepare_scene_and_commit(&mut exact, 7, scene_for(1, 124)); + assert_eq!(next_image, base); + + // A new inline revision has no native key mapping on the client and is + // uploaded at the logical base ID. The next native revision must therefore + // stage into bank 1 rather than collide with that visible base image. + let mut revised = test_headless_server(); + let (_control, _render) = add_client(&mut revised, 7); + let (first_transfer, first_image) = prepare_scene_and_commit(&mut revised, 7, scene()); + ack_native(&mut revised, 7, first_transfer, first_image); + commit_scene(&mut revised, 7, &scene_for(1, 124)); + let (_next_transfer, next_image) = prepare_scene_and_commit(&mut revised, 7, scene_for(1, 125)); + assert_eq!(next_image, base ^ NATIVE_SLOT_BIT); +} + +#[test] +fn rejected_omitted_scene_does_not_prune_visible_bank_history() { + let mut server = test_headless_server(); + let (_control, _render) = add_client(&mut server, 7); + let base = crate::kitty_graphics::surface::native_host_image_id( + &server.client_shell_boot_id, + &scene().placements[0].asset, + ); + let (first_transfer, first_image) = prepare_scene_and_commit(&mut server, 7, scene()); + assert_eq!(first_image, base ^ NATIVE_SLOT_BIT); + ack_native(&mut server, 7, first_transfer, first_image); + + // Preparing an omitted-source scene is speculative. Model the full render + // path rejecting it because its render lane is already occupied. + let writer = ClientWriter::test_paused(); + server.clients.get_mut(&7).unwrap().writer = Some(writer.clone()); + writer.render.try_send(vec![1]).unwrap(); + let mut omitted = SurfaceGraphicsScene::default(); + assert!(server + .prepare_native_scene( + 7, + &mut omitted, + &mut DeliveryCache::default(), + &mut Default::default(), + ) + .is_none()); + assert!(matches!( + writer.render.try_send(vec![2]), + Err(std::sync::mpsc::TrySendError::Full(_)) + )); + + // The rejected omission did not change what is visible. Stage opposite the + // still-visible bank 1 when the source returns. + let (_next_transfer, next_image) = prepare_scene_and_commit(&mut server, 7, scene_for(1, 124)); + assert_eq!(next_image, base); +} + +#[test] +fn unqueued_native_prepare_does_not_advance_image_bank() { + let mut server = test_headless_server(); + let (_control, _render) = add_client(&mut server, 7); + let base = crate::kitty_graphics::surface::native_host_image_id( + &server.client_shell_boot_id, + &scene_for(3, 200).placements[0].asset, + ); + let mut dropped_scene = scene_for(3, 200); + let (dropped, dropped_message) = server + .native_graphics + .prepare( + 7, + &server.client_shell_boot_id, + &mut dropped_scene, + &DeliveryCache::default(), + ) + .unwrap(); + let ServerMessage::GraphicsFile { + transfer_id: dropped_transfer, + image_id: dropped_image, + .. + } = dropped_message + else { + panic!("expected file upload"); + }; + assert_eq!(dropped_image, base ^ NATIVE_SLOT_BIT); + drop(dropped); // Model serialization/send failure before commit. + + let (transfer, image) = prepare_scene_and_commit(&mut server, 7, scene_for(3, 201)); + assert_eq!(image, dropped_image); + assert!(!native_result( + &mut server, + 7, + dropped_transfer, + dropped_image, + true + )); + assert!(server.native_graphics.is_pending(7)); + ack_native(&mut server, 7, transfer, image); + let (_next_transfer, next_image) = prepare_scene_and_commit(&mut server, 7, scene_for(3, 202)); + assert_eq!(next_image, base); +} + +#[test] +fn native_timeout_uses_control_lane_and_falls_back_inline() { + let mut server = test_headless_server(); + let (control, render) = add_client(&mut server, 7); + let (path, token, image) = prepare_and_commit(&mut server, 7); + // Occupy the render lane: retirement must not wait for file/render backpressure. + server.clients[&7] + .writer + .as_ref() + .unwrap() + .render + .send_ordered(vec![99]) + .unwrap(); + assert!(server.expire_native_graphics(Instant::now() + Duration::from_secs(60))); + assert_retirement(&control, token, image); + assert_eq!( + render.recv_timeout(Duration::from_millis(100)).unwrap(), + vec![99] + ); + assert!(render.try_recv().is_err()); + assert!(!path.exists()); + assert!(!server.native_graphics.is_pending(7)); + assert!(!server.clients[&7].shell_graphics_delivery.has_pending()); + assert_eq!(server.clients[&7].deferred_render(), DeferredRender::Full); + assert_native_disabled(&mut server, 7); + assert!(!native_result(&mut server, 7, token, image, true)); +} + +#[test] +fn native_rejection_falls_back_without_disabling_other_client() { + let mut server = test_headless_server(); + let (control, render) = add_client(&mut server, 7); + let (_other_control, _other_render) = add_client(&mut server, 8); + let (path, token, image) = prepare_and_commit(&mut server, 7); + let (other_path, other_token, other_image) = prepare_and_commit(&mut server, 8); + assert!(native_result(&mut server, 7, token, image, false)); + assert_retirement(&control, token, image); + assert!(render.try_recv().is_err()); + assert!(!path.exists()); + assert_native_disabled(&mut server, 7); + assert!(other_path.exists()); + assert!(server.native_graphics.is_pending(8)); + native_started(&mut server, 8, other_token, other_image); + native_result(&mut server, 8, other_token, other_image, true); + assert!(!other_path.exists()); + assert!(!server.native_graphics.is_pending(8)); + assert!(server.clients[&8].direct_graphics); +} + +#[test] +fn disconnect_unlinks_only_that_clients_native_export() { + let mut server = test_headless_server(); + let (_control, _render) = add_client(&mut server, 7); + let (_other_control, _other_render) = add_client(&mut server, 8); + let (path, token, image) = prepare_and_commit(&mut server, 7); + let (other_path, other_token, other_image) = prepare_and_commit(&mut server, 8); + server.remove_client(7); + assert!(!path.exists()); + assert!(other_path.exists()); + assert!(!server.native_graphics.is_pending(7)); + assert!(server.native_graphics.is_pending(8)); + assert!(!native_result(&mut server, 7, token, image, true)); + native_started(&mut server, 8, other_token, other_image); + native_result(&mut server, 8, other_token, other_image, true); + assert!(!other_path.exists()); + assert!(!server.native_graphics.is_pending(8)); + assert!(server.clients[&8].direct_graphics); +} + +#[test] +fn pending_hold_accepts_pixel_revision_but_not_geometry_or_source_change() { + let mut server = test_headless_server(); + let (_control, _render) = add_client(&mut server, 7); + let (_path, _token, _image) = prepare_and_commit(&mut server, 7); + let original = scene(); + assert!(server.native_graphics.can_hold(7, &original)); + let mut next = original.clone(); + next.placements[0].asset.data_fingerprint += 1; + next.assets[0].key.data_fingerprint += 1; + next.assets[0].data[0] += 1; + assert!(server.native_graphics.can_hold(7, &next)); + assert_eq!( + server.native_graphics.hold(7).unwrap().0.placements, + original.placements + ); + next.placements[0].cols += 1; + assert!(!server.native_graphics.can_hold(7, &next)); + next = original.clone(); + next.placements[0].x += 1; + assert!(!server.native_graphics.can_hold(7, &next)); + next = original.clone(); + next.placements[0].asset.image_width += 1; + assert!(!server.native_graphics.can_hold(7, &next)); + next = original.clone(); + next.placements[0].asset.source = SurfaceGraphicsSource::Terminal { + target: SurfaceGraphicsTarget::Pane { + pane_id: "another-pane".into(), + }, + image_id: 1, + }; + assert!(!server.native_graphics.can_hold(7, &next)); + assert!(!server + .native_graphics + .can_hold(7, &SurfaceGraphicsScene::default())); + next = original; + next.retained_assets.push(next.placements[0].asset.clone()); + assert!(!server.native_graphics.can_hold(7, &next)); +} + +fn drain_native_render_messages(writer: &ClientWriter) -> Vec<ServerMessage> { + let mut messages = Vec::new(); + for batch in writer.test_drain() { + let len = batch.len() as u64; + let mut cursor = std::io::Cursor::new(batch); + while cursor.position() < len { + messages.push( + protocol::read_message(&mut cursor, protocol::MAX_GRAPHICS_FRAME_SIZE) + .expect("decode ordered scene/file transaction"), + ); + } + } + messages +} + +fn write_chunked_test_image( + server: &mut HeadlessServer, + pane: crate::layout::PaneId, + image_id: u32, + side: u32, + pixels: &[u8], +) { + use base64::Engine as _; + + let encoded = base64::engine::general_purpose::STANDARD.encode(pixels); + let chunks = encoded.as_bytes().chunks(4096); + let count = chunks.len(); + for (index, chunk) in chunks.enumerate() { + let more = u8::from(index + 1 < count); + let mut command = if index == 0 { + format!( + "\x1b_Ga=T,f=32,t=d,i={image_id},p={image_id},s={side},v={side},c=1,r=1,q=2,m={more};" + ) + .into_bytes() + } else { + format!("\x1b_Gm={more};").into_bytes() + }; + command.extend_from_slice(chunk); + command.extend_from_slice(b"\x1b\\"); + write_shared_test_pane(server, pane, &command); + } +} + +#[tokio::test] +async fn delta_encoded_inline_commit_updates_next_native_bank() { + let (mut server, _control, _render, pane) = retained_test_server_with_control( + b"\x1b_Ga=T,f=32,t=d,i=7,p=3,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\", + ); + let writer = ClientWriter::test_paused(); + let client = server.clients.get_mut(&1).unwrap(); + client.writer = Some(writer.clone()); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.render_state.enable_surface_delta(true); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + client.direct_graphics = true; + client.pixel_mouse = true; + server.app.state.kitty_graphics_enabled = true; + + server.render_and_stream(); + let first = drain_native_render_messages(&writer); + let (first_transfer, first_image) = first + .iter() + .find_map(|message| match message { + ServerMessage::GraphicsFile { + transfer_id, + image_id, + .. + } => Some((*transfer_id, *image_id)), + _ => None, + }) + .expect("first native upload"); + assert!(native_started(&mut server, 1, first_transfer, first_image)); + native_result(&mut server, 1, first_transfer, first_image, true); + + // Model a client without direct-file capability for this replacement. + // Delta encoding + // wraps its surface in EndpointControl, but the queued asset still commits + // bank 0 bookkeeping. + server.clients.get_mut(&1).unwrap().direct_graphics = false; + write_shared_test_pane( + &mut server, + pane, + b"\x1b_Ga=T,f=32,t=d,i=7,p=3,s=1,v=1,c=1,r=1,q=2;AP8A/w==\x1b\\", + ); + assert!(server.render_retained_pane_surface_and_stream(&HashSet::from([pane]))); + let inline = drain_native_render_messages(&writer); + assert!(inline.iter().any(|message| matches!( + message, + ServerMessage::EndpointControl { kind, .. } + if kind == protocol::surface_delta::MESSAGE_KIND + ))); + assert!(!inline + .iter() + .any(|message| matches!(message, ServerMessage::GraphicsFile { .. }))); + + server.clients.get_mut(&1).unwrap().direct_graphics = true; + write_shared_test_pane( + &mut server, + pane, + b"\x1b_Ga=T,f=32,t=d,i=7,p=3,s=1,v=1,c=1,r=1,q=2;AAD//w==\x1b\\", + ); + server.render_and_stream(); + let next = drain_native_render_messages(&writer); + assert!(next.iter().any(|message| matches!( + message, + ServerMessage::EndpointControl { kind, .. } + if kind == protocol::surface_delta::MESSAGE_KIND + ))); + let next_image = next + .iter() + .find_map(|message| match message { + ServerMessage::GraphicsFile { image_id, .. } => Some(*image_id), + _ => None, + }) + .expect("native upload after inline delta"); + assert_eq!(next_image, first_image, "must stage opposite inline bank 0"); + shutdown_test_runtimes(&mut server); +} + +#[tokio::test] +async fn oversized_native_scene_queues_file_and_continues_stripped_assets_after_ack() { + let (mut server, _control, _render, pane) = retained_test_server_with_control(b""); + let writer = ClientWriter::test_paused(); + let client = server.clients.get_mut(&1).unwrap(); + client.writer = Some(writer.clone()); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + client.direct_graphics = true; + client.pixel_mouse = true; + server.app.state.kitty_graphics_enabled = true; + + const SIDE: u32 = 128; + let pixels = vec![0x7f; SIDE as usize * SIDE as usize * 4]; + for image_id in [7, 8, 9] { + write_chunked_test_image(&mut server, pane, image_id, SIDE, &pixels); + } + + // The limit is deliberately above one 64 KiB asset plus metadata but below + // the two inline assets left after selecting one native upload. + server.render_and_stream_with_test_graphics_limit(100_000); + let first = drain_native_render_messages(&writer); + let first_surface = first + .iter() + .find_map(|message| match message { + ServerMessage::PaneSurface(surface) => Some(surface), + _ => None, + }) + .expect("trimmed metadata scene"); + assert_eq!(first_surface.graphics.placements.len(), 3); + assert_eq!(first_surface.graphics.assets.len(), 1); + let first_inline = first_surface.graphics.assets[0].key.clone(); + let (transfer, image, first_native) = first + .iter() + .find_map(|message| match message { + ServerMessage::GraphicsFile { + transfer_id, + image_id, + surface_asset: Some(asset), + .. + } => Some((*transfer_id, *image_id, asset.clone())), + _ => None, + }) + .expect("selected native asset survives oversized recovery"); + assert_ne!(first_inline.source, first_native.source); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::Full); + + assert!(native_started(&mut server, 1, transfer, image)); + native_result(&mut server, 1, transfer, image, true); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::Full); + + server.render_and_stream_with_test_graphics_limit(100_000); + let continuation = drain_native_render_messages(&writer); + let continued_asset = continuation.iter().find_map(|message| match message { + ServerMessage::GraphicsFile { + surface_asset: Some(asset), + .. + } => Some(asset), + ServerMessage::PaneSurface(surface) => { + surface.graphics.assets.first().map(|asset| &asset.key) + } + _ => None, + }); + let continued_asset = continued_asset.expect("stripped asset delivered on continuation"); + assert_ne!(continued_asset.source, first_inline.source); + assert_ne!(continued_asset.source, first_native.source); + shutdown_test_runtimes(&mut server); +} + +#[tokio::test] +async fn oversized_inline_trims_largest_and_sends_fitting_asset_without_retry_spin() { + let (mut server, _control, _render, pane) = retained_test_server_with_control(b""); + let writer = ClientWriter::test_paused(); + let client = server.clients.get_mut(&1).unwrap(); + client.writer = Some(writer.clone()); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + client.direct_graphics = false; + client.pixel_mouse = true; + server.app.state.kitty_graphics_enabled = true; + + for (image_id, side) in [(7, 128u32), (8, 16u32)] { + let pixels = vec![image_id as u8; side as usize * side as usize * 4]; + write_chunked_test_image(&mut server, pane, image_id, side, &pixels); + } + + // Probe the exact payload size of this surface with only the small asset. + // Resetting the server-side delivery/baseline makes the measured render + // observational only; the next call exercises the production recovery path. + server.render_and_stream(); + let mut measured_surface = drain_native_render_messages(&writer) + .into_iter() + .find_map(|message| match message { + ServerMessage::PaneSurface(surface) => Some(surface), + _ => None, + }) + .expect("measurement surface"); + assert_eq!(measured_surface.graphics.assets.len(), 2); + measured_surface + .graphics + .assets + .retain(|asset| asset.key.image_width == 16); + let exact_small_limit = HeadlessServer::frame_server_message_with_max( + &ServerMessage::PaneSurface(measured_surface), + crate::protocol::MAX_GRAPHICS_FRAME_SIZE, + ) + .unwrap() + .len() + .saturating_sub(4); + let client = server.clients.get_mut(&1).unwrap(); + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.shell_graphics_delivery = DeliveryCache::default(); + client.clear_deferred_render(); + + // Exactly metadata plus the 1 KiB image fits; adding the 64 KiB image does not. + server.render_and_stream_with_test_graphics_limit(exact_small_limit); + let first = drain_native_render_messages(&writer); + let first_surface = first + .iter() + .find_map(|message| match message { + ServerMessage::PaneSurface(surface) => Some(surface), + _ => None, + }) + .expect("surface with fitting inline asset"); + assert_eq!(first_surface.graphics.assets.len(), 1); + assert_eq!(first_surface.graphics.assets[0].key.image_width, 16); + assert!(!first + .iter() + .any(|message| matches!(message, ServerMessage::GraphicsFile { .. }))); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::Full); + + // The remaining large asset cannot fit even alone. It remains absent from + // delivery state, but the identical impossible frame is not rescheduled forever. + server.render_and_stream_with_test_graphics_limit(exact_small_limit); + let second = drain_native_render_messages(&writer); + let second_surface = second + .iter() + .find_map(|message| match message { + ServerMessage::PaneSurface(surface) => Some(surface), + _ => None, + }) + .expect("metadata-only bounded fallback"); + assert!(second_surface.graphics.assets.is_empty()); + assert!(server.clients[&1].shell_graphics_delivery.has_pending()); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); + shutdown_test_runtimes(&mut server); +} + +#[tokio::test] +async fn quiet_native_producer_geometry_retirement_schedules_full_inline_recovery() { + for retained in [false, true] { + let (mut server, _control, _render, pane) = retained_test_server_with_control( + b"\x1b_Ga=T,f=32,t=d,i=7,p=3,s=1,v=1,c=1,r=1,q=2;/wAA/w==\x1b\\", + ); + let writer = ClientWriter::test_paused(); + let client = server.clients.get_mut(&1).unwrap(); + client.writer = Some(writer.clone()); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + client.direct_graphics = true; + client.pixel_mouse = true; + server.app.state.kitty_graphics_enabled = true; + server.app.render_dirty.take(); + server.render_and_stream(); + + // The writer has already drained, so no future writer progress can wake + // the server after geometry cancellation. The native ACK is still pending. + let initial = drain_native_render_messages(&writer); + let metadata_index = initial + .iter() + .position(|message| matches!(message, ServerMessage::PaneSurface(_))) + .expect("native metadata scene"); + let file_index = initial + .iter() + .position(|message| matches!(message, ServerMessage::GraphicsFile { .. })) + .expect("native file upload"); + assert!(metadata_index < file_index); + let ServerMessage::PaneSurface(metadata) = &initial[metadata_index] else { + unreachable!() + }; + assert!(metadata.graphics.assets.is_empty()); + assert_eq!(metadata.graphics.placements.len(), 1); + assert_eq!(metadata.graphics.placements[0].cols, 1); + let ServerMessage::GraphicsFile { + transfer_id, + image_id, + path, + .. + } = &initial[file_index] + else { + unreachable!() + }; + let (token, image, native_path) = (*transfer_id, *image_id, PathBuf::from(path)); + assert!(native_path.exists()); + assert!(server.native_graphics.is_pending(1)); + assert!(writer.test_drain().is_empty()); + native_started(&mut server, 1, token, image); + + // A single producer update moves/resizes the placement, then goes quiet. + // Re-upload the same pixels to replace placement p=3 with two columns. + write_shared_test_pane( + &mut server, + pane, + b"\x1b_Ga=T,f=32,t=d,i=7,p=3,s=1,v=1,c=2,r=1,q=2;/wAA/w==\x1b\\", + ); + server.app.render_dirty.take(); // model the main loop consuming this PTY wake + assert!(!server.app.render_dirty.is_pending()); + if retained { + assert!(server.render_retained_pane_surface_and_stream(&HashSet::from([pane]))); + } else { + server.render_and_stream(); + } + assert!(!server.native_graphics.is_pending(1), "retained={retained}"); + assert!(!native_path.exists()); + assert!( + server.app.render_dirty.is_pending(), + "geometry retirement must independently wake the quiet server; retained={retained}" + ); + assert!(server.app.render_dirty.has_immediate_work()); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::Full); + let retired = drain_native_render_messages(&writer); + assert!(retired.iter().any(|message| matches!(message, + ServerMessage::GraphicsTransmissionRetired { transfer_id, image_id } + if *transfer_id == token && *image_id == image + ))); + assert!(!retired.iter().any(|message| matches!( + message, + ServerMessage::PaneSurface(_) | ServerMessage::GraphicsFile { .. } + ))); + + // A stale success must not restore native residency or consume the full + // recovery wake. No producer bytes or writer events occur after this ACK. + assert!(!native_result(&mut server, 1, token, image, true)); + assert!(!server.native_graphics.is_pending(1)); + let request = server.app.render_dirty.take(); + assert!( + request.generic, + "generic full-render request must survive late ACK; retained={retained}" + ); + server.render_and_stream(); + let recovered = drain_native_render_messages(&writer); + assert!(!recovered + .iter() + .any(|message| matches!(message, ServerMessage::GraphicsFile { .. }))); + let surface = recovered + .iter() + .find_map(|message| match message { + ServerMessage::PaneSurface(surface) => Some(surface), + _ => None, + }) + .expect( + "next scheduled full render delivers inline fallback without new producer activity", + ); + assert_eq!(surface.graphics.placements.len(), 1); + assert_eq!(surface.graphics.placements[0].cols, 2); + assert_eq!(surface.graphics.assets.len(), 1); + assert_eq!(surface.graphics.assets[0].data, [255, 0, 0, 255]); + assert!(server.clients[&1].direct_graphics); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); + shutdown_test_runtimes(&mut server); + } +} + +#[tokio::test] +#[ignore = "manual native-file retained-render scaling profile; one image, 1/15 populated panes"] +async fn native_file_render_scale_profile() { + use base64::Engine as _; + use ratatui::layout::Direction; + + const WARMUP: usize = 5; + const SAMPLES: usize = 35; + const IMAGE_WIDTH: u32 = 800; + const IMAGE_HEIGHT: u32 = 480; + for count in [1, 15] { + for (native, source_retention) in [(false, false), (true, false), (true, true)] { + let (mut server, _control, _render, root) = + retained_test_server_with_control(b"populated root terminal\r\n"); + if source_retention { + // Test runtimes normally leave file transmission disabled. Enable + // it and the snapshot callback only for this source-producing pane. + server + .app + .state + .runtime_for_pane_in_workspace(&server.app.terminal_runtimes, 0, root) + .unwrap() + .test_enable_kitty_source_forwarding(); + } + let mut panes = vec![root]; + for index in 1..count { + let workspace = &mut server.app.state.workspaces[0]; + workspace.tabs[0].layout.focus_pane(panes[(index - 1) / 2]); + let pane = workspace.test_split(if index % 2 == 0 { + Direction::Vertical + } else { + Direction::Horizontal + }); + workspace.insert_test_runtime( + pane, + crate::terminal::TerminalRuntime::test_with_screen_bytes( + 160, + 48, + b"populated terminal\r\nsecond row\r\nthird row\r\n", + ), + ); + panes.push(pane); + } + let writer = ClientWriter::test_paused(); + let client = server.clients.get_mut(&1).unwrap(); + client.writer = Some(writer.clone()); + client.mode = ClientConnectionMode::ClientShell; + client.terminal_size = (160, 48); + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + client.cell_size = crate::kitty_graphics::HostCellSize { + width_px: 10, + height_px: 20, + }; + // Compare actual capability fallback with automatic native delivery. + client.direct_graphics = native; + client.pixel_mouse = true; + server.app.state.kitty_graphics_enabled = true; + server.render_and_stream(); + let baseline = drain_native_render_messages(&writer); + assert!(baseline + .iter() + .any(|message| matches!(message, ServerMessage::PaneSurface(_)))); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); + let sources = panes.iter().copied().collect::<HashSet<_>>(); + + // Deterministic noisy RGB, opaque alpha. Only the first pixel changes + // each submission; no timing includes producer/base64 construction. + let mut rgba = vec![0u8; IMAGE_WIDTH as usize * IMAGE_HEIGHT as usize * 4]; + let mut noise = 0x1234_5678u32; + for pixel in rgba.chunks_exact_mut(4) { + noise = noise.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + pixel[..3].copy_from_slice(&noise.to_le_bytes()[..3]); + pixel[3] = 255; + } + // Private /var/tmp native-store generation owns the producer path: + // bounded to one image and removed even if an assertion unwinds. + let producer_store = crate::pane_graphics_files::FileStore::native_sources(); + let producer = source_retention + .then(|| producer_store.export(&rgba).expect("private producer file")); + let mut previous_source_path: Option<PathBuf> = None; + let mut times = Vec::with_capacity(SAMPLES); + for sample in 0..WARMUP + SAMPLES { + assert!(!server.native_graphics.is_pending(1)); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); + assert!(writer.test_drain().is_empty()); + rgba[..3].copy_from_slice(&(sample as u32 + 1).to_le_bytes()[..3]); + let mut payload = Vec::new(); + payload.extend_from_slice(b"\x1b[H"); + if let Some(producer) = &producer { + std::fs::write(producer.path(), &rgba).unwrap(); + let encoded_path = base64::engine::general_purpose::STANDARD + .encode(producer.path().as_os_str().as_encoded_bytes()); + payload.extend_from_slice(format!( + "\x1b_Ga=T,f=32,t=f,i=7,p=3,s={IMAGE_WIDTH},v={IMAGE_HEIGHT},c=1,r=1,q=2;{encoded_path}\x1b\\" + ).as_bytes()); + } else { + let encoded = base64::engine::general_purpose::STANDARD.encode(&rgba); + let chunks = encoded.as_bytes().chunks(4096); + let chunk_count = chunks.len(); + for (index, chunk) in chunks.enumerate() { + let more = u8::from(index + 1 < chunk_count); + if index == 0 { + payload.extend_from_slice(format!( + "\x1b_Ga=T,f=32,t=d,i=7,p=3,s={IMAGE_WIDTH},v={IMAGE_HEIGHT},c=1,r=1,q=2,m={more};" + ).as_bytes()); + } else { + payload.extend_from_slice(format!("\x1b_Gm={more};").as_bytes()); + } + payload.extend_from_slice(chunk); + payload.extend_from_slice(b"\x1b\\"); + } + } + for pane in &panes { + write_shared_test_pane( + &mut server, + *pane, + format!("\x1b[Hframe {sample:03}").as_bytes(), + ); + } + write_shared_test_pane(&mut server, root, &payload); + let retained_source_path = if source_retention { + let runtime = server + .app + .state + .runtime_for_pane_in_workspace(&server.app.terminal_runtimes, 0, root) + .unwrap(); + let placements = + runtime.kitty_image_placements_with_data_filter(|descriptor| { + assert!(descriptor.source_file, + "source profile requires working CoW retention, not decoded fallback"); + false + }); + assert_eq!(placements.len(), 1); + assert!(placements[0].data.is_empty()); + let source = placements[0].source_file.as_ref().expect("retained source"); + let path = source.path().to_owned(); + if let Some(previous) = previous_source_path.take() { + assert_ne!(previous, path); + assert!(!previous.exists(), "replacement releases previous source"); + } + Some(path) + } else { + None + }; + server.app.render_dirty.take(); + + let started = Instant::now(); + let retained = server.render_retained_pane_surface_and_stream(&sources); + let elapsed = started.elapsed(); + assert!( + retained, + "profile must not silently fall back to full rendering" + ); + if sample >= WARMUP { + times.push(elapsed); + } + + // Validate outside timing. Decoded exports close on ACK; source + // snapshots remain owned by the image until its replacement. + let messages = drain_native_render_messages(&writer); + let surfaces = messages + .iter() + .filter_map(|message| match message { + ServerMessage::PaneSurface(surface) => Some(surface), + _ => None, + }) + .collect::<Vec<_>>(); + assert_eq!(surfaces.len(), 1); + let graphics = &surfaces[0].graphics; + assert_eq!(graphics.placements.len(), 1); + assert_eq!(graphics.placements[0].asset.image_width, IMAGE_WIDTH); + assert_eq!(graphics.placements[0].asset.image_height, IMAGE_HEIGHT); + let files = messages + .iter() + .filter_map(|message| match message { + ServerMessage::GraphicsFile { + path, + expected_len, + transfer_id, + image_id, + .. + } => Some((path, *expected_len, *transfer_id, *image_id)), + _ => None, + }) + .collect::<Vec<_>>(); + if native { + assert!(graphics.assets.is_empty()); + assert_eq!(files.len(), 1); + let (path, expected_len, token, image) = files[0]; + assert_eq!(expected_len, rgba.len() as u64); + assert_eq!(std::fs::read(path).unwrap(), rgba); + assert_ne!(token & NATIVE_TRANSFER_BIT, 0); + native_started(&mut server, 1, token, image); + native_result(&mut server, 1, token, image, true); + if source_retention { + assert_eq!( + retained_source_path.as_deref(), + Some(std::path::Path::new(path)) + ); + assert!( + std::path::Path::new(path).exists(), + "ACK must preserve image backing" + ); + previous_source_path = Some(PathBuf::from(path)); + } else { + assert!(!std::path::Path::new(path).exists()); + } + } else { + assert!(files.is_empty()); + assert_eq!(graphics.assets.len(), 1); + assert_eq!(graphics.assets[0].data, rgba); + } + assert!(!server.native_graphics.is_pending(1)); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); + } + times.sort_unstable(); + println!( + "native_file_render_scale retained=true native_files={native} source_retention={source_retention} viewport=160x48 populated_panes={count} changing_images=1 image=800x480 warmup={WARMUP} samples={SAMPLES} median_us={} p95_us={}", + times[SAMPLES / 2].as_micros(), times[SAMPLES * 95 / 100].as_micros(), + ); + shutdown_test_runtimes(&mut server); + drop(server); + if let Some(path) = previous_source_path { + // Aborted compression tasks release their terminal reference on poll. + for _ in 0..100 { + if !path.exists() { + break; + } + tokio::task::yield_now().await; + } + assert!(!path.exists(), "terminal shutdown releases final source"); + } + } + } +} + +fn source_backed_scene( + source: &Arc<crate::pane_graphics_files::OwnedExport>, +) -> ( + SurfaceGraphicsScene, + crate::kitty_graphics::surface::SourceFiles, +) { + let mut desired = scene(); + let key = &mut desired.assets[0].key; + key.image_width = (source.len() / 4) as u32; + key.data_len = source.len() as u64; + key.data_fingerprint = source.fingerprint(); + desired.placements[0].asset = key.clone(); + desired.placements[0].source_width = key.image_width; + let sources = std::collections::HashMap::from([(key.clone(), Arc::clone(source))]); + desired.assets[0].data.clear(); + (desired, sources) +} + +fn assert_source_upload_and_ack( + source: &Arc<crate::pane_graphics_files::OwnedExport>, + pixels: &[u8], +) { + let mut server = test_headless_server(); + let (_control, _render) = add_client(&mut server, 7); + let (mut desired, mut sources) = source_backed_scene(source); + let mut delivery = DeliveryCache::default(); + let (pending, message) = server + .prepare_native_scene(7, &mut desired, &mut delivery, &mut sources) + .expect("source-backed native upload"); + assert!(desired.assets.is_empty(), "metadata must not carry pixels"); + assert_eq!(desired.placements.len(), 1); + assert!(sources.is_empty()); + let ServerMessage::GraphicsFile { + path, + transfer_id, + image_id, + expected_len, + surface_asset, + leading, + control, + } = message + else { + panic!("expected path-only upload"); + }; + assert_eq!( + PathBuf::from(&path), + source.path(), + "reuse source, not decoded export" + ); + assert_eq!(expected_len, pixels.len() as u64); + assert_eq!(surface_asset.as_ref(), Some(&desired.placements[0].asset)); + assert!(leading.is_empty()); + assert_eq!( + control, + format!("a=t,f=32,s={},v=1,i={image_id},q=0", pixels.len() / 4) + ); + assert_ne!(transfer_id & NATIVE_TRANSFER_BIT, 0); + assert_eq!(std::fs::read(&path).unwrap(), pixels); + commit_scene(&mut server, 7, &desired); + server.native_graphics.commit(7, pending); + native_started(&mut server, 7, transfer_id, image_id); + native_result(&mut server, 7, transfer_id, image_id, true); + assert!(!server.native_graphics.is_pending(7)); + // The producer image's Arc still owns the backing after the transfer ACK. + assert_eq!(std::fs::read(&path).unwrap(), pixels); + assert_eq!(source.copy_rgba().unwrap(), pixels); +} + +#[test] +fn source_backed_native_upload_reuses_path_and_ack_preserves_image_backing() { + let store = crate::pane_graphics_files::FileStore::native_sources(); + let pixels = [1, 2, 3, 255]; + let source = Arc::new(store.export(&pixels).unwrap()); + let path = source.path().to_owned(); + assert_source_upload_and_ack(&source, &pixels); + drop(source); + assert!(!path.exists(), "final image owner releases the backing"); +} + +#[test] +fn source_backed_client_without_direct_support_materializes_exact_pixels() { + let store = crate::pane_graphics_files::FileStore::native_sources(); + let pixels = [1, 2, 3, 255, 200, 100, 50, 0]; + let source = Arc::new(store.export(&pixels).unwrap()); + let (mut desired, mut sources) = source_backed_scene(&source); + let expected_placements = desired.placements.clone(); + let mut server = test_headless_server(); + let (_control, _render) = add_client(&mut server, 7); + server.clients.get_mut(&7).unwrap().direct_graphics = false; + assert!(server + .prepare_native_scene(7, &mut desired, &mut DeliveryCache::default(), &mut sources) + .is_none()); + assert!(sources.is_empty()); + assert_eq!(desired.assets.len(), 1); + assert_eq!(desired.assets[0].data, pixels); + assert_eq!(desired.placements, expected_placements); + assert!(!server.native_graphics.is_pending(7)); +} + +#[cfg(target_os = "linux")] +#[test] +fn snapshotted_source_survives_producer_unlink_and_native_ack() { + use std::os::fd::AsRawFd; + use std::os::unix::fs::OpenOptionsExt as _; + + let store = crate::pane_graphics_files::FileStore::native_sources(); + let producer_path = store.source_directory().unwrap().join("producer.rgba"); + let pixels = vec![73; 4096]; + let mut producer = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .mode(0o600) + .open(&producer_path) + .unwrap(); + std::io::Write::write_all(&mut producer, &pixels).unwrap(); + let snapshot = store.snapshot(i64::from(producer.as_raw_fd()), pixels.len()); + std::fs::remove_file(&producer_path).unwrap(); + drop(producer); + let source = match snapshot { + Ok(source) => Arc::new(source), + Err(error) => { + assert!( + error.kind() == std::io::ErrorKind::Unsupported + || matches!( + error.raw_os_error(), + Some( + libc::EOPNOTSUPP + | libc::ENOTTY + | libc::EXDEV + | libc::EINVAL + | libc::ENOSYS + ) + ), + "unexpected snapshot failure: {error}" + ); + eprintln!("native snapshot unsupported on test filesystem: {error}"); + return; + } + }; + assert!(!producer_path.exists()); + let path = source.path().to_owned(); + assert_source_upload_and_ack(&source, &pixels); + drop(source); + assert!(!path.exists()); +} + +#[tokio::test] +async fn failed_source_read_retries_after_identical_suppression_without_producer_activity() { + let (mut server, _control, _render, _pane) = + retained_test_server_with_control(b"quiet source producer"); + let writer = ClientWriter::test_paused(); + let client = server.clients.get_mut(&1).unwrap(); + client.writer = Some(writer.clone()); + client.mode = ClientConnectionMode::ClientShell; + client.render_state = + crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); + server.clients.get_mut(&1).unwrap().direct_graphics = false; + server.render_and_stream(); + let mut surface = drain_native_render_messages(&writer) + .into_iter() + .find_map(|message| match message { + ServerMessage::PaneSurface(surface) => Some(surface), + _ => None, + }) + .expect("initial quiet surface"); + + // Inject the collected sidecar at the scheduler boundary, avoiding global + // environment changes and a filesystem-dependent CoW requirement. + let store = crate::pane_graphics_files::FileStore::native_sources(); + let pixels = [1, 2, 3, 255]; + let source = Arc::new(store.export(&pixels).unwrap()); + std::fs::write(source.path(), []).unwrap(); + let mut delivery = DeliveryCache::default(); + for attempt in 0..2 { + let (mut graphics, mut sources) = source_backed_scene(&source); + assert!(server + .prepare_native_scene(1, &mut graphics, &mut delivery, &mut sources) + .is_none()); + assert!(graphics.assets.is_empty()); + surface.graphics = graphics; + let client = server.clients.get_mut(&1).unwrap(); + let prepared = client.render_state.prepare_pane_surface(surface.clone()); + if attempt == 0 { + client + .render_state + .commit_sent_frame(prepared.expect("first failed scene")); + } else { + assert!( + prepared.is_none(), + "repeated read failure produces identical metadata" + ); + // Exactly the full-render identical-frame suppression branch. + client.clear_deferred_render(); + } + } + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); + server.app.render_dirty.take(); + + // The timer is one-shot: an inactive client does not receive repeated wakes + // unless a real materialization attempt fails and explicitly re-arms it. + let tick = Instant::now() + Duration::from_secs(2); + assert!(server.expire_native_graphics(tick)); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::Full); + server.clients.get_mut(&1).unwrap().clear_deferred_render(); + assert!(!server.expire_native_graphics(tick + Duration::from_secs(3600))); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); + + let (mut failed_graphics, mut failed_sources) = source_backed_scene(&source); + assert!(server + .prepare_native_scene(1, &mut failed_graphics, &mut delivery, &mut failed_sources) + .is_none()); + assert!(failed_graphics.assets.is_empty()); + let rearmed_tick = tick + Duration::from_secs(7200); + assert!(server.expire_native_graphics(rearmed_tick)); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::Full); + + // No PTY bytes, writer progress, or native ACK occurs after recovery. + std::fs::write(source.path(), pixels).unwrap(); + let (mut graphics, mut sources) = source_backed_scene(&source); + assert!(server + .prepare_native_scene(1, &mut graphics, &mut delivery, &mut sources) + .is_none()); + assert_eq!(graphics.assets[0].data, pixels); + surface.graphics = graphics; + let client = server.clients.get_mut(&1).unwrap(); + let prepared = client + .render_state + .prepare_pane_surface(surface) + .expect("recovered pixels must not be suppressed as identical"); + let ServerMessage::PaneSurface(recovered) = prepared.message() else { + panic!("expected recovered surface"); + }; + assert_eq!(recovered.graphics.assets[0].data, pixels); + client.render_state.commit_sent_frame(prepared); + client.clear_deferred_render(); + assert!( + !server.expire_native_graphics(rearmed_tick + Duration::from_secs(2)), + "successful materialization leaves the one-shot retry timer disarmed" + ); + assert_eq!(server.clients[&1].deferred_render(), DeferredRender::None); + shutdown_test_runtimes(&mut server); +} diff --git a/src/server/headless/tests/pane_graphics.rs b/src/server/headless/tests/pane_graphics.rs index 3e36a8fc..190eaa80 100644 --- a/src/server/headless/tests/pane_graphics.rs +++ b/src/server/headless/tests/pane_graphics.rs @@ -5,93 +5,6 @@ fn receive_render(receiver: &std::sync::mpsc::Receiver<Vec<u8>>, timeout: Durati } #[tokio::test] -async fn client_shell_surface_sends_complete_placements_and_each_live_asset_once() { - let (mut server, _control_rx, client_rx, pane_id) = - retained_test_server_with_control(b"client shell graphics"); - let client = server.clients.get_mut(&1).unwrap(); - client.mode = ClientConnectionMode::ClientShell; - client.render_state = - crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); - client.cell_size = crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }; - set_graphics_layer(&mut server, pane_id, vec![1, 2, 3, 4]); - - server.render_and_stream(); - let first = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); - let ServerMessage::PaneSurface(first) = first else { - panic!("expected client shell pane surface"); - }; - assert_eq!(first.graphics.placements.len(), 1); - assert_eq!(first.graphics.assets.len(), 1); - assert_eq!(first.graphics.assets[0].data, vec![1, 2, 3, 4]); - assert!(matches!( - first.graphics.placements[0].asset.source, - crate::protocol::SurfaceGraphicsSource::PaneLayer { .. } - )); - - server.clients.get_mut(&1).unwrap().request_repaint(); - server.render_and_stream(); - let second = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); - let ServerMessage::PaneSurface(second) = second else { - panic!("expected replacement client shell pane surface"); - }; - assert_eq!(second.graphics.placements, first.graphics.placements); - assert!(second.graphics.assets.is_empty()); - - // A post-commit typed surface.set(true) resets only this viewer's delivery cache, so the - // already selected target receives the asset again without relying on graphics that may have - // arrived during the frozen source frame. - assert!(server.set_client_shell_surface_active(1, true).is_some()); - server.render_and_stream(); - let replay = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); - let ServerMessage::PaneSurface(replay) = replay else { - panic!("expected post-commit graphics replay"); - }; - assert_eq!(replay.graphics.placements, first.graphics.placements); - assert_eq!(replay.graphics.assets.len(), 1); - assert_eq!(replay.graphics.assets[0].data, vec![1, 2, 3, 4]); -} - -#[tokio::test] -async fn client_shell_asset_delivery_is_bounded_to_the_current_live_scene() { - let (mut server, _control_rx, client_rx, pane_id) = - retained_test_server_with_control(b"client shell graphics"); - let client = server.clients.get_mut(&1).unwrap(); - client.mode = ClientConnectionMode::ClientShell; - client.render_state = - crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); - client.cell_size = crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }; - set_graphics_layer(&mut server, pane_id, vec![1, 2, 3, 4]); - - server.render_and_stream(); - let _first = receive_render(&client_rx, Duration::from_millis(100)); - server.app.pane_graphics.slots.clear(); - server.app.pane_graphics.mark_changed(); - server.clients.get_mut(&1).unwrap().request_repaint(); - server.render_and_stream(); - let removed = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); - let ServerMessage::PaneSurface(removed) = removed else { - panic!("expected removed client shell scene"); - }; - assert!(removed.graphics.placements.is_empty()); - - set_graphics_layer(&mut server, pane_id, vec![1, 2, 3, 4]); - server.clients.get_mut(&1).unwrap().request_repaint(); - server.render_and_stream(); - let restored = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); - let ServerMessage::PaneSurface(restored) = restored else { - panic!("expected restored client shell scene"); - }; - assert_eq!(restored.graphics.assets.len(), 1); - assert_eq!(restored.graphics.assets[0].data, vec![1, 2, 3, 4]); -} - -#[tokio::test] async fn first_kitty_image_updates_retained_surface_without_full_redraw() { let (mut server, _control_rx, client_rx, pane_id) = retained_test_server_with_control(b"text before image"); @@ -409,87 +322,6 @@ async fn client_shell_delivers_equal_pixels_for_distinct_terminal_image_ids() { assert!(surface.graphics.assets.is_empty()); } -#[tokio::test] -async fn full_client_shell_render_lane_does_not_commit_graphics_delivery() { - let (mut server, _control_rx, client_rx, pane_id) = - retained_test_server_with_control(b"client shell graphics"); - let client = server.clients.get_mut(&1).unwrap(); - client.mode = ClientConnectionMode::ClientShell; - client.render_state = - crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); - client.cell_size = crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }; - set_graphics_layer(&mut server, pane_id, vec![5, 6, 7, 8]); - - fill_render_lane(&server); - server.render_and_stream(); - let _older = receive_render(&client_rx, Duration::from_millis(100)); - server.render_and_stream(); - let message = read_server_message(receive_render(&client_rx, Duration::from_millis(100))); - let ServerMessage::PaneSurface(surface) = message else { - panic!("expected client shell pane surface"); - }; - assert_eq!(surface.graphics.assets.len(), 1); - assert_eq!(surface.graphics.assets[0].data, vec![5, 6, 7, 8]); -} - -fn graphics_key(pane_id: crate::layout::PaneId) -> crate::app::pane_graphics::Key { - (pane_id, api::schema::PANE_GRAPHICS_PRIMARY_LAYER_ID.into()) -} - -fn active_gate() -> std::sync::Arc<std::sync::atomic::AtomicBool> { - std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)) -} - -fn set_graphics_layer(server: &mut HeadlessServer, pane_id: crate::layout::PaneId, data: Vec<u8>) { - set_named_graphics_layer( - server, - pane_id, - api::schema::PANE_GRAPHICS_PRIMARY_LAYER_ID, - data, - 0, - ); -} - -fn set_named_graphics_layer( - server: &mut HeadlessServer, - pane_id: crate::layout::PaneId, - layer_id: &str, - data: Vec<u8>, - z_index: i32, -) { - let key = (pane_id, layer_id.into()); - let host_image_id = server.app.pane_graphics.reserve_image_id(&key).unwrap(); - let layer = crate::app::pane_graphics::Layer::inline( - api::schema::PaneGraphicsFormat::Png, - 1, - 1, - data, - Default::default(), - z_index, - ); - server.app.pane_graphics.slots.insert( - key, - crate::app::pane_graphics::Slot::test(host_image_id, Some(layer)), - ); -} - -fn set_stream_owner(server: &mut HeadlessServer, pane_id: crate::layout::PaneId, owner: &str) { - let key = graphics_key(pane_id); - if let Some(slot) = server.app.pane_graphics.slots.get_mut(&key) { - slot.stream_owner = Some(owner.into()); - slot.stream_active = Some(active_gate()); - } else { - let host_image_id = server.app.pane_graphics.reserve_image_id(&key).unwrap(); - let mut slot = crate::app::pane_graphics::Slot::test(host_image_id, None); - slot.stream_owner = Some(owner.into()); - slot.stream_active = Some(active_gate()); - server.app.pane_graphics.slots.insert(key, slot); - } -} - fn fill_render_lane(server: &HeadlessServer) { let queued = HeadlessServer::frame_server_message(&ServerMessage::ReloadSoundConfig) .expect("dummy frame"); @@ -500,105 +332,6 @@ fn fill_render_lane(server: &HeadlessServer) { .test_fill_render(queued); } -fn stream_set_message( - id: &str, - pane_id: &str, - owner: &str, - data: Vec<u8>, -) -> (api::ApiRequestMessage, std::sync::mpsc::Receiver<String>) { - let (respond_to, response_rx) = std::sync::mpsc::channel(); - ( - api::ApiRequestMessage { - request: api::schema::Request { - id: id.into(), - method: api::schema::Method::PaneGraphicsStreamSet( - api::schema::PaneGraphicsSetParams { - pane_id: pane_id.into(), - layer_id: None, - z_index: 0, - owner: owner.into(), - format: api::schema::PaneGraphicsFormat::Png, - image_width: 1, - image_height: 1, - data: Some(data), - data_base64: String::new(), - placement: api::schema::PaneGraphicsPlacementParams::default(), - }, - ), - }, - respond_to, - response_write_complete: None, - stream_active: None, - }, - response_rx, - ) -} - -#[cfg(unix)] -fn sparse_direct_frame( - server: &HeadlessServer, - name: &str, - image_width: u32, - image_height: u32, -) -> String { - use std::os::unix::fs::OpenOptionsExt as _; - - let path = server - .app - .pane_graphics_files - .source_directory() - .unwrap() - .join(name); - let file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&path) - .unwrap(); - file.set_len(u64::from(image_width) * u64::from(image_height) * 4) - .unwrap(); - path.to_string_lossy().into_owned() -} - -#[cfg(unix)] -fn direct_stream_message( - id: &str, - pane_id: &str, - owner: &str, - path: String, - image_width: u32, - image_height: u32, - sequence: u64, -) -> (api::ApiRequestMessage, std::sync::mpsc::Receiver<String>) { - let (respond_to, response_rx) = std::sync::mpsc::channel(); - ( - api::ApiRequestMessage { - request: api::schema::Request { - id: id.into(), - method: api::schema::Method::PaneGraphicsStreamDirect( - api::schema::PaneGraphicsDirectParams { - pane_id: pane_id.into(), - layer_id: None, - z_index: 0, - owner: owner.into(), - image_width, - image_height, - format: api::schema::PaneGraphicsFormat::Rgba, - path, - sequence, - revision: 1, - placement: Default::default(), - }, - ), - }, - respond_to, - response_write_complete: None, - stream_active: None, - }, - response_rx, - ) -} - #[tokio::test] async fn pixel_mouse_activation_follows_child_1016_without_graphics_demand() { let (mut server, _client_rx, _pane_id) = @@ -614,7 +347,6 @@ async fn pixel_mouse_activation_follows_child_1016_without_graphics_demand() { }; client.host_mouse_capture_active = None; client.host_sgr_pixels_active = None; - server.app.direct_graphics_available = false; server.stream_host_mouse_capture_mode(); assert!(matches!( @@ -625,868 +357,3 @@ async fn pixel_mouse_activation_follows_child_1016_without_graphics_demand() { } )); } - -#[tokio::test] -async fn graphics_pruning_preserves_live_panes_and_removes_closed_panes() { - let (mut server, _client_rx, pane_id) = retained_test_server(b"aaaa"); - set_graphics_layer(&mut server, pane_id, vec![1, 2, 3]); - - assert!(!server - .app - .pane_graphics - .retain_live_panes(&server.app.state)); - assert!(server - .app - .pane_graphics - .slots - .contains_key(&graphics_key(pane_id))); - - server.app.state.workspaces.clear(); - assert!(server - .app - .pane_graphics - .retain_live_panes(&server.app.state)); - assert!(server.app.pane_graphics.slots.is_empty()); -} - -#[test] -fn stream_open_gate_is_owned_by_the_layer_and_cancels_on_removal() { - let mut server = test_headless_server(); - server.app.state.kitty_graphics_enabled = true; - let workspace = crate::workspace::Workspace::test_new("gated"); - let pane_id = workspace.tabs[0].root_pane; - let public = format!("{}:p1", workspace.id); - server.app.state.workspaces = vec![workspace]; - server.app.state.active = Some(0); - let active = active_gate(); - let (respond_to, response_rx) = std::sync::mpsc::channel(); - - server.handle_api_request_with_shutdown_check(api::ApiRequestMessage { - request: api::schema::Request { - id: "open-gated".into(), - method: api::schema::Method::PaneGraphicsStreamOpen( - api::schema::PaneGraphicsStreamParams { - pane_id: public.clone(), - layer_id: None, - z_index: 0, - owner: "worker-1".into(), - }, - ), - }, - respond_to, - response_write_complete: None, - stream_active: Some(active.clone()), - }); - assert!( - serde_json::from_str::<api::schema::SuccessResponse>(&response_rx.recv().unwrap()).is_ok() - ); - let (frame, frame_response) = - stream_set_message("gated-frame", &public, "worker-1", vec![1, 2, 3]); - assert_eq!( - server.handle_api_request_with_render_impact(frame), - RenderImpact::Graphics - ); - assert!(frame_response.recv().is_ok()); - assert!(active.load(std::sync::atomic::Ordering::Acquire)); - active.store(false, std::sync::atomic::Ordering::Release); - let (delayed, delayed_response) = - stream_set_message("delayed-frame", &public, "worker-1", vec![4, 5, 6]); - assert_eq!( - server.handle_api_request_with_render_impact(delayed), - RenderImpact::None - ); - let error: api::schema::ErrorResponse = - serde_json::from_str(&delayed_response.recv().unwrap()).unwrap(); - assert_eq!(error.error.code, "stream_closed"); - assert!(server - .app - .pane_graphics - .slots - .remove(&graphics_key(pane_id)) - .is_some()); - assert!(!active.load(std::sync::atomic::Ordering::Acquire)); -} - -#[test] -fn stream_set_has_graphics_only_render_impact() { - let mut server = test_headless_server(); - let workspace = crate::workspace::Workspace::test_new("graphics"); - let pane_id = workspace.tabs[0].root_pane; - let public_pane_id = format!("{}:p1", workspace.id); - server.app.state.workspaces = vec![workspace]; - server.app.state.active = Some(0); - server.app.state.selected = 0; - server.app.state.kitty_graphics_enabled = true; - set_stream_owner(&mut server, pane_id, "owner-a"); - - let (request, response_rx) = - stream_set_message("wrong-owner", &public_pane_id, "owner-b", vec![1, 2, 3]); - assert_eq!( - server.handle_api_request_with_render_impact(request), - RenderImpact::None - ); - assert!(serde_json::from_str::<api::schema::ErrorResponse>( - &response_rx - .recv_timeout(Duration::from_millis(100)) - .unwrap() - ) - .is_ok()); - - let (request, response_rx) = - stream_set_message("stream-frame", &public_pane_id, "owner-a", vec![1, 2, 3]); - assert_eq!( - server.handle_api_request_with_render_impact(request), - RenderImpact::Graphics - ); - assert!(serde_json::from_str::<api::schema::SuccessResponse>( - &response_rx - .recv_timeout(Duration::from_millis(100)) - .unwrap() - ) - .is_ok()); - - server - .app - .event_tx - .try_send(AppEvent::UpdateReady { - version: "9.9.9".into(), - install_command: "herdr update".into(), - }) - .unwrap(); - let (request, _response_rx) = stream_set_message( - "stream-frame-with-internal-event", - &public_pane_id, - "owner-a", - vec![4, 5, 6], - ); - assert_eq!( - server.handle_api_request_with_render_impact(request), - RenderImpact::Full - ); - - server.app.pane_graphics.clear(); - let (respond_to, _response_rx) = std::sync::mpsc::channel(); - let impact = server.handle_api_request_with_render_impact(api::ApiRequestMessage { - request: api::schema::Request { - id: "direct-frame".into(), - method: api::schema::Method::PaneGraphicsSet(api::schema::PaneGraphicsSetParams { - pane_id: public_pane_id, - layer_id: None, - z_index: 0, - owner: String::new(), - format: api::schema::PaneGraphicsFormat::Png, - image_width: 1, - image_height: 1, - data: Some(vec![1, 2, 3]), - data_base64: String::new(), - placement: api::schema::PaneGraphicsPlacementParams::default(), - }), - }, - respond_to, - response_write_complete: None, - stream_active: None, - }); - assert_eq!(impact, RenderImpact::Full); -} - -#[test] -fn rejected_or_stale_requests_do_not_schedule_rendering() { - let mut server = test_headless_server(); - let workspace = crate::workspace::Workspace::test_new("graphics"); - let pane_id = workspace.tabs[0].root_pane; - let public_pane_id = format!("{}:p1", workspace.id); - server.app.state.workspaces = vec![workspace]; - server.app.state.active = Some(0); - server.app.state.selected = 0; - server.app.state.kitty_graphics_enabled = false; - - let (respond_to, response_rx) = std::sync::mpsc::channel(); - let changed = server.handle_api_request_with_shutdown_check(api::ApiRequestMessage { - request: api::schema::Request { - id: "disabled-set".into(), - method: api::schema::Method::PaneGraphicsSet(api::schema::PaneGraphicsSetParams { - pane_id: public_pane_id.clone(), - layer_id: None, - z_index: 0, - owner: String::new(), - format: api::schema::PaneGraphicsFormat::Png, - image_width: 1, - image_height: 1, - data: Some(vec![1, 2, 3]), - data_base64: String::new(), - placement: api::schema::PaneGraphicsPlacementParams::default(), - }), - }, - respond_to, - response_write_complete: None, - stream_active: None, - }); - assert!(!changed); - let response = response_rx - .recv_timeout(Duration::from_millis(100)) - .unwrap(); - assert_eq!( - serde_json::from_str::<api::schema::ErrorResponse>(&response) - .unwrap() - .error - .code, - "feature_disabled" - ); - - server.app.state.kitty_graphics_enabled = true; - set_stream_owner(&mut server, pane_id, "current-owner"); - let (respond_to, response_rx) = std::sync::mpsc::channel(); - let impact = server.handle_api_request_with_render_impact(api::ApiRequestMessage { - request: api::schema::Request { - id: "stale-close".into(), - method: api::schema::Method::PaneGraphicsStreamClose( - api::schema::PaneGraphicsStreamParams { - pane_id: public_pane_id, - layer_id: None, - z_index: 0, - owner: "stale-owner".into(), - }, - ), - }, - respond_to, - response_write_complete: None, - stream_active: None, - }); - assert_eq!(impact, RenderImpact::None); - assert_eq!( - server - .app - .pane_graphics - .slots - .get(&graphics_key(pane_id)) - .and_then(|slot| slot.stream_owner.as_deref()), - Some("current-owner") - ); - assert!(serde_json::from_str::<api::schema::SuccessResponse>( - &response_rx - .recv_timeout(Duration::from_millis(100)) - .unwrap() - ) - .is_ok()); -} - -#[cfg(unix)] -#[tokio::test] -async fn direct_graphics_availability_follows_foreground_client_with_background_clients() { - let (mut server, _client_rx, _pane_id) = retained_test_server(b"direct eligibility"); - let foreground = server.clients.get_mut(&1).unwrap(); - foreground.direct_graphics = true; - foreground.pixel_mouse = true; - - let (background_writer, _background_control_rx, _background_render_rx) = test_client_writer(); - server.clients.insert( - 2, - ClientConnection::new( - (80, 24), - crate::kitty_graphics::HostCellSize::default(), - 2, - RenderEncoding::SemanticFrame, - Some(background_writer), - ), - ); - - server.sync_foreground_client_state(); - assert!(server.direct_graphics_available()); - assert!(server.app.direct_graphics_available); - - server.foreground_client_id = Some(2); - server.sync_foreground_client_state(); - assert!(!server.direct_graphics_available()); - assert!(!server.app.direct_graphics_available); -} - -#[cfg(unix)] -#[test] -fn direct_graphics_routing_prefers_the_target_stream_owner() { - let (mut server, first_key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - add_direct_client(&mut server, 7); - add_direct_client(&mut server, 8); - server.clients.get_mut(&8).unwrap().last_activity = 2; - let (transfer_id, image_id) = direct_ids(&server, &first_key); - assert!(server.complete_direct_graphics(7, transfer_id, image_id, true)); - assert_eq!(response_rx.recv().unwrap(), "ack"); - - let second_key = (first_key.0, "second".into()); - let second_layer = crate::app::pane_graphics::Layer { - format: crate::api::schema::PaneGraphicsFormat::Rgba, - image_width: 1, - image_height: 1, - backing: crate::app::pane_graphics::Backing::Resident { - len: 4, - client_id: 8, - }, - data_fingerprint: 2, - render: Default::default(), - z_index: 0, - }; - server.app.pane_graphics.slots.insert( - second_key.clone(), - crate::app::pane_graphics::Slot::test((1 << 31) | 901, Some(second_layer)), - ); - - assert_eq!(server.direct_graphics_client_for_key(&first_key), Some(7)); - assert_eq!(server.direct_graphics_client_for_key(&second_key), Some(8)); - assert_eq!( - server.direct_graphics_client_for_key(&(first_key.0, "new".into())), - Some(8) - ); -} - -#[cfg(unix)] -#[test] -fn resident_direct_stream_survives_non_direct_client_becoming_foreground() { - let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - add_direct_client(&mut server, 7); - server.foreground_client_id = Some(7); - server.sync_foreground_client_state(); - let (transfer_id, image_id) = direct_ids(&server, &key); - assert!(server.complete_direct_graphics(7, transfer_id, image_id, true)); - assert_eq!(response_rx.recv().unwrap(), "ack"); - - let (writer, _control_rx, _render_rx) = test_client_writer(); - server.clients.insert( - 8, - ClientConnection::new( - (80, 24), - crate::kitty_graphics::HostCellSize::default(), - 2, - RenderEncoding::SemanticFrame, - Some(writer), - ), - ); - server.foreground_client_id = Some(8); - server.sync_foreground_client_state(); - - assert!(server.direct_graphics_available()); - assert!(server.app.direct_graphics_available); - assert_eq!( - server.app.pane_graphics.slots[&key] - .layer - .as_ref() - .and_then(crate::app::pane_graphics::Layer::resident_client), - Some(7) - ); - - server.remove_client_and_resize_if_needed(7); - assert!(!server.app.pane_graphics.slots.contains_key(&key)); - assert!(!server.direct_graphics_available()); - assert!(!server.app.direct_graphics_available); -} - -#[cfg(unix)] -#[tokio::test] -async fn client_shell_direct_graphics_uploads_without_server_authored_coordinates() { - let (mut server, _control_rx, client_rx, pane_id) = - retained_test_server_with_control(b"client shell direct"); - server.app.state.kitty_graphics_enabled = true; - let client = server.clients.get_mut(&1).unwrap(); - client.mode = ClientConnectionMode::ClientShell; - client.render_state = - crate::server::render_stream::ClientRenderState::new(RenderEncoding::SemanticFrame); - client.cell_size = crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }; - client.direct_graphics = true; - client.pixel_mouse = true; - server.app.direct_graphics_available = true; - let (background_writer, _background_control_rx, background_render_rx) = test_client_writer(); - server.clients.insert( - 2, - ClientConnection::new( - (80, 24), - crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }, - 2, - RenderEncoding::SemanticFrame, - Some(background_writer), - ), - ); - set_stream_owner(&mut server, pane_id, "browser"); - let public_pane_id = server.app.public_pane_id(0, pane_id).unwrap(); - let path = sparse_direct_frame(&server, "client-shell-direct.rgba", 1, 1); - let (message, response_rx) = - direct_stream_message("shell-direct", &public_pane_id, "browser", path, 1, 1, 1); - - assert_eq!( - server.handle_pane_graphics_stream_frame(message), - RenderImpact::None - ); - let (transfer_id, image_id, asset) = match read_server_message( - client_rx - .recv_timeout(Duration::from_secs(1)) - .expect("client shell direct upload"), - ) { - ServerMessage::GraphicsFile { - transfer_id, - image_id, - leading, - control, - surface_asset: Some(asset), - .. - } => { - assert!(leading.is_empty()); - assert!(control.starts_with("a=t,"), "{control}"); - assert!(!control.contains("\u{1b}["), "{control}"); - (transfer_id, image_id, asset) - } - other => panic!("expected client shell graphics file, got {other:?}"), - }; - assert!(background_render_rx.try_recv().is_err()); - assert_eq!( - image_id, - crate::kitty_graphics::surface::host_image_id(&server.client_shell_boot_id, &asset) - ); - let (pending, _) = crate::server::client_shell_graphics::collect( - &server.app, - &[], - &[], - None, - Some(crate::ui::TabSurfaceTarget { - workspace_index: 0, - tab_index: 0, - }), - crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }, - &crate::kitty_graphics::surface::DeliveryCache::default(), - 1, - ); - assert_eq!(pending.retained_assets, vec![asset.clone()]); - - server.start_direct_graphics_response(1, transfer_id, image_id); - assert!(server.complete_direct_graphics(1, transfer_id, image_id, true)); - assert!(serde_json::from_str::<api::schema::SuccessResponse>( - &response_rx.recv_timeout(Duration::from_secs(1)).unwrap() - ) - .is_ok()); - - server.foreground_client_id = Some(2); - server.sync_foreground_client_state(); - let next_path = sparse_direct_frame(&server, "client-shell-direct-next.rgba", 1, 1); - let (next_message, next_response_rx) = direct_stream_message( - "shell-direct-next", - &public_pane_id, - "browser", - next_path, - 1, - 1, - 2, - ); - assert_eq!( - server.handle_pane_graphics_stream_frame(next_message), - RenderImpact::None - ); - let (next_transfer_id, next_image_id, next_asset) = match read_server_message( - client_rx - .recv_timeout(Duration::from_secs(1)) - .expect("sticky owner direct upload"), - ) { - ServerMessage::GraphicsFile { - transfer_id, - image_id, - surface_asset: Some(asset), - .. - } => (transfer_id, image_id, asset), - other => panic!("expected sticky owner graphics file, got {other:?}"), - }; - assert!(background_render_rx.try_recv().is_err()); - server.start_direct_graphics_response(1, next_transfer_id, next_image_id); - assert!(server.complete_direct_graphics(1, next_transfer_id, next_image_id, true)); - assert!(serde_json::from_str::<api::schema::SuccessResponse>( - &next_response_rx - .recv_timeout(Duration::from_secs(1)) - .unwrap() - ) - .is_ok()); - assert!(server.app.direct_graphics_available); - - server.clients.get_mut(&1).unwrap().request_repaint(); - server.render_and_stream(); - let surface = read_server_message( - client_rx - .recv_timeout(Duration::from_secs(1)) - .expect("resident client shell scene"), - ); - let ServerMessage::PaneSurface(surface) = surface else { - panic!("expected client shell pane surface"); - }; - assert_eq!(surface.graphics.placements.len(), 1); - assert!(surface.graphics.assets.is_empty()); - assert_eq!(surface.graphics.placements[0].asset, next_asset); - assert_eq!(surface.graphics.retained_assets, vec![next_asset.clone()]); - - let background_surface = read_server_message( - background_render_rx - .recv_timeout(Duration::from_secs(1)) - .expect("non-owning client shell scene"), - ); - let ServerMessage::PaneSurface(background_surface) = background_surface else { - panic!("expected non-owning client shell pane surface"); - }; - assert!(background_surface.graphics.assets.is_empty()); - assert!(background_surface.graphics.placements.is_empty()); - assert!(background_surface.graphics.retained_assets.is_empty()); - - let (hidden, _) = crate::server::client_shell_graphics::collect( - &server.app, - &[], - &[], - None, - Some(crate::ui::TabSurfaceTarget { - workspace_index: 0, - tab_index: 0, - }), - crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }, - &crate::kitty_graphics::surface::DeliveryCache::default(), - 1, - ); - assert!(hidden.placements.is_empty()); - assert_eq!(hidden.retained_assets, vec![next_asset]); -} - -#[cfg(unix)] -fn direct_gate_server( - data: &[u8], -) -> ( - HeadlessServer, - crate::app::pane_graphics::Key, - std::sync::mpsc::Receiver<String>, -) { - direct_gate_server_with_file(data.len(), Some(data)) -} - -#[cfg(unix)] -fn direct_gate_server_with_file( - len: usize, - data: Option<&[u8]>, -) -> ( - HeadlessServer, - crate::app::pane_graphics::Key, - std::sync::mpsc::Receiver<String>, -) { - use std::io::Write as _; - use std::os::unix::fs::OpenOptionsExt as _; - let mut server = test_headless_server(); - let workspace = crate::workspace::Workspace::test_new("direct-gate"); - let pane_id = workspace.tabs[0].root_pane; - server.app.state.workspaces = vec![workspace]; - server.app.state.active = Some(0); - let key = graphics_key(pane_id); - let path = server - .app - .pane_graphics_files - .source_directory() - .unwrap() - .join("gate-frame"); - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&path) - .unwrap(); - if let Some(data) = data { - file.write_all(data).unwrap(); - } else { - file.set_len(len as u64).unwrap(); - } - drop(file); - let lease = server.app.pane_graphics_files.lease(&path, len).unwrap(); - let (respond_to, response_rx) = std::sync::mpsc::channel(); - let layer = - crate::app::pane_graphics::Layer::direct(1, 1, lease.clone(), Default::default(), 0); - let mut slot = crate::app::pane_graphics::Slot::test((1 << 31) | 900, Some(layer)); - slot.stream_owner = Some("owner".into()); - slot.stream_active = Some(active_gate()); - slot.direct_gate = Some(crate::app::pane_graphics::DirectGate { - transfer_id: lease.fingerprint(), - image_id: (1 << 31) | 900, - client_id: 7, - deadline: std::time::Instant::now() + Duration::from_secs(1), - written: true, - success_response: "ack".into(), - respond_to, - }); - server.app.pane_graphics.slots.insert(key.clone(), slot); - (server, key, response_rx) -} - -#[cfg(unix)] -fn direct_ids(server: &HeadlessServer, key: &crate::app::pane_graphics::Key) -> (u64, u32) { - let slot = &server.app.pane_graphics.slots[key]; - let gate = slot.direct_gate.as_ref().unwrap(); - (gate.transfer_id, gate.image_id) -} - -#[cfg(unix)] -fn add_direct_client(server: &mut HeadlessServer, client_id: u64) { - let (writer, control_rx, render_rx) = test_client_writer(); - std::mem::forget((control_rx, render_rx)); - let mut client = ClientConnection::new( - (80, 24), - crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }, - 1, - RenderEncoding::SemanticFrame, - Some(writer), - ); - client.direct_graphics = true; - client.pixel_mouse = true; - server.clients.insert(client_id, client); -} - -#[cfg(unix)] -#[test] -fn terminal_response_deadline_starts_only_after_client_flush() { - let (mut server, key, _response_rx) = direct_gate_server(&[1, 2, 3, 4]); - let slot = server.app.pane_graphics.slots.get_mut(&key).unwrap(); - let gate = slot.direct_gate.as_mut().unwrap(); - gate.written = false; - let (transfer_id, image_id) = (gate.transfer_id, slot.host_image_id); - assert!(!server.complete_direct_graphics(7, transfer_id, image_id, true)); - assert!(!server.start_direct_graphics_response(7, transfer_id, image_id)); - let gate = server.app.pane_graphics.slots[&key] - .direct_gate - .as_ref() - .unwrap(); - assert!(gate.written && gate.deadline > std::time::Instant::now()); -} - -#[cfg(unix)] -#[test] -fn outer_timeout_covers_both_direct_phases_and_cancellation_blocks_late_results() { - assert!( - crate::app::pane_graphics::DIRECT_OUTER_TIMEOUT - > crate::app::pane_graphics::DIRECT_DELIVERY_TIMEOUT - + crate::app::pane_graphics::DIRECT_RESPONSE_TIMEOUT - ); - let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - let slot = server.app.pane_graphics.slots.get_mut(&key).unwrap(); - slot.stream_active - .as_ref() - .unwrap() - .store(false, std::sync::atomic::Ordering::Release); - let (transfer_id, image_id) = ( - slot.direct_gate.as_ref().unwrap().transfer_id, - slot.host_image_id, - ); - - assert!(!server.complete_direct_graphics(7, transfer_id, image_id, true)); - assert!(response_rx.try_recv().is_err()); - assert!(server.app.pane_graphics.slots[&key] - .layer - .as_ref() - .unwrap() - .direct_lease() - .is_some()); -} - -#[cfg(unix)] -#[test] -fn matching_terminal_ok_releases_producer_and_acknowledges() { - let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - let (transfer_id, image_id) = direct_ids(&server, &key); - - assert!(server.complete_direct_graphics(7, transfer_id, image_id, true)); - - assert_eq!(response_rx.recv().unwrap(), "ack"); - let layer = server.app.pane_graphics.slots[&key].layer.as_ref().unwrap(); - assert!(layer.terminal_only()); - assert!(layer.direct_lease().is_none()); -} - -#[cfg(unix)] -#[test] -fn unwritten_direct_full_falls_back_without_stickiness_but_disconnect_retires() { - for error in [ - std::sync::mpsc::TrySendError::Full(Vec::new()), - std::sync::mpsc::TrySendError::Disconnected(Vec::new()), - ] { - let should_ack = matches!(error, std::sync::mpsc::TrySendError::Full(_)); - let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - add_direct_client(&mut server, 7); - let gate = server - .app - .pane_graphics - .slots - .get_mut(&key) - .and_then(|slot| slot.direct_gate.take()) - .unwrap(); - let result = server.handle_unwritten_direct_failure( - &key, - gate.success_response, - gate.respond_to, - error, - ); - let inline = server - .app - .pane_graphics - .slots - .get(&key) - .and_then(|slot| slot.layer.as_ref()?.inline_data()) - .is_some(); - assert_eq!( - ( - result, - response_rx.try_recv().ok().as_deref() == Some("ack"), - inline, - server.clients[&7].direct_graphics, - ), - (should_ack, should_ack, should_ack, true) - ); - } -} - -#[cfg(unix)] -#[test] -fn eligibility_loss_cancels_the_queued_direct_upload() { - let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - let (writer, control_rx, _render_rx) = test_client_writer(); - let mut client = ClientConnection::new( - (80, 24), - crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }, - 1, - RenderEncoding::SemanticFrame, - Some(writer), - ); - client.direct_graphics = true; - client.pixel_mouse = true; - server.clients.insert(7, client); - let (transfer_id, image_id) = direct_ids(&server, &key); - - server.retire_all_direct_graphics(); - - assert!(matches!( - read_server_message( - control_rx - .recv_timeout(Duration::from_millis(100)) - .expect("direct retirement") - ), - ServerMessage::GraphicsTransmissionRetired { - transfer_id: retired_transfer, - image_id: retired_image, - } if retired_transfer == transfer_id && retired_image == image_id - )); - assert!(!server.app.pane_graphics.slots.contains_key(&key)); - assert!(response_rx.recv().is_err()); -} - -#[cfg(unix)] -#[test] -fn client_loss_retires_only_its_direct_stream() { - let (mut pending, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - pending.retire_direct_graphics_for_client(8); - assert!(pending.app.pane_graphics.slots.contains_key(&key)); - pending.retire_direct_graphics_for_client(7); - assert!(!pending.app.pane_graphics.slots.contains_key(&key)); - assert!(response_rx.recv().is_err()); - - let (mut resident, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - let slot = resident.app.pane_graphics.slots.get(&key).unwrap(); - assert!(resident.complete_direct_graphics( - 7, - slot.direct_gate.as_ref().unwrap().transfer_id, - slot.host_image_id, - true, - )); - assert_eq!(response_rx.recv().unwrap(), "ack"); - resident.retire_direct_graphics_for_client(8); - assert!(resident.app.pane_graphics.slots.contains_key(&key)); - resident.retire_direct_graphics_for_client(7); - assert!(!resident.app.pane_graphics.slots.contains_key(&key)); -} - -#[cfg(unix)] -#[test] -fn pane_removal_cancels_the_pending_client_upload() { - let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - let (writer, control_rx, _render_rx) = test_client_writer(); - let mut client = ClientConnection::new( - (80, 24), - crate::kitty_graphics::HostCellSize { - width_px: 10, - height_px: 20, - }, - 1, - RenderEncoding::SemanticFrame, - Some(writer), - ); - client.direct_graphics = true; - client.pixel_mouse = true; - server.clients.insert(7, client); - let (transfer_id, image_id) = direct_ids(&server, &key); - server.app.state.workspaces.clear(); - - assert!(server.retain_live_pane_graphics()); - - assert!(matches!( - read_server_message( - control_rx - .recv_timeout(Duration::from_millis(100)) - .expect("pane removal retirement") - ), - ServerMessage::GraphicsTransmissionRetired { - transfer_id: retired_transfer, - image_id: retired_image, - } if retired_transfer == transfer_id && retired_image == image_id - )); - assert!(!server.app.pane_graphics.slots.contains_key(&key)); - assert!(response_rx.recv().is_err()); -} - -#[cfg(unix)] -#[test] -fn pane_removal_and_shutdown_drop_direct_without_ack() { - let setups: [fn(&mut HeadlessServer); 2] = [ - |server| server.app.state.workspaces.clear(), - |server| server.shutting_down = true, - ]; - for setup in setups { - let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - let (transfer_id, image_id) = direct_ids(&server, &key); - setup(&mut server); - assert!(!server.complete_direct_graphics(7, transfer_id, image_id, true)); - assert!(response_rx.recv().is_err()); - assert!(!server.app.pane_graphics.slots.contains_key(&key)); - } -} - -#[cfg(unix)] -#[test] -fn timeout_retires_stream_without_producer_ack() { - let (mut server, key, response_rx) = direct_gate_server(&[1, 2, 3, 4]); - add_direct_client(&mut server, 7); - server - .app - .pane_graphics - .slots - .get_mut(&key) - .unwrap() - .direct_gate - .as_mut() - .unwrap() - .deadline = std::time::Instant::now() - Duration::from_millis(1); - - assert!(server.expire_direct_graphics(std::time::Instant::now())); - - assert!(response_rx.recv().is_err()); - assert!(!server.app.pane_graphics.slots.contains_key(&key)); - assert!(!server.clients[&7].direct_graphics); - assert!(server.clients[&7].pixel_mouse); -} diff --git a/src/server/headless/tests/pane_move.rs b/src/server/headless/tests/pane_move.rs index 789b144f..99b39c8b 100644 --- a/src/server/headless/tests/pane_move.rs +++ b/src/server/headless/tests/pane_move.rs @@ -29,7 +29,6 @@ fn public_move( }, respond_to, response_write_complete: None, - stream_active: None, }); let response = response_rx.recv().expect("pane move response"); match serde_json::from_str::<SuccessResponse>(&response) { diff --git a/src/server/render_stream.rs b/src/server/render_stream.rs index db613e96..ee874e52 100644 --- a/src/server/render_stream.rs +++ b/src/server/render_stream.rs @@ -7,7 +7,7 @@ use crate::app::state::AppState; use crate::protocol::render_ansi::{BlitEncoder, EncodedBlit}; use crate::protocol::{ CursorState, FrameData, PaneSurfaceFrame, PaneSurfacePatch, RenderEncoding, ServerMessage, - TerminalFrame, + SurfaceGraphicsAssetKey, SurfaceGraphicsScene, TerminalFrame, }; use crate::terminal::TerminalRuntimeRegistry; @@ -152,9 +152,18 @@ impl ClientRenderState { } } + #[cfg(test)] pub(crate) fn prepare_pane_surface( &mut self, + surface: PaneSurfaceFrame, + ) -> Option<PreparedRender> { + self.prepare_pane_surface_with_file(surface, false) + } + + pub(crate) fn prepare_pane_surface_with_file( + &mut self, mut surface: PaneSurfaceFrame, + has_file_upload: bool, ) -> Option<PreparedRender> { let Self::Semantic { last_surface, @@ -166,7 +175,8 @@ impl ClientRenderState { else { return None; }; - if !*recompute_pending + if !has_file_upload + && !*recompute_pending && surface.graphics.assets.is_empty() && last_surface.as_deref().is_some_and(|last| { last.projection_revision == surface.projection_revision @@ -182,6 +192,7 @@ impl ClientRenderState { } surface.surface_revision = surface_revision.saturating_add(1); let assets = std::mem::take(&mut surface.graphics.assets); + let queued_graphics_assets = assets.iter().map(|asset| asset.key.clone()).collect(); let committed_surface = surface.clone(); surface.graphics.assets = assets; let mut message = ServerMessage::PaneSurface(surface); @@ -217,6 +228,7 @@ impl ClientRenderState { Some(PreparedRender::Semantic { message: delta.or(reused).unwrap_or(message), committed_surface: Box::new(committed_surface), + queued_graphics_assets, }) } @@ -342,6 +354,7 @@ pub(crate) enum PreparedRender { Semantic { message: ServerMessage, committed_surface: Box<PaneSurfaceFrame>, + queued_graphics_assets: Vec<SurfaceGraphicsAssetKey>, }, SemanticPatch { message: ServerMessage, @@ -362,19 +375,56 @@ impl PreparedRender { } } - pub(crate) fn strip_pane_surface_assets(&mut self) -> bool { + /// Graphics metadata represented by this semantic update plus only the + /// asset keys whose pixel payloads were queued. This is independent of the + /// selected wire codec and avoids cloning asset byte vectors. + pub(crate) fn queued_surface_graphics( + &self, + ) -> Option<(&SurfaceGraphicsScene, &[SurfaceGraphicsAssetKey])> { + match self { + Self::Semantic { + committed_surface, + queued_graphics_assets, + .. + } => Some((&committed_surface.graphics, queued_graphics_assets)), + Self::SemanticPatch { .. } | Self::TerminalAnsi { .. } => None, + } + } + + pub(crate) fn has_queued_surface_assets(&self) -> bool { + matches!(self, Self::Semantic { queued_graphics_assets, .. } if !queued_graphics_assets.is_empty()) + } + + /// Removes the largest inline payload from a full semantic surface while + /// preserving placement metadata. Largest-first guarantees that a fitting + /// smaller asset is not discarded behind an oversized one. Equal sizes use + /// deterministic scene order. Encoded delta/reuse messages return `None`; callers + /// can invalidate that baseline and retry as a full surface. + pub(crate) fn pop_pane_surface_asset(&mut self) -> Option<SurfaceGraphicsAssetKey> { let Self::Semantic { message: ServerMessage::PaneSurface(surface), + queued_graphics_assets, .. } = self else { - return false; + return None; }; - if surface.graphics.assets.is_empty() { - return false; + let index = surface + .graphics + .assets + .iter() + .enumerate() + .max_by_key(|(index, asset)| (asset.data.len(), *index))? + .0; + let asset = surface.graphics.assets.remove(index); + let key = asset.key; + if let Some(index) = queued_graphics_assets + .iter() + .position(|queued| *queued == key) + { + queued_graphics_assets.remove(index); } - surface.graphics.assets.clear(); - true + Some(key) } } @@ -731,6 +781,31 @@ mod tests { } #[test] + fn deferred_file_upload_keeps_identical_metadata_and_retries_without_committing() { + for reuse in [false, true] { + let mut state = ClientRenderState::new(RenderEncoding::SemanticFrame); + state.enable_surface_reuse(reuse); + let surface = popup_surface("native"); + let first = state.prepare_pane_surface(surface.clone()).unwrap(); + state.commit_sent_frame(first); + assert!(state.prepare_pane_surface(surface.clone()).is_none()); + let file = state + .prepare_pane_surface_with_file(surface.clone(), true) + .unwrap(); + let retry = state + .prepare_pane_surface_with_file(surface.clone(), true) + .unwrap(); + let config = bincode::config::standard(); + assert_eq!( + bincode::serde::encode_to_vec(file.message(), config).unwrap(), + bincode::serde::encode_to_vec(retry.message(), config).unwrap() + ); + state.commit_sent_frame(retry); + assert!(state.prepare_pane_surface(surface).is_none()); + } + } + + #[test] fn popup_only_surface_changes_are_not_deduplicated() { let mut state = ClientRenderState::new(RenderEncoding::SemanticFrame); let prepared = state diff --git a/src/terminal/runtime.rs b/src/terminal/runtime.rs index 3a460312..ce7cab92 100644 --- a/src/terminal/runtime.rs +++ b/src/terminal/runtime.rs @@ -593,6 +593,11 @@ impl TerminalRuntime { #[cfg(test)] impl TerminalRuntime { + #[cfg(unix)] + pub(crate) fn test_enable_kitty_source_forwarding(&self) { + self.0.test_enable_kitty_source_forwarding(); + } + pub(crate) fn test_contend_during_dirty_collection( &self, bytes: Vec<u8>, diff --git a/vendor/libghostty-vt.patches.md b/vendor/libghostty-vt.patches.md index cd7f44f6..351b9560 100644 --- a/vendor/libghostty-vt.patches.md +++ b/vendor/libghostty-vt.patches.md @@ -170,3 +170,61 @@ just test-one clear_pane just maintenance-test just check ``` + +## 0007 experimental encoded PNG and immutable source retention + +status: active + +patch: `vendor/patches/libghostty-vt/0007-experimental-png-retention.patch` + +herdr issue: none; maintainer-directed native Kitty forwarding experiment + +upstream discussion: not opened + +upstream pr: not opened + +vendored base: `44f2a44df7e8c4a0c6df3f7d872ef3d7ead88e51` + +local files: + +- `vendor/libghostty-vt/include/ghostty/vt/terminal.h` +- `vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h` +- `vendor/libghostty-vt/src/terminal/c/terminal.zig` +- `vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig` +- `vendor/libghostty-vt/src/terminal/kitty/graphics_image.zig` +- `vendor/libghostty-vt/src/terminal/kitty/graphics_exec.zig` +- `vendor/libghostty-vt/src/terminal/kitty/graphics_storage.zig` + +reason: An explicitly enabled embedding mode retains structurally validated, +quiet PNG uploads as encoded bytes, avoiding pixel decoding before forwarding +through a multiplexer. The existing raw getters retain their meaning; separate +getters expose retained PNG bytes. Queries and response-bearing uploads retain +full validation, and animation operations materialize pixels transactionally. +Storage reserves both encoded bytes and expected decoded size. + +This is default-off and experimental: CRC-valid corrupt compressed pixels may +be rejected later than in normal mode, including after placement. Quiet mode +suppresses replies, not validation semantics; this patch is not a claim of full +protocol-equivalent transparent forwarding. Herdr exercises this mode only in +tests; production PNG uploads retain full decoding and validation. + +A separate default-off snapshot callback retains host-owned immutable raw RGBA +file backing before reading pixels. Herdr installs this callback automatically +on Linux, using same-filesystem CoW snapshots. It never retains a mutable producer pathname. Unsupported snapshots +use the original loader; animation materializes pixels transactionally. Backing +ownership and bounded reads are explicit in the embedding ABI. + +remove when: upstream provides an equivalent opt-in owned encoded-image +representation and immutable host-backed raw sources with bounded storage, +strict query handling and lazy pixel materialization, or this experiment is retired. + +verification: + +```sh +just test-one native_source +just test-one png_forward_tests +just test-one kitty_png_replacement +just test-one kitty_file_image_survives +(cd vendor/libghostty-vt && zig build test-lib-vt -Dtest-filter='experimental PNG') +just check +``` diff --git a/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h b/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h index 99cee60a..8dad8086 100644 --- a/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h +++ b/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h @@ -391,12 +391,13 @@ typedef enum GHOSTTY_ENUM_TYPED { /** * Borrowed pointer to the raw pixel data. Valid as long as the * underlying terminal is not mutated. Returns GHOSTTY_NO_VALUE when - * the image metadata is resident but its pixel payload is pending. + * the image metadata is resident but its pixel payload is pending, or + * when the experimental PRESERVE_PNG option retained encoded PNG data. * * The data is always fully decoded, uncompressed pixels in the * format reported by GHOSTTY_KITTY_IMAGE_DATA_FORMAT: zlib payloads - * are inflated and PNG payloads are decoded to RGBA at transmission - * time, before the image is stored. Consumers can upload this + * are inflated and PNG payloads are normally decoded to RGBA at transmission + * time, before the image is stored. Consumers can upload returned pixels * directly to the GPU without any decode step. * * For an animated image (Kitty graphics animation, actions a=f/a=a) @@ -412,7 +413,7 @@ typedef enum GHOSTTY_ENUM_TYPED { /** * Length of the raw pixel data in bytes. Always equal to * width * height * bytes-per-pixel for the reported format. For a - * pending image, this is the expected length reserved against the + * pending or encoded PNG image, this is the expected decoded length reserved against the * storage limit even though DATA_PTR is not available yet. * * Output type: size_t * @@ -439,6 +440,17 @@ typedef enum GHOSTTY_ENUM_TYPED { */ GHOSTTY_KITTY_IMAGE_DATA_GENERATION = 9, + /** Owned encoded PNG bytes, borrowed until the next mutation. NO_VALUE + * unless experimentally retained. Output type: const uint8_t **. + */ + GHOSTTY_KITTY_IMAGE_DATA_ENCODED_PNG_PTR = 10, + /** Encoded PNG length; NO_VALUE unless retained. Output type: size_t *. */ + GHOSTTY_KITTY_IMAGE_DATA_ENCODED_PNG_LEN = 11, + /** Borrowed void* backing context; valid only while image is unchanged. */ + GHOSTTY_KITTY_IMAGE_DATA_FILE_CONTEXT = 12, + /** uint64_t immutable revision fingerprint; never reads image pixels. */ + GHOSTTY_KITTY_IMAGE_DATA_FILE_IDENTITY = 13, + GHOSTTY_KITTY_IMAGE_DATA_MAX_VALUE = GHOSTTY_ENUM_MAX_VALUE, } GhosttyKittyGraphicsImageData; diff --git a/vendor/libghostty-vt/include/ghostty/vt/terminal.h b/vendor/libghostty-vt/include/ghostty/vt/terminal.h index 81f9ebd0..6404af53 100644 --- a/vendor/libghostty-vt/include/ghostty/vt/terminal.h +++ b/vendor/libghostty-vt/include/ghostty/vt/terminal.h @@ -27,6 +27,39 @@ extern "C" { #endif +/** Borrowed opened descriptor; size is sizeof this request. No path is supplied. */ +typedef struct { + size_t size; + int64_t fd; + uint64_t expected_len; +} GhosttyKittyImageSnapshotFileRequest; + +typedef bool (*GhosttyKittyImageFileReadFn)(void *context, uint8_t *dest, size_t len); +typedef void (*GhosttyKittyImageFileReleaseFn)(void *context); + +typedef struct { + void *context; + uint64_t identity; + size_t len; + GhosttyKittyImageFileReadFn read; + GhosttyKittyImageFileReleaseFn release; +} GhosttyKittyImageFileBacking; + +/** Optional synchronous callback, default NULL. Verify exact descriptor size + * and snapshot immutable bytes before returning true. True transfers one context + * reference; false transfers nothing. Do not close the borrowed descriptor or + * reenter the terminal. Both read and release MUST be non-NULL on success. + * Read must fill exactly len bytes; release is called once. An incomplete + * backing is rejected and ordinary file loading is used instead. If release + * is provided, rejection calls it once; without release, the host remains + * responsible for cleanup (the terminal cannot release that context). + * identity identifies the immutable revision, not a pathname or source inode. + */ +typedef bool (*GhosttyKittyImageSnapshotFileFn)( + GhosttyTerminal terminal, void *userdata, + const GhosttyKittyImageSnapshotFileRequest *request, + GhosttyKittyImageFileBacking *out_backing); + /** @defgroup terminal Terminal * * Complete terminal emulator state and rendering. @@ -1545,6 +1578,14 @@ typedef enum GHOSTTY_ENUM_TYPED { * Input type: size_t* */ GHOSTTY_TERMINAL_OPT_CLIPBOARD_WRITE_MAX_BYTES = 39, + + /** Experimental, default-off PNG retention for quiet base transmissions. + * Input type: bool*. NULL is a no-op. Corrupt compressed pixels may only + * be detected when an animation operation materializes the image. + */ + GHOSTTY_TERMINAL_OPT_KITTY_IMAGE_PRESERVE_PNG = 40, + /** Optional synchronous GhosttyKittyImageSnapshotFileFn; NULL disables. */ + GHOSTTY_TERMINAL_OPT_KITTY_IMAGE_SNAPSHOT_FILE = 41, GHOSTTY_TERMINAL_OPT_MAX_VALUE = GHOSTTY_ENUM_MAX_VALUE, } GhosttyTerminalOption; diff --git a/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig b/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig index 2bd5e862..c66b13be 100644 --- a/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig +++ b/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig @@ -183,6 +183,10 @@ pub const ImageData = enum(c_int) { data_ptr = 7, data_len = 8, generation = 9, + encoded_png_ptr = 10, + encoded_png_len = 11, + file_context = 12, + file_identity = 13, pub fn OutType(comptime self: ImageData) type { return switch (self) { @@ -190,9 +194,10 @@ pub const ImageData = enum(c_int) { .id, .number, .width, .height => u32, .format => ImageFormat, .compression => ImageCompression, - .data_ptr => [*]const u8, - .data_len => usize, - .generation => u64, + .data_ptr, .encoded_png_ptr => [*]const u8, + .data_len, .encoded_png_len => usize, + .generation, .file_identity => u64, + .file_context => ?*anyopaque, }; } }; @@ -266,7 +271,26 @@ fn imageGetTyped( // data; the image generation changes whenever the current // frame does, so generation-keyed caches stay coherent. .data_ptr => out.* = (image.renderData().bytes() orelse return .no_value).ptr, - .data_len => out.* = image.renderData().len(), + .data_len => out.* = switch (image.renderData()) { + .encoded_png => |png| png.decoded_len, + else => image.renderData().len(), + }, + .encoded_png_ptr => out.* = switch (image.data) { + .encoded_png => |png| png.bytes.ptr, + else => return .no_value, + }, + .encoded_png_len => out.* = switch (image.data) { + .encoded_png => |png| png.bytes.len, + else => return .no_value, + }, + .file_context => out.* = switch (image.data) { + .native_file => |file| file.context, + else => return .no_value, + }, + .file_identity => out.* = switch (image.data) { + .native_file => |file| file.identity, + else => return .no_value, + }, .generation => out.* = image.generation, } diff --git a/vendor/libghostty-vt/src/terminal/c/terminal.zig b/vendor/libghostty-vt/src/terminal/c/terminal.zig index 91bbe310..e825183e 100644 --- a/vendor/libghostty-vt/src/terminal/c/terminal.zig +++ b/vendor/libghostty-vt/src/terminal/c/terminal.zig @@ -13,6 +13,7 @@ const ScreenSet = @import("../ScreenSet.zig"); const PageList = @import("../PageList.zig"); const apc = @import("../apc.zig"); const kitty = @import("../kitty/key.zig"); +const kitty_image = @import("../kitty/graphics_image.zig"); const kitty_gfx_c = @import("kitty_graphics.zig"); const modes = @import("../modes.zig"); const point = @import("../point.zig"); @@ -257,6 +258,7 @@ const Effects = struct { clipboard_write: ?ClipboardWriteFn = null, clipboard_read: ?ClipboardReadFn = null, unknown_sequence: ?UnknownSequenceFn = null, + snapshot_file: ?SnapshotFileFn = null, /// Scratch buffer for DA1 feature codes. The device attributes /// trampoline converts C feature codes into this buffer and returns @@ -265,6 +267,14 @@ const Effects = struct { /// (`reportDeviceAttributes`) reads it before any re-entrant call. da_features_buf: [64]device_attributes.Primary.Feature = undefined, + pub const SnapshotFileFn = *const fn (Terminal, ?*anyopaque, *const kitty_image.SnapshotFileRequest, *kitty_image.FileBacking) callconv(lib.calling_conv) bool; + + fn snapshotFile(context: ?*anyopaque, request: *const kitty_image.SnapshotFileRequest, out: *kitty_image.FileBacking) bool { + const wrapper: *TerminalWrapper = @ptrCast(@alignCast(context.?)); + const callback = wrapper.effects.snapshot_file orelse return false; + return callback(@ptrCast(wrapper), wrapper.effects.userdata, request, out); + } + /// C function pointer type for the write_pty callback. pub const WritePtyFn = *const fn (Terminal, ?*anyopaque, [*]const u8, usize) callconv(lib.calling_conv) void; @@ -1174,12 +1184,15 @@ pub const Option = enum(c_int) { terminfo_name = 37, clipboard_read = 38, clipboard_write_max_bytes = 39, + kitty_image_preserve_png = 40, + kitty_image_snapshot_file = 41, /// Input type expected for setting the option. pub fn InType(comptime self: Option) type { return switch (self) { .userdata => ?*const anyopaque, .write_pty => ?Effects.WritePtyFn, + .kitty_image_snapshot_file => ?Effects.SnapshotFileFn, .bell => ?Effects.BellFn, .color_scheme => ?Effects.ColorSchemeFn, .desktop_notification => ?Effects.DesktopNotificationFn, @@ -1197,6 +1210,7 @@ pub const Option = enum(c_int) { .color_foreground, .color_background, .color_cursor => ?*const color.RGB.C, .color_palette => ?*const color.PaletteC, .kitty_image_storage_limit => ?*const u64, + .kitty_image_preserve_png, .kitty_image_medium_file, .kitty_image_medium_shared_mem, .glyph_protocol, @@ -1250,6 +1264,17 @@ fn setTyped( switch (option) { .userdata => wrapper.effects.userdata = @constCast(value), .write_pty => wrapper.effects.write_pty = value, + .kitty_image_snapshot_file => { + if (comptime !build_options.kitty_graphics) return .success; + wrapper.effects.snapshot_file = value; + var it = wrapper.terminal.screens.all.iterator(); + while (it.next()) |entry| { + entry.value.*.kitty_images.image_limits.snapshot_file = if (value != null) + .{ .context = @ptrCast(wrapper), .callback = &Effects.snapshotFile } + else + null; + } + }, .bell => wrapper.effects.bell = value, .color_scheme => wrapper.effects.color_scheme = value, .desktop_notification => wrapper.effects.desktop_notification = value, @@ -1330,6 +1355,7 @@ fn setTyped( screen.kitty_images.setLimit(screen.io, screen.alloc, screen, limit); } }, + .kitty_image_preserve_png, .kitty_image_medium_file, .kitty_image_medium_shared_mem, => { @@ -1339,6 +1365,7 @@ fn setTyped( while (it.next()) |entry| { const screen = entry.value.*; switch (option) { + .kitty_image_preserve_png => screen.kitty_images.image_limits.preserve_png = val, .kitty_image_medium_file => screen.kitty_images.image_limits.file = val, .kitty_image_medium_shared_mem => screen.kitty_images.image_limits.shared_memory = val, else => unreachable, diff --git a/vendor/libghostty-vt/src/terminal/kitty/graphics_exec.zig b/vendor/libghostty-vt/src/terminal/kitty/graphics_exec.zig index b1b968b2..203190f1 100644 --- a/vendor/libghostty-vt/src/terminal/kitty/graphics_exec.zig +++ b/vendor/libghostty-vt/src/terminal/kitty/graphics_exec.zig @@ -509,6 +509,11 @@ fn transmitAnimationFrame( result.message = "ENOENT: image not found"; return result; }; + storage.materializeBacking(io, alloc, img) catch |err| { + encodeError(&result, err); + return result; + }; + result.id = img.id; var loading = LoadingImage.init( @@ -773,6 +778,10 @@ fn controlAnimation( result.message = "ENOENT: image not found"; return result; }; + storage.materializeBacking(io, alloc, img) catch |err| { + encodeError(&result, err); + return result; + }; const anim = ensureAnimation(alloc, img) catch |err| { encodeError(&result, err); @@ -857,6 +866,11 @@ fn composeAnimation( result.message = "ENOENT: image not found"; return result; }; + storage.materializeBacking(io, alloc, img) catch |err| { + encodeError(&result, err); + return result; + }; + result.id = img.id; if (img.data.bytes() == null) { result.message = "EINVAL: image data incomplete"; @@ -1058,7 +1072,10 @@ fn loadAndAddImage( // loading.debugDump() catch unreachable; // Validate and store our image - var img = try loading.complete(alloc); + var img = if (storage.image_limits.preserve_png) + try loading.completePreservingPng(alloc) + else + try loading.complete(alloc); errdefer img.deinit(alloc); try storage.addImage(io, alloc, terminal.screens.active, img); @@ -3601,3 +3618,87 @@ test "kittygfx animation: control negative gap makes frame gapless" { const anim = storage.imagePtrByIdOrNumber(1, 0).?.animation.?; try testing.expectEqual(@as(u32, 0), anim.frames.items[0].gap_ms); } + +test "kittygfx experimental PNG retention quiet inheritance and transactional materialization" { + const testing = std.testing; + const alloc = testing.allocator; + const io = testing.io; + const sys = @import("../sys.zig"); + const Decoder = struct { + var calls: usize = 0; + var fail: bool = false; + fn decode(a: Allocator, _: []const u8) sys.DecodeError!sys.Image { + calls += 1; + if (fail) return error.InvalidData; + const pixels = try a.alloc(u8, 50 * 76 * 4); + @memset(pixels, 0); + return .{ .width = 50, .height = 76, .data = pixels }; + } + }; + const original = sys.decode_png; + defer sys.decode_png = original; + sys.decode_png = &Decoder.decode; + Decoder.calls = 0; + Decoder.fail = false; + const png = @embedFile("testdata/image-png-none-50x76-2147483647-raw.data"); + var t = try Terminal.init(io, alloc, .{ .rows = 5, .cols = 5 }); + defer t.deinit(alloc); + const storage = &t.screens.active.kitty_images; + var cmd: Command = .{ + .quiet = .failures, + .control = .{ .transmit = .{ .format = .png, .image_id = 1 } }, + .data = @constCast(png), + }; + // Default off is the normal decoder path even for q=2. + try testing.expect(!storage.image_limits.preserve_png); + _ = execute(io, alloc, &t, &cmd); + try testing.expectEqual(@as(usize, 1), Decoder.calls); + try testing.expect(storage.imageById(1).?.data == .complete); + storage.image_limits.preserve_png = true; + for ([_]Command.Quiet{ .no, .ok }) |quiet| { + cmd.quiet = quiet; + _ = execute(io, alloc, &t, &cmd); + try testing.expect(storage.imageById(1).?.data == .complete); + } + try testing.expectEqual(@as(usize, 3), Decoder.calls); + cmd.quiet = .failures; + cmd.control = .{ .query = .{ .format = .png, .image_id = 2 } }; + _ = execute(io, alloc, &t, &cmd); + try testing.expectEqual(@as(usize, 4), Decoder.calls); + try testing.expect(storage.imageById(2) == null); + cmd.control = .{ .transmit = .{ .format = .png, .image_id = 1, .more_chunks = true } }; + cmd.data = @constCast(png[0..33]); + _ = execute(io, alloc, &t, &cmd); + cmd.quiet = .no; + cmd.control = .{ .transmit = .{} }; + cmd.data = @constCast(png[33..]); + _ = execute(io, alloc, &t, &cmd); + try testing.expectEqual(@as(usize, 4), Decoder.calls); + const img = storage.imagePtrByIdOrNumber(1, 0).?; + try testing.expectEqualSlices(u8, png, img.data.encoded_png.bytes); + try testing.expect(img.data.bytes() == null); + try testing.expectEqual(png.len + 50 * 76 * 4, storage.total_bytes); + const generation = img.generation; + Decoder.fail = true; + const animate: Command = .{ .control = .{ .control_animation = .{ .image_id = 1 } } }; + _ = execute(io, alloc, &t, &animate); + try testing.expectEqual(generation, img.generation); + try testing.expect(img.data == .encoded_png); + try testing.expectEqual(png.len + 50 * 76 * 4, storage.total_bytes); + Decoder.fail = false; + _ = execute(io, alloc, &t, &animate); + try testing.expect(img.generation > generation); + try testing.expect(img.data == .complete); + try testing.expectEqual(@as(usize, 50 * 76 * 4), storage.total_bytes); + + // The reservation includes both the encoded and decoded representation. + storage.setLimit(io, alloc, t.screens.active, 50 * 76 * 4); + cmd.quiet = .failures; + cmd.control = .{ .transmit = .{ .format = .png, .image_id = 3 } }; + cmd.data = @constCast(png); + const calls = Decoder.calls; + _ = execute(io, alloc, &t, &cmd); + try testing.expectEqual(calls, Decoder.calls); + try testing.expect(storage.imageById(3) == null); + try testing.expect(storage.loading == null); +} diff --git a/vendor/libghostty-vt/src/terminal/kitty/graphics_image.zig b/vendor/libghostty-vt/src/terminal/kitty/graphics_image.zig index e411bafe..c841c48d 100644 --- a/vendor/libghostty-vt/src/terminal/kitty/graphics_image.zig +++ b/vendor/libghostty-vt/src/terminal/kitty/graphics_image.zig @@ -21,6 +21,24 @@ const max_dimension = 10000; /// Maximum size in bytes, taken from Kitty. const max_size = 400 * 1024 * 1024; // 400MB +/// Host-owned immutable raw RGBA snapshot. A successful callback transfers one reference. +pub const FileBacking = extern struct { + context: ?*anyopaque, + identity: u64, + len: usize, + read: ?*const fn (?*anyopaque, [*]u8, usize) callconv(.c) bool, + release: ?*const fn (?*anyopaque) callconv(.c) void, +}; +pub const SnapshotFileRequest = extern struct { + size: usize = @sizeOf(SnapshotFileRequest), + fd: i64, + expected_len: u64, +}; +pub const SnapshotFileHook = struct { + context: ?*anyopaque, + callback: *const fn (?*anyopaque, *const SnapshotFileRequest, *FileBacking) bool, +}; + /// An image that is still being loaded. The image should be initialized /// using init on the first chunk and then addData for each subsequent /// chunk. Once all chunks have been added, complete should be called @@ -29,6 +47,7 @@ pub const LoadingImage = struct { /// The in-progress image. The first chunk must have all the metadata /// so this comes from that initially. image: Image, + snapshot_file: ?SnapshotFileHook = null, /// The data that is being built up. data: std.ArrayListUnmanaged(u8) = .empty, @@ -85,6 +104,8 @@ pub const LoadingImage = struct { disabled: void, }, shared_memory: bool, + preserve_png: bool = false, + snapshot_file: ?SnapshotFileHook = null, /// Enables all filesystem-related image transmission mediums. `path` /// is the temporary directory to expect files in when files are @@ -132,6 +153,10 @@ pub const LoadingImage = struct { .metadata = .{ .transient = t.usage.transient }, }, + .snapshot_file = if (cmd.control == .transmit or cmd.control == .transmit_and_display) + limits.snapshot_file + else + null, .display = cmd.display(), .quiet = cmd.quiet, .response = .{ @@ -427,6 +452,16 @@ pub const LoadingImage = struct { return error.InvalidData; } + // Only whole, uncompressed base RGBA files qualify. The host must + // verify the descriptor size and create an immutable private snapshot. + if (comptime builtin.os.tag != .windows and builtin.os.tag != .wasi and builtin.os.tag != .freestanding) { + if (self.snapshot_file) |hook| { + if (nativeFileExpectedLen(t)) |expected| { + if (try self.trySnapshotFile(hook, .{ .fd = @intCast(file.handle), .expected_len = expected })) return; + } + } + } + var buf: [4096]u8 = undefined; var buf_reader = file.reader(io, &buf); if (t.offset > 0) { @@ -468,6 +503,32 @@ pub const LoadingImage = struct { self.data = .{ .items = managed.items, .capacity = managed.capacity }; } + /// Accept only complete callback records. A malformed success falls back + /// to the ordinary reader, releasing the transferred reference if possible. + fn trySnapshotFile(self: *LoadingImage, hook: SnapshotFileHook, request: SnapshotFileRequest) !bool { + var backing: FileBacking = .{ .context = null, .identity = 0, .len = 0, .read = null, .release = null }; + if (!hook.callback(hook.context, &request, &backing)) return false; + if (backing.read == null or backing.release == null) { + if (backing.release) |release| release(backing.context); + return false; + } + if (backing.len != request.expected_len) { + backing.release.?(backing.context); + return error.InvalidData; + } + self.image.data = .{ .native_file = backing }; + return true; + } + + fn nativeFileExpectedLen(t: command.Transmission) ?usize { + if (t.medium != .file or t.format != .rgba or t.compression != .none or + t.more_chunks or t.offset != 0 or t.width == 0 or t.height == 0 or + t.width > max_dimension or t.height > max_dimension) return null; + const len = @as(usize, t.width) * @as(usize, t.height) * 4; + if (len > 16 * 1024 * 1024 or (t.size != 0 and t.size != len)) return null; + return len; + } + /// Returns the canonical path of an open file after applying the file /// transmission blocklist. fn validatedFilePath(io: std.Io, file: std.Io.File, buf: []u8) ![]const u8 { @@ -569,7 +630,7 @@ pub const LoadingImage = struct { // Data length must be what we expect. const bpp = command.Transmission.formatBpp(img.format); const expected_len = img.width * img.height * bpp; - const actual_len = self.data.items.len; + const actual_len = if (img.data == .native_file) img.data.native_file.len else self.data.items.len; if (self.frame != null) { // Kitty allows animation frames to exceed their expected length // and just truncates it. Not sure if thats expected but lets @@ -592,12 +653,37 @@ pub const LoadingImage = struct { // Everything looks good, copy the image data over. var result = self.image; - result.data = .{ .complete = try self.data.toOwnedSlice(alloc) }; + if (result.data != .native_file) result.data = .{ .complete = try self.data.toOwnedSlice(alloc) }; errdefer result.deinit(alloc); self.image = .{}; return result; } + /// Experimental forwarding path: only invoked for assembled quiet base + /// transmissions. Never used by queries or animation frame loading. + pub fn completePreservingPng(self: *LoadingImage, alloc: Allocator) !Image { + if (self.image.format != .png or self.image.compression != .none or + self.quiet != .failures or self.frame != null) return self.complete(alloc); + const header = inspectPng(self.data.items) catch |err| switch (err) { + // Do not expand the deferred-corruption contract to metadata we + // have not validated. Such PNGs keep the normal decoder semantics. + error.UnsupportedPngRetention => return self.complete(alloc), + error.InvalidData => return error.InvalidData, + error.DimensionsRequired => return error.DimensionsRequired, + error.DimensionsTooLarge => return error.DimensionsTooLarge, + }; + var result = self.image; + result.width = header.width; + result.height = header.height; + result.format = .rgba; + result.data = .{ .encoded_png = .{ + .bytes = try self.data.toOwnedSlice(alloc), + .decoded_len = header.decoded_len, + } }; + self.image = .{}; + return result; + } + /// Debug function to write the data to a file. This is useful for /// capturing some test data for unit tests. pub fn debugDump(io: std.Io, self: LoadingImage) !void { @@ -768,11 +854,16 @@ pub const Image = struct { /// Expected decoded byte length for a payload that has not arrived. pending: usize, + encoded_png: struct { bytes: []const u8, decoded_len: usize }, + native_file: FileBacking, + /// Bytes reserved against the storage limit. pub fn len(self: Data) usize { return switch (self) { .complete => |data| data.len, .pending => |expected_len| expected_len, + .native_file => |file| file.len, + .encoded_png => |png| png.bytes.len + png.decoded_len, }; } @@ -780,7 +871,7 @@ pub const Image = struct { pub fn bytes(self: Data) ?[]const u8 { return switch (self) { .complete => |data| data, - .pending => null, + .pending, .encoded_png, .native_file => null, }; } @@ -792,10 +883,43 @@ pub const Image = struct { switch (self.*) { .complete => |data| if (data.len > 0) alloc.free(data), .pending => {}, + .encoded_png => |png| alloc.free(png.bytes), + .native_file => |file| if (file.release) |release| release(file.context), } } }; + /// Transactional: a failed host read leaves the authoritative backing owned. + pub fn materializeFile(self: *Image, alloc: Allocator) !void { + const file = switch (self.data) { + .native_file => |file| file, + else => return, + }; + const read = file.read orelse return error.InvalidData; + const release = file.release orelse return error.InvalidData; + const bytes = try alloc.alloc(u8, file.len); + errdefer alloc.free(bytes); + if (!read(file.context, bytes.ptr, bytes.len)) return error.InvalidData; + self.data = .{ .complete = bytes }; + release(file.context); + } + + /// Transactional: neither ownership nor metadata changes on failure. + pub fn materializePng(self: *Image, alloc: Allocator) !void { + const png = switch (self.data) { + .encoded_png => |png| png, + else => return, + }; + const decode = sys.decode_png orelse return error.UnsupportedFormat; + var limited: LimitedAllocator = .init(alloc, max_size); + const result = try decode(limited.allocator(), png.bytes); + errdefer alloc.free(result.data); + if (result.width != self.width or result.height != self.height or + result.data.len != png.decoded_len) return error.InvalidData; + alloc.free(png.bytes); + self.data = .{ .complete = result.data }; + } + pub fn deinit(self: *Image, alloc: Allocator) void { self.data.deinit(alloc); if (self.animation) |anim| { @@ -2159,3 +2283,404 @@ test "limits: temporary file medium allowed by limits" { ); defer loading.deinit(alloc); } + +const PngHeader = struct { width: u32, height: u32, decoded_len: usize }; + +/// Bounded structural validation, deliberately without inflating IDAT. +/// CRC-valid but corrupt compressed pixels can fail later in this experiment. +fn inspectPng(bytes: []const u8) !PngHeader { + if (bytes.len > max_size or bytes.len < 33 or + !std.mem.eql(u8, bytes[0..8], "\x89PNG\r\n\x1a\n")) return error.InvalidData; + var offset: usize = 8; + var header: ?PngHeader = null; + var idat = false; + var ended_idat = false; + var palette = false; + var indexed = false; + var color_type: u8 = 0; + var bit_depth: u8 = 0; + var palette_entries: usize = 0; + var transparency = false; + while (offset < bytes.len) { + if (bytes.len - offset < 12) return error.InvalidData; + const len: usize = std.mem.readInt(u32, bytes[offset..][0..4], .big); + if (len > bytes.len - offset - 12) return error.InvalidData; + const kind = bytes[offset + 4 ..][0..4]; + for (kind) |ch| if (!std.ascii.isAlphabetic(ch)) return error.InvalidData; + if (kind[2] & 0x20 != 0) return error.InvalidData; + const body = bytes[offset + 8 ..][0..len]; + const crc = std.mem.readInt(u32, bytes[offset + 8 + len ..][0..4], .big); + if (std.hash.crc.Crc32IsoHdlc.hash(bytes[offset + 4 ..][0 .. len + 4]) != crc) + return error.InvalidData; + offset += len + 12; + if (std.mem.eql(u8, kind, "IHDR")) { + if (header != null or len != 13) return error.InvalidData; + const width = std.mem.readInt(u32, body[0..4], .big); + const height = std.mem.readInt(u32, body[4..8], .big); + if (width == 0 or height == 0) return error.DimensionsRequired; + if (width > max_dimension or height > max_dimension) return error.DimensionsTooLarge; + const depth = body[8]; + const supported = switch (body[9]) { + 0 => depth == 1 or depth == 2 or depth == 4 or depth == 8 or depth == 16, + 2, 4, 6 => depth == 8 or depth == 16, + 3 => depth == 1 or depth == 2 or depth == 4 or depth == 8, + else => false, + }; + if (!supported or body[10] != 0 or body[11] != 0 or body[12] > 1) + return error.InvalidData; + color_type = body[9]; + bit_depth = depth; + indexed = color_type == 3; + const pixels = std.math.mul(usize, width, height) catch return error.InvalidData; + const decoded_len = std.math.mul(usize, pixels, 4) catch return error.InvalidData; + if (decoded_len > max_size) return error.InvalidData; + header = .{ .width = width, .height = height, .decoded_len = decoded_len }; + } else { + if (header == null) return error.InvalidData; + if (std.mem.eql(u8, kind, "IDAT")) { + if (ended_idat or (indexed and !palette)) return error.InvalidData; + idat = true; + } else { + if (idat) ended_idat = true; + if (std.mem.eql(u8, kind, "IEND")) { + if (!idat or len != 0 or offset != bytes.len) return error.InvalidData; + return header.?; + } else if (std.mem.eql(u8, kind, "PLTE")) { + if (palette or idat or transparency or len == 0 or len > 768 or len % 3 != 0 or + color_type == 0 or color_type == 4) return error.InvalidData; + palette_entries = len / 3; + if (indexed and palette_entries > (@as(usize, 1) << @intCast(bit_depth))) + return error.InvalidData; + palette = true; + } else if (std.mem.eql(u8, kind, "tRNS")) { + if (transparency or idat) return error.InvalidData; + switch (color_type) { + 0, 2 => { + const expected: usize = if (color_type == 0) 2 else 6; + if (len != expected) return error.InvalidData; + const max_sample: u32 = (@as(u32, 1) << @intCast(bit_depth)) - 1; + var i: usize = 0; + while (i < len) : (i += 2) { + if (std.mem.readInt(u16, body[i..][0..2], .big) > max_sample) + return error.InvalidData; + } + }, + 3 => if (!palette or len == 0 or len > palette_entries) + return error.InvalidData, + // Alpha-bearing color types must not carry tRNS. + else => return error.InvalidData, + } + transparency = true; + } else if (kind[0] & 0x20 == 0) { + return error.InvalidData; + } else { + // sBIT, bKGD, color profiles, APNG, text, and any other + // ancillary extensions are deliberately not eligible. + // The strict decoder handles their validity/semantics. + return error.UnsupportedPngRetention; + } + } + } + } + return error.InvalidData; +} + +test "experimental PNG inspection validates CRC and complete structure" { + const testing = std.testing; + const original = @embedFile("testdata/image-png-none-50x76-2147483647-raw.data"); + const header = try inspectPng(original); + try testing.expectEqual(@as(u32, 50), header.width); + try testing.expectEqual(@as(u32, 76), header.height); + try testing.expectEqual(@as(usize, 50 * 76 * 4), header.decoded_len); + try testing.expectError(error.InvalidData, inspectPng(original[0 .. original.len - 1])); + var corrupt = original.*; + corrupt[29] ^= 1; // IHDR CRC + try testing.expectError(error.InvalidData, inspectPng(&corrupt)); + corrupt = original.*; + corrupt[0] = 0; + try testing.expectError(error.InvalidData, inspectPng(&corrupt)); +} + +test "experimental PNG inspection transparency and palette metadata" { + const testing = std.testing; + const Chunk = struct { kind: *const [4]u8, body: []const u8 }; + const Helper = struct { + fn append(out: *std.ArrayList(u8), kind: *const [4]u8, body: []const u8) !void { + var word: [4]u8 = undefined; + std.mem.writeInt(u32, &word, @intCast(body.len), .big); + try out.appendSlice(testing.allocator, &word); + const crc_start = out.items.len; + try out.appendSlice(testing.allocator, kind); + try out.appendSlice(testing.allocator, body); + std.mem.writeInt(u32, &word, std.hash.crc.Crc32IsoHdlc.hash(out.items[crc_start..]), .big); + try out.appendSlice(testing.allocator, &word); + } + fn make(color: u8, depth: u8, chunks: []const Chunk) ![]u8 { + var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(testing.allocator); + try out.appendSlice(testing.allocator, "\x89PNG\r\n\x1a\n"); + const ihdr = [_]u8{ 0, 0, 0, 1, 0, 0, 0, 1, depth, color, 0, 0, 0 }; + try append(&out, "IHDR", &ihdr); + for (chunks) |chunk| try append(&out, chunk.kind, chunk.body); + try append(&out, "IEND", ""); + return out.toOwnedSlice(testing.allocator); + } + fn check(color: u8, depth: u8, chunks: []const Chunk, expected: ?anyerror) !void { + const png = try make(color, depth, chunks); + defer testing.allocator.free(png); + if (expected) |err| { + try testing.expectError(err, inspectPng(png)); + } else { + _ = try inspectPng(png); + } + } + }; + // IDAT is intentionally opaque; this test only exercises structure. + const idat: Chunk = .{ .kind = "IDAT", .body = "" }; + const palette: Chunk = .{ .kind = "PLTE", .body = &.{ 0, 0, 0, 255, 255, 255 } }; + const alpha: Chunk = .{ .kind = "tRNS", .body = &.{128} }; + const gray: Chunk = .{ .kind = "tRNS", .body = &.{ 0, 1 } }; + const rgb: Chunk = .{ .kind = "tRNS", .body = &.{ 0, 1, 0, 2, 0, 3 } }; + try Helper.check(3, 1, &.{ palette, alpha, idat }, null); + try Helper.check(0, 1, &.{ gray, idat }, null); + try Helper.check(2, 8, &.{ rgb, idat }, null); + try Helper.check(2, 8, &.{ palette, rgb, idat }, null); + try Helper.check(3, 1, &.{ alpha, palette, idat }, error.InvalidData); + try Helper.check(3, 1, &.{ palette, idat, alpha }, error.InvalidData); + try Helper.check(3, 1, &.{ palette, alpha, alpha, idat }, error.InvalidData); + try Helper.check(0, 8, &.{ alpha, idat }, error.InvalidData); + try Helper.check(2, 8, &.{ gray, idat }, error.InvalidData); + try Helper.check(4, 8, &.{ gray, idat }, error.InvalidData); + try Helper.check(6, 8, &.{ rgb, idat }, error.InvalidData); + try Helper.check(0, 8, &.{ palette, idat }, error.InvalidData); + try Helper.check(4, 8, &.{ palette, idat }, error.InvalidData); + try Helper.check(2, 8, &.{ rgb, palette, idat }, error.InvalidData); + try Helper.check(3, 1, &.{ palette, .{ .kind = "tRNS", .body = "" }, idat }, error.InvalidData); + try Helper.check(3, 1, &.{ palette, .{ .kind = "tRNS", .body = &.{ 1, 2, 3 } }, idat }, error.InvalidData); + try Helper.check(3, 1, &.{ .{ .kind = "PLTE", .body = &.{ 0, 0, 0, 1, 1, 1, 2, 2, 2 } }, idat }, error.InvalidData); + try Helper.check(0, 1, &.{ .{ .kind = "tRNS", .body = &.{ 0, 2 } }, idat }, error.InvalidData); + try Helper.check(2, 8, &.{ .{ .kind = "tRNS", .body = &.{ 1, 0, 0, 0, 0, 0 } }, idat }, error.InvalidData); + try Helper.check(0, 16, &.{ .{ .kind = "tRNS", .body = &.{ 255, 255 } }, idat }, null); + for ([_]*const [4]u8{ "sBIT", "bKGD", "iCCP", "gAMA" }) |kind| { + try Helper.check(2, 8, &.{ .{ .kind = kind, .body = "" }, idat }, error.UnsupportedPngRetention); + } +} + +test "experimental PNG unsupported metadata falls back to strict decoding" { + const testing = std.testing; + const Decoder = struct { + var calls: usize = 0; + fn decode(_: Allocator, _: []const u8) sys.DecodeError!sys.Image { + calls += 1; + return error.InvalidData; + } + }; + const original_decoder = sys.decode_png; + defer sys.decode_png = original_decoder; + sys.decode_png = &Decoder.decode; + Decoder.calls = 0; + const png = @embedFile("testdata/image-png-none-50x76-2147483647-raw.data"); + var loading: LoadingImage = .{ + .image = .{ .format = .png }, + .quiet = .failures, + .temporary_directory = null, + }; + defer loading.deinit(testing.allocator); + try loading.addData(testing.allocator, png[0..33]); + // CRC-valid ancillary chunk: inspection must delegate, not retain or + // return UnsupportedPngRetention to the caller. + var chunk = [_]u8{ 0, 0, 0, 1, 's', 'B', 'I', 'T', 8, 0, 0, 0, 0 }; + std.mem.writeInt(u32, chunk[9..13], std.hash.crc.Crc32IsoHdlc.hash(chunk[4..9]), .big); + try loading.addData(testing.allocator, &chunk); + try loading.addData(testing.allocator, png[33..]); + try testing.expectError(error.InvalidData, loading.completePreservingPng(testing.allocator)); + try testing.expectEqual(@as(usize, 1), Decoder.calls); +} + +test "native file completion and transactional lazy materialization" { + const testing = std.testing; + const Fake = struct { + reads: usize = 0, + releases: usize = 0, + fail: bool = false, + fn read(ctx: ?*anyopaque, dest: [*]u8, len: usize) callconv(.c) bool { + const self: *@This() = @ptrCast(@alignCast(ctx.?)); + self.reads += 1; + if (self.fail) return false; + @memset(dest[0..len], 123); + return true; + } + fn release(ctx: ?*anyopaque) callconv(.c) void { + const self: *@This() = @ptrCast(@alignCast(ctx.?)); + self.releases += 1; + } + fn backing(self: *@This(), len: usize) FileBacking { + return .{ .context = self, .identity = 7, .len = len, .read = read, .release = release }; + } + }; + var fake: Fake = .{}; + var loading: LoadingImage = .{ + .image = .{ .width = 1, .height = 1, .format = .rgba, .data = .{ .native_file = fake.backing(4) } }, + .quiet = .no, + .temporary_directory = null, + }; + defer loading.deinit(testing.allocator); + var img = try loading.complete(testing.allocator); + defer img.deinit(testing.allocator); + try testing.expectEqual(@as(usize, 0), fake.reads); + try testing.expectEqual(@as(usize, 4), img.data.len()); + try testing.expect(!img.data.isPending()); + try testing.expect(img.data.bytes() == null); + fake.fail = true; + try testing.expectError(error.InvalidData, img.materializeFile(testing.allocator)); + try testing.expect(img.data == .native_file); + try testing.expectEqual(@as(usize, 0), fake.releases); + fake.fail = false; + try img.materializeFile(testing.allocator); + try testing.expectEqual(@as(usize, 1), fake.releases); + try testing.expectEqualSlices(u8, &.{ 123, 123, 123, 123 }, img.data.bytes().?); + + var invalid: LoadingImage = .{ + .image = .{ .width = 1, .height = 1, .format = .rgba, .data = .{ .native_file = fake.backing(3) } }, + .quiet = .no, + .temporary_directory = null, + }; + try testing.expectError(error.InvalidData, invalid.complete(testing.allocator)); + invalid.deinit(testing.allocator); + try testing.expectEqual(@as(usize, 2), fake.releases); + // q=1 also validates length without reading pixels. + var quiet_one: LoadingImage = .{ + .image = .{ .width = 1, .height = 1, .format = .rgba, .data = .{ .native_file = fake.backing(4) } }, + .quiet = .ok, + .temporary_directory = null, + }; + const reads_before = fake.reads; + var quiet_img = try quiet_one.complete(testing.allocator); + quiet_one.deinit(testing.allocator); + try testing.expectEqual(reads_before, fake.reads); + quiet_img.deinit(testing.allocator); + try testing.expectEqual(@as(usize, 3), fake.releases); + var canceled: LoadingImage = .{ + .image = .{ .data = .{ .native_file = fake.backing(4) } }, + .quiet = .failures, + .temporary_directory = null, + }; + canceled.deinit(testing.allocator); + try testing.expectEqual(@as(usize, 4), fake.releases); +} + +test "native file eligibility is whole raw RGBA only" { + const testing = std.testing; + var t: command.Transmission = .{ .medium = .file, .format = .rgba, .width = 1, .height = 2 }; + try testing.expectEqual(@as(?usize, 8), LoadingImage.nativeFileExpectedLen(t)); + t.size = 8; + try testing.expectEqual(@as(?usize, 8), LoadingImage.nativeFileExpectedLen(t)); + t.size = 7; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.size = 0; + t.more_chunks = true; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.more_chunks = false; + t.offset = 1; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.offset = 0; + t.format = .rgb; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.format = .png; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.format = .rgba; + t.medium = .temporary_file; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.medium = .shared_memory; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.medium = .file; + t.compression = .zlib_deflate; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.compression = .none; + t.width = 0; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.width = 10001; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); + t.width = 10000; + t.height = 10000; + try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); +} + +test "native file snapshot rejects missing callbacks before retaining" { + const testing = std.testing; + const Fake = struct { + reads: usize = 0, + releases: usize = 0, + missing_read: bool = true, + missing_release: bool = false, + + fn read(ctx: ?*anyopaque, dest: [*]u8, len: usize) callconv(.c) bool { + const self: *@This() = @ptrCast(@alignCast(ctx.?)); + self.reads += 1; + @memset(dest[0..len], 42); + return true; + } + + fn release(ctx: ?*anyopaque) callconv(.c) void { + const self: *@This() = @ptrCast(@alignCast(ctx.?)); + self.releases += 1; + } + + fn snapshot(ctx: ?*anyopaque, request: *const SnapshotFileRequest, out: *FileBacking) bool { + const self: *@This() = @ptrCast(@alignCast(ctx.?)); + out.* = .{ + .context = ctx, + .identity = 1, + .len = @intCast(request.expected_len), + .read = if (self.missing_read) null else &read, + .release = if (self.missing_release) null else &release, + }; + return true; + } + }; + var fake: Fake = .{}; + const hook: SnapshotFileHook = .{ .context = &fake, .callback = &Fake.snapshot }; + const request: SnapshotFileRequest = .{ .fd = -1, .expected_len = 4 }; + var loading: LoadingImage = .{ + .image = .{ .width = 1, .height = 1, .format = .rgba }, + .quiet = .no, + .temporary_directory = null, + }; + // Missing read: transferred context is immediately released, not stored. + try testing.expect(!try loading.trySnapshotFile(hook, request)); + try testing.expect(loading.image.data != .native_file); + try testing.expectEqual(@as(usize, 1), fake.releases); + + // Missing release (or both): reject without calling a null pointer. + fake.missing_read = false; + fake.missing_release = true; + try testing.expect(!try loading.trySnapshotFile(hook, request)); + try testing.expect(loading.image.data != .native_file); + fake.missing_read = true; + try testing.expect(!try loading.trySnapshotFile(hook, request)); + try testing.expect(loading.image.data != .native_file); + try testing.expectEqual(@as(usize, 0), fake.reads); + try testing.expectEqual(@as(usize, 1), fake.releases); + + // The ordinary reader can still supply bytes after rejection. + try loading.addData(testing.allocator, &.{ 1, 2, 3, 4 }); + var img = try loading.complete(testing.allocator); + loading.deinit(testing.allocator); + try testing.expectEqualSlices(u8, &.{ 1, 2, 3, 4 }, img.data.bytes().?); + img.deinit(testing.allocator); + try testing.expectEqual(@as(usize, 1), fake.releases); + + // Valid callback ownership is unchanged: zero-read acceptance, one release. + fake.missing_read = false; + fake.missing_release = false; + var accepted: LoadingImage = .{ + .image = .{ .width = 1, .height = 1, .format = .rgba }, + .quiet = .no, + .temporary_directory = null, + }; + try testing.expect(try accepted.trySnapshotFile(hook, request)); + try testing.expect(accepted.image.data == .native_file); + try testing.expectEqual(@as(usize, 0), fake.reads); + accepted.deinit(testing.allocator); + try testing.expectEqual(@as(usize, 2), fake.releases); +} diff --git a/vendor/libghostty-vt/src/terminal/kitty/graphics_storage.zig b/vendor/libghostty-vt/src/terminal/kitty/graphics_storage.zig index 897c8944..bd8cd424 100644 --- a/vendor/libghostty-vt/src/terminal/kitty/graphics_storage.zig +++ b/vendor/libghostty-vt/src/terminal/kitty/graphics_storage.zig @@ -154,7 +154,7 @@ pub const ImageStorage = struct { if (img.generation != self.generation) return false; const expected_len = switch (img.data) { - .complete => return false, + .complete, .encoded_png, .native_file => return false, .pending => |len| len, }; if (data.len != expected_len) return false; @@ -975,6 +975,15 @@ pub const ImageStorage = struct { img.generation = self.generation; } + pub fn materializeBacking(self: *ImageStorage, io: std.Io, alloc: Allocator, img: *Image) !void { + if (img.data != .encoded_png and img.data != .native_file) return; + const before = img.data.len(); + try img.materializeFile(alloc); + try img.materializePng(alloc); + self.total_bytes -= before - img.data.len(); + self.markImageContentChanged(io, img); + } + /// Convert a stored image's base data to RGBA in place, adjusting /// byte accounting. All animation composition happens in RGBA; /// this is called before the first composition into an image. diff --git a/vendor/patches/libghostty-vt/0007-experimental-png-retention.patch b/vendor/patches/libghostty-vt/0007-experimental-png-retention.patch new file mode 100644 index 00000000..e9964ab0 --- /dev/null +++ b/vendor/patches/libghostty-vt/0007-experimental-png-retention.patch @@ -0,0 +1,1055 @@ +diff --git a/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h b/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h +index 99cee60a..8dad8086 100644 +--- a/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h ++++ b/vendor/libghostty-vt/include/ghostty/vt/kitty_graphics.h +@@ -391,12 +391,13 @@ typedef enum GHOSTTY_ENUM_TYPED { + /** + * Borrowed pointer to the raw pixel data. Valid as long as the + * underlying terminal is not mutated. Returns GHOSTTY_NO_VALUE when +- * the image metadata is resident but its pixel payload is pending. ++ * the image metadata is resident but its pixel payload is pending, or ++ * when the experimental PRESERVE_PNG option retained encoded PNG data. + * + * The data is always fully decoded, uncompressed pixels in the + * format reported by GHOSTTY_KITTY_IMAGE_DATA_FORMAT: zlib payloads +- * are inflated and PNG payloads are decoded to RGBA at transmission +- * time, before the image is stored. Consumers can upload this ++ * are inflated and PNG payloads are normally decoded to RGBA at transmission ++ * time, before the image is stored. Consumers can upload returned pixels + * directly to the GPU without any decode step. + * + * For an animated image (Kitty graphics animation, actions a=f/a=a) +@@ -412,7 +413,7 @@ typedef enum GHOSTTY_ENUM_TYPED { + /** + * Length of the raw pixel data in bytes. Always equal to + * width * height * bytes-per-pixel for the reported format. For a +- * pending image, this is the expected length reserved against the ++ * pending or encoded PNG image, this is the expected decoded length reserved against the + * storage limit even though DATA_PTR is not available yet. + * + * Output type: size_t * +@@ -439,6 +440,17 @@ typedef enum GHOSTTY_ENUM_TYPED { + */ + GHOSTTY_KITTY_IMAGE_DATA_GENERATION = 9, + ++ /** Owned encoded PNG bytes, borrowed until the next mutation. NO_VALUE ++ * unless experimentally retained. Output type: const uint8_t **. ++ */ ++ GHOSTTY_KITTY_IMAGE_DATA_ENCODED_PNG_PTR = 10, ++ /** Encoded PNG length; NO_VALUE unless retained. Output type: size_t *. */ ++ GHOSTTY_KITTY_IMAGE_DATA_ENCODED_PNG_LEN = 11, ++ /** Borrowed void* backing context; valid only while image is unchanged. */ ++ GHOSTTY_KITTY_IMAGE_DATA_FILE_CONTEXT = 12, ++ /** uint64_t immutable revision fingerprint; never reads image pixels. */ ++ GHOSTTY_KITTY_IMAGE_DATA_FILE_IDENTITY = 13, ++ + GHOSTTY_KITTY_IMAGE_DATA_MAX_VALUE = GHOSTTY_ENUM_MAX_VALUE, + } GhosttyKittyGraphicsImageData; + +diff --git a/vendor/libghostty-vt/include/ghostty/vt/terminal.h b/vendor/libghostty-vt/include/ghostty/vt/terminal.h +index 81f9ebd0..6404af53 100644 +--- a/vendor/libghostty-vt/include/ghostty/vt/terminal.h ++++ b/vendor/libghostty-vt/include/ghostty/vt/terminal.h +@@ -27,6 +27,39 @@ + extern "C" { + #endif + ++/** Borrowed opened descriptor; size is sizeof this request. No path is supplied. */ ++typedef struct { ++ size_t size; ++ int64_t fd; ++ uint64_t expected_len; ++} GhosttyKittyImageSnapshotFileRequest; ++ ++typedef bool (*GhosttyKittyImageFileReadFn)(void *context, uint8_t *dest, size_t len); ++typedef void (*GhosttyKittyImageFileReleaseFn)(void *context); ++ ++typedef struct { ++ void *context; ++ uint64_t identity; ++ size_t len; ++ GhosttyKittyImageFileReadFn read; ++ GhosttyKittyImageFileReleaseFn release; ++} GhosttyKittyImageFileBacking; ++ ++/** Optional synchronous callback, default NULL. Verify exact descriptor size ++ * and snapshot immutable bytes before returning true. True transfers one context ++ * reference; false transfers nothing. Do not close the borrowed descriptor or ++ * reenter the terminal. Both read and release MUST be non-NULL on success. ++ * Read must fill exactly len bytes; release is called once. An incomplete ++ * backing is rejected and ordinary file loading is used instead. If release ++ * is provided, rejection calls it once; without release, the host remains ++ * responsible for cleanup (the terminal cannot release that context). ++ * identity identifies the immutable revision, not a pathname or source inode. ++ */ ++typedef bool (*GhosttyKittyImageSnapshotFileFn)( ++ GhosttyTerminal terminal, void *userdata, ++ const GhosttyKittyImageSnapshotFileRequest *request, ++ GhosttyKittyImageFileBacking *out_backing); ++ + /** @defgroup terminal Terminal + * + * Complete terminal emulator state and rendering. +@@ -1545,6 +1578,14 @@ typedef enum GHOSTTY_ENUM_TYPED { + * Input type: size_t* + */ + GHOSTTY_TERMINAL_OPT_CLIPBOARD_WRITE_MAX_BYTES = 39, ++ ++ /** Experimental, default-off PNG retention for quiet base transmissions. ++ * Input type: bool*. NULL is a no-op. Corrupt compressed pixels may only ++ * be detected when an animation operation materializes the image. ++ */ ++ GHOSTTY_TERMINAL_OPT_KITTY_IMAGE_PRESERVE_PNG = 40, ++ /** Optional synchronous GhosttyKittyImageSnapshotFileFn; NULL disables. */ ++ GHOSTTY_TERMINAL_OPT_KITTY_IMAGE_SNAPSHOT_FILE = 41, + GHOSTTY_TERMINAL_OPT_MAX_VALUE = GHOSTTY_ENUM_MAX_VALUE, + } GhosttyTerminalOption; + +diff --git a/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig b/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig +index 2bd5e862..c66b13be 100644 +--- a/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig ++++ b/vendor/libghostty-vt/src/terminal/c/kitty_graphics.zig +@@ -183,6 +183,10 @@ pub const ImageData = enum(c_int) { + data_ptr = 7, + data_len = 8, + generation = 9, ++ encoded_png_ptr = 10, ++ encoded_png_len = 11, ++ file_context = 12, ++ file_identity = 13, + + pub fn OutType(comptime self: ImageData) type { + return switch (self) { +@@ -190,9 +194,10 @@ pub const ImageData = enum(c_int) { + .id, .number, .width, .height => u32, + .format => ImageFormat, + .compression => ImageCompression, +- .data_ptr => [*]const u8, +- .data_len => usize, +- .generation => u64, ++ .data_ptr, .encoded_png_ptr => [*]const u8, ++ .data_len, .encoded_png_len => usize, ++ .generation, .file_identity => u64, ++ .file_context => ?*anyopaque, + }; + } + }; +@@ -266,7 +271,26 @@ fn imageGetTyped( + // data; the image generation changes whenever the current + // frame does, so generation-keyed caches stay coherent. + .data_ptr => out.* = (image.renderData().bytes() orelse return .no_value).ptr, +- .data_len => out.* = image.renderData().len(), ++ .data_len => out.* = switch (image.renderData()) { ++ .encoded_png => |png| png.decoded_len, ++ else => image.renderData().len(), ++ }, ++ .encoded_png_ptr => out.* = switch (image.data) { ++ .encoded_png => |png| png.bytes.ptr, ++ else => return .no_value, ++ }, ++ .encoded_png_len => out.* = switch (image.data) { ++ .encoded_png => |png| png.bytes.len, ++ else => return .no_value, ++ }, ++ .file_context => out.* = switch (image.data) { ++ .native_file => |file| file.context, ++ else => return .no_value, ++ }, ++ .file_identity => out.* = switch (image.data) { ++ .native_file => |file| file.identity, ++ else => return .no_value, ++ }, + .generation => out.* = image.generation, + } + +diff --git a/vendor/libghostty-vt/src/terminal/c/terminal.zig b/vendor/libghostty-vt/src/terminal/c/terminal.zig +index 91bbe310..e825183e 100644 +--- a/vendor/libghostty-vt/src/terminal/c/terminal.zig ++++ b/vendor/libghostty-vt/src/terminal/c/terminal.zig +@@ -13,6 +13,7 @@ const ScreenSet = @import("../ScreenSet.zig"); + const PageList = @import("../PageList.zig"); + const apc = @import("../apc.zig"); + const kitty = @import("../kitty/key.zig"); ++const kitty_image = @import("../kitty/graphics_image.zig"); + const kitty_gfx_c = @import("kitty_graphics.zig"); + const modes = @import("../modes.zig"); + const point = @import("../point.zig"); +@@ -257,6 +258,7 @@ const Effects = struct { + clipboard_write: ?ClipboardWriteFn = null, + clipboard_read: ?ClipboardReadFn = null, + unknown_sequence: ?UnknownSequenceFn = null, ++ snapshot_file: ?SnapshotFileFn = null, + + /// Scratch buffer for DA1 feature codes. The device attributes + /// trampoline converts C feature codes into this buffer and returns +@@ -265,6 +267,14 @@ const Effects = struct { + /// (`reportDeviceAttributes`) reads it before any re-entrant call. + da_features_buf: [64]device_attributes.Primary.Feature = undefined, + ++ pub const SnapshotFileFn = *const fn (Terminal, ?*anyopaque, *const kitty_image.SnapshotFileRequest, *kitty_image.FileBacking) callconv(lib.calling_conv) bool; ++ ++ fn snapshotFile(context: ?*anyopaque, request: *const kitty_image.SnapshotFileRequest, out: *kitty_image.FileBacking) bool { ++ const wrapper: *TerminalWrapper = @ptrCast(@alignCast(context.?)); ++ const callback = wrapper.effects.snapshot_file orelse return false; ++ return callback(@ptrCast(wrapper), wrapper.effects.userdata, request, out); ++ } ++ + /// C function pointer type for the write_pty callback. + pub const WritePtyFn = *const fn (Terminal, ?*anyopaque, [*]const u8, usize) callconv(lib.calling_conv) void; + +@@ -1174,12 +1184,15 @@ pub const Option = enum(c_int) { + terminfo_name = 37, + clipboard_read = 38, + clipboard_write_max_bytes = 39, ++ kitty_image_preserve_png = 40, ++ kitty_image_snapshot_file = 41, + + /// Input type expected for setting the option. + pub fn InType(comptime self: Option) type { + return switch (self) { + .userdata => ?*const anyopaque, + .write_pty => ?Effects.WritePtyFn, ++ .kitty_image_snapshot_file => ?Effects.SnapshotFileFn, + .bell => ?Effects.BellFn, + .color_scheme => ?Effects.ColorSchemeFn, + .desktop_notification => ?Effects.DesktopNotificationFn, +@@ -1197,6 +1210,7 @@ pub const Option = enum(c_int) { + .color_foreground, .color_background, .color_cursor => ?*const color.RGB.C, + .color_palette => ?*const color.PaletteC, + .kitty_image_storage_limit => ?*const u64, ++ .kitty_image_preserve_png, + .kitty_image_medium_file, + .kitty_image_medium_shared_mem, + .glyph_protocol, +@@ -1250,6 +1264,17 @@ fn setTyped( + switch (option) { + .userdata => wrapper.effects.userdata = @constCast(value), + .write_pty => wrapper.effects.write_pty = value, ++ .kitty_image_snapshot_file => { ++ if (comptime !build_options.kitty_graphics) return .success; ++ wrapper.effects.snapshot_file = value; ++ var it = wrapper.terminal.screens.all.iterator(); ++ while (it.next()) |entry| { ++ entry.value.*.kitty_images.image_limits.snapshot_file = if (value != null) ++ .{ .context = @ptrCast(wrapper), .callback = &Effects.snapshotFile } ++ else ++ null; ++ } ++ }, + .bell => wrapper.effects.bell = value, + .color_scheme => wrapper.effects.color_scheme = value, + .desktop_notification => wrapper.effects.desktop_notification = value, +@@ -1330,6 +1355,7 @@ fn setTyped( + screen.kitty_images.setLimit(screen.io, screen.alloc, screen, limit); + } + }, ++ .kitty_image_preserve_png, + .kitty_image_medium_file, + .kitty_image_medium_shared_mem, + => { +@@ -1339,6 +1365,7 @@ fn setTyped( + while (it.next()) |entry| { + const screen = entry.value.*; + switch (option) { ++ .kitty_image_preserve_png => screen.kitty_images.image_limits.preserve_png = val, + .kitty_image_medium_file => screen.kitty_images.image_limits.file = val, + .kitty_image_medium_shared_mem => screen.kitty_images.image_limits.shared_memory = val, + else => unreachable, +diff --git a/vendor/libghostty-vt/src/terminal/kitty/graphics_exec.zig b/vendor/libghostty-vt/src/terminal/kitty/graphics_exec.zig +index b1b968b2..203190f1 100644 +--- a/vendor/libghostty-vt/src/terminal/kitty/graphics_exec.zig ++++ b/vendor/libghostty-vt/src/terminal/kitty/graphics_exec.zig +@@ -509,6 +509,11 @@ fn transmitAnimationFrame( + result.message = "ENOENT: image not found"; + return result; + }; ++ storage.materializeBacking(io, alloc, img) catch |err| { ++ encodeError(&result, err); ++ return result; ++ }; ++ + result.id = img.id; + + var loading = LoadingImage.init( +@@ -773,6 +778,10 @@ fn controlAnimation( + result.message = "ENOENT: image not found"; + return result; + }; ++ storage.materializeBacking(io, alloc, img) catch |err| { ++ encodeError(&result, err); ++ return result; ++ }; + + const anim = ensureAnimation(alloc, img) catch |err| { + encodeError(&result, err); +@@ -857,6 +866,11 @@ fn composeAnimation( + result.message = "ENOENT: image not found"; + return result; + }; ++ storage.materializeBacking(io, alloc, img) catch |err| { ++ encodeError(&result, err); ++ return result; ++ }; ++ + result.id = img.id; + if (img.data.bytes() == null) { + result.message = "EINVAL: image data incomplete"; +@@ -1058,7 +1072,10 @@ fn loadAndAddImage( + // loading.debugDump() catch unreachable; + + // Validate and store our image +- var img = try loading.complete(alloc); ++ var img = if (storage.image_limits.preserve_png) ++ try loading.completePreservingPng(alloc) ++ else ++ try loading.complete(alloc); + errdefer img.deinit(alloc); + try storage.addImage(io, alloc, terminal.screens.active, img); + +@@ -3601,3 +3618,87 @@ test "kittygfx animation: control negative gap makes frame gapless" { + const anim = storage.imagePtrByIdOrNumber(1, 0).?.animation.?; + try testing.expectEqual(@as(u32, 0), anim.frames.items[0].gap_ms); + } ++ ++test "kittygfx experimental PNG retention quiet inheritance and transactional materialization" { ++ const testing = std.testing; ++ const alloc = testing.allocator; ++ const io = testing.io; ++ const sys = @import("../sys.zig"); ++ const Decoder = struct { ++ var calls: usize = 0; ++ var fail: bool = false; ++ fn decode(a: Allocator, _: []const u8) sys.DecodeError!sys.Image { ++ calls += 1; ++ if (fail) return error.InvalidData; ++ const pixels = try a.alloc(u8, 50 * 76 * 4); ++ @memset(pixels, 0); ++ return .{ .width = 50, .height = 76, .data = pixels }; ++ } ++ }; ++ const original = sys.decode_png; ++ defer sys.decode_png = original; ++ sys.decode_png = &Decoder.decode; ++ Decoder.calls = 0; ++ Decoder.fail = false; ++ const png = @embedFile("testdata/image-png-none-50x76-2147483647-raw.data"); ++ var t = try Terminal.init(io, alloc, .{ .rows = 5, .cols = 5 }); ++ defer t.deinit(alloc); ++ const storage = &t.screens.active.kitty_images; ++ var cmd: Command = .{ ++ .quiet = .failures, ++ .control = .{ .transmit = .{ .format = .png, .image_id = 1 } }, ++ .data = @constCast(png), ++ }; ++ // Default off is the normal decoder path even for q=2. ++ try testing.expect(!storage.image_limits.preserve_png); ++ _ = execute(io, alloc, &t, &cmd); ++ try testing.expectEqual(@as(usize, 1), Decoder.calls); ++ try testing.expect(storage.imageById(1).?.data == .complete); ++ storage.image_limits.preserve_png = true; ++ for ([_]Command.Quiet{ .no, .ok }) |quiet| { ++ cmd.quiet = quiet; ++ _ = execute(io, alloc, &t, &cmd); ++ try testing.expect(storage.imageById(1).?.data == .complete); ++ } ++ try testing.expectEqual(@as(usize, 3), Decoder.calls); ++ cmd.quiet = .failures; ++ cmd.control = .{ .query = .{ .format = .png, .image_id = 2 } }; ++ _ = execute(io, alloc, &t, &cmd); ++ try testing.expectEqual(@as(usize, 4), Decoder.calls); ++ try testing.expect(storage.imageById(2) == null); ++ cmd.control = .{ .transmit = .{ .format = .png, .image_id = 1, .more_chunks = true } }; ++ cmd.data = @constCast(png[0..33]); ++ _ = execute(io, alloc, &t, &cmd); ++ cmd.quiet = .no; ++ cmd.control = .{ .transmit = .{} }; ++ cmd.data = @constCast(png[33..]); ++ _ = execute(io, alloc, &t, &cmd); ++ try testing.expectEqual(@as(usize, 4), Decoder.calls); ++ const img = storage.imagePtrByIdOrNumber(1, 0).?; ++ try testing.expectEqualSlices(u8, png, img.data.encoded_png.bytes); ++ try testing.expect(img.data.bytes() == null); ++ try testing.expectEqual(png.len + 50 * 76 * 4, storage.total_bytes); ++ const generation = img.generation; ++ Decoder.fail = true; ++ const animate: Command = .{ .control = .{ .control_animation = .{ .image_id = 1 } } }; ++ _ = execute(io, alloc, &t, &animate); ++ try testing.expectEqual(generation, img.generation); ++ try testing.expect(img.data == .encoded_png); ++ try testing.expectEqual(png.len + 50 * 76 * 4, storage.total_bytes); ++ Decoder.fail = false; ++ _ = execute(io, alloc, &t, &animate); ++ try testing.expect(img.generation > generation); ++ try testing.expect(img.data == .complete); ++ try testing.expectEqual(@as(usize, 50 * 76 * 4), storage.total_bytes); ++ ++ // The reservation includes both the encoded and decoded representation. ++ storage.setLimit(io, alloc, t.screens.active, 50 * 76 * 4); ++ cmd.quiet = .failures; ++ cmd.control = .{ .transmit = .{ .format = .png, .image_id = 3 } }; ++ cmd.data = @constCast(png); ++ const calls = Decoder.calls; ++ _ = execute(io, alloc, &t, &cmd); ++ try testing.expectEqual(calls, Decoder.calls); ++ try testing.expect(storage.imageById(3) == null); ++ try testing.expect(storage.loading == null); ++} +diff --git a/vendor/libghostty-vt/src/terminal/kitty/graphics_image.zig b/vendor/libghostty-vt/src/terminal/kitty/graphics_image.zig +index e411bafe..c841c48d 100644 +--- a/vendor/libghostty-vt/src/terminal/kitty/graphics_image.zig ++++ b/vendor/libghostty-vt/src/terminal/kitty/graphics_image.zig +@@ -21,6 +21,24 @@ const max_dimension = 10000; + /// Maximum size in bytes, taken from Kitty. + const max_size = 400 * 1024 * 1024; // 400MB + ++/// Host-owned immutable raw RGBA snapshot. A successful callback transfers one reference. ++pub const FileBacking = extern struct { ++ context: ?*anyopaque, ++ identity: u64, ++ len: usize, ++ read: ?*const fn (?*anyopaque, [*]u8, usize) callconv(.c) bool, ++ release: ?*const fn (?*anyopaque) callconv(.c) void, ++}; ++pub const SnapshotFileRequest = extern struct { ++ size: usize = @sizeOf(SnapshotFileRequest), ++ fd: i64, ++ expected_len: u64, ++}; ++pub const SnapshotFileHook = struct { ++ context: ?*anyopaque, ++ callback: *const fn (?*anyopaque, *const SnapshotFileRequest, *FileBacking) bool, ++}; ++ + /// An image that is still being loaded. The image should be initialized + /// using init on the first chunk and then addData for each subsequent + /// chunk. Once all chunks have been added, complete should be called +@@ -29,6 +47,7 @@ pub const LoadingImage = struct { + /// The in-progress image. The first chunk must have all the metadata + /// so this comes from that initially. + image: Image, ++ snapshot_file: ?SnapshotFileHook = null, + + /// The data that is being built up. + data: std.ArrayListUnmanaged(u8) = .empty, +@@ -85,6 +104,8 @@ pub const LoadingImage = struct { + disabled: void, + }, + shared_memory: bool, ++ preserve_png: bool = false, ++ snapshot_file: ?SnapshotFileHook = null, + + /// Enables all filesystem-related image transmission mediums. `path` + /// is the temporary directory to expect files in when files are +@@ -132,6 +153,10 @@ pub const LoadingImage = struct { + .metadata = .{ .transient = t.usage.transient }, + }, + ++ .snapshot_file = if (cmd.control == .transmit or cmd.control == .transmit_and_display) ++ limits.snapshot_file ++ else ++ null, + .display = cmd.display(), + .quiet = cmd.quiet, + .response = .{ +@@ -427,6 +452,16 @@ pub const LoadingImage = struct { + return error.InvalidData; + } + ++ // Only whole, uncompressed base RGBA files qualify. The host must ++ // verify the descriptor size and create an immutable private snapshot. ++ if (comptime builtin.os.tag != .windows and builtin.os.tag != .wasi and builtin.os.tag != .freestanding) { ++ if (self.snapshot_file) |hook| { ++ if (nativeFileExpectedLen(t)) |expected| { ++ if (try self.trySnapshotFile(hook, .{ .fd = @intCast(file.handle), .expected_len = expected })) return; ++ } ++ } ++ } ++ + var buf: [4096]u8 = undefined; + var buf_reader = file.reader(io, &buf); + if (t.offset > 0) { +@@ -468,6 +503,32 @@ pub const LoadingImage = struct { + self.data = .{ .items = managed.items, .capacity = managed.capacity }; + } + ++ /// Accept only complete callback records. A malformed success falls back ++ /// to the ordinary reader, releasing the transferred reference if possible. ++ fn trySnapshotFile(self: *LoadingImage, hook: SnapshotFileHook, request: SnapshotFileRequest) !bool { ++ var backing: FileBacking = .{ .context = null, .identity = 0, .len = 0, .read = null, .release = null }; ++ if (!hook.callback(hook.context, &request, &backing)) return false; ++ if (backing.read == null or backing.release == null) { ++ if (backing.release) |release| release(backing.context); ++ return false; ++ } ++ if (backing.len != request.expected_len) { ++ backing.release.?(backing.context); ++ return error.InvalidData; ++ } ++ self.image.data = .{ .native_file = backing }; ++ return true; ++ } ++ ++ fn nativeFileExpectedLen(t: command.Transmission) ?usize { ++ if (t.medium != .file or t.format != .rgba or t.compression != .none or ++ t.more_chunks or t.offset != 0 or t.width == 0 or t.height == 0 or ++ t.width > max_dimension or t.height > max_dimension) return null; ++ const len = @as(usize, t.width) * @as(usize, t.height) * 4; ++ if (len > 16 * 1024 * 1024 or (t.size != 0 and t.size != len)) return null; ++ return len; ++ } ++ + /// Returns the canonical path of an open file after applying the file + /// transmission blocklist. + fn validatedFilePath(io: std.Io, file: std.Io.File, buf: []u8) ![]const u8 { +@@ -569,7 +630,7 @@ pub const LoadingImage = struct { + // Data length must be what we expect. + const bpp = command.Transmission.formatBpp(img.format); + const expected_len = img.width * img.height * bpp; +- const actual_len = self.data.items.len; ++ const actual_len = if (img.data == .native_file) img.data.native_file.len else self.data.items.len; + if (self.frame != null) { + // Kitty allows animation frames to exceed their expected length + // and just truncates it. Not sure if thats expected but lets +@@ -592,12 +653,37 @@ pub const LoadingImage = struct { + + // Everything looks good, copy the image data over. + var result = self.image; +- result.data = .{ .complete = try self.data.toOwnedSlice(alloc) }; ++ if (result.data != .native_file) result.data = .{ .complete = try self.data.toOwnedSlice(alloc) }; + errdefer result.deinit(alloc); + self.image = .{}; + return result; + } + ++ /// Experimental forwarding path: only invoked for assembled quiet base ++ /// transmissions. Never used by queries or animation frame loading. ++ pub fn completePreservingPng(self: *LoadingImage, alloc: Allocator) !Image { ++ if (self.image.format != .png or self.image.compression != .none or ++ self.quiet != .failures or self.frame != null) return self.complete(alloc); ++ const header = inspectPng(self.data.items) catch |err| switch (err) { ++ // Do not expand the deferred-corruption contract to metadata we ++ // have not validated. Such PNGs keep the normal decoder semantics. ++ error.UnsupportedPngRetention => return self.complete(alloc), ++ error.InvalidData => return error.InvalidData, ++ error.DimensionsRequired => return error.DimensionsRequired, ++ error.DimensionsTooLarge => return error.DimensionsTooLarge, ++ }; ++ var result = self.image; ++ result.width = header.width; ++ result.height = header.height; ++ result.format = .rgba; ++ result.data = .{ .encoded_png = .{ ++ .bytes = try self.data.toOwnedSlice(alloc), ++ .decoded_len = header.decoded_len, ++ } }; ++ self.image = .{}; ++ return result; ++ } ++ + /// Debug function to write the data to a file. This is useful for + /// capturing some test data for unit tests. + pub fn debugDump(io: std.Io, self: LoadingImage) !void { +@@ -768,11 +854,16 @@ pub const Image = struct { + /// Expected decoded byte length for a payload that has not arrived. + pending: usize, + ++ encoded_png: struct { bytes: []const u8, decoded_len: usize }, ++ native_file: FileBacking, ++ + /// Bytes reserved against the storage limit. + pub fn len(self: Data) usize { + return switch (self) { + .complete => |data| data.len, + .pending => |expected_len| expected_len, ++ .native_file => |file| file.len, ++ .encoded_png => |png| png.bytes.len + png.decoded_len, + }; + } + +@@ -780,7 +871,7 @@ pub const Image = struct { + pub fn bytes(self: Data) ?[]const u8 { + return switch (self) { + .complete => |data| data, +- .pending => null, ++ .pending, .encoded_png, .native_file => null, + }; + } + +@@ -792,10 +883,43 @@ pub const Image = struct { + switch (self.*) { + .complete => |data| if (data.len > 0) alloc.free(data), + .pending => {}, ++ .encoded_png => |png| alloc.free(png.bytes), ++ .native_file => |file| if (file.release) |release| release(file.context), + } + } + }; + ++ /// Transactional: a failed host read leaves the authoritative backing owned. ++ pub fn materializeFile(self: *Image, alloc: Allocator) !void { ++ const file = switch (self.data) { ++ .native_file => |file| file, ++ else => return, ++ }; ++ const read = file.read orelse return error.InvalidData; ++ const release = file.release orelse return error.InvalidData; ++ const bytes = try alloc.alloc(u8, file.len); ++ errdefer alloc.free(bytes); ++ if (!read(file.context, bytes.ptr, bytes.len)) return error.InvalidData; ++ self.data = .{ .complete = bytes }; ++ release(file.context); ++ } ++ ++ /// Transactional: neither ownership nor metadata changes on failure. ++ pub fn materializePng(self: *Image, alloc: Allocator) !void { ++ const png = switch (self.data) { ++ .encoded_png => |png| png, ++ else => return, ++ }; ++ const decode = sys.decode_png orelse return error.UnsupportedFormat; ++ var limited: LimitedAllocator = .init(alloc, max_size); ++ const result = try decode(limited.allocator(), png.bytes); ++ errdefer alloc.free(result.data); ++ if (result.width != self.width or result.height != self.height or ++ result.data.len != png.decoded_len) return error.InvalidData; ++ alloc.free(png.bytes); ++ self.data = .{ .complete = result.data }; ++ } ++ + pub fn deinit(self: *Image, alloc: Allocator) void { + self.data.deinit(alloc); + if (self.animation) |anim| { +@@ -2159,3 +2283,404 @@ test "limits: temporary file medium allowed by limits" { + ); + defer loading.deinit(alloc); + } ++ ++const PngHeader = struct { width: u32, height: u32, decoded_len: usize }; ++ ++/// Bounded structural validation, deliberately without inflating IDAT. ++/// CRC-valid but corrupt compressed pixels can fail later in this experiment. ++fn inspectPng(bytes: []const u8) !PngHeader { ++ if (bytes.len > max_size or bytes.len < 33 or ++ !std.mem.eql(u8, bytes[0..8], "\x89PNG\r\n\x1a\n")) return error.InvalidData; ++ var offset: usize = 8; ++ var header: ?PngHeader = null; ++ var idat = false; ++ var ended_idat = false; ++ var palette = false; ++ var indexed = false; ++ var color_type: u8 = 0; ++ var bit_depth: u8 = 0; ++ var palette_entries: usize = 0; ++ var transparency = false; ++ while (offset < bytes.len) { ++ if (bytes.len - offset < 12) return error.InvalidData; ++ const len: usize = std.mem.readInt(u32, bytes[offset..][0..4], .big); ++ if (len > bytes.len - offset - 12) return error.InvalidData; ++ const kind = bytes[offset + 4 ..][0..4]; ++ for (kind) |ch| if (!std.ascii.isAlphabetic(ch)) return error.InvalidData; ++ if (kind[2] & 0x20 != 0) return error.InvalidData; ++ const body = bytes[offset + 8 ..][0..len]; ++ const crc = std.mem.readInt(u32, bytes[offset + 8 + len ..][0..4], .big); ++ if (std.hash.crc.Crc32IsoHdlc.hash(bytes[offset + 4 ..][0 .. len + 4]) != crc) ++ return error.InvalidData; ++ offset += len + 12; ++ if (std.mem.eql(u8, kind, "IHDR")) { ++ if (header != null or len != 13) return error.InvalidData; ++ const width = std.mem.readInt(u32, body[0..4], .big); ++ const height = std.mem.readInt(u32, body[4..8], .big); ++ if (width == 0 or height == 0) return error.DimensionsRequired; ++ if (width > max_dimension or height > max_dimension) return error.DimensionsTooLarge; ++ const depth = body[8]; ++ const supported = switch (body[9]) { ++ 0 => depth == 1 or depth == 2 or depth == 4 or depth == 8 or depth == 16, ++ 2, 4, 6 => depth == 8 or depth == 16, ++ 3 => depth == 1 or depth == 2 or depth == 4 or depth == 8, ++ else => false, ++ }; ++ if (!supported or body[10] != 0 or body[11] != 0 or body[12] > 1) ++ return error.InvalidData; ++ color_type = body[9]; ++ bit_depth = depth; ++ indexed = color_type == 3; ++ const pixels = std.math.mul(usize, width, height) catch return error.InvalidData; ++ const decoded_len = std.math.mul(usize, pixels, 4) catch return error.InvalidData; ++ if (decoded_len > max_size) return error.InvalidData; ++ header = .{ .width = width, .height = height, .decoded_len = decoded_len }; ++ } else { ++ if (header == null) return error.InvalidData; ++ if (std.mem.eql(u8, kind, "IDAT")) { ++ if (ended_idat or (indexed and !palette)) return error.InvalidData; ++ idat = true; ++ } else { ++ if (idat) ended_idat = true; ++ if (std.mem.eql(u8, kind, "IEND")) { ++ if (!idat or len != 0 or offset != bytes.len) return error.InvalidData; ++ return header.?; ++ } else if (std.mem.eql(u8, kind, "PLTE")) { ++ if (palette or idat or transparency or len == 0 or len > 768 or len % 3 != 0 or ++ color_type == 0 or color_type == 4) return error.InvalidData; ++ palette_entries = len / 3; ++ if (indexed and palette_entries > (@as(usize, 1) << @intCast(bit_depth))) ++ return error.InvalidData; ++ palette = true; ++ } else if (std.mem.eql(u8, kind, "tRNS")) { ++ if (transparency or idat) return error.InvalidData; ++ switch (color_type) { ++ 0, 2 => { ++ const expected: usize = if (color_type == 0) 2 else 6; ++ if (len != expected) return error.InvalidData; ++ const max_sample: u32 = (@as(u32, 1) << @intCast(bit_depth)) - 1; ++ var i: usize = 0; ++ while (i < len) : (i += 2) { ++ if (std.mem.readInt(u16, body[i..][0..2], .big) > max_sample) ++ return error.InvalidData; ++ } ++ }, ++ 3 => if (!palette or len == 0 or len > palette_entries) ++ return error.InvalidData, ++ // Alpha-bearing color types must not carry tRNS. ++ else => return error.InvalidData, ++ } ++ transparency = true; ++ } else if (kind[0] & 0x20 == 0) { ++ return error.InvalidData; ++ } else { ++ // sBIT, bKGD, color profiles, APNG, text, and any other ++ // ancillary extensions are deliberately not eligible. ++ // The strict decoder handles their validity/semantics. ++ return error.UnsupportedPngRetention; ++ } ++ } ++ } ++ } ++ return error.InvalidData; ++} ++ ++test "experimental PNG inspection validates CRC and complete structure" { ++ const testing = std.testing; ++ const original = @embedFile("testdata/image-png-none-50x76-2147483647-raw.data"); ++ const header = try inspectPng(original); ++ try testing.expectEqual(@as(u32, 50), header.width); ++ try testing.expectEqual(@as(u32, 76), header.height); ++ try testing.expectEqual(@as(usize, 50 * 76 * 4), header.decoded_len); ++ try testing.expectError(error.InvalidData, inspectPng(original[0 .. original.len - 1])); ++ var corrupt = original.*; ++ corrupt[29] ^= 1; // IHDR CRC ++ try testing.expectError(error.InvalidData, inspectPng(&corrupt)); ++ corrupt = original.*; ++ corrupt[0] = 0; ++ try testing.expectError(error.InvalidData, inspectPng(&corrupt)); ++} ++ ++test "experimental PNG inspection transparency and palette metadata" { ++ const testing = std.testing; ++ const Chunk = struct { kind: *const [4]u8, body: []const u8 }; ++ const Helper = struct { ++ fn append(out: *std.ArrayList(u8), kind: *const [4]u8, body: []const u8) !void { ++ var word: [4]u8 = undefined; ++ std.mem.writeInt(u32, &word, @intCast(body.len), .big); ++ try out.appendSlice(testing.allocator, &word); ++ const crc_start = out.items.len; ++ try out.appendSlice(testing.allocator, kind); ++ try out.appendSlice(testing.allocator, body); ++ std.mem.writeInt(u32, &word, std.hash.crc.Crc32IsoHdlc.hash(out.items[crc_start..]), .big); ++ try out.appendSlice(testing.allocator, &word); ++ } ++ fn make(color: u8, depth: u8, chunks: []const Chunk) ![]u8 { ++ var out: std.ArrayList(u8) = .empty; ++ errdefer out.deinit(testing.allocator); ++ try out.appendSlice(testing.allocator, "\x89PNG\r\n\x1a\n"); ++ const ihdr = [_]u8{ 0, 0, 0, 1, 0, 0, 0, 1, depth, color, 0, 0, 0 }; ++ try append(&out, "IHDR", &ihdr); ++ for (chunks) |chunk| try append(&out, chunk.kind, chunk.body); ++ try append(&out, "IEND", ""); ++ return out.toOwnedSlice(testing.allocator); ++ } ++ fn check(color: u8, depth: u8, chunks: []const Chunk, expected: ?anyerror) !void { ++ const png = try make(color, depth, chunks); ++ defer testing.allocator.free(png); ++ if (expected) |err| { ++ try testing.expectError(err, inspectPng(png)); ++ } else { ++ _ = try inspectPng(png); ++ } ++ } ++ }; ++ // IDAT is intentionally opaque; this test only exercises structure. ++ const idat: Chunk = .{ .kind = "IDAT", .body = "" }; ++ const palette: Chunk = .{ .kind = "PLTE", .body = &.{ 0, 0, 0, 255, 255, 255 } }; ++ const alpha: Chunk = .{ .kind = "tRNS", .body = &.{128} }; ++ const gray: Chunk = .{ .kind = "tRNS", .body = &.{ 0, 1 } }; ++ const rgb: Chunk = .{ .kind = "tRNS", .body = &.{ 0, 1, 0, 2, 0, 3 } }; ++ try Helper.check(3, 1, &.{ palette, alpha, idat }, null); ++ try Helper.check(0, 1, &.{ gray, idat }, null); ++ try Helper.check(2, 8, &.{ rgb, idat }, null); ++ try Helper.check(2, 8, &.{ palette, rgb, idat }, null); ++ try Helper.check(3, 1, &.{ alpha, palette, idat }, error.InvalidData); ++ try Helper.check(3, 1, &.{ palette, idat, alpha }, error.InvalidData); ++ try Helper.check(3, 1, &.{ palette, alpha, alpha, idat }, error.InvalidData); ++ try Helper.check(0, 8, &.{ alpha, idat }, error.InvalidData); ++ try Helper.check(2, 8, &.{ gray, idat }, error.InvalidData); ++ try Helper.check(4, 8, &.{ gray, idat }, error.InvalidData); ++ try Helper.check(6, 8, &.{ rgb, idat }, error.InvalidData); ++ try Helper.check(0, 8, &.{ palette, idat }, error.InvalidData); ++ try Helper.check(4, 8, &.{ palette, idat }, error.InvalidData); ++ try Helper.check(2, 8, &.{ rgb, palette, idat }, error.InvalidData); ++ try Helper.check(3, 1, &.{ palette, .{ .kind = "tRNS", .body = "" }, idat }, error.InvalidData); ++ try Helper.check(3, 1, &.{ palette, .{ .kind = "tRNS", .body = &.{ 1, 2, 3 } }, idat }, error.InvalidData); ++ try Helper.check(3, 1, &.{ .{ .kind = "PLTE", .body = &.{ 0, 0, 0, 1, 1, 1, 2, 2, 2 } }, idat }, error.InvalidData); ++ try Helper.check(0, 1, &.{ .{ .kind = "tRNS", .body = &.{ 0, 2 } }, idat }, error.InvalidData); ++ try Helper.check(2, 8, &.{ .{ .kind = "tRNS", .body = &.{ 1, 0, 0, 0, 0, 0 } }, idat }, error.InvalidData); ++ try Helper.check(0, 16, &.{ .{ .kind = "tRNS", .body = &.{ 255, 255 } }, idat }, null); ++ for ([_]*const [4]u8{ "sBIT", "bKGD", "iCCP", "gAMA" }) |kind| { ++ try Helper.check(2, 8, &.{ .{ .kind = kind, .body = "" }, idat }, error.UnsupportedPngRetention); ++ } ++} ++ ++test "experimental PNG unsupported metadata falls back to strict decoding" { ++ const testing = std.testing; ++ const Decoder = struct { ++ var calls: usize = 0; ++ fn decode(_: Allocator, _: []const u8) sys.DecodeError!sys.Image { ++ calls += 1; ++ return error.InvalidData; ++ } ++ }; ++ const original_decoder = sys.decode_png; ++ defer sys.decode_png = original_decoder; ++ sys.decode_png = &Decoder.decode; ++ Decoder.calls = 0; ++ const png = @embedFile("testdata/image-png-none-50x76-2147483647-raw.data"); ++ var loading: LoadingImage = .{ ++ .image = .{ .format = .png }, ++ .quiet = .failures, ++ .temporary_directory = null, ++ }; ++ defer loading.deinit(testing.allocator); ++ try loading.addData(testing.allocator, png[0..33]); ++ // CRC-valid ancillary chunk: inspection must delegate, not retain or ++ // return UnsupportedPngRetention to the caller. ++ var chunk = [_]u8{ 0, 0, 0, 1, 's', 'B', 'I', 'T', 8, 0, 0, 0, 0 }; ++ std.mem.writeInt(u32, chunk[9..13], std.hash.crc.Crc32IsoHdlc.hash(chunk[4..9]), .big); ++ try loading.addData(testing.allocator, &chunk); ++ try loading.addData(testing.allocator, png[33..]); ++ try testing.expectError(error.InvalidData, loading.completePreservingPng(testing.allocator)); ++ try testing.expectEqual(@as(usize, 1), Decoder.calls); ++} ++ ++test "native file completion and transactional lazy materialization" { ++ const testing = std.testing; ++ const Fake = struct { ++ reads: usize = 0, ++ releases: usize = 0, ++ fail: bool = false, ++ fn read(ctx: ?*anyopaque, dest: [*]u8, len: usize) callconv(.c) bool { ++ const self: *@This() = @ptrCast(@alignCast(ctx.?)); ++ self.reads += 1; ++ if (self.fail) return false; ++ @memset(dest[0..len], 123); ++ return true; ++ } ++ fn release(ctx: ?*anyopaque) callconv(.c) void { ++ const self: *@This() = @ptrCast(@alignCast(ctx.?)); ++ self.releases += 1; ++ } ++ fn backing(self: *@This(), len: usize) FileBacking { ++ return .{ .context = self, .identity = 7, .len = len, .read = read, .release = release }; ++ } ++ }; ++ var fake: Fake = .{}; ++ var loading: LoadingImage = .{ ++ .image = .{ .width = 1, .height = 1, .format = .rgba, .data = .{ .native_file = fake.backing(4) } }, ++ .quiet = .no, ++ .temporary_directory = null, ++ }; ++ defer loading.deinit(testing.allocator); ++ var img = try loading.complete(testing.allocator); ++ defer img.deinit(testing.allocator); ++ try testing.expectEqual(@as(usize, 0), fake.reads); ++ try testing.expectEqual(@as(usize, 4), img.data.len()); ++ try testing.expect(!img.data.isPending()); ++ try testing.expect(img.data.bytes() == null); ++ fake.fail = true; ++ try testing.expectError(error.InvalidData, img.materializeFile(testing.allocator)); ++ try testing.expect(img.data == .native_file); ++ try testing.expectEqual(@as(usize, 0), fake.releases); ++ fake.fail = false; ++ try img.materializeFile(testing.allocator); ++ try testing.expectEqual(@as(usize, 1), fake.releases); ++ try testing.expectEqualSlices(u8, &.{ 123, 123, 123, 123 }, img.data.bytes().?); ++ ++ var invalid: LoadingImage = .{ ++ .image = .{ .width = 1, .height = 1, .format = .rgba, .data = .{ .native_file = fake.backing(3) } }, ++ .quiet = .no, ++ .temporary_directory = null, ++ }; ++ try testing.expectError(error.InvalidData, invalid.complete(testing.allocator)); ++ invalid.deinit(testing.allocator); ++ try testing.expectEqual(@as(usize, 2), fake.releases); ++ // q=1 also validates length without reading pixels. ++ var quiet_one: LoadingImage = .{ ++ .image = .{ .width = 1, .height = 1, .format = .rgba, .data = .{ .native_file = fake.backing(4) } }, ++ .quiet = .ok, ++ .temporary_directory = null, ++ }; ++ const reads_before = fake.reads; ++ var quiet_img = try quiet_one.complete(testing.allocator); ++ quiet_one.deinit(testing.allocator); ++ try testing.expectEqual(reads_before, fake.reads); ++ quiet_img.deinit(testing.allocator); ++ try testing.expectEqual(@as(usize, 3), fake.releases); ++ var canceled: LoadingImage = .{ ++ .image = .{ .data = .{ .native_file = fake.backing(4) } }, ++ .quiet = .failures, ++ .temporary_directory = null, ++ }; ++ canceled.deinit(testing.allocator); ++ try testing.expectEqual(@as(usize, 4), fake.releases); ++} ++ ++test "native file eligibility is whole raw RGBA only" { ++ const testing = std.testing; ++ var t: command.Transmission = .{ .medium = .file, .format = .rgba, .width = 1, .height = 2 }; ++ try testing.expectEqual(@as(?usize, 8), LoadingImage.nativeFileExpectedLen(t)); ++ t.size = 8; ++ try testing.expectEqual(@as(?usize, 8), LoadingImage.nativeFileExpectedLen(t)); ++ t.size = 7; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.size = 0; ++ t.more_chunks = true; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.more_chunks = false; ++ t.offset = 1; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.offset = 0; ++ t.format = .rgb; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.format = .png; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.format = .rgba; ++ t.medium = .temporary_file; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.medium = .shared_memory; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.medium = .file; ++ t.compression = .zlib_deflate; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.compression = .none; ++ t.width = 0; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.width = 10001; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++ t.width = 10000; ++ t.height = 10000; ++ try testing.expect(LoadingImage.nativeFileExpectedLen(t) == null); ++} ++ ++test "native file snapshot rejects missing callbacks before retaining" { ++ const testing = std.testing; ++ const Fake = struct { ++ reads: usize = 0, ++ releases: usize = 0, ++ missing_read: bool = true, ++ missing_release: bool = false, ++ ++ fn read(ctx: ?*anyopaque, dest: [*]u8, len: usize) callconv(.c) bool { ++ const self: *@This() = @ptrCast(@alignCast(ctx.?)); ++ self.reads += 1; ++ @memset(dest[0..len], 42); ++ return true; ++ } ++ ++ fn release(ctx: ?*anyopaque) callconv(.c) void { ++ const self: *@This() = @ptrCast(@alignCast(ctx.?)); ++ self.releases += 1; ++ } ++ ++ fn snapshot(ctx: ?*anyopaque, request: *const SnapshotFileRequest, out: *FileBacking) bool { ++ const self: *@This() = @ptrCast(@alignCast(ctx.?)); ++ out.* = .{ ++ .context = ctx, ++ .identity = 1, ++ .len = @intCast(request.expected_len), ++ .read = if (self.missing_read) null else &read, ++ .release = if (self.missing_release) null else &release, ++ }; ++ return true; ++ } ++ }; ++ var fake: Fake = .{}; ++ const hook: SnapshotFileHook = .{ .context = &fake, .callback = &Fake.snapshot }; ++ const request: SnapshotFileRequest = .{ .fd = -1, .expected_len = 4 }; ++ var loading: LoadingImage = .{ ++ .image = .{ .width = 1, .height = 1, .format = .rgba }, ++ .quiet = .no, ++ .temporary_directory = null, ++ }; ++ // Missing read: transferred context is immediately released, not stored. ++ try testing.expect(!try loading.trySnapshotFile(hook, request)); ++ try testing.expect(loading.image.data != .native_file); ++ try testing.expectEqual(@as(usize, 1), fake.releases); ++ ++ // Missing release (or both): reject without calling a null pointer. ++ fake.missing_read = false; ++ fake.missing_release = true; ++ try testing.expect(!try loading.trySnapshotFile(hook, request)); ++ try testing.expect(loading.image.data != .native_file); ++ fake.missing_read = true; ++ try testing.expect(!try loading.trySnapshotFile(hook, request)); ++ try testing.expect(loading.image.data != .native_file); ++ try testing.expectEqual(@as(usize, 0), fake.reads); ++ try testing.expectEqual(@as(usize, 1), fake.releases); ++ ++ // The ordinary reader can still supply bytes after rejection. ++ try loading.addData(testing.allocator, &.{ 1, 2, 3, 4 }); ++ var img = try loading.complete(testing.allocator); ++ loading.deinit(testing.allocator); ++ try testing.expectEqualSlices(u8, &.{ 1, 2, 3, 4 }, img.data.bytes().?); ++ img.deinit(testing.allocator); ++ try testing.expectEqual(@as(usize, 1), fake.releases); ++ ++ // Valid callback ownership is unchanged: zero-read acceptance, one release. ++ fake.missing_read = false; ++ fake.missing_release = false; ++ var accepted: LoadingImage = .{ ++ .image = .{ .width = 1, .height = 1, .format = .rgba }, ++ .quiet = .no, ++ .temporary_directory = null, ++ }; ++ try testing.expect(try accepted.trySnapshotFile(hook, request)); ++ try testing.expect(accepted.image.data == .native_file); ++ try testing.expectEqual(@as(usize, 0), fake.reads); ++ accepted.deinit(testing.allocator); ++ try testing.expectEqual(@as(usize, 2), fake.releases); ++} +diff --git a/vendor/libghostty-vt/src/terminal/kitty/graphics_storage.zig b/vendor/libghostty-vt/src/terminal/kitty/graphics_storage.zig +index 897c8944..bd8cd424 100644 +--- a/vendor/libghostty-vt/src/terminal/kitty/graphics_storage.zig ++++ b/vendor/libghostty-vt/src/terminal/kitty/graphics_storage.zig +@@ -154,7 +154,7 @@ pub const ImageStorage = struct { + if (img.generation != self.generation) return false; + + const expected_len = switch (img.data) { +- .complete => return false, ++ .complete, .encoded_png, .native_file => return false, + .pending => |len| len, + }; + if (data.len != expected_len) return false; +@@ -975,6 +975,15 @@ pub const ImageStorage = struct { + img.generation = self.generation; + } + ++ pub fn materializeBacking(self: *ImageStorage, io: std.Io, alloc: Allocator, img: *Image) !void { ++ if (img.data != .encoded_png and img.data != .native_file) return; ++ const before = img.data.len(); ++ try img.materializeFile(alloc); ++ try img.materializePng(alloc); ++ self.total_bytes -= before - img.data.len(); ++ self.markImageContentChanged(io, img); ++ } ++ + /// Convert a stored image's base data to RGBA in place, adjusting + /// byte accounting. All animation composition happens in RGBA; + /// this is called before the first composition into an image. |