aboutsummaryrefslogtreecommitdiff
path: root/docs/next
diff options
context:
space:
mode:
authorCan Celik <ogulcancelik@gmail.com>2026-08-09 02:04:39 +0300
committerGitHub <noreply@github.com>2026-08-09 02:04:39 +0300
commit1777e9bba32b953ed1ad203b4a16d01105539000 (patch)
tree22096973812c9baba822453d8470bdcfe64fc2a6 /docs/next
parent36a8d36a7258e08b40c01bf521b1f4006525f143 (diff)
feat(graphics): add direct pane frame streaming (#2523)
* feat(graphics): add direct pane frame streaming * feat(graphics): expose pane visibility * test(graphics): preserve legacy terminal image output * fix(graphics): harden streaming lifecycle * fix(input): preserve mouse releases outside panes
Diffstat (limited to 'docs/next')
-rw-r--r--docs/next/CHANGELOG.md1
-rw-r--r--docs/next/api/herdr-api.schema.json92
-rw-r--r--docs/next/website/src/content/docs/socket-api.mdx41
3 files changed, 125 insertions, 9 deletions
diff --git a/docs/next/CHANGELOG.md b/docs/next/CHANGELOG.md
index 343086aa..5d123a8c 100644
--- a/docs/next/CHANGELOG.md
+++ b/docs/next/CHANGELOG.md
@@ -11,6 +11,7 @@
### Changed
- Bumped the client/server protocol version to 20 for pane terminal bell forwarding.
+- Experimental pane graphics now support bounded named layers, acknowledged full-RGBA primary-layer direct file frames on audited local terminals, owned BGRA fallback, exact pixel mouse input, and placement-only resize replay.
### Fixed
- `herdr config check` now reports unknown built-in theme names instead of silently accepting them. (#2452)
diff --git a/docs/next/api/herdr-api.schema.json b/docs/next/api/herdr-api.schema.json
index 1e8d7830..8a552da0 100644
--- a/docs/next/api/herdr-api.schema.json
+++ b/docs/next/api/herdr-api.schema.json
@@ -2452,6 +2452,12 @@
},
"PaneGraphicsClearParams": {
"properties": {
+ "layer_id": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
"pane_id": {
"type": "string"
}
@@ -2465,7 +2471,8 @@
"enum": [
"png",
"rgb",
- "rgba"
+ "rgba",
+ "bgra"
],
"type": "string"
},
@@ -2515,6 +2522,12 @@
"minimum": 0,
"type": "integer"
},
+ "layer_id": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
"pane_id": {
"type": "string"
},
@@ -2526,6 +2539,11 @@
"viewport_col": 0,
"viewport_row": 0
}
+ },
+ "z_index": {
+ "default": 0,
+ "format": "int32",
+ "type": "integer"
}
},
"required": [
@@ -9236,6 +9254,30 @@
},
{
"properties": {
+ "revision": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "sequence": {
+ "format": "uint64",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "type": {
+ "const": "pane_graphics_frame_ack",
+ "type": "string"
+ }
+ },
+ "required": [
+ "type",
+ "sequence",
+ "revision"
+ ],
+ "type": "object"
+ },
+ {
+ "properties": {
"cell_height_px": {
"format": "uint32",
"minimum": 0,
@@ -9246,6 +9288,51 @@
"minimum": 0,
"type": "integer"
},
+ "file_frame_damage": {
+ "default": false,
+ "description": "Accepts damage metadata while still consuming a complete canonical file.",
+ "type": "boolean"
+ },
+ "file_frame_directory": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "file_frame_formats": {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ "file_frame_max_bytes": {
+ "format": "uint",
+ "minimum": 0,
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "file_frame_transport": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "max_layers_per_pane": {
+ "default": 0,
+ "format": "uint",
+ "minimum": 0,
+ "type": "integer"
+ },
+ "pane_visible": {
+ "description": "True only when this pane is on the currently rendered terminal surface.",
+ "type": "boolean"
+ },
+ "pixel_mouse": {
+ "default": false,
+ "type": "boolean"
+ },
"type": {
"const": "pane_graphics_info",
"type": "string"
@@ -9254,7 +9341,8 @@
"required": [
"type",
"cell_width_px",
- "cell_height_px"
+ "cell_height_px",
+ "pane_visible"
],
"type": "object"
},
diff --git a/docs/next/website/src/content/docs/socket-api.mdx b/docs/next/website/src/content/docs/socket-api.mdx
index bcf0f14c..86551c03 100644
--- a/docs/next/website/src/content/docs/socket-api.mdx
+++ b/docs/next/website/src/content/docs/socket-api.mdx
@@ -177,10 +177,20 @@ Clients can treat `offset_from_bottom == 0` as at-bottom state.
Pane graphics let a plugin place image data over a pane. They are available
only when `[experimental].kitty_graphics = true`; otherwise every pane graphics
-method returns `feature_disabled`. Calling `pane.graphics.info` returns the
-attached client's cell width and height in pixels without creating a graphics
-layer. `pane.graphics.set` accepts `png`, `rgb`, or `rgba` data in `data_base64`,
-and `pane.graphics.clear` removes the layer.
+method returns `feature_disabled`. Calling `pane.graphics.info` explicitly
+activates capability discovery and returns the attached client's cell size,
+file-frame options, pixel-mouse support, the 16-layer limit, and `pane_visible`.
+`pane_visible` is true only when the target is in the active workspace and tab
+and is not hidden by zoom. Short-lived UI modes do not change it.
+
+`pane.graphics.set`, `pane.graphics.clear`, and `pane.graphics.stream` accept an
+optional `layer_id` (default `primary`). Set and stream also accept `z_index`;
+layers are placed in stable `(z_index, layer_id)` order. Each stream exclusively owns its layer, and
+closing it removes that layer. Inline frames accept `png`, `rgb`, `rgba`, or
+`bgra`; BGRA is normalized once to owned RGBA. Herdr advances the host cache
+one image transaction per render pass, so arbitrary layer sets progress without
+an aggregate frame. Headless transport keeps each transaction within its 32 MiB
+wire limit; local monolithic rendering does not apply that transport limit.
```json
{"id":"graphics_info","method":"pane.graphics.info","params":{"pane_id":"w1:p1"}}
@@ -190,14 +200,31 @@ and `pane.graphics.clear` removes the layer.
For repeated frames, open a dedicated socket with `pane.graphics.stream`. After
Herdr replies with `ok`, send one JSON header and then exactly `data_length` raw
-bytes per frame. A stream owns that pane's graphics layer until the socket
-closes; concurrent set, clear, or stream requests return `stream_conflict`.
+bytes per inline frame. Concurrent operations on that layer return
+`stream_conflict`.
```json
-{"id":"graphics_stream","method":"pane.graphics.stream","params":{"pane_id":"w1:p1"}}
+{"id":"graphics_stream","method":"pane.graphics.stream","params":{"pane_id":"w1:p1","z_index":0}}
{"format":"png","image_width":800,"image_height":600,"data_length":12345,"placement":{"viewport_col":0,"viewport_row":0,"grid_cols":80,"grid_rows":30}}
```
+When `pane.graphics.info` advertises `file_frame_transport: "direct-kitty"`, an
+eligible local Ghostty, kitty, or WezTerm client may submit an immutable private
+`rgba` or `bgra` file with `file.path`, `sequence`, and `revision`. Direct Kitty
+file transport is reserved for the default `primary` page layer; named secondary
+layers use owned inline RGBA. BGRA is always copied, swizzled, and rendered
+inline. Herdr replies with a `pane_graphics_frame_ack` only after the terminal
+accepts the file, or after a safe owned inline fallback is installed. Confirmed
+file-transport failure disables direct files for that client connection without
+disabling exact pixel mouse. A timeout or client loss closes the stream without
+acknowledging source reuse. Monolithic `--no-session` mode advertises neither
+fast file transport nor exact pixel mouse and remains on owned inline fallback.
+
+Direct files are always complete canonical `width * height * 4` RGBA frames.
+`file_frame_damage: true` means Herdr accepts optional damage metadata for
+producer-side canonical-ring efficiency; it still copies or presents the full file.
+Resize and full redraw replay placements without retransmitting pixels.
+
`pane.layout` returns the tab layout snapshot with `workspace_id`, `tab_id`,
`zoomed`, outer `area`, `focused_pane_id`, pane rects, and split rects/ratios.
`pane.neighbor` and `pane.edges` include that same layout snapshot so clients