From 0d1a39dcee6a0f244a036c55935970f40cfa76b8 Mon Sep 17 00:00:00 2001 From: Can Celik Date: Sun, 27 Sep 2026 20:29:49 +0300 Subject: feat: accept mouse events on terminal session control (#4685) --- .../website/src/content/docs/cli-reference.mdx | 13 ++++- .../website/src/content/docs/ja/cli-reference.mdx | 2 +- .../src/content/docs/ja/persistence-remote.mdx | 2 +- .../src/content/docs/persistence-remote.mdx | 5 +- .../src/content/docs/zh-cn/cli-reference.mdx | 2 +- .../src/content/docs/zh-cn/persistence-remote.mdx | 2 +- src/client/terminal_sessions.rs | 60 ++++++++++++++++++++- src/client/tests/mod.rs | 61 ++++++++++++++++++++++ src/server/headless/tests/mod.rs | 46 ++++++++++++++++ 9 files changed, 183 insertions(+), 10 deletions(-) diff --git a/docs/next/website/src/content/docs/cli-reference.mdx b/docs/next/website/src/content/docs/cli-reference.mdx index f9aff8fe..05a539d6 100644 --- a/docs/next/website/src/content/docs/cli-reference.mdx +++ b/docs/next/website/src/content/docs/cli-reference.mdx @@ -377,8 +377,17 @@ Detach from direct attach with `ctrl+b q`. Send literal `ctrl+b` with `ctrl+b ct terminal, or agent target. It prints the same newline-delimited `terminal.frame` and `terminal.closed` records as observe mode. It reads newline-delimited JSON commands on stdin: `terminal.input`, `terminal.resize`, -`terminal.scroll`, and `terminal.release`. One controller can own a terminal at -a time; use `--takeover` to replace it. +`terminal.scroll`, `terminal.mouse`, and `terminal.release`. One controller can +own a terminal at a time; use `--takeover` to replace it. +`terminal.mouse` sends one mouse event at a zero-based cell of the controller's +viewport, for example +`{"type":"terminal.mouse","action":"down","button":"left","column":12,"row":5}`. +`action` is `down`, `up`, `drag`, or `move`; `button` is `left` (default), +`right`, or `middle`; `modifiers` is optional and combines Shift = 1, Ctrl = 2, +and Alt = 4 (default 0). A click is `down` followed by `up`. Herdr encodes the +event for the application's mouse mode and drops it when the application has +not enabled mouse reporting, so a plain shell never receives stray input. +`move` reaches only applications that track all mouse motion. `terminal session observe` opens a read-only live terminal stream for a pane, terminal, or agent target. It prints newline-delimited JSON `terminal.frame` records with base64-encoded ANSI bytes, then a `terminal.closed` record when diff --git a/docs/next/website/src/content/docs/ja/cli-reference.mdx b/docs/next/website/src/content/docs/ja/cli-reference.mdx index 81894daa..b622c245 100644 --- a/docs/next/website/src/content/docs/ja/cli-reference.mdx +++ b/docs/next/website/src/content/docs/ja/cli-reference.mdx @@ -351,7 +351,7 @@ herdr terminal title clear ``` ダイレクトアタッチからは `ctrl+b q` でデタッチします。リテラルの `ctrl+b` は `ctrl+b ctrl+b` で送ります。 -`terminal session control` は、ペイン、ターミナル、エージェントを対象に書き込み可能なライブストリームを開きます。observe と同じ改行区切り JSON の `terminal.frame` と `terminal.closed` を出力し、標準入力から `terminal.input`、`terminal.resize`、`terminal.scroll`、`terminal.release` を受け取ります。制御クライアントは同時に 1 つだけで、`--takeover` で置き換えられます。 +`terminal session control` は、ペイン、ターミナル、エージェントを対象に書き込み可能なライブストリームを開きます。observe と同じ改行区切り JSON の `terminal.frame` と `terminal.closed` を出力し、標準入力から `terminal.input`、`terminal.resize`、`terminal.scroll`、`terminal.mouse`、`terminal.release` を受け取ります。制御クライアントは同時に 1 つだけで、`--takeover` で置き換えられます。`terminal.mouse` は 0 始まりのセル位置にマウスイベントを 1 つ送ります(例: `{"type":"terminal.mouse","action":"down","button":"left","column":12,"row":5}`)。`action` は `down`、`up`、`drag`、`move` のいずれかで、`button` は `left`(既定)、`right`、`middle`、`modifiers` は省略可能で Shift = 1、Ctrl = 2、Alt = 4 の組み合わせです(既定 0)。クリックは `down` の後に `up` を送ります。Herdr はアプリのマウスモードに合わせてエンコードし、マウス報告を有効にしていないアプリには送らないため、通常のシェルに余計な入力が届くことはありません。`move` はすべてのマウス移動を追跡するアプリにのみ届きます。 `terminal session observe` は同じ対象の読み取り専用ストリームを開きます。base64 の ANSI バイト列を含む `terminal.frame` と、サーバー終了時の `terminal.closed` を改行区切り JSON で出力します。複数の観測クライアントは入力、リサイズ、スクロール、制御権を取得せずに同じターミナルを監視できます。Linux/macOS ではソケットへの書き込みが 30 秒間進まない観測クライアントを切断します。その場合は最後の `terminal.closed` が届かないことがありますが、ペインや他のクライアントは動き続けます。出力のない静かなペインはこのタイムアウトの対象ではありません。 diff --git a/docs/next/website/src/content/docs/ja/persistence-remote.mdx b/docs/next/website/src/content/docs/ja/persistence-remote.mdx index 94bffb8b..ae695ab3 100644 --- a/docs/next/website/src/content/docs/ja/persistence-remote.mdx +++ b/docs/next/website/src/content/docs/ja/persistence-remote.mdx @@ -148,4 +148,4 @@ base64 の ANSI データを含む `terminal.frame` を改行区切り JSON で herdr terminal session control w1:p1 --takeover --cols 120 --rows 40 ``` -同じフレームを出力し、標準入力から改行区切り JSON コマンドを読み取ります。`terminal.input` はテキストまたは base64 データ、`terminal.resize` は表示サイズ、`terminal.scroll` はビューポートのスクロール、`terminal.release` はコントローラーの終了を扱います。入力とリサイズを所有できるコントローラーは 1 つだけです。 +同じフレームを出力し、標準入力から改行区切り JSON コマンドを読み取ります。`terminal.input` はテキストまたは base64 データ、`terminal.resize` は表示サイズ、`terminal.scroll` はビューポートのスクロール、`terminal.mouse` はマウス報告を有効にしたアプリへのセル位置のマウスイベント、`terminal.release` はコントローラーの終了を扱います。入力とリサイズを所有できるコントローラーは 1 つだけです。 diff --git a/docs/next/website/src/content/docs/persistence-remote.mdx b/docs/next/website/src/content/docs/persistence-remote.mdx index 4f42cef5..0ffa1406 100644 --- a/docs/next/website/src/content/docs/persistence-remote.mdx +++ b/docs/next/website/src/content/docs/persistence-remote.mdx @@ -155,5 +155,6 @@ herdr terminal session control w1:p1 --takeover --cols 120 --rows 40 Control mode prints the same newline-delimited frame records and reads newline-delimited JSON commands on stdin. `terminal.input` sends text or base64 bytes, `terminal.resize` changes the controller viewport, -`terminal.scroll` scrolls the attached viewport, and `terminal.release` closes -the controller. Only one controller owns input and resize at a time. +`terminal.scroll` scrolls the attached viewport, `terminal.mouse` sends a mouse +event at a cell to applications that enabled mouse reporting, and +`terminal.release` closes the controller. Only one controller owns input and resize at a time. diff --git a/docs/next/website/src/content/docs/zh-cn/cli-reference.mdx b/docs/next/website/src/content/docs/zh-cn/cli-reference.mdx index 080abb48..987e0c4a 100644 --- a/docs/next/website/src/content/docs/zh-cn/cli-reference.mdx +++ b/docs/next/website/src/content/docs/zh-cn/cli-reference.mdx @@ -351,7 +351,7 @@ herdr terminal title clear ``` 从直接附加中用 `ctrl+b q` 分离。用 `ctrl+b ctrl+b` 发送字面的 `ctrl+b`。 -`terminal session control` 为窗格、终端或智能体打开可写的实时终端流，输出与 observe 相同的逐行 JSON `terminal.frame` 和 `terminal.closed`，并从标准输入读取 `terminal.input`、`terminal.resize`、`terminal.scroll`、`terminal.release` 命令。同一终端同时只能有一个控制方，可用 `--takeover` 接管。 +`terminal session control` 为窗格、终端或智能体打开可写的实时终端流，输出与 observe 相同的逐行 JSON `terminal.frame` 和 `terminal.closed`，并从标准输入读取 `terminal.input`、`terminal.resize`、`terminal.scroll`、`terminal.mouse`、`terminal.release` 命令。同一终端同时只能有一个控制方，可用 `--takeover` 接管。`terminal.mouse` 在从 0 开始计数的单元格位置发送一个鼠标事件，例如 `{"type":"terminal.mouse","action":"down","button":"left","column":12,"row":5}`。`action` 为 `down`、`up`、`drag` 或 `move`；`button` 为 `left`（默认）、`right` 或 `middle`；`modifiers` 可省略，由 Shift = 1、Ctrl = 2、Alt = 4 组合（默认 0）。一次点击就是先 `down` 再 `up`。Herdr 会按应用的鼠标模式编码事件；应用未开启鼠标报告时直接丢弃，因此普通 shell 不会收到多余输入。`move` 只会发给追踪全部鼠标移动的应用。 `terminal session observe` 为相同目标打开只读实时流，输出包含 base64 ANSI 字节的逐行 JSON `terminal.frame`，服务器关闭流时输出 `terminal.closed`。多个观察方可以同时查看终端，不获取输入、调整尺寸、滚动或接管权限。Linux/macOS 上，若套接字写入连续 30 秒没有进展，观察方会被断开；此时可能没有最后的 `terminal.closed` 记录，但窗格和其他客户端继续运行。安静、没有输出的窗格不会触发此超时。 diff --git a/docs/next/website/src/content/docs/zh-cn/persistence-remote.mdx b/docs/next/website/src/content/docs/zh-cn/persistence-remote.mdx index ddd0b0fb..6b0be4d8 100644 --- a/docs/next/website/src/content/docs/zh-cn/persistence-remote.mdx +++ b/docs/next/website/src/content/docs/zh-cn/persistence-remote.mdx @@ -148,4 +148,4 @@ herdr terminal session observe w1:p1 --cols 120 --rows 40 herdr terminal session control w1:p1 --takeover --cols 120 --rows 40 ``` -控制模式输出相同的帧，并从标准输入读取逐行 JSON 命令。`terminal.input` 发送文本或 base64 数据，`terminal.resize` 更改视口尺寸，`terminal.scroll` 滚动视口，`terminal.release` 关闭控制器。同一时间只有一个控制器能拥有输入和调整尺寸的权限。 +控制模式输出相同的帧，并从标准输入读取逐行 JSON 命令。`terminal.input` 发送文本或 base64 数据，`terminal.resize` 更改视口尺寸，`terminal.scroll` 滚动视口，`terminal.mouse` 向已开启鼠标报告的应用发送单元格位置的鼠标事件，`terminal.release` 关闭控制器。同一时间只有一个控制器能拥有输入和调整尺寸的权限。 diff --git a/src/client/terminal_sessions.rs b/src/client/terminal_sessions.rs index 6d3160c8..aeecbf47 100644 --- a/src/client/terminal_sessions.rs +++ b/src/client/terminal_sessions.rs @@ -7,8 +7,8 @@ use tracing::info; use crate::ipc::LocalStream; use crate::protocol::{ - self, AttachScrollDirection, AttachScrollSource, ClientMessage, RenderEncoding, ServerMessage, - MAX_GRAPHICS_FRAME_SIZE, + self, AttachScrollDirection, AttachScrollSource, ClientMessage, ClientMouseButton, + ClientMouseKind, ClientMousePosition, RenderEncoding, ServerMessage, MAX_GRAPHICS_FRAME_SIZE, }; use crate::server::socket_paths::client_socket_path; @@ -187,10 +187,38 @@ enum TerminalControlCommand { #[serde(default)] modifiers: u8, }, + #[serde(rename = "terminal.mouse")] + Mouse { + action: TerminalControlMouseAction, + #[serde(default)] + button: TerminalControlMouseButton, + column: u16, + row: u16, + #[serde(default)] + modifiers: u8, + }, #[serde(rename = "terminal.release")] Release {}, } +#[derive(Clone, Copy, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +enum TerminalControlMouseAction { + Down, + Up, + Drag, + Move, +} + +#[derive(Clone, Copy, Default, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +enum TerminalControlMouseButton { + #[default] + Left, + Right, + Middle, +} + #[derive(Clone, Copy, serde::Deserialize)] #[serde(rename_all = "snake_case")] enum TerminalControlScrollDirection { @@ -273,6 +301,34 @@ pub(super) fn terminal_control_command_from_json(raw: &str) -> Result { + let button = match button { + TerminalControlMouseButton::Left => ClientMouseButton::Left, + TerminalControlMouseButton::Right => ClientMouseButton::Right, + TerminalControlMouseButton::Middle => ClientMouseButton::Middle, + }; + let kind = match action { + TerminalControlMouseAction::Down => ClientMouseKind::Down(button), + TerminalControlMouseAction::Up => ClientMouseKind::Up(button), + TerminalControlMouseAction::Drag => ClientMouseKind::Drag(button), + TerminalControlMouseAction::Move => ClientMouseKind::Moved, + }; + // The server encodes the event for the pane's mouse mode and drops it + // when the application has not enabled mouse reporting. + Ok(ClientMessage::AttachMouse { + kind, + position: ClientMousePosition::Cell { column, row }, + geometry: None, + modifiers, + lines: 1, + }) + } TerminalControlCommand::Release {} => Ok(ClientMessage::Detach), } } diff --git a/src/client/tests/mod.rs b/src/client/tests/mod.rs index 8519ea07..2fcac6eb 100644 --- a/src/client/tests/mod.rs +++ b/src/client/tests/mod.rs @@ -886,6 +886,67 @@ fn terminal_control_resize_command_maps_to_client_resize() { assert!(!pixel_mouse); } +#[test] +fn terminal_control_mouse_command_maps_to_attach_mouse() { + let action = terminal_control_command_from_json( + r#"{"type":"terminal.mouse","action":"down","column":12,"row":5}"#, + ) + .unwrap(); + let ClientMessage::AttachMouse { + kind, + position, + geometry, + modifiers, + lines, + } = action + else { + panic!("expected attach mouse command"); + }; + assert_eq!( + kind, + crate::protocol::ClientMouseKind::Down(crate::protocol::ClientMouseButton::Left) + ); + assert_eq!( + position, + crate::protocol::ClientMousePosition::Cell { column: 12, row: 5 } + ); + assert_eq!((geometry, modifiers, lines), (None, 0, 1)); +} + +#[test] +fn terminal_control_mouse_command_maps_every_action_and_button() { + use crate::protocol::{ClientMouseButton as Button, ClientMouseKind as Kind}; + for (json, expected) in [ + (r#""action":"up","button":"right""#, Kind::Up(Button::Right)), + ( + r#""action":"drag","button":"middle""#, + Kind::Drag(Button::Middle), + ), + (r#""action":"move""#, Kind::Moved), + ] { + let raw = format!(r#"{{"type":"terminal.mouse",{json},"column":0,"row":0,"modifiers":4}}"#); + let ClientMessage::AttachMouse { + kind, modifiers, .. + } = terminal_control_command_from_json(&raw).unwrap() + else { + panic!("expected attach mouse command"); + }; + assert_eq!((kind, modifiers), (expected, 4), "{raw}"); + } +} + +#[test] +fn terminal_control_mouse_command_rejects_unknown_actions_and_missing_cells() { + assert!(terminal_control_command_from_json( + r#"{"type":"terminal.mouse","action":"click","column":1,"row":1}"# + ) + .is_err()); + assert!(terminal_control_command_from_json( + r#"{"type":"terminal.mouse","action":"down","column":1}"# + ) + .is_err()); +} + #[test] fn terminal_control_scroll_command_maps_to_attach_scroll() { let action = terminal_control_command_from_json( diff --git a/src/server/headless/tests/mod.rs b/src/server/headless/tests/mod.rs index 34258dd3..4def3fa4 100644 --- a/src/server/headless/tests/mod.rs +++ b/src/server/headless/tests/mod.rs @@ -6309,6 +6309,52 @@ fn direct_terminal_mouse_uses_runtime_protocol_encoding() { }); } +#[test] +fn direct_terminal_mouse_sends_nothing_without_mouse_reporting() { + with_terminal_session_test_server(|server, runtime_terminal_id, terminal_id, _pane_id| { + // A plain shell never enabled mouse reporting, so a controller's click + // must not reach it as stray escape bytes. + let (runtime, mut input_rx) = + crate::terminal::TerminalRuntime::test_with_channel_and_scrollback_bytes( + 80, 24, 0, b"", 4, + ); + server + .app + .terminal_runtimes + .insert(runtime_terminal_id, runtime); + server.clients.insert( + 1, + ClientConnection::new_with_mode( + ClientConnectionMode::TerminalAttach { + terminal_id: terminal_id.clone(), + }, + (80, 24), + crate::kitty_graphics::HostCellSize::default(), + 1, + RenderEncoding::TerminalAnsi, + None, + ), + ); + + for kind in [ + protocol::ClientMouseKind::Down(protocol::ClientMouseButton::Left), + protocol::ClientMouseKind::Drag(protocol::ClientMouseButton::Left), + protocol::ClientMouseKind::Up(protocol::ClientMouseButton::Left), + protocol::ClientMouseKind::Moved, + ] { + server.handle_server_event(ServerEvent::ClientAttachMouse { + client_id: 1, + kind, + position: protocol::ClientMousePosition::Cell { column: 10, row: 5 }, + geometry: None, + modifiers: 0, + lines: 1, + }); + } + assert!(input_rx.try_recv().is_err()); + }); +} + #[test] fn direct_terminal_pixel_mouse_uses_runtime_tracking_and_coordinates() { with_terminal_session_test_server(|server, runtime_terminal_id, terminal_id, _pane_id| { -- cgit v1.2.3