1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
|
import { Type, type Static } from "typebox";
import { Text } from "@mariozechner/pi-tui";
import type { BrowserClient } from "../client";
import { Coords, MouseButton } from "../schemas/common";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr } from "../util/tool";
import { captureWithCrosshair } from "./screenshot";
const ClickArgs = Type.Object({
...Coords.properties,
button: Type.Optional(MouseButton),
count: Type.Optional(
Type.Integer({
minimum: 1,
maximum: 3,
default: 1,
description: "Number of clicks (1 = single, 2 = double). Default: 1",
}),
),
});
type ClickArgsT = Static<typeof ClickArgs>;
const dispatchClick = async (
client: BrowserClient,
args: ClickArgsT,
): Promise<Result<void, ToolErr>> => {
const button = args.button ?? "left";
const count = args.count ?? 1;
const pressed = await client.session().call("Input.dispatchMouseEvent", {
type: "mousePressed", x: args.x, y: args.y, button, clickCount: count,
});
if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message });
const released = await client.session().call("Input.dispatchMouseEvent", {
type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count,
});
if (!released.success) return err({ kind: "cdp_error", message: released.error.message });
return ok(undefined);
};
export const clickTool = defineBrowserTool({
name: "browser_click",
label: "Browser Click",
description:
"Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.",
promptSnippet: "Click at pixel coordinates (x, y) on the page",
promptGuidelines: [
"Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).",
"Capture a browser_screenshot BEFORE clicking to find the right coordinates.",
"Capture another browser_screenshot AFTER clicking to verify the action worked.",
"Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.",
"If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.",
"For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.",
],
parameters: ClickArgs,
renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0),
async handler(args, { client }) {
const clicked = await dispatchClick(client, args);
if (!clicked.success) return clicked;
if (process.env["BH_DEBUG_CLICKS"]) {
const debug = await captureWithCrosshair(client, { x: args.x, y: args.y });
if (debug.success) {
return ok({
text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`,
details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y },
});
}
}
return ok({
text: `Clicked at (${args.x}, ${args.y})`,
details: { x: args.x, y: args.y },
});
},
});
|
