1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
import { Type } from "typebox";
import { Text } from "@mariozechner/pi-tui";
import type { BrowserClient } from "../client";
import { Coords, MouseButton } from "../schemas/common";
import { type Result, ok } from "../util/result";
import { debugClicksEnabled } from "../util/debug";
import { defineBrowserTool, type ToolErr } from "../util/tool";
import { cdpCall } from "./cdp-call";
import { captureWithCrosshair } from "./screenshot-capture";
import { interactiveDiff } from "./ref-resolve";
import { resolveTarget } from "./target";
const ClickArgs = Type.Object({
ref: Type.Optional(
Type.String({
description:
"Stable element ref from browser_snapshot (e.g. 'e12'). PREFERRED over x/y — survives re-renders. When set, x/y are ignored.",
}),
),
x: Type.Optional(Coords.properties.x),
y: Type.Optional(Coords.properties.y),
button: Type.Optional(MouseButton),
count: Type.Optional(
Type.Integer({
minimum: 1,
maximum: 3,
default: 1,
description: "Number of clicks (1 = single, 2 = double). Default: 1",
}),
),
});
const dispatchClick = async (
client: BrowserClient,
x: number,
y: number,
button: "left" | "right" | "middle",
count: number,
): Promise<Result<void, ToolErr>> => {
// Some focus/hover handlers and React synthetic events only fire when a mousemove precedes the press, so without it a click lands without focusing.
const moved = await cdpCall(client, "Input.dispatchMouseEvent", { type: "mouseMoved", x, y });
if (!moved.success) return moved;
const pressed = await cdpCall(client, "Input.dispatchMouseEvent", {
type: "mousePressed", x, y, button, clickCount: count,
});
if (!pressed.success) return pressed;
const released = await cdpCall(client, "Input.dispatchMouseEvent", {
type: "mouseReleased", x, y, button, clickCount: count,
});
if (!released.success) return released;
return ok(undefined);
};
export const clickTool = defineBrowserTool({
name: "browser_click",
label: "Browser Click",
description:
"Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.",
promptSnippet: "Click an element by ref (preferred) or pixel coordinates",
promptGuidelines: [
"PREFER `ref` from browser_snapshot (the outline shows '[eN]' for every interactive element) — it survives re-renders, unlike coordinates which go stale after a save/edit reflows the page.",
"Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.",
"A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.",
"After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).",
"Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.",
"If a click doesn't register, run pi with --browser-debug-clicks (or set BH_DEBUG_CLICKS=1) to get annotated screenshots. For React/Vue components ignoring clicks, try browser_dispatch_key.",
],
parameters: ClickArgs,
concurrency: "serialized",
renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0),
async handler(args, { client }) {
const button = args.button ?? "left";
const count = args.count ?? 1;
const resolved = await resolveTarget(client, args);
if (!resolved.success) return resolved;
const { x, y, label: target } = resolved.data;
const clicked = await dispatchClick(client, x, y, button, count);
if (!clicked.success) return clicked;
const diff = await interactiveDiff(client);
if (debugClicksEnabled()) {
const debug = await captureWithCrosshair(client, { x, y });
if (debug.success) {
return ok({
text: `Clicked at ${target}\n[DEBUG] Overlay screenshot: ${debug.data.path}${diff}`,
details: { debugScreenshotPath: debug.data.path, x, y, ref: args.ref },
});
}
}
return ok({
text: `Clicked at ${target}${diff}`,
details: { x, y, ref: args.ref },
});
},
});
|
