aboutsummaryrefslogtreecommitdiff
path: root/src/domains/click.ts
diff options
context:
space:
mode:
Diffstat (limited to 'src/domains/click.ts')
-rw-r--r--src/domains/click.ts79
1 files changed, 79 insertions, 0 deletions
diff --git a/src/domains/click.ts b/src/domains/click.ts
new file mode 100644
index 0000000..b3aa80a
--- /dev/null
+++ b/src/domains/click.ts
@@ -0,0 +1,79 @@
+import { writeFile } from "node:fs/promises";
+import { Type, type Static } from "typebox";
+import { Text } from "@mariozechner/pi-tui";
+import type { BrowserClient } from "../client";
+import { Coords, MouseButton } from "../schemas/common";
+import { screenshotPath } from "../util/paths";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr } from "../util/tool";
+
+const ClickArgs = Type.Object({
+ ...Coords.properties,
+ button: Type.Optional(MouseButton),
+ count: Type.Optional(
+ Type.Integer({
+ minimum: 1,
+ maximum: 3,
+ default: 1,
+ description: "Number of clicks (1 = single, 2 = double). Default: 1",
+ }),
+ ),
+});
+type ClickArgsT = Static<typeof ClickArgs>;
+
+const dispatchClick = async (
+ client: BrowserClient,
+ args: ClickArgsT,
+): Promise<Result<void, ToolErr>> => {
+ const button = args.button ?? "left";
+ const count = args.count ?? 1;
+ const pressed = await client.session().call("Input.dispatchMouseEvent", {
+ type: "mousePressed", x: args.x, y: args.y, button, clickCount: count,
+ });
+ if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message });
+ const released = await client.session().call("Input.dispatchMouseEvent", {
+ type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count,
+ });
+ if (!released.success) return err({ kind: "cdp_error", message: released.error.message });
+ return ok(undefined);
+};
+
+export const clickTool = defineBrowserTool({
+ name: "browser_click",
+ label: "Browser Click",
+ description:
+ "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.",
+ promptSnippet: "Click at pixel coordinates (x, y) on the page",
+ promptGuidelines: [
+ "Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).",
+ "Capture a browser_screenshot BEFORE clicking to find the right coordinates.",
+ "Capture another browser_screenshot AFTER clicking to verify the action worked.",
+ "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.",
+ "If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.",
+ "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.",
+ ],
+ parameters: ClickArgs,
+ renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0),
+ async handler(args, { client }) {
+ const clicked = await dispatchClick(client, args);
+ if (!clicked.success) return clicked;
+ if (process.env["BH_DEBUG_CLICKS"]) {
+ // Plain screenshot for now; the crosshair overlay returns in Task 14
+ // (domains/screenshot.ts) once sharp-shim lands.
+ const path = screenshotPath(client.namespace, "png");
+ const shot = await client.session().call("Page.captureScreenshot", { format: "png" });
+ if (shot.success) {
+ const data = (shot.data as { data: string }).data;
+ await writeFile(path, Buffer.from(data, "base64"));
+ return ok({
+ text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Screenshot: ${path}`,
+ details: { debugScreenshotPath: path, x: args.x, y: args.y },
+ });
+ }
+ }
+ return ok({
+ text: `Clicked at (${args.x}, ${args.y})`,
+ details: { x: args.x, y: args.y },
+ });
+ },
+});