diff options
| author | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-05-02 16:05:09 +0530 |
|---|---|---|
| committer | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-05-02 16:05:09 +0530 |
| commit | 0f962ef99b82207239e5dacb399be3cabce0ece3 (patch) | |
| tree | eb45efdf8821dc882a6c578fe6e767af24cc5266 /src/domains/click.ts | |
| parent | 6a4bfa31a925af67c026cd68700fe7779fd7c4cb (diff) | |
feat(domains): migrate browser_click to defineBrowserTool — vertical slice
Implements Task 9: adds src/domains/click.ts with the new architecture,
updates registry.ts to register clickTool, removes the legacy browser_click
block from tools.ts, and wires BrowserClient alongside BrowserDaemon in
index.ts so both systems coexist during migration.
Also fixes AnyBrowserToolDefinition variance: uses never-arg renderCall in
the erased type and a boundary cast in registerBrowserTool so concrete tool
definitions (with specific Static<S> args) are assignable to the registry
array without widening.
Diffstat (limited to 'src/domains/click.ts')
| -rw-r--r-- | src/domains/click.ts | 79 |
1 files changed, 79 insertions, 0 deletions
diff --git a/src/domains/click.ts b/src/domains/click.ts new file mode 100644 index 0000000..b3aa80a --- /dev/null +++ b/src/domains/click.ts @@ -0,0 +1,79 @@ +import { writeFile } from "node:fs/promises"; +import { Type, type Static } from "typebox"; +import { Text } from "@mariozechner/pi-tui"; +import type { BrowserClient } from "../client"; +import { Coords, MouseButton } from "../schemas/common"; +import { screenshotPath } from "../util/paths"; +import { type Result, err, ok } from "../util/result"; +import { defineBrowserTool, type ToolErr } from "../util/tool"; + +const ClickArgs = Type.Object({ + ...Coords.properties, + button: Type.Optional(MouseButton), + count: Type.Optional( + Type.Integer({ + minimum: 1, + maximum: 3, + default: 1, + description: "Number of clicks (1 = single, 2 = double). Default: 1", + }), + ), +}); +type ClickArgsT = Static<typeof ClickArgs>; + +const dispatchClick = async ( + client: BrowserClient, + args: ClickArgsT, +): Promise<Result<void, ToolErr>> => { + const button = args.button ?? "left"; + const count = args.count ?? 1; + const pressed = await client.session().call("Input.dispatchMouseEvent", { + type: "mousePressed", x: args.x, y: args.y, button, clickCount: count, + }); + if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message }); + const released = await client.session().call("Input.dispatchMouseEvent", { + type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count, + }); + if (!released.success) return err({ kind: "cdp_error", message: released.error.message }); + return ok(undefined); +}; + +export const clickTool = defineBrowserTool({ + name: "browser_click", + label: "Browser Click", + description: + "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.", + promptSnippet: "Click at pixel coordinates (x, y) on the page", + promptGuidelines: [ + "Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).", + "Capture a browser_screenshot BEFORE clicking to find the right coordinates.", + "Capture another browser_screenshot AFTER clicking to verify the action worked.", + "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.", + "If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.", + "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.", + ], + parameters: ClickArgs, + renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0), + async handler(args, { client }) { + const clicked = await dispatchClick(client, args); + if (!clicked.success) return clicked; + if (process.env["BH_DEBUG_CLICKS"]) { + // Plain screenshot for now; the crosshair overlay returns in Task 14 + // (domains/screenshot.ts) once sharp-shim lands. + const path = screenshotPath(client.namespace, "png"); + const shot = await client.session().call("Page.captureScreenshot", { format: "png" }); + if (shot.success) { + const data = (shot.data as { data: string }).data; + await writeFile(path, Buffer.from(data, "base64")); + return ok({ + text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Screenshot: ${path}`, + details: { debugScreenshotPath: path, x: args.x, y: args.y }, + }); + } + } + return ok({ + text: `Clicked at (${args.x}, ${args.y})`, + details: { x: args.x, y: args.y }, + }); + }, +}); |
