diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/domains/click.ts | 79 | ||||
| -rw-r--r-- | src/index.ts | 12 | ||||
| -rw-r--r-- | src/registry.ts | 3 | ||||
| -rw-r--r-- | src/tools.ts | 77 | ||||
| -rw-r--r-- | src/util/tool.ts | 23 |
5 files changed, 110 insertions, 84 deletions
diff --git a/src/domains/click.ts b/src/domains/click.ts new file mode 100644 index 0000000..b3aa80a --- /dev/null +++ b/src/domains/click.ts @@ -0,0 +1,79 @@ +import { writeFile } from "node:fs/promises"; +import { Type, type Static } from "typebox"; +import { Text } from "@mariozechner/pi-tui"; +import type { BrowserClient } from "../client"; +import { Coords, MouseButton } from "../schemas/common"; +import { screenshotPath } from "../util/paths"; +import { type Result, err, ok } from "../util/result"; +import { defineBrowserTool, type ToolErr } from "../util/tool"; + +const ClickArgs = Type.Object({ + ...Coords.properties, + button: Type.Optional(MouseButton), + count: Type.Optional( + Type.Integer({ + minimum: 1, + maximum: 3, + default: 1, + description: "Number of clicks (1 = single, 2 = double). Default: 1", + }), + ), +}); +type ClickArgsT = Static<typeof ClickArgs>; + +const dispatchClick = async ( + client: BrowserClient, + args: ClickArgsT, +): Promise<Result<void, ToolErr>> => { + const button = args.button ?? "left"; + const count = args.count ?? 1; + const pressed = await client.session().call("Input.dispatchMouseEvent", { + type: "mousePressed", x: args.x, y: args.y, button, clickCount: count, + }); + if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message }); + const released = await client.session().call("Input.dispatchMouseEvent", { + type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count, + }); + if (!released.success) return err({ kind: "cdp_error", message: released.error.message }); + return ok(undefined); +}; + +export const clickTool = defineBrowserTool({ + name: "browser_click", + label: "Browser Click", + description: + "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.", + promptSnippet: "Click at pixel coordinates (x, y) on the page", + promptGuidelines: [ + "Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).", + "Capture a browser_screenshot BEFORE clicking to find the right coordinates.", + "Capture another browser_screenshot AFTER clicking to verify the action worked.", + "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.", + "If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.", + "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.", + ], + parameters: ClickArgs, + renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0), + async handler(args, { client }) { + const clicked = await dispatchClick(client, args); + if (!clicked.success) return clicked; + if (process.env["BH_DEBUG_CLICKS"]) { + // Plain screenshot for now; the crosshair overlay returns in Task 14 + // (domains/screenshot.ts) once sharp-shim lands. + const path = screenshotPath(client.namespace, "png"); + const shot = await client.session().call("Page.captureScreenshot", { format: "png" }); + if (shot.success) { + const data = (shot.data as { data: string }).data; + await writeFile(path, Buffer.from(data, "base64")); + return ok({ + text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Screenshot: ${path}`, + details: { debugScreenshotPath: path, x: args.x, y: args.y }, + }); + } + } + return ok({ + text: `Clicked at (${args.x}, ${args.y})`, + details: { x: args.x, y: args.y }, + }); + }, +}); diff --git a/src/index.ts b/src/index.ts index 7a3464c..14ea8eb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,8 +21,10 @@ import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent"; import { BrowserDaemon } from "./daemon"; +import { type BrowserClient, createBrowserClient } from "./client"; import { getBrowserSystemPrompt } from "./prompt"; import { registerRenderers } from "./renderers"; +import { registerAllTools } from "./registry"; import { registerSetupCommand } from "./setup"; import { type BrowserState, defaultState, persistState, restoreState } from "./state"; import { cleanupTempDirs, registerTools } from "./tools"; @@ -35,6 +37,7 @@ export default function browserHarnessExtension(pi: ExtensionAPI) { // ── State ────────────────────────────────────────────────────────────────── let state: BrowserState = defaultState(namespace); let daemon: BrowserDaemon | null = null; + let client: BrowserClient | null = null; // ── Tool initialization flag ─────────────────────────────────────────────── let toolsRegistered = false; @@ -133,9 +136,13 @@ export default function browserHarnessExtension(pi: ExtensionAPI) { // Don't block — user can run /browser-setup later } + client = createBrowserClient({ namespace: state.namespace }); + try { await client.start(); } catch { /* surfaced via tool errors */ } + // Register tools (once) if (!toolsRegistered && daemon) { registerTools(pi, daemon); + if (client) registerAllTools(pi, client); toolsRegistered = true; } @@ -171,6 +178,11 @@ export default function browserHarnessExtension(pi: ExtensionAPI) { daemon = null; } + if (client) { + try { await client.stop(); } catch { /* best-effort */ } + client = null; + } + toolsRegistered = false; // Clean up temp files from truncated tool outputs diff --git a/src/registry.ts b/src/registry.ts index 014ba51..8d1fd27 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -1,9 +1,10 @@ import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; import type { BrowserClient } from "./client"; import { type AnyBrowserToolDefinition, registerBrowserTool } from "./util/tool"; +import { clickTool } from "./domains/click"; const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [ - // Domain tools registered as they're migrated, one per domain file. + clickTool, ]; export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => { diff --git a/src/tools.ts b/src/tools.ts index 3ea0b54..134531c 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -319,83 +319,6 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { }); // ═══════════════════════════════════════════════════════════════════════════ - // browser_click - // ═══════════════════════════════════════════════════════════════════════════ - pi.registerTool({ - name: "browser_click", - label: "Browser Click", - description: - "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.", - promptSnippet: "Click at pixel coordinates (x, y) on the page", - promptGuidelines: [ - "Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).", - "Capture a browser_screenshot BEFORE clicking to find the right coordinates.", - "Capture another browser_screenshot AFTER clicking to verify the action worked.", - "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.", - "If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.", - "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.", - ], - parameters: Type.Object({ - x: Type.Number({ description: "X coordinate in CSS pixels from left edge of viewport" }), - y: Type.Number({ description: "Y coordinate in CSS pixels from top edge of viewport" }), - button: Type.Optional(Type.String({ description: 'Mouse button: "left", "right", or "middle". Default: "left"' })), - clicks: Type.Optional(Type.Number({ description: "Number of clicks (1 = single, 2 = double). Default: 1" })), - }), - async execute(_id, params) { - try { - await daemon.ensureAlive(); - await daemon.cdp("Input.dispatchMouseEvent", { - type: "mousePressed", - x: params.x, - y: params.y, - button: params.button || "left", - clickCount: params.clicks || 1, - }); - await daemon.cdp("Input.dispatchMouseEvent", { - type: "mouseReleased", - x: params.x, - y: params.y, - button: params.button || "left", - clickCount: params.clicks || 1, - }); - - // Debug click overlay (when BH_DEBUG_CLICKS is set) - if (process.env.BH_DEBUG_CLICKS) { - try { - const debugPath = nextScreenshotPath(); - await daemon.captureScreenshotWithDebugOverlay( - debugPath, - params.x, - params.y, - ); - return { - content: [{ - type: "text" as const, - text: `Clicked at (${params.x}, ${params.y})\n[DEBUG] Overlay screenshot: ${debugPath}`, - }], - details: { debugScreenshotPath: debugPath }, - }; - } catch { - // Overlay failed — still report the click - } - } - - return { content: [{ type: "text" as const, text: `Clicked at (${params.x}, ${params.y})` }], details: undefined }; - } catch (err) { - return { - isError: true, - content: [{ type: "text" as const, text: `Click failed: ${err instanceof Error ? err.message : String(err)}` }], - details: undefined, - }; - } - }, - renderCall(args, theme) { - const { x, y } = args as { x: number; y: number }; - return new Text(`🖱️ Click at (${x}, ${y})`, 0, 0); - }, - }); - - // ═══════════════════════════════════════════════════════════════════════════ // browser_type // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ diff --git a/src/util/tool.ts b/src/util/tool.ts index fc1c5e2..d269ba2 100644 --- a/src/util/tool.ts +++ b/src/util/tool.ts @@ -56,9 +56,14 @@ export type BrowserToolDefinition<S extends TSchema> = { readonly ensureAlive?: boolean; }; -// Type-erased entry for arrays. AnyBrowserToolDefinition lets the registry -// hold a heterogeneous list without losing the per-tool Static<S> binding. -export type AnyBrowserToolDefinition = BrowserToolDefinition<TSchema>; +// Type-erased entry for the registry's heterogeneous TOOLS array. +// renderCall accepts `never` so any concrete args type satisfies it via +// function-parameter contravariance (a handler that takes SpecificT is +// assignable to one that takes never, because never is a subtype of everything). +export type AnyBrowserToolDefinition = Omit<BrowserToolDefinition<TSchema>, "renderCall" | "renderResult"> & { + readonly renderCall?: (args: never, theme: Theme) => Component; + readonly renderResult?: (result: AgentToolResult<unknown>, expanded: boolean, theme: Theme) => Component; +}; export const defineBrowserTool = <S extends TSchema>( def: BrowserToolDefinition<S>, @@ -94,11 +99,15 @@ const toToolResult = ( }; }; -export const registerBrowserTool = <S extends TSchema>( +export const registerBrowserTool = ( pi: ExtensionAPI, client: BrowserClient, - def: BrowserToolDefinition<S>, + def: AnyBrowserToolDefinition, ): void => { + // AnyBrowserToolDefinition is type-erased; we recover the schema-typed surface + // by casting def to the generic form. The cast is safe because defineBrowserTool + // ensures the handler, parameters, and renderCall were all created with the same S. + type S = TSchema; const td: ToolDefinition<S> = { name: def.name, label: def.label, @@ -110,7 +119,9 @@ export const registerBrowserTool = <S extends TSchema>( ? { renderCall: (args: Static<S>, theme: Theme, _ctx: Parameters<NonNullable<ToolDefinition<S>["renderCall"]>>[2]) => // Non-null assertion safe: this branch only runs when renderCall was provided. - def.renderCall!(args, theme), + // Cast from never->Component to unknown->Component: safe because at runtime + // args is always the concrete Static<S> value the tool was defined with. + (def.renderCall as (args: unknown, theme: Theme) => Component)(args, theme), } : {}), ...(def.renderResult |