diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/domains/keyboard.ts | 126 | ||||
| -rw-r--r-- | src/registry.ts | 4 | ||||
| -rw-r--r-- | src/tools.ts | 152 |
3 files changed, 130 insertions, 152 deletions
diff --git a/src/domains/keyboard.ts b/src/domains/keyboard.ts new file mode 100644 index 0000000..d953b76 --- /dev/null +++ b/src/domains/keyboard.ts @@ -0,0 +1,126 @@ +import { Type } from "typebox"; +import { safeJs } from "../util/js-template"; +import { virtualKeyCode } from "../util/keycodes"; +import { type Result, err, ok } from "../util/result"; +import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; + +const TypeArgs = Type.Object({ + text: Type.String({ description: "Text to type" }), +}); + +export const typeTool = defineBrowserTool({ + name: "browser_type", + label: "Browser Type", + description: "Type text into the currently focused element. Use browser_click first to focus an input field.", + promptSnippet: "Type text into the focused element", + promptGuidelines: [ + "Use browser_type to enter text. Click on an input field with browser_click first to focus it.", + "For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.", + ], + parameters: TypeArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const r = await client.session().call("Input.insertText", { text: args.text }); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + return ok({ text: `Typed: "${args.text}"` }); + }, +}); + +const PressKeyArgs = Type.Object({ + key: Type.String({ + description: + 'Key to press. Special keys: Enter, Tab, Backspace, Escape, Delete, ArrowLeft/Up/Right/Down, Home, End, PageUp, PageDown. Space as " "', + }), + modifiers: Type.Optional( + Type.Integer({ + minimum: 0, + maximum: 15, + description: "Modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with OR.", + }), + ), +}); + +export const pressKeyTool = defineBrowserTool({ + name: "browser_press_key", + label: "Browser Press Key", + description: + "Press a keyboard key. Supports special keys (Enter, Tab, Backspace, Escape, Delete, arrows, Home, End, PageUp, PageDown, Space as ' ') and regular characters. Optional modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta/Cmd, 8=Shift.", + promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)", + promptGuidelines: [ + "Use browser_press_key for keyboard shortcuts and navigation keys.", + "Special key names: Enter, Tab, Backspace, Escape, Delete, ArrowLeft, ArrowUp, ArrowRight, ArrowDown, Home, End, PageUp, PageDown.", + "Use Space as ' ' (a single space character).", + "Modifiers: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with bitwise OR: Ctrl+Shift = 2|8 = 10.", + ], + parameters: PressKeyArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const k = args.key; + const code = virtualKeyCode(k); + const modifiers = args.modifiers ?? 0; + const isChar = k.length === 1; + const downParams: Record<string, unknown> = { + type: "keyDown", + key: k, + code: k, + modifiers, + ...(code ? { windowsVirtualKeyCode: code, nativeVirtualKeyCode: code } : {}), + ...(isChar ? { text: k, unmodifiedText: k } : {}), + }; + const down = await client.session().call("Input.dispatchKeyEvent", downParams); + if (!down.success) return err({ kind: "cdp_error", message: down.error.message }); + const up = await client.session().call("Input.dispatchKeyEvent", { type: "keyUp", key: k, code: k, modifiers }); + if (!up.success) return err({ kind: "cdp_error", message: up.error.message }); + return ok({ text: `Pressed: ${k}${modifiers ? ` (modifiers=${modifiers})` : ""}` }); + }, +}); + +const DispatchKeyArgs = Type.Object({ + selector: Type.String({ description: "CSS selector of the target element" }), + key: Type.String({ description: "Key value (e.g., 'Enter', 'a')" }), + eventType: Type.Optional( + Type.Union( + [Type.Literal("keydown"), Type.Literal("keyup"), Type.Literal("keypress")], + { default: "keydown", description: "Event type to dispatch. Default: keydown" }, + ), + ), +}); + +export const dispatchKeyTool = defineBrowserTool({ + name: "browser_dispatch_key", + label: "Browser Dispatch Key", + description: + "Dispatch a DOM KeyboardEvent on a specific element via JS injection. Use for React/Vue components that listen to synthetic events more reliably than CDP input.", + promptSnippet: "Dispatch a DOM KeyboardEvent on a specific element", + promptGuidelines: [ + "Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.", + "The selector must match exactly one or more elements; zero matches is reported as an error.", + "eventType defaults to 'keydown'.", + ], + parameters: DispatchKeyArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const eventType = args.eventType ?? "keydown"; + const expr = safeJs` + (() => { + const els = document.querySelectorAll(${args.selector}); + if (els.length === 0) return 0; + for (const el of els) { + el.dispatchEvent(new KeyboardEvent(${eventType}, { key: ${args.key}, bubbles: true, cancelable: true })); + } + return els.length; + })() + `; + const r = await client.evaluateJs(expr); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + const matched = Number(r.data ?? 0); + if (matched === 0) { + return err({ + kind: "invalid_state", + message: `Selector matched 0 elements: ${args.selector}`, + details: { matched: 0, selector: args.selector }, + }); + } + return ok({ + text: `Dispatched ${eventType}('${args.key}') on ${matched} element(s)`, + details: { matched, selector: args.selector }, + }); + }, +}); diff --git a/src/registry.ts b/src/registry.ts index 8d1fd27..e2b13b4 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -2,9 +2,13 @@ import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; import type { BrowserClient } from "./client"; import { type AnyBrowserToolDefinition, registerBrowserTool } from "./util/tool"; import { clickTool } from "./domains/click"; +import { typeTool, pressKeyTool, dispatchKeyTool } from "./domains/keyboard"; const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [ clickTool, + typeTool, + pressKeyTool, + dispatchKeyTool, ]; export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => { diff --git a/src/tools.ts b/src/tools.ts index 134531c..834d9c3 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -319,111 +319,6 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { }); // ═══════════════════════════════════════════════════════════════════════════ - // browser_type - // ═══════════════════════════════════════════════════════════════════════════ - pi.registerTool({ - name: "browser_type", - label: "Browser Type", - description: "Type text into the currently focused element. Use browser_click first to focus an input field.", - promptSnippet: "Type text into the focused element", - promptGuidelines: [ - "Use browser_type to enter text. Click on an input field with browser_click first to focus it.", - "For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.", - ], - parameters: Type.Object({ - text: Type.String({ description: "Text to type" }), - }), - async execute(_id, params) { - try { - await daemon.ensureAlive(); - await daemon.cdp("Input.insertText", { text: params.text }); - return { content: [{ type: "text" as const, text: `Typed: "${params.text}"` }], details: undefined }; - } catch (err) { - return { - isError: true, - content: [{ type: "text" as const, text: `Type failed: ${err instanceof Error ? err.message : String(err)}` }], - details: undefined, - }; - } - }, - }); - - // ═══════════════════════════════════════════════════════════════════════════ - // browser_press_key - // ═══════════════════════════════════════════════════════════════════════════ - pi.registerTool({ - name: "browser_press_key", - label: "Browser Press Key", - description: - "Press a keyboard key. Supports special keys (Enter, Tab, Backspace, Escape, Delete, arrows, Home, End, PageUp, PageDown, Space as ' ') and regular characters. Optional modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta/Cmd, 8=Shift.", - promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)", - promptGuidelines: [ - "Use browser_press_key for keyboard shortcuts and navigation keys.", - "Special key names: Enter, Tab, Backspace, Escape, Delete, ArrowLeft, ArrowUp, ArrowRight, ArrowDown, Home, End, PageUp, PageDown.", - "Use Space as ' ' (a single space character).", - "Modifiers: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with bitwise OR: Ctrl+Shift = 2|8 = 10.", - ], - parameters: Type.Object({ - key: Type.String({ description: 'Key to press. Special keys: Enter, Tab, Backspace, Escape, Delete, ArrowLeft/Up/Right/Down, Home, End, PageUp, PageDown. Space as " "' }), - modifiers: Type.Optional(Type.Number({ description: "Modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with OR." })), - }), - async execute(_id, params) { - try { - await daemon.ensureAlive(); - const k = params.key; - const mod = params.modifiers || 0; - const vk = daemon.getVirtualKeyCode(k); - const code = daemon.getKeyCode(k); - const isSpecial = SPECIAL_KEYS.includes(k); - const text = k.length === 1 && !isSpecial ? k : ""; - - // keyDown - const base = { - type: "keyDown" as const, - key: k, - code, - modifiers: mod, - windowsVirtualKeyCode: vk, - nativeVirtualKeyCode: vk, - ...(text ? { text } : {}), - }; - await daemon.cdp("Input.dispatchKeyEvent", base); - - // char event for printable single characters - if (text.length === 1) { - await daemon.cdp("Input.dispatchKeyEvent", { - type: "char", - text, - key: k, - code, - modifiers: mod, - windowsVirtualKeyCode: vk, - nativeVirtualKeyCode: vk, - }); - } - - // keyUp - await daemon.cdp("Input.dispatchKeyEvent", { - type: "keyUp", - key: k, - code, - modifiers: mod, - windowsVirtualKeyCode: vk, - nativeVirtualKeyCode: vk, - }); - - return { content: [{ type: "text" as const, text: `Pressed: "${k}"${mod ? ` (modifiers: ${mod})` : ""}` }], details: undefined }; - } catch (err) { - return { - isError: true, - content: [{ type: "text" as const, text: `Key press failed: ${err instanceof Error ? err.message : String(err)}` }], - details: undefined, - }; - } - }, - }); - - // ═══════════════════════════════════════════════════════════════════════════ // browser_scroll // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ @@ -943,53 +838,6 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { }); // ═══════════════════════════════════════════════════════════════════════════ - // browser_dispatch_key - // ═══════════════════════════════════════════════════════════════════════════ - pi.registerTool({ - name: "browser_dispatch_key", - label: "Browser Dispatch Key", - description: - "Dispatch a DOM KeyboardEvent on a matched element. Use this when a site " + - "reacts to synthetic DOM key events more reliably than raw CDP input events " + - "(e.g., React/Vue comboboxes, autocomplete widgets, custom form inputs).", - promptSnippet: "Dispatch a DOM KeyboardEvent on a specific element (for framework inputs)", - promptGuidelines: [ - "Use browser_dispatch_key when browser_press_key doesn't work — some React/Vue components only listen for DOM-level events.", - "The selector must match a single element that can receive focus.", - "Common keys: 'Enter' (13), 'Tab' (9), 'Escape' (27), 'Backspace' (8), ' ' (32).", - "Try browser_press_key first. Only fall back to browser_dispatch_key if the page ignores raw CDP input.", - ], - parameters: Type.Object({ - selector: Type.String({ description: "CSS selector for the target element" }), - key: Type.String({ description: "Key to dispatch (e.g. 'Enter', 'Tab', 'Escape', 'a', ' ')" }), - event: Type.Optional(Type.String({ description: "DOM event type: 'keydown', 'keypress', or 'keyup'. Default: 'keypress'" })), - }), - async execute(_id, params) { - try { - await daemon.ensureAlive(); - - const key = params.key; - const eventType = params.event || "keypress"; - const keyCode = daemon.getVirtualKeyCode(key); - - const expression = `(()=>{const e=document.querySelector(${JSON.stringify(params.selector)});if(e){e.focus();e.dispatchEvent(new KeyboardEvent(${JSON.stringify(eventType)},{key:${JSON.stringify(key)},code:${JSON.stringify(key)},keyCode:${keyCode},which:${keyCode},bubbles:true}))}else{throw new Error('No element found for selector: ${params.selector.replace(/'/g, "\\'")}')}})()`; - - const result = await daemon.evaluateJS(expression); - return { - content: [{ type: "text" as const, text: `Dispatched ${eventType} "${key}" (keyCode: ${keyCode}) on "${params.selector}".` }], - details: undefined, - }; - } catch (err) { - return { - isError: true, - content: [{ type: "text" as const, text: `Dispatch key failed: ${err instanceof Error ? err.message : String(err)}` }], - details: undefined, - }; - } - }, - }); - - // ═══════════════════════════════════════════════════════════════════════════ // browser_wait // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ |