diff options
| author | Aman Kumar <145339995+amankumarsingh77@users.noreply.github.com> | 2026-06-29 00:02:54 +0530 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-06-29 00:02:54 +0530 |
| commit | 0b32e88f84f36a759d7191e8bdfd09a44789fa95 (patch) | |
| tree | 832fb2e1d636e2c979384fd78e2910e02a74db2e /src/domains | |
| parent | 9f428cceb9c070e01df43dd55633c22678b8c406 (diff) | |
feat: address elements by stable refs across re-renders (#8)
Interaction tools could only target elements by raw viewport coordinates
(browser_click) or model-guessed CSS selectors (browser_fill/focus/
select_option/upload). Neither survives a re-render: on React/SPA forms,
every save or edit reflows the page, so captured @(x,y) coordinates go
stale and guessed selectors miss — leaving the agent to fall back to
hand-rolled execute_js DOM walks and unnoticed failed saves.
Add a stable ref system, mirroring Playwright MCP / browser-use:
- snapshot assigns each interactive element a ref (e1, e2, …), shown as
[eN] in the outline, and publishes a per-tab ref → backendNodeId map
- interaction tools accept an optional `ref` that re-resolves the element
at call time (via DOM.resolveNode / getBoxModel), so it stays valid
across re-renders; coordinates/selectors remain as fallbacks
- mutating tools append a compact diff of changed / newly-appeared (*) /
removed interactive elements, so silent failed saves surface immediately
- stale refs return an actionable "re-snapshot" error instead of a miss
Ref map is replaced per snapshot (bounded memory); fill keeps the native
_valueTracker write path so React controlled inputs still update.
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Diffstat (limited to 'src/domains')
| -rw-r--r-- | src/domains/click.ts | 84 | ||||
| -rw-r--r-- | src/domains/files.ts | 37 | ||||
| -rw-r--r-- | src/domains/form.ts | 270 | ||||
| -rw-r--r-- | src/domains/keyboard.ts | 66 | ||||
| -rw-r--r-- | src/domains/ref-resolve.ts | 152 | ||||
| -rw-r--r-- | src/domains/snapshot.ts | 98 |
6 files changed, 663 insertions, 44 deletions
diff --git a/src/domains/click.ts b/src/domains/click.ts index 9946664..db340a8 100644 --- a/src/domains/click.ts +++ b/src/domains/click.ts @@ -1,13 +1,21 @@ -import { Type, type Static } from "typebox"; +import { Type } from "typebox"; import { Text } from "@mariozechner/pi-tui"; import type { BrowserClient } from "../client"; import { Coords, MouseButton } from "../schemas/common"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr } from "../util/tool"; import { captureWithCrosshair } from "./screenshot"; +import { interactiveDiff, resolveRefToBox } from "./ref-resolve"; const ClickArgs = Type.Object({ - ...Coords.properties, + ref: Type.Optional( + Type.String({ + description: + "Stable element ref from browser_snapshot (e.g. 'e12'). PREFERRED over x/y — survives re-renders. When set, x/y are ignored.", + }), + ), + x: Type.Optional(Coords.properties.x), + y: Type.Optional(Coords.properties.y), button: Type.Optional(MouseButton), count: Type.Optional( Type.Integer({ @@ -18,20 +26,27 @@ const ClickArgs = Type.Object({ }), ), }); -type ClickArgsT = Static<typeof ClickArgs>; const dispatchClick = async ( client: BrowserClient, - args: ClickArgsT, + x: number, + y: number, + button: "left" | "right" | "middle", + count: number, ): Promise<Result<void, ToolErr>> => { - const button = args.button ?? "left"; - const count = args.count ?? 1; + // Move the pointer to the target first. Some focus/hover handlers (and React + // synthetic events) only fire when a mousemove precedes the press, so without + // it a click can land without focusing the element. + const moved = await client.session().call("Input.dispatchMouseEvent", { + type: "mouseMoved", x, y, + }); + if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message }); const pressed = await client.session().call("Input.dispatchMouseEvent", { - type: "mousePressed", x: args.x, y: args.y, button, clickCount: count, + type: "mousePressed", x, y, button, clickCount: count, }); if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message }); const released = await client.session().call("Input.dispatchMouseEvent", { - type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count, + type: "mouseReleased", x, y, button, clickCount: count, }); if (!released.success) return err({ kind: "cdp_error", message: released.error.message }); return ok(undefined); @@ -41,34 +56,55 @@ export const clickTool = defineBrowserTool({ name: "browser_click", label: "Browser Click", description: - "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Get coordinates from browser_snapshot's @(x,y) hints — do not screenshot to find click targets.", - promptSnippet: "Click at pixel coordinates (x, y) on the page", + "Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.", + promptSnippet: "Click an element by ref (preferred) or pixel coordinates", promptGuidelines: [ - "Get (x, y) from browser_snapshot — the outline shows '@(x,y)' for every interactive element. NO screenshot needed.", - "If the target isn't an interactive role in the snapshot, fall back to browser_execute_js with `el.getBoundingClientRect()` — still NO screenshot.", - "Coordinates are viewport CSS pixels (not device pixels).", - "After clicking, verify with browser_snapshot (or browser_execute_js for a specific value). Only screenshot if you need to confirm visual rendering.", - "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.", - "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions (debug only).", - "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.", + "PREFER `ref` from browser_snapshot (the outline shows '[eN]' for every interactive element) — it survives re-renders, unlike coordinates which go stale after a save/edit reflows the page.", + "Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.", + "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.", + "After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).", + "Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.", + "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots (debug only). For React/Vue components ignoring clicks, try browser_dispatch_key.", ], parameters: ClickArgs, - renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0), + renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0), async handler(args, { client }) { - const clicked = await dispatchClick(client, args); + const button = args.button ?? "left"; + const count = args.count ?? 1; + + // Resolve the click point: ref re-resolves to the element's CURRENT box + // (so re-renders don't matter); otherwise use the literal x/y. + let x: number; + let y: number; + if (args.ref !== undefined) { + const box = await resolveRefToBox(client, args.ref); + if (!box.success) return box; + x = box.data.cx; + y = box.data.cy; + } else if (args.x !== undefined && args.y !== undefined) { + x = args.x; + y = args.y; + } else { + return err({ kind: "invalid_state", message: "Provide either `ref` or both `x` and `y`." }); + } + + const clicked = await dispatchClick(client, x, y, button, count); if (!clicked.success) return clicked; + + const target = args.ref !== undefined ? `[${args.ref}] (${x}, ${y})` : `(${x}, ${y})`; + const diff = await interactiveDiff(client); if (process.env["BH_DEBUG_CLICKS"]) { - const debug = await captureWithCrosshair(client, { x: args.x, y: args.y }); + const debug = await captureWithCrosshair(client, { x, y }); if (debug.success) { return ok({ - text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`, - details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y }, + text: `Clicked at ${target}\n[DEBUG] Overlay screenshot: ${debug.data.path}${diff}`, + details: { debugScreenshotPath: debug.data.path, x, y, ref: args.ref }, }); } } return ok({ - text: `Clicked at (${args.x}, ${args.y})`, - details: { x: args.x, y: args.y }, + text: `Clicked at ${target}${diff}`, + details: { x, y, ref: args.ref }, }); }, }); diff --git a/src/domains/files.ts b/src/domains/files.ts index 91f8761..d68d236 100644 --- a/src/domains/files.ts +++ b/src/domains/files.ts @@ -6,12 +6,28 @@ import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { safeJs } from "../util/js-template"; import { pdfPath } from "../util/paths"; +import { resolveRefToBackendId } from "./ref-resolve"; const UploadArgs = Type.Object({ - selector: Type.String({ description: "CSS selector of the file <input>" }), + ref: Type.Optional( + Type.String({ description: "Stable ref of the file <input> from browser_snapshot. PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the file <input> (fallback when no ref)" })), filePath: Type.String({ description: "Absolute path to the file to upload" }), }); +// Ref path: set files directly on the resolved backendNodeId via CDP — simplest +// and most reliable, no selector needed. +const tryRefUpload = async ( + client: BrowserClient, + backendId: number, + filePath: string, +): Promise<Result<void, ToolErr>> => { + const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId }); + if (!set.success) return err({ kind: "cdp_error", message: set.error.message }); + return ok(undefined); +}; + const verifyReadable = async (filePath: string): Promise<Result<void, ToolErr>> => { try { await access(filePath, constants.R_OK); @@ -89,16 +105,29 @@ const jsFallbackUpload = async ( export const uploadFileTool = defineBrowserTool({ name: "browser_upload_file", label: "Browser Upload File", - description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages.", - promptSnippet: "Upload a file to a file input", + description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.", + promptSnippet: "Upload a file to a file input by ref (preferred) or selector", promptGuidelines: [ + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders. (File inputs are often hidden; snapshot still surfaces them.)", "File path must be absolute and readable.", - "Selector must match a file input.", ], parameters: UploadArgs, async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { const readable = await verifyReadable(args.filePath); if (!readable.success) return readable; + + // Ref path: resolve to backendNodeId and set files directly via CDP. + if (args.ref !== undefined) { + const backendId = resolveRefToBackendId(client, args.ref); + if (!backendId.success) return backendId; + const up = await tryRefUpload(client, backendId.data, args.filePath); + if (!up.success) return up; + return ok({ text: `Uploaded ${args.filePath} via CDP (ref ${args.ref})`, details: { mode: "cdp", ref: args.ref, filePath: args.filePath } }); + } + + if (args.selector === undefined) { + return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); + } const cdp = await tryCdpUpload(client, args.selector, args.filePath); if (cdp.success) { return ok({ diff --git a/src/domains/form.ts b/src/domains/form.ts new file mode 100644 index 0000000..a10646c --- /dev/null +++ b/src/domains/form.ts @@ -0,0 +1,270 @@ +import { Type } from "typebox"; +import type { BrowserClient } from "../client"; +import { safeJs } from "../util/js-template"; +import { type Result, err, ok } from "../util/result"; +import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; +import { interactiveDiff, resolveRefToObjectId } from "./ref-resolve"; + +const FillArgs = Type.Object({ + ref: Type.Optional( + Type.String({ + description: + "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector — survives re-renders. When set, selector is ignored.", + }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the form field to fill (fallback when no ref)" })), + value: Type.String({ description: "Value to set in the field" }), +}); + +/** + * Run an element-scoped function either against a CSS selector (querySelector + + * apply) or a ref (resolve to objectId, callFunctionOn with element as `this`). + * The shared `fnBody` is a function body where `this` is the target element and + * the single argument is the value — identical logic on both paths. Returns the + * function's return value (parsed) or a ToolErr. + * + * On the ref path a detached node yields a stale-ref error; on the selector path + * a missing element returns { status: "not_found" } from the function body. + */ +const runOnElement = async ( + client: BrowserClient, + opts: { ref?: string | undefined; selector?: string | undefined; fnBody: string; arg: unknown }, +): Promise<Result<unknown, ToolErr>> => { + if (opts.ref !== undefined) { + const objectId = await resolveRefToObjectId(client, opts.ref); + if (!objectId.success) return objectId; + const r = await client.session().call("Runtime.callFunctionOn", { + objectId: objectId.data, + functionDeclaration: `function (arg) { ${opts.fnBody} }`, + arguments: [{ value: opts.arg }], + returnByValue: true, + }); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + return ok((r.data as { result?: { value?: unknown } }).result?.value); + } + if (opts.selector === undefined) { + return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); + } + // Selector path: bind the same trusted function body to the matched element + // via .call(). fnBody is harness-authored (not user input); only the selector + // and value are interpolated, and those go through safeJs. + const prelude = safeJs` + (() => { + const el = document.querySelector(${opts.selector}); + if (!el) return { status: "not_found" }; + const __arg = ${opts.arg}; + return (function (arg) {`; + const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`; + const r = await client.evaluateJs(expr); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + return ok(r.data); +}; + +/** + * Result shape returned by the in-page fill script. Discriminated by `status` + * so the handler can map page-level outcomes to typed ToolErr kinds. + */ +type FillResult = + | { status: "ok"; tag: string; kind: string; value: string } + | { status: "not_found" } + | { status: "not_fillable"; tag: string }; + +// Element-scoped fill logic. `this` is the target element, `arg` is the value. +// Write via the native prototype value setter so React's _valueTracker +// registers the change — assigning el.value directly does NOT, which is why +// controlled inputs revert on re-render. Shared verbatim by the ref path +// (callFunctionOn) and the selector path (querySelector + .call). +const FILL_FN = ` + const el = this; + const tag = el.tagName; + const proto = + el instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype + : el instanceof HTMLInputElement ? HTMLInputElement.prototype + : null; + if (proto) { + const desc = Object.getOwnPropertyDescriptor(proto, "value"); + const setter = desc && desc.set; + if (setter) setter.call(el, arg); + else el.value = arg; + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + return { status: "ok", tag, kind: el.type || tag.toLowerCase(), value: el.value }; + } + if (el.isContentEditable) { + // Rich-text editors (Slack/Notion/ProseMirror) listen for beforeinput/input. + el.focus(); + let done = false; + try { + const sel = window.getSelection(); + if (sel) { sel.removeAllRanges(); const rng = document.createRange(); rng.selectNodeContents(el); sel.addRange(rng); } + done = document.execCommand("insertText", false, arg); + } catch (e) { done = false; } + if (!done) { + el.textContent = arg; + el.dispatchEvent(new InputEvent("input", { bubbles: true })); + } + return { status: "ok", tag, kind: "contenteditable", value: el.textContent || "" }; + } + return { status: "not_fillable", tag }; +`; + +export const fillTool = defineBrowserTool({ + name: "browser_fill", + label: "Browser Fill", + description: + "Fill a form field (input/textarea/contenteditable). PREFERRED: pass `ref` (e.g. 'e7') from browser_snapshot — survives re-renders. Fallback: a CSS `selector`. Writes through the native value setter and fires bubbling 'input'/'change' events, so React/Vue/Angular controlled components and rich-text editors update correctly — unlike browser_type. Returns the field's value after writing, plus a compact diff of page changes.", + promptSnippet: "Fill a form field by ref (preferred) or selector (works with React/Vue controlled inputs)", + promptGuidelines: [ + "PREFER `ref` from browser_snapshot (the '[eN]' handle) over a guessed CSS selector — refs survive re-renders and don't require you to invent a selector.", + "Pass the desired value; no browser_click is needed first.", + "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.", + "Use browser_type instead only for keystroke-sensitive widgets (autocomplete, masked/segmented inputs).", + "The result reports the field's value after writing and an appended page-changes diff — confirm both match what you intended before moving on.", + ], + parameters: FillArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: FILL_FN, arg: args.value }); + if (!r.success) return r; + const res = r.data as FillResult | undefined; + const target = args.ref ?? args.selector ?? ""; + if (res === undefined || res.status === "not_found") { + return err({ + kind: "invalid_state", + message: `No element matched: ${target}`, + details: { ref: args.ref, selector: args.selector }, + }); + } + if (res.status === "not_fillable") { + return err({ + kind: "invalid_state", + message: `Element <${res.tag.toLowerCase()}> is not fillable (not an input, textarea, or contenteditable): ${target}. For <select> use browser_select_option.`, + details: { ref: args.ref, selector: args.selector, tag: res.tag }, + }); + } + const diff = await interactiveDiff(client); + return ok({ + text: `Filled ${target} = ${JSON.stringify(res.value)}${diff}`, + details: { ref: args.ref, selector: args.selector, value: args.value, verified: res.value, tag: res.tag, kind: res.kind }, + }); + }, +}); + +const FocusArgs = Type.Object({ + ref: Type.Optional( + Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the element to focus (fallback when no ref)" })), +}); + +type FocusResult = { status: "ok"; tag: string } | { status: "not_found" } | { status: "not_focusable"; tag: string }; + +const FOCUS_FN = ` + const el = this; + if (typeof el.focus !== "function") return { status: "not_focusable", tag: el.tagName }; + el.focus(); + return document.activeElement === el ? { status: "ok", tag: el.tagName } : { status: "not_focusable", tag: el.tagName }; +`; + +export const focusTool = defineBrowserTool({ + name: "browser_focus", + label: "Browser Focus", + description: + "Focus an element via the DOM .focus() method — deterministic, no coordinate accuracy needed. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use before browser_type when a click might miss the field.", + promptSnippet: "Focus an element by ref (preferred) or selector", + promptGuidelines: [ + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", + "Use before browser_type to guarantee the right field is focused without relying on click coordinates.", + "For simply setting a value, prefer browser_fill (it doesn't need a separate focus step).", + ], + parameters: FocusArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: FOCUS_FN, arg: "" }); + if (!r.success) return r; + const res = r.data as FocusResult | undefined; + const target = args.ref ?? args.selector ?? ""; + if (res === undefined || res.status === "not_found") { + return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector } }); + } + if (res.status === "not_focusable") { + return err({ kind: "invalid_state", message: `Element <${res.tag.toLowerCase()}> could not be focused: ${target}`, details: { ref: args.ref, selector: args.selector, tag: res.tag } }); + } + return ok({ text: `Focused ${target}`, details: { ref: args.ref, selector: args.selector, tag: res.tag } }); + }, +}); + +const SelectOptionArgs = Type.Object({ + ref: Type.Optional( + Type.String({ description: "Stable element ref of the <select> from browser_snapshot. PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the <select> element (fallback when no ref)" })), + value: Type.Optional(Type.String({ description: "Option value attribute to select" })), + label: Type.Optional(Type.String({ description: "Visible option text to select (exact match)" })), + index: Type.Optional(Type.Integer({ minimum: 0, description: "Zero-based option index to select" })), +}); + +type SelectResult = + | { status: "ok"; value: string; label: string } + | { status: "not_found" } + | { status: "not_select"; tag: string } + | { status: "no_match"; options: ReadonlyArray<{ value: string; label: string }> }; + +// `this` is the <select>; `arg` is { wv, wl, wi } (null = not provided). +const SELECT_FN = ` + const el = this; + if (el.tagName !== "SELECT") return { status: "not_select", tag: el.tagName }; + const opts = Array.from(el.options); + const wv = arg.wv, wl = arg.wl, wi = arg.wi; + let opt = null; + if (wi !== null) opt = opts[wi] || null; + else if (wv !== null) opt = opts.find(o => o.value === wv) || null; + else if (wl !== null) opt = opts.find(o => o.text === wl) || null; + if (!opt) return { status: "no_match", options: opts.map(o => ({ value: o.value, label: o.text })) }; + el.value = opt.value; + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + return { status: "ok", value: opt.value, label: opt.text }; +`; + +export const selectOptionTool = defineBrowserTool({ + name: "browser_select_option", + label: "Browser Select Option", + description: + "Select an option in a native <select> element by value, visible label, or index. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Sets the selection and fires bubbling 'input'/'change' events so framework listeners update. On no match, returns the available options.", + promptSnippet: "Select an option in a native <select> by ref (preferred) or selector", + promptGuidelines: [ + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", + "Provide exactly one of value, label, or index to choose the option.", + "For custom (non-native) dropdowns built from divs, click to open then browser_click the option instead.", + "If no option matches, the error details list the available options so you can retry.", + ], + parameters: SelectOptionArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + if (args.value === undefined && args.label === undefined && args.index === undefined) { + return err({ kind: "invalid_state", message: "Provide one of value, label, or index." }); + } + // null sentinels distinguish "not provided" from a real empty-string match. + const arg = { wv: args.value ?? null, wl: args.label ?? null, wi: args.index ?? null }; + const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: SELECT_FN, arg }); + if (!r.success) return r; + const res = r.data as SelectResult | undefined; + const target = args.ref ?? args.selector ?? ""; + if (res === undefined || res.status === "not_found") { + return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector } }); + } + if (res.status === "not_select") { + return err({ kind: "invalid_state", message: `Element <${res.tag.toLowerCase()}> is not a <select>: ${target}`, details: { ref: args.ref, selector: args.selector, tag: res.tag } }); + } + if (res.status === "no_match") { + return err({ + kind: "invalid_state", + message: `No matching option in ${target}. Available: ${res.options.map((o) => o.label).join(", ")}`, + details: { ref: args.ref, selector: args.selector, options: res.options }, + }); + } + const diff = await interactiveDiff(client); + return ok({ + text: `Selected "${res.label}" (value=${JSON.stringify(res.value)}) in ${target}${diff}`, + details: { ref: args.ref, selector: args.selector, value: res.value, label: res.label }, + }); + }, +}); diff --git a/src/domains/keyboard.ts b/src/domains/keyboard.ts index c39814e..69a7396 100644 --- a/src/domains/keyboard.ts +++ b/src/domains/keyboard.ts @@ -3,6 +3,7 @@ import { safeJs } from "../util/js-template"; import { virtualKeyCode } from "../util/keycodes"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; +import { resolveRefToObjectId } from "./ref-resolve"; const TypeArgs = Type.Object({ text: Type.String({ description: "Text to type" }), @@ -19,6 +20,29 @@ export const typeTool = defineBrowserTool({ ], parameters: TypeArgs, async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + // Guard against silent text loss: Input.insertText succeeds regardless of + // whether an editable element is focused, so without this check typed text + // can vanish into the void with no error. Fail loudly with an actionable + // message instead. IIFE starts with "(" so evaluateJs returns it directly. + const focused = await client.evaluateJs(safeJs` + (() => { + const el = document.activeElement; + const ok = !!el && el !== document.body && + (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable); + return { ok, tag: el ? el.tagName : null }; + })() + `); + if (focused.success) { + const f = focused.data as { ok: boolean; tag: string | null }; + if (!f.ok) { + return err({ + kind: "invalid_state", + message: + "No editable element is focused — typed text would be lost. Click the field with browser_click first, or use browser_fill with a selector.", + details: { activeElement: f.tag }, + }); + } + } const r = await client.session().call("Input.insertText", { text: args.text }); if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); return ok({ text: `Typed: "${args.text}"` }); @@ -67,6 +91,14 @@ export const pressKeyTool = defineBrowserTool({ }; const down = await client.session().call("Input.dispatchKeyEvent", downParams); if (!down.success) return err({ kind: "cdp_error", message: down.error.message }); + // For a printable character with no command modifier (Ctrl/Meta/Alt = 1|2|4), + // emit a `char` event so the page receives keypress/textInput and the + // character is actually inserted. Shift (8) is allowed (capitals/symbols). + const hasCommandModifier = (modifiers & (1 | 2 | 4)) !== 0; + if (isChar && !hasCommandModifier) { + const charEv = await client.session().call("Input.dispatchKeyEvent", { type: "char", text: k, key: k }); + if (!charEv.success) return err({ kind: "cdp_error", message: charEv.error.message }); + } const up = await client.session().call("Input.dispatchKeyEvent", { type: "keyUp", key: k, code: k, modifiers }); if (!up.success) return err({ kind: "cdp_error", message: up.error.message }); return ok({ text: `Pressed: ${k}${modifiers ? ` (modifiers=${modifiers})` : ""}` }); @@ -74,7 +106,10 @@ export const pressKeyTool = defineBrowserTool({ }); const DispatchKeyArgs = Type.Object({ - selector: Type.String({ description: "CSS selector of the target element" }), + ref: Type.Optional( + Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the target element (fallback when no ref)" })), key: Type.String({ description: "Key value (e.g., 'Enter', 'a')" }), eventType: Type.Optional( Type.Union( @@ -88,17 +123,38 @@ export const dispatchKeyTool = defineBrowserTool({ name: "browser_dispatch_key", label: "Browser Dispatch Key", description: - "Dispatch a DOM KeyboardEvent on a specific element via JS injection. Use for React/Vue components that listen to synthetic events more reliably than CDP input.", - promptSnippet: "Dispatch a DOM KeyboardEvent on a specific element", + "Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.", + promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector", promptGuidelines: [ - "Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text into inputs (use browser_type or browser_press_key for actual typing).", + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", + "Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text (use browser_type or browser_press_key for actual typing).", "Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.", - "The selector must match exactly one or more elements; zero matches is reported as an error.", "eventType defaults to 'keydown'.", ], parameters: DispatchKeyArgs, async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { const eventType = args.eventType ?? "keydown"; + const target = args.ref ?? args.selector ?? ""; + // Ref path: dispatch on the single resolved node. Selector path: dispatch on + // all matches (preserves the original multi-match behavior). + if (args.ref !== undefined) { + const objectId = await resolveRefToObjectId(client, args.ref); + if (!objectId.success) return objectId; + const r = await client.session().call("Runtime.callFunctionOn", { + objectId: objectId.data, + functionDeclaration: `function (type, key) { this.dispatchEvent(new KeyboardEvent(type, { key, bubbles: true, cancelable: true })); return 1; }`, + arguments: [{ value: eventType }, { value: args.key }], + returnByValue: true, + }); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + return ok({ + text: `Dispatched ${eventType}('${args.key}') on ${target}`, + details: { matched: 1, ref: args.ref }, + }); + } + if (args.selector === undefined) { + return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); + } const expr = safeJs` (() => { const els = document.querySelectorAll(${args.selector}); diff --git a/src/domains/ref-resolve.ts b/src/domains/ref-resolve.ts new file mode 100644 index 0000000..3f636df --- /dev/null +++ b/src/domains/ref-resolve.ts @@ -0,0 +1,152 @@ +import type { BrowserClient } from "../client"; +import { type Result, err, ok } from "../util/result"; +import type { ToolErr } from "../util/tool"; +import { buildTree, collectInteractiveTargets, type RawAxNode, type SlimNode } from "./snapshot"; + +type Box = { x: number; y: number; width: number; height: number; cx: number; cy: number }; + +const staleErr = (ref: string): ToolErr => ({ + kind: "invalid_state", + message: `Ref ${ref} is unknown or stale — re-run browser_snapshot to get fresh refs.`, + details: { ref }, +}); + +/** + * Resolve a ref to its CDP objectId (a handle to the live JS node). Use when a + * tool needs to run JS against the element (fill, focus, select, dispatch_key). + * Re-resolves through the backendNodeId at call time, so it reflects the current + * DOM even after re-renders — and fails loudly if the node was detached. + */ +export const resolveRefToObjectId = async ( + client: BrowserClient, + ref: string, +): Promise<Result<string, ToolErr>> => { + const backendId = client.session().resolveRef(ref); + if (backendId === undefined) return err(staleErr(ref)); + const resolved = await client.session().call("DOM.resolveNode", { backendNodeId: backendId }); + if (!resolved.success) { + // A detached node (removed by a re-render) fails to resolve — surface it as + // a stale ref rather than a raw CDP error so the agent knows to re-snapshot. + return err(staleErr(ref)); + } + const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId; + if (!objectId) return err(staleErr(ref)); + return ok(objectId); +}; + +/** Resolve a ref to its current backendNodeId (for CDP calls that take one, e.g. DOM.setFileInputFiles). */ +export const resolveRefToBackendId = (client: BrowserClient, ref: string): Result<number, ToolErr> => { + const backendId = client.session().resolveRef(ref); + if (backendId === undefined) return err(staleErr(ref)); + return ok(backendId); +}; + +/** + * Resolve a ref to the current center of its bounding box, in viewport CSS + * pixels. Use for browser_click. Fetched fresh via DOM.getBoxModel, so a + * re-rendered/moved element is clicked at its new position. + */ +export const resolveRefToBox = async ( + client: BrowserClient, + ref: string, +): Promise<Result<Box, ToolErr>> => { + const backendId = client.session().resolveRef(ref); + if (backendId === undefined) return err(staleErr(ref)); + const r = await client.session().call("DOM.getBoxModel", { backendNodeId: backendId }); + if (!r.success) return err(staleErr(ref)); + const data = r.data as { model?: { content?: ReadonlyArray<number>; width?: number; height?: number } }; + const content = data.model?.content; + if (!content || content.length < 8) return err(staleErr(ref)); + const x = content[0]!; + const y = content[1]!; + const width = data.model!.width ?? 0; + const height = data.model!.height ?? 0; + if (width <= 0 || height <= 0) return err(staleErr(ref)); + return ok({ + x: Math.round(x), + y: Math.round(y), + width: Math.round(width), + height: Math.round(height), + cx: Math.round(x + width / 2), + cy: Math.round(y + height / 2), + }); +}; + +const sigOf = (n: SlimNode): string => `${n.role}|${n.name ?? ""}|${n.value ?? ""}|${n.state ?? ""}`; +const keyOf = (n: SlimNode): string => `${n.role}|${n.name ?? ""}`; + +/** + * Recompute the interactive elements after a mutation and return a compact, + * human/LLM-readable diff vs. the prior snapshot's baseline — changed values, + * newly-appeared elements (marked `*`), and removed ones. Re-publishes the fresh + * ref map so subsequent ref-based calls target the post-mutation DOM. + * + * Deliberately cheap: it re-reads the AX tree (no per-node box fetch — the + * expensive part of a full snapshot) and emits a few lines, not a full tree, so + * a long action sequence doesn't blow up the context. + * + * Returns "" on any failure or when nothing changed — callers append it only if + * non-empty, so a diff is best-effort and never blocks the primary action. + */ +export const interactiveDiff = async (client: BrowserClient): Promise<string> => { + const session = client.session(); + const prev = session.refSignatures(); + + const axRes = await session.call("Accessibility.getFullAXTree", {}); + if (!axRes.success) return ""; + const rawNodes = (axRes.data as { nodes: ReadonlyArray<RawAxNode> }).nodes; + const slim = buildTree(rawNodes, { interestingOnly: true, maxNodes: 1000 }); + const targets = collectInteractiveTargets(slim); + + // Re-publish fresh refs so later ref calls resolve against the new DOM. + const refMap = new Map<string, number>(); + const refSig = new Map<string, string>(); + targets.forEach(({ node, backendId }, i) => { + const ref = `e${i + 1}`; + node.ref = ref; + refMap.set(ref, backendId); + refSig.set(ref, sigOf(node)); + }); + session.setRefMap(refMap, refSig); + + // Compare by stable identity (role|name) rather than ref position, since refs + // renumber when the element set changes. Build prev/current identity → value. + const prevByKey = new Map<string, string>(); + for (const sig of prev.values()) { + const parts = sig.split("|"); + prevByKey.set(`${parts[0]}|${parts[1]}`, sig); + } + const curByKey = new Map<string, { ref: string; sig: string }>(); + for (const { node } of targets) { + if (node.ref) curByKey.set(keyOf(node), { ref: node.ref, sig: sigOf(node) }); + } + + const appeared: string[] = []; + const changed: string[] = []; + for (const [key, { ref, sig }] of curByKey) { + const before = prevByKey.get(key); + if (before === undefined) { + appeared.push(` *[${ref}] ${key.replace("|", ' "')}"`); + } else if (before !== sig) { + const va = before.split("|")[2] ?? ""; + const vb = sig.split("|")[2] ?? ""; + const sa = before.split("|")[3] ?? ""; + const sb = sig.split("|")[3] ?? ""; + const detail = va !== vb ? `value ${JSON.stringify(va)} → ${JSON.stringify(vb)}` : `state ${JSON.stringify(sa)} → ${JSON.stringify(sb)}`; + changed.push(` [${ref}] ${key.replace("|", ' "')}": ${detail}`); + } + } + const removed: string[] = []; + for (const key of prevByKey.keys()) { + if (!curByKey.has(key)) removed.push(` ${key.replace("|", ' "')}"`); + } + + if (appeared.length === 0 && changed.length === 0 && removed.length === 0) return ""; + const lines: string[] = ["", "Page changes (re-snapshot for full tree):"]; + if (changed.length) lines.push("Changed:", ...changed.slice(0, 12)); + if (appeared.length) lines.push("New (*):", ...appeared.slice(0, 12)); + if (removed.length) lines.push("Removed:", ...removed.slice(0, 8)); + const overflow = Math.max(0, changed.length - 12) + Math.max(0, appeared.length - 12) + Math.max(0, removed.length - 8); + if (overflow > 0) lines.push(` …and ${overflow} more (browser_snapshot for full state)`); + return lines.join("\n"); +}; diff --git a/src/domains/snapshot.ts b/src/domains/snapshot.ts index 39c493a..8665e0b 100644 --- a/src/domains/snapshot.ts +++ b/src/domains/snapshot.ts @@ -2,6 +2,7 @@ import { readFileSync, writeFileSync } from "node:fs"; import { Type } from "typebox"; import { Container, Image, type ImageTheme, Markdown, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui"; import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent"; +import type { CdpSession } from "../cdp/session"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { applyTruncation } from "../util/truncate"; @@ -27,7 +28,7 @@ const SnapshotArgs = Type.Object({ // Raw shapes from CDP Accessibility.getFullAXTree. Cast at this boundary only. type AxValue = { value?: unknown }; -type RawAxNode = { +export type RawAxNode = { nodeId: string; parentId?: string; childIds?: ReadonlyArray<string>; @@ -42,7 +43,7 @@ type RawAxNode = { type Box = { x: number; y: number; width: number; height: number; cx: number; cy: number }; -type SlimNode = { +export type SlimNode = { role: string; name?: string; value?: string; @@ -51,6 +52,8 @@ type SlimNode = { children: SlimNode[]; /** Internal: kept for the post-build box fetch. Stripped before returning to the LLM. */ _backendId?: number; + /** Stable handle (e.g. "e12") for interaction tools. Survives re-renders via backendNodeId. */ + ref?: string; /** Click target — center of the element's bounding box, in viewport CSS pixels. */ box?: Box; }; @@ -70,6 +73,20 @@ const INTERACTIVE_ROLES = new Set([ "spinbutton", ]); +// Roles whose live DOM value/checked state should override the (often stale) +// AX-tree value. The AX `value` is unreliable for freshly-typed or controlled +// inputs, so for these we read the real DOM property at snapshot time. +const VALUE_ROLES = new Set([ + "textbox", + "searchbox", + "spinbutton", + "combobox", + "checkbox", + "radio", + "switch", + "slider", +]); + const stringOf = (v: AxValue | undefined): string | undefined => { const x = v?.value; if (typeof x === "string" && x !== "") return x; @@ -93,7 +110,7 @@ const collectState = (props: RawAxNode["properties"]): string | undefined => { return flags.length > 0 ? flags.join(", ") : undefined; }; -const buildTree = ( +export const buildTree = ( rawNodes: ReadonlyArray<RawAxNode>, opts: { interestingOnly: boolean; maxNodes: number }, ): SlimNode[] => { @@ -204,8 +221,9 @@ const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => { if (n.name) line += ` "${n.name}"`; if (n.value && n.value !== n.name) line += ` = ${JSON.stringify(n.value)}`; if (n.state) line += ` (${n.state})`; - // Surface click coordinates for interactive nodes — agents can pass these - // straight to browser_click without a screenshot round-trip. + // Surface the stable ref + click coordinates for interactive nodes. Agents + // pass `ref` to interaction tools (survives re-renders); @(x,y) is a fallback. + if (n.ref) line += ` [${n.ref}]`; if (n.box && INTERACTIVE_ROLES.has(n.role)) { line += ` @(${n.box.cx},${n.box.cy})`; } @@ -229,7 +247,7 @@ const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] => * roles. Caller fetches DOM.getBoxModel for each and writes box back into the * slim node by reference. */ -const collectInteractiveTargets = ( +export const collectInteractiveTargets = ( nodes: ReadonlyArray<SlimNode>, ): Array<{ node: SlimNode; backendId: number }> => { const out: Array<{ node: SlimNode; backendId: number }> = []; @@ -245,6 +263,33 @@ const collectInteractiveTargets = ( return out; }; +/** + * Read the live DOM value for a form control by backend node id. Resolves the + * node to a JS object then reads the property that matters for its type: + * `.value` for inputs/selects/textareas, `.checked` (→ "checked"/"unchecked") + * for checkboxes/radios. Returns undefined on any failure so the caller falls + * back to the AX value. Mirrors the box-fetch error handling: never throws. + */ +const liveValue = async (session: CdpSession, backendId: number): Promise<string | undefined> => { + const resolved = await session.call("DOM.resolveNode", { backendNodeId: backendId }); + if (!resolved.success) return undefined; + const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId; + if (!objectId) return undefined; + const fn = `function () { + if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked"; + if (typeof this.value === "string") return this.value; + return null; + }`; + const r = await session.call("Runtime.callFunctionOn", { + objectId, + functionDeclaration: fn, + returnByValue: true, + }); + if (!r.success) return undefined; + const value = (r.data as { result?: { value?: unknown } }).result?.value; + return typeof value === "string" ? value : undefined; +}; + type SnapshotDetails = { nodeCount: number; truncated: boolean; @@ -258,14 +303,15 @@ export const snapshotTool = defineBrowserTool({ name: "browser_snapshot", label: "Browser Snapshot", description: - "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy) plus click coordinates for every interactive element as @(x,y). Use this BEFORE deciding whether you need a screenshot — almost always sufficient on its own and far cheaper. Pair with browser_execute_js for surgical reads of specific element values.", - promptSnippet: "Get accessibility-tree snapshot of the current page (default for page inspection)", + "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy). Every interactive element gets a stable ref shown as [eN] plus click coordinates @(x,y). Pass the ref to browser_click/browser_fill/etc. — refs survive re-renders, coordinates don't. Use this BEFORE deciding whether you need a screenshot. Pair with browser_execute_js for surgical reads of specific element values.", + promptSnippet: "Get accessibility-tree snapshot with stable element refs (default for page inspection)", promptGuidelines: [ "DEFAULT — use this whenever you need to know what's on a page, what's clickable, or how the page is structured.", - "Click coordinates come for free: every interactive element shows '@(x,y)' in the outline. Pass these straight to browser_click. NO screenshot round-trip needed.", - "DO NOT call browser_screenshot just to understand the page. This tool already gives you structure, labels, states, and click targets.", + "Refs come for free: every interactive element shows '[eN]' in the outline. Pass eN as `ref` to browser_click/browser_fill/browser_select_option/browser_focus/browser_upload_file — refs survive re-renders, so prefer them over the '@(x,y)' coordinates (a fallback).", + "DO NOT call browser_screenshot just to understand the page. This tool already gives you structure, labels, states, refs, and click targets.", + "Re-run after a navigation or a major re-render to get fresh refs; a 'ref is stale' error from an interaction tool means you need a new snapshot.", "Pass includeScreenshot:true ONLY if you also need to verify visual rendering (rare).", - "format:'json' returns the raw slim structure (with `box` per node) for programmatic use; default 'outline' is human/LLM-readable.", + "format:'json' returns the raw slim structure (with `ref` and `box` per node) for programmatic use; default 'outline' is human/LLM-readable.", ], parameters: SnapshotArgs, @@ -293,6 +339,19 @@ export const snapshotTool = defineBrowserTool({ // without a screenshot round-trip. Capped budget so a slow page can't // wedge the call. const targets = collectInteractiveTargets(slim); + + // Assign stable refs (e1, e2, …) to every interactive target. Interaction + // tools resolve these refs to backendNodeIds at call time, so they stay + // valid across re-renders. The ref → backendNodeId map (and the per-ref + // signature baseline for the post-mutation diff) is published below, after + // the box/live-value fetch so the signature reflects the live DOM value. + const refMap = new Map<string, number>(); + targets.forEach(({ node, backendId }, i) => { + const ref = `e${i + 1}`; + node.ref = ref; + refMap.set(ref, backendId); + }); + if (targets.length > 0) { const ac = new AbortController(); const timer = setTimeout(() => ac.abort(), 1_500); @@ -320,11 +379,28 @@ export const snapshotTool = defineBrowserTool({ cx: Math.round(x + width / 2), cy: Math.round(y + height / 2), }; + // Override the AX value with the live DOM property for form controls. + // The AX `value` is stale for freshly-typed / controlled inputs, so the + // agent can't trust it as a read-back signal. Reading .value/.checked + // from the actual element fixes that. + if (ac.signal.aborted || !VALUE_ROLES.has(node.role)) return; + const live = await liveValue(session, backendId); + if (live !== undefined) node.value = live; }), ); clearTimeout(timer); } + // Publish refs to the active tab. The signature (role|name|value|state) is + // the baseline the post-mutation auto-diff compares against, so it must use + // the live values resolved above. + const refSig = new Map<string, string>(); + for (const { node } of targets) { + if (node.ref === undefined) continue; + refSig.set(node.ref, `${node.role}|${node.name ?? ""}|${node.value ?? ""}|${node.state ?? ""}`); + } + session.setRefMap(refMap, refSig); + const stripped = stripInternals(slim); const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripped, null, 2); const trunc = await applyTruncation(text, "snapshot"); |
