diff options
| -rw-r--r-- | package-lock.json | 4 | ||||
| -rw-r--r-- | package.json | 2 | ||||
| -rw-r--r-- | src/cdp/session.ts | 31 | ||||
| -rw-r--r-- | src/domains/click.ts | 84 | ||||
| -rw-r--r-- | src/domains/files.ts | 37 | ||||
| -rw-r--r-- | src/domains/form.ts | 270 | ||||
| -rw-r--r-- | src/domains/keyboard.ts | 66 | ||||
| -rw-r--r-- | src/domains/ref-resolve.ts | 152 | ||||
| -rw-r--r-- | src/domains/snapshot.ts | 98 | ||||
| -rw-r--r-- | src/prompt.ts | 33 | ||||
| -rw-r--r-- | src/registry.ts | 4 |
11 files changed, 727 insertions, 54 deletions
diff --git a/package-lock.json b/package-lock.json index baa1b14..f072373 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-browser-harness", - "version": "0.5.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-browser-harness", - "version": "0.5.0", + "version": "0.7.0", "license": "MIT", "dependencies": { "ws": "^8.20.0" diff --git a/package.json b/package.json index e7768b5..9219afe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-browser-harness", - "version": "0.6.0", + "version": "0.7.0", "description": "Browser control extension for pi — navigate, click, type, screenshot, and extract data from real Chrome via CDP", "keywords": [ "pi-package", diff --git a/src/cdp/session.ts b/src/cdp/session.ts index ac19456..e090858 100644 --- a/src/cdp/session.ts +++ b/src/cdp/session.ts @@ -17,12 +17,25 @@ type TabSession = { pageInfoDirty: boolean; networkBuffer: ReturnType<typeof createNetworkBuffer>; consoleBuffer: ReturnType<typeof createConsoleBuffer>; + // Stable element refs (e1, e2, …) → CDP backendNodeId, from the latest + // snapshot of this tab. Replaced wholesale on every snapshot — old refs are + // stale anyway, so this stays bounded (~50–60 KB worst case). + refMap: Map<string, number>; + // Per-ref signature ("role|name|value") from the latest snapshot, used as the + // baseline for the post-mutation auto-diff. Replaced alongside refMap. + refSig: Map<string, string>; }; export type CdpSession = { attachFirstPage(): Promise<Result<{ readonly targetId: string; readonly sessionId: string }, CdpError>>; switchTo(targetId: string): Promise<Result<void, CdpError>>; current(): { readonly sessionId: string; readonly targetId: string } | null; + /** Replace the active tab's ref map and ref signatures (from a fresh snapshot). */ + setRefMap(refMap: Map<string, number>, refSig: Map<string, string>): void; + /** Resolve a ref (e.g. "e12") to its backendNodeId on the active tab, or undefined if unknown/stale. */ + resolveRef(ref: string): number | undefined; + /** The active tab's prior ref signatures — baseline for the post-mutation diff. */ + refSignatures(): ReadonlyMap<string, string>; call(method: string, params?: Record<string, unknown>, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>; callOnTarget(method: string, params: Record<string, unknown>, sessionId: string, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>; callBrowser(method: string, params?: Record<string, unknown>, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>; @@ -211,6 +224,8 @@ export const createCdpSession = ( pageInfoDirty: false, networkBuffer: createNetworkBuffer(), consoleBuffer: createConsoleBuffer(), + refMap: new Map(), + refSig: new Map(), }); sessionIdToTargetId.set(a.sessionId, pickTargetId); return ok({ targetId: pickTargetId, sessionId: a.sessionId }); @@ -230,6 +245,8 @@ export const createCdpSession = ( pageInfoDirty: true, networkBuffer: createNetworkBuffer(), consoleBuffer: createConsoleBuffer(), + refMap: new Map(), + refSig: new Map(), }; if (existing) { // Each Target.attachToTarget produces a new sessionId — update it. @@ -249,6 +266,20 @@ export const createCdpSession = ( current() { return sessionId && targetId ? { sessionId, targetId } : null; }, + setRefMap(refMap, refSig) { + const tab = targetId ? tabs.get(targetId) : undefined; + if (!tab) return; + tab.refMap = refMap; + tab.refSig = refSig; + }, + resolveRef(ref) { + const tab = targetId ? tabs.get(targetId) : undefined; + return tab?.refMap.get(ref); + }, + refSignatures() { + const tab = targetId ? tabs.get(targetId) : undefined; + return tab?.refSig ?? new Map(); + }, call(method, params = {}, opts = {}) { return transport.request(method, params, { sessionId, ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}) }); }, diff --git a/src/domains/click.ts b/src/domains/click.ts index 9946664..db340a8 100644 --- a/src/domains/click.ts +++ b/src/domains/click.ts @@ -1,13 +1,21 @@ -import { Type, type Static } from "typebox"; +import { Type } from "typebox"; import { Text } from "@mariozechner/pi-tui"; import type { BrowserClient } from "../client"; import { Coords, MouseButton } from "../schemas/common"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr } from "../util/tool"; import { captureWithCrosshair } from "./screenshot"; +import { interactiveDiff, resolveRefToBox } from "./ref-resolve"; const ClickArgs = Type.Object({ - ...Coords.properties, + ref: Type.Optional( + Type.String({ + description: + "Stable element ref from browser_snapshot (e.g. 'e12'). PREFERRED over x/y — survives re-renders. When set, x/y are ignored.", + }), + ), + x: Type.Optional(Coords.properties.x), + y: Type.Optional(Coords.properties.y), button: Type.Optional(MouseButton), count: Type.Optional( Type.Integer({ @@ -18,20 +26,27 @@ const ClickArgs = Type.Object({ }), ), }); -type ClickArgsT = Static<typeof ClickArgs>; const dispatchClick = async ( client: BrowserClient, - args: ClickArgsT, + x: number, + y: number, + button: "left" | "right" | "middle", + count: number, ): Promise<Result<void, ToolErr>> => { - const button = args.button ?? "left"; - const count = args.count ?? 1; + // Move the pointer to the target first. Some focus/hover handlers (and React + // synthetic events) only fire when a mousemove precedes the press, so without + // it a click can land without focusing the element. + const moved = await client.session().call("Input.dispatchMouseEvent", { + type: "mouseMoved", x, y, + }); + if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message }); const pressed = await client.session().call("Input.dispatchMouseEvent", { - type: "mousePressed", x: args.x, y: args.y, button, clickCount: count, + type: "mousePressed", x, y, button, clickCount: count, }); if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message }); const released = await client.session().call("Input.dispatchMouseEvent", { - type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count, + type: "mouseReleased", x, y, button, clickCount: count, }); if (!released.success) return err({ kind: "cdp_error", message: released.error.message }); return ok(undefined); @@ -41,34 +56,55 @@ export const clickTool = defineBrowserTool({ name: "browser_click", label: "Browser Click", description: - "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Get coordinates from browser_snapshot's @(x,y) hints — do not screenshot to find click targets.", - promptSnippet: "Click at pixel coordinates (x, y) on the page", + "Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.", + promptSnippet: "Click an element by ref (preferred) or pixel coordinates", promptGuidelines: [ - "Get (x, y) from browser_snapshot — the outline shows '@(x,y)' for every interactive element. NO screenshot needed.", - "If the target isn't an interactive role in the snapshot, fall back to browser_execute_js with `el.getBoundingClientRect()` — still NO screenshot.", - "Coordinates are viewport CSS pixels (not device pixels).", - "After clicking, verify with browser_snapshot (or browser_execute_js for a specific value). Only screenshot if you need to confirm visual rendering.", - "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.", - "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions (debug only).", - "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.", + "PREFER `ref` from browser_snapshot (the outline shows '[eN]' for every interactive element) — it survives re-renders, unlike coordinates which go stale after a save/edit reflows the page.", + "Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.", + "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.", + "After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).", + "Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.", + "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots (debug only). For React/Vue components ignoring clicks, try browser_dispatch_key.", ], parameters: ClickArgs, - renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0), + renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0), async handler(args, { client }) { - const clicked = await dispatchClick(client, args); + const button = args.button ?? "left"; + const count = args.count ?? 1; + + // Resolve the click point: ref re-resolves to the element's CURRENT box + // (so re-renders don't matter); otherwise use the literal x/y. + let x: number; + let y: number; + if (args.ref !== undefined) { + const box = await resolveRefToBox(client, args.ref); + if (!box.success) return box; + x = box.data.cx; + y = box.data.cy; + } else if (args.x !== undefined && args.y !== undefined) { + x = args.x; + y = args.y; + } else { + return err({ kind: "invalid_state", message: "Provide either `ref` or both `x` and `y`." }); + } + + const clicked = await dispatchClick(client, x, y, button, count); if (!clicked.success) return clicked; + + const target = args.ref !== undefined ? `[${args.ref}] (${x}, ${y})` : `(${x}, ${y})`; + const diff = await interactiveDiff(client); if (process.env["BH_DEBUG_CLICKS"]) { - const debug = await captureWithCrosshair(client, { x: args.x, y: args.y }); + const debug = await captureWithCrosshair(client, { x, y }); if (debug.success) { return ok({ - text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`, - details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y }, + text: `Clicked at ${target}\n[DEBUG] Overlay screenshot: ${debug.data.path}${diff}`, + details: { debugScreenshotPath: debug.data.path, x, y, ref: args.ref }, }); } } return ok({ - text: `Clicked at (${args.x}, ${args.y})`, - details: { x: args.x, y: args.y }, + text: `Clicked at ${target}${diff}`, + details: { x, y, ref: args.ref }, }); }, }); diff --git a/src/domains/files.ts b/src/domains/files.ts index 91f8761..d68d236 100644 --- a/src/domains/files.ts +++ b/src/domains/files.ts @@ -6,12 +6,28 @@ import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { safeJs } from "../util/js-template"; import { pdfPath } from "../util/paths"; +import { resolveRefToBackendId } from "./ref-resolve"; const UploadArgs = Type.Object({ - selector: Type.String({ description: "CSS selector of the file <input>" }), + ref: Type.Optional( + Type.String({ description: "Stable ref of the file <input> from browser_snapshot. PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the file <input> (fallback when no ref)" })), filePath: Type.String({ description: "Absolute path to the file to upload" }), }); +// Ref path: set files directly on the resolved backendNodeId via CDP — simplest +// and most reliable, no selector needed. +const tryRefUpload = async ( + client: BrowserClient, + backendId: number, + filePath: string, +): Promise<Result<void, ToolErr>> => { + const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId }); + if (!set.success) return err({ kind: "cdp_error", message: set.error.message }); + return ok(undefined); +}; + const verifyReadable = async (filePath: string): Promise<Result<void, ToolErr>> => { try { await access(filePath, constants.R_OK); @@ -89,16 +105,29 @@ const jsFallbackUpload = async ( export const uploadFileTool = defineBrowserTool({ name: "browser_upload_file", label: "Browser Upload File", - description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages.", - promptSnippet: "Upload a file to a file input", + description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.", + promptSnippet: "Upload a file to a file input by ref (preferred) or selector", promptGuidelines: [ + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders. (File inputs are often hidden; snapshot still surfaces them.)", "File path must be absolute and readable.", - "Selector must match a file input.", ], parameters: UploadArgs, async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { const readable = await verifyReadable(args.filePath); if (!readable.success) return readable; + + // Ref path: resolve to backendNodeId and set files directly via CDP. + if (args.ref !== undefined) { + const backendId = resolveRefToBackendId(client, args.ref); + if (!backendId.success) return backendId; + const up = await tryRefUpload(client, backendId.data, args.filePath); + if (!up.success) return up; + return ok({ text: `Uploaded ${args.filePath} via CDP (ref ${args.ref})`, details: { mode: "cdp", ref: args.ref, filePath: args.filePath } }); + } + + if (args.selector === undefined) { + return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); + } const cdp = await tryCdpUpload(client, args.selector, args.filePath); if (cdp.success) { return ok({ diff --git a/src/domains/form.ts b/src/domains/form.ts new file mode 100644 index 0000000..a10646c --- /dev/null +++ b/src/domains/form.ts @@ -0,0 +1,270 @@ +import { Type } from "typebox"; +import type { BrowserClient } from "../client"; +import { safeJs } from "../util/js-template"; +import { type Result, err, ok } from "../util/result"; +import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; +import { interactiveDiff, resolveRefToObjectId } from "./ref-resolve"; + +const FillArgs = Type.Object({ + ref: Type.Optional( + Type.String({ + description: + "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector — survives re-renders. When set, selector is ignored.", + }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the form field to fill (fallback when no ref)" })), + value: Type.String({ description: "Value to set in the field" }), +}); + +/** + * Run an element-scoped function either against a CSS selector (querySelector + + * apply) or a ref (resolve to objectId, callFunctionOn with element as `this`). + * The shared `fnBody` is a function body where `this` is the target element and + * the single argument is the value — identical logic on both paths. Returns the + * function's return value (parsed) or a ToolErr. + * + * On the ref path a detached node yields a stale-ref error; on the selector path + * a missing element returns { status: "not_found" } from the function body. + */ +const runOnElement = async ( + client: BrowserClient, + opts: { ref?: string | undefined; selector?: string | undefined; fnBody: string; arg: unknown }, +): Promise<Result<unknown, ToolErr>> => { + if (opts.ref !== undefined) { + const objectId = await resolveRefToObjectId(client, opts.ref); + if (!objectId.success) return objectId; + const r = await client.session().call("Runtime.callFunctionOn", { + objectId: objectId.data, + functionDeclaration: `function (arg) { ${opts.fnBody} }`, + arguments: [{ value: opts.arg }], + returnByValue: true, + }); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + return ok((r.data as { result?: { value?: unknown } }).result?.value); + } + if (opts.selector === undefined) { + return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); + } + // Selector path: bind the same trusted function body to the matched element + // via .call(). fnBody is harness-authored (not user input); only the selector + // and value are interpolated, and those go through safeJs. + const prelude = safeJs` + (() => { + const el = document.querySelector(${opts.selector}); + if (!el) return { status: "not_found" }; + const __arg = ${opts.arg}; + return (function (arg) {`; + const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`; + const r = await client.evaluateJs(expr); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + return ok(r.data); +}; + +/** + * Result shape returned by the in-page fill script. Discriminated by `status` + * so the handler can map page-level outcomes to typed ToolErr kinds. + */ +type FillResult = + | { status: "ok"; tag: string; kind: string; value: string } + | { status: "not_found" } + | { status: "not_fillable"; tag: string }; + +// Element-scoped fill logic. `this` is the target element, `arg` is the value. +// Write via the native prototype value setter so React's _valueTracker +// registers the change — assigning el.value directly does NOT, which is why +// controlled inputs revert on re-render. Shared verbatim by the ref path +// (callFunctionOn) and the selector path (querySelector + .call). +const FILL_FN = ` + const el = this; + const tag = el.tagName; + const proto = + el instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype + : el instanceof HTMLInputElement ? HTMLInputElement.prototype + : null; + if (proto) { + const desc = Object.getOwnPropertyDescriptor(proto, "value"); + const setter = desc && desc.set; + if (setter) setter.call(el, arg); + else el.value = arg; + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + return { status: "ok", tag, kind: el.type || tag.toLowerCase(), value: el.value }; + } + if (el.isContentEditable) { + // Rich-text editors (Slack/Notion/ProseMirror) listen for beforeinput/input. + el.focus(); + let done = false; + try { + const sel = window.getSelection(); + if (sel) { sel.removeAllRanges(); const rng = document.createRange(); rng.selectNodeContents(el); sel.addRange(rng); } + done = document.execCommand("insertText", false, arg); + } catch (e) { done = false; } + if (!done) { + el.textContent = arg; + el.dispatchEvent(new InputEvent("input", { bubbles: true })); + } + return { status: "ok", tag, kind: "contenteditable", value: el.textContent || "" }; + } + return { status: "not_fillable", tag }; +`; + +export const fillTool = defineBrowserTool({ + name: "browser_fill", + label: "Browser Fill", + description: + "Fill a form field (input/textarea/contenteditable). PREFERRED: pass `ref` (e.g. 'e7') from browser_snapshot — survives re-renders. Fallback: a CSS `selector`. Writes through the native value setter and fires bubbling 'input'/'change' events, so React/Vue/Angular controlled components and rich-text editors update correctly — unlike browser_type. Returns the field's value after writing, plus a compact diff of page changes.", + promptSnippet: "Fill a form field by ref (preferred) or selector (works with React/Vue controlled inputs)", + promptGuidelines: [ + "PREFER `ref` from browser_snapshot (the '[eN]' handle) over a guessed CSS selector — refs survive re-renders and don't require you to invent a selector.", + "Pass the desired value; no browser_click is needed first.", + "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.", + "Use browser_type instead only for keystroke-sensitive widgets (autocomplete, masked/segmented inputs).", + "The result reports the field's value after writing and an appended page-changes diff — confirm both match what you intended before moving on.", + ], + parameters: FillArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: FILL_FN, arg: args.value }); + if (!r.success) return r; + const res = r.data as FillResult | undefined; + const target = args.ref ?? args.selector ?? ""; + if (res === undefined || res.status === "not_found") { + return err({ + kind: "invalid_state", + message: `No element matched: ${target}`, + details: { ref: args.ref, selector: args.selector }, + }); + } + if (res.status === "not_fillable") { + return err({ + kind: "invalid_state", + message: `Element <${res.tag.toLowerCase()}> is not fillable (not an input, textarea, or contenteditable): ${target}. For <select> use browser_select_option.`, + details: { ref: args.ref, selector: args.selector, tag: res.tag }, + }); + } + const diff = await interactiveDiff(client); + return ok({ + text: `Filled ${target} = ${JSON.stringify(res.value)}${diff}`, + details: { ref: args.ref, selector: args.selector, value: args.value, verified: res.value, tag: res.tag, kind: res.kind }, + }); + }, +}); + +const FocusArgs = Type.Object({ + ref: Type.Optional( + Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the element to focus (fallback when no ref)" })), +}); + +type FocusResult = { status: "ok"; tag: string } | { status: "not_found" } | { status: "not_focusable"; tag: string }; + +const FOCUS_FN = ` + const el = this; + if (typeof el.focus !== "function") return { status: "not_focusable", tag: el.tagName }; + el.focus(); + return document.activeElement === el ? { status: "ok", tag: el.tagName } : { status: "not_focusable", tag: el.tagName }; +`; + +export const focusTool = defineBrowserTool({ + name: "browser_focus", + label: "Browser Focus", + description: + "Focus an element via the DOM .focus() method — deterministic, no coordinate accuracy needed. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use before browser_type when a click might miss the field.", + promptSnippet: "Focus an element by ref (preferred) or selector", + promptGuidelines: [ + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", + "Use before browser_type to guarantee the right field is focused without relying on click coordinates.", + "For simply setting a value, prefer browser_fill (it doesn't need a separate focus step).", + ], + parameters: FocusArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: FOCUS_FN, arg: "" }); + if (!r.success) return r; + const res = r.data as FocusResult | undefined; + const target = args.ref ?? args.selector ?? ""; + if (res === undefined || res.status === "not_found") { + return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector } }); + } + if (res.status === "not_focusable") { + return err({ kind: "invalid_state", message: `Element <${res.tag.toLowerCase()}> could not be focused: ${target}`, details: { ref: args.ref, selector: args.selector, tag: res.tag } }); + } + return ok({ text: `Focused ${target}`, details: { ref: args.ref, selector: args.selector, tag: res.tag } }); + }, +}); + +const SelectOptionArgs = Type.Object({ + ref: Type.Optional( + Type.String({ description: "Stable element ref of the <select> from browser_snapshot. PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the <select> element (fallback when no ref)" })), + value: Type.Optional(Type.String({ description: "Option value attribute to select" })), + label: Type.Optional(Type.String({ description: "Visible option text to select (exact match)" })), + index: Type.Optional(Type.Integer({ minimum: 0, description: "Zero-based option index to select" })), +}); + +type SelectResult = + | { status: "ok"; value: string; label: string } + | { status: "not_found" } + | { status: "not_select"; tag: string } + | { status: "no_match"; options: ReadonlyArray<{ value: string; label: string }> }; + +// `this` is the <select>; `arg` is { wv, wl, wi } (null = not provided). +const SELECT_FN = ` + const el = this; + if (el.tagName !== "SELECT") return { status: "not_select", tag: el.tagName }; + const opts = Array.from(el.options); + const wv = arg.wv, wl = arg.wl, wi = arg.wi; + let opt = null; + if (wi !== null) opt = opts[wi] || null; + else if (wv !== null) opt = opts.find(o => o.value === wv) || null; + else if (wl !== null) opt = opts.find(o => o.text === wl) || null; + if (!opt) return { status: "no_match", options: opts.map(o => ({ value: o.value, label: o.text })) }; + el.value = opt.value; + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + return { status: "ok", value: opt.value, label: opt.text }; +`; + +export const selectOptionTool = defineBrowserTool({ + name: "browser_select_option", + label: "Browser Select Option", + description: + "Select an option in a native <select> element by value, visible label, or index. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Sets the selection and fires bubbling 'input'/'change' events so framework listeners update. On no match, returns the available options.", + promptSnippet: "Select an option in a native <select> by ref (preferred) or selector", + promptGuidelines: [ + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", + "Provide exactly one of value, label, or index to choose the option.", + "For custom (non-native) dropdowns built from divs, click to open then browser_click the option instead.", + "If no option matches, the error details list the available options so you can retry.", + ], + parameters: SelectOptionArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + if (args.value === undefined && args.label === undefined && args.index === undefined) { + return err({ kind: "invalid_state", message: "Provide one of value, label, or index." }); + } + // null sentinels distinguish "not provided" from a real empty-string match. + const arg = { wv: args.value ?? null, wl: args.label ?? null, wi: args.index ?? null }; + const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: SELECT_FN, arg }); + if (!r.success) return r; + const res = r.data as SelectResult | undefined; + const target = args.ref ?? args.selector ?? ""; + if (res === undefined || res.status === "not_found") { + return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector } }); + } + if (res.status === "not_select") { + return err({ kind: "invalid_state", message: `Element <${res.tag.toLowerCase()}> is not a <select>: ${target}`, details: { ref: args.ref, selector: args.selector, tag: res.tag } }); + } + if (res.status === "no_match") { + return err({ + kind: "invalid_state", + message: `No matching option in ${target}. Available: ${res.options.map((o) => o.label).join(", ")}`, + details: { ref: args.ref, selector: args.selector, options: res.options }, + }); + } + const diff = await interactiveDiff(client); + return ok({ + text: `Selected "${res.label}" (value=${JSON.stringify(res.value)}) in ${target}${diff}`, + details: { ref: args.ref, selector: args.selector, value: res.value, label: res.label }, + }); + }, +}); diff --git a/src/domains/keyboard.ts b/src/domains/keyboard.ts index c39814e..69a7396 100644 --- a/src/domains/keyboard.ts +++ b/src/domains/keyboard.ts @@ -3,6 +3,7 @@ import { safeJs } from "../util/js-template"; import { virtualKeyCode } from "../util/keycodes"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; +import { resolveRefToObjectId } from "./ref-resolve"; const TypeArgs = Type.Object({ text: Type.String({ description: "Text to type" }), @@ -19,6 +20,29 @@ export const typeTool = defineBrowserTool({ ], parameters: TypeArgs, async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + // Guard against silent text loss: Input.insertText succeeds regardless of + // whether an editable element is focused, so without this check typed text + // can vanish into the void with no error. Fail loudly with an actionable + // message instead. IIFE starts with "(" so evaluateJs returns it directly. + const focused = await client.evaluateJs(safeJs` + (() => { + const el = document.activeElement; + const ok = !!el && el !== document.body && + (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable); + return { ok, tag: el ? el.tagName : null }; + })() + `); + if (focused.success) { + const f = focused.data as { ok: boolean; tag: string | null }; + if (!f.ok) { + return err({ + kind: "invalid_state", + message: + "No editable element is focused — typed text would be lost. Click the field with browser_click first, or use browser_fill with a selector.", + details: { activeElement: f.tag }, + }); + } + } const r = await client.session().call("Input.insertText", { text: args.text }); if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); return ok({ text: `Typed: "${args.text}"` }); @@ -67,6 +91,14 @@ export const pressKeyTool = defineBrowserTool({ }; const down = await client.session().call("Input.dispatchKeyEvent", downParams); if (!down.success) return err({ kind: "cdp_error", message: down.error.message }); + // For a printable character with no command modifier (Ctrl/Meta/Alt = 1|2|4), + // emit a `char` event so the page receives keypress/textInput and the + // character is actually inserted. Shift (8) is allowed (capitals/symbols). + const hasCommandModifier = (modifiers & (1 | 2 | 4)) !== 0; + if (isChar && !hasCommandModifier) { + const charEv = await client.session().call("Input.dispatchKeyEvent", { type: "char", text: k, key: k }); + if (!charEv.success) return err({ kind: "cdp_error", message: charEv.error.message }); + } const up = await client.session().call("Input.dispatchKeyEvent", { type: "keyUp", key: k, code: k, modifiers }); if (!up.success) return err({ kind: "cdp_error", message: up.error.message }); return ok({ text: `Pressed: ${k}${modifiers ? ` (modifiers=${modifiers})` : ""}` }); @@ -74,7 +106,10 @@ export const pressKeyTool = defineBrowserTool({ }); const DispatchKeyArgs = Type.Object({ - selector: Type.String({ description: "CSS selector of the target element" }), + ref: Type.Optional( + Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the target element (fallback when no ref)" })), key: Type.String({ description: "Key value (e.g., 'Enter', 'a')" }), eventType: Type.Optional( Type.Union( @@ -88,17 +123,38 @@ export const dispatchKeyTool = defineBrowserTool({ name: "browser_dispatch_key", label: "Browser Dispatch Key", description: - "Dispatch a DOM KeyboardEvent on a specific element via JS injection. Use for React/Vue components that listen to synthetic events more reliably than CDP input.", - promptSnippet: "Dispatch a DOM KeyboardEvent on a specific element", + "Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.", + promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector", promptGuidelines: [ - "Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text into inputs (use browser_type or browser_press_key for actual typing).", + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", + "Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text (use browser_type or browser_press_key for actual typing).", "Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.", - "The selector must match exactly one or more elements; zero matches is reported as an error.", "eventType defaults to 'keydown'.", ], parameters: DispatchKeyArgs, async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { const eventType = args.eventType ?? "keydown"; + const target = args.ref ?? args.selector ?? ""; + // Ref path: dispatch on the single resolved node. Selector path: dispatch on + // all matches (preserves the original multi-match behavior). + if (args.ref !== undefined) { + const objectId = await resolveRefToObjectId(client, args.ref); + if (!objectId.success) return objectId; + const r = await client.session().call("Runtime.callFunctionOn", { + objectId: objectId.data, + functionDeclaration: `function (type, key) { this.dispatchEvent(new KeyboardEvent(type, { key, bubbles: true, cancelable: true })); return 1; }`, + arguments: [{ value: eventType }, { value: args.key }], + returnByValue: true, + }); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + return ok({ + text: `Dispatched ${eventType}('${args.key}') on ${target}`, + details: { matched: 1, ref: args.ref }, + }); + } + if (args.selector === undefined) { + return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); + } const expr = safeJs` (() => { const els = document.querySelectorAll(${args.selector}); diff --git a/src/domains/ref-resolve.ts b/src/domains/ref-resolve.ts new file mode 100644 index 0000000..3f636df --- /dev/null +++ b/src/domains/ref-resolve.ts @@ -0,0 +1,152 @@ +import type { BrowserClient } from "../client"; +import { type Result, err, ok } from "../util/result"; +import type { ToolErr } from "../util/tool"; +import { buildTree, collectInteractiveTargets, type RawAxNode, type SlimNode } from "./snapshot"; + +type Box = { x: number; y: number; width: number; height: number; cx: number; cy: number }; + +const staleErr = (ref: string): ToolErr => ({ + kind: "invalid_state", + message: `Ref ${ref} is unknown or stale — re-run browser_snapshot to get fresh refs.`, + details: { ref }, +}); + +/** + * Resolve a ref to its CDP objectId (a handle to the live JS node). Use when a + * tool needs to run JS against the element (fill, focus, select, dispatch_key). + * Re-resolves through the backendNodeId at call time, so it reflects the current + * DOM even after re-renders — and fails loudly if the node was detached. + */ +export const resolveRefToObjectId = async ( + client: BrowserClient, + ref: string, +): Promise<Result<string, ToolErr>> => { + const backendId = client.session().resolveRef(ref); + if (backendId === undefined) return err(staleErr(ref)); + const resolved = await client.session().call("DOM.resolveNode", { backendNodeId: backendId }); + if (!resolved.success) { + // A detached node (removed by a re-render) fails to resolve — surface it as + // a stale ref rather than a raw CDP error so the agent knows to re-snapshot. + return err(staleErr(ref)); + } + const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId; + if (!objectId) return err(staleErr(ref)); + return ok(objectId); +}; + +/** Resolve a ref to its current backendNodeId (for CDP calls that take one, e.g. DOM.setFileInputFiles). */ +export const resolveRefToBackendId = (client: BrowserClient, ref: string): Result<number, ToolErr> => { + const backendId = client.session().resolveRef(ref); + if (backendId === undefined) return err(staleErr(ref)); + return ok(backendId); +}; + +/** + * Resolve a ref to the current center of its bounding box, in viewport CSS + * pixels. Use for browser_click. Fetched fresh via DOM.getBoxModel, so a + * re-rendered/moved element is clicked at its new position. + */ +export const resolveRefToBox = async ( + client: BrowserClient, + ref: string, +): Promise<Result<Box, ToolErr>> => { + const backendId = client.session().resolveRef(ref); + if (backendId === undefined) return err(staleErr(ref)); + const r = await client.session().call("DOM.getBoxModel", { backendNodeId: backendId }); + if (!r.success) return err(staleErr(ref)); + const data = r.data as { model?: { content?: ReadonlyArray<number>; width?: number; height?: number } }; + const content = data.model?.content; + if (!content || content.length < 8) return err(staleErr(ref)); + const x = content[0]!; + const y = content[1]!; + const width = data.model!.width ?? 0; + const height = data.model!.height ?? 0; + if (width <= 0 || height <= 0) return err(staleErr(ref)); + return ok({ + x: Math.round(x), + y: Math.round(y), + width: Math.round(width), + height: Math.round(height), + cx: Math.round(x + width / 2), + cy: Math.round(y + height / 2), + }); +}; + +const sigOf = (n: SlimNode): string => `${n.role}|${n.name ?? ""}|${n.value ?? ""}|${n.state ?? ""}`; +const keyOf = (n: SlimNode): string => `${n.role}|${n.name ?? ""}`; + +/** + * Recompute the interactive elements after a mutation and return a compact, + * human/LLM-readable diff vs. the prior snapshot's baseline — changed values, + * newly-appeared elements (marked `*`), and removed ones. Re-publishes the fresh + * ref map so subsequent ref-based calls target the post-mutation DOM. + * + * Deliberately cheap: it re-reads the AX tree (no per-node box fetch — the + * expensive part of a full snapshot) and emits a few lines, not a full tree, so + * a long action sequence doesn't blow up the context. + * + * Returns "" on any failure or when nothing changed — callers append it only if + * non-empty, so a diff is best-effort and never blocks the primary action. + */ +export const interactiveDiff = async (client: BrowserClient): Promise<string> => { + const session = client.session(); + const prev = session.refSignatures(); + + const axRes = await session.call("Accessibility.getFullAXTree", {}); + if (!axRes.success) return ""; + const rawNodes = (axRes.data as { nodes: ReadonlyArray<RawAxNode> }).nodes; + const slim = buildTree(rawNodes, { interestingOnly: true, maxNodes: 1000 }); + const targets = collectInteractiveTargets(slim); + + // Re-publish fresh refs so later ref calls resolve against the new DOM. + const refMap = new Map<string, number>(); + const refSig = new Map<string, string>(); + targets.forEach(({ node, backendId }, i) => { + const ref = `e${i + 1}`; + node.ref = ref; + refMap.set(ref, backendId); + refSig.set(ref, sigOf(node)); + }); + session.setRefMap(refMap, refSig); + + // Compare by stable identity (role|name) rather than ref position, since refs + // renumber when the element set changes. Build prev/current identity → value. + const prevByKey = new Map<string, string>(); + for (const sig of prev.values()) { + const parts = sig.split("|"); + prevByKey.set(`${parts[0]}|${parts[1]}`, sig); + } + const curByKey = new Map<string, { ref: string; sig: string }>(); + for (const { node } of targets) { + if (node.ref) curByKey.set(keyOf(node), { ref: node.ref, sig: sigOf(node) }); + } + + const appeared: string[] = []; + const changed: string[] = []; + for (const [key, { ref, sig }] of curByKey) { + const before = prevByKey.get(key); + if (before === undefined) { + appeared.push(` *[${ref}] ${key.replace("|", ' "')}"`); + } else if (before !== sig) { + const va = before.split("|")[2] ?? ""; + const vb = sig.split("|")[2] ?? ""; + const sa = before.split("|")[3] ?? ""; + const sb = sig.split("|")[3] ?? ""; + const detail = va !== vb ? `value ${JSON.stringify(va)} → ${JSON.stringify(vb)}` : `state ${JSON.stringify(sa)} → ${JSON.stringify(sb)}`; + changed.push(` [${ref}] ${key.replace("|", ' "')}": ${detail}`); + } + } + const removed: string[] = []; + for (const key of prevByKey.keys()) { + if (!curByKey.has(key)) removed.push(` ${key.replace("|", ' "')}"`); + } + + if (appeared.length === 0 && changed.length === 0 && removed.length === 0) return ""; + const lines: string[] = ["", "Page changes (re-snapshot for full tree):"]; + if (changed.length) lines.push("Changed:", ...changed.slice(0, 12)); + if (appeared.length) lines.push("New (*):", ...appeared.slice(0, 12)); + if (removed.length) lines.push("Removed:", ...removed.slice(0, 8)); + const overflow = Math.max(0, changed.length - 12) + Math.max(0, appeared.length - 12) + Math.max(0, removed.length - 8); + if (overflow > 0) lines.push(` …and ${overflow} more (browser_snapshot for full state)`); + return lines.join("\n"); +}; diff --git a/src/domains/snapshot.ts b/src/domains/snapshot.ts index 39c493a..8665e0b 100644 --- a/src/domains/snapshot.ts +++ b/src/domains/snapshot.ts @@ -2,6 +2,7 @@ import { readFileSync, writeFileSync } from "node:fs"; import { Type } from "typebox"; import { Container, Image, type ImageTheme, Markdown, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui"; import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent"; +import type { CdpSession } from "../cdp/session"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { applyTruncation } from "../util/truncate"; @@ -27,7 +28,7 @@ const SnapshotArgs = Type.Object({ // Raw shapes from CDP Accessibility.getFullAXTree. Cast at this boundary only. type AxValue = { value?: unknown }; -type RawAxNode = { +export type RawAxNode = { nodeId: string; parentId?: string; childIds?: ReadonlyArray<string>; @@ -42,7 +43,7 @@ type RawAxNode = { type Box = { x: number; y: number; width: number; height: number; cx: number; cy: number }; -type SlimNode = { +export type SlimNode = { role: string; name?: string; value?: string; @@ -51,6 +52,8 @@ type SlimNode = { children: SlimNode[]; /** Internal: kept for the post-build box fetch. Stripped before returning to the LLM. */ _backendId?: number; + /** Stable handle (e.g. "e12") for interaction tools. Survives re-renders via backendNodeId. */ + ref?: string; /** Click target — center of the element's bounding box, in viewport CSS pixels. */ box?: Box; }; @@ -70,6 +73,20 @@ const INTERACTIVE_ROLES = new Set([ "spinbutton", ]); +// Roles whose live DOM value/checked state should override the (often stale) +// AX-tree value. The AX `value` is unreliable for freshly-typed or controlled +// inputs, so for these we read the real DOM property at snapshot time. +const VALUE_ROLES = new Set([ + "textbox", + "searchbox", + "spinbutton", + "combobox", + "checkbox", + "radio", + "switch", + "slider", +]); + const stringOf = (v: AxValue | undefined): string | undefined => { const x = v?.value; if (typeof x === "string" && x !== "") return x; @@ -93,7 +110,7 @@ const collectState = (props: RawAxNode["properties"]): string | undefined => { return flags.length > 0 ? flags.join(", ") : undefined; }; -const buildTree = ( +export const buildTree = ( rawNodes: ReadonlyArray<RawAxNode>, opts: { interestingOnly: boolean; maxNodes: number }, ): SlimNode[] => { @@ -204,8 +221,9 @@ const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => { if (n.name) line += ` "${n.name}"`; if (n.value && n.value !== n.name) line += ` = ${JSON.stringify(n.value)}`; if (n.state) line += ` (${n.state})`; - // Surface click coordinates for interactive nodes — agents can pass these - // straight to browser_click without a screenshot round-trip. + // Surface the stable ref + click coordinates for interactive nodes. Agents + // pass `ref` to interaction tools (survives re-renders); @(x,y) is a fallback. + if (n.ref) line += ` [${n.ref}]`; if (n.box && INTERACTIVE_ROLES.has(n.role)) { line += ` @(${n.box.cx},${n.box.cy})`; } @@ -229,7 +247,7 @@ const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] => * roles. Caller fetches DOM.getBoxModel for each and writes box back into the * slim node by reference. */ -const collectInteractiveTargets = ( +export const collectInteractiveTargets = ( nodes: ReadonlyArray<SlimNode>, ): Array<{ node: SlimNode; backendId: number }> => { const out: Array<{ node: SlimNode; backendId: number }> = []; @@ -245,6 +263,33 @@ const collectInteractiveTargets = ( return out; }; +/** + * Read the live DOM value for a form control by backend node id. Resolves the + * node to a JS object then reads the property that matters for its type: + * `.value` for inputs/selects/textareas, `.checked` (→ "checked"/"unchecked") + * for checkboxes/radios. Returns undefined on any failure so the caller falls + * back to the AX value. Mirrors the box-fetch error handling: never throws. + */ +const liveValue = async (session: CdpSession, backendId: number): Promise<string | undefined> => { + const resolved = await session.call("DOM.resolveNode", { backendNodeId: backendId }); + if (!resolved.success) return undefined; + const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId; + if (!objectId) return undefined; + const fn = `function () { + if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked"; + if (typeof this.value === "string") return this.value; + return null; + }`; + const r = await session.call("Runtime.callFunctionOn", { + objectId, + functionDeclaration: fn, + returnByValue: true, + }); + if (!r.success) return undefined; + const value = (r.data as { result?: { value?: unknown } }).result?.value; + return typeof value === "string" ? value : undefined; +}; + type SnapshotDetails = { nodeCount: number; truncated: boolean; @@ -258,14 +303,15 @@ export const snapshotTool = defineBrowserTool({ name: "browser_snapshot", label: "Browser Snapshot", description: - "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy) plus click coordinates for every interactive element as @(x,y). Use this BEFORE deciding whether you need a screenshot — almost always sufficient on its own and far cheaper. Pair with browser_execute_js for surgical reads of specific element values.", - promptSnippet: "Get accessibility-tree snapshot of the current page (default for page inspection)", + "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy). Every interactive element gets a stable ref shown as [eN] plus click coordinates @(x,y). Pass the ref to browser_click/browser_fill/etc. — refs survive re-renders, coordinates don't. Use this BEFORE deciding whether you need a screenshot. Pair with browser_execute_js for surgical reads of specific element values.", + promptSnippet: "Get accessibility-tree snapshot with stable element refs (default for page inspection)", promptGuidelines: [ "DEFAULT — use this whenever you need to know what's on a page, what's clickable, or how the page is structured.", - "Click coordinates come for free: every interactive element shows '@(x,y)' in the outline. Pass these straight to browser_click. NO screenshot round-trip needed.", - "DO NOT call browser_screenshot just to understand the page. This tool already gives you structure, labels, states, and click targets.", + "Refs come for free: every interactive element shows '[eN]' in the outline. Pass eN as `ref` to browser_click/browser_fill/browser_select_option/browser_focus/browser_upload_file — refs survive re-renders, so prefer them over the '@(x,y)' coordinates (a fallback).", + "DO NOT call browser_screenshot just to understand the page. This tool already gives you structure, labels, states, refs, and click targets.", + "Re-run after a navigation or a major re-render to get fresh refs; a 'ref is stale' error from an interaction tool means you need a new snapshot.", "Pass includeScreenshot:true ONLY if you also need to verify visual rendering (rare).", - "format:'json' returns the raw slim structure (with `box` per node) for programmatic use; default 'outline' is human/LLM-readable.", + "format:'json' returns the raw slim structure (with `ref` and `box` per node) for programmatic use; default 'outline' is human/LLM-readable.", ], parameters: SnapshotArgs, @@ -293,6 +339,19 @@ export const snapshotTool = defineBrowserTool({ // without a screenshot round-trip. Capped budget so a slow page can't // wedge the call. const targets = collectInteractiveTargets(slim); + + // Assign stable refs (e1, e2, …) to every interactive target. Interaction + // tools resolve these refs to backendNodeIds at call time, so they stay + // valid across re-renders. The ref → backendNodeId map (and the per-ref + // signature baseline for the post-mutation diff) is published below, after + // the box/live-value fetch so the signature reflects the live DOM value. + const refMap = new Map<string, number>(); + targets.forEach(({ node, backendId }, i) => { + const ref = `e${i + 1}`; + node.ref = ref; + refMap.set(ref, backendId); + }); + if (targets.length > 0) { const ac = new AbortController(); const timer = setTimeout(() => ac.abort(), 1_500); @@ -320,11 +379,28 @@ export const snapshotTool = defineBrowserTool({ cx: Math.round(x + width / 2), cy: Math.round(y + height / 2), }; + // Override the AX value with the live DOM property for form controls. + // The AX `value` is stale for freshly-typed / controlled inputs, so the + // agent can't trust it as a read-back signal. Reading .value/.checked + // from the actual element fixes that. + if (ac.signal.aborted || !VALUE_ROLES.has(node.role)) return; + const live = await liveValue(session, backendId); + if (live !== undefined) node.value = live; }), ); clearTimeout(timer); } + // Publish refs to the active tab. The signature (role|name|value|state) is + // the baseline the post-mutation auto-diff compares against, so it must use + // the live values resolved above. + const refSig = new Map<string, string>(); + for (const { node } of targets) { + if (node.ref === undefined) continue; + refSig.set(node.ref, `${node.role}|${node.name ?? ""}|${node.value ?? ""}|${node.state ?? ""}`); + } + session.setRefMap(refMap, refSig); + const stripped = stripInternals(slim); const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripped, null, 2); const trunc = await applyTruncation(text, "snapshot"); diff --git a/src/prompt.ts b/src/prompt.ts index ae94db3..7483730 100644 --- a/src/prompt.ts +++ b/src/prompt.ts @@ -21,9 +21,13 @@ to interact with pages visually, and other tools alongside them. | Tool | Purpose | |------|---------| +| browser_snapshot | Accessibility-tree snapshot. Assigns every interactive element a stable **ref** (\`[eN]\`) plus \`@(x,y)\`. Refs are the primary handle for interaction tools — they survive re-renders, coordinates don't. | | browser_screenshot | Capture a PNG or JPEG screenshot of the current page (rendered inline in the TUI; the LLM reads the image via the saved file path) | -| browser_click | Click at viewport coordinates | -| browser_type | Type text into the focused element | +| browser_click | Click an element by **ref** (preferred — re-resolves position, survives re-renders) or viewport x/y. Appends a compact diff of page changes. | +| browser_type | Type text into the focused element (real keystrokes; requires a focused field) | +| browser_fill | Fill an input/textarea by **ref** (preferred) or selector — updates React/Vue controlled components, verifies the value, appends a page-changes diff | +| browser_select_option | Select an option in a native <select> by **ref** (preferred) or selector, choosing by value/label/index (fires change) | +| browser_focus | Focus an element by **ref** (preferred) or selector (deterministic; use before browser_type) | | browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) | | browser_scroll | Scroll the page at coordinates | | browser_navigate | Navigate to a URL (result details.outcome.kind is "in_place" or "new_tab_created") | @@ -38,6 +42,7 @@ to interact with pages visually, and other tools alongside them. | browser_http_get | Direct HTTP GET (outside browser, for APIs) | | browser_console | Read JS errors / console output on the current tab (diagnostic — use when an action looks broken; pass sinceSeq from the previous nextCursor to see only new messages) | | browser_wait | Wait N seconds | +| browser_wait_for | Wait until a selector/text appears (or a selector disappears) — use for SPA content before interacting | | browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) | | browser_handle_dialog | Accept or dismiss a JS dialog | | browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) | @@ -83,11 +88,25 @@ browser_http_get("https://api.example.com/data") + browser_click(x, y) browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot() \`\`\` -**Form filling:** +**Form filling (ref-first — this is the reliable path on React/SPA forms):** \`\`\` -browser_screenshot() → find input coordinates → browser_click(x, y) -→ browser_type("text") → browser_press_key("Tab") → browser_screenshot() +browser_wait_for({ selector: "#email" }) // ensure SPA field is rendered +browser_snapshot() // each interactive element gets a [eN] ref +→ browser_fill({ ref: "e7", value: "a@b.com" }) // target by ref, not a guessed selector +→ browser_select_option({ ref: "e9", label: "India" }) // native <select> by ref +→ browser_click({ ref: "e12" }) // e.g. Save — re-resolves position even if the form reflowed +// each mutating call appends a compact "Page changes" diff — read it to confirm +// the change landed (e.g. a save closed the form, a new field appeared as *[eN]). \`\`\` +**Why ref over selector/coordinates:** refs are keyed to the element's identity, so +they survive the re-renders that React/Vue forms trigger on every edit and save — +coordinates go stale and selectors you guess often don't match. If a call returns +"ref is stale", the page changed: re-run browser_snapshot for fresh refs. + +Prefer browser_fill for text inputs/textareas — it fires input/change so controlled +components update, and returns the field's value for verification. Use browser_click + +browser_type only for keystroke-sensitive widgets (autocomplete, masked inputs); +browser_type requires an already-focused field and errors if none is focused. **Data extraction:** \`\`\` @@ -121,8 +140,8 @@ write("/tmp/extract.js", "...script with daemon access...") → browser_run_scri | Tool | Purpose | |------|---------| -| browser_upload_file | Upload a file to a file input (bypasses file picker) | -| browser_dispatch_key | Dispatch a DOM KeyboardEvent on a specific element (for React/Vue inputs); returns details.matched (number of elements) | +| browser_upload_file | Upload a file to a file input by **ref** (preferred) or selector (bypasses file picker) | +| browser_dispatch_key | Dispatch a DOM KeyboardEvent on an element by **ref** (preferred) or selector (for React/Vue inputs); returns details.matched | | browser_download | Configure download directory and disable save-as prompts | | browser_viewport_resize | Resize the viewport for responsive testing | | browser_drag_and_drop | Perform drag-and-drop from one coordinate to another | diff --git a/src/registry.ts b/src/registry.ts index fcdb66a..d09f1cc 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -3,6 +3,7 @@ import type { BrowserClient } from "./client"; import { type AnyBrowserToolDefinition, registerBrowserTool } from "./util/tool"; import { clickTool } from "./domains/click"; import { typeTool, pressKeyTool, dispatchKeyTool } from "./domains/keyboard"; +import { fillTool, selectOptionTool, focusTool } from "./domains/form"; import { pageInfoTool, waitTool, waitForLoadTool } from "./domains/page"; import { scrollTool } from "./domains/scroll"; import { handleDialogTool } from "./domains/dialog"; @@ -27,6 +28,9 @@ const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [ setupTool, { ...clickTool, serialized: true }, { ...typeTool, serialized: true }, + { ...fillTool, serialized: true }, + { ...selectOptionTool, serialized: true }, + { ...focusTool, serialized: true }, { ...pressKeyTool, serialized: true }, { ...dispatchKeyTool, serialized: true }, { ...scrollTool, serialized: true }, |