From 0b32e88f84f36a759d7191e8bdfd09a44789fa95 Mon Sep 17 00:00:00 2001 From: Aman Kumar <145339995+amankumarsingh77@users.noreply.github.com> Date: Mon, 29 Jun 2026 00:02:54 +0530 Subject: feat: address elements by stable refs across re-renders (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interaction tools could only target elements by raw viewport coordinates (browser_click) or model-guessed CSS selectors (browser_fill/focus/ select_option/upload). Neither survives a re-render: on React/SPA forms, every save or edit reflows the page, so captured @(x,y) coordinates go stale and guessed selectors miss — leaving the agent to fall back to hand-rolled execute_js DOM walks and unnoticed failed saves. Add a stable ref system, mirroring Playwright MCP / browser-use: - snapshot assigns each interactive element a ref (e1, e2, …), shown as [eN] in the outline, and publishes a per-tab ref → backendNodeId map - interaction tools accept an optional `ref` that re-resolves the element at call time (via DOM.resolveNode / getBoxModel), so it stays valid across re-renders; coordinates/selectors remain as fallbacks - mutating tools append a compact diff of changed / newly-appeared (*) / removed interactive elements, so silent failed saves surface immediately - stale refs return an actionable "re-snapshot" error instead of a miss Ref map is replaced per snapshot (bounded memory); fill keeps the native _valueTracker write path so React controlled inputs still update. Co-authored-by: Claude Opus 4.8 --- package-lock.json | 4 +- package.json | 2 +- src/cdp/session.ts | 31 ++++++ src/domains/click.ts | 84 ++++++++++---- src/domains/files.ts | 37 ++++++- src/domains/form.ts | 270 +++++++++++++++++++++++++++++++++++++++++++++ src/domains/keyboard.ts | 66 ++++++++++- src/domains/ref-resolve.ts | 152 +++++++++++++++++++++++++ src/domains/snapshot.ts | 98 ++++++++++++++-- src/prompt.ts | 33 ++++-- src/registry.ts | 4 + 11 files changed, 727 insertions(+), 54 deletions(-) create mode 100644 src/domains/form.ts create mode 100644 src/domains/ref-resolve.ts diff --git a/package-lock.json b/package-lock.json index baa1b14..f072373 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-browser-harness", - "version": "0.5.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-browser-harness", - "version": "0.5.0", + "version": "0.7.0", "license": "MIT", "dependencies": { "ws": "^8.20.0" diff --git a/package.json b/package.json index e7768b5..9219afe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-browser-harness", - "version": "0.6.0", + "version": "0.7.0", "description": "Browser control extension for pi — navigate, click, type, screenshot, and extract data from real Chrome via CDP", "keywords": [ "pi-package", diff --git a/src/cdp/session.ts b/src/cdp/session.ts index ac19456..e090858 100644 --- a/src/cdp/session.ts +++ b/src/cdp/session.ts @@ -17,12 +17,25 @@ type TabSession = { pageInfoDirty: boolean; networkBuffer: ReturnType; consoleBuffer: ReturnType; + // Stable element refs (e1, e2, …) → CDP backendNodeId, from the latest + // snapshot of this tab. Replaced wholesale on every snapshot — old refs are + // stale anyway, so this stays bounded (~50–60 KB worst case). + refMap: Map; + // Per-ref signature ("role|name|value") from the latest snapshot, used as the + // baseline for the post-mutation auto-diff. Replaced alongside refMap. + refSig: Map; }; export type CdpSession = { attachFirstPage(): Promise>; switchTo(targetId: string): Promise>; current(): { readonly sessionId: string; readonly targetId: string } | null; + /** Replace the active tab's ref map and ref signatures (from a fresh snapshot). */ + setRefMap(refMap: Map, refSig: Map): void; + /** Resolve a ref (e.g. "e12") to its backendNodeId on the active tab, or undefined if unknown/stale. */ + resolveRef(ref: string): number | undefined; + /** The active tab's prior ref signatures — baseline for the post-mutation diff. */ + refSignatures(): ReadonlyMap; call(method: string, params?: Record, opts?: { timeoutMs?: number }): Promise>; callOnTarget(method: string, params: Record, sessionId: string, opts?: { timeoutMs?: number }): Promise>; callBrowser(method: string, params?: Record, opts?: { timeoutMs?: number }): Promise>; @@ -211,6 +224,8 @@ export const createCdpSession = ( pageInfoDirty: false, networkBuffer: createNetworkBuffer(), consoleBuffer: createConsoleBuffer(), + refMap: new Map(), + refSig: new Map(), }); sessionIdToTargetId.set(a.sessionId, pickTargetId); return ok({ targetId: pickTargetId, sessionId: a.sessionId }); @@ -230,6 +245,8 @@ export const createCdpSession = ( pageInfoDirty: true, networkBuffer: createNetworkBuffer(), consoleBuffer: createConsoleBuffer(), + refMap: new Map(), + refSig: new Map(), }; if (existing) { // Each Target.attachToTarget produces a new sessionId — update it. @@ -249,6 +266,20 @@ export const createCdpSession = ( current() { return sessionId && targetId ? { sessionId, targetId } : null; }, + setRefMap(refMap, refSig) { + const tab = targetId ? tabs.get(targetId) : undefined; + if (!tab) return; + tab.refMap = refMap; + tab.refSig = refSig; + }, + resolveRef(ref) { + const tab = targetId ? tabs.get(targetId) : undefined; + return tab?.refMap.get(ref); + }, + refSignatures() { + const tab = targetId ? tabs.get(targetId) : undefined; + return tab?.refSig ?? new Map(); + }, call(method, params = {}, opts = {}) { return transport.request(method, params, { sessionId, ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}) }); }, diff --git a/src/domains/click.ts b/src/domains/click.ts index 9946664..db340a8 100644 --- a/src/domains/click.ts +++ b/src/domains/click.ts @@ -1,13 +1,21 @@ -import { Type, type Static } from "typebox"; +import { Type } from "typebox"; import { Text } from "@mariozechner/pi-tui"; import type { BrowserClient } from "../client"; import { Coords, MouseButton } from "../schemas/common"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr } from "../util/tool"; import { captureWithCrosshair } from "./screenshot"; +import { interactiveDiff, resolveRefToBox } from "./ref-resolve"; const ClickArgs = Type.Object({ - ...Coords.properties, + ref: Type.Optional( + Type.String({ + description: + "Stable element ref from browser_snapshot (e.g. 'e12'). PREFERRED over x/y — survives re-renders. When set, x/y are ignored.", + }), + ), + x: Type.Optional(Coords.properties.x), + y: Type.Optional(Coords.properties.y), button: Type.Optional(MouseButton), count: Type.Optional( Type.Integer({ @@ -18,20 +26,27 @@ const ClickArgs = Type.Object({ }), ), }); -type ClickArgsT = Static; const dispatchClick = async ( client: BrowserClient, - args: ClickArgsT, + x: number, + y: number, + button: "left" | "right" | "middle", + count: number, ): Promise> => { - const button = args.button ?? "left"; - const count = args.count ?? 1; + // Move the pointer to the target first. Some focus/hover handlers (and React + // synthetic events) only fire when a mousemove precedes the press, so without + // it a click can land without focusing the element. + const moved = await client.session().call("Input.dispatchMouseEvent", { + type: "mouseMoved", x, y, + }); + if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message }); const pressed = await client.session().call("Input.dispatchMouseEvent", { - type: "mousePressed", x: args.x, y: args.y, button, clickCount: count, + type: "mousePressed", x, y, button, clickCount: count, }); if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message }); const released = await client.session().call("Input.dispatchMouseEvent", { - type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count, + type: "mouseReleased", x, y, button, clickCount: count, }); if (!released.success) return err({ kind: "cdp_error", message: released.error.message }); return ok(undefined); @@ -41,34 +56,55 @@ export const clickTool = defineBrowserTool({ name: "browser_click", label: "Browser Click", description: - "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Get coordinates from browser_snapshot's @(x,y) hints — do not screenshot to find click targets.", - promptSnippet: "Click at pixel coordinates (x, y) on the page", + "Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.", + promptSnippet: "Click an element by ref (preferred) or pixel coordinates", promptGuidelines: [ - "Get (x, y) from browser_snapshot — the outline shows '@(x,y)' for every interactive element. NO screenshot needed.", - "If the target isn't an interactive role in the snapshot, fall back to browser_execute_js with `el.getBoundingClientRect()` — still NO screenshot.", - "Coordinates are viewport CSS pixels (not device pixels).", - "After clicking, verify with browser_snapshot (or browser_execute_js for a specific value). Only screenshot if you need to confirm visual rendering.", - "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.", - "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions (debug only).", - "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.", + "PREFER `ref` from browser_snapshot (the outline shows '[eN]' for every interactive element) — it survives re-renders, unlike coordinates which go stale after a save/edit reflows the page.", + "Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.", + "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.", + "After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).", + "Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.", + "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots (debug only). For React/Vue components ignoring clicks, try browser_dispatch_key.", ], parameters: ClickArgs, - renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0), + renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0), async handler(args, { client }) { - const clicked = await dispatchClick(client, args); + const button = args.button ?? "left"; + const count = args.count ?? 1; + + // Resolve the click point: ref re-resolves to the element's CURRENT box + // (so re-renders don't matter); otherwise use the literal x/y. + let x: number; + let y: number; + if (args.ref !== undefined) { + const box = await resolveRefToBox(client, args.ref); + if (!box.success) return box; + x = box.data.cx; + y = box.data.cy; + } else if (args.x !== undefined && args.y !== undefined) { + x = args.x; + y = args.y; + } else { + return err({ kind: "invalid_state", message: "Provide either `ref` or both `x` and `y`." }); + } + + const clicked = await dispatchClick(client, x, y, button, count); if (!clicked.success) return clicked; + + const target = args.ref !== undefined ? `[${args.ref}] (${x}, ${y})` : `(${x}, ${y})`; + const diff = await interactiveDiff(client); if (process.env["BH_DEBUG_CLICKS"]) { - const debug = await captureWithCrosshair(client, { x: args.x, y: args.y }); + const debug = await captureWithCrosshair(client, { x, y }); if (debug.success) { return ok({ - text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`, - details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y }, + text: `Clicked at ${target}\n[DEBUG] Overlay screenshot: ${debug.data.path}${diff}`, + details: { debugScreenshotPath: debug.data.path, x, y, ref: args.ref }, }); } } return ok({ - text: `Clicked at (${args.x}, ${args.y})`, - details: { x: args.x, y: args.y }, + text: `Clicked at ${target}${diff}`, + details: { x, y, ref: args.ref }, }); }, }); diff --git a/src/domains/files.ts b/src/domains/files.ts index 91f8761..d68d236 100644 --- a/src/domains/files.ts +++ b/src/domains/files.ts @@ -6,12 +6,28 @@ import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { safeJs } from "../util/js-template"; import { pdfPath } from "../util/paths"; +import { resolveRefToBackendId } from "./ref-resolve"; const UploadArgs = Type.Object({ - selector: Type.String({ description: "CSS selector of the file " }), + ref: Type.Optional( + Type.String({ description: "Stable ref of the file from browser_snapshot. PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the file (fallback when no ref)" })), filePath: Type.String({ description: "Absolute path to the file to upload" }), }); +// Ref path: set files directly on the resolved backendNodeId via CDP — simplest +// and most reliable, no selector needed. +const tryRefUpload = async ( + client: BrowserClient, + backendId: number, + filePath: string, +): Promise> => { + const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId }); + if (!set.success) return err({ kind: "cdp_error", message: set.error.message }); + return ok(undefined); +}; + const verifyReadable = async (filePath: string): Promise> => { try { await access(filePath, constants.R_OK); @@ -89,16 +105,29 @@ const jsFallbackUpload = async ( export const uploadFileTool = defineBrowserTool({ name: "browser_upload_file", label: "Browser Upload File", - description: "Set files on a file via CDP, with a JS-DataTransfer fallback for stubborn pages.", - promptSnippet: "Upload a file to a file input", + description: "Set files on a file via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.", + promptSnippet: "Upload a file to a file input by ref (preferred) or selector", promptGuidelines: [ + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders. (File inputs are often hidden; snapshot still surfaces them.)", "File path must be absolute and readable.", - "Selector must match a file input.", ], parameters: UploadArgs, async handler(args, { client }): Promise> { const readable = await verifyReadable(args.filePath); if (!readable.success) return readable; + + // Ref path: resolve to backendNodeId and set files directly via CDP. + if (args.ref !== undefined) { + const backendId = resolveRefToBackendId(client, args.ref); + if (!backendId.success) return backendId; + const up = await tryRefUpload(client, backendId.data, args.filePath); + if (!up.success) return up; + return ok({ text: `Uploaded ${args.filePath} via CDP (ref ${args.ref})`, details: { mode: "cdp", ref: args.ref, filePath: args.filePath } }); + } + + if (args.selector === undefined) { + return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); + } const cdp = await tryCdpUpload(client, args.selector, args.filePath); if (cdp.success) { return ok({ diff --git a/src/domains/form.ts b/src/domains/form.ts new file mode 100644 index 0000000..a10646c --- /dev/null +++ b/src/domains/form.ts @@ -0,0 +1,270 @@ +import { Type } from "typebox"; +import type { BrowserClient } from "../client"; +import { safeJs } from "../util/js-template"; +import { type Result, err, ok } from "../util/result"; +import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; +import { interactiveDiff, resolveRefToObjectId } from "./ref-resolve"; + +const FillArgs = Type.Object({ + ref: Type.Optional( + Type.String({ + description: + "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector — survives re-renders. When set, selector is ignored.", + }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the form field to fill (fallback when no ref)" })), + value: Type.String({ description: "Value to set in the field" }), +}); + +/** + * Run an element-scoped function either against a CSS selector (querySelector + + * apply) or a ref (resolve to objectId, callFunctionOn with element as `this`). + * The shared `fnBody` is a function body where `this` is the target element and + * the single argument is the value — identical logic on both paths. Returns the + * function's return value (parsed) or a ToolErr. + * + * On the ref path a detached node yields a stale-ref error; on the selector path + * a missing element returns { status: "not_found" } from the function body. + */ +const runOnElement = async ( + client: BrowserClient, + opts: { ref?: string | undefined; selector?: string | undefined; fnBody: string; arg: unknown }, +): Promise> => { + if (opts.ref !== undefined) { + const objectId = await resolveRefToObjectId(client, opts.ref); + if (!objectId.success) return objectId; + const r = await client.session().call("Runtime.callFunctionOn", { + objectId: objectId.data, + functionDeclaration: `function (arg) { ${opts.fnBody} }`, + arguments: [{ value: opts.arg }], + returnByValue: true, + }); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + return ok((r.data as { result?: { value?: unknown } }).result?.value); + } + if (opts.selector === undefined) { + return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); + } + // Selector path: bind the same trusted function body to the matched element + // via .call(). fnBody is harness-authored (not user input); only the selector + // and value are interpolated, and those go through safeJs. + const prelude = safeJs` + (() => { + const el = document.querySelector(${opts.selector}); + if (!el) return { status: "not_found" }; + const __arg = ${opts.arg}; + return (function (arg) {`; + const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`; + const r = await client.evaluateJs(expr); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + return ok(r.data); +}; + +/** + * Result shape returned by the in-page fill script. Discriminated by `status` + * so the handler can map page-level outcomes to typed ToolErr kinds. + */ +type FillResult = + | { status: "ok"; tag: string; kind: string; value: string } + | { status: "not_found" } + | { status: "not_fillable"; tag: string }; + +// Element-scoped fill logic. `this` is the target element, `arg` is the value. +// Write via the native prototype value setter so React's _valueTracker +// registers the change — assigning el.value directly does NOT, which is why +// controlled inputs revert on re-render. Shared verbatim by the ref path +// (callFunctionOn) and the selector path (querySelector + .call). +const FILL_FN = ` + const el = this; + const tag = el.tagName; + const proto = + el instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype + : el instanceof HTMLInputElement ? HTMLInputElement.prototype + : null; + if (proto) { + const desc = Object.getOwnPropertyDescriptor(proto, "value"); + const setter = desc && desc.set; + if (setter) setter.call(el, arg); + else el.value = arg; + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + return { status: "ok", tag, kind: el.type || tag.toLowerCase(), value: el.value }; + } + if (el.isContentEditable) { + // Rich-text editors (Slack/Notion/ProseMirror) listen for beforeinput/input. + el.focus(); + let done = false; + try { + const sel = window.getSelection(); + if (sel) { sel.removeAllRanges(); const rng = document.createRange(); rng.selectNodeContents(el); sel.addRange(rng); } + done = document.execCommand("insertText", false, arg); + } catch (e) { done = false; } + if (!done) { + el.textContent = arg; + el.dispatchEvent(new InputEvent("input", { bubbles: true })); + } + return { status: "ok", tag, kind: "contenteditable", value: el.textContent || "" }; + } + return { status: "not_fillable", tag }; +`; + +export const fillTool = defineBrowserTool({ + name: "browser_fill", + label: "Browser Fill", + description: + "Fill a form field (input/textarea/contenteditable). PREFERRED: pass `ref` (e.g. 'e7') from browser_snapshot — survives re-renders. Fallback: a CSS `selector`. Writes through the native value setter and fires bubbling 'input'/'change' events, so React/Vue/Angular controlled components and rich-text editors update correctly — unlike browser_type. Returns the field's value after writing, plus a compact diff of page changes.", + promptSnippet: "Fill a form field by ref (preferred) or selector (works with React/Vue controlled inputs)", + promptGuidelines: [ + "PREFER `ref` from browser_snapshot (the '[eN]' handle) over a guessed CSS selector — refs survive re-renders and don't require you to invent a selector.", + "Pass the desired value; no browser_click is needed first.", + "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.", + "Use browser_type instead only for keystroke-sensitive widgets (autocomplete, masked/segmented inputs).", + "The result reports the field's value after writing and an appended page-changes diff — confirm both match what you intended before moving on.", + ], + parameters: FillArgs, + async handler(args, { client }): Promise> { + const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: FILL_FN, arg: args.value }); + if (!r.success) return r; + const res = r.data as FillResult | undefined; + const target = args.ref ?? args.selector ?? ""; + if (res === undefined || res.status === "not_found") { + return err({ + kind: "invalid_state", + message: `No element matched: ${target}`, + details: { ref: args.ref, selector: args.selector }, + }); + } + if (res.status === "not_fillable") { + return err({ + kind: "invalid_state", + message: `Element <${res.tag.toLowerCase()}> is not fillable (not an input, textarea, or contenteditable): ${target}. For from browser_snapshot. PREFERRED over selector." }), + ), + selector: Type.Optional(Type.String({ description: "CSS selector of the ; `arg` is { wv, wl, wi } (null = not provided). +const SELECT_FN = ` + const el = this; + if (el.tagName !== "SELECT") return { status: "not_select", tag: el.tagName }; + const opts = Array.from(el.options); + const wv = arg.wv, wl = arg.wl, wi = arg.wi; + let opt = null; + if (wi !== null) opt = opts[wi] || null; + else if (wv !== null) opt = opts.find(o => o.value === wv) || null; + else if (wl !== null) opt = opts.find(o => o.text === wl) || null; + if (!opt) return { status: "no_match", options: opts.map(o => ({ value: o.value, label: o.text })) }; + el.value = opt.value; + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + return { status: "ok", value: opt.value, label: opt.text }; +`; + +export const selectOptionTool = defineBrowserTool({ + name: "browser_select_option", + label: "Browser Select Option", + description: + "Select an option in a native by ref (preferred) or selector", + promptGuidelines: [ + "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", + "Provide exactly one of value, label, or index to choose the option.", + "For custom (non-native) dropdowns built from divs, click to open then browser_click the option instead.", + "If no option matches, the error details list the available options so you can retry.", + ], + parameters: SelectOptionArgs, + async handler(args, { client }): Promise> { + if (args.value === undefined && args.label === undefined && args.index === undefined) { + return err({ kind: "invalid_state", message: "Provide one of value, label, or index." }); + } + // null sentinels distinguish "not provided" from a real empty-string match. + const arg = { wv: args.value ?? null, wl: args.label ?? null, wi: args.index ?? null }; + const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: SELECT_FN, arg }); + if (!r.success) return r; + const res = r.data as SelectResult | undefined; + const target = args.ref ?? args.selector ?? ""; + if (res === undefined || res.status === "not_found") { + return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector } }); + } + if (res.status === "not_select") { + return err({ kind: "invalid_state", message: `Element <${res.tag.toLowerCase()}> is not a by **ref** (preferred) or selector, choosing by value/label/index (fires change) | +| browser_focus | Focus an element by **ref** (preferred) or selector (deterministic; use before browser_type) | | browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) | | browser_scroll | Scroll the page at coordinates | | browser_navigate | Navigate to a URL (result details.outcome.kind is "in_place" or "new_tab_created") | @@ -38,6 +42,7 @@ to interact with pages visually, and other tools alongside them. | browser_http_get | Direct HTTP GET (outside browser, for APIs) | | browser_console | Read JS errors / console output on the current tab (diagnostic — use when an action looks broken; pass sinceSeq from the previous nextCursor to see only new messages) | | browser_wait | Wait N seconds | +| browser_wait_for | Wait until a selector/text appears (or a selector disappears) — use for SPA content before interacting | | browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) | | browser_handle_dialog | Accept or dismiss a JS dialog | | browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) | @@ -83,11 +88,25 @@ browser_http_get("https://api.example.com/data") + browser_click(x, y) browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot() \`\`\` -**Form filling:** +**Form filling (ref-first — this is the reliable path on React/SPA forms):** \`\`\` -browser_screenshot() → find input coordinates → browser_click(x, y) -→ browser_type("text") → browser_press_key("Tab") → browser_screenshot() +browser_wait_for({ selector: "#email" }) // ensure SPA field is rendered +browser_snapshot() // each interactive element gets a [eN] ref +→ browser_fill({ ref: "e7", value: "a@b.com" }) // target by ref, not a guessed selector +→ browser_select_option({ ref: "e9", label: "India" }) // native