import { Type } from "typebox"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { detailsOf, isPageResult, type PageResult, resolveAndCall, runOnElement } from "./element-call"; import { fillBody } from "./fill-engine"; import { interactiveDiff } from "./ref-resolve"; const FillArgs = Type.Object({ ref: Type.Optional( Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector — survives re-renders. When set, selector is ignored.", }), ), selector: Type.Optional(Type.String({ description: "CSS selector of the form field to fill (fallback when no ref)" })), value: Type.String({ description: "Value to set in the field" }), }); const FILL_BODY = `const value = arg;\n${fillBody({ rejectSelect: true, focusFirst: false })}`; const NOT_FOUND: PageResult = { ok: false, reason: "not_found" }; export const fillTool = defineBrowserTool({ name: "browser_fill", label: "Browser Fill", description: "Fill a form field (input/textarea/contenteditable), or tick a checkbox/radio by passing 'true'/'false'. PREFERRED: pass `ref` (e.g. 'e7') from browser_snapshot — survives re-renders. Fallback: a CSS `selector`. Writes through the native value setter and fires bubbling 'input'/'change' events, so React/Vue/Angular controlled components and rich-text editors update correctly — unlike browser_type. Refuses a disabled field rather than writing a value the page will ignore. Returns the field's value after writing, plus a compact diff of page changes.", promptSnippet: "Fill a form field by ref (preferred) or selector (works with React/Vue controlled inputs)", promptGuidelines: [ "PREFER `ref` from browser_snapshot (the '[eN]' handle) over a guessed CSS selector — refs survive re-renders and don't require you to invent a selector.", "Pass the desired value; no browser_click is needed first.", "For a checkbox or radio pass 'true' or 'false' — it toggles `checked`, not the submit value. browser_set_checked does the same thing with a boolean.", "A disabled field is refused: the error says so, rather than reporting a write the page never accepted.", "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.", "Use browser_type instead only for keystroke-sensitive widgets (autocomplete, masked/segmented inputs).", "Fill sets the value but does not focus or submit. To submit a tag/autocomplete input, follow with browser_dispatch_key({ ref, key: 'Enter' }) — not browser_press_key, which targets the focused element and may miss this field.", "For a custom (div-based) dropdown, browser_fill won't validate against its option list — open it, run browser_snapshot, then browser_click the option's ref.", "The result reports the field's value after writing and an appended page-changes diff — confirm both match what you intended before moving on.", ], parameters: FillArgs, concurrency: "serialized", async handler(args, { client }): Promise> { const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: FILL_BODY, arg: args.value, notFound: NOT_FOUND, }); if (!r.success) return r; const res = isPageResult(r.data) ? r.data : undefined; const target = args.ref ?? args.selector ?? ""; if (res === undefined || res.reason === "not_found") { return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector }, }); } if (!res.ok) { const details = { ref: args.ref, selector: args.selector, ...(res.tag !== undefined ? { tag: res.tag } : {}) }; if (res.kind === "select") { return err({ kind: "invalid_state", message: `Element use browser_select_option.`, details, }); } return err({ kind: "invalid_state", message: `Could not fill ${target}: ${res.reason ?? "unknown reason"}`, details, }); } const shown = res.value !== undefined ? JSON.stringify(res.value) : `checked=${res.checked === true}`; const diff = await interactiveDiff(client); return ok({ text: `Filled ${target} = ${shown}${diff}`, details: { ref: args.ref, selector: args.selector, value: args.value, verified: res.value ?? res.checked, ...(res.tag !== undefined ? { tag: res.tag } : {}), ...(res.kind !== undefined ? { kind: res.kind } : {}), }, }); }, }); const FocusArgs = Type.Object({ ref: Type.Optional( Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector." }), ), selector: Type.Optional(Type.String({ description: "CSS selector of the element to focus (fallback when no ref)" })), }); type FocusResult = { status: "ok"; tag: string } | { status: "not_found" } | { status: "not_focusable"; tag: string }; const isFocusResult = (v: unknown): v is FocusResult => { if (typeof v !== "object" || v === null || !("status" in v)) return false; if (v.status === "not_found") return true; if (v.status === "ok" || v.status === "not_focusable") return "tag" in v && typeof v.tag === "string"; return false; }; const FOCUS_FN = ` const el = this; if (typeof el.focus !== "function") return { status: "not_focusable", tag: el.tagName }; el.focus(); return document.activeElement === el ? { status: "ok", tag: el.tagName } : { status: "not_focusable", tag: el.tagName }; `; export const focusTool = defineBrowserTool({ name: "browser_focus", label: "Browser Focus", description: "Focus an element via the DOM .focus() method — deterministic, no coordinate accuracy needed. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use before browser_type when a click might miss the field.", promptSnippet: "Focus an element by ref (preferred) or selector", promptGuidelines: [ "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", "Use before browser_type to guarantee the right field is focused without relying on click coordinates.", "For simply setting a value, prefer browser_fill (it doesn't need a separate focus step).", ], parameters: FocusArgs, concurrency: "serialized", async handler(args, { client }): Promise> { const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: FOCUS_FN, arg: "" }); if (!r.success) return r; const res = isFocusResult(r.data) ? r.data : undefined; const target = args.ref ?? args.selector ?? ""; if (res === undefined || res.status === "not_found") { return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector } }); } if (res.status === "not_focusable") { return err({ kind: "invalid_state", message: `Element <${res.tag.toLowerCase()}> could not be focused: ${target}`, details: { ref: args.ref, selector: args.selector, tag: res.tag } }); } return ok({ text: `Focused ${target}`, details: { ref: args.ref, selector: args.selector, tag: res.tag } }); }, }); const SelectOptionArgs = Type.Object({ ref: Type.Optional( Type.String({ description: "Stable element ref of the element (fallback when no ref)" })), value: Type.Optional(Type.String({ description: "Option value attribute to select" })), label: Type.Optional(Type.String({ description: "Visible option text to select (exact match)" })), index: Type.Optional(Type.Integer({ minimum: 0, description: "Zero-based option index to select" })), }); type SelectResult = | { status: "ok"; value: string; label: string } | { status: "not_found" } | { status: "not_select"; tag: string } | { status: "no_match"; options: ReadonlyArray<{ value: string; label: string }> }; const isSelectOption = (v: unknown): v is { value: string; label: string } => typeof v === "object" && v !== null && "value" in v && typeof v.value === "string" && "label" in v && typeof v.label === "string"; const isSelectResult = (v: unknown): v is SelectResult => { if (typeof v !== "object" || v === null || !("status" in v)) return false; if (v.status === "not_found") return true; if (v.status === "not_select") return "tag" in v && typeof v.tag === "string"; if (v.status === "ok") { return "value" in v && typeof v.value === "string" && "label" in v && typeof v.label === "string"; } if (v.status === "no_match") { return "options" in v && Array.isArray(v.options) && v.options.every(isSelectOption); } return false; }; const SELECT_FN = ` const el = this; if (el.tagName !== "SELECT") return { status: "not_select", tag: el.tagName }; const opts = Array.from(el.options); const wv = arg.wv, wl = arg.wl, wi = arg.wi; let opt = null; if (wi !== null) opt = opts[wi] || null; else if (wv !== null) opt = opts.find(o => o.value === wv) || null; else if (wl !== null) opt = opts.find(o => o.text === wl) || null; if (!opt) return { status: "no_match", options: opts.map(o => ({ value: o.value, label: o.text })) }; el.value = opt.value; el.dispatchEvent(new Event("input", { bubbles: true })); el.dispatchEvent(new Event("change", { bubbles: true })); return { status: "ok", value: opt.value, label: opt.text }; `; export const selectOptionTool = defineBrowserTool({ name: "browser_select_option", label: "Browser Select Option", description: "Select an option in a native by ref (preferred) or selector", promptGuidelines: [ "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", "Provide exactly one of value, label, or index to choose the option.", "For custom (non-native) dropdowns built from divs, click to open then browser_click the option instead.", "If no option matches, the error details list the available options so you can retry.", ], parameters: SelectOptionArgs, concurrency: "serialized", async handler(args, { client }): Promise> { if (args.value === undefined && args.label === undefined && args.index === undefined) { return err({ kind: "invalid_state", message: "Provide one of value, label, or index." }); } // null sentinels distinguish "not provided" from a real empty-string match. const arg = { wv: args.value ?? null, wl: args.label ?? null, wi: args.index ?? null }; const r = await runOnElement(client, { ref: args.ref, selector: args.selector, fnBody: SELECT_FN, arg }); if (!r.success) return r; const res = isSelectResult(r.data) ? r.data : undefined; const target = args.ref ?? args.selector ?? ""; if (res === undefined || res.status === "not_found") { return err({ kind: "invalid_state", message: `No element matched: ${target}`, details: { ref: args.ref, selector: args.selector } }); } if (res.status === "not_select") { return err({ kind: "invalid_state", message: `Element <${res.tag.toLowerCase()}> is not a