1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
|
import { Type } from "typebox";
import type { BrowserClient } from "../client";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
// ── Page-side functions ──────────────────────────────────────────────────────
// These run inside the page via Runtime.callFunctionOn with `this` bound to the
// resolved element. They are NOT string-interpolated with user data — the value
// is passed as a structured `arguments` entry, so safeJs is unnecessary here.
/**
* Universal field setter. Auto-detects the element type and applies the right
* strategy, then reads the resulting value back so the caller can confirm.
*
* The <input>/<textarea> branch uses the "native setter" trick: React (and
* other frameworks) overload the element's `value` setter to track state, so a
* plain `el.value = x` updates the DOM but not the framework. Calling the
* prototype's original setter and then dispatching a bubbling `input` event is
* what makes React's onChange fire.
*/
const FILL_FN = `
function(value) {
const el = this;
if (!el || el.nodeType !== 1) return { ok: false, reason: "ref does not point to an element" };
if (el.disabled) return { ok: false, reason: "element is disabled" };
const tag = (el.tagName || "").toLowerCase();
const type = (el.type || "").toLowerCase();
try { el.focus(); } catch (e) {}
const fire = function(t) { el.dispatchEvent(new Event(t, { bubbles: true })); };
if (tag === "select") {
const want = String(value);
let matched = null;
for (let i = 0; i < el.options.length; i++) {
const o = el.options[i];
if (o.value === want || o.label === want || o.text === want) { matched = o; break; }
}
if (!matched) {
const opts = [];
for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text });
return { ok: false, reason: "no matching option", kind: "select", options: opts };
}
el.value = matched.value;
fire("input"); fire("change");
return { ok: true, kind: "select", value: el.value, text: matched.text };
}
if (tag === "input" && (type === "checkbox" || type === "radio")) {
const want = value === true || value === "true" || value === "on" || value === 1;
if (el.checked !== want) {
try { el.click(); } catch (e) {}
if (el.checked !== want) { el.checked = want; fire("input"); fire("change"); }
}
return { ok: true, kind: type, checked: el.checked };
}
if (el.isContentEditable) {
el.textContent = String(value);
fire("input"); fire("change");
return { ok: true, kind: "contenteditable", value: el.textContent };
}
if (tag === "input" || tag === "textarea") {
const proto = tag === "textarea" ? window.HTMLTextAreaElement.prototype : window.HTMLInputElement.prototype;
const desc = Object.getOwnPropertyDescriptor(proto, "value");
if (desc && desc.set) desc.set.call(el, String(value));
else el.value = String(value);
fire("input"); fire("change");
return { ok: true, kind: tag, value: el.value };
}
return { ok: false, reason: "element is not a fillable field (tag=" + tag + ")" };
}`;
/** Specialized <select> setter — matches by value first, then by label/text. */
const SELECT_FN = `
function(value, label) {
const el = this;
if (!el || (el.tagName || "").toLowerCase() !== "select") return { ok: false, reason: "ref is not a <select> element" };
if (el.disabled) return { ok: false, reason: "select is disabled" };
let matched = null;
for (let i = 0; i < el.options.length; i++) {
const o = el.options[i];
if (value != null && o.value === String(value)) { matched = o; break; }
if (label != null && (o.text === String(label) || o.label === String(label))) { matched = o; break; }
}
if (!matched) {
const opts = [];
for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text });
return { ok: false, reason: "no matching option", options: opts };
}
try { el.focus(); } catch (e) {}
el.value = matched.value;
el.dispatchEvent(new Event("input", { bubbles: true }));
el.dispatchEvent(new Event("change", { bubbles: true }));
return { ok: true, value: el.value, text: matched.text };
}`;
/** Specialized checkbox/radio setter — only acts when the state differs. */
const CHECK_FN = `
function(checked) {
const el = this;
const tag = (el.tagName || "").toLowerCase();
const type = (el.type || "").toLowerCase();
if (tag !== "input" || (type !== "checkbox" && type !== "radio")) return { ok: false, reason: "ref is not a checkbox or radio" };
if (el.disabled) return { ok: false, reason: "element is disabled" };
const want = checked === true;
let changed = false;
if (el.checked !== want) {
try { el.click(); } catch (e) {}
if (el.checked !== want) { el.checked = want; el.dispatchEvent(new Event("change", { bubbles: true })); }
changed = true;
}
return { ok: true, checked: el.checked, changed: changed };
}`;
// ── Resolver ─────────────────────────────────────────────────────────────────
/** Shape returned by the page-side functions above. */
type PageResult = {
readonly ok: boolean;
readonly reason?: string;
readonly kind?: string;
readonly value?: unknown;
readonly text?: string;
readonly checked?: boolean;
readonly changed?: boolean;
readonly options?: ReadonlyArray<{ value: string; text: string }>;
};
/**
* Resolve a ref (CDP backendDOMNodeId) to a live JS handle and invoke a
* page-side function on it. DOM + Runtime are enabled on every session
* (see cdp/session.ts enableDomains), so no extra setup is required.
*/
const resolveAndCall = async (
client: BrowserClient,
ref: number,
functionDeclaration: string,
args: ReadonlyArray<unknown>,
): Promise<Result<PageResult, ToolErr>> => {
const session = client.session();
const resolved = await session.call("DOM.resolveNode", { backendNodeId: ref });
if (!resolved.success) {
return err({
kind: "invalid_state",
message: `ref ${ref} could not be resolved — the page may have changed. Re-run browser_snapshot to get fresh refs.`,
details: { ref },
});
}
const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId;
if (objectId === undefined) {
return err({
kind: "invalid_state",
message: `ref ${ref} resolved to no JS object — re-run browser_snapshot to get fresh refs.`,
details: { ref },
});
}
try {
const called = await session.call("Runtime.callFunctionOn", {
objectId,
functionDeclaration,
arguments: args.map((v) => ({ value: v })),
returnByValue: true,
awaitPromise: true,
});
if (!called.success) return err({ kind: "cdp_error", message: called.error.message, details: { ref } });
const data = called.data as { result?: { value?: unknown }; exceptionDetails?: unknown };
if (data.exceptionDetails !== undefined) {
return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(data.exceptionDetails)}`, details: { ref } });
}
const value = data.result?.value;
if (typeof value !== "object" || value === null) {
return err({ kind: "internal", message: "page function returned no result object", details: { ref } });
}
return ok(value as PageResult);
} finally {
// Best-effort: release the remote handle so it can be GC'd. Failure is
// harmless (the page may have navigated), so we ignore the result.
await session.call("Runtime.releaseObject", { objectId });
}
};
/** Drop the `ok` flag before stashing a page result in tool `details` — the
* tool-result envelope reserves `ok` for its own success discriminant. */
const detailsOf = (ref: number, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => {
const { ok: _ok, ...rest } = res;
return { ref, ...rest, ...(extra ?? {}) };
};
// ── Tools ────────────────────────────────────────────────────────────────────
const FieldValue = Type.Union([Type.String(), Type.Boolean()], {
description: "Value to set. String for text inputs / textareas / selects / contenteditable; boolean for checkboxes & radios.",
});
const FillArgs = Type.Object({
ref: Type.Integer({ description: "Element ref from browser_snapshot ([ref=N])." }),
value: FieldValue,
});
export const fillTool = defineBrowserTool({
name: "browser_fill",
label: "Browser Fill",
description:
"Fill a single form field by ref. Auto-detects input/textarea/select/checkbox/radio/contenteditable, clears then sets the value, fires input+change events (drives React/Vue controlled inputs correctly), and reads the value back to confirm. Get the ref from browser_snapshot's [ref=N].",
promptSnippet: "Fill a form field by ref (text, select, checkbox) — React-safe, self-confirming",
promptGuidelines: [
"Get the ref from browser_snapshot's [ref=N] for the target field. No click-to-focus needed.",
"value is a string for text fields/selects/contenteditable, or a boolean for checkboxes/radios.",
"The result reports the value the field actually holds after filling — check it matches what you intended.",
"For autocomplete/typeahead that needs real keystrokes, click the field's @(x,y) then use browser_type instead.",
"If a ref no longer resolves, re-run browser_snapshot to get fresh refs.",
],
parameters: FillArgs,
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
const r = await resolveAndCall(client, args.ref, FILL_FN, [args.value]);
if (!r.success) return r;
const res = r.data;
if (!res.ok) {
return err({ kind: "invalid_state", message: `Could not fill ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
}
const wanted = typeof args.value === "boolean" ? undefined : String(args.value);
const got = res.value !== undefined ? String(res.value) : undefined;
const mismatch = wanted !== undefined && got !== undefined && got !== wanted;
let text: string;
if (mismatch) {
text = `Filled ref ${args.ref}, but it now reads ${JSON.stringify(got)} (wanted ${JSON.stringify(wanted)})`;
} else if (got !== undefined) {
text = `Filled ref ${args.ref} = ${JSON.stringify(got)}`;
} else if (res.checked !== undefined) {
text = `Filled ref ${args.ref} (checked=${res.checked})`;
} else {
text = `Filled ref ${args.ref}`;
}
return ok({ text, details: detailsOf(args.ref, res, { mismatch }) });
},
});
const FillFormArgs = Type.Object({
fields: Type.Array(
Type.Object({
ref: Type.Integer({ description: "Element ref from browser_snapshot ([ref=N])." }),
value: FieldValue,
}),
{ minItems: 1, description: "Fields to fill in one batch, each by ref." },
),
});
export const fillFormTool = defineBrowserTool({
name: "browser_fill_form",
label: "Browser Fill Form",
description:
"Fill multiple form fields in one call — the efficient way to complete a form. Each field is { ref, value }; refs come from browser_snapshot's [ref=N]. Handles text inputs, textareas, selects, checkboxes, radios, and contenteditable, drives React/Vue controlled inputs correctly, and reports the resulting value of every field.",
promptSnippet: "Fill many form fields at once by ref — React-safe, self-confirming",
promptGuidelines: [
"Preferred for forms: snapshot once, then fill all fields in a single browser_fill_form call.",
"Each field: { ref, value }. value is a string for text/select/contenteditable, boolean for checkbox/radio.",
"The result summarizes how many fields filled cleanly and flags any value mismatches or per-field errors.",
"Re-run browser_snapshot afterwards to verify the form state if needed.",
],
parameters: FillFormArgs,
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
const results: Array<Readonly<Record<string, unknown>>> = [];
let okCount = 0;
const issues: string[] = [];
for (const f of args.fields) {
const r = await resolveAndCall(client, f.ref, FILL_FN, [f.value]);
if (!r.success) {
results.push({ ref: f.ref, ok: false, error: r.error.message });
issues.push(`ref ${f.ref} (${r.error.message})`);
continue;
}
const res = r.data;
const wanted = typeof f.value === "boolean" ? undefined : String(f.value);
const got = res.value !== undefined ? String(res.value) : undefined;
const mismatch = wanted !== undefined && got !== undefined && got !== wanted;
const fieldOk = res.ok === true && !mismatch;
if (fieldOk) okCount++;
else issues.push(`ref ${f.ref} (${res.ok === false ? (res.reason ?? "failed") : "value mismatch"})`);
results.push({ ref: f.ref, ok: fieldOk, mismatch, ...detailsOf(f.ref, res) });
}
const text =
`Filled ${okCount}/${args.fields.length} fields` +
(issues.length > 0 ? `; issues: ${issues.join(", ")}` : "");
return ok({ text, details: { results } });
},
});
const SelectOptionArgs = Type.Object({
ref: Type.Integer({ description: "Element ref of a <select> from browser_snapshot ([ref=N])." }),
value: Type.Optional(Type.String({ description: "Option value attribute to select." })),
label: Type.Optional(Type.String({ description: "Visible option text/label to select (used if value is omitted)." })),
});
export const selectOptionTool = defineBrowserTool({
name: "browser_select_option",
label: "Browser Select Option",
description:
"Select an option in a native <select> dropdown by ref. Matches by option value, then by visible label/text, and fires change. Native selects cannot be driven by clicking coordinates — use this instead. Get the ref from browser_snapshot's [ref=N].",
promptSnippet: "Choose an option in a <select> dropdown by ref",
promptGuidelines: [
"Provide value (the option's value attribute) or label (the visible text). value wins if both given.",
"On no match, the result lists the available options so you can retry with a valid one.",
"Use this for native <select> elements — clicking coordinates does not work for OS dropdowns.",
],
parameters: SelectOptionArgs,
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
if (args.value === undefined && args.label === undefined) {
return err({ kind: "invalid_state", message: "Provide value or label to select an option." });
}
const r = await resolveAndCall(client, args.ref, SELECT_FN, [args.value ?? null, args.label ?? null]);
if (!r.success) return r;
const res = r.data;
if (!res.ok) {
return err({ kind: "invalid_state", message: `Could not select option on ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
}
return ok({
text: `Selected ${JSON.stringify(res.text)} (value=${JSON.stringify(res.value)}) on ref ${args.ref}`,
details: detailsOf(args.ref, res),
});
},
});
const SetCheckedArgs = Type.Object({
ref: Type.Integer({ description: "Element ref of a checkbox or radio from browser_snapshot ([ref=N])." }),
checked: Type.Boolean({ description: "Desired checked state." }),
});
export const setCheckedTool = defineBrowserTool({
name: "browser_set_checked",
label: "Browser Set Checked",
description:
"Set a checkbox or radio to a desired checked state by ref. Only acts if the current state differs, fires change, and reports the final state. Get the ref from browser_snapshot's [ref=N].",
promptSnippet: "Set a checkbox/radio checked state by ref",
promptGuidelines: [
"checked:true ticks the box, checked:false unticks it. Idempotent — no-op if already in the desired state.",
"For radios, set checked:true on the option you want; the group deselects the others.",
],
parameters: SetCheckedArgs,
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
const r = await resolveAndCall(client, args.ref, CHECK_FN, [args.checked]);
if (!r.success) return r;
const res = r.data;
if (!res.ok) {
return err({ kind: "invalid_state", message: `Could not set checked on ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
}
return ok({
text: `Set ref ${args.ref} checked=${res.checked}` + (res.changed === false ? " (already)" : ""),
details: detailsOf(args.ref, res),
});
},
});
|