aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAman Kumar <145339995+amankumarsingh77@users.noreply.github.com>2026-07-22 00:59:58 +0530
committerGitHub <noreply@github.com>2026-07-22 00:59:58 +0530
commitfe00d3361dece3d35f788d3eeb1c750eedbf6294 (patch)
treea51b8ffe3cddb1aa8d457784f71cc1b2b0b61d56
parent7de0beaf2cf713afc50c573e941fa279ec6df9d9 (diff)
feat: add forms domain and durable window binding (v0.9.0) (#14)
- Add forms domain with a universal field setter that auto-detects element type and fires framework-compatible input events (React native-setter trick) - Add window-binding helper to capture and bind the real Chrome windowId as the session's durable window identity on every new-window code path - Wire supporting changes across CDP session/ownership, client, navigate, isolated-tab, prompt, and state - Add manual tests for forms and window binding - Bump version 0.8.3 -> 0.9.0 Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
-rw-r--r--.pi/settings.json6
-rw-r--r--package-lock.json4
-rw-r--r--package.json2
-rw-r--r--src/cdp/ownership.ts20
-rw-r--r--src/cdp/session.ts55
-rw-r--r--src/cdp/window-binding.ts15
-rw-r--r--src/client.ts61
-rw-r--r--src/domains/forms.ts355
-rw-r--r--src/domains/isolated-tab.ts6
-rw-r--r--src/domains/navigate.ts35
-rw-r--r--src/index.ts17
-rw-r--r--src/prompt.ts208
-rw-r--r--src/state.ts1
-rw-r--r--test/manual/forms-test.ts149
-rw-r--r--test/manual/window-binding-test.ts239
15 files changed, 1027 insertions, 146 deletions
diff --git a/.pi/settings.json b/.pi/settings.json
index 59c7b93..d2f3b27 100644
--- a/.pi/settings.json
+++ b/.pi/settings.json
@@ -1,4 +1,4 @@
{
- "extensions": ["../src/index.ts"],
- "skills": ["../skills"]
-}
+ "extensions": ["./src/index.ts"],
+ "skills":["./skills"]
+} \ No newline at end of file
diff --git a/package-lock.json b/package-lock.json
index 6542fd6..25e866c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "pi-browser-harness",
- "version": "0.8.3",
+ "version": "0.9.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "pi-browser-harness",
- "version": "0.8.3",
+ "version": "0.9.0",
"license": "MIT",
"dependencies": {
"ws": "^8.20.0"
diff --git a/package.json b/package.json
index 1efe449..780db4f 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "pi-browser-harness",
- "version": "0.8.3",
+ "version": "0.9.0",
"description": "Browser control extension for pi — navigate, click, type, screenshot, and extract data from real Chrome via CDP",
"keywords": [
"pi-package",
diff --git a/src/cdp/ownership.ts b/src/cdp/ownership.ts
index f221d6a..727b3d8 100644
--- a/src/cdp/ownership.ts
+++ b/src/cdp/ownership.ts
@@ -20,14 +20,24 @@ export type OwnershipRegistry = {
replaceAll(ids: ReadonlyArray<string>): void;
setHarnessWindow(targetId: string | undefined): void;
harnessWindow(): string | undefined;
+ /** The real Chrome windowId of the dedicated harness window — the durable
+ * identity of "the window this session initialized", surviving seed-tab
+ * closure. Undefined until the window is created (or after teardown). */
+ setHarnessWindowId(windowId: number | undefined): void;
+ harnessWindowId(): number | undefined;
onChange(cb: () => void): void;
};
export const createOwnershipRegistry = (
- initial?: { readonly ownedTargetIds?: ReadonlyArray<string>; readonly harnessWindowTargetId?: string },
+ initial?: {
+ readonly ownedTargetIds?: ReadonlyArray<string>;
+ readonly harnessWindowTargetId?: string;
+ readonly harnessWindowId?: number;
+ },
): OwnershipRegistry => {
const owned = new Set<string>(initial?.ownedTargetIds ?? []);
let harnessWindow: string | undefined = initial?.harnessWindowTargetId;
+ let harnessWindowId: number | undefined = initial?.harnessWindowId;
let listener: (() => void) | null = null;
const notify = (): void => {
@@ -65,6 +75,14 @@ export const createOwnershipRegistry = (
harnessWindow() {
return harnessWindow;
},
+ setHarnessWindowId(windowId) {
+ if (harnessWindowId === windowId) return;
+ harnessWindowId = windowId;
+ notify();
+ },
+ harnessWindowId() {
+ return harnessWindowId;
+ },
onChange(cb) {
listener = cb;
},
diff --git a/src/cdp/session.ts b/src/cdp/session.ts
index e090858..03410ba 100644
--- a/src/cdp/session.ts
+++ b/src/cdp/session.ts
@@ -92,6 +92,17 @@ export const createCdpSession = (
const tab = targetId ? tabs.get(targetId) : undefined;
if (tab) tab.pageInfoDirty = true;
}
+ if (ev.method === "Target.targetCreated" && ownership) {
+ // Adopt tabs opened BY a tab we already own (window.open, target=_blank,
+ // popups). Chrome sets openerId to the opener target; if that opener is
+ // owned, the child belongs to this session's window and must be
+ // controllable + cleaned up. Targets the user opens themselves have no
+ // owned opener, so they are never adopted.
+ const info = (ev.params as { targetInfo?: { targetId?: string; type?: string; openerId?: string } } | undefined)?.targetInfo;
+ if (info?.type === "page" && info.targetId && info.openerId && ownership.has(info.openerId)) {
+ ownership.add(info.targetId);
+ }
+ }
if (ev.method === "Target.targetDestroyed" && ownership) {
const params = ev.params as { targetId?: string } | undefined;
if (params?.targetId) {
@@ -179,6 +190,43 @@ export const createCdpSession = (
if (survivors.length !== ownership.list().length) ownership.replaceAll(survivors);
const hw = ownership.harnessWindow();
if (hw && !live.has(hw)) ownership.setHarnessWindow(undefined);
+
+ if (survivors.length === 0) {
+ // No owned tab survived — the harness window is gone (user closed it,
+ // or Chrome restarted and reassigned all ids). The persisted windowId
+ // is now meaningless and could even collide with one of the user's
+ // windows, so drop it. attachFirstPage will mint a fresh window below.
+ ownership.setHarnessWindowId(undefined);
+ } else {
+ // Self-heal ownership from the window itself. We anchor on a tab we
+ // KNOW we own (a survivor) and re-adopt only its live window-mates —
+ // popups/children from last session, tabs reopened in our window.
+ // Ownership is thus DERIVED from the real window, never from the bare
+ // persisted integer (which Chrome may reassign to a user window on
+ // restart — anchoring on a survivor makes that misfire impossible).
+ const anchorId = survivors[0]!;
+ const anchorWin = await transport.request("Browser.getWindowForTarget", { targetId: anchorId }, { sessionId: null });
+ if (anchorWin.success) {
+ const anchorWindowId = (anchorWin.data as { windowId?: number }).windowId;
+ if (typeof anchorWindowId === "number") {
+ ownership.setHarnessWindowId(anchorWindowId); // refresh to the live id
+ const windows = await Promise.all(
+ allPages.map((p) =>
+ p.targetId === anchorId
+ ? Promise.resolve(anchorWin)
+ : transport.request("Browser.getWindowForTarget", { targetId: p.targetId }, { sessionId: null }),
+ ),
+ );
+ allPages.forEach((p, i) => {
+ const r = windows[i];
+ if (r?.success && (r.data as { windowId?: number }).windowId === anchorWindowId) {
+ ownership.add(p.targetId);
+ }
+ });
+ }
+ }
+ if (!ownership.harnessWindow()) ownership.setHarnessWindow(anchorId);
+ }
}
// Prune per-tab state for pages that no longer exist
const liveTargetIds = new Set(allPages.map((p) => p.targetId));
@@ -208,6 +256,13 @@ export const createCdpSession = (
if (ownership) {
ownership.setHarnessWindow(c.targetId);
ownership.add(c.targetId);
+ // Capture the real Chrome windowId — the durable identity of the
+ // window this session initialized (survives seed-tab closure).
+ const win = await transport.request("Browser.getWindowForTarget", { targetId: c.targetId }, { sessionId: null });
+ if (win.success) {
+ const wid = (win.data as { windowId?: number }).windowId;
+ if (typeof wid === "number") ownership.setHarnessWindowId(wid);
+ }
}
}
diff --git a/src/cdp/window-binding.ts b/src/cdp/window-binding.ts
new file mode 100644
index 0000000..4ae1ac3
--- /dev/null
+++ b/src/cdp/window-binding.ts
@@ -0,0 +1,15 @@
+import type { BrowserClient } from "../client";
+
+/**
+ * Capture the real Chrome windowId of a freshly-created harness window and bind
+ * it as the session's durable window identity. Best-effort: a failed CDP call
+ * leaves the previous binding untouched. Call this on every code path that
+ * spawns a NEW dedicated window (newWindow:true), so the windowId never points
+ * at a window that has since been replaced.
+ */
+export const bindHarnessWindowId = async (client: BrowserClient, targetId: string): Promise<void> => {
+ const r = await client.session().callBrowser("Browser.getWindowForTarget", { targetId });
+ if (!r.success) return;
+ const wid = (r.data as { windowId?: number }).windowId;
+ if (typeof wid === "number") client.ownership().setHarnessWindowId(wid);
+};
diff --git a/src/client.ts b/src/client.ts
index 724a711..9a97369 100644
--- a/src/client.ts
+++ b/src/client.ts
@@ -16,10 +16,12 @@ export type BrowserClientOptions = {
readonly initialOwnership?: {
readonly ownedTargetIds?: ReadonlyArray<string>;
readonly harnessWindowTargetId?: string;
+ readonly harnessWindowId?: number;
};
readonly onOwnershipChange?: (snapshot: {
readonly ownedTargetIds: ReadonlyArray<string>;
readonly harnessWindowTargetId: string | undefined;
+ readonly harnessWindowId: number | undefined;
}) => void;
};
@@ -32,6 +34,9 @@ export type BrowserClient = {
/** Detach from the current page target (removes the "Chrome is being controlled" banner)
* while keeping the transport connection alive. Call on session shutdown. */
detach(): Promise<void>;
+ /** Close every tab this session owns and clear the window binding. Best-effort —
+ * used on session shutdown so no stale harness tabs survive. */
+ closeOwnedTabs(): Promise<void>;
evaluateJs(expression: string, sessionId?: string): Promise<Result<unknown, CdpError>>;
pageInfo(): Promise<Result<PageInfo | { readonly dialog: DialogInfo }, CdpError>>;
takeDialog(): DialogInfo | null;
@@ -82,13 +87,16 @@ const parsePageInfoPayload = (v: unknown): Result<PageInfo, CdpError> => {
export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient => {
const transport = opts.transport ?? createCdpTransport();
- const ownershipInit: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string } = {};
+ const ownershipInit: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string; harnessWindowId?: number } = {};
if (opts.initialOwnership?.ownedTargetIds !== undefined) {
ownershipInit.ownedTargetIds = opts.initialOwnership.ownedTargetIds;
}
if (opts.initialOwnership?.harnessWindowTargetId !== undefined) {
ownershipInit.harnessWindowTargetId = opts.initialOwnership.harnessWindowTargetId;
}
+ if (opts.initialOwnership?.harnessWindowId !== undefined) {
+ ownershipInit.harnessWindowId = opts.initialOwnership.harnessWindowId;
+ }
const ownership = createOwnershipRegistry(ownershipInit);
if (opts.onOwnershipChange) {
const cb = opts.onOwnershipChange;
@@ -96,6 +104,7 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
cb({
ownedTargetIds: ownership.list(),
harnessWindowTargetId: ownership.harnessWindow(),
+ harnessWindowId: ownership.harnessWindowId(),
});
});
}
@@ -256,6 +265,15 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
return ok(undefined);
};
+ // Ask Chrome for the real windowId hosting a target. Undefined on failure —
+ // callers treat the window binding as best-effort.
+ const getWindowId = async (targetId: string): Promise<number | undefined> => {
+ const r = await session.callBrowser("Browser.getWindowForTarget", { targetId });
+ if (!r.success) return undefined;
+ const wid = (r.data as { windowId?: number }).windowId;
+ return typeof wid === "number" ? wid : undefined;
+ };
+
const newTab = async (url?: string): Promise<Result<string, CdpError>> => {
// Verify the recorded harness window still exists. Querying the live
// target list also gives listTabs's reconciliation a chance to run.
@@ -263,22 +281,35 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
if (!tabsResult.success) return tabsResult;
const live = new Set(tabsResult.data.map((t) => t.targetId));
const hw = ownership.harnessWindow();
+ // Pick an opener that keeps us inside the harness window. Prefer the seed
+ // tab; if the user closed it but other owned tabs survive, reuse one of
+ // those instead of spawning a second window (avoids window scatter).
+ const opener = hw && live.has(hw)
+ ? hw
+ : ownership.list().find((id) => live.has(id));
const params: Record<string, unknown> = { url: "about:blank" };
- if (hw && live.has(hw)) {
- // Open as a child of the harness window's seed tab — Chrome places
- // it in the same window, giving us visual grouping for free.
- params["openerId"] = hw;
+ if (opener) {
+ // Open as a child of an owned tab — Chrome places it in the same window,
+ // giving us visual grouping for free and keeping the session single-window.
+ params["openerId"] = opener;
} else {
- // Either fresh session or the harness window was closed by the user.
- // Spawn a new dedicated window.
+ // Fresh session, or the whole harness window was closed by the user.
+ // Spawn a new dedicated window and rebind to it.
params["newWindow"] = true;
}
const created = await session.callBrowser("Target.createTarget", params);
if (!created.success) return created;
const c = created.data as { targetId: string };
- if (params["newWindow"]) ownership.setHarnessWindow(c.targetId);
+ if (params["newWindow"]) {
+ ownership.setHarnessWindow(c.targetId);
+ const wid = await getWindowId(c.targetId);
+ if (wid !== undefined) ownership.setHarnessWindowId(wid);
+ } else if (opener && opener !== hw) {
+ // Re-seeded onto a surviving owned tab; keep the seed pointer live.
+ ownership.setHarnessWindow(opener);
+ }
ownership.add(c.targetId);
const switched = await switchTab(c.targetId);
@@ -297,6 +328,18 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
return ok(undefined);
};
+ const closeOwnedTabs = async (): Promise<void> => {
+ // Snapshot first — remove() mutates the list, and targetDestroyed events
+ // may also prune concurrently. Failures are swallowed: a tab the user
+ // already closed, or a dead transport, must not block shutdown.
+ for (const id of ownership.list()) {
+ await session.callBrowser("Target.closeTarget", { targetId: id }).catch(() => {});
+ ownership.remove(id);
+ }
+ ownership.setHarnessWindow(undefined);
+ ownership.setHarnessWindowId(undefined);
+ };
+
const status = (): DaemonStatus => ({
alive: transport.state() === "open" && session.current() !== null,
sessionId: session.current()?.sessionId ?? null,
@@ -316,7 +359,7 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient =
return {
namespace: opts.namespace,
- ensureAlive, status, start, stop, detach,
+ ensureAlive, status, start, stop, detach, closeOwnedTabs,
evaluateJs, pageInfo,
takeDialog: () => session.takeDialog(),
listTabs, switchTab, newTab, closeTab,
diff --git a/src/domains/forms.ts b/src/domains/forms.ts
new file mode 100644
index 0000000..1d83ba6
--- /dev/null
+++ b/src/domains/forms.ts
@@ -0,0 +1,355 @@
+import { Type } from "typebox";
+import type { BrowserClient } from "../client";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+
+// ── Page-side functions ──────────────────────────────────────────────────────
+// These run inside the page via Runtime.callFunctionOn with `this` bound to the
+// resolved element. They are NOT string-interpolated with user data — the value
+// is passed as a structured `arguments` entry, so safeJs is unnecessary here.
+
+/**
+ * Universal field setter. Auto-detects the element type and applies the right
+ * strategy, then reads the resulting value back so the caller can confirm.
+ *
+ * The <input>/<textarea> branch uses the "native setter" trick: React (and
+ * other frameworks) overload the element's `value` setter to track state, so a
+ * plain `el.value = x` updates the DOM but not the framework. Calling the
+ * prototype's original setter and then dispatching a bubbling `input` event is
+ * what makes React's onChange fire.
+ */
+const FILL_FN = `
+function(value) {
+ const el = this;
+ if (!el || el.nodeType !== 1) return { ok: false, reason: "ref does not point to an element" };
+ if (el.disabled) return { ok: false, reason: "element is disabled" };
+ const tag = (el.tagName || "").toLowerCase();
+ const type = (el.type || "").toLowerCase();
+ try { el.focus(); } catch (e) {}
+ const fire = function(t) { el.dispatchEvent(new Event(t, { bubbles: true })); };
+
+ if (tag === "select") {
+ const want = String(value);
+ let matched = null;
+ for (let i = 0; i < el.options.length; i++) {
+ const o = el.options[i];
+ if (o.value === want || o.label === want || o.text === want) { matched = o; break; }
+ }
+ if (!matched) {
+ const opts = [];
+ for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text });
+ return { ok: false, reason: "no matching option", kind: "select", options: opts };
+ }
+ el.value = matched.value;
+ fire("input"); fire("change");
+ return { ok: true, kind: "select", value: el.value, text: matched.text };
+ }
+
+ if (tag === "input" && (type === "checkbox" || type === "radio")) {
+ const want = value === true || value === "true" || value === "on" || value === 1;
+ if (el.checked !== want) {
+ try { el.click(); } catch (e) {}
+ if (el.checked !== want) { el.checked = want; fire("input"); fire("change"); }
+ }
+ return { ok: true, kind: type, checked: el.checked };
+ }
+
+ if (el.isContentEditable) {
+ el.textContent = String(value);
+ fire("input"); fire("change");
+ return { ok: true, kind: "contenteditable", value: el.textContent };
+ }
+
+ if (tag === "input" || tag === "textarea") {
+ const proto = tag === "textarea" ? window.HTMLTextAreaElement.prototype : window.HTMLInputElement.prototype;
+ const desc = Object.getOwnPropertyDescriptor(proto, "value");
+ if (desc && desc.set) desc.set.call(el, String(value));
+ else el.value = String(value);
+ fire("input"); fire("change");
+ return { ok: true, kind: tag, value: el.value };
+ }
+
+ return { ok: false, reason: "element is not a fillable field (tag=" + tag + ")" };
+}`;
+
+/** Specialized <select> setter — matches by value first, then by label/text. */
+const SELECT_FN = `
+function(value, label) {
+ const el = this;
+ if (!el || (el.tagName || "").toLowerCase() !== "select") return { ok: false, reason: "ref is not a <select> element" };
+ if (el.disabled) return { ok: false, reason: "select is disabled" };
+ let matched = null;
+ for (let i = 0; i < el.options.length; i++) {
+ const o = el.options[i];
+ if (value != null && o.value === String(value)) { matched = o; break; }
+ if (label != null && (o.text === String(label) || o.label === String(label))) { matched = o; break; }
+ }
+ if (!matched) {
+ const opts = [];
+ for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text });
+ return { ok: false, reason: "no matching option", options: opts };
+ }
+ try { el.focus(); } catch (e) {}
+ el.value = matched.value;
+ el.dispatchEvent(new Event("input", { bubbles: true }));
+ el.dispatchEvent(new Event("change", { bubbles: true }));
+ return { ok: true, value: el.value, text: matched.text };
+}`;
+
+/** Specialized checkbox/radio setter — only acts when the state differs. */
+const CHECK_FN = `
+function(checked) {
+ const el = this;
+ const tag = (el.tagName || "").toLowerCase();
+ const type = (el.type || "").toLowerCase();
+ if (tag !== "input" || (type !== "checkbox" && type !== "radio")) return { ok: false, reason: "ref is not a checkbox or radio" };
+ if (el.disabled) return { ok: false, reason: "element is disabled" };
+ const want = checked === true;
+ let changed = false;
+ if (el.checked !== want) {
+ try { el.click(); } catch (e) {}
+ if (el.checked !== want) { el.checked = want; el.dispatchEvent(new Event("change", { bubbles: true })); }
+ changed = true;
+ }
+ return { ok: true, checked: el.checked, changed: changed };
+}`;
+
+// ── Resolver ─────────────────────────────────────────────────────────────────
+
+/** Shape returned by the page-side functions above. */
+type PageResult = {
+ readonly ok: boolean;
+ readonly reason?: string;
+ readonly kind?: string;
+ readonly value?: unknown;
+ readonly text?: string;
+ readonly checked?: boolean;
+ readonly changed?: boolean;
+ readonly options?: ReadonlyArray<{ value: string; text: string }>;
+};
+
+/**
+ * Resolve a ref (CDP backendDOMNodeId) to a live JS handle and invoke a
+ * page-side function on it. DOM + Runtime are enabled on every session
+ * (see cdp/session.ts enableDomains), so no extra setup is required.
+ */
+const resolveAndCall = async (
+ client: BrowserClient,
+ ref: number,
+ functionDeclaration: string,
+ args: ReadonlyArray<unknown>,
+): Promise<Result<PageResult, ToolErr>> => {
+ const session = client.session();
+
+ const resolved = await session.call("DOM.resolveNode", { backendNodeId: ref });
+ if (!resolved.success) {
+ return err({
+ kind: "invalid_state",
+ message: `ref ${ref} could not be resolved — the page may have changed. Re-run browser_snapshot to get fresh refs.`,
+ details: { ref },
+ });
+ }
+ const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId;
+ if (objectId === undefined) {
+ return err({
+ kind: "invalid_state",
+ message: `ref ${ref} resolved to no JS object — re-run browser_snapshot to get fresh refs.`,
+ details: { ref },
+ });
+ }
+
+ try {
+ const called = await session.call("Runtime.callFunctionOn", {
+ objectId,
+ functionDeclaration,
+ arguments: args.map((v) => ({ value: v })),
+ returnByValue: true,
+ awaitPromise: true,
+ });
+ if (!called.success) return err({ kind: "cdp_error", message: called.error.message, details: { ref } });
+ const data = called.data as { result?: { value?: unknown }; exceptionDetails?: unknown };
+ if (data.exceptionDetails !== undefined) {
+ return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(data.exceptionDetails)}`, details: { ref } });
+ }
+ const value = data.result?.value;
+ if (typeof value !== "object" || value === null) {
+ return err({ kind: "internal", message: "page function returned no result object", details: { ref } });
+ }
+ return ok(value as PageResult);
+ } finally {
+ // Best-effort: release the remote handle so it can be GC'd. Failure is
+ // harmless (the page may have navigated), so we ignore the result.
+ await session.call("Runtime.releaseObject", { objectId });
+ }
+};
+
+/** Drop the `ok` flag before stashing a page result in tool `details` — the
+ * tool-result envelope reserves `ok` for its own success discriminant. */
+const detailsOf = (ref: number, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => {
+ const { ok: _ok, ...rest } = res;
+ return { ref, ...rest, ...(extra ?? {}) };
+};
+
+// ── Tools ────────────────────────────────────────────────────────────────────
+
+const FieldValue = Type.Union([Type.String(), Type.Boolean()], {
+ description: "Value to set. String for text inputs / textareas / selects / contenteditable; boolean for checkboxes & radios.",
+});
+
+const FillArgs = Type.Object({
+ ref: Type.Integer({ description: "Element ref from browser_snapshot ([ref=N])." }),
+ value: FieldValue,
+});
+
+export const fillTool = defineBrowserTool({
+ name: "browser_fill",
+ label: "Browser Fill",
+ description:
+ "Fill a single form field by ref. Auto-detects input/textarea/select/checkbox/radio/contenteditable, clears then sets the value, fires input+change events (drives React/Vue controlled inputs correctly), and reads the value back to confirm. Get the ref from browser_snapshot's [ref=N].",
+ promptSnippet: "Fill a form field by ref (text, select, checkbox) — React-safe, self-confirming",
+ promptGuidelines: [
+ "Get the ref from browser_snapshot's [ref=N] for the target field. No click-to-focus needed.",
+ "value is a string for text fields/selects/contenteditable, or a boolean for checkboxes/radios.",
+ "The result reports the value the field actually holds after filling — check it matches what you intended.",
+ "For autocomplete/typeahead that needs real keystrokes, click the field's @(x,y) then use browser_type instead.",
+ "If a ref no longer resolves, re-run browser_snapshot to get fresh refs.",
+ ],
+ parameters: FillArgs,
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+ const r = await resolveAndCall(client, args.ref, FILL_FN, [args.value]);
+ if (!r.success) return r;
+ const res = r.data;
+ if (!res.ok) {
+ return err({ kind: "invalid_state", message: `Could not fill ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
+ }
+ const wanted = typeof args.value === "boolean" ? undefined : String(args.value);
+ const got = res.value !== undefined ? String(res.value) : undefined;
+ const mismatch = wanted !== undefined && got !== undefined && got !== wanted;
+ let text: string;
+ if (mismatch) {
+ text = `Filled ref ${args.ref}, but it now reads ${JSON.stringify(got)} (wanted ${JSON.stringify(wanted)})`;
+ } else if (got !== undefined) {
+ text = `Filled ref ${args.ref} = ${JSON.stringify(got)}`;
+ } else if (res.checked !== undefined) {
+ text = `Filled ref ${args.ref} (checked=${res.checked})`;
+ } else {
+ text = `Filled ref ${args.ref}`;
+ }
+ return ok({ text, details: detailsOf(args.ref, res, { mismatch }) });
+ },
+});
+
+const FillFormArgs = Type.Object({
+ fields: Type.Array(
+ Type.Object({
+ ref: Type.Integer({ description: "Element ref from browser_snapshot ([ref=N])." }),
+ value: FieldValue,
+ }),
+ { minItems: 1, description: "Fields to fill in one batch, each by ref." },
+ ),
+});
+
+export const fillFormTool = defineBrowserTool({
+ name: "browser_fill_form",
+ label: "Browser Fill Form",
+ description:
+ "Fill multiple form fields in one call — the efficient way to complete a form. Each field is { ref, value }; refs come from browser_snapshot's [ref=N]. Handles text inputs, textareas, selects, checkboxes, radios, and contenteditable, drives React/Vue controlled inputs correctly, and reports the resulting value of every field.",
+ promptSnippet: "Fill many form fields at once by ref — React-safe, self-confirming",
+ promptGuidelines: [
+ "Preferred for forms: snapshot once, then fill all fields in a single browser_fill_form call.",
+ "Each field: { ref, value }. value is a string for text/select/contenteditable, boolean for checkbox/radio.",
+ "The result summarizes how many fields filled cleanly and flags any value mismatches or per-field errors.",
+ "Re-run browser_snapshot afterwards to verify the form state if needed.",
+ ],
+ parameters: FillFormArgs,
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+ const results: Array<Readonly<Record<string, unknown>>> = [];
+ let okCount = 0;
+ const issues: string[] = [];
+ for (const f of args.fields) {
+ const r = await resolveAndCall(client, f.ref, FILL_FN, [f.value]);
+ if (!r.success) {
+ results.push({ ref: f.ref, ok: false, error: r.error.message });
+ issues.push(`ref ${f.ref} (${r.error.message})`);
+ continue;
+ }
+ const res = r.data;
+ const wanted = typeof f.value === "boolean" ? undefined : String(f.value);
+ const got = res.value !== undefined ? String(res.value) : undefined;
+ const mismatch = wanted !== undefined && got !== undefined && got !== wanted;
+ const fieldOk = res.ok === true && !mismatch;
+ if (fieldOk) okCount++;
+ else issues.push(`ref ${f.ref} (${res.ok === false ? (res.reason ?? "failed") : "value mismatch"})`);
+ results.push({ ref: f.ref, ok: fieldOk, mismatch, ...detailsOf(f.ref, res) });
+ }
+ const text =
+ `Filled ${okCount}/${args.fields.length} fields` +
+ (issues.length > 0 ? `; issues: ${issues.join(", ")}` : "");
+ return ok({ text, details: { results } });
+ },
+});
+
+const SelectOptionArgs = Type.Object({
+ ref: Type.Integer({ description: "Element ref of a <select> from browser_snapshot ([ref=N])." }),
+ value: Type.Optional(Type.String({ description: "Option value attribute to select." })),
+ label: Type.Optional(Type.String({ description: "Visible option text/label to select (used if value is omitted)." })),
+});
+
+export const selectOptionTool = defineBrowserTool({
+ name: "browser_select_option",
+ label: "Browser Select Option",
+ description:
+ "Select an option in a native <select> dropdown by ref. Matches by option value, then by visible label/text, and fires change. Native selects cannot be driven by clicking coordinates — use this instead. Get the ref from browser_snapshot's [ref=N].",
+ promptSnippet: "Choose an option in a <select> dropdown by ref",
+ promptGuidelines: [
+ "Provide value (the option's value attribute) or label (the visible text). value wins if both given.",
+ "On no match, the result lists the available options so you can retry with a valid one.",
+ "Use this for native <select> elements — clicking coordinates does not work for OS dropdowns.",
+ ],
+ parameters: SelectOptionArgs,
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+ if (args.value === undefined && args.label === undefined) {
+ return err({ kind: "invalid_state", message: "Provide value or label to select an option." });
+ }
+ const r = await resolveAndCall(client, args.ref, SELECT_FN, [args.value ?? null, args.label ?? null]);
+ if (!r.success) return r;
+ const res = r.data;
+ if (!res.ok) {
+ return err({ kind: "invalid_state", message: `Could not select option on ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
+ }
+ return ok({
+ text: `Selected ${JSON.stringify(res.text)} (value=${JSON.stringify(res.value)}) on ref ${args.ref}`,
+ details: detailsOf(args.ref, res),
+ });
+ },
+});
+
+const SetCheckedArgs = Type.Object({
+ ref: Type.Integer({ description: "Element ref of a checkbox or radio from browser_snapshot ([ref=N])." }),
+ checked: Type.Boolean({ description: "Desired checked state." }),
+});
+
+export const setCheckedTool = defineBrowserTool({
+ name: "browser_set_checked",
+ label: "Browser Set Checked",
+ description:
+ "Set a checkbox or radio to a desired checked state by ref. Only acts if the current state differs, fires change, and reports the final state. Get the ref from browser_snapshot's [ref=N].",
+ promptSnippet: "Set a checkbox/radio checked state by ref",
+ promptGuidelines: [
+ "checked:true ticks the box, checked:false unticks it. Idempotent — no-op if already in the desired state.",
+ "For radios, set checked:true on the option you want; the group deselects the others.",
+ ],
+ parameters: SetCheckedArgs,
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+ const r = await resolveAndCall(client, args.ref, CHECK_FN, [args.checked]);
+ if (!r.success) return r;
+ const res = r.data;
+ if (!res.ok) {
+ return err({ kind: "invalid_state", message: `Could not set checked on ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) });
+ }
+ return ok({
+ text: `Set ref ${args.ref} checked=${res.checked}` + (res.changed === false ? " (already)" : ""),
+ details: detailsOf(args.ref, res),
+ });
+ },
+});
diff --git a/src/domains/isolated-tab.ts b/src/domains/isolated-tab.ts
index c759cec..73ed158 100644
--- a/src/domains/isolated-tab.ts
+++ b/src/domains/isolated-tab.ts
@@ -13,6 +13,7 @@
import type { BrowserClient } from "../client";
import { type Result, err, ok } from "../util/result";
import { safeJs } from "../util/js-template";
+import { bindHarnessWindowId } from "../cdp/window-binding";
import type { ToolErr } from "../util/tool";
/** A tab attached by its own sessionId, ready to drive via callOnTarget. */
@@ -40,7 +41,10 @@ export const openIsolatedTab = async (client: BrowserClient): Promise<Result<Iso
if (!created.success) return err(toToolErr(created.error.message));
const { targetId } = created.data as { targetId: string };
- if (createParams["newWindow"]) client.ownership().setHarnessWindow(targetId);
+ if (createParams["newWindow"]) {
+ client.ownership().setHarnessWindow(targetId);
+ await bindHarnessWindowId(client, targetId);
+ }
client.ownership().add(targetId);
const attached = await client.session().callBrowser("Target.attachToTarget", { targetId, flatten: true });
diff --git a/src/domains/navigate.ts b/src/domains/navigate.ts
index 4477485..e402efc 100644
--- a/src/domains/navigate.ts
+++ b/src/domains/navigate.ts
@@ -4,6 +4,7 @@ import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
import { applyTruncation } from "../util/truncate";
import { safeJs } from "../util/js-template";
+import { bindHarnessWindowId } from "../cdp/window-binding";
const NavigateArgs = Type.Object({
url: Type.String({ description: "Full URL to navigate to (e.g. https://github.com)" }),
@@ -75,7 +76,7 @@ export const openUrlsTool = defineBrowserTool({
description: "Open multiple URLs in parallel new tabs. Returns per-URL outcomes.",
promptSnippet: "Open multiple URLs in new tabs (parallel)",
promptGuidelines: [
- "Use after web_search to open citations in parallel.",
+ "Use after browser_web_search to open citations in parallel.",
"After opening, use browser_list_tabs / browser_switch_tab to navigate, then browser_snapshot to inspect each tab.",
"Use browser_wait_for_load on a tab before extracting data from SPAs.",
],
@@ -83,17 +84,31 @@ export const openUrlsTool = defineBrowserTool({
renderCall: (a) => new Text(`🌐 Opening ${a.urls.length} URL${a.urls.length !== 1 ? "s" : ""}…`, 0, 0),
async handler(args, { client, onUpdate }): Promise<Result<ToolOk, ToolErr>> {
const total = args.urls.length;
- // Phase 1: create tabs in parallel. Own them so they appear in
- // browser_list_tabs scope:'owned' and group in the harness window.
- const hw = client.ownership().harnessWindow();
- const created = await Promise.all(args.urls.map(async (url): Promise<TabResult> => {
- const params: Record<string, unknown> = { url: "about:blank" };
- if (hw) params["openerId"] = hw;
- else params["newWindow"] = true;
- const r = await client.session().callBrowser("Target.createTarget", params);
+ // Phase 1: create tabs. All must land in ONE window. If the harness window
+ // already exists, open every url as its child in parallel. If it doesn't
+ // (fresh/closed), create the FIRST url as a new dedicated window (binding
+ // its windowId) and open the rest as children of it — a parallel
+ // newWindow:true per url would otherwise scatter tabs across N windows.
+ let seed = client.ownership().harnessWindow();
+ let seedResult: TabResult | undefined;
+ if (!seed) {
+ const url0 = args.urls[0];
+ if (url0 === undefined) return err({ kind: "invalid_state", message: "no URLs provided" });
+ const r0 = await client.session().callBrowser("Target.createTarget", { url: "about:blank", newWindow: true });
+ if (!r0.success) return err({ kind: "cdp_error", message: r0.error.message });
+ seed = (r0.data as { targetId: string }).targetId;
+ client.ownership().setHarnessWindow(seed);
+ await bindHarnessWindowId(client, seed);
+ client.ownership().add(seed);
+ seedResult = { url: url0, targetId: seed, ok: true };
+ }
+ const seededWindow = seed;
+ const created = await Promise.all(args.urls.map(async (url, i): Promise<TabResult> => {
+ // Reuse the seed tab (created above) for the first url instead of duplicating it.
+ if (i === 0 && seedResult) return seedResult;
+ const r = await client.session().callBrowser("Target.createTarget", { url: "about:blank", openerId: seededWindow });
if (!r.success) return { url, targetId: "", ok: false, error: r.error.message };
const c = r.data as { targetId: string };
- if (params["newWindow"]) client.ownership().setHarnessWindow(c.targetId);
client.ownership().add(c.targetId);
return { url, targetId: c.targetId, ok: true };
}));
diff --git a/src/index.ts b/src/index.ts
index d5f3435..7109ea0 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -109,9 +109,10 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
if (!client) {
const transport = createDaemonTransport(state.namespace);
- const initialOwnership: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string } = {};
+ const initialOwnership: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string; harnessWindowId?: number } = {};
if (state.ownedTargetIds !== undefined) initialOwnership.ownedTargetIds = state.ownedTargetIds;
if (state.harnessWindowTargetId !== undefined) initialOwnership.harnessWindowTargetId = state.harnessWindowTargetId;
+ if (state.harnessWindowId !== undefined) initialOwnership.harnessWindowId = state.harnessWindowId;
client = createBrowserClient({
namespace: state.namespace,
@@ -124,6 +125,9 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
...(snap.harnessWindowTargetId !== undefined
? { harnessWindowTargetId: snap.harnessWindowTargetId }
: {}),
+ ...(snap.harnessWindowId !== undefined
+ ? { harnessWindowId: snap.harnessWindowId }
+ : {}),
};
// Ownership changes can fire from CDP events at any time,
// including after session replacement when pi's ctx is stale.
@@ -147,20 +151,27 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void {
});
pi.on("session_shutdown", async () => {
- persistState(pi, state);
if (client) {
try {
// Detach from the page target (removes the "Chrome is being controlled"
// banner) but keep the transport alive for the next session.
await client.detach();
+ // Close every tab this session opened so no stale harness tabs linger
+ // in the user's browser between sessions. This also clears the persisted
+ // ownership + window binding via onOwnershipChange, so the next session
+ // starts with a fresh dedicated window rather than reattaching to leftovers.
+ await client.closeOwnedTabs();
} catch (e) {
- console.warn("[pi-browser-harness] client.detach() failed during shutdown:", e);
+ console.warn("[pi-browser-harness] browser teardown failed during shutdown:", e);
}
// ponytail: keep the transport alive across sessions.
// Do NOT call client.stop() or null out client — the daemon connection
// persists and eliminates the per-session "Allow Remote Debugging" prompt.
// Only Chrome restart or daemon death triggers a new prompt.
}
+ // Persist last — closeOwnedTabs mutates `state` through onOwnershipChange,
+ // so this captures the cleared ownership set.
+ persistState(pi, state);
await cleanupTempDirs();
});
diff --git a/src/prompt.ts b/src/prompt.ts
index 7483730..c336c0c 100644
--- a/src/prompt.ts
+++ b/src/prompt.ts
@@ -2,150 +2,126 @@
* System prompt guidance for browser usage.
*
* Injects a compact browser control reference into the pi system prompt.
- * Per-tool guidelines are injected via promptGuidelines on each tool registration.
- * This prompt provides only the cross-cutting patterns and tool reference table.
+ * This prompt carries only cross-cutting knowledge — the mental model, tool
+ * routing, workflows, and gotchas. Per-tool specifics (schemas, detailed usage)
+ * are injected separately via promptGuidelines on each tool registration, so this
+ * file intentionally does NOT restate full tool descriptions.
*/
export function getBrowserSystemPrompt(): string {
return `
## Browser Control
-You have full control of a real Chrome browser via \`browser_*\` tools.
-The browser connects to the user's running Chrome (not a headless instance).
-
-**These tools coexist with all other standard pi tools** — read, bash, edit,
-write, and any other built-in tools are still available. Use browser_* tools
-to interact with pages visually, and other tools alongside them.
-
-### Browser-Specific Tools
-
-| Tool | Purpose |
-|------|---------|
-| browser_snapshot | Accessibility-tree snapshot. Assigns every interactive element a stable **ref** (\`[eN]\`) plus \`@(x,y)\`. Refs are the primary handle for interaction tools — they survive re-renders, coordinates don't. |
-| browser_screenshot | Capture a PNG or JPEG screenshot of the current page (rendered inline in the TUI; the LLM reads the image via the saved file path) |
-| browser_click | Click an element by **ref** (preferred — re-resolves position, survives re-renders) or viewport x/y. Appends a compact diff of page changes. |
-| browser_type | Type text into the focused element (real keystrokes; requires a focused field) |
-| browser_fill | Fill an input/textarea by **ref** (preferred) or selector — updates React/Vue controlled components, verifies the value, appends a page-changes diff |
-| browser_select_option | Select an option in a native <select> by **ref** (preferred) or selector, choosing by value/label/index (fires change) |
-| browser_focus | Focus an element by **ref** (preferred) or selector (deterministic; use before browser_type) |
-| browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) |
-| browser_scroll | Scroll the page at coordinates |
-| browser_navigate | Navigate to a URL (result details.outcome.kind is "in_place" or "new_tab_created") |
-| browser_new_tab | Open a new tab, optionally navigate |
-| browser_open_urls | Open multiple URLs in new tabs (parallel) |
-| browser_go_back / browser_go_forward / browser_reload | History navigation |
-| browser_page_info | Get page URL, title, viewport, scroll position, or dialog info |
-| browser_list_tabs | List all open browser tabs |
-| browser_current_tab | Get current tab info |
-| browser_switch_tab | Switch to a different tab by targetId |
-| browser_execute_js | Execute JavaScript and return the result |
-| browser_http_get | Direct HTTP GET (outside browser, for APIs) |
-| browser_console | Read JS errors / console output on the current tab (diagnostic — use when an action looks broken; pass sinceSeq from the previous nextCursor to see only new messages) |
-| browser_wait | Wait N seconds |
-| browser_wait_for | Wait until a selector/text appears (or a selector disappears) — use for SPA content before interacting |
-| browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) |
-| browser_handle_dialog | Accept or dismiss a JS dialog |
-| browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) |
-
-### Temporary Scripts
-
-When the built-in tools aren't enough for a multi-step workflow, write a temporary
-script to disk and execute it with browser_run_script. The script runs in the
-harness process with direct access to the browser daemon and Node.js APIs.
+You drive a **real Chrome** (the user's running instance, not headless) through
+\`browser_*\` tools. These coexist with every standard pi tool — read, bash, edit,
+write — so mix them freely: \`browser_*\` for the page, everything else alongside.
-\`\`\`
-write("/tmp/scrape-pages.js", \`
- const results = [];
- for (const url of params.urls) {
- await daemon.cdp("Page.navigate", { url });
- await new Promise(r => setTimeout(r, 2000));
- const data = await daemon.evaluateJS("document.title");
- results.push({ url, title: data });
- }
- return { content: [{ type: "text", text: JSON.stringify(results, null, 2) }] };
-\`)
-browser_run_script("/tmp/scrape-pages.js", { urls: [...] })
-\`\`\`
+### The core handle: refs
-Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout.
+Interaction is ref-first. Call \`browser_snapshot\` to get an accessibility tree
+where every interactive element has a stable ref (\`[eN]\`) and \`@(x,y)\`. **Target
+elements by ref, not by coordinates or guessed selectors** — refs are keyed to
+element identity, so they survive the re-renders that React/Vue forms trigger on
+every edit and save. Coordinates go stale; guessed selectors miss.
-### Parallel Execution
+- Prefer \`browser_fill\` for text — it fires input/change so controlled components
+ update, and returns the value for verification. Use \`browser_click\` + \`browser_type\`
+ only for keystroke-sensitive widgets (autocomplete, masked inputs).
+- Mutating calls append a compact **"Page changes"** diff — read it to confirm the
+ change landed (form closed, new \`*[eN]\` appeared).
+- \`"ref is stale"\` means the page changed: re-run \`browser_snapshot\` for fresh refs.
-Observation tools (browser_screenshot, browser_page_info, browser_execute_js, browser_list_tabs, browser_http_get, etc.) can run in parallel with each other and with mutation tools. The harness automatically serializes mutation tools (click, type, scroll, navigate, switch_tab, etc.) so they never race on shared state. When operations are independent, emit them in the same turn for better performance.
+### Choosing a tool
-**Examples of safe parallel calls:**
-\`\`\`
-browser_screenshot() + browser_page_info() + browser_execute_js("document.title")
-browser_http_get("https://api.example.com/data") + browser_click(x, y)
-\`\`\`
+| To… | Reach for |
+|-----|-----------|
+| See the page structure (for interaction) | \`browser_snapshot\` (refs) |
+| See the page visually | \`browser_screenshot\` |
+| Search the web | \`browser_web_search\` → ranked {title, url, snippet} |
+| Read an article's clean text | \`browser_read_page\` (reader mode; url or owned tab) |
+| Extract specific DOM values | \`browser_execute_js\` |
+| Hit a JSON/API endpoint | \`browser_http_get\` (raw GET, outside the browser) |
+| Click / fill / type / select / focus / press a key | \`browser_click\` · \`browser_fill\` · \`browser_type\` · \`browser_select_option\` · \`browser_focus\` · \`browser_press_key\` |
+| Upload a file · drag · resize viewport | \`browser_upload_file\` · \`browser_drag_and_drop\` · \`browser_viewport_resize\` |
+| Go to a URL / open many / manage tabs | \`browser_navigate\` · \`browser_new_tab\` · \`browser_open_urls\` · \`browser_list_tabs\` · \`browser_switch_tab\` · \`browser_go_back\`/\`go_forward\`/\`reload\` |
+| Wait for content / load / a fixed delay | \`browser_wait_for\` (selector/text) · \`browser_wait_for_load\` (readyState) · \`browser_wait\` |
+| Inspect page / handle a dialog | \`browser_page_info\` · \`browser_current_tab\` · \`browser_handle_dialog\` |
+| Diagnose a broken action | \`browser_console\` (JS errors) · \`browser_get_network_log\` |
+| Save output / config downloads | \`browser_print_to_pdf\` · \`browser_download\` |
+| Anything the tools can't express | \`browser_run_script\` (see Extending) |
-**Multi-agent note**: When multiple subagents use the browser, tab switching by one agent changes the active tab for all agents. Per-tab data (console buffers, network traces, dialogs, page info cache) is isolated — switching tabs does not destroy another agent's collected data. Call \`browser_current_tab\` before mutation tools to confirm you're on the expected tab. If \`browser_page_info\` returns a dialog, handle it with \`browser_handle_dialog\` promptly — dialogs block page interaction and are not queued across agents.
+Each tool carries its own detailed guidelines — this table is only for routing.
-### Common Patterns
+### Workflows
-**Navigation:**
+**Research (search → read):**
\`\`\`
-browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot()
+browser_web_search("query") // ranked links, no page content
+→ browser_read_page(url) // clean article text for the top hits
+// fall back to browser_open_urls + browser_execute_js only when reader mode misses
\`\`\`
-**Form filling (ref-first — this is the reliable path on React/SPA forms):**
+**Form filling (ref-first — the reliable path on SPA forms):**
\`\`\`
-browser_wait_for({ selector: "#email" }) // ensure SPA field is rendered
-browser_snapshot() // each interactive element gets a [eN] ref
-→ browser_fill({ ref: "e7", value: "a@b.com" }) // target by ref, not a guessed selector
-→ browser_select_option({ ref: "e9", label: "India" }) // native <select> by ref
-→ browser_click({ ref: "e12" }) // e.g. Save — re-resolves position even if the form reflowed
-// each mutating call appends a compact "Page changes" diff — read it to confirm
-// the change landed (e.g. a save closed the form, a new field appeared as *[eN]).
+browser_wait_for({ selector: "#email" }) // ensure the field is rendered
+browser_snapshot() // interactive elements get [eN] refs
+→ browser_fill({ ref: "e7", value: "a@b.com" })
+→ browser_select_option({ ref: "e9", label: "India" })
+→ browser_click({ ref: "e12" }) // re-resolves position even after reflow
+// read each "Page changes" diff to confirm the step landed
\`\`\`
-**Why ref over selector/coordinates:** refs are keyed to the element's identity, so
-they survive the re-renders that React/Vue forms trigger on every edit and save —
-coordinates go stale and selectors you guess often don't match. If a call returns
-"ref is stale", the page changed: re-run browser_snapshot for fresh refs.
-
-Prefer browser_fill for text inputs/textareas — it fires input/change so controlled
-components update, and returns the field's value for verification. Use browser_click +
-browser_type only for keystroke-sensitive widgets (autocomplete, masked inputs);
-browser_type requires an already-focused field and errors if none is focused.
-**Data extraction:**
+**Navigate + verify:**
\`\`\`
-browser_execute_js("document.querySelector('.price').innerText")
-// or for APIs:
-browser_http_get("https://api.example.com/data")
+browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot()
\`\`\`
-**Scrolling:**
-\`\`\`
-browser_screenshot() → browser_scroll({ deltaY: 500 }) → browser_screenshot()
-\`\`\`
-Note: deltaY follows W3C wheel-event convention: positive=down, negative=up. Default deltaY=300 scrolls down.
+**Scroll:** \`browser_scroll({ deltaY: 500 })\` — W3C wheel convention: positive = down,
+negative = up (default 300, down).
-**Research Workflow (search + browser):**
+### Parallelism
+
+Observation tools (\`browser_screenshot\`, \`browser_page_info\`, \`browser_execute_js\`,
+\`browser_list_tabs\`, \`browser_http_get\`, \`browser_web_search\`, \`browser_read_page\`, …)
+run in parallel with each other and with mutations. The harness serializes mutations
+(click, type, scroll, navigate, switch_tab) so they never race. Emit independent calls
+in one turn:
\`\`\`
-browser_navigate("https://google.com/search?q=...") → search engine in a tab
-browser_open_urls(urls: ["url1", "url2", ...]) → open result pages in parallel tabs
-browser_list_tabs() → see all open tabs with targetIds
-browser_switch_tab(targetId: "...") → switch to a tab
-browser_screenshot() → visually inspect the page
-browser_execute_js("document.querySelector('.main').innerText") → extract content
+browser_screenshot() + browser_page_info() + browser_execute_js("document.title")
\`\`\`
-**Temporary Scripts (extending the harness):**
+**Multi-agent:** tab switching by one agent changes the active tab for all agents,
+but per-tab data (console, network, dialogs) stays isolated. Call \`browser_current_tab\`
+before a mutation to confirm you're on the expected tab. Handle any dialog reported by
+\`browser_page_info\` promptly — dialogs block interaction and aren't queued across agents.
+
+**Session boundary & tab hygiene:** you operate only inside this session's own Chrome
+window. \`browser_switch_tab\`/\`browser_close_tab\` refuse tabs this session didn't open,
+and \`browser_list_tabs\` defaults to owned tabs — never reach into the user's other tabs
+or windows. Close tabs when you're done with them: call \`browser_close_tab\` on any tab
+you opened (via \`browser_new_tab\`/\`browser_open_urls\`) once you've extracted what you
+need. Don't leave a pile of stale tabs behind — keep only what a later step still needs.
+(Leftover tabs are closed automatically when the session ends, but close them yourself
+as you go.)
+
+### Extending: temporary scripts
+
+When built-in tools can't express a multi-step workflow, write a script to a temp file
+and run it with \`browser_run_script\`. It executes in the harness process with direct
+daemon + Node access.
\`\`\`
-write("/tmp/extract.js", "...script with daemon access...") → browser_run_script("/tmp/extract.js")
+write(<tempfile>.js, \`
+ const results = [];
+ for (const url of params.urls) {
+ await daemon.cdp("Page.navigate", { url });
+ await new Promise(r => setTimeout(r, 2000));
+ results.push({ url, title: await daemon.evaluateJS("document.title") });
+ }
+ return { content: [{ type: "text", text: JSON.stringify(results, null, 2) }] };
+\`)
+browser_run_script(<tempfile>.js, { urls: [...] })
\`\`\`
-
-### Additional Tools
-
-| Tool | Purpose |
-|------|---------|
-| browser_upload_file | Upload a file to a file input by **ref** (preferred) or selector (bypasses file picker) |
-| browser_dispatch_key | Dispatch a DOM KeyboardEvent on an element by **ref** (preferred) or selector (for React/Vue inputs); returns details.matched |
-| browser_download | Configure download directory and disable save-as prompts |
-| browser_viewport_resize | Resize the viewport for responsive testing |
-| browser_drag_and_drop | Perform drag-and-drop from one coordinate to another |
-| browser_print_to_pdf | Print the current page to a PDF file |
-| browser_get_network_log | Get buffered network request/response events |
+Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON,
+Buffer, setTimeout, clearTimeout. Use a real temp path from the OS — don't hardcode \`/tmp\`.
`;
}
diff --git a/src/state.ts b/src/state.ts
index 0629f40..d4b1534 100644
--- a/src/state.ts
+++ b/src/state.ts
@@ -12,6 +12,7 @@ export type BrowserState = {
readonly remoteBrowserId?: string;
readonly ownedTargetIds?: ReadonlyArray<string>;
readonly harnessWindowTargetId?: string;
+ readonly harnessWindowId?: number;
};
export const defaultState = (namespace = "default"): BrowserState => ({
diff --git a/test/manual/forms-test.ts b/test/manual/forms-test.ts
new file mode 100644
index 0000000..f1c79f6
--- /dev/null
+++ b/test/manual/forms-test.ts
@@ -0,0 +1,149 @@
+/**
+ * Manual integration test for the form-filling tools.
+ *
+ * Exercises browser_snapshot (ref surfacing) + browser_fill_form / browser_fill /
+ * browser_select_option / browser_set_checked against a data:-URL fixture that
+ * includes a plain input, a textarea, a "controlled" input (commits its model
+ * only on the `input` event — the realistic failure mode for naive value-setting),
+ * a native <select>, a checkbox, and a contenteditable.
+ *
+ * Requires the daemon running + a real Chrome (same as live-daemon-test.ts).
+ * Run: npx tsx test/manual/forms-test.ts
+ */
+import { createDaemonTransport } from "../../src/cdp/daemon-transport";
+import { createBrowserClient } from "../../src/client";
+import { fillTool, fillFormTool, selectOptionTool, setCheckedTool } from "../../src/domains/forms";
+
+// NOTE: we intentionally do NOT import snapshotTool here — it pulls in pi-tui /
+// pi-coding-agent value imports that don't resolve under bare tsx. Instead we
+// derive refs from Accessibility.getFullAXTree's backendDOMNodeId, which is the
+// exact value browser_snapshot surfaces as [ref=N]. The snapshot's [ref=N]
+// rendering is covered by `npm run typecheck` + the real-page smoke test.
+
+let passed = 0;
+let failed = 0;
+const check = (cond: boolean, label: string): void => {
+ if (cond) { passed++; console.log(` ✓ ${label}`); }
+ else { failed++; console.error(` ✗ ${label}`); }
+};
+
+// Minimal handler context — the form/snapshot handlers only read `client`.
+const mkCtx = (client: any): any => ({
+ client,
+ signal: undefined,
+ onUpdate: () => {},
+ extensionCtx: { cwd: process.cwd() },
+});
+
+const FIXTURE = `
+<!doctype html><html><body>
+ <h1>Form fixture</h1>
+ <input id="plain" aria-label="plain" />
+ <textarea id="ta" aria-label="ta"></textarea>
+ <input id="controlled" aria-label="controlled" />
+ <span id="model"></span>
+ <select id="sel" aria-label="country">
+ <option value="us">United States</option>
+ <option value="uk">United Kingdom</option>
+ <option value="ca">Canada</option>
+ </select>
+ <input type="checkbox" id="chk" aria-label="agree" />
+ <div id="editable" role="textbox" aria-label="bio" contenteditable="true"></div>
+ <script>
+ // Controlled input: its "model" updates ONLY when an input event fires.
+ // A naive value-set that skips the input event leaves the model stale.
+ var model = "";
+ var c = document.getElementById("controlled");
+ c.addEventListener("input", function () { model = c.value; document.getElementById("model").textContent = model; });
+ window.__model = function () { return model; };
+ </script>
+</body></html>`;
+
+// Resolve a ref by accessible name via the AX tree — the same backendDOMNodeId
+// that browser_snapshot prints as [ref=N].
+const refByName = async (client: any, name: string): Promise<number | undefined> => {
+ const ax = await client.session().call("Accessibility.getFullAXTree", {});
+ if (!ax.success) return undefined;
+ for (const n of (ax.data as any).nodes as any[]) {
+ if (n?.name?.value === name && typeof n.backendDOMNodeId === "number") return n.backendDOMNodeId;
+ }
+ return undefined;
+};
+
+async function main(): Promise<void> {
+ const transport = createDaemonTransport("pi-forms-test");
+ const client = createBrowserClient({ namespace: "pi-forms-test", transport });
+ const ctx = mkCtx(client);
+
+ const started = await client.start();
+ if (!started.success) { console.error("Could not start client:", started.error.message); process.exit(1); }
+
+ const url = "data:text/html," + encodeURIComponent(FIXTURE);
+ const tab = await client.newTab(url);
+ check(tab.success, `Opened fixture tab: ${tab.success ? "ok" : tab.error.message}`);
+ // Give the inline script a beat to wire up listeners.
+ await new Promise((r) => setTimeout(r, 300));
+
+ // ── Resolve refs (the backendDOMNodeId browser_snapshot prints as [ref=N]) ──
+ const refs = {
+ plain: await refByName(client, "plain"),
+ ta: await refByName(client, "ta"),
+ controlled: await refByName(client, "controlled"),
+ sel: await refByName(client, "country"),
+ chk: await refByName(client, "agree"),
+ editable: await refByName(client, "bio"),
+ };
+ for (const [k, v] of Object.entries(refs)) check(typeof v === "number", `ref present for ${k} (${v})`);
+
+ // ── Batch fill text fields ──────────────────────────────────────────────────
+ const fill = await fillFormTool.handler({
+ fields: [
+ { ref: refs.plain!, value: "hello plain" },
+ { ref: refs.ta!, value: "multi\nline" },
+ { ref: refs.controlled!, value: "ctrl-value" },
+ { ref: refs.editable!, value: "my bio" },
+ ],
+ }, ctx);
+ check(fill.success, "browser_fill_form succeeded");
+ if (fill.success) console.log(" " + fill.data.text);
+
+ // ── Select + checkbox ───────────────────────────────────────────────────────
+ const sel = await selectOptionTool.handler({ ref: refs.sel!, label: "Canada" }, ctx);
+ check(sel.success, `browser_select_option by label: ${sel.success ? sel.data.text : (sel as any).error.message}`);
+ const chk = await setCheckedTool.handler({ ref: refs.chk!, checked: true }, ctx);
+ check(chk.success, `browser_set_checked: ${chk.success ? chk.data.text : (chk as any).error.message}`);
+
+ // ── Verify committed state via JS reads ─────────────────────────────────────
+ const read = async (expr: string): Promise<unknown> => {
+ const r = await client.evaluateJs(expr);
+ return r.success ? r.data : `ERR:${r.error.message}`;
+ };
+ check((await read("document.getElementById('plain').value")) === "hello plain", "plain input value committed");
+ check((await read("document.getElementById('ta').value")) === "multi\nline", "textarea value committed");
+ // The controlled input's MODEL (input-event-driven) must reflect the fill —
+ // proves browser_fill fired the input event, not just set the DOM value.
+ check((await read("window.__model()")) === "ctrl-value", "controlled input model committed (input event fired)");
+ check((await read("document.getElementById('editable').textContent")) === "my bio", "contenteditable text committed");
+ check((await read("document.getElementById('sel').value")) === "ca", "select value committed");
+ check((await read("document.getElementById('chk').checked")) === true, "checkbox checked committed");
+
+ // ── No-matching-option path ─────────────────────────────────────────────────
+ const badSel = await selectOptionTool.handler({ ref: refs.sel!, label: "Atlantis" }, ctx);
+ check(!badSel.success && (badSel as any).error.kind === "invalid_state", "select with no match → invalid_state error");
+
+ // ── Stale ref path ──────────────────────────────────────────────────────────
+ const staleRef = refs.plain!;
+ const nav = await client.session().call("Page.navigate", { url: "data:text/html," + encodeURIComponent("<body>gone</body>") });
+ check(nav.success, "navigated away to invalidate refs");
+ await new Promise((r) => setTimeout(r, 300));
+ const stale = await fillTool.handler({ ref: staleRef, value: "x" }, ctx);
+ check(!stale.success && (stale as any).error.kind === "invalid_state", "fill on stale ref → invalid_state error");
+
+ await client.closeTab(tab.success ? (tab.data as string) : "");
+ await client.stop();
+
+ console.log(`\n${passed} passed, ${failed} failed`);
+ process.exit(failed === 0 ? 0 : 1);
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });
diff --git a/test/manual/window-binding-test.ts b/test/manual/window-binding-test.ts
new file mode 100644
index 0000000..d743ae5
--- /dev/null
+++ b/test/manual/window-binding-test.ts
@@ -0,0 +1,239 @@
+/**
+ * Unit test (mock transport) for window-scoped session binding + tab teardown.
+ *
+ * Covers the two fixes:
+ * 1. The session binds to a real Chrome windowId. New tabs land in that window;
+ * tabs opened by an owned page (popups / window.open) are auto-adopted so the
+ * session controls them; nothing outside the window is ever owned.
+ * 2. closeOwnedTabs() closes every owned tab and clears the window binding — the
+ * teardown the extension runs on session shutdown so no stale tabs survive.
+ *
+ * Run: npx tsx test/manual/window-binding-test.ts
+ */
+import { createBrowserClient } from "../../src/client";
+import { createOwnershipRegistry } from "../../src/cdp/ownership";
+import type { CdpTransport } from "../../src/cdp/transport";
+import type { CdpEvent } from "../../src/cdp/types";
+import { ok, type Result } from "../../src/util/result";
+import type { CdpError } from "../../src/cdp/errors";
+
+let passed = 0;
+let failed = 0;
+const check = (cond: boolean, label: string): void => {
+ if (cond) { passed++; console.log(` ✓ ${label}`); }
+ else { failed++; console.error(` ✗ ${label}`); }
+};
+
+// ── A minimal in-memory CDP browser the mock transport drives. ──────────────
+type FakeTarget = { targetId: string; type: string; url: string; title: string; windowId: number; openerId?: string };
+
+const createMockTransport = (): CdpTransport & {
+ seed: (t: FakeTarget) => void;
+ targets: Map<string, FakeTarget>;
+ closed: ReadonlyArray<string>;
+} => {
+ const targets = new Map<string, FakeTarget>();
+ const closed: string[] = [];
+ let sessionSeq = 0;
+ let targetSeq = 0;
+ let windowSeq = 100;
+
+ // Event queue plumbing so session.ts's consumeEvents() can await targetCreated.
+ const listeners: Array<() => void> = [];
+ let buffer: CdpEvent[] = [];
+ let waiter: (() => void) | null = null;
+ const push = (ev: CdpEvent): void => {
+ buffer.push(ev);
+ if (waiter) { const w = waiter; waiter = null; w(); }
+ };
+ const iterable: AsyncIterable<CdpEvent> = {
+ [Symbol.asyncIterator]() {
+ return {
+ async next(): Promise<IteratorResult<CdpEvent>> {
+ if (buffer.length === 0) {
+ await new Promise<void>((r) => { waiter = r; });
+ }
+ const v = buffer.shift()!;
+ return { value: v, done: false };
+ },
+ };
+ },
+ };
+
+ const request = async (
+ method: string,
+ params: Record<string, unknown>,
+ ): Promise<Result<unknown, CdpError>> => {
+ switch (method) {
+ case "Target.setDiscoverTargets":
+ case "Page.enable": case "DOM.enable": case "Runtime.enable":
+ case "Network.enable": case "Accessibility.enable": case "Log.enable":
+ case "Target.activateTarget":
+ return ok({});
+ case "Target.getTargets":
+ return ok({ targetInfos: [...targets.values()].map((t) => ({ targetId: t.targetId, type: t.type, url: t.url, title: t.title })) });
+ case "Browser.getWindowForTarget": {
+ const t = targets.get(params["targetId"] as string);
+ return ok({ windowId: t?.windowId ?? -1, bounds: {} });
+ }
+ case "Target.createTarget": {
+ const id = `tgt-${++targetSeq}`;
+ const opener = params["openerId"] as string | undefined;
+ const newWindow = params["newWindow"] === true;
+ const windowId = newWindow || !opener ? ++windowSeq : (targets.get(opener)?.windowId ?? ++windowSeq);
+ const t: FakeTarget = { targetId: id, type: "page", url: (params["url"] as string) ?? "about:blank", title: "", windowId, ...(opener ? { openerId: opener } : {}) };
+ targets.set(id, t);
+ // Chrome fires targetCreated for every new target (discover:true).
+ push({ method: "Target.targetCreated", params: { targetInfo: { targetId: id, type: "page", url: t.url, openerId: opener } } });
+ return ok({ targetId: id });
+ }
+ case "Target.attachToTarget":
+ return ok({ sessionId: `sess-${++sessionSeq}` });
+ case "Target.detachFromTarget":
+ return ok({});
+ case "Target.closeTarget": {
+ const id = params["targetId"] as string;
+ if (targets.delete(id)) { closed.push(id); push({ method: "Target.targetDestroyed", params: { targetId: id } }); }
+ return ok({});
+ }
+ case "Runtime.evaluate":
+ return ok({ result: { value: "complete" } });
+ case "Page.navigate":
+ return ok({ frameId: "f1" });
+ default:
+ return ok({});
+ }
+ };
+
+ return {
+ connect: async () => ok(undefined),
+ close: async () => { for (const l of listeners) l(); },
+ request,
+ events: () => iterable,
+ state: () => "open",
+ onClose: (cb) => { listeners.push(cb); return () => {}; },
+ seed: (t) => targets.set(t.targetId, t),
+ targets,
+ get closed() { return closed; },
+ };
+};
+
+const tick = (): Promise<void> => new Promise((r) => setTimeout(r, 20));
+
+async function main(): Promise<void> {
+ console.log("window-binding-test\n");
+
+ // ── Registry: windowId binding is stored + cleared. ───────────────────────
+ {
+ const reg = createOwnershipRegistry();
+ check(reg.harnessWindowId() === undefined, "windowId starts undefined");
+ reg.setHarnessWindowId(101);
+ check(reg.harnessWindowId() === 101, "setHarnessWindowId stores the id");
+ let notified = 0;
+ reg.onChange(() => { notified++; });
+ reg.setHarnessWindowId(101);
+ check(notified === 0, "setting the same windowId does not notify");
+ reg.setHarnessWindowId(undefined);
+ check(reg.harnessWindowId() === undefined && notified === 1, "clearing windowId notifies");
+ }
+
+ // ── newTab: creates the dedicated window and captures its real windowId. ──
+ {
+ const transport = createMockTransport();
+ const client = createBrowserClient({ namespace: "t", transport });
+ await client.start();
+ const wid = client.ownership().harnessWindowId();
+ check(typeof wid === "number", "attach captures a real windowId for the harness window");
+
+ const first = await client.newTab("https://a.test");
+ check(first.success, "newTab succeeds");
+ if (first.success) {
+ const t = transport.targets.get(first.data)!;
+ check(t.windowId === wid, "new tab lands in the harness window (via openerId)");
+ check(client.owns(first.data), "new tab is owned");
+ }
+ }
+
+ // ── Popup adoption: a tab opened by an owned page is auto-owned. ──────────
+ {
+ const transport = createMockTransport();
+ const client = createBrowserClient({ namespace: "t", transport });
+ await client.start();
+ const cur = client.current()!;
+
+ // A page the session controls opens a child target (window.open / _blank).
+ // Chrome emits Target.targetCreated with openerId = our owned tab; the
+ // session must adopt it so the tab is controllable and cleaned up later.
+ const child = await client.session().callBrowser("Target.createTarget", { url: "https://child.test", openerId: cur.targetId });
+ await tick();
+ check(child.success, "owned page opens a child target");
+ if (child.success) {
+ const id = (child.data as { targetId: string }).targetId;
+ check(client.owns(id), "child target opened by an owned tab is auto-adopted");
+ }
+
+ // A target the user opens in their own window has no owned opener → never adopted.
+ const stranger = await client.session().callBrowser("Target.createTarget", { url: "https://user.test", newWindow: true });
+ await tick();
+ if (stranger.success) {
+ const id = (stranger.data as { targetId: string }).targetId;
+ check(!client.owns(id), "target with no owned opener is not adopted");
+ }
+ }
+
+ // ── Reattach self-heal: re-adopt live window-mates of a surviving owned tab. ─
+ {
+ const transport = createMockTransport();
+ transport.seed({ targetId: "own-1", type: "page", url: "https://a.test", title: "", windowId: 500 });
+ transport.seed({ targetId: "sib-2", type: "page", url: "https://b.test", title: "", windowId: 500, openerId: "own-1" });
+ transport.seed({ targetId: "user-9", type: "page", url: "https://user.test", title: "", windowId: 700 });
+ const client = createBrowserClient({
+ namespace: "t",
+ transport,
+ initialOwnership: { ownedTargetIds: ["own-1"], harnessWindowTargetId: "own-1", harnessWindowId: 500 },
+ });
+ await client.start();
+ check(client.owns("own-1"), "reattach: persisted owned tab stays owned");
+ check(client.owns("sib-2"), "reattach: live window-mate (last session's popup) is re-adopted");
+ check(!client.owns("user-9"), "reattach: a tab in a DIFFERENT window is never adopted");
+ }
+
+ // ── Restart-collision guard: stale windowId must not adopt the user's tabs. ─
+ {
+ const transport = createMockTransport();
+ // Chrome restarted: our owned tab id is dead; windowId 500 now belongs to
+ // the user's own window full of their tabs.
+ transport.seed({ targetId: "user-a", type: "page", url: "https://user-a.test", title: "", windowId: 500 });
+ transport.seed({ targetId: "user-b", type: "page", url: "https://user-b.test", title: "", windowId: 500 });
+ const client = createBrowserClient({
+ namespace: "t",
+ transport,
+ initialOwnership: { ownedTargetIds: ["dead-1"], harnessWindowTargetId: "dead-1", harnessWindowId: 500 },
+ });
+ await client.start();
+ check(!client.owns("user-a") && !client.owns("user-b"), "restart: stale windowId does NOT adopt the user's tabs");
+ check(client.ownership().harnessWindowId() !== 500, "restart: binds a fresh window instead of the colliding id");
+ check(client.current() !== null && !["user-a", "user-b"].includes(client.current()!.targetId), "restart: attaches to a fresh harness tab, not a user tab");
+ }
+
+ // ── Teardown: closeOwnedTabs closes every owned tab + clears binding. ─────
+ {
+ const transport = createMockTransport();
+ const client = createBrowserClient({ namespace: "t", transport });
+ await client.start();
+ await client.newTab("https://a.test");
+ await client.newTab("https://b.test");
+ const ownedBefore = client.ownership().list();
+ check(ownedBefore.length >= 2, "several owned tabs before teardown");
+
+ await client.closeOwnedTabs();
+ check(client.ownership().list().length === 0, "closeOwnedTabs clears the owned set");
+ check(client.ownership().harnessWindowId() === undefined, "closeOwnedTabs clears the windowId binding");
+ check(ownedBefore.every((id) => transport.closed.includes(id)), "every owned tab was actually closed in the browser");
+ }
+
+ console.log(`\n${passed} passed, ${failed} failed`);
+ process.exit(failed === 0 ? 0 : 1);
+}
+
+void main();