diff options
| author | Aman Kumar <145339995+amankumarsingh77@users.noreply.github.com> | 2026-07-22 00:59:58 +0530 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-07-22 00:59:58 +0530 |
| commit | fe00d3361dece3d35f788d3eeb1c750eedbf6294 (patch) | |
| tree | a51b8ffe3cddb1aa8d457784f71cc1b2b0b61d56 | |
| parent | 7de0beaf2cf713afc50c573e941fa279ec6df9d9 (diff) | |
feat: add forms domain and durable window binding (v0.9.0) (#14)
- Add forms domain with a universal field setter that auto-detects element
type and fires framework-compatible input events (React native-setter trick)
- Add window-binding helper to capture and bind the real Chrome windowId as the
session's durable window identity on every new-window code path
- Wire supporting changes across CDP session/ownership, client, navigate,
isolated-tab, prompt, and state
- Add manual tests for forms and window binding
- Bump version 0.8.3 -> 0.9.0
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
| -rw-r--r-- | .pi/settings.json | 6 | ||||
| -rw-r--r-- | package-lock.json | 4 | ||||
| -rw-r--r-- | package.json | 2 | ||||
| -rw-r--r-- | src/cdp/ownership.ts | 20 | ||||
| -rw-r--r-- | src/cdp/session.ts | 55 | ||||
| -rw-r--r-- | src/cdp/window-binding.ts | 15 | ||||
| -rw-r--r-- | src/client.ts | 61 | ||||
| -rw-r--r-- | src/domains/forms.ts | 355 | ||||
| -rw-r--r-- | src/domains/isolated-tab.ts | 6 | ||||
| -rw-r--r-- | src/domains/navigate.ts | 35 | ||||
| -rw-r--r-- | src/index.ts | 17 | ||||
| -rw-r--r-- | src/prompt.ts | 208 | ||||
| -rw-r--r-- | src/state.ts | 1 | ||||
| -rw-r--r-- | test/manual/forms-test.ts | 149 | ||||
| -rw-r--r-- | test/manual/window-binding-test.ts | 239 |
15 files changed, 1027 insertions, 146 deletions
diff --git a/.pi/settings.json b/.pi/settings.json index 59c7b93..d2f3b27 100644 --- a/.pi/settings.json +++ b/.pi/settings.json @@ -1,4 +1,4 @@ { - "extensions": ["../src/index.ts"], - "skills": ["../skills"] -} + "extensions": ["./src/index.ts"], + "skills":["./skills"] +}
\ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 6542fd6..25e866c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-browser-harness", - "version": "0.8.3", + "version": "0.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-browser-harness", - "version": "0.8.3", + "version": "0.9.0", "license": "MIT", "dependencies": { "ws": "^8.20.0" diff --git a/package.json b/package.json index 1efe449..780db4f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-browser-harness", - "version": "0.8.3", + "version": "0.9.0", "description": "Browser control extension for pi — navigate, click, type, screenshot, and extract data from real Chrome via CDP", "keywords": [ "pi-package", diff --git a/src/cdp/ownership.ts b/src/cdp/ownership.ts index f221d6a..727b3d8 100644 --- a/src/cdp/ownership.ts +++ b/src/cdp/ownership.ts @@ -20,14 +20,24 @@ export type OwnershipRegistry = { replaceAll(ids: ReadonlyArray<string>): void; setHarnessWindow(targetId: string | undefined): void; harnessWindow(): string | undefined; + /** The real Chrome windowId of the dedicated harness window — the durable + * identity of "the window this session initialized", surviving seed-tab + * closure. Undefined until the window is created (or after teardown). */ + setHarnessWindowId(windowId: number | undefined): void; + harnessWindowId(): number | undefined; onChange(cb: () => void): void; }; export const createOwnershipRegistry = ( - initial?: { readonly ownedTargetIds?: ReadonlyArray<string>; readonly harnessWindowTargetId?: string }, + initial?: { + readonly ownedTargetIds?: ReadonlyArray<string>; + readonly harnessWindowTargetId?: string; + readonly harnessWindowId?: number; + }, ): OwnershipRegistry => { const owned = new Set<string>(initial?.ownedTargetIds ?? []); let harnessWindow: string | undefined = initial?.harnessWindowTargetId; + let harnessWindowId: number | undefined = initial?.harnessWindowId; let listener: (() => void) | null = null; const notify = (): void => { @@ -65,6 +75,14 @@ export const createOwnershipRegistry = ( harnessWindow() { return harnessWindow; }, + setHarnessWindowId(windowId) { + if (harnessWindowId === windowId) return; + harnessWindowId = windowId; + notify(); + }, + harnessWindowId() { + return harnessWindowId; + }, onChange(cb) { listener = cb; }, diff --git a/src/cdp/session.ts b/src/cdp/session.ts index e090858..03410ba 100644 --- a/src/cdp/session.ts +++ b/src/cdp/session.ts @@ -92,6 +92,17 @@ export const createCdpSession = ( const tab = targetId ? tabs.get(targetId) : undefined; if (tab) tab.pageInfoDirty = true; } + if (ev.method === "Target.targetCreated" && ownership) { + // Adopt tabs opened BY a tab we already own (window.open, target=_blank, + // popups). Chrome sets openerId to the opener target; if that opener is + // owned, the child belongs to this session's window and must be + // controllable + cleaned up. Targets the user opens themselves have no + // owned opener, so they are never adopted. + const info = (ev.params as { targetInfo?: { targetId?: string; type?: string; openerId?: string } } | undefined)?.targetInfo; + if (info?.type === "page" && info.targetId && info.openerId && ownership.has(info.openerId)) { + ownership.add(info.targetId); + } + } if (ev.method === "Target.targetDestroyed" && ownership) { const params = ev.params as { targetId?: string } | undefined; if (params?.targetId) { @@ -179,6 +190,43 @@ export const createCdpSession = ( if (survivors.length !== ownership.list().length) ownership.replaceAll(survivors); const hw = ownership.harnessWindow(); if (hw && !live.has(hw)) ownership.setHarnessWindow(undefined); + + if (survivors.length === 0) { + // No owned tab survived — the harness window is gone (user closed it, + // or Chrome restarted and reassigned all ids). The persisted windowId + // is now meaningless and could even collide with one of the user's + // windows, so drop it. attachFirstPage will mint a fresh window below. + ownership.setHarnessWindowId(undefined); + } else { + // Self-heal ownership from the window itself. We anchor on a tab we + // KNOW we own (a survivor) and re-adopt only its live window-mates — + // popups/children from last session, tabs reopened in our window. + // Ownership is thus DERIVED from the real window, never from the bare + // persisted integer (which Chrome may reassign to a user window on + // restart — anchoring on a survivor makes that misfire impossible). + const anchorId = survivors[0]!; + const anchorWin = await transport.request("Browser.getWindowForTarget", { targetId: anchorId }, { sessionId: null }); + if (anchorWin.success) { + const anchorWindowId = (anchorWin.data as { windowId?: number }).windowId; + if (typeof anchorWindowId === "number") { + ownership.setHarnessWindowId(anchorWindowId); // refresh to the live id + const windows = await Promise.all( + allPages.map((p) => + p.targetId === anchorId + ? Promise.resolve(anchorWin) + : transport.request("Browser.getWindowForTarget", { targetId: p.targetId }, { sessionId: null }), + ), + ); + allPages.forEach((p, i) => { + const r = windows[i]; + if (r?.success && (r.data as { windowId?: number }).windowId === anchorWindowId) { + ownership.add(p.targetId); + } + }); + } + } + if (!ownership.harnessWindow()) ownership.setHarnessWindow(anchorId); + } } // Prune per-tab state for pages that no longer exist const liveTargetIds = new Set(allPages.map((p) => p.targetId)); @@ -208,6 +256,13 @@ export const createCdpSession = ( if (ownership) { ownership.setHarnessWindow(c.targetId); ownership.add(c.targetId); + // Capture the real Chrome windowId — the durable identity of the + // window this session initialized (survives seed-tab closure). + const win = await transport.request("Browser.getWindowForTarget", { targetId: c.targetId }, { sessionId: null }); + if (win.success) { + const wid = (win.data as { windowId?: number }).windowId; + if (typeof wid === "number") ownership.setHarnessWindowId(wid); + } } } diff --git a/src/cdp/window-binding.ts b/src/cdp/window-binding.ts new file mode 100644 index 0000000..4ae1ac3 --- /dev/null +++ b/src/cdp/window-binding.ts @@ -0,0 +1,15 @@ +import type { BrowserClient } from "../client"; + +/** + * Capture the real Chrome windowId of a freshly-created harness window and bind + * it as the session's durable window identity. Best-effort: a failed CDP call + * leaves the previous binding untouched. Call this on every code path that + * spawns a NEW dedicated window (newWindow:true), so the windowId never points + * at a window that has since been replaced. + */ +export const bindHarnessWindowId = async (client: BrowserClient, targetId: string): Promise<void> => { + const r = await client.session().callBrowser("Browser.getWindowForTarget", { targetId }); + if (!r.success) return; + const wid = (r.data as { windowId?: number }).windowId; + if (typeof wid === "number") client.ownership().setHarnessWindowId(wid); +}; diff --git a/src/client.ts b/src/client.ts index 724a711..9a97369 100644 --- a/src/client.ts +++ b/src/client.ts @@ -16,10 +16,12 @@ export type BrowserClientOptions = { readonly initialOwnership?: { readonly ownedTargetIds?: ReadonlyArray<string>; readonly harnessWindowTargetId?: string; + readonly harnessWindowId?: number; }; readonly onOwnershipChange?: (snapshot: { readonly ownedTargetIds: ReadonlyArray<string>; readonly harnessWindowTargetId: string | undefined; + readonly harnessWindowId: number | undefined; }) => void; }; @@ -32,6 +34,9 @@ export type BrowserClient = { /** Detach from the current page target (removes the "Chrome is being controlled" banner) * while keeping the transport connection alive. Call on session shutdown. */ detach(): Promise<void>; + /** Close every tab this session owns and clear the window binding. Best-effort — + * used on session shutdown so no stale harness tabs survive. */ + closeOwnedTabs(): Promise<void>; evaluateJs(expression: string, sessionId?: string): Promise<Result<unknown, CdpError>>; pageInfo(): Promise<Result<PageInfo | { readonly dialog: DialogInfo }, CdpError>>; takeDialog(): DialogInfo | null; @@ -82,13 +87,16 @@ const parsePageInfoPayload = (v: unknown): Result<PageInfo, CdpError> => { export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient => { const transport = opts.transport ?? createCdpTransport(); - const ownershipInit: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string } = {}; + const ownershipInit: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string; harnessWindowId?: number } = {}; if (opts.initialOwnership?.ownedTargetIds !== undefined) { ownershipInit.ownedTargetIds = opts.initialOwnership.ownedTargetIds; } if (opts.initialOwnership?.harnessWindowTargetId !== undefined) { ownershipInit.harnessWindowTargetId = opts.initialOwnership.harnessWindowTargetId; } + if (opts.initialOwnership?.harnessWindowId !== undefined) { + ownershipInit.harnessWindowId = opts.initialOwnership.harnessWindowId; + } const ownership = createOwnershipRegistry(ownershipInit); if (opts.onOwnershipChange) { const cb = opts.onOwnershipChange; @@ -96,6 +104,7 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient = cb({ ownedTargetIds: ownership.list(), harnessWindowTargetId: ownership.harnessWindow(), + harnessWindowId: ownership.harnessWindowId(), }); }); } @@ -256,6 +265,15 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient = return ok(undefined); }; + // Ask Chrome for the real windowId hosting a target. Undefined on failure — + // callers treat the window binding as best-effort. + const getWindowId = async (targetId: string): Promise<number | undefined> => { + const r = await session.callBrowser("Browser.getWindowForTarget", { targetId }); + if (!r.success) return undefined; + const wid = (r.data as { windowId?: number }).windowId; + return typeof wid === "number" ? wid : undefined; + }; + const newTab = async (url?: string): Promise<Result<string, CdpError>> => { // Verify the recorded harness window still exists. Querying the live // target list also gives listTabs's reconciliation a chance to run. @@ -263,22 +281,35 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient = if (!tabsResult.success) return tabsResult; const live = new Set(tabsResult.data.map((t) => t.targetId)); const hw = ownership.harnessWindow(); + // Pick an opener that keeps us inside the harness window. Prefer the seed + // tab; if the user closed it but other owned tabs survive, reuse one of + // those instead of spawning a second window (avoids window scatter). + const opener = hw && live.has(hw) + ? hw + : ownership.list().find((id) => live.has(id)); const params: Record<string, unknown> = { url: "about:blank" }; - if (hw && live.has(hw)) { - // Open as a child of the harness window's seed tab — Chrome places - // it in the same window, giving us visual grouping for free. - params["openerId"] = hw; + if (opener) { + // Open as a child of an owned tab — Chrome places it in the same window, + // giving us visual grouping for free and keeping the session single-window. + params["openerId"] = opener; } else { - // Either fresh session or the harness window was closed by the user. - // Spawn a new dedicated window. + // Fresh session, or the whole harness window was closed by the user. + // Spawn a new dedicated window and rebind to it. params["newWindow"] = true; } const created = await session.callBrowser("Target.createTarget", params); if (!created.success) return created; const c = created.data as { targetId: string }; - if (params["newWindow"]) ownership.setHarnessWindow(c.targetId); + if (params["newWindow"]) { + ownership.setHarnessWindow(c.targetId); + const wid = await getWindowId(c.targetId); + if (wid !== undefined) ownership.setHarnessWindowId(wid); + } else if (opener && opener !== hw) { + // Re-seeded onto a surviving owned tab; keep the seed pointer live. + ownership.setHarnessWindow(opener); + } ownership.add(c.targetId); const switched = await switchTab(c.targetId); @@ -297,6 +328,18 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient = return ok(undefined); }; + const closeOwnedTabs = async (): Promise<void> => { + // Snapshot first — remove() mutates the list, and targetDestroyed events + // may also prune concurrently. Failures are swallowed: a tab the user + // already closed, or a dead transport, must not block shutdown. + for (const id of ownership.list()) { + await session.callBrowser("Target.closeTarget", { targetId: id }).catch(() => {}); + ownership.remove(id); + } + ownership.setHarnessWindow(undefined); + ownership.setHarnessWindowId(undefined); + }; + const status = (): DaemonStatus => ({ alive: transport.state() === "open" && session.current() !== null, sessionId: session.current()?.sessionId ?? null, @@ -316,7 +359,7 @@ export const createBrowserClient = (opts: BrowserClientOptions): BrowserClient = return { namespace: opts.namespace, - ensureAlive, status, start, stop, detach, + ensureAlive, status, start, stop, detach, closeOwnedTabs, evaluateJs, pageInfo, takeDialog: () => session.takeDialog(), listTabs, switchTab, newTab, closeTab, diff --git a/src/domains/forms.ts b/src/domains/forms.ts new file mode 100644 index 0000000..1d83ba6 --- /dev/null +++ b/src/domains/forms.ts @@ -0,0 +1,355 @@ +import { Type } from "typebox"; +import type { BrowserClient } from "../client"; +import { type Result, err, ok } from "../util/result"; +import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; + +// ── Page-side functions ────────────────────────────────────────────────────── +// These run inside the page via Runtime.callFunctionOn with `this` bound to the +// resolved element. They are NOT string-interpolated with user data — the value +// is passed as a structured `arguments` entry, so safeJs is unnecessary here. + +/** + * Universal field setter. Auto-detects the element type and applies the right + * strategy, then reads the resulting value back so the caller can confirm. + * + * The <input>/<textarea> branch uses the "native setter" trick: React (and + * other frameworks) overload the element's `value` setter to track state, so a + * plain `el.value = x` updates the DOM but not the framework. Calling the + * prototype's original setter and then dispatching a bubbling `input` event is + * what makes React's onChange fire. + */ +const FILL_FN = ` +function(value) { + const el = this; + if (!el || el.nodeType !== 1) return { ok: false, reason: "ref does not point to an element" }; + if (el.disabled) return { ok: false, reason: "element is disabled" }; + const tag = (el.tagName || "").toLowerCase(); + const type = (el.type || "").toLowerCase(); + try { el.focus(); } catch (e) {} + const fire = function(t) { el.dispatchEvent(new Event(t, { bubbles: true })); }; + + if (tag === "select") { + const want = String(value); + let matched = null; + for (let i = 0; i < el.options.length; i++) { + const o = el.options[i]; + if (o.value === want || o.label === want || o.text === want) { matched = o; break; } + } + if (!matched) { + const opts = []; + for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text }); + return { ok: false, reason: "no matching option", kind: "select", options: opts }; + } + el.value = matched.value; + fire("input"); fire("change"); + return { ok: true, kind: "select", value: el.value, text: matched.text }; + } + + if (tag === "input" && (type === "checkbox" || type === "radio")) { + const want = value === true || value === "true" || value === "on" || value === 1; + if (el.checked !== want) { + try { el.click(); } catch (e) {} + if (el.checked !== want) { el.checked = want; fire("input"); fire("change"); } + } + return { ok: true, kind: type, checked: el.checked }; + } + + if (el.isContentEditable) { + el.textContent = String(value); + fire("input"); fire("change"); + return { ok: true, kind: "contenteditable", value: el.textContent }; + } + + if (tag === "input" || tag === "textarea") { + const proto = tag === "textarea" ? window.HTMLTextAreaElement.prototype : window.HTMLInputElement.prototype; + const desc = Object.getOwnPropertyDescriptor(proto, "value"); + if (desc && desc.set) desc.set.call(el, String(value)); + else el.value = String(value); + fire("input"); fire("change"); + return { ok: true, kind: tag, value: el.value }; + } + + return { ok: false, reason: "element is not a fillable field (tag=" + tag + ")" }; +}`; + +/** Specialized <select> setter — matches by value first, then by label/text. */ +const SELECT_FN = ` +function(value, label) { + const el = this; + if (!el || (el.tagName || "").toLowerCase() !== "select") return { ok: false, reason: "ref is not a <select> element" }; + if (el.disabled) return { ok: false, reason: "select is disabled" }; + let matched = null; + for (let i = 0; i < el.options.length; i++) { + const o = el.options[i]; + if (value != null && o.value === String(value)) { matched = o; break; } + if (label != null && (o.text === String(label) || o.label === String(label))) { matched = o; break; } + } + if (!matched) { + const opts = []; + for (let i = 0; i < el.options.length; i++) opts.push({ value: el.options[i].value, text: el.options[i].text }); + return { ok: false, reason: "no matching option", options: opts }; + } + try { el.focus(); } catch (e) {} + el.value = matched.value; + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + return { ok: true, value: el.value, text: matched.text }; +}`; + +/** Specialized checkbox/radio setter — only acts when the state differs. */ +const CHECK_FN = ` +function(checked) { + const el = this; + const tag = (el.tagName || "").toLowerCase(); + const type = (el.type || "").toLowerCase(); + if (tag !== "input" || (type !== "checkbox" && type !== "radio")) return { ok: false, reason: "ref is not a checkbox or radio" }; + if (el.disabled) return { ok: false, reason: "element is disabled" }; + const want = checked === true; + let changed = false; + if (el.checked !== want) { + try { el.click(); } catch (e) {} + if (el.checked !== want) { el.checked = want; el.dispatchEvent(new Event("change", { bubbles: true })); } + changed = true; + } + return { ok: true, checked: el.checked, changed: changed }; +}`; + +// ── Resolver ───────────────────────────────────────────────────────────────── + +/** Shape returned by the page-side functions above. */ +type PageResult = { + readonly ok: boolean; + readonly reason?: string; + readonly kind?: string; + readonly value?: unknown; + readonly text?: string; + readonly checked?: boolean; + readonly changed?: boolean; + readonly options?: ReadonlyArray<{ value: string; text: string }>; +}; + +/** + * Resolve a ref (CDP backendDOMNodeId) to a live JS handle and invoke a + * page-side function on it. DOM + Runtime are enabled on every session + * (see cdp/session.ts enableDomains), so no extra setup is required. + */ +const resolveAndCall = async ( + client: BrowserClient, + ref: number, + functionDeclaration: string, + args: ReadonlyArray<unknown>, +): Promise<Result<PageResult, ToolErr>> => { + const session = client.session(); + + const resolved = await session.call("DOM.resolveNode", { backendNodeId: ref }); + if (!resolved.success) { + return err({ + kind: "invalid_state", + message: `ref ${ref} could not be resolved — the page may have changed. Re-run browser_snapshot to get fresh refs.`, + details: { ref }, + }); + } + const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId; + if (objectId === undefined) { + return err({ + kind: "invalid_state", + message: `ref ${ref} resolved to no JS object — re-run browser_snapshot to get fresh refs.`, + details: { ref }, + }); + } + + try { + const called = await session.call("Runtime.callFunctionOn", { + objectId, + functionDeclaration, + arguments: args.map((v) => ({ value: v })), + returnByValue: true, + awaitPromise: true, + }); + if (!called.success) return err({ kind: "cdp_error", message: called.error.message, details: { ref } }); + const data = called.data as { result?: { value?: unknown }; exceptionDetails?: unknown }; + if (data.exceptionDetails !== undefined) { + return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(data.exceptionDetails)}`, details: { ref } }); + } + const value = data.result?.value; + if (typeof value !== "object" || value === null) { + return err({ kind: "internal", message: "page function returned no result object", details: { ref } }); + } + return ok(value as PageResult); + } finally { + // Best-effort: release the remote handle so it can be GC'd. Failure is + // harmless (the page may have navigated), so we ignore the result. + await session.call("Runtime.releaseObject", { objectId }); + } +}; + +/** Drop the `ok` flag before stashing a page result in tool `details` — the + * tool-result envelope reserves `ok` for its own success discriminant. */ +const detailsOf = (ref: number, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => { + const { ok: _ok, ...rest } = res; + return { ref, ...rest, ...(extra ?? {}) }; +}; + +// ── Tools ──────────────────────────────────────────────────────────────────── + +const FieldValue = Type.Union([Type.String(), Type.Boolean()], { + description: "Value to set. String for text inputs / textareas / selects / contenteditable; boolean for checkboxes & radios.", +}); + +const FillArgs = Type.Object({ + ref: Type.Integer({ description: "Element ref from browser_snapshot ([ref=N])." }), + value: FieldValue, +}); + +export const fillTool = defineBrowserTool({ + name: "browser_fill", + label: "Browser Fill", + description: + "Fill a single form field by ref. Auto-detects input/textarea/select/checkbox/radio/contenteditable, clears then sets the value, fires input+change events (drives React/Vue controlled inputs correctly), and reads the value back to confirm. Get the ref from browser_snapshot's [ref=N].", + promptSnippet: "Fill a form field by ref (text, select, checkbox) — React-safe, self-confirming", + promptGuidelines: [ + "Get the ref from browser_snapshot's [ref=N] for the target field. No click-to-focus needed.", + "value is a string for text fields/selects/contenteditable, or a boolean for checkboxes/radios.", + "The result reports the value the field actually holds after filling — check it matches what you intended.", + "For autocomplete/typeahead that needs real keystrokes, click the field's @(x,y) then use browser_type instead.", + "If a ref no longer resolves, re-run browser_snapshot to get fresh refs.", + ], + parameters: FillArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const r = await resolveAndCall(client, args.ref, FILL_FN, [args.value]); + if (!r.success) return r; + const res = r.data; + if (!res.ok) { + return err({ kind: "invalid_state", message: `Could not fill ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) }); + } + const wanted = typeof args.value === "boolean" ? undefined : String(args.value); + const got = res.value !== undefined ? String(res.value) : undefined; + const mismatch = wanted !== undefined && got !== undefined && got !== wanted; + let text: string; + if (mismatch) { + text = `Filled ref ${args.ref}, but it now reads ${JSON.stringify(got)} (wanted ${JSON.stringify(wanted)})`; + } else if (got !== undefined) { + text = `Filled ref ${args.ref} = ${JSON.stringify(got)}`; + } else if (res.checked !== undefined) { + text = `Filled ref ${args.ref} (checked=${res.checked})`; + } else { + text = `Filled ref ${args.ref}`; + } + return ok({ text, details: detailsOf(args.ref, res, { mismatch }) }); + }, +}); + +const FillFormArgs = Type.Object({ + fields: Type.Array( + Type.Object({ + ref: Type.Integer({ description: "Element ref from browser_snapshot ([ref=N])." }), + value: FieldValue, + }), + { minItems: 1, description: "Fields to fill in one batch, each by ref." }, + ), +}); + +export const fillFormTool = defineBrowserTool({ + name: "browser_fill_form", + label: "Browser Fill Form", + description: + "Fill multiple form fields in one call — the efficient way to complete a form. Each field is { ref, value }; refs come from browser_snapshot's [ref=N]. Handles text inputs, textareas, selects, checkboxes, radios, and contenteditable, drives React/Vue controlled inputs correctly, and reports the resulting value of every field.", + promptSnippet: "Fill many form fields at once by ref — React-safe, self-confirming", + promptGuidelines: [ + "Preferred for forms: snapshot once, then fill all fields in a single browser_fill_form call.", + "Each field: { ref, value }. value is a string for text/select/contenteditable, boolean for checkbox/radio.", + "The result summarizes how many fields filled cleanly and flags any value mismatches or per-field errors.", + "Re-run browser_snapshot afterwards to verify the form state if needed.", + ], + parameters: FillFormArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const results: Array<Readonly<Record<string, unknown>>> = []; + let okCount = 0; + const issues: string[] = []; + for (const f of args.fields) { + const r = await resolveAndCall(client, f.ref, FILL_FN, [f.value]); + if (!r.success) { + results.push({ ref: f.ref, ok: false, error: r.error.message }); + issues.push(`ref ${f.ref} (${r.error.message})`); + continue; + } + const res = r.data; + const wanted = typeof f.value === "boolean" ? undefined : String(f.value); + const got = res.value !== undefined ? String(res.value) : undefined; + const mismatch = wanted !== undefined && got !== undefined && got !== wanted; + const fieldOk = res.ok === true && !mismatch; + if (fieldOk) okCount++; + else issues.push(`ref ${f.ref} (${res.ok === false ? (res.reason ?? "failed") : "value mismatch"})`); + results.push({ ref: f.ref, ok: fieldOk, mismatch, ...detailsOf(f.ref, res) }); + } + const text = + `Filled ${okCount}/${args.fields.length} fields` + + (issues.length > 0 ? `; issues: ${issues.join(", ")}` : ""); + return ok({ text, details: { results } }); + }, +}); + +const SelectOptionArgs = Type.Object({ + ref: Type.Integer({ description: "Element ref of a <select> from browser_snapshot ([ref=N])." }), + value: Type.Optional(Type.String({ description: "Option value attribute to select." })), + label: Type.Optional(Type.String({ description: "Visible option text/label to select (used if value is omitted)." })), +}); + +export const selectOptionTool = defineBrowserTool({ + name: "browser_select_option", + label: "Browser Select Option", + description: + "Select an option in a native <select> dropdown by ref. Matches by option value, then by visible label/text, and fires change. Native selects cannot be driven by clicking coordinates — use this instead. Get the ref from browser_snapshot's [ref=N].", + promptSnippet: "Choose an option in a <select> dropdown by ref", + promptGuidelines: [ + "Provide value (the option's value attribute) or label (the visible text). value wins if both given.", + "On no match, the result lists the available options so you can retry with a valid one.", + "Use this for native <select> elements — clicking coordinates does not work for OS dropdowns.", + ], + parameters: SelectOptionArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + if (args.value === undefined && args.label === undefined) { + return err({ kind: "invalid_state", message: "Provide value or label to select an option." }); + } + const r = await resolveAndCall(client, args.ref, SELECT_FN, [args.value ?? null, args.label ?? null]); + if (!r.success) return r; + const res = r.data; + if (!res.ok) { + return err({ kind: "invalid_state", message: `Could not select option on ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) }); + } + return ok({ + text: `Selected ${JSON.stringify(res.text)} (value=${JSON.stringify(res.value)}) on ref ${args.ref}`, + details: detailsOf(args.ref, res), + }); + }, +}); + +const SetCheckedArgs = Type.Object({ + ref: Type.Integer({ description: "Element ref of a checkbox or radio from browser_snapshot ([ref=N])." }), + checked: Type.Boolean({ description: "Desired checked state." }), +}); + +export const setCheckedTool = defineBrowserTool({ + name: "browser_set_checked", + label: "Browser Set Checked", + description: + "Set a checkbox or radio to a desired checked state by ref. Only acts if the current state differs, fires change, and reports the final state. Get the ref from browser_snapshot's [ref=N].", + promptSnippet: "Set a checkbox/radio checked state by ref", + promptGuidelines: [ + "checked:true ticks the box, checked:false unticks it. Idempotent — no-op if already in the desired state.", + "For radios, set checked:true on the option you want; the group deselects the others.", + ], + parameters: SetCheckedArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const r = await resolveAndCall(client, args.ref, CHECK_FN, [args.checked]); + if (!r.success) return r; + const res = r.data; + if (!res.ok) { + return err({ kind: "invalid_state", message: `Could not set checked on ref ${args.ref}: ${res.reason ?? "unknown reason"}`, details: detailsOf(args.ref, res) }); + } + return ok({ + text: `Set ref ${args.ref} checked=${res.checked}` + (res.changed === false ? " (already)" : ""), + details: detailsOf(args.ref, res), + }); + }, +}); diff --git a/src/domains/isolated-tab.ts b/src/domains/isolated-tab.ts index c759cec..73ed158 100644 --- a/src/domains/isolated-tab.ts +++ b/src/domains/isolated-tab.ts @@ -13,6 +13,7 @@ import type { BrowserClient } from "../client"; import { type Result, err, ok } from "../util/result"; import { safeJs } from "../util/js-template"; +import { bindHarnessWindowId } from "../cdp/window-binding"; import type { ToolErr } from "../util/tool"; /** A tab attached by its own sessionId, ready to drive via callOnTarget. */ @@ -40,7 +41,10 @@ export const openIsolatedTab = async (client: BrowserClient): Promise<Result<Iso if (!created.success) return err(toToolErr(created.error.message)); const { targetId } = created.data as { targetId: string }; - if (createParams["newWindow"]) client.ownership().setHarnessWindow(targetId); + if (createParams["newWindow"]) { + client.ownership().setHarnessWindow(targetId); + await bindHarnessWindowId(client, targetId); + } client.ownership().add(targetId); const attached = await client.session().callBrowser("Target.attachToTarget", { targetId, flatten: true }); diff --git a/src/domains/navigate.ts b/src/domains/navigate.ts index 4477485..e402efc 100644 --- a/src/domains/navigate.ts +++ b/src/domains/navigate.ts @@ -4,6 +4,7 @@ import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { applyTruncation } from "../util/truncate"; import { safeJs } from "../util/js-template"; +import { bindHarnessWindowId } from "../cdp/window-binding"; const NavigateArgs = Type.Object({ url: Type.String({ description: "Full URL to navigate to (e.g. https://github.com)" }), @@ -75,7 +76,7 @@ export const openUrlsTool = defineBrowserTool({ description: "Open multiple URLs in parallel new tabs. Returns per-URL outcomes.", promptSnippet: "Open multiple URLs in new tabs (parallel)", promptGuidelines: [ - "Use after web_search to open citations in parallel.", + "Use after browser_web_search to open citations in parallel.", "After opening, use browser_list_tabs / browser_switch_tab to navigate, then browser_snapshot to inspect each tab.", "Use browser_wait_for_load on a tab before extracting data from SPAs.", ], @@ -83,17 +84,31 @@ export const openUrlsTool = defineBrowserTool({ renderCall: (a) => new Text(`🌐 Opening ${a.urls.length} URL${a.urls.length !== 1 ? "s" : ""}…`, 0, 0), async handler(args, { client, onUpdate }): Promise<Result<ToolOk, ToolErr>> { const total = args.urls.length; - // Phase 1: create tabs in parallel. Own them so they appear in - // browser_list_tabs scope:'owned' and group in the harness window. - const hw = client.ownership().harnessWindow(); - const created = await Promise.all(args.urls.map(async (url): Promise<TabResult> => { - const params: Record<string, unknown> = { url: "about:blank" }; - if (hw) params["openerId"] = hw; - else params["newWindow"] = true; - const r = await client.session().callBrowser("Target.createTarget", params); + // Phase 1: create tabs. All must land in ONE window. If the harness window + // already exists, open every url as its child in parallel. If it doesn't + // (fresh/closed), create the FIRST url as a new dedicated window (binding + // its windowId) and open the rest as children of it — a parallel + // newWindow:true per url would otherwise scatter tabs across N windows. + let seed = client.ownership().harnessWindow(); + let seedResult: TabResult | undefined; + if (!seed) { + const url0 = args.urls[0]; + if (url0 === undefined) return err({ kind: "invalid_state", message: "no URLs provided" }); + const r0 = await client.session().callBrowser("Target.createTarget", { url: "about:blank", newWindow: true }); + if (!r0.success) return err({ kind: "cdp_error", message: r0.error.message }); + seed = (r0.data as { targetId: string }).targetId; + client.ownership().setHarnessWindow(seed); + await bindHarnessWindowId(client, seed); + client.ownership().add(seed); + seedResult = { url: url0, targetId: seed, ok: true }; + } + const seededWindow = seed; + const created = await Promise.all(args.urls.map(async (url, i): Promise<TabResult> => { + // Reuse the seed tab (created above) for the first url instead of duplicating it. + if (i === 0 && seedResult) return seedResult; + const r = await client.session().callBrowser("Target.createTarget", { url: "about:blank", openerId: seededWindow }); if (!r.success) return { url, targetId: "", ok: false, error: r.error.message }; const c = r.data as { targetId: string }; - if (params["newWindow"]) client.ownership().setHarnessWindow(c.targetId); client.ownership().add(c.targetId); return { url, targetId: c.targetId, ok: true }; })); diff --git a/src/index.ts b/src/index.ts index d5f3435..7109ea0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -109,9 +109,10 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void { if (!client) { const transport = createDaemonTransport(state.namespace); - const initialOwnership: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string } = {}; + const initialOwnership: { ownedTargetIds?: ReadonlyArray<string>; harnessWindowTargetId?: string; harnessWindowId?: number } = {}; if (state.ownedTargetIds !== undefined) initialOwnership.ownedTargetIds = state.ownedTargetIds; if (state.harnessWindowTargetId !== undefined) initialOwnership.harnessWindowTargetId = state.harnessWindowTargetId; + if (state.harnessWindowId !== undefined) initialOwnership.harnessWindowId = state.harnessWindowId; client = createBrowserClient({ namespace: state.namespace, @@ -124,6 +125,9 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void { ...(snap.harnessWindowTargetId !== undefined ? { harnessWindowTargetId: snap.harnessWindowTargetId } : {}), + ...(snap.harnessWindowId !== undefined + ? { harnessWindowId: snap.harnessWindowId } + : {}), }; // Ownership changes can fire from CDP events at any time, // including after session replacement when pi's ctx is stale. @@ -147,20 +151,27 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void { }); pi.on("session_shutdown", async () => { - persistState(pi, state); if (client) { try { // Detach from the page target (removes the "Chrome is being controlled" // banner) but keep the transport alive for the next session. await client.detach(); + // Close every tab this session opened so no stale harness tabs linger + // in the user's browser between sessions. This also clears the persisted + // ownership + window binding via onOwnershipChange, so the next session + // starts with a fresh dedicated window rather than reattaching to leftovers. + await client.closeOwnedTabs(); } catch (e) { - console.warn("[pi-browser-harness] client.detach() failed during shutdown:", e); + console.warn("[pi-browser-harness] browser teardown failed during shutdown:", e); } // ponytail: keep the transport alive across sessions. // Do NOT call client.stop() or null out client — the daemon connection // persists and eliminates the per-session "Allow Remote Debugging" prompt. // Only Chrome restart or daemon death triggers a new prompt. } + // Persist last — closeOwnedTabs mutates `state` through onOwnershipChange, + // so this captures the cleared ownership set. + persistState(pi, state); await cleanupTempDirs(); }); diff --git a/src/prompt.ts b/src/prompt.ts index 7483730..c336c0c 100644 --- a/src/prompt.ts +++ b/src/prompt.ts @@ -2,150 +2,126 @@ * System prompt guidance for browser usage. * * Injects a compact browser control reference into the pi system prompt. - * Per-tool guidelines are injected via promptGuidelines on each tool registration. - * This prompt provides only the cross-cutting patterns and tool reference table. + * This prompt carries only cross-cutting knowledge — the mental model, tool + * routing, workflows, and gotchas. Per-tool specifics (schemas, detailed usage) + * are injected separately via promptGuidelines on each tool registration, so this + * file intentionally does NOT restate full tool descriptions. */ export function getBrowserSystemPrompt(): string { return ` ## Browser Control -You have full control of a real Chrome browser via \`browser_*\` tools. -The browser connects to the user's running Chrome (not a headless instance). - -**These tools coexist with all other standard pi tools** — read, bash, edit, -write, and any other built-in tools are still available. Use browser_* tools -to interact with pages visually, and other tools alongside them. - -### Browser-Specific Tools - -| Tool | Purpose | -|------|---------| -| browser_snapshot | Accessibility-tree snapshot. Assigns every interactive element a stable **ref** (\`[eN]\`) plus \`@(x,y)\`. Refs are the primary handle for interaction tools — they survive re-renders, coordinates don't. | -| browser_screenshot | Capture a PNG or JPEG screenshot of the current page (rendered inline in the TUI; the LLM reads the image via the saved file path) | -| browser_click | Click an element by **ref** (preferred — re-resolves position, survives re-renders) or viewport x/y. Appends a compact diff of page changes. | -| browser_type | Type text into the focused element (real keystrokes; requires a focused field) | -| browser_fill | Fill an input/textarea by **ref** (preferred) or selector — updates React/Vue controlled components, verifies the value, appends a page-changes diff | -| browser_select_option | Select an option in a native <select> by **ref** (preferred) or selector, choosing by value/label/index (fires change) | -| browser_focus | Focus an element by **ref** (preferred) or selector (deterministic; use before browser_type) | -| browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) | -| browser_scroll | Scroll the page at coordinates | -| browser_navigate | Navigate to a URL (result details.outcome.kind is "in_place" or "new_tab_created") | -| browser_new_tab | Open a new tab, optionally navigate | -| browser_open_urls | Open multiple URLs in new tabs (parallel) | -| browser_go_back / browser_go_forward / browser_reload | History navigation | -| browser_page_info | Get page URL, title, viewport, scroll position, or dialog info | -| browser_list_tabs | List all open browser tabs | -| browser_current_tab | Get current tab info | -| browser_switch_tab | Switch to a different tab by targetId | -| browser_execute_js | Execute JavaScript and return the result | -| browser_http_get | Direct HTTP GET (outside browser, for APIs) | -| browser_console | Read JS errors / console output on the current tab (diagnostic — use when an action looks broken; pass sinceSeq from the previous nextCursor to see only new messages) | -| browser_wait | Wait N seconds | -| browser_wait_for | Wait until a selector/text appears (or a selector disappears) — use for SPA content before interacting | -| browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) | -| browser_handle_dialog | Accept or dismiss a JS dialog | -| browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) | - -### Temporary Scripts - -When the built-in tools aren't enough for a multi-step workflow, write a temporary -script to disk and execute it with browser_run_script. The script runs in the -harness process with direct access to the browser daemon and Node.js APIs. +You drive a **real Chrome** (the user's running instance, not headless) through +\`browser_*\` tools. These coexist with every standard pi tool — read, bash, edit, +write — so mix them freely: \`browser_*\` for the page, everything else alongside. -\`\`\` -write("/tmp/scrape-pages.js", \` - const results = []; - for (const url of params.urls) { - await daemon.cdp("Page.navigate", { url }); - await new Promise(r => setTimeout(r, 2000)); - const data = await daemon.evaluateJS("document.title"); - results.push({ url, title: data }); - } - return { content: [{ type: "text", text: JSON.stringify(results, null, 2) }] }; -\`) -browser_run_script("/tmp/scrape-pages.js", { urls: [...] }) -\`\`\` +### The core handle: refs -Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout. +Interaction is ref-first. Call \`browser_snapshot\` to get an accessibility tree +where every interactive element has a stable ref (\`[eN]\`) and \`@(x,y)\`. **Target +elements by ref, not by coordinates or guessed selectors** — refs are keyed to +element identity, so they survive the re-renders that React/Vue forms trigger on +every edit and save. Coordinates go stale; guessed selectors miss. -### Parallel Execution +- Prefer \`browser_fill\` for text — it fires input/change so controlled components + update, and returns the value for verification. Use \`browser_click\` + \`browser_type\` + only for keystroke-sensitive widgets (autocomplete, masked inputs). +- Mutating calls append a compact **"Page changes"** diff — read it to confirm the + change landed (form closed, new \`*[eN]\` appeared). +- \`"ref is stale"\` means the page changed: re-run \`browser_snapshot\` for fresh refs. -Observation tools (browser_screenshot, browser_page_info, browser_execute_js, browser_list_tabs, browser_http_get, etc.) can run in parallel with each other and with mutation tools. The harness automatically serializes mutation tools (click, type, scroll, navigate, switch_tab, etc.) so they never race on shared state. When operations are independent, emit them in the same turn for better performance. +### Choosing a tool -**Examples of safe parallel calls:** -\`\`\` -browser_screenshot() + browser_page_info() + browser_execute_js("document.title") -browser_http_get("https://api.example.com/data") + browser_click(x, y) -\`\`\` +| To… | Reach for | +|-----|-----------| +| See the page structure (for interaction) | \`browser_snapshot\` (refs) | +| See the page visually | \`browser_screenshot\` | +| Search the web | \`browser_web_search\` → ranked {title, url, snippet} | +| Read an article's clean text | \`browser_read_page\` (reader mode; url or owned tab) | +| Extract specific DOM values | \`browser_execute_js\` | +| Hit a JSON/API endpoint | \`browser_http_get\` (raw GET, outside the browser) | +| Click / fill / type / select / focus / press a key | \`browser_click\` · \`browser_fill\` · \`browser_type\` · \`browser_select_option\` · \`browser_focus\` · \`browser_press_key\` | +| Upload a file · drag · resize viewport | \`browser_upload_file\` · \`browser_drag_and_drop\` · \`browser_viewport_resize\` | +| Go to a URL / open many / manage tabs | \`browser_navigate\` · \`browser_new_tab\` · \`browser_open_urls\` · \`browser_list_tabs\` · \`browser_switch_tab\` · \`browser_go_back\`/\`go_forward\`/\`reload\` | +| Wait for content / load / a fixed delay | \`browser_wait_for\` (selector/text) · \`browser_wait_for_load\` (readyState) · \`browser_wait\` | +| Inspect page / handle a dialog | \`browser_page_info\` · \`browser_current_tab\` · \`browser_handle_dialog\` | +| Diagnose a broken action | \`browser_console\` (JS errors) · \`browser_get_network_log\` | +| Save output / config downloads | \`browser_print_to_pdf\` · \`browser_download\` | +| Anything the tools can't express | \`browser_run_script\` (see Extending) | -**Multi-agent note**: When multiple subagents use the browser, tab switching by one agent changes the active tab for all agents. Per-tab data (console buffers, network traces, dialogs, page info cache) is isolated — switching tabs does not destroy another agent's collected data. Call \`browser_current_tab\` before mutation tools to confirm you're on the expected tab. If \`browser_page_info\` returns a dialog, handle it with \`browser_handle_dialog\` promptly — dialogs block page interaction and are not queued across agents. +Each tool carries its own detailed guidelines — this table is only for routing. -### Common Patterns +### Workflows -**Navigation:** +**Research (search → read):** \`\`\` -browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot() +browser_web_search("query") // ranked links, no page content +→ browser_read_page(url) // clean article text for the top hits +// fall back to browser_open_urls + browser_execute_js only when reader mode misses \`\`\` -**Form filling (ref-first — this is the reliable path on React/SPA forms):** +**Form filling (ref-first — the reliable path on SPA forms):** \`\`\` -browser_wait_for({ selector: "#email" }) // ensure SPA field is rendered -browser_snapshot() // each interactive element gets a [eN] ref -→ browser_fill({ ref: "e7", value: "a@b.com" }) // target by ref, not a guessed selector -→ browser_select_option({ ref: "e9", label: "India" }) // native <select> by ref -→ browser_click({ ref: "e12" }) // e.g. Save — re-resolves position even if the form reflowed -// each mutating call appends a compact "Page changes" diff — read it to confirm -// the change landed (e.g. a save closed the form, a new field appeared as *[eN]). +browser_wait_for({ selector: "#email" }) // ensure the field is rendered +browser_snapshot() // interactive elements get [eN] refs +→ browser_fill({ ref: "e7", value: "a@b.com" }) +→ browser_select_option({ ref: "e9", label: "India" }) +→ browser_click({ ref: "e12" }) // re-resolves position even after reflow +// read each "Page changes" diff to confirm the step landed \`\`\` -**Why ref over selector/coordinates:** refs are keyed to the element's identity, so -they survive the re-renders that React/Vue forms trigger on every edit and save — -coordinates go stale and selectors you guess often don't match. If a call returns -"ref is stale", the page changed: re-run browser_snapshot for fresh refs. - -Prefer browser_fill for text inputs/textareas — it fires input/change so controlled -components update, and returns the field's value for verification. Use browser_click + -browser_type only for keystroke-sensitive widgets (autocomplete, masked inputs); -browser_type requires an already-focused field and errors if none is focused. -**Data extraction:** +**Navigate + verify:** \`\`\` -browser_execute_js("document.querySelector('.price').innerText") -// or for APIs: -browser_http_get("https://api.example.com/data") +browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot() \`\`\` -**Scrolling:** -\`\`\` -browser_screenshot() → browser_scroll({ deltaY: 500 }) → browser_screenshot() -\`\`\` -Note: deltaY follows W3C wheel-event convention: positive=down, negative=up. Default deltaY=300 scrolls down. +**Scroll:** \`browser_scroll({ deltaY: 500 })\` — W3C wheel convention: positive = down, +negative = up (default 300, down). -**Research Workflow (search + browser):** +### Parallelism + +Observation tools (\`browser_screenshot\`, \`browser_page_info\`, \`browser_execute_js\`, +\`browser_list_tabs\`, \`browser_http_get\`, \`browser_web_search\`, \`browser_read_page\`, …) +run in parallel with each other and with mutations. The harness serializes mutations +(click, type, scroll, navigate, switch_tab) so they never race. Emit independent calls +in one turn: \`\`\` -browser_navigate("https://google.com/search?q=...") → search engine in a tab -browser_open_urls(urls: ["url1", "url2", ...]) → open result pages in parallel tabs -browser_list_tabs() → see all open tabs with targetIds -browser_switch_tab(targetId: "...") → switch to a tab -browser_screenshot() → visually inspect the page -browser_execute_js("document.querySelector('.main').innerText") → extract content +browser_screenshot() + browser_page_info() + browser_execute_js("document.title") \`\`\` -**Temporary Scripts (extending the harness):** +**Multi-agent:** tab switching by one agent changes the active tab for all agents, +but per-tab data (console, network, dialogs) stays isolated. Call \`browser_current_tab\` +before a mutation to confirm you're on the expected tab. Handle any dialog reported by +\`browser_page_info\` promptly — dialogs block interaction and aren't queued across agents. + +**Session boundary & tab hygiene:** you operate only inside this session's own Chrome +window. \`browser_switch_tab\`/\`browser_close_tab\` refuse tabs this session didn't open, +and \`browser_list_tabs\` defaults to owned tabs — never reach into the user's other tabs +or windows. Close tabs when you're done with them: call \`browser_close_tab\` on any tab +you opened (via \`browser_new_tab\`/\`browser_open_urls\`) once you've extracted what you +need. Don't leave a pile of stale tabs behind — keep only what a later step still needs. +(Leftover tabs are closed automatically when the session ends, but close them yourself +as you go.) + +### Extending: temporary scripts + +When built-in tools can't express a multi-step workflow, write a script to a temp file +and run it with \`browser_run_script\`. It executes in the harness process with direct +daemon + Node access. \`\`\` -write("/tmp/extract.js", "...script with daemon access...") → browser_run_script("/tmp/extract.js") +write(<tempfile>.js, \` + const results = []; + for (const url of params.urls) { + await daemon.cdp("Page.navigate", { url }); + await new Promise(r => setTimeout(r, 2000)); + results.push({ url, title: await daemon.evaluateJS("document.title") }); + } + return { content: [{ type: "text", text: JSON.stringify(results, null, 2) }] }; +\`) +browser_run_script(<tempfile>.js, { urls: [...] }) \`\`\` - -### Additional Tools - -| Tool | Purpose | -|------|---------| -| browser_upload_file | Upload a file to a file input by **ref** (preferred) or selector (bypasses file picker) | -| browser_dispatch_key | Dispatch a DOM KeyboardEvent on an element by **ref** (preferred) or selector (for React/Vue inputs); returns details.matched | -| browser_download | Configure download directory and disable save-as prompts | -| browser_viewport_resize | Resize the viewport for responsive testing | -| browser_drag_and_drop | Perform drag-and-drop from one coordinate to another | -| browser_print_to_pdf | Print the current page to a PDF file | -| browser_get_network_log | Get buffered network request/response events | +Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, +Buffer, setTimeout, clearTimeout. Use a real temp path from the OS — don't hardcode \`/tmp\`. `; } diff --git a/src/state.ts b/src/state.ts index 0629f40..d4b1534 100644 --- a/src/state.ts +++ b/src/state.ts @@ -12,6 +12,7 @@ export type BrowserState = { readonly remoteBrowserId?: string; readonly ownedTargetIds?: ReadonlyArray<string>; readonly harnessWindowTargetId?: string; + readonly harnessWindowId?: number; }; export const defaultState = (namespace = "default"): BrowserState => ({ diff --git a/test/manual/forms-test.ts b/test/manual/forms-test.ts new file mode 100644 index 0000000..f1c79f6 --- /dev/null +++ b/test/manual/forms-test.ts @@ -0,0 +1,149 @@ +/** + * Manual integration test for the form-filling tools. + * + * Exercises browser_snapshot (ref surfacing) + browser_fill_form / browser_fill / + * browser_select_option / browser_set_checked against a data:-URL fixture that + * includes a plain input, a textarea, a "controlled" input (commits its model + * only on the `input` event — the realistic failure mode for naive value-setting), + * a native <select>, a checkbox, and a contenteditable. + * + * Requires the daemon running + a real Chrome (same as live-daemon-test.ts). + * Run: npx tsx test/manual/forms-test.ts + */ +import { createDaemonTransport } from "../../src/cdp/daemon-transport"; +import { createBrowserClient } from "../../src/client"; +import { fillTool, fillFormTool, selectOptionTool, setCheckedTool } from "../../src/domains/forms"; + +// NOTE: we intentionally do NOT import snapshotTool here — it pulls in pi-tui / +// pi-coding-agent value imports that don't resolve under bare tsx. Instead we +// derive refs from Accessibility.getFullAXTree's backendDOMNodeId, which is the +// exact value browser_snapshot surfaces as [ref=N]. The snapshot's [ref=N] +// rendering is covered by `npm run typecheck` + the real-page smoke test. + +let passed = 0; +let failed = 0; +const check = (cond: boolean, label: string): void => { + if (cond) { passed++; console.log(` ✓ ${label}`); } + else { failed++; console.error(` ✗ ${label}`); } +}; + +// Minimal handler context — the form/snapshot handlers only read `client`. +const mkCtx = (client: any): any => ({ + client, + signal: undefined, + onUpdate: () => {}, + extensionCtx: { cwd: process.cwd() }, +}); + +const FIXTURE = ` +<!doctype html><html><body> + <h1>Form fixture</h1> + <input id="plain" aria-label="plain" /> + <textarea id="ta" aria-label="ta"></textarea> + <input id="controlled" aria-label="controlled" /> + <span id="model"></span> + <select id="sel" aria-label="country"> + <option value="us">United States</option> + <option value="uk">United Kingdom</option> + <option value="ca">Canada</option> + </select> + <input type="checkbox" id="chk" aria-label="agree" /> + <div id="editable" role="textbox" aria-label="bio" contenteditable="true"></div> + <script> + // Controlled input: its "model" updates ONLY when an input event fires. + // A naive value-set that skips the input event leaves the model stale. + var model = ""; + var c = document.getElementById("controlled"); + c.addEventListener("input", function () { model = c.value; document.getElementById("model").textContent = model; }); + window.__model = function () { return model; }; + </script> +</body></html>`; + +// Resolve a ref by accessible name via the AX tree — the same backendDOMNodeId +// that browser_snapshot prints as [ref=N]. +const refByName = async (client: any, name: string): Promise<number | undefined> => { + const ax = await client.session().call("Accessibility.getFullAXTree", {}); + if (!ax.success) return undefined; + for (const n of (ax.data as any).nodes as any[]) { + if (n?.name?.value === name && typeof n.backendDOMNodeId === "number") return n.backendDOMNodeId; + } + return undefined; +}; + +async function main(): Promise<void> { + const transport = createDaemonTransport("pi-forms-test"); + const client = createBrowserClient({ namespace: "pi-forms-test", transport }); + const ctx = mkCtx(client); + + const started = await client.start(); + if (!started.success) { console.error("Could not start client:", started.error.message); process.exit(1); } + + const url = "data:text/html," + encodeURIComponent(FIXTURE); + const tab = await client.newTab(url); + check(tab.success, `Opened fixture tab: ${tab.success ? "ok" : tab.error.message}`); + // Give the inline script a beat to wire up listeners. + await new Promise((r) => setTimeout(r, 300)); + + // ── Resolve refs (the backendDOMNodeId browser_snapshot prints as [ref=N]) ── + const refs = { + plain: await refByName(client, "plain"), + ta: await refByName(client, "ta"), + controlled: await refByName(client, "controlled"), + sel: await refByName(client, "country"), + chk: await refByName(client, "agree"), + editable: await refByName(client, "bio"), + }; + for (const [k, v] of Object.entries(refs)) check(typeof v === "number", `ref present for ${k} (${v})`); + + // ── Batch fill text fields ────────────────────────────────────────────────── + const fill = await fillFormTool.handler({ + fields: [ + { ref: refs.plain!, value: "hello plain" }, + { ref: refs.ta!, value: "multi\nline" }, + { ref: refs.controlled!, value: "ctrl-value" }, + { ref: refs.editable!, value: "my bio" }, + ], + }, ctx); + check(fill.success, "browser_fill_form succeeded"); + if (fill.success) console.log(" " + fill.data.text); + + // ── Select + checkbox ─────────────────────────────────────────────────────── + const sel = await selectOptionTool.handler({ ref: refs.sel!, label: "Canada" }, ctx); + check(sel.success, `browser_select_option by label: ${sel.success ? sel.data.text : (sel as any).error.message}`); + const chk = await setCheckedTool.handler({ ref: refs.chk!, checked: true }, ctx); + check(chk.success, `browser_set_checked: ${chk.success ? chk.data.text : (chk as any).error.message}`); + + // ── Verify committed state via JS reads ───────────────────────────────────── + const read = async (expr: string): Promise<unknown> => { + const r = await client.evaluateJs(expr); + return r.success ? r.data : `ERR:${r.error.message}`; + }; + check((await read("document.getElementById('plain').value")) === "hello plain", "plain input value committed"); + check((await read("document.getElementById('ta').value")) === "multi\nline", "textarea value committed"); + // The controlled input's MODEL (input-event-driven) must reflect the fill — + // proves browser_fill fired the input event, not just set the DOM value. + check((await read("window.__model()")) === "ctrl-value", "controlled input model committed (input event fired)"); + check((await read("document.getElementById('editable').textContent")) === "my bio", "contenteditable text committed"); + check((await read("document.getElementById('sel').value")) === "ca", "select value committed"); + check((await read("document.getElementById('chk').checked")) === true, "checkbox checked committed"); + + // ── No-matching-option path ───────────────────────────────────────────────── + const badSel = await selectOptionTool.handler({ ref: refs.sel!, label: "Atlantis" }, ctx); + check(!badSel.success && (badSel as any).error.kind === "invalid_state", "select with no match → invalid_state error"); + + // ── Stale ref path ────────────────────────────────────────────────────────── + const staleRef = refs.plain!; + const nav = await client.session().call("Page.navigate", { url: "data:text/html," + encodeURIComponent("<body>gone</body>") }); + check(nav.success, "navigated away to invalidate refs"); + await new Promise((r) => setTimeout(r, 300)); + const stale = await fillTool.handler({ ref: staleRef, value: "x" }, ctx); + check(!stale.success && (stale as any).error.kind === "invalid_state", "fill on stale ref → invalid_state error"); + + await client.closeTab(tab.success ? (tab.data as string) : ""); + await client.stop(); + + console.log(`\n${passed} passed, ${failed} failed`); + process.exit(failed === 0 ? 0 : 1); +} + +main().catch((e) => { console.error(e); process.exit(1); }); diff --git a/test/manual/window-binding-test.ts b/test/manual/window-binding-test.ts new file mode 100644 index 0000000..d743ae5 --- /dev/null +++ b/test/manual/window-binding-test.ts @@ -0,0 +1,239 @@ +/** + * Unit test (mock transport) for window-scoped session binding + tab teardown. + * + * Covers the two fixes: + * 1. The session binds to a real Chrome windowId. New tabs land in that window; + * tabs opened by an owned page (popups / window.open) are auto-adopted so the + * session controls them; nothing outside the window is ever owned. + * 2. closeOwnedTabs() closes every owned tab and clears the window binding — the + * teardown the extension runs on session shutdown so no stale tabs survive. + * + * Run: npx tsx test/manual/window-binding-test.ts + */ +import { createBrowserClient } from "../../src/client"; +import { createOwnershipRegistry } from "../../src/cdp/ownership"; +import type { CdpTransport } from "../../src/cdp/transport"; +import type { CdpEvent } from "../../src/cdp/types"; +import { ok, type Result } from "../../src/util/result"; +import type { CdpError } from "../../src/cdp/errors"; + +let passed = 0; +let failed = 0; +const check = (cond: boolean, label: string): void => { + if (cond) { passed++; console.log(` ✓ ${label}`); } + else { failed++; console.error(` ✗ ${label}`); } +}; + +// ── A minimal in-memory CDP browser the mock transport drives. ────────────── +type FakeTarget = { targetId: string; type: string; url: string; title: string; windowId: number; openerId?: string }; + +const createMockTransport = (): CdpTransport & { + seed: (t: FakeTarget) => void; + targets: Map<string, FakeTarget>; + closed: ReadonlyArray<string>; +} => { + const targets = new Map<string, FakeTarget>(); + const closed: string[] = []; + let sessionSeq = 0; + let targetSeq = 0; + let windowSeq = 100; + + // Event queue plumbing so session.ts's consumeEvents() can await targetCreated. + const listeners: Array<() => void> = []; + let buffer: CdpEvent[] = []; + let waiter: (() => void) | null = null; + const push = (ev: CdpEvent): void => { + buffer.push(ev); + if (waiter) { const w = waiter; waiter = null; w(); } + }; + const iterable: AsyncIterable<CdpEvent> = { + [Symbol.asyncIterator]() { + return { + async next(): Promise<IteratorResult<CdpEvent>> { + if (buffer.length === 0) { + await new Promise<void>((r) => { waiter = r; }); + } + const v = buffer.shift()!; + return { value: v, done: false }; + }, + }; + }, + }; + + const request = async ( + method: string, + params: Record<string, unknown>, + ): Promise<Result<unknown, CdpError>> => { + switch (method) { + case "Target.setDiscoverTargets": + case "Page.enable": case "DOM.enable": case "Runtime.enable": + case "Network.enable": case "Accessibility.enable": case "Log.enable": + case "Target.activateTarget": + return ok({}); + case "Target.getTargets": + return ok({ targetInfos: [...targets.values()].map((t) => ({ targetId: t.targetId, type: t.type, url: t.url, title: t.title })) }); + case "Browser.getWindowForTarget": { + const t = targets.get(params["targetId"] as string); + return ok({ windowId: t?.windowId ?? -1, bounds: {} }); + } + case "Target.createTarget": { + const id = `tgt-${++targetSeq}`; + const opener = params["openerId"] as string | undefined; + const newWindow = params["newWindow"] === true; + const windowId = newWindow || !opener ? ++windowSeq : (targets.get(opener)?.windowId ?? ++windowSeq); + const t: FakeTarget = { targetId: id, type: "page", url: (params["url"] as string) ?? "about:blank", title: "", windowId, ...(opener ? { openerId: opener } : {}) }; + targets.set(id, t); + // Chrome fires targetCreated for every new target (discover:true). + push({ method: "Target.targetCreated", params: { targetInfo: { targetId: id, type: "page", url: t.url, openerId: opener } } }); + return ok({ targetId: id }); + } + case "Target.attachToTarget": + return ok({ sessionId: `sess-${++sessionSeq}` }); + case "Target.detachFromTarget": + return ok({}); + case "Target.closeTarget": { + const id = params["targetId"] as string; + if (targets.delete(id)) { closed.push(id); push({ method: "Target.targetDestroyed", params: { targetId: id } }); } + return ok({}); + } + case "Runtime.evaluate": + return ok({ result: { value: "complete" } }); + case "Page.navigate": + return ok({ frameId: "f1" }); + default: + return ok({}); + } + }; + + return { + connect: async () => ok(undefined), + close: async () => { for (const l of listeners) l(); }, + request, + events: () => iterable, + state: () => "open", + onClose: (cb) => { listeners.push(cb); return () => {}; }, + seed: (t) => targets.set(t.targetId, t), + targets, + get closed() { return closed; }, + }; +}; + +const tick = (): Promise<void> => new Promise((r) => setTimeout(r, 20)); + +async function main(): Promise<void> { + console.log("window-binding-test\n"); + + // ── Registry: windowId binding is stored + cleared. ─────────────────────── + { + const reg = createOwnershipRegistry(); + check(reg.harnessWindowId() === undefined, "windowId starts undefined"); + reg.setHarnessWindowId(101); + check(reg.harnessWindowId() === 101, "setHarnessWindowId stores the id"); + let notified = 0; + reg.onChange(() => { notified++; }); + reg.setHarnessWindowId(101); + check(notified === 0, "setting the same windowId does not notify"); + reg.setHarnessWindowId(undefined); + check(reg.harnessWindowId() === undefined && notified === 1, "clearing windowId notifies"); + } + + // ── newTab: creates the dedicated window and captures its real windowId. ── + { + const transport = createMockTransport(); + const client = createBrowserClient({ namespace: "t", transport }); + await client.start(); + const wid = client.ownership().harnessWindowId(); + check(typeof wid === "number", "attach captures a real windowId for the harness window"); + + const first = await client.newTab("https://a.test"); + check(first.success, "newTab succeeds"); + if (first.success) { + const t = transport.targets.get(first.data)!; + check(t.windowId === wid, "new tab lands in the harness window (via openerId)"); + check(client.owns(first.data), "new tab is owned"); + } + } + + // ── Popup adoption: a tab opened by an owned page is auto-owned. ────────── + { + const transport = createMockTransport(); + const client = createBrowserClient({ namespace: "t", transport }); + await client.start(); + const cur = client.current()!; + + // A page the session controls opens a child target (window.open / _blank). + // Chrome emits Target.targetCreated with openerId = our owned tab; the + // session must adopt it so the tab is controllable and cleaned up later. + const child = await client.session().callBrowser("Target.createTarget", { url: "https://child.test", openerId: cur.targetId }); + await tick(); + check(child.success, "owned page opens a child target"); + if (child.success) { + const id = (child.data as { targetId: string }).targetId; + check(client.owns(id), "child target opened by an owned tab is auto-adopted"); + } + + // A target the user opens in their own window has no owned opener → never adopted. + const stranger = await client.session().callBrowser("Target.createTarget", { url: "https://user.test", newWindow: true }); + await tick(); + if (stranger.success) { + const id = (stranger.data as { targetId: string }).targetId; + check(!client.owns(id), "target with no owned opener is not adopted"); + } + } + + // ── Reattach self-heal: re-adopt live window-mates of a surviving owned tab. ─ + { + const transport = createMockTransport(); + transport.seed({ targetId: "own-1", type: "page", url: "https://a.test", title: "", windowId: 500 }); + transport.seed({ targetId: "sib-2", type: "page", url: "https://b.test", title: "", windowId: 500, openerId: "own-1" }); + transport.seed({ targetId: "user-9", type: "page", url: "https://user.test", title: "", windowId: 700 }); + const client = createBrowserClient({ + namespace: "t", + transport, + initialOwnership: { ownedTargetIds: ["own-1"], harnessWindowTargetId: "own-1", harnessWindowId: 500 }, + }); + await client.start(); + check(client.owns("own-1"), "reattach: persisted owned tab stays owned"); + check(client.owns("sib-2"), "reattach: live window-mate (last session's popup) is re-adopted"); + check(!client.owns("user-9"), "reattach: a tab in a DIFFERENT window is never adopted"); + } + + // ── Restart-collision guard: stale windowId must not adopt the user's tabs. ─ + { + const transport = createMockTransport(); + // Chrome restarted: our owned tab id is dead; windowId 500 now belongs to + // the user's own window full of their tabs. + transport.seed({ targetId: "user-a", type: "page", url: "https://user-a.test", title: "", windowId: 500 }); + transport.seed({ targetId: "user-b", type: "page", url: "https://user-b.test", title: "", windowId: 500 }); + const client = createBrowserClient({ + namespace: "t", + transport, + initialOwnership: { ownedTargetIds: ["dead-1"], harnessWindowTargetId: "dead-1", harnessWindowId: 500 }, + }); + await client.start(); + check(!client.owns("user-a") && !client.owns("user-b"), "restart: stale windowId does NOT adopt the user's tabs"); + check(client.ownership().harnessWindowId() !== 500, "restart: binds a fresh window instead of the colliding id"); + check(client.current() !== null && !["user-a", "user-b"].includes(client.current()!.targetId), "restart: attaches to a fresh harness tab, not a user tab"); + } + + // ── Teardown: closeOwnedTabs closes every owned tab + clears binding. ───── + { + const transport = createMockTransport(); + const client = createBrowserClient({ namespace: "t", transport }); + await client.start(); + await client.newTab("https://a.test"); + await client.newTab("https://b.test"); + const ownedBefore = client.ownership().list(); + check(ownedBefore.length >= 2, "several owned tabs before teardown"); + + await client.closeOwnedTabs(); + check(client.ownership().list().length === 0, "closeOwnedTabs clears the owned set"); + check(client.ownership().harnessWindowId() === undefined, "closeOwnedTabs clears the windowId binding"); + check(ownedBefore.every((id) => transport.closed.includes(id)), "every owned tab was actually closed in the browser"); + } + + console.log(`\n${passed} passed, ${failed} failed`); + process.exit(failed === 0 ? 0 : 1); +} + +void main(); |
