From 014de0ec21936a7a2fd64db9d151e73c79bb44ad Mon Sep 17 00:00:00 2001 From: Patrick Barattin Date: Sun, 11 Oct 2026 20:25:38 +0200 Subject: refactor: remove duplicate browser prompt instructions (#35) Remove browser instruction text repeated across tool descriptions, parameter schemas, tool guidelines and the global browser prompt. Tool names, schemas, handlers and concurrency are unchanged; shared recovery and ownership guidance is now available before and after the browser connects, and the prompt names which tools are serialized versus observations. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 ++ CONTRIBUTING.md | 2 +- README.md | 8 +++ src/domains/click.ts | 5 -- src/domains/console.ts | 2 - src/domains/files.ts | 2 +- src/domains/form-batch.ts | 1 - src/domains/form.ts | 10 +-- src/domains/history.ts | 6 +- src/domains/js.ts | 6 +- src/domains/keyboard.ts | 9 +-- src/domains/navigate.ts | 3 - src/domains/network.ts | 4 +- src/domains/page.ts | 8 +-- src/domains/readpage/read-page.ts | 1 - src/domains/screenshot.ts | 4 -- src/domains/scroll.ts | 1 - src/domains/search/web-search.ts | 1 - src/domains/setup.ts | 2 - src/domains/snapshot.ts | 5 +- src/domains/tabs.ts | 10 +-- src/domains/viewport.ts | 1 - src/index.ts | 11 +-- src/prompt.ts | 145 ++++++-------------------------------- test/util/prompt-guidance-test.ts | 72 +++++++++++++++++++ 25 files changed, 122 insertions(+), 201 deletions(-) create mode 100644 test/util/prompt-guidance-test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f0778b..aaf0a11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to pi-browser-harness will be documented in this file. ## Unreleased +### Changed + +- **Browser guidance no longer repeats tool declarations.** The browser prompt now contains only cross-tool coordination, available before and after connection, and tool guidelines omit usage and argument details already present in descriptions and schemas. Tool names, schemas, handlers, and safety warnings are unchanged. + ### Added - **Per-tab extra HTTP request headers.** `browser_set_headers` applies headers to subsequent document, subresource, fetch, and XHR requests on the current owned tab through CDP, while `browser_clear_headers` removes them. Header values are not echoed in tool output, and guidance requires clearing them before switching tabs or navigating to another origin. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa73342..8436e4f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,7 +87,7 @@ Use the normal browser workflow when testing: screenshot → act → screenshot - No unchecked casts. `npm run boundaries` fails the build on `as any`, `as {`, `as unknown as`, `@ts-ignore`, and non-null assertions in `src/`. A green `tsc` is not evidence of type safety. - Prefer no comments. Explain a non-obvious convention in the commit message or PR description, not inline. - Prefer small, focused changes over broad rewrites. -- Keep tool descriptions, parameter descriptions, and prompt guidelines clear and user-facing. +- Keep tool descriptions and parameter schemas clear and user-facing. Use prompt guidelines only for decisions and caveats not already covered there. Keep `src/prompt.ts` limited to cross-tool coordination; leave recipes in the README and skill. - Preserve the existing error-handling style: return useful, actionable messages to the agent instead of leaking low-level details when possible. - Avoid adding dependencies unless they are necessary for the browser-control experience. - Keep README and CHANGELOG updates in the same PR when behavior changes. diff --git a/README.md b/README.md index fb7d072..a1fd9b2 100644 --- a/README.md +++ b/README.md @@ -353,6 +353,14 @@ A long-lived daemon owns the CDP connection, so it outlives individual pi sessio you only see Chrome's "Allow Remote Debugging" prompt once. `browser_run_script` executes in the harness process with direct daemon and Node access. +### Prompt guidance + +Tool descriptions and parameter schemas own per-tool usage. Tool guidelines add only decisions +and caveats not already covered there. The browser prompt carries cross-tool rules: +tab isolation, concurrency, checking mutations, and stale-ref recovery. These rules are available +before and after connection. Longer recipes remain in this README and the bundled skill +rather than being copied into the browser prompt. + --- ## Troubleshooting diff --git a/src/domains/click.ts b/src/domains/click.ts index 79bb5ee..47d6dc5 100644 --- a/src/domains/click.ts +++ b/src/domains/click.ts @@ -58,11 +58,6 @@ export const clickTool = defineBrowserTool({ "Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.", promptSnippet: "Click an element by ref (preferred) or pixel coordinates", promptGuidelines: [ - "PREFER `ref` from browser_snapshot (the outline shows '[eN]' for every interactive element) — it survives re-renders, unlike coordinates which go stale after a save/edit reflows the page.", - "Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.", - "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.", - "After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).", - "Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.", "If a click doesn't register, run pi with --browser-debug-clicks (or set BH_DEBUG_CLICKS=1) to get annotated screenshots. For React/Vue components ignoring clicks, try browser_dispatch_key.", ], parameters: ClickArgs, diff --git a/src/domains/console.ts b/src/domains/console.ts index 7809f77..799fe2e 100644 --- a/src/domains/console.ts +++ b/src/domains/console.ts @@ -158,8 +158,6 @@ export const consoleTool = defineBrowserTool({ "Read JS errors and console messages from the active tab. Diagnostic — use when an action looks broken or silent.", promptSnippet: "Read JS errors / console output on the current tab", promptGuidelines: [ - "Use after a click, submit, or navigate produced no visible change — errors often explain it.", - "Pass `sinceSeq` from the previous call's `nextCursor` to see only new messages after an action.", "Buffer is reset on tab switch (capacity 500 records). bufferOverflowed:true means older entries were dropped.", "Don't use as a default observation tool — prefer browser_snapshot for page structure.", ], diff --git a/src/domains/files.ts b/src/domains/files.ts index 89fdaf9..9a3a705 100644 --- a/src/domains/files.ts +++ b/src/domains/files.ts @@ -104,7 +104,7 @@ export const uploadFileTool = defineBrowserTool({ description: "Set files on a file via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.", promptSnippet: "Upload a file to a file input by ref (preferred) or selector", promptGuidelines: [ - "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders. (File inputs are often hidden; snapshot still surfaces them.)", + "File inputs are often hidden; browser_snapshot still surfaces them.", "File path must be absolute and readable.", ], parameters: UploadArgs, diff --git a/src/domains/form-batch.ts b/src/domains/form-batch.ts index 567108b..3ec8d26 100644 --- a/src/domains/form-batch.ts +++ b/src/domains/form-batch.ts @@ -28,7 +28,6 @@ export const fillFormTool = defineBrowserTool({ promptSnippet: "Fill many form fields at once by ref — React-safe, self-confirming", promptGuidelines: [ "Preferred for forms: snapshot once, then fill all fields in a single browser_fill_form call.", - "Each field: { ref, value }. value is a string for text/select/contenteditable, boolean for checkbox/radio.", "The result summarizes how many fields filled cleanly and flags any value mismatches or per-field errors.", "Re-run browser_snapshot afterwards to verify the form state if needed.", ], diff --git a/src/domains/form.ts b/src/domains/form.ts index 577eb5d..1673738 100644 --- a/src/domains/form.ts +++ b/src/domains/form.ts @@ -27,15 +27,12 @@ export const fillTool = defineBrowserTool({ "Fill a form field (input/textarea/contenteditable), or tick a checkbox/radio by passing 'true'/'false'. PREFERRED: pass `ref` (e.g. 'e7') from browser_snapshot — survives re-renders. Fallback: a CSS `selector`. Writes through the native value setter and fires bubbling 'input'/'change' events, so React/Vue/Angular controlled components and rich-text editors update correctly — unlike browser_type. Refuses a disabled field rather than writing a value the page will ignore. Returns the field's value after writing, plus a compact diff of page changes.", promptSnippet: "Fill a form field by ref (preferred) or selector (works with React/Vue controlled inputs)", promptGuidelines: [ - "PREFER `ref` from browser_snapshot (the '[eN]' handle) over a guessed CSS selector — refs survive re-renders and don't require you to invent a selector.", "Pass the desired value; no browser_click is needed first.", "For a checkbox or radio pass 'true' or 'false' — it toggles `checked`, not the submit value. browser_set_checked does the same thing with a boolean.", - "A disabled field is refused: the error says so, rather than reporting a write the page never accepted.", - "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.", "Use browser_type instead only for keystroke-sensitive widgets (autocomplete, masked/segmented inputs).", "Fill sets the value but does not focus or submit. To submit a tag/autocomplete input, follow with browser_dispatch_key({ ref, key: 'Enter' }) — not browser_press_key, which targets the focused element and may miss this field.", "For a custom (div-based) dropdown, browser_fill won't validate against its option list — open it, run browser_snapshot, then browser_click the option's ref.", - "The result reports the field's value after writing and an appended page-changes diff — confirm both match what you intended before moving on.", + "Confirm the field value browser_fill returns matches what you intended.", ], parameters: FillArgs, concurrency: "serialized", @@ -118,8 +115,6 @@ export const focusTool = defineBrowserTool({ "Focus an element via the DOM .focus() method — deterministic, no coordinate accuracy needed. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use before browser_type when a click might miss the field.", promptSnippet: "Focus an element by ref (preferred) or selector", promptGuidelines: [ - "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", - "Use before browser_type to guarantee the right field is focused without relying on click coordinates.", "For simply setting a value, prefer browser_fill (it doesn't need a separate focus step).", ], parameters: FocusArgs, @@ -196,10 +191,8 @@ export const selectOptionTool = defineBrowserTool({ "Select an option in a native by ref (preferred) or selector", promptGuidelines: [ - "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", "Provide exactly one of value, label, or index to choose the option.", "For custom (non-native) dropdowns built from divs, click to open then browser_click the option instead.", - "If no option matches, the error details list the available options so you can retry.", ], parameters: SelectOptionArgs, concurrency: "serialized", @@ -263,7 +256,6 @@ export const setCheckedTool = defineBrowserTool({ "Set a checkbox or radio to a desired checked state by ref. Only acts if the current state differs, fires change, and reports the final state. Get the ref from browser_snapshot's [eN].", promptSnippet: "Set a checkbox/radio checked state by ref", promptGuidelines: [ - "checked:true ticks the box, checked:false unticks it. Idempotent — no-op if already in the desired state.", "For radios, set checked:true on the option you want; the group deselects the others.", ], parameters: SetCheckedArgs, diff --git a/src/domains/history.ts b/src/domains/history.ts index 4b5fa56..b19de2e 100644 --- a/src/domains/history.ts +++ b/src/domains/history.ts @@ -18,7 +18,7 @@ export const goBackTool = defineBrowserTool({ label: "Browser Go Back", description: "Navigate back one page in history.", promptSnippet: "Go back one page", - promptGuidelines: ["Use to return to the previous page after navigating."], + promptGuidelines: [], parameters: Type.Object({}), concurrency: "serialized", async handler(_a, { client }): Promise> { @@ -38,7 +38,7 @@ export const goForwardTool = defineBrowserTool({ label: "Browser Go Forward", description: "Navigate forward one page in history.", promptSnippet: "Go forward one page", - promptGuidelines: ["Use to undo a browser_go_back."], + promptGuidelines: [], parameters: Type.Object({}), concurrency: "serialized", async handler(_a, { client }): Promise> { @@ -58,7 +58,7 @@ export const reloadTool = defineBrowserTool({ label: "Browser Reload", description: "Reload the current page.", promptSnippet: "Reload the page", - promptGuidelines: ["Use to refresh the page, e.g. after server-side changes."], + promptGuidelines: [], parameters: Type.Object({}), concurrency: "serialized", async handler(_a, { client }): Promise> { diff --git a/src/domains/js.ts b/src/domains/js.ts index 19c8913..9d7cbd1 100644 --- a/src/domains/js.ts +++ b/src/domains/js.ts @@ -36,8 +36,6 @@ export const executeJsTool = defineBrowserTool({ "DEFAULT for extracting a specific value: `document.querySelector('.price').innerText`, `input.value`, `el.getAttribute('aria-label')`, etc.", "For click coordinates not already in browser_snapshot: `JSON.stringify(document.querySelector('SELECTOR').getBoundingClientRect())` — beats a screenshot every time.", "For 'is this element visible / enabled / checked / focused', a one-liner here beats a screenshot every time.", - "DO NOT call browser_screenshot to read a value or check element state. Use this tool.", - "`return foo` is auto-wrapped in an IIFE — both `foo` and `(() => foo)()` work.", "For iframes, first run `Object.values(document.querySelectorAll('iframe'))` to find the iframe, then pass its targetId.", "Result must be JSON-serializable (Runtime.evaluate returnByValue=true).", ], @@ -138,9 +136,7 @@ export const runScriptTool = defineBrowserTool({ "Execute a temporary JavaScript script with full Node.js + browser-daemon access. Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR. Mandatory timeout. The script is full RCE on the harness's process — only invoke scripts you wrote and reviewed.", promptSnippet: "Run a temporary script with daemon + Node access (security-bounded)", promptGuidelines: [ - "Write the script with the write tool first; pass its absolute path.", - "Path must be inside tmpdir, cwd, or BH_SCRIPT_DIR — otherwise rejected.", - "Default timeout is 60s; pass timeoutMs to extend (max 600s).", + "Write the script with the write tool first; pass its absolute path. Use the OS temp directory rather than hardcoding /tmp.", "Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout.", "Script MUST return { content: [{ type: 'text', text: '...' }], details?: {...} }. Throw on errors.", ], diff --git a/src/domains/keyboard.ts b/src/domains/keyboard.ts index 17b56e7..10b1b5c 100644 --- a/src/domains/keyboard.ts +++ b/src/domains/keyboard.ts @@ -31,7 +31,6 @@ export const typeTool = defineBrowserTool({ description: "Type text into the currently focused element. Use browser_click first to focus an input field.", promptSnippet: "Type text into the focused element", promptGuidelines: [ - "Use browser_type to enter text. Click on an input field with browser_click first to focus it.", "For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.", ], parameters: TypeArgs, @@ -84,9 +83,7 @@ export const pressKeyTool = defineBrowserTool({ promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)", promptGuidelines: [ "Use browser_press_key for keyboard shortcuts and navigation keys.", - "Special key names: Enter, Tab, Backspace, Escape, Delete, ArrowLeft, ArrowUp, ArrowRight, ArrowDown, Home, End, PageUp, PageDown.", - "Use Space as ' ' (a single space character).", - "Modifiers: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with bitwise OR: Ctrl+Shift = 2|8 = 10.", + "browser_press_key modifiers combine with bitwise OR: Ctrl+Shift = 2|8 = 10.", ], parameters: PressKeyArgs, concurrency: "serialized", @@ -138,11 +135,9 @@ export const dispatchKeyTool = defineBrowserTool({ "Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.", promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector", promptGuidelines: [ - "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.", - "Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text (use browser_type or browser_press_key for actual typing).", + "Does NOT insert text (use browser_type or browser_press_key for actual typing).", "Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.", "The event carries keyCode/which (e.g. 13 for Enter) for legacy handlers, but is untrusted (isTrusted === false) — a few libraries may still ignore it.", - "eventType defaults to 'keydown'.", ], parameters: DispatchKeyArgs, concurrency: "serialized", diff --git a/src/domains/navigate.ts b/src/domains/navigate.ts index 151a742..5d490ae 100644 --- a/src/domains/navigate.ts +++ b/src/domains/navigate.ts @@ -22,8 +22,6 @@ export const navigateTool = defineBrowserTool({ "Navigate to a URL. Creates a new tab on first call (when no real tabs exist). Otherwise navigates the current tab in place.", promptSnippet: "Navigate to a URL", promptGuidelines: [ - "Use browser_navigate to go to URLs.", - "Use browser_wait_for_load after browser_navigate to wait for the page to finish loading.", "For extracting data from a page you already navigated to, use browser_execute_js or browser_http_get (faster for APIs).", "Note: Google and some sites with strict anti-bot detection may reject CDP navigation. Use browser_http_get for search results.", ], @@ -80,7 +78,6 @@ export const openUrlsTool = defineBrowserTool({ promptGuidelines: [ "Use after browser_web_search to open citations in parallel.", "After opening, use browser_list_tabs / browser_switch_tab to navigate, then browser_snapshot to inspect each tab.", - "Use browser_wait_for_load on a tab before extracting data from SPAs.", ], parameters: OpenUrlsArgs, concurrency: "serialized", diff --git a/src/domains/network.ts b/src/domains/network.ts index a34dde3..438aaa1 100644 --- a/src/domains/network.ts +++ b/src/domains/network.ts @@ -24,7 +24,6 @@ export const httpGetTool = defineBrowserTool({ "Fetch a URL outside the browser (faster than browser_navigate for APIs and static pages). Timeout covers headers AND body read.", promptSnippet: "HTTP GET (outside browser; for APIs/static pages)", promptGuidelines: [ - "Faster than navigate+execute_js for JSON/HTML APIs.", "No JS rendering — for SPAs use browser_navigate.", ], parameters: HttpGetArgs, @@ -224,8 +223,7 @@ export const networkRequestsTool = defineBrowserTool({ promptSnippet: "Inspect recent network requests on the current page", promptGuidelines: [ "Buffer is reset on tab switch — only requests on the current tab since attach are visible.", - 'urlPattern is a substring; wrap in slashes for regex (e.g. "/\\\\.json$/").', - "includeResponseBodies:true costs an extra CDP call per matched request — use only when you need payloads.", + "Pass includeResponseBodies:true to browser_network_requests only when you need payloads.", "If bufferOverflowed:true in the result, older events were dropped (capacity 500 records/tab).", ], parameters: NetworkRequestsArgs, diff --git a/src/domains/page.ts b/src/domains/page.ts index 909963f..65bd4a2 100644 --- a/src/domains/page.ts +++ b/src/domains/page.ts @@ -28,10 +28,7 @@ export const pageInfoTool = defineBrowserTool({ "Get current page state: URL, title, viewport size, scroll position, page dimensions. If a JS dialog is open, returns dialog info instead.", promptSnippet: "Get current page URL, title, viewport, and scroll position", promptGuidelines: [ - "Use browser_page_info to quickly check what page you're on and whether a JS dialog is blocking interaction.", - "If browser_page_info returns a dialog, use browser_handle_dialog before any other browser actions.", - "JS dialogs freeze the page's JS thread, so no other interaction works until the dialog is handled.", - "browser_page_info auto-detects alert, confirm, prompt, and beforeunload dialogs.", + "If browser_page_info returns a dialog, call browser_handle_dialog before any other browser actions: dialogs freeze the page's JS thread.", ], parameters: Type.Object({}), concurrency: "parallel", @@ -87,7 +84,6 @@ export const waitForLoadTool = defineBrowserTool({ promptSnippet: "Wait for the page to finish loading", promptGuidelines: [ "Call after browser_navigate / browser_open_urls before extracting data.", - "Returns when readyState becomes 'complete' OR the timeout elapses.", ], parameters: WaitForLoadArgs, concurrency: "serialized", @@ -122,8 +118,6 @@ export const waitForTool = defineBrowserTool({ promptSnippet: "Wait for an element or text to appear (or an element to disappear)", promptGuidelines: [ "Use before browser_fill / browser_click on dynamic pages so you act after the element renders.", - "Provide selector (default: wait until present; gone:true: wait until absent) or text.", - "Returns a typed timeout error if the condition isn't met within the timeout.", ], parameters: WaitForArgs, concurrency: "parallel", diff --git a/src/domains/readpage/read-page.ts b/src/domains/readpage/read-page.ts index fadee38..744a5ce 100644 --- a/src/domains/readpage/read-page.ts +++ b/src/domains/readpage/read-page.ts @@ -157,7 +157,6 @@ export const readPageTool = defineBrowserTool({ "Read a page as clean article text — main content with nav/ads/boilerplate stripped. Pass a url (opened in an isolated tab, read, then closed) or a targetId of an owned tab. Reader-mode counterpart to browser_web_search.", promptSnippet: "Read a page's main content as clean text", promptGuidelines: [ - "Pass url to read an arbitrary page (opened + closed in its own tab, never disturbing your current tab), or targetId to read an already-open owned tab.", "Returns readable main-article text — use this over browser_snapshot/browser_execute_js when you want an article's content for reading or research.", "Boilerplate-heavy or structure-less pages fall back to bounded body text rather than erroring.", ], diff --git a/src/domains/screenshot.ts b/src/domains/screenshot.ts index 9f2587c..162e309 100644 --- a/src/domains/screenshot.ts +++ b/src/domains/screenshot.ts @@ -20,10 +20,6 @@ export const screenshotTool = defineBrowserTool({ "Capture the page as a JPEG/PNG image. NOT a default exploration tool. Use ONLY when (a) you need to verify visual properties (layout, color, spacing, rendered text legibility), or (b) browser_snapshot and browser_execute_js cannot answer your question. For understanding what's on the page use browser_snapshot. For specific element values use browser_execute_js. For click coordinates use the @(x,y) hints in browser_snapshot's outline.", promptSnippet: "Capture a screenshot — visual verification only", promptGuidelines: [ - "DO NOT use as a default exploration tool. browser_snapshot is the default for understanding page structure — far cheaper and more reliable.", - "DO NOT use to find click coordinates. browser_snapshot already includes @(x,y) for every interactive element. If a target isn't there, use browser_execute_js with element.getBoundingClientRect().", - "DO use to verify visual rendering after an action: did the modal animate in correctly, did the chart render, are colors right, did the layout reflow as expected.", - "DO use as a debugging fallback when browser_execute_js or browser_snapshot return surprising results and you suspect the rendered page differs from the DOM.", "format='jpeg' with quality 60-90 keeps screenshots small for photo-heavy pages. Use maxDim if the page is huge.", ], parameters: ScreenshotArgs, diff --git a/src/domains/scroll.ts b/src/domains/scroll.ts index 832f584..4169422 100644 --- a/src/domains/scroll.ts +++ b/src/domains/scroll.ts @@ -20,7 +20,6 @@ export const scrollTool = defineBrowserTool({ "Scroll the page at given coordinates. deltaY follows W3C wheel-event convention: positive = scroll down, negative = scroll up. Default scrolls down (deltaY = 300).", promptSnippet: "Scroll the page (deltaY positive=down, negative=up)", promptGuidelines: [ - "Default behavior scrolls down 300px (deltaY=300). Pass a negative deltaY to scroll up.", "Pass x/y to target a specific scrollable region (e.g., a div with overflow); otherwise scrolls the page at viewport center.", ], parameters: ScrollArgs, diff --git a/src/domains/search/web-search.ts b/src/domains/search/web-search.ts index 553135c..f54dd3c 100644 --- a/src/domains/search/web-search.ts +++ b/src/domains/search/web-search.ts @@ -77,7 +77,6 @@ export const webSearchTool = defineBrowserTool({ "Search the web and return ranked results (title, url, snippet, rank). Scrapes a Google results page in the real Chrome — no API key. Returns links only; use browser_read_page to read a result's content.", promptSnippet: "Search the web for a query", promptGuidelines: [ - "Returns ranked {title, url, snippet, rank} — links only, no page content. Follow up with browser_read_page.", "On a CAPTCHA / bot wall the call fails with kind:'invalid_state' and details.reason:'captcha' (plus serpUrl) — surface it to the user; do not retry in a tight loop.", "On zero results it fails with details.reason:'no_results' — rephrase the query and try again.", "Runs in its own isolated tab and never disturbs the user's current tab.", diff --git a/src/domains/setup.ts b/src/domains/setup.ts index 9291431..3f241a9 100644 --- a/src/domains/setup.ts +++ b/src/domains/setup.ts @@ -15,8 +15,6 @@ export const setupTool = defineBrowserTool({ "Call when browser tools fail with 'Browser harness not initialized'. Idempotent.", promptSnippet: "Initialize browser connection", promptGuidelines: [ - "Call browser_setup when you get a 'Browser harness not initialized' error from any browser tool.", - "This tool is idempotent — calling it when already connected is harmless.", "After browser_setup succeeds, retry the browser tool that failed.", ], parameters: SetupArgs, diff --git a/src/domains/snapshot.ts b/src/domains/snapshot.ts index 86c5fe3..68f49b4 100644 --- a/src/domains/snapshot.ts +++ b/src/domains/snapshot.ts @@ -98,10 +98,7 @@ export const snapshotTool = defineBrowserTool({ "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy). Every interactive element gets a stable ref shown as [eN] plus click coordinates @(x,y). Pass the ref to browser_click/browser_fill/etc. — refs survive re-renders, coordinates don't. Use this BEFORE deciding whether you need a screenshot. Pair with browser_execute_js for surgical reads of specific element values.", promptSnippet: "Get accessibility-tree snapshot with stable element refs (default for page inspection)", promptGuidelines: [ - "DEFAULT — use this whenever you need to know what's on a page, what's clickable, or how the page is structured.", - "Refs come for free: every interactive element shows '[eN]' in the outline. Pass eN as `ref` to browser_click/browser_fill/browser_select_option/browser_focus/browser_upload_file — refs survive re-renders, so prefer them over the '@(x,y)' coordinates (a fallback).", - "DO NOT call browser_screenshot just to understand the page. This tool already gives you structure, labels, states, refs, and click targets.", - "Re-run after a navigation or a major re-render to get fresh refs; a 'ref is stale' error from an interaction tool means you need a new snapshot.", + "Re-run browser_snapshot after a navigation or a major re-render to get fresh refs.", "Pass includeScreenshot:true ONLY if you also need to verify visual rendering (rare).", "format:'json' returns the raw slim structure (with `ref` and `box` per node) for programmatic use; default 'outline' is human/LLM-readable.", ], diff --git a/src/domains/tabs.ts b/src/domains/tabs.ts index a2c3831..1b225ff 100644 --- a/src/domains/tabs.ts +++ b/src/domains/tabs.ts @@ -19,10 +19,7 @@ export const listTabsTool = defineBrowserTool({ description: "List browser tabs. Defaults to tabs this harness session opened (scope:'owned'). Pass scope:'all' to see the user's other tabs too.", promptSnippet: "List browser tabs", promptGuidelines: [ - "Defaults to scope:'owned' — only tabs this session created. Pass scope:'all' to inspect the user's other tabs.", - "browser_switch_tab and browser_close_tab refuse non-owned tabs; use browser_new_tab to open one this session controls.", "Each tab shows its full 32-char hex targetId — use the exact value with browser_switch_tab.", - "Internal tabs (chrome://) included by default; pass includeInternal=false to exclude them.", ], parameters: ListTabsArgs, concurrency: "parallel", @@ -83,7 +80,6 @@ export const switchTabTool = defineBrowserTool({ promptGuidelines: [ "Get a targetId via browser_list_tabs first.", "Accepts exact targetId or a unique prefix of at least 8 hex characters.", - "Refuses tabs not owned by this session — use browser_new_tab to open a controllable tab instead.", ], parameters: SwitchTabArgs, concurrency: "serialized", @@ -138,10 +134,7 @@ export const newTabTool = defineBrowserTool({ label: "Browser New Tab", description: "Open a new tab in the harness's dedicated Chrome window and switch to it. Optionally navigate to a URL.", promptSnippet: "Open a new tab", - promptGuidelines: [ - "All tabs opened by this tool live in a single dedicated Chrome window separate from the user's main browsing.", - "Pass url to navigate immediately.", - ], + promptGuidelines: [], parameters: NewTabArgs, concurrency: "serialized", async handler(args, { client }): Promise> { @@ -166,7 +159,6 @@ export const closeTabTool = defineBrowserTool({ description: "Close a tab the harness owns. Refuses tabs not opened by this session.", promptSnippet: "Close an owned tab", promptGuidelines: [ - "Only closes tabs this session opened (visible in browser_list_tabs default scope).", "Accepts exact targetId or a unique hex prefix (>=8 chars).", ], parameters: CloseTabArgs, diff --git a/src/domains/viewport.ts b/src/domains/viewport.ts index e4ba5d5..d330df1 100644 --- a/src/domains/viewport.ts +++ b/src/domains/viewport.ts @@ -17,7 +17,6 @@ export const viewportResizeTool = defineBrowserTool({ description: "Override the viewport size and device pixel ratio for responsive testing.", promptSnippet: "Resize the viewport (responsive testing)", promptGuidelines: [ - "Width/height in CSS pixels (e.g., 375x667 for iPhone SE).", "Pass deviceScaleFactor=2 to simulate retina displays.", ], parameters: ViewportArgs, diff --git a/src/index.ts b/src/index.ts index d69940a..a2e0c11 100644 --- a/src/index.ts +++ b/src/index.ts @@ -150,13 +150,8 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void { }); pi.on("before_agent_start", async (event) => { - if (!client || !client.status().alive) { - return { - systemPrompt: - event.systemPrompt + - `\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser is not connected. Run /browser-setup.`, - }; - } - return { systemPrompt: event.systemPrompt + getBrowserSystemPrompt() }; + return { + systemPrompt: event.systemPrompt + getBrowserSystemPrompt(client?.status().alive ?? false), + }; }); } diff --git a/src/prompt.ts b/src/prompt.ts index 9a28abd..4fb78a7 100644 --- a/src/prompt.ts +++ b/src/prompt.ts @@ -1,128 +1,27 @@ -/** - * System prompt guidance for browser usage. - * - * Injects a compact browser control reference into the pi system prompt. - * This prompt carries only cross-cutting knowledge — the mental model, tool - * routing, workflows, and gotchas. Per-tool specifics (schemas, detailed usage) - * are injected separately via promptGuidelines on each tool registration, so this - * file intentionally does NOT restate full tool descriptions. - */ - -export function getBrowserSystemPrompt(): string { +export function getBrowserSystemPrompt(connected = true): string { + const connectionNotice = connected + ? "" + : "Browser is not connected. Run browser_setup or /browser-setup to connect.\n\n"; return ` ## Browser Control -You drive a **real Chrome** (the user's running instance, not headless) through -\`browser_*\` tools. These coexist with every standard pi tool — read, bash, edit, -write — so mix them freely: \`browser_*\` for the page, everything else alongside. - -### The core handle: refs - -Interaction is ref-first. Call \`browser_snapshot\` to get an accessibility tree -where every interactive element has a stable ref (\`[eN]\`) and \`@(x,y)\`. **Target -elements by ref, not by coordinates or guessed selectors** — refs are keyed to -element identity, so they survive the re-renders that React/Vue forms trigger on -every edit and save. Coordinates go stale; guessed selectors miss. - -- Prefer \`browser_fill\` for text — it fires input/change so controlled components - update, and returns the value for verification. Use \`browser_click\` + \`browser_type\` - only for keystroke-sensitive widgets (autocomplete, masked inputs). -- Mutating calls append a compact **"Page changes"** diff — read it to confirm the - change landed (form closed, new \`*[eN]\` appeared). -- \`"ref is stale"\` means the page changed: re-run \`browser_snapshot\` for fresh refs. - -### Choosing a tool - -| To… | Reach for | -|-----|-----------| -| See the page structure (for interaction) | \`browser_snapshot\` (refs) | -| See the page visually | \`browser_screenshot\` | -| Search the web | \`browser_web_search\` → ranked {title, url, snippet} | -| Read an article's clean text | \`browser_read_page\` (reader mode; url or owned tab) | -| Extract specific DOM values | \`browser_execute_js\` | -| Hit a JSON/API endpoint | \`browser_http_get\` (raw GET, outside the browser) | -| Set or clear request headers on the current tab | \`browser_set_headers\` · \`browser_clear_headers\` | -| Click / fill / type / select / focus / press a key | \`browser_click\` · \`browser_fill\` · \`browser_type\` · \`browser_select_option\` · \`browser_focus\` · \`browser_press_key\` | -| Upload a file · drag · resize viewport | \`browser_upload_file\` · \`browser_drag_and_drop\` · \`browser_viewport_resize\` | -| Go to a URL / open many / manage tabs | \`browser_navigate\` · \`browser_new_tab\` · \`browser_open_urls\` · \`browser_list_tabs\` · \`browser_switch_tab\` · \`browser_go_back\`/\`go_forward\`/\`reload\` | -| Wait for content / load / a fixed delay | \`browser_wait_for\` (selector/text) · \`browser_wait_for_load\` (readyState) · \`browser_wait\` | -| Inspect page / handle a dialog | \`browser_page_info\` · \`browser_current_tab\` · \`browser_handle_dialog\` | -| Diagnose a broken action | \`browser_console\` (JS errors) · \`browser_get_network_log\` | -| Save output / config downloads | \`browser_print_to_pdf\` · \`browser_download\` | -| Anything the tools can't express | \`browser_run_script\` (see Extending) | - -Each tool carries its own detailed guidelines — this table is only for routing. - -### Workflows - -**Research (search → read):** -\`\`\` -browser_web_search("query") // ranked links, no page content -→ browser_read_page(url) // clean article text for the top hits -// fall back to browser_open_urls + browser_execute_js only when reader mode misses -\`\`\` - -**Form filling (ref-first — the reliable path on SPA forms):** -\`\`\` -browser_wait_for({ selector: "#email" }) // ensure the field is rendered -browser_snapshot() // interactive elements get [eN] refs -→ browser_fill({ ref: "e7", value: "a@b.com" }) -→ browser_select_option({ ref: "e9", label: "India" }) -→ browser_click({ ref: "e12" }) // re-resolves position even after reflow -// read each "Page changes" diff to confirm the step landed -\`\`\` - -**Navigate + verify:** -\`\`\` -browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot() -\`\`\` - -**Scroll:** \`browser_scroll({ deltaY: 500 })\` — W3C wheel convention: positive = down, -negative = up (default 300, down). - -### Parallelism - -Observation tools (\`browser_screenshot\`, \`browser_page_info\`, \`browser_execute_js\`, -\`browser_list_tabs\`, \`browser_http_get\`, \`browser_web_search\`, \`browser_read_page\`, …) -run in parallel with each other and with mutations. The harness serializes mutations -(click, type, scroll, navigate, switch_tab) so they never race. Emit independent calls -in one turn: -\`\`\` -browser_screenshot() + browser_page_info() + browser_execute_js("document.title") -\`\`\` - -**Multi-agent:** tab switching by one agent changes the active tab for all agents, -but per-tab data (console, network, dialogs) stays isolated. Call \`browser_current_tab\` -before a mutation to confirm you're on the expected tab. Handle any dialog reported by -\`browser_page_info\` promptly — dialogs block interaction and aren't queued across agents. - -**Session boundary & tab hygiene:** you operate only inside this session's own Chrome -window. \`browser_switch_tab\`/\`browser_close_tab\` refuse tabs this session didn't open, -and \`browser_list_tabs\` defaults to owned tabs — never reach into the user's other tabs -or windows. Close tabs when you're done with them: call \`browser_close_tab\` on any tab -you opened (via \`browser_new_tab\`/\`browser_open_urls\`) once you've extracted what you -need. Don't leave a pile of stale tabs behind — keep only what a later step still needs. -(Leftover tabs are closed automatically when the session ends, but close them yourself -as you go.) - -### Extending: temporary scripts - -When built-in tools can't express a multi-step workflow, write a script to a temp file -and run it with \`browser_run_script\`. It executes in the harness process with direct -daemon + Node access. -\`\`\` -write(.js, \` - const results = []; - for (const url of params.urls) { - await daemon.cdp("Page.navigate", { url }); - await new Promise(r => setTimeout(r, 2000)); - results.push({ url, title: await daemon.evaluateJS("document.title") }); - } - return { content: [{ type: "text", text: JSON.stringify(results, null, 2) }] }; -\`) -browser_run_script(.js, { urls: [...] }) -\`\`\` -Script bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, -Buffer, setTimeout, clearTimeout. Use a real temp path from the OS — don't hardcode \`/tmp\`. +${connectionNotice}You control the user's running Chrome through \`browser_*\` tools alongside the +standard pi tools. Per-tool usage lives in tool descriptions, parameter schemas, +and tool-specific guidelines. + +- Operate only inside this session's owned Chrome window. Never reach into the + user's other tabs or windows. Close owned tabs once no later step needs them. +- Mutating tools (click, type, fill, press_key, scroll, navigate, history, tab + switch/new/close, upload, download, headers, viewport, drag, web_search) are + serialized. Observations (snapshot, screenshot, page_info, current_tab, list_tabs, + console, network_requests, http_get, read_page, execute_js, wait, wait_for) may run + concurrently with them. \`browser_run_script\` bypasses that lane: keep mutating + scripts sequential with other browser calls. Parallelize only independent calls. +- Agents sharing a session share its active tab. Check \`browser_current_tab\` + before a mutation to confirm the expected tab when another agent may switch it. +- Read any appended "Page changes" diff to confirm an interaction. Tools without a + diff, including \`browser_run_script\`, need explicit state verification. +- If a tool reports "ref is stale", call \`browser_snapshot\` for fresh refs before + retrying; do not reuse the stale handle. `; } diff --git a/test/util/prompt-guidance-test.ts b/test/util/prompt-guidance-test.ts new file mode 100644 index 0000000..29e54e2 --- /dev/null +++ b/test/util/prompt-guidance-test.ts @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { ALL_TOOLS } from "../../src/registry"; +import { getBrowserSystemPrompt } from "../../src/prompt"; + +const prompt = getBrowserSystemPrompt(); +const names = new Set(ALL_TOOLS.map((tool) => tool.name)); + +const toolNamed = (name: string) => { + const tool = ALL_TOOLS.find((item) => item.name === name); + assert.ok(tool, `${name} must remain registered`); + return tool; +}; + +describe("browser prompt guidance", () => { + it("references only registered browser tools", () => { + for (const name of prompt.match(/\bbrowser_[a-z]+(?:_[a-z]+)*\b/g) ?? []) { + assert.ok(names.has(name), `Unknown tool in connected prompt: ${name}`); + } + }); + + it("does not repeat identical guidelines across tools", () => { + const seen = new Map(); + for (const tool of ALL_TOOLS) { + for (const guideline of tool.promptGuidelines) { + assert.ok(!seen.has(guideline), `${tool.name} repeats a guideline from ${seen.get(guideline)}`); + seen.set(guideline, tool.name); + } + } + }); + + it("preserves cross-tool recovery, verification, and tab isolation", () => { + assert.match(prompt, /ref is stale/); + assert.match(prompt, /browser_snapshot/); + assert.match(prompt, /Page changes/); + assert.match(prompt, /own(?:ed)?.*Chrome\s+window/); + assert.match(prompt, /Close.*tabs/); + assert.match(prompt, /serializ/); + assert.match(prompt, /concurrent|parallel/); + assert.match(prompt, /browser_current_tab/); + }); + + it("does not promise serialization or automatic diffs for scripts", () => { + assert.equal(toolNamed("browser_run_script").concurrency, "parallel"); + const normalized = prompt.replaceAll("`", "").replace(/\s+/g, " "); + assert.match(normalized, /browser_run_script bypasses that lane/); + assert.match(normalized, /keep mutating scripts sequential with other browser calls/); + assert.match(normalized, /Tools without a diff.*need explicit state verification/); + }); + + it("keeps shared guidance available before the browser connects", () => { + const disconnected = getBrowserSystemPrompt(false); + assert.match(disconnected, /browser_setup/); + assert.match(disconnected, /\/browser-setup/); + for (const rule of ["ref is stale", "Page changes", "owned Chrome window", "browser_current_tab"]) { + assert.ok(disconnected.includes(rule), `Disconnected prompt lost: ${rule}`); + } + assert.doesNotMatch(prompt, /Browser is not connected/); + }); + + it("keeps script execution safeguards and return contract available", () => { + const tool = toolNamed("browser_run_script"); + assert.match(tool.description, /tmpdir, cwd, or BH_SCRIPT_DIR/); + assert.match(tool.description, /only invoke scripts you wrote and reviewed/); + assert.match(tool.promptGuidelines.join("\n"), /MUST return.*content/); + }); + + it("keeps CAPTCHA recovery and header cleanup instructions available", () => { + assert.match(toolNamed("browser_web_search").promptGuidelines.join("\n"), /CAPTCHA.*do not retry in a tight loop/); + assert.match(toolNamed("browser_set_headers").promptGuidelines.join("\n"), /browser_clear_headers before switching tabs or navigating.*another origin/); + }); +}); -- cgit v1.2.3