diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/index.ts | 205 | ||||
| -rw-r--r-- | src/prompt.ts | 9 | ||||
| -rw-r--r-- | src/setup.ts | 41 | ||||
| -rw-r--r-- | src/state.ts | 76 |
4 files changed, 102 insertions, 229 deletions
diff --git a/src/index.ts b/src/index.ts index 14ea8eb..f5c7676 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,8 +2,9 @@ * pi-browser-harness — browser control extension for pi. * * Gives pi agents full control of a real Chrome browser via CDP. - * Spawns the browser-harness daemon, registers browser_* tools, - * and injects browser control guidance into the system prompt. + * Connects to the user's running Chrome (chrome://inspect/#remote-debugging), + * registers browser_* tools, and injects browser control guidance into the + * system prompt. * * Install: * pi install npm:pi-browser-harness @@ -11,42 +12,32 @@ * * Commands: * /browser-setup — guided setup wizard - * /browser-status — show daemon status and current page - * /browser-reload-daemon — restart the daemon + * /browser-status — show client status and current page + * /browser-reload-daemon — restart the browser client * * Flags: - * --browser-namespace <name> — override BU_NAME (default: auto) + * --browser-namespace <name> — override namespace (default: auto) * --browser-debug-clicks — enable debug click overlay */ -import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent"; -import { BrowserDaemon } from "./daemon"; +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; import { type BrowserClient, createBrowserClient } from "./client"; import { getBrowserSystemPrompt } from "./prompt"; -import { registerRenderers } from "./renderers"; -import { registerAllTools } from "./registry"; import { registerSetupCommand } from "./setup"; import { type BrowserState, defaultState, persistState, restoreState } from "./state"; -import { cleanupTempDirs, registerTools } from "./tools"; +import { registerAllTools } from "./registry"; +import { cleanupTempDirs } from "./util/truncate"; -export default function browserHarnessExtension(pi: ExtensionAPI) { - // ── Resolve namespace ────────────────────────────────────────────────────── +export default function browserHarnessExtension(pi: ExtensionAPI): void { const flagNs = pi.getFlag("browser-namespace") as string | undefined; - const namespace = flagNs || `pi-${Math.random().toString(36).slice(2, 10)}`; + const namespace = flagNs ?? `pi-${Math.random().toString(36).slice(2, 10)}`; - // ── State ────────────────────────────────────────────────────────────────── let state: BrowserState = defaultState(namespace); - let daemon: BrowserDaemon | null = null; let client: BrowserClient | null = null; - - // ── Tool initialization flag ─────────────────────────────────────────────── let toolsRegistered = false; - // ═══════════════════════════════════════════════════════════════════════════ - // Flags - // ═══════════════════════════════════════════════════════════════════════════ pi.registerFlag("browser-namespace", { - description: "Browser daemon namespace (BU_NAME). Default: auto-generated", + description: "Browser daemon namespace. Default: auto-generated", type: "string", }); pi.registerFlag("browser-debug-clicks", { @@ -55,181 +46,95 @@ export default function browserHarnessExtension(pi: ExtensionAPI) { default: false, }); - // ═══════════════════════════════════════════════════════════════════════════ - // Commands - // ═══════════════════════════════════════════════════════════════════════════ pi.registerCommand("browser-status", { description: "Show browser connection status and current page", handler: async (_args, ctx) => { - if (!daemon) { - ctx.ui.notify("Browser daemon not started. Run /browser-setup first.", "warning"); + if (!client) { + ctx.ui.notify("Browser client not started. Run /browser-setup first.", "warning"); return; } - - const status = daemon.getStatus(); + const s = client.status(); const lines = [ - `Browser: ${status.alive ? "🟢 Connected" : "🔴 Disconnected"}`, - `Session: ${status.sessionId || "none"}`, + `Browser: ${s.alive ? "🟢 Connected" : "🔴 Disconnected"}`, + `Session: ${s.sessionId ?? "none"}`, ]; - - if (status.remoteBrowserId) { - lines.push(`Browser ID: ${status.remoteBrowserId}`); - } - - // Try to get current page info - if (status.alive) { - try { - const info = await daemon.getPageInfo(); - if ("dialog" in info) { - lines.push(`\n⚠️ Dialog open: ${info.dialog.type} — "${info.dialog.message}"`); + if (s.remoteBrowserId) lines.push(`Browser ID: ${s.remoteBrowserId}`); + if (s.alive) { + const info = await client.pageInfo(); + if (info.success) { + if ("dialog" in info.data) { + lines.push(`\n⚠️ Dialog open: ${info.data.dialog.type} — "${info.data.dialog.message}"`); } else { - lines.push(`\nCurrent Page:`); - lines.push(` URL: ${info.url}`); - lines.push(` Title: ${info.title}`); - lines.push(` Viewport: ${info.width}x${info.height}`); + lines.push( + `\nCurrent Page:`, + ` URL: ${info.data.url}`, + ` Title: ${info.data.title}`, + ` Viewport: ${info.data.width}x${info.data.height}`, + ); } - } catch { - lines.push("\n(Could not read page info)"); } } - ctx.ui.notify(lines.join("\n"), "info"); }, }); pi.registerCommand("browser-reload-daemon", { - description: "Restart the browser daemon", + description: "Restart the browser client", handler: async (_args, ctx) => { - if (!daemon) { - ctx.ui.notify("Browser daemon not started.", "warning"); + if (!client) { + ctx.ui.notify("Browser client not started.", "warning"); return; } - - ctx.ui.notify("Restarting browser daemon...", "info"); - try { - await daemon.stop(); - await daemon.start(); - ctx.ui.notify("Browser daemon restarted ✓", "info"); - } catch (err) { - ctx.ui.notify( - `Restart failed: ${err instanceof Error ? err.message : String(err)}`, - "error", - ); + ctx.ui.notify("Restarting browser client...", "info"); + await client.stop(); + const r = await client.start(); + if (r.success) { + ctx.ui.notify("Browser client restarted ✓", "info"); + } else { + ctx.ui.notify(`Restart failed: ${r.error.message}`, "error"); } }, }); - // ═══════════════════════════════════════════════════════════════════════════ - // Session lifecycle - // ═══════════════════════════════════════════════════════════════════════════ pi.on("session_start", async (_event, ctx) => { - // Restore persisted state (preserving the current namespace from flag) state = restoreState(ctx, state.namespace); - - // Create daemon instance - daemon = new BrowserDaemon(state.namespace); - - // Attempt to start daemon (will fail gracefully if Chrome not connected) - try { - await daemon.start(); - } catch { - // Don't block — user can run /browser-setup later - } - client = createBrowserClient({ namespace: state.namespace }); - try { await client.start(); } catch { /* surfaced via tool errors */ } - - // Register tools (once) - if (!toolsRegistered && daemon) { - registerTools(pi, daemon); - if (client) registerAllTools(pi, client); + // failure is fine — surfaced via tool errors when the agent first tries to use the browser + await client.start(); + if (!toolsRegistered) { + registerAllTools(pi, client); toolsRegistered = true; } - - // Register renderers - registerRenderers(pi); - - // Setup command (always available) - if (daemon) { - registerSetupCommand(pi, daemon); - } - - // Update status - if (daemon.getStatus().alive) { - ctx.ui.setStatus("browser", "🟢 Browser connected"); - } else { - ctx.ui.setStatus("browser", "🔴 Browser — run /browser-setup"); - } + registerSetupCommand(pi, client); + ctx.ui.setStatus( + "browser", + client.status().alive ? "🟢 Browser connected" : "🔴 Browser — run /browser-setup", + ); }); - pi.on("session_shutdown", async (_event) => { - // Persist current state - if (state) { - persistState(pi, state); - } - - // Stop daemon - if (daemon) { - try { - await daemon.stop(); - } catch { - // best-effort - } - daemon = null; - } - + pi.on("session_shutdown", async () => { + persistState(pi, state); if (client) { try { await client.stop(); } catch { /* best-effort */ } client = null; } - toolsRegistered = false; - - // Clean up temp files from truncated tool outputs await cleanupTempDirs(); }); - // Restore state on tree navigation pi.on("session_tree", async (_event, ctx) => { - state = restoreState(ctx, daemon?.namespace); - if (daemon) { - persistState(pi, state); - } + state = restoreState(ctx, client?.namespace); + persistState(pi, state); }); - // ═══════════════════════════════════════════════════════════════════════════ - // System prompt injection - // ═══════════════════════════════════════════════════════════════════════════ pi.on("before_agent_start", async (event) => { - if (!daemon || !daemon.getStatus().alive) { - // Browser not connected — inject minimal note + if (!client || !client.status().alive) { return { systemPrompt: event.systemPrompt + - `\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser daemon is not connected. ` + - `Run /browser-setup to connect to Chrome, or /browser-status to check.`, + `\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser is not connected. Run /browser-setup.`, }; } - - return { - systemPrompt: event.systemPrompt + getBrowserSystemPrompt(), - }; - }); - - // ═══════════════════════════════════════════════════════════════════════════ - // Tool result hook: track tab history - // ═══════════════════════════════════════════════════════════════════════════ - pi.on("tool_result", async (event) => { - if (event.toolName === "browser_new_tab" || event.toolName === "browser_navigate") { - // Track new tabs in history - const details = event.details as { targetId?: string } | undefined; - if (details?.targetId) { - state = { - ...state, - tabHistory: [details.targetId, ...state.tabHistory.filter((id) => id !== details.targetId)].slice(0, 20), - }; - persistState(pi, state); - } - } + return { systemPrompt: event.systemPrompt + getBrowserSystemPrompt() }; }); } diff --git a/src/prompt.ts b/src/prompt.ts index a585124..d4ac8a3 100644 --- a/src/prompt.ts +++ b/src/prompt.ts @@ -21,12 +21,12 @@ to interact with pages visually, and other tools alongside them. | Tool | Purpose | |------|---------| -| browser_screenshot | Capture a PNG screenshot of the current page | +| browser_screenshot | Capture a PNG screenshot of the current page (result details include path and attached: false — the LLM reads the image via the saved file path) | | browser_click | Click at viewport coordinates | | browser_type | Type text into the focused element | | browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) | | browser_scroll | Scroll the page at coordinates | -| browser_navigate | Navigate to a URL | +| browser_navigate | Navigate to a URL (result details.outcome.kind is "in_place" or "new_tab_created") | | browser_new_tab | Open a new tab, optionally navigate | | browser_open_urls | Open multiple URLs in new tabs (parallel) | | browser_go_back / browser_go_forward / browser_reload | History navigation | @@ -37,7 +37,7 @@ to interact with pages visually, and other tools alongside them. | browser_execute_js | Execute JavaScript and return the result | | browser_http_get | Direct HTTP GET (outside browser, for APIs) | | browser_wait | Wait N seconds | -| browser_wait_for_load | Wait for document.readyState === 'complete' | +| browser_wait_for_load | Wait for document.readyState === 'complete' (returns a typed timeout error if the page doesn't reach readyState=complete in N seconds, default 15) | | browser_handle_dialog | Accept or dismiss a JS dialog | | browser_run_script | Execute a temporary script file with daemon access (write script to disk, then run) | @@ -87,6 +87,7 @@ browser_http_get("https://api.example.com/data") \`\`\` browser_screenshot() → browser_scroll({ deltaY: -500 }) → browser_screenshot() \`\`\` +Note: deltaY follows W3C wheel-event convention: positive=up, negative=down. Default deltaY=-300 scrolls down. **Research Workflow (search + browser):** \`\`\` @@ -108,7 +109,7 @@ write("/tmp/extract.js", "...script with daemon access...") → browser_run_scri | Tool | Purpose | |------|---------| | browser_upload_file | Upload a file to a file input (bypasses file picker) | -| browser_dispatch_key | Dispatch a DOM KeyboardEvent on a specific element (for React/Vue inputs) | +| browser_dispatch_key | Dispatch a DOM KeyboardEvent on a specific element (for React/Vue inputs); returns details.matched (number of elements) | | browser_download | Configure download directory and disable save-as prompts | | browser_viewport_resize | Resize the viewport for responsive testing | | browser_drag_and_drop | Perform drag-and-drop from one coordinate to another | diff --git a/src/setup.ts b/src/setup.ts index f99e9d9..0d715ab 100644 --- a/src/setup.ts +++ b/src/setup.ts @@ -5,18 +5,18 @@ import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent"; import { execSync } from "node:child_process"; -import type { BrowserDaemon } from "./daemon"; +import type { BrowserClient } from "./client"; -export function registerSetupCommand(pi: ExtensionAPI, daemon: BrowserDaemon): void { +export function registerSetupCommand(pi: ExtensionAPI, client: BrowserClient): void { pi.registerCommand("browser-setup", { description: "Connect pi to your Chrome browser", handler: async (_args, ctx) => { - await runSetup(ctx, daemon); + await runSetup(ctx, client); }, }); } -async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<void> { +async function runSetup(ctx: ExtensionContext, client: BrowserClient): Promise<void> { ctx.ui.notify("browser setup: checking Chrome...", "info"); // Step 1: Check Chrome is running @@ -32,11 +32,9 @@ async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<v // Step 2: Try to connect to Chrome DevTools ctx.ui.notify("Connecting to Chrome DevTools...", "info"); - try { - await daemon.start(); - ctx.ui.notify("Connected to Chrome ✓", "info"); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const startResult = await client.start(); + if (!startResult.success) { + const msg = startResult.error.message; const lower = msg.toLowerCase(); if ( @@ -58,25 +56,26 @@ async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<v ctx.ui.notify(`Connection failed: ${msg}`, "error"); return; } + ctx.ui.notify("Connected to Chrome ✓", "info"); // Step 3: Verify with test navigation ctx.ui.notify("Testing browser control...", "info"); - try { - await daemon.newTab("https://github.com"); - const info = await daemon.getPageInfo(); - if ("dialog" in info) { - await daemon.cdp("Page.handleJavaScriptDialog", { accept: true }); - } - ctx.ui.notify( - `Browser control verified ✓\nNavigated to: ${"url" in info ? info.url : "github.com"}`, - "info", - ); - } catch (err) { + const tabResult = await client.newTab("https://github.com"); + if (!tabResult.success) { ctx.ui.notify( - `Browser connected but test navigation failed: ${err instanceof Error ? err.message : String(err)}`, + `Browser connected but test navigation failed: ${tabResult.error.message}`, "warning", ); + return; + } + + const info = await client.pageInfo(); + if (info.success && "dialog" in info.data) { + await client.session().call("Page.handleJavaScriptDialog", { accept: true }); } + + const pageUrl = info.success && !("dialog" in info.data) ? info.data.url : "github.com"; + ctx.ui.notify(`Browser control verified ✓\nNavigated to: ${pageUrl}`, "info"); } function checkChromeRunning(): boolean { diff --git a/src/state.ts b/src/state.ts index 9423309..9dcef69 100644 --- a/src/state.ts +++ b/src/state.ts @@ -1,77 +1,45 @@ /** * Session persistence for pi-browser-harness. * - * Persists daemon namespace, tab history, and preferences across - * session reloads and branch navigation. + * Persists the daemon namespace and (when applicable) the remote browser ID + * across session reloads and branch navigation. */ import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent"; -// ── Types ──────────────────────────────────────────────────────────────────── +export type BrowserState = { + readonly namespace: string; + readonly remoteBrowserId?: string; +}; -export interface BrowserState { - /** Daemon namespace (BU_NAME) for this session */ - namespace: string; - /** Recently active tab target IDs */ - tabHistory: string[]; - /** Cloud browser ID (for remote sessions) */ - remoteBrowserId?: string; - /** Screenshot output directory */ - screenshotDir: string; - /** Whether debug click overlay is enabled */ - debugClicks: boolean; -} +export const defaultState = (namespace = "default"): BrowserState => ({ namespace }); -// ── Defaults ───────────────────────────────────────────────────────────────── - -export function defaultState(namespace = "default"): BrowserState { - return { - namespace, - tabHistory: [], - screenshotDir: "", - debugClicks: false, - }; -} - -// ── Persist ────────────────────────────────────────────────────────────────── - -export function persistState(pi: ExtensionAPI, state: BrowserState): void { +export const persistState = (pi: ExtensionAPI, state: BrowserState): void => { pi.appendEntry<BrowserState>("browser-harness-state", state); -} - -// ── Restore ────────────────────────────────────────────────────────────────── +}; /** * Find the last browser-harness-state entry in the current branch * and return the restored state, merged with defaults. + * + * If `currentNamespace` is supplied (e.g. from the --browser-namespace flag), + * it overrides whatever is in the persisted entry. */ -export function restoreState(ctx: ExtensionContext, currentNamespace?: string): BrowserState { +export const restoreState = (ctx: ExtensionContext, currentNamespace?: string): BrowserState => { const branchEntries = ctx.sessionManager.getBranch(); - const defaults = defaultState(currentNamespace); - - // Walk branch from newest to oldest to find last persisted state + const fallback = defaultState(currentNamespace); for (let i = branchEntries.length - 1; i >= 0; i--) { const entry = branchEntries[i]; - if (entry.type === "custom" && entry.customType === "browser-harness-state") { + if (entry?.type === "custom" && entry.customType === "browser-harness-state") { const data = entry.data as Partial<BrowserState> | undefined; if (data) { - // Merge persisted data onto defaults, but always keep the current - // namespace (which comes from the --browser-namespace flag or auto-gen). - return { ...defaults, ...data, namespace: currentNamespace || data.namespace || defaults.namespace }; + return { + ...fallback, + ...data, + namespace: currentNamespace ?? data.namespace ?? fallback.namespace, + }; } } } - - return defaults; -} - -// ── Merge helpers ──────────────────────────────────────────────────────────── - -export function addToTabHistory(state: BrowserState, targetId: string): BrowserState { - const filtered = state.tabHistory.filter((id) => id !== targetId); - return { ...state, tabHistory: [targetId, ...filtered].slice(0, 20) }; -} - -export function setRemoteBrowser(state: BrowserState, browserId: string): BrowserState { - return { ...state, remoteBrowserId: browserId }; -} + return fallback; +}; |