diff options
| author | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-04-29 00:40:36 +0530 |
|---|---|---|
| committer | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-04-29 00:40:36 +0530 |
| commit | 8fff91522918a8309dea2e9fa725e15f6ed1c3ea (patch) | |
| tree | a2f5fb863fd6ac260a3f37e4ed308d0c6df0a9ed /src | |
Initial release: pi-browser-harness
Browser control extension for pi — 20 browser_* tools for navigating,
clicking, typing, screenshotting, and extracting data from real Chrome
via CDP. Includes self-extending harness (register_tool/remove_tool),
guided /browser-setup, session persistence, and custom TUI renderers.
Diffstat (limited to 'src')
| -rw-r--r-- | src/daemon.ts | 551 | ||||
| -rw-r--r-- | src/dynamic-tools.ts | 335 | ||||
| -rw-r--r-- | src/index.ts | 228 | ||||
| -rw-r--r-- | src/prompt.ts | 78 | ||||
| -rw-r--r-- | src/protocol.ts | 201 | ||||
| -rw-r--r-- | src/renderers.ts | 21 | ||||
| -rw-r--r-- | src/setup.ts | 178 | ||||
| -rw-r--r-- | src/state.ts | 77 | ||||
| -rw-r--r-- | src/tools.ts | 1200 |
9 files changed, 2869 insertions, 0 deletions
diff --git a/src/daemon.ts b/src/daemon.ts new file mode 100644 index 0000000..babae15 --- /dev/null +++ b/src/daemon.ts @@ -0,0 +1,551 @@ +/** + * BrowserDaemon — manages the browser-harness daemon process lifecycle. + * + * Architecture: + * Node.js → Unix socket (/tmp/bu-<namespace>.sock) → daemon.py → CDP WS → Chrome + * + * The daemon is spawned as a child process on start() and communicates via + * the protocol defined in protocol.ts. All CDP calls go through this class. + */ + +import { spawn, type ChildProcess } from "node:child_process"; +import { existsSync } from "node:fs"; +import { access, readFile, unlink, writeFile } from "node:fs/promises"; +import { homedir, tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { + sendRequest, + daemonAlive, + healthProbe, + isInternalUrl, + type CDPEvent, + type DaemonRequest, + type DaemonResponse, + type DaemonStatus, + type DialogInfo, + type PageInfo, + type PageInfoResult, + type RemoteConfig, + type TabInfo, +} from "./protocol"; + +// ── Constants ──────────────────────────────────────────────────────────────── + +/** Common locations where browser-harness might be cloned */ +const BH_SEARCH_PATHS = [ + join(homedir(), "Developer", "browser-harness"), + join(homedir(), "src", "browser-harness"), + join(homedir(), "browser-harness"), + join(homedir(), "dev", "browser-harness"), + join(homedir(), "Projects", "browser-harness"), +]; + +const DAEMON_START_TIMEOUT_MS = 30_000; +const DAEMON_POLL_INTERVAL_MS = 200; +const DAEMON_SHUTDOWN_TIMEOUT_MS = 5_000; +const SESSION_ATTACH_TIMEOUT_MS = 5_000; + +const VIRTUAL_KEY_CODES: Record<string, number> = { + Enter: 13, + Tab: 9, + Backspace: 8, + Escape: 27, + Delete: 46, + " ": 32, + ArrowLeft: 37, + ArrowUp: 38, + ArrowRight: 39, + ArrowDown: 40, + Home: 36, + End: 35, + PageUp: 33, + PageDown: 34, +}; + +// ── BrowserDaemon ──────────────────────────────────────────────────────────── + +export class BrowserDaemon { + readonly namespace: string; + readonly socketPath: string; + readonly pidPath: string; + readonly logPath: string; + + private proc: ChildProcess | null = null; + private _sessionId: string | null = null; + private _pid: number | null = null; + private remote: RemoteConfig | null = null; + + constructor(namespace = "default", remote?: RemoteConfig) { + this.namespace = namespace; + this.socketPath = `/tmp/bu-${namespace}.sock`; + this.pidPath = `/tmp/bu-${namespace}.pid`; + this.logPath = `/tmp/bu-${namespace}.log`; + this.remote = remote ?? null; + } + + // ── Public getters ─────────────────────────────────────────────────────── + + get sessionId(): string | null { + return this._sessionId; + } + + get pid(): number | null { + return this._pid; + } + + getStatus(): DaemonStatus { + return { + alive: this._sessionId !== null, + sessionId: this._sessionId, + pid: this._pid, + namespace: this.namespace, + socketPath: this.socketPath, + remoteBrowserId: this.remote?.browserId, + }; + } + + // ── Lifecycle ──────────────────────────────────────────────────────────── + + /** + * Start the daemon process and wait for it to become ready. + * Idempotent — returns immediately if already alive and healthy. + */ + async start(): Promise<void> { + // Already healthy? + if (await healthProbe(this.socketPath)) { + await this.refreshSessionId(); + return; + } + + // Stale socket but daemon alive? Stop it first. + if (await daemonAlive(this.socketPath)) { + await this.stop(); + } + + const bhDir = await findBrowserHarnessDir(); + const env = this.buildEnv(); + + this.proc = spawn("uv", ["run", "daemon.py"], { + cwd: bhDir, + env: { ...process.env, ...env }, + stdio: ["ignore", "ignore", "ignore"], + detached: true, + }); + this.proc.unref(); + + if (this.proc.pid) { + this._pid = this.proc.pid; + } + + // Poll until socket accepts connections + const deadline = Date.now() + DAEMON_START_TIMEOUT_MS; + let lastError = ""; + + while (Date.now() < deadline) { + if (this.proc.exitCode !== null) { + lastError = await this.readLogTail(); + throw new Error( + `Browser daemon exited (code ${this.proc.exitCode}): ${lastError || "unknown error"}`, + ); + } + + if (await daemonAlive(this.socketPath)) { + // Verify CDP is healthy (not just socket) + if (await healthProbe(this.socketPath)) { + await this.refreshSessionId(); + return; + } + // Socket alive but CDP not ready — keep polling + } + + await sleep(DAEMON_POLL_INTERVAL_MS); + } + + lastError = await this.readLogTail(); + // Check if Chrome needs remote debugging setup + if (this.needsChromeSetup(lastError)) { + throw new Error( + `Chrome remote debugging not enabled. Start Chrome with --remote-debugging-port=9222, then retry.\n\nDaemon log: ${lastError}`, + ); + } + + throw new Error( + `Daemon "${this.namespace}" failed to start after ${DAEMON_START_TIMEOUT_MS / 1000}s. ` + + `Log: ${lastError || "no log output"}`, + ); + } + + /** + * Gracefully stop the daemon: send shutdown, wait for exit, force-kill if needed. + */ + async stop(): Promise<void> { + // Send shutdown via socket + try { + await sendRequest(this.socketPath, { meta: "shutdown" }, 3000); + } catch { + // Socket may already be dead — that's fine + } + + // Wait for process to exit + if (this.proc && this.proc.exitCode === null) { + const deadline = Date.now() + DAEMON_SHUTDOWN_TIMEOUT_MS; + while (Date.now() < deadline) { + if (this.proc.exitCode !== null) break; + await sleep(100); + } + + // Force kill if still alive + if (this.proc.exitCode === null) { + try { + this.proc.kill("SIGTERM"); + } catch { + // already dead + } + } + } + + // Clean up files + for (const f of [this.socketPath, this.pidPath]) { + try { + await unlink(f); + } catch { + // file doesn't exist + } + } + + this.proc = null; + this._sessionId = null; + this._pid = null; + } + + /** + * Ensure daemon is alive and healthy. Restarts if stale. + * Call before every tool execution. + */ + async ensureAlive(): Promise<void> { + if (!(await healthProbe(this.socketPath))) { + await this.stop(); + await this.start(); + } + } + + // ── CDP Communication ──────────────────────────────────────────────────── + + /** + * Send a raw request to the daemon and get the response. + */ + async send(request: DaemonRequest): Promise<DaemonResponse> { + await this.ensureAlive(); + + try { + const response = await sendRequest(this.socketPath, request); + return response; + } catch (err) { + // Socket died mid-request — restart and retry once + const message = err instanceof Error ? err.message : String(err); + if (message.includes("ENOENT") || message.includes("ECONNREFUSED") || message.includes("EPIPE")) { + await this.stop(); + await this.start(); + return sendRequest(this.socketPath, request); + } + throw err; + } + } + + /** + * Convenience: send a CDP method and return the result. + * Handles session management — Target.* methods skip session, + * others use explicit sessionId or the daemon's default session. + */ + async cdp( + method: string, + params?: Record<string, unknown>, + sessionId?: string | null, + ): Promise<unknown> { + // Browser-level Target.* calls must NOT use a session + const sid = method.startsWith("Target.") ? null : (sessionId ?? this._sessionId); + + const response = await this.send({ + method, + params: params ?? {}, + session_id: sid, + }); + + if (response.error) { + // Auto-recover from stale session + if ( + response.error.includes("Session with given id not found") && + sid === this._sessionId + ) { + await this.attachFirstPage(); + const retry = await this.send({ + method, + params: params ?? {}, + session_id: this._sessionId, + }); + if (retry.error) { + throw new Error(`CDP error after session recovery (${method}): ${retry.error}`); + } + return retry.result; + } + throw new Error(`CDP error (${method}): ${response.error}`); + } + + return response.result; + } + + // ── High-level Browser Helpers ──────────────────────────────────────────── + + /** Get page state or dialog info */ + async getPageInfo(): Promise<PageInfoResult> { + // Check for JS dialog first + try { + const dialogResp = await sendRequest(this.socketPath, { meta: "pending_dialog" }); + if (dialogResp.dialog) { + return { dialog: dialogResp.dialog }; + } + } catch { + // ignore — dialog check is best-effort + } + + const js = `JSON.stringify({url:location.href,title:document.title,w:innerWidth,h:innerHeight,sx:scrollX,sy:scrollY,pw:document.documentElement.scrollWidth,ph:document.documentElement.scrollHeight})`; + const raw = await this.evaluateJS(js); + return JSON.parse(raw as string) as PageInfo; + } + + /** Capture a PNG screenshot, write to path, return path */ + async captureScreenshot(outputPath: string, fullPage = false): Promise<string> { + const result = (await this.cdp("Page.captureScreenshot", { + format: "png", + captureBeyondViewport: fullPage, + })) as { data: string }; + + const buf = Buffer.from(result.data, "base64"); + await writeFile(outputPath, buf); + return outputPath; + } + + /** List browser tabs */ + async listTabs(includeChrome = true): Promise<TabInfo[]> { + const result = (await this.cdp("Target.getTargets")) as { + targetInfos: Array<{ targetId: string; type: string; title: string; url: string }>; + }; + + const tabs: TabInfo[] = []; + for (const t of result.targetInfos) { + if (t.type !== "page") continue; + if (!includeChrome && isInternalUrl(t.url)) continue; + tabs.push({ targetId: t.targetId, title: t.title || "", url: t.url || "" }); + } + return tabs; + } + + /** Get current tab info */ + async currentTab(): Promise<TabInfo> { + const result = (await this.cdp("Target.getTargetInfo")) as { + targetInfo: { targetId: string; url: string; title: string }; + }; + const t = result.targetInfo; + return { targetId: t.targetId, url: t.url || "", title: t.title || "" }; + } + + /** Create a new tab and switch to it. Optionally navigate. */ + async newTab(url?: string): Promise<string> { + const result = (await this.cdp("Target.createTarget", { + url: "about:blank", + })) as { targetId: string }; + + await this.switchTab(result.targetId); + if (url && url !== "about:blank") { + await this.cdp("Page.navigate", { url }); + } + return result.targetId; + } + + /** Activate and attach to a tab */ + async switchTab(targetId: string): Promise<void> { + await this.cdp("Target.activateTarget", { targetId }); + const result = (await this.cdp("Target.attachToTarget", { + targetId, + flatten: true, + })) as { sessionId: string }; + + // Tell the daemon to use this as default session + this._sessionId = result.sessionId; + await sendRequest(this.socketPath, { + meta: "set_session", + session_id: result.sessionId, + }); + + // Mark the tab with a green circle for user visibility + try { + await this.cdp("Runtime.evaluate", { + expression: `if(!document.title.startsWith('🟢'))document.title='🟢 '+document.title`, + }); + } catch { + // best-effort + } + } + + /** Evaluate JavaScript in the page (or a specific iframe target) */ + async evaluateJS(expression: string, targetId?: string): Promise<unknown> { + // Auto-wrap return statements in IIFE + if (expression.includes("return ") && !expression.trim().startsWith("(")) { + expression = `(function(){${expression}})()`; + } + + const sid = targetId || null; // explicit target or daemon default + const result = (await this.cdp( + "Runtime.evaluate", + { + expression, + returnByValue: true, + awaitPromise: true, + }, + targetId ? sid : undefined, + )) as { result?: { value?: unknown }; exceptionDetails?: unknown }; + + if (result.exceptionDetails) { + throw new Error(`JS evaluation failed: ${JSON.stringify(result.exceptionDetails)}`); + } + return result.result?.value; + } + + /** Drain buffered CDP events from the daemon */ + async drainEvents(): Promise<CDPEvent[]> { + const resp = await sendRequest(this.socketPath, { meta: "drain_events" }); + return resp.events ?? []; + } + + /** Get virtual key code for a key name or character */ + getVirtualKeyCode(key: string): number { + return VIRTUAL_KEY_CODES[key] ?? (key.length === 1 ? key.charCodeAt(0) : 0); + } + + /** Get the key code string for CDP (uses the key name or the key itself) */ + getKeyCode(key: string): string { + return key.length === 1 && !VIRTUAL_KEY_CODES[key] ? key : key; + } + + // ── Internal Helpers ────────────────────────────────────────────────────── + + /** Attach to the first real page target. Creates about:blank if none exist. */ + private async attachFirstPage(): Promise<string> { + const result = (await this.cdp("Target.getTargets")) as { + targetInfos: Array<{ targetId: string; type: string; url: string }>; + }; + + const pages = result.targetInfos.filter( + (t) => t.type === "page" && !isInternalUrl(t.url), + ); + + if (pages.length === 0) { + const created = (await this.cdp("Target.createTarget", { + url: "about:blank", + })) as { targetId: string }; + pages.push({ targetId: created.targetId, type: "page", url: "about:blank" }); + } + + const attachResult = (await this.cdp("Target.attachToTarget", { + targetId: pages[0].targetId, + flatten: true, + })) as { sessionId: string }; + + this._sessionId = attachResult.sessionId; + + // Enable core domains + for (const domain of ["Page", "DOM", "Runtime", "Network"]) { + try { + await this.cdp(`${domain}.enable`); + } catch { + // best-effort + } + } + + return attachResult.sessionId; + } + + /** Refresh session ID from the daemon's current session */ + private async refreshSessionId(): Promise<void> { + try { + const resp = await sendRequest(this.socketPath, { meta: "session" }); + if (resp.session_id) { + this._sessionId = resp.session_id; + } + } catch { + // will be set on first attach + } + } + + /** Build environment variables for the daemon process */ + private buildEnv(): Record<string, string> { + const env: Record<string, string> = { + BU_NAME: this.namespace, + }; + + if (this.remote) { + env.BU_CDP_WS = this.remote.cdpUrl; + env.BU_BROWSER_ID = this.remote.browserId; + } + + return env; + } + + /** Read the last line of the daemon log */ + private async readLogTail(): Promise<string> { + try { + const content = await readFile(this.logPath, "utf-8"); + const lines = content.trim().split("\n"); + return lines[lines.length - 1] || ""; + } catch { + return ""; + } + } + + /** Check if the error message suggests Chrome needs remote debugging setup */ + private needsChromeSetup(errorMsg: string): boolean { + const lower = errorMsg.toLowerCase(); + return ( + lower.includes("devtoolsactiveport not found") || + lower.includes("enable chrome://inspect") || + lower.includes("not live yet") || + (lower.includes("ws handshake failed") && + (lower.includes("403") || lower.includes("timed out"))) + ); + } +} + +// ── Utility ────────────────────────────────────────────────────────────────── + +function sleep(ms: number): Promise<void> { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Find the browser-harness repository directory. + * Searches common locations and falls back to cwd. + */ +async function findBrowserHarnessDir(): Promise<string> { + // Check common locations + for (const dir of BH_SEARCH_PATHS) { + if ( + existsSync(join(dir, "daemon.py")) && + existsSync(join(dir, "helpers.py")) + ) { + return dir; + } + } + + // Check cwd + const cwd = process.cwd(); + if (existsSync(join(cwd, "daemon.py")) && existsSync(join(cwd, "helpers.py"))) { + return cwd; + } + + throw new Error( + "browser-harness not found. Clone it first:\n" + + " git clone https://github.com/browser-use/browser-harness ~/Developer/browser-harness\n" + + " cd ~/Developer/browser-harness && uv sync\n\n" + + `Searched: ${BH_SEARCH_PATHS.join(", ")}`, + ); +} diff --git a/src/dynamic-tools.ts b/src/dynamic-tools.ts new file mode 100644 index 0000000..5aecaf8 --- /dev/null +++ b/src/dynamic-tools.ts @@ -0,0 +1,335 @@ +/** + * dynamic-tools.ts — self-writing tools for pi-browser-harness. + * + * Mirrors the browser-harness philosophy: the agent writes what's missing, + * mid-task. No framework, no recipes, no rails. + * + * Provides three management tools: + * list_dynamic_tools — see what's already registered + * register_tool — inject a new tool into the harness + * remove_tool — retire a tool that's no longer needed + * + * Tools registered via register_tool (which internally calls pi.registerTool()) + * are available immediately in the same session — the LLM can call them on + * the very next sub-turn within the same agent processing cycle, without + * needing /reload. The agent decides to extend the harness the same way + * it decides to write any code: it sees a gap and fills it. + */ + +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import { Type } from "typebox"; +import type { BrowserDaemon } from "./daemon"; + +// ── Types ──────────────────────────────────────────────────────────────────── + +interface DynamicToolDef { + name: string; + label: string; + description: string; + execute: (params: Record<string, unknown>, signal: AbortSignal | undefined, onUpdate: ((update: unknown) => void) | undefined, ctx: unknown) => Promise<{ content: Array<{ type: "text"; text: string }>; details?: Record<string, unknown> }>; +} + +// ── AsyncFunction constructor ──────────────────────────────────────────────── + +const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor as new (...args: string[]) => (...args: unknown[]) => Promise<unknown>; + +// ── Registration ───────────────────────────────────────────────────────────── + +export function registerDynamicTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { + const registry = new Map<string, DynamicToolDef>(); + + // ═══════════════════════════════════════════════════════════════════════════ + // list_dynamic_tools + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "list_dynamic_tools", + label: "List Dynamic Tools", + description: + "List all dynamically registered tools with their names, labels, and descriptions. " + + "Call this FIRST when you're unsure whether a needed capability already exists — " + + "before writing a new tool.", + promptSnippet: "List all dynamically registered tools (names + descriptions)", + promptGuidelines: [ + "Call list_dynamic_tools before register_tool to avoid duplicating existing capabilities.", + "If a needed tool exists but has the wrong shape, use remove_tool to replace it with a corrected version.", + ], + parameters: Type.Object({}), + async execute() { + if (registry.size === 0) { + return { + content: [{ + type: "text" as const, + text: "No dynamic tools registered yet. The harness is clean — use register_tool to teach it something new.", + }], + details: { tools: [], count: 0 }, + }; + } + + const tools = Array.from(registry.values()).map((t) => ({ + name: t.name, + label: t.label, + description: t.description, + })); + + return { + content: [{ + type: "text" as const, + text: `${tools.length} dynamic tool(s) registered:\n\n` + + tools.map((t) => `• ${t.name} — ${t.description}`).join("\n"), + }], + details: { tools, count: tools.length }, + }; + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // register_tool + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "register_tool", + label: "Register Dynamic Tool", + description: + "Register a new tool by providing its JavaScript implementation. " + + "The tool becomes available immediately in the same session — " + + "the LLM can call it on the very next sub-turn within the same " + + "agent processing cycle, without reloading. " + + "Use this whenever you discover you need a capability the harness doesn't " + + "already have. Don't work around limitations, extend the harness.\n\n" + + "The implementation runs as an async function body. It receives these bindings:\n" + + " params — the tool's arguments (as passed by the caller)\n" + + " daemon — the browser daemon (for composing browser actions: daemon.cdp(), daemon.evaluateJS(), daemon.getPageInfo(), etc.)\n" + + " require — Node.js require() for builtins and installed packages\n" + + " signal — AbortSignal for cancellation\n" + + " onUpdate — progress callback: onUpdate({ content: [{ type: 'text', text: '...' }] })\n" + + " ctx — ExtensionContext with cwd, sessionManager, ui, signal, etc.\n" + + " console, fetch, JSON, Buffer, setTimeout, clearTimeout\n\n" + + "The implementation MUST return { content: [{ type: 'text', text: '...' }], details?: {...} }.\n" + + "For errors, throw: throw new Error('something went wrong').\n\n" + + "Example implementation:\n" + + ' const info = await daemon.getPageInfo();\n' + + ' if ("dialog" in info) throw new Error("Dialog is blocking the page");\n' + + ' return { content: [{ type: "text", text: `Current URL: ${info.url}` }], details: { url: info.url } };', + promptSnippet: "Register a new tool with custom JavaScript logic (available immediately, next sub-turn)", + promptGuidelines: [ + "Use register_tool when you need a capability not provided by the built-in browser_* tools. The new tool is available immediately — the LLM can call it on the very next sub-turn within the same agent processing cycle, without reloading.", + "Before registering, always call list_dynamic_tools to check if a suitable tool already exists.", + "The implementation has access to the browser daemon via `daemon` — use daemon.cdp(), daemon.evaluateJS(), daemon.getPageInfo(), etc. to compose browser actions inside the tool.", + "For pure data processing or file operations, use require() to access Node.js modules like fs, path, crypto.", + "If the implementation needs existing browser tools (browser_navigate, browser_screenshot, etc.), break those into separate tool calls before or after the dynamic tool — do not try to invoke them from within the implementation.", + ], + parameters: Type.Object({ + name: Type.String({ + description: "Unique tool name in snake_case (e.g. 'scrape_paginated_results', 'extract_structured_data')", + }), + label: Type.String({ + description: "Human-readable label for the tool (e.g. 'Scrape Paginated Results')", + }), + description: Type.String({ + description: "What the tool does, including expected parameter names and their types. This is shown to the LLM when it considers which tool to call.", + }), + implementation: Type.String({ + description: + "JavaScript code (async function body). Bindings: params, daemon, require, signal, onUpdate, ctx, console, fetch, JSON, Buffer, setTimeout, clearTimeout. Must return { content: [{ type: 'text', text: '...' }], details?: {...} }.", + }), + }), + async execute(_id, p, _signal, _onUpdate) { + const { name, label, description, implementation } = p as { + name: string; + label: string; + description: string; + implementation: string; + }; + + // ── Validation ────────────────────────────────────────────────── + if (!/^[a-z][a-z0-9_]*$/.test(name)) { + return { + content: [{ + type: "text" as const, + text: `Invalid tool name '${name}'. Must be snake_case: lowercase letters, digits, and underscores, starting with a letter.`, + }], + details: { error: "invalid_name" }, + }; + } + + if (registry.has(name)) { + const existing = registry.get(name)!; + return { + content: [{ + type: "text" as const, + text: `Tool '${name}' already exists: ${existing.description}\n\nUse remove_tool("${name}") first if you want to replace it, then call register_tool again.`, + }], + details: { exists: true, existingLabel: existing.label }, + }; + } + + // ── Compile ───────────────────────────────────────────────────── + let executeFn: (...args: unknown[]) => Promise<unknown>; + try { + // The LLM writes a function body. We wrap it in an async function. + // Prepend "use strict" so accidental global leaks become errors. + const source = `"use strict";\n${implementation}`; + executeFn = new AsyncFunction( + "params", "daemon", "require", "signal", "onUpdate", "ctx", + "console", "fetch", "JSON", "Buffer", "setTimeout", "clearTimeout", + source, + ); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + return { + content: [{ + type: "text" as const, + text: `Syntax error in implementation: ${msg}\n\nFix the JavaScript syntax and try again.`, + }], + details: { error: "syntax_error", message: msg }, + }; + } + + // ── Smoke test ────────────────────────────────────────────────── + // Run the function once with empty params to catch obvious errors early. + try { + const testResult = await executeFn( + {}, + daemon, + require, + undefined, + (_update: unknown) => {}, + { cwd: process.cwd() }, + console, + fetch, + JSON, + Buffer, + setTimeout, + clearTimeout, + ); + if (!testResult || typeof testResult !== "object" || !Array.isArray((testResult as any).content)) { + return { + content: [{ + type: "text" as const, + text: `Implementation must return an object with a 'content' array. Got: ${JSON.stringify(testResult)}`, + }], + details: { error: "invalid_return" }, + }; + } + } catch (err: unknown) { + // Smoke test failure is ok — the params were empty, so runtime errors + // from missing params are expected. Only reject if the error looks + // like a code bug (e.g. ReferenceError for undefined variables). + const msg = err instanceof Error ? err.message : String(err); + if (msg.includes("is not defined") || msg.includes("Cannot find module")) { + return { + content: [{ + type: "text" as const, + text: `Implementation failed smoke test: ${msg}\n\nCheck for undefined variables, missing imports, or broken require() calls.`, + }], + details: { error: "smoke_test_failed", message: msg }, + }; + } + // Otherwise it's likely just missing params — proceed. + } + + // ── Register ──────────────────────────────────────────────────── + const def: DynamicToolDef = { + name, + label, + description, + execute: async (params, signal, onUpdate, ctx) => { + const result = await executeFn( + params, + daemon, + require, + signal, + onUpdate, + ctx, + console, + fetch, + JSON, + Buffer, + setTimeout, + clearTimeout, + ); + return result as { content: Array<{ type: "text"; text: string }>; details?: Record<string, unknown> }; + }, + }; + + registry.set(name, def); + + // Register with pi — available immediately on the next sub-turn + pi.registerTool({ + name, + label, + description, + parameters: Type.Object({}, { additionalProperties: true }), + async execute(_toolCallId, execParams, execSignal, execOnUpdate, execCtx) { + const tool = registry.get(name); + if (!tool) { + throw new Error(`Tool '${name}' was removed. Use list_dynamic_tools to see available tools.`); + } + return tool.execute( + (execParams ?? {}) as Record<string, unknown>, + execSignal, + execOnUpdate, + execCtx, + ); + }, + }); + + return { + content: [{ + type: "text" as const, + text: `✅ Tool '${name}' registered and available immediately.\n\nLabel: ${label}\nDescription: ${description}\n\nYou can now call it as: ${name}(...)`, + }], + details: { name, label }, + }; + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // remove_tool + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "remove_tool", + label: "Remove Dynamic Tool", + description: + "Remove a previously registered dynamic tool. The tool will not be available on subsequent turns. " + + "Use this to replace a broken tool or clean up tools you no longer need.", + promptSnippet: "Remove a dynamically registered tool", + promptGuidelines: [ + "Use remove_tool to retire a dynamic tool you no longer need or to replace it with a corrected version.", + "After removing a tool, you can immediately register a replacement with the same name via register_tool.", + ], + parameters: Type.Object({ + name: Type.String({ description: "Name of the tool to remove (as shown by list_dynamic_tools)" }), + }), + async execute(_id, p) { + const { name } = p as { name: string }; + const existed = registry.has(name); + + if (!existed) { + return { + content: [{ + type: "text" as const, + text: `Tool '${name}' is not in the dynamic registry. Use list_dynamic_tools to see registered tools.`, + }], + details: { removed: false }, + }; + } + + // Remove from registry + registry.delete(name); + + // Deactivate from pi's active tool set so it doesn't appear in the system prompt + const activeNames = pi.getActiveTools() + .map((t: unknown) => (t as { name: string }).name) + .filter((n: string) => n !== name); + pi.setActiveTools(activeNames); + + return { + content: [{ + type: "text" as const, + text: `🗑️ Tool '${name}' removed. It will no longer appear in the system prompt or be callable.`, + }], + details: { removed: true }, + }; + }, + }); +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..2226cd7 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,228 @@ +/** + * pi-browser-harness — browser control extension for pi. + * + * Gives pi agents full control of a real Chrome browser via CDP. + * Spawns the browser-harness daemon, registers browser_* tools, + * and injects browser control guidance into the system prompt. + * + * Install: + * pi install npm:pi-browser-harness + * # or copy to .pi/extensions/pi-browser-harness/ + * + * Commands: + * /browser-setup — guided setup wizard + * /browser-status — show daemon status and current page + * /browser-reload-daemon — restart the daemon + * + * Flags: + * --browser-namespace <name> — override BU_NAME (default: auto) + * --browser-debug-clicks — enable debug click overlay + */ + +import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent"; +import { BrowserDaemon } from "./daemon"; +import { getBrowserSystemPrompt } from "./prompt"; +import { registerRenderers } from "./renderers"; +import { registerSetupCommand } from "./setup"; +import { type BrowserState, defaultState, persistState, restoreState } from "./state"; +import { cleanupTempDirs, registerTools } from "./tools"; +import { registerDynamicTools } from "./dynamic-tools"; + +export default function browserHarnessExtension(pi: ExtensionAPI) { + // ── Resolve namespace ────────────────────────────────────────────────────── + const flagNs = pi.getFlag("browser-namespace") as string | undefined; + const namespace = flagNs || `pi-${Math.random().toString(36).slice(2, 10)}`; + + // ── State ────────────────────────────────────────────────────────────────── + let state: BrowserState = defaultState(namespace); + let daemon: BrowserDaemon | null = null; + + // ── Tool initialization flag ─────────────────────────────────────────────── + let toolsRegistered = false; + + // ═══════════════════════════════════════════════════════════════════════════ + // Flags + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerFlag("browser-namespace", { + description: "Browser daemon namespace (BU_NAME). Default: auto-generated", + type: "string", + }); + pi.registerFlag("browser-debug-clicks", { + description: "Enable debug click overlay (saves annotated screenshots to /tmp)", + type: "boolean", + default: false, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // Commands + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerCommand("browser-status", { + description: "Show browser daemon status and current page", + handler: async (_args, ctx) => { + if (!daemon) { + ctx.ui.notify("Browser daemon not started. Run /browser-setup first.", "warning"); + return; + } + + const status = daemon.getStatus(); + const lines = [ + `Browser Daemon: ${status.alive ? "🟢 Connected" : "🔴 Disconnected"}`, + `Namespace: ${status.namespace}`, + `Socket: ${status.socketPath}`, + `Session: ${status.sessionId || "none"}`, + `PID: ${status.pid || "none"}`, + ]; + + if (status.remoteBrowserId) { + lines.push(`Remote Browser: ${status.remoteBrowserId}`); + } + + // Try to get current page info + if (status.alive) { + try { + const info = await daemon.getPageInfo(); + if ("dialog" in info) { + lines.push(`\n⚠️ Dialog open: ${info.dialog.type} — "${info.dialog.message}"`); + } else { + lines.push(`\nCurrent Page:`); + lines.push(` URL: ${info.url}`); + lines.push(` Title: ${info.title}`); + lines.push(` Viewport: ${info.width}x${info.height}`); + } + } catch { + lines.push("\n(Could not read page info)"); + } + } + + ctx.ui.notify(lines.join("\n"), "info"); + }, + }); + + pi.registerCommand("browser-reload-daemon", { + description: "Restart the browser daemon", + handler: async (_args, ctx) => { + if (!daemon) { + ctx.ui.notify("Browser daemon not started.", "warning"); + return; + } + + ctx.ui.notify("Restarting browser daemon...", "info"); + try { + await daemon.stop(); + await daemon.start(); + ctx.ui.notify("Browser daemon restarted ✓", "info"); + } catch (err) { + ctx.ui.notify( + `Restart failed: ${err instanceof Error ? err.message : String(err)}`, + "error", + ); + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // Session lifecycle + // ═══════════════════════════════════════════════════════════════════════════ + pi.on("session_start", async (_event, ctx) => { + // Restore persisted state (preserving the current namespace from flag) + state = restoreState(ctx, state.namespace); + + // Create daemon instance + daemon = new BrowserDaemon(state.namespace); + + // Attempt to start daemon (will fail gracefully if Chrome not connected) + try { + await daemon.start(); + } catch { + // Don't block — user can run /browser-setup later + } + + // Register tools (once) + if (!toolsRegistered && daemon) { + registerTools(pi, daemon); + registerDynamicTools(pi, daemon); + toolsRegistered = true; + } + + // Register renderers + registerRenderers(pi); + + // Setup command (always available) + if (daemon) { + registerSetupCommand(pi, daemon); + } + + // Update status + if (daemon.getStatus().alive) { + ctx.ui.setStatus("browser", "🟢 Browser connected"); + } else { + ctx.ui.setStatus("browser", "🔴 Browser — run /browser-setup"); + } + }); + + pi.on("session_shutdown", async (_event) => { + // Persist current state + if (state) { + persistState(pi, state); + } + + // Stop daemon + if (daemon) { + try { + await daemon.stop(); + } catch { + // best-effort + } + daemon = null; + } + + toolsRegistered = false; + + // Clean up temp files from truncated tool outputs + await cleanupTempDirs(); + }); + + // Restore state on tree navigation + pi.on("session_tree", async (_event, ctx) => { + state = restoreState(ctx, daemon?.namespace); + if (daemon) { + persistState(pi, state); + } + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // System prompt injection + // ═══════════════════════════════════════════════════════════════════════════ + pi.on("before_agent_start", async (event) => { + if (!daemon || !daemon.getStatus().alive) { + // Browser not connected — inject minimal note + return { + systemPrompt: + event.systemPrompt + + `\n\n## Browser Control\n\nBrowser tools (browser_*) are available but the browser daemon is not connected. ` + + `Run /browser-setup to connect to Chrome, or /browser-status to check.`, + }; + } + + return { + systemPrompt: event.systemPrompt + getBrowserSystemPrompt(), + }; + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // Tool result hook: track tab history + // ═══════════════════════════════════════════════════════════════════════════ + pi.on("tool_result", async (event) => { + if (event.toolName === "browser_new_tab" || event.toolName === "browser_navigate") { + // Track new tabs in history + const details = event.details as { targetId?: string } | undefined; + if (details?.targetId) { + state = { + ...state, + tabHistory: [details.targetId, ...state.tabHistory.filter((id) => id !== details.targetId)].slice(0, 20), + }; + persistState(pi, state); + } + } + }); +} diff --git a/src/prompt.ts b/src/prompt.ts new file mode 100644 index 0000000..26c41b5 --- /dev/null +++ b/src/prompt.ts @@ -0,0 +1,78 @@ +/** + * System prompt guidance for browser usage. + * + * Injects a compact browser control reference into the pi system prompt. + * Per-tool guidelines are injected via promptGuidelines on each tool registration. + * This prompt provides only the cross-cutting patterns and tool reference table. + */ + +export function getBrowserSystemPrompt(): string { + return ` +## Browser Control + +You have full control of a real Chrome browser via \`browser_*\` tools. +The browser connects to the user's running Chrome (not a headless instance). + +**These tools coexist with all other standard pi tools** — read, bash, edit, +write, and any other built-in tools are still available. Use browser_* tools +to interact with pages visually, and other tools alongside them. + +### Browser-Specific Tools + +| Tool | Purpose | +|------|---------| +| browser_screenshot | Capture a PNG screenshot of the current page | +| browser_click | Click at viewport coordinates | +| browser_type | Type text into the focused element | +| browser_press_key | Press a keyboard key (Enter, Tab, Escape, arrows, etc.) | +| browser_scroll | Scroll the page at coordinates | +| browser_navigate | Navigate to a URL | +| browser_new_tab | Open a new tab, optionally navigate | +| browser_open_urls | Open multiple URLs in new tabs (parallel) | +| browser_go_back / browser_go_forward / browser_reload | History navigation | +| browser_page_info | Get page URL, title, viewport, scroll position, or dialog info | +| browser_list_tabs | List all open browser tabs | +| browser_current_tab | Get current tab info | +| browser_switch_tab | Switch to a different tab by targetId | +| browser_execute_js | Execute JavaScript and return the result | +| browser_http_get | Direct HTTP GET (outside browser, for APIs) | +| browser_wait | Wait N seconds | +| browser_wait_for_load | Wait for document.readyState === 'complete' | +| browser_handle_dialog | Accept or dismiss a JS dialog | + +### Common Patterns + +**Navigation:** +\`\`\` +browser_new_tab("https://example.com") → browser_wait_for_load() → browser_screenshot() +\`\`\` + +**Form filling:** +\`\`\` +browser_screenshot() → find input coordinates → browser_click(x, y) +→ browser_type("text") → browser_press_key("Tab") → browser_screenshot() +\`\`\` + +**Data extraction:** +\`\`\` +browser_execute_js("document.querySelector('.price').innerText") +// or for APIs: +browser_http_get("https://api.example.com/data") +\`\`\` + +**Scrolling:** +\`\`\` +browser_screenshot() → browser_scroll({ deltaY: -500 }) → browser_screenshot() +\`\`\` + +**Research Workflow (search + browser):** +\`\`\` +browser_navigate("https://google.com/search?q=...") → search engine in a tab +browser_open_urls(urls: ["url1", "url2", ...]) → open result pages in parallel tabs +browser_list_tabs() → see all open tabs with targetIds +browser_switch_tab(targetId: "...") → switch to a tab +browser_screenshot() → visually inspect the page +browser_execute_js("document.querySelector('.main').innerText") → extract content +\`\`\` +`; +} diff --git a/src/protocol.ts b/src/protocol.ts new file mode 100644 index 0000000..8edc79d --- /dev/null +++ b/src/protocol.ts @@ -0,0 +1,201 @@ +/** + * Protocol types and low-level socket communication for the browser-harness daemon. + * + * The daemon exposes a Unix socket at /tmp/bu-<NAME>.sock. + * Protocol: one JSON-line request → one JSON-line response per connection. + */ + +import { createConnection } from "node:net"; + +// ── Request ────────────────────────────────────────────────────────────────── + +export interface DaemonRequest { + /** CDP method (e.g. "Page.navigate") — absent for meta requests */ + method?: string; + /** CDP parameters */ + params?: Record<string, unknown>; + /** CDP session ID — null/absent for browser-level (Target.*) calls */ + session_id?: string | null; + /** Meta command: "drain_events" | "set_session" | "pending_dialog" | "shutdown" */ + meta?: string; +} + +// ── Response ───────────────────────────────────────────────────────────────── + +export interface DaemonResponse { + result?: unknown; + error?: string; + events?: CDPEvent[]; + session_id?: string; + dialog?: DialogInfo; + ok?: boolean; +} + +// ── CDP Event ──────────────────────────────────────────────────────────────── + +export interface CDPEvent { + method: string; + params: unknown; + session_id?: string; +} + +// ── Dialog ─────────────────────────────────────────────────────────────────── + +export interface DialogInfo { + type: "alert" | "confirm" | "prompt" | "beforeunload"; + message: string; + defaultPrompt?: string; +} + +// ── Tab ────────────────────────────────────────────────────────────────────── + +export interface TabInfo { + targetId: string; + title: string; + url: string; +} + +// ── Page State ─────────────────────────────────────────────────────────────── + +export interface PageInfo { + url: string; + title: string; + width: number; + height: number; + scrollX: number; + scrollY: number; + pageWidth: number; + pageHeight: number; +} + +export type PageInfoResult = PageInfo | { dialog: DialogInfo }; + +// ── Daemon Status ──────────────────────────────────────────────────────────── + +export interface DaemonStatus { + alive: boolean; + sessionId: string | null; + pid: number | null; + namespace: string; + socketPath: string; + remoteBrowserId?: string; +} + +// ── Remote Config ──────────────────────────────────────────────────────────── + +export interface RemoteConfig { + cdpUrl: string; + browserId: string; +} + +// ── Internal URL prefixes to filter ────────────────────────────────────────── + +const INTERNAL_PREFIXES = [ + "chrome://", + "chrome-untrusted://", + "devtools://", + "chrome-extension://", + "about:", +]; + +export function isInternalUrl(url: string): boolean { + return INTERNAL_PREFIXES.some((p) => url.startsWith(p)); +} + +// ── Low-level socket send ──────────────────────────────────────────────────── + +/** + * Send a single JSON-line request to the daemon socket and read one JSON-line response. + * Opens a fresh connection per request (stateless, matches helpers.py _send pattern). + */ +export function sendRequest( + socketPath: string, + request: DaemonRequest, + timeoutMs = 15_000, +): Promise<DaemonResponse> { + return new Promise((resolve, reject) => { + const socket = createConnection(socketPath); + + const timer = setTimeout(() => { + socket.destroy(); + reject(new Error(`Daemon request timed out after ${timeoutMs}ms: ${JSON.stringify(request.method ?? request.meta)}`)); + }, timeoutMs); + + let buffer = ""; + + socket.on("connect", () => { + socket.write(JSON.stringify(request) + "\n"); + }); + + socket.on("data", (chunk: Buffer) => { + buffer += chunk.toString(); + // Response is one JSON line ending with \n + if (buffer.includes("\n")) { + clearTimeout(timer); + const line = buffer.split("\n")[0]; + try { + const response: DaemonResponse = JSON.parse(line); + socket.end(); + resolve(response); + } catch (e) { + socket.end(); + reject(new Error(`Failed to parse daemon response: ${line.slice(0, 200)}`)); + } + } + }); + + socket.on("error", (err: NodeJS.ErrnoException) => { + clearTimeout(timer); + socket.destroy(); + reject(new Error(`Daemon socket error (${socketPath}): ${err.message}`)); + }); + + socket.on("close", () => { + clearTimeout(timer); + if (buffer.length === 0) { + reject(new Error(`Daemon socket closed without response (${socketPath})`)); + } + }); + }); +} + +/** + * Quick check if the daemon socket accepts connections. + */ +export function daemonAlive(socketPath: string, timeoutMs = 1000): Promise<boolean> { + return new Promise((resolve) => { + const socket = createConnection(socketPath); + const timer = setTimeout(() => { + socket.destroy(); + resolve(false); + }, timeoutMs); + + socket.on("connect", () => { + clearTimeout(timer); + socket.end(); + resolve(true); + }); + + socket.on("error", () => { + clearTimeout(timer); + resolve(false); + }); + }); +} + +/** + * True health check: send Target.getTargets and verify "result" key in response. + * A stale daemon can accept socket connections but have a dead CDP websocket. + */ +export async function healthProbe(socketPath: string): Promise<boolean> { + try { + const response = await sendRequest( + socketPath, + { method: "Target.getTargets", params: {} }, + 3000, + ); + return "result" in response && response.result !== undefined; + } catch { + return false; + } +} diff --git a/src/renderers.ts b/src/renderers.ts new file mode 100644 index 0000000..5882ff4 --- /dev/null +++ b/src/renderers.ts @@ -0,0 +1,21 @@ +/** + * TUI renderers for pi-browser-harness. + * + * Registers custom message renderers and tool renderCall/renderResult + * for browser tools. + */ + +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import { Text } from "@mariozechner/pi-tui"; + +export function registerRenderers(pi: ExtensionAPI): void { + // Register a message renderer for screenshot notifications + pi.registerMessageRenderer("browser-screenshot", (message, _options, theme) => { + const path = (message as { details?: { path?: string } }).details?.path || "unknown"; + return new Text( + theme.fg("accent", `📸 Screenshot: ${path}`), + 0, + 0, + ); + }); +} diff --git a/src/setup.ts b/src/setup.ts new file mode 100644 index 0000000..839a540 --- /dev/null +++ b/src/setup.ts @@ -0,0 +1,178 @@ +/** + * Setup command — guides the user through installing and connecting + * browser-harness to their Chrome browser. + */ + +import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent"; +import { execSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { BrowserDaemon } from "./daemon"; + +export function registerSetupCommand(pi: ExtensionAPI, daemon: BrowserDaemon): void { + pi.registerCommand("browser-setup", { + description: "Install and connect browser-harness to Chrome", + handler: async (_args, ctx) => { + await runSetup(ctx, daemon); + }, + }); +} + +async function runSetup(ctx: ExtensionContext, daemon: BrowserDaemon): Promise<void> { + ctx.ui.notify("browser-harness setup: checking installation...", "info"); + + // Step 1: Check for browser-harness installation + const bhInstalled = await checkBrowserHarnessInstalled(); + if (!bhInstalled) { + ctx.ui.notify( + "browser-harness not found. Installing...", + "warning", + ); + const installed = await installBrowserHarness(ctx); + if (!installed) { + ctx.ui.notify( + "Installation failed. Please install manually:\n" + + " git clone https://github.com/browser-use/browser-harness ~/Developer/browser-harness\n" + + " cd ~/Developer/browser-harness && uv sync", + "error", + ); + return; + } + } + + ctx.ui.notify("browser-harness found ✓", "info"); + + // Step 2: Check Chrome is running + const chromeRunning = checkChromeRunning(); + if (!chromeRunning) { + ctx.ui.notify( + "Chrome/Chromium/Edge not detected. Please start your browser and retry /browser-setup.", + "error", + ); + return; + } + ctx.ui.notify("Chrome is running ✓", "info"); + + // Step 3: Try to start the daemon + ctx.ui.notify("Connecting to Chrome...", "info"); + try { + await daemon.start(); + ctx.ui.notify("Daemon connected ✓", "info"); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + const lower = msg.toLowerCase(); + + if ( + lower.includes("devtoolsactiveport") || + lower.includes("remote debugging") + ) { + ctx.ui.notify( + "Chrome remote debugging needs to be enabled.\n\n" + + "Start Chrome with: --remote-debugging-port=9222\n" + + "Then run /browser-setup again.", + "warning", + ); + return; + } + + ctx.ui.notify(`Connection failed: ${msg}`, "error"); + return; + } + + // Step 4: Verify with test navigation + ctx.ui.notify("Testing browser control...", "info"); + try { + await daemon.newTab("https://github.com/browser-use/browser-harness"); + const info = await daemon.getPageInfo(); + if ("dialog" in info) { + await daemon.cdp("Page.handleJavaScriptDialog", { accept: true }); + } + ctx.ui.notify( + `Browser control verified ✓\nNavigated to: ${"url" in info ? info.url : "github.com"}`, + "info", + ); + } catch (err) { + ctx.ui.notify( + `Browser connected but test navigation failed: ${err instanceof Error ? err.message : String(err)}`, + "warning", + ); + } +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +async function checkBrowserHarnessInstalled(): Promise<boolean> { + const searchPaths = [ + join(homedir(), "Developer", "browser-harness"), + join(homedir(), "src", "browser-harness"), + join(homedir(), "browser-harness"), + join(homedir(), "dev", "browser-harness"), + join(homedir(), "Projects", "browser-harness"), + process.cwd(), + ]; + + for (const dir of searchPaths) { + if ( + existsSync(join(dir, "daemon.py")) && + existsSync(join(dir, "helpers.py")) + ) { + return true; + } + } + + // Check if uv tool is installed globally + try { + execSync("uv tool list 2>/dev/null | grep browser-harness", { timeout: 5000 }); + return true; + } catch { + // not found via uv + } + + return false; +} + +async function installBrowserHarness(ctx: ExtensionContext): Promise<boolean> { + // Try uv tool install + try { + ctx.ui.notify("Installing browser-harness via uv...", "info"); + execSync("uv tool install browser-harness", { + timeout: 60_000, + stdio: "pipe", + }); + return true; + } catch { + // uv install failed + } + + // Try git clone + try { + const targetDir = join(homedir(), "Developer", "browser-harness"); + ctx.ui.notify(`Cloning browser-harness to ${targetDir}...`, "info"); + execSync( + `git clone https://github.com/browser-use/browser-harness "${targetDir}"`, + { timeout: 30_000, stdio: "pipe" }, + ); + execSync("uv sync", { cwd: targetDir, timeout: 60_000, stdio: "pipe" }); + return true; + } catch { + return false; + } +} + +function checkChromeRunning(): boolean { + try { + if (process.platform === "darwin" || process.platform === "linux") { + const out = execSync("ps -A -o comm=", { timeout: 5000 }).toString().toLowerCase(); + return ["google chrome", "chrome", "chromium", "microsoft edge", "msedge"].some( + (n) => out.includes(n), + ); + } else if (process.platform === "win32") { + const out = execSync("tasklist", { timeout: 5000 }).toString().toLowerCase(); + return ["chrome.exe", "msedge.exe"].some((n) => out.includes(n)); + } + } catch { + // best-effort + } + return false; +} diff --git a/src/state.ts b/src/state.ts new file mode 100644 index 0000000..9423309 --- /dev/null +++ b/src/state.ts @@ -0,0 +1,77 @@ +/** + * Session persistence for pi-browser-harness. + * + * Persists daemon namespace, tab history, and preferences across + * session reloads and branch navigation. + */ + +import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent"; + +// ── Types ──────────────────────────────────────────────────────────────────── + +export interface BrowserState { + /** Daemon namespace (BU_NAME) for this session */ + namespace: string; + /** Recently active tab target IDs */ + tabHistory: string[]; + /** Cloud browser ID (for remote sessions) */ + remoteBrowserId?: string; + /** Screenshot output directory */ + screenshotDir: string; + /** Whether debug click overlay is enabled */ + debugClicks: boolean; +} + +// ── Defaults ───────────────────────────────────────────────────────────────── + +export function defaultState(namespace = "default"): BrowserState { + return { + namespace, + tabHistory: [], + screenshotDir: "", + debugClicks: false, + }; +} + +// ── Persist ────────────────────────────────────────────────────────────────── + +export function persistState(pi: ExtensionAPI, state: BrowserState): void { + pi.appendEntry<BrowserState>("browser-harness-state", state); +} + +// ── Restore ────────────────────────────────────────────────────────────────── + +/** + * Find the last browser-harness-state entry in the current branch + * and return the restored state, merged with defaults. + */ +export function restoreState(ctx: ExtensionContext, currentNamespace?: string): BrowserState { + const branchEntries = ctx.sessionManager.getBranch(); + const defaults = defaultState(currentNamespace); + + // Walk branch from newest to oldest to find last persisted state + for (let i = branchEntries.length - 1; i >= 0; i--) { + const entry = branchEntries[i]; + if (entry.type === "custom" && entry.customType === "browser-harness-state") { + const data = entry.data as Partial<BrowserState> | undefined; + if (data) { + // Merge persisted data onto defaults, but always keep the current + // namespace (which comes from the --browser-namespace flag or auto-gen). + return { ...defaults, ...data, namespace: currentNamespace || data.namespace || defaults.namespace }; + } + } + } + + return defaults; +} + +// ── Merge helpers ──────────────────────────────────────────────────────────── + +export function addToTabHistory(state: BrowserState, targetId: string): BrowserState { + const filtered = state.tabHistory.filter((id) => id !== targetId); + return { ...state, tabHistory: [targetId, ...filtered].slice(0, 20) }; +} + +export function setRemoteBrowser(state: BrowserState, browserId: string): BrowserState { + return { ...state, remoteBrowserId: browserId }; +} diff --git a/src/tools.ts b/src/tools.ts new file mode 100644 index 0000000..befd355 --- /dev/null +++ b/src/tools.ts @@ -0,0 +1,1200 @@ +/** + * Tool registrations for pi-browser-harness. + * + * Maps browser-harness daemon methods to pi tools with TypeBox schemas, + * prompt snippets, guidelines, and TUI renderers. + */ + +import { + DEFAULT_MAX_BYTES, + DEFAULT_MAX_LINES, + formatSize, + truncateHead, + withFileMutationQueue, +} from "@mariozechner/pi-coding-agent"; +import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; +import { Text } from "@mariozechner/pi-tui"; +import { Type } from "typebox"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { BrowserDaemon } from "./daemon"; +import type { TabInfo } from "./protocol"; + +// ── Key code map ───────────────────────────────────────────────────────────── + +const SPECIAL_KEYS = [ + "Enter", "Tab", "Backspace", "Escape", "Delete", + "ArrowLeft", "ArrowUp", "ArrowRight", "ArrowDown", + "Home", "End", "PageUp", "PageDown", " ", +]; + +// ── Screenshot path ────────────────────────────────────────────────────────── + +let screenshotCounter = 0; +function nextScreenshotPath(): string { + screenshotCounter++; + return join(tmpdir(), `pi-browser-screenshot-${Date.now()}-${screenshotCounter}.png`); +} + +// ── Output truncation ──────────────────────────────────────────────────────── + +/** Track temp directories for cleanup on session shutdown. */ +const tempDirs: string[] = []; + +/** Best-effort cleanup of all tracked temp directories. */ +export async function cleanupTempDirs(): Promise<void> { + const dirs = tempDirs.splice(0); + await Promise.all( + dirs.map((d) => rm(d, { recursive: true, force: true }).catch(() => {})), + ); +} + +/** + * Truncate output to pi's built-in limits (50KB / 2000 lines). + * When truncated, writes the full output to a temp file so the LLM + * can read it via the read tool and so CTRL+O expand can show it. + */ +async function applyTruncation( + output: string, + prefix: string, +): Promise<{ + text: string; + fullOutputPath?: string; + wasTruncated: boolean; +}> { + const truncation = truncateHead(output, { + maxLines: DEFAULT_MAX_LINES, + maxBytes: DEFAULT_MAX_BYTES, + }); + + if (!truncation.truncated) { + return { text: truncation.content, wasTruncated: false }; + } + + const tempDir = await mkdtemp(join(tmpdir(), `pi-bh-${prefix}-`)); + tempDirs.push(tempDir); + + const tempFile = join(tempDir, "output.txt"); + await withFileMutationQueue(tempFile, async () => { + await writeFile(tempFile, output, "utf8"); + }); + + const omittedBytes = truncation.totalBytes - truncation.outputBytes; + let text = truncation.content; + text += `\n\n[Output truncated: ${truncation.outputLines} of ${truncation.totalLines} lines`; + text += ` (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}).`; + text += ` ${formatSize(omittedBytes)} omitted. Full output: ${tempFile}]`; + + return { text, fullOutputPath: tempFile, wasTruncated: true }; +} + +// ── Registration ───────────────────────────────────────────────────────────── + +export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { + // ═══════════════════════════════════════════════════════════════════════════ + // browser_navigate + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_navigate", + label: "Browser Navigate", + description: + "Navigate to a URL. Creates a new tab on first call (safe), navigates current tab on subsequent calls.", + promptSnippet: "Navigate to a URL (creates new tab on first call)", + promptGuidelines: [ + "Use browser_navigate to go to URLs. The first call creates a new tab so it doesn't clobber the user's active tab.", + "Use browser_wait_for_load after browser_navigate to wait for the page to finish loading.", + "For extracting data from a page you already navigated to, use browser_execute_js or browser_http_get (faster for APIs).", + ], + parameters: Type.Object({ + url: Type.String({ description: "Full URL to navigate to (e.g. https://github.com)" }), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + + // Determine if we need a new tab: create one on first-ever call, + // or if the current tab is an internal Chrome page. Otherwise + // navigate the current tab in-place. + let targetId: string; + let isNewTab = false; + try { + const tabs = await daemon.listTabs(false); + if (tabs.length === 0) { + // No real tabs exist — create one + targetId = await daemon.newTab(params.url); + isNewTab = true; + } else { + // Navigate the current tab in-place + await daemon.cdp("Page.navigate", { url: params.url }); + const tab = await daemon.currentTab(); + targetId = tab.targetId; + } + } catch { + // Fallback: create a new tab + targetId = await daemon.newTab(params.url); + isNewTab = true; + } + + const info = await daemon.getPageInfo(); + if ("dialog" in info) { + return { + content: [{ type: "text" as const, text: `Navigated to ${params.url}\nTarget: ${targetId}\n⚠️ A dialog is open: ${info.dialog.type} — "${info.dialog.message}"\nUse browser_handle_dialog to dismiss or accept it.` }], + }; + } + + const prefix = isNewTab ? "New tab: " : ""; + return { + content: [{ + type: "text" as const, + text: `${prefix}Navigated to: ${info.url}\nTitle: ${info.title}\nViewport: ${info.width}x${info.height}\nTarget: ${targetId}`, + }], + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Navigation failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + renderCall(args, theme) { + return new Text(`🌐 Navigate to ${(args as { url: string }).url}`, 0, 0); + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_page_info + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_page_info", + label: "Browser Page Info", + description: + "Get current page state: URL, title, viewport size, scroll position, page dimensions. If a JS dialog (alert/confirm/prompt) is open, returns dialog info instead.", + promptSnippet: "Get current page URL, title, viewport, and scroll position", + promptGuidelines: [ + "Use browser_page_info to quickly check what page you're on and whether a JS dialog is blocking interaction.", + "If browser_page_info returns a dialog, use browser_handle_dialog before any other browser actions.", + ], + parameters: Type.Object({}), + async execute() { + try { + await daemon.ensureAlive(); + const info = await daemon.getPageInfo(); + + if ("dialog" in info) { + return { + content: [{ + type: "text" as const, + text: `⚠️ DIALOG OPEN: ${info.dialog.type}\nMessage: ${info.dialog.message}${info.dialog.defaultPrompt ? `\nDefault: ${info.dialog.defaultPrompt}` : ""}\n\nUse browser_handle_dialog to accept or dismiss.`, + }], + }; + } + + return { + content: [{ + type: "text" as const, + text: `URL: ${info.url}\nTitle: ${info.title}\nViewport: ${info.width}x${info.height}\nScroll: (${info.scrollX}, ${info.scrollY})\nPage size: ${info.pageWidth}x${info.pageHeight}`, + }], + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_go_back + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_go_back", + label: "Browser Go Back", + description: "Navigate back one page in history.", + promptSnippet: "Go back one page in browser history", + promptGuidelines: [ + "Use browser_go_back to return to the previous page after navigating.", + ], + parameters: Type.Object({}), + async execute() { + try { + await daemon.ensureAlive(); + const history = (await daemon.cdp("Page.getNavigationHistory")) as { + entries: Array<{ id: number; url: string; title: string }>; + currentIndex: number; + }; + if (history.currentIndex <= 0) { + return { content: [{ type: "text" as const, text: "Already at the beginning of history." }] }; + } + const targetEntry = history.entries[history.currentIndex - 1]; + await daemon.cdp("Page.navigateToHistoryEntry", { entryId: targetEntry.id }); + return { content: [{ type: "text" as const, text: `Navigated back to: ${targetEntry.url}` }] }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_go_forward + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_go_forward", + label: "Browser Go Forward", + description: "Navigate forward one page in history.", + promptSnippet: "Go forward one page in browser history", + promptGuidelines: [ + "Use browser_go_forward to go forward after navigating back.", + ], + parameters: Type.Object({}), + async execute() { + try { + await daemon.ensureAlive(); + const history = (await daemon.cdp("Page.getNavigationHistory")) as { + entries: Array<{ id: number; url: string; title: string }>; + currentIndex: number; + }; + if (history.currentIndex >= history.entries.length - 1) { + return { content: [{ type: "text" as const, text: "Already at the end of history." }] }; + } + const targetEntry = history.entries[history.currentIndex + 1]; + await daemon.cdp("Page.navigateToHistoryEntry", { entryId: targetEntry.id }); + return { content: [{ type: "text" as const, text: `Navigated forward to: ${targetEntry.url}` }] }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_reload + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_reload", + label: "Browser Reload", + description: "Reload the current page.", + promptSnippet: "Reload the current page", + promptGuidelines: [ + "Use browser_reload to refresh the current page, e.g. after making changes that should be reflected.", + ], + parameters: Type.Object({}), + async execute() { + try { + await daemon.ensureAlive(); + await daemon.cdp("Page.reload"); + const info = await daemon.getPageInfo(); + if ("dialog" in info) { + return { content: [{ type: "text" as const, text: `Reloaded. ⚠️ Dialog open: ${info.dialog.type}` }] }; + } + return { content: [{ type: "text" as const, text: `Reloaded: ${info.title} (${info.url})` }] }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_click + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_click", + label: "Browser Click", + description: + "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.", + promptSnippet: "Click at pixel coordinates (x, y) on the page", + promptGuidelines: [ + "Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).", + "Capture a browser_screenshot BEFORE clicking to find the right coordinates.", + "Capture another browser_screenshot AFTER clicking to verify the action worked.", + "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.", + ], + parameters: Type.Object({ + x: Type.Number({ description: "X coordinate in CSS pixels from left edge of viewport" }), + y: Type.Number({ description: "Y coordinate in CSS pixels from top edge of viewport" }), + button: Type.Optional(Type.String({ description: 'Mouse button: "left", "right", or "middle". Default: "left"' })), + clicks: Type.Optional(Type.Number({ description: "Number of clicks (1 = single, 2 = double). Default: 1" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + await daemon.cdp("Input.dispatchMouseEvent", { + type: "mousePressed", + x: params.x, + y: params.y, + button: params.button || "left", + clickCount: params.clicks || 1, + }); + await daemon.cdp("Input.dispatchMouseEvent", { + type: "mouseReleased", + x: params.x, + y: params.y, + button: params.button || "left", + clickCount: params.clicks || 1, + }); + return { content: [{ type: "text" as const, text: `Clicked at (${params.x}, ${params.y})` }] }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Click failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + renderCall(args, theme) { + const { x, y } = args as { x: number; y: number }; + return new Text(`🖱️ Click at (${x}, ${y})`, 0, 0); + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_type + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_type", + label: "Browser Type", + description: "Type text into the currently focused element. Use browser_click first to focus an input field.", + promptSnippet: "Type text into the focused element", + promptGuidelines: [ + "Use browser_type to enter text. Click on an input field with browser_click first to focus it.", + "For special keys (Enter, Tab, Escape, arrows), use browser_press_key instead.", + ], + parameters: Type.Object({ + text: Type.String({ description: "Text to type" }), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + await daemon.cdp("Input.insertText", { text: params.text }); + return { content: [{ type: "text" as const, text: `Typed: "${params.text}"` }] }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Type failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_press_key + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_press_key", + label: "Browser Press Key", + description: + "Press a keyboard key. Supports special keys (Enter, Tab, Backspace, Escape, Delete, arrows, Home, End, PageUp, PageDown, Space as ' ') and regular characters. Optional modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta/Cmd, 8=Shift.", + promptSnippet: "Press a key (Enter, Tab, Escape, arrows, or any character)", + promptGuidelines: [ + "Use browser_press_key for keyboard shortcuts and navigation keys.", + "Special key names: Enter, Tab, Backspace, Escape, Delete, ArrowLeft, ArrowUp, ArrowRight, ArrowDown, Home, End, PageUp, PageDown.", + "Use Space as ' ' (a single space character).", + "Modifiers: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with bitwise OR: Ctrl+Shift = 2|8 = 10.", + ], + parameters: Type.Object({ + key: Type.String({ description: 'Key to press. Special keys: Enter, Tab, Backspace, Escape, Delete, ArrowLeft/Up/Right/Down, Home, End, PageUp, PageDown. Space as " "' }), + modifiers: Type.Optional(Type.Number({ description: "Modifier bitfield: 1=Alt, 2=Ctrl, 4=Meta(Cmd), 8=Shift. Combine with OR." })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + const k = params.key; + const mod = params.modifiers || 0; + const vk = daemon.getVirtualKeyCode(k); + const code = daemon.getKeyCode(k); + const isSpecial = SPECIAL_KEYS.includes(k); + const text = k.length === 1 && !isSpecial ? k : ""; + + // keyDown + const base = { + type: "keyDown" as const, + key: k, + code, + modifiers: mod, + windowsVirtualKeyCode: vk, + nativeVirtualKeyCode: vk, + ...(text ? { text } : {}), + }; + await daemon.cdp("Input.dispatchKeyEvent", base); + + // char event for printable single characters + if (text.length === 1) { + await daemon.cdp("Input.dispatchKeyEvent", { + type: "char", + text, + key: k, + code, + modifiers: mod, + windowsVirtualKeyCode: vk, + nativeVirtualKeyCode: vk, + }); + } + + // keyUp + await daemon.cdp("Input.dispatchKeyEvent", { + type: "keyUp", + key: k, + code, + modifiers: mod, + windowsVirtualKeyCode: vk, + nativeVirtualKeyCode: vk, + }); + + return { content: [{ type: "text" as const, text: `Pressed: "${k}"${mod ? ` (modifiers: ${mod})` : ""}` }] }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Key press failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_scroll + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_scroll", + label: "Browser Scroll", + description: + "Scroll the page at given coordinates (or center of viewport if not specified). Default scrolls down 300px.", + promptSnippet: "Scroll the page by deltaY/deltaX pixels at coordinates", + promptGuidelines: [ + "Use browser_scroll to scroll pages. Default deltaY=-300 scrolls down.", + "Positive deltaY scrolls up, negative scrolls down.", + "Capture browser_screenshot after scrolling to see the new viewport content.", + ], + parameters: Type.Object({ + deltaY: Type.Optional(Type.Number({ description: "Vertical scroll delta in pixels. Negative = down, positive = up. Default: -300" })), + deltaX: Type.Optional(Type.Number({ description: "Horizontal scroll delta in pixels. Default: 0" })), + x: Type.Optional(Type.Number({ description: "X position to scroll from. Default: viewport center" })), + y: Type.Optional(Type.Number({ description: "Y position to scroll from. Default: viewport center" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + const dy = params.deltaY ?? -300; + const dx = params.deltaX ?? 0; + + // Default to center of viewport if not specified + let cx = params.x; + let cy = params.y; + if (cx === undefined || cy === undefined) { + try { + const info = await daemon.getPageInfo(); + if (!("dialog" in info)) { + if (cx === undefined) cx = Math.round(info.width / 2); + if (cy === undefined) cy = Math.round(info.height / 2); + } + } catch { + cx = cx ?? 500; + cy = cy ?? 400; + } + } + + await daemon.cdp("Input.dispatchMouseEvent", { + type: "mouseWheel", + x: cx, + y: cy, + deltaX: dx, + deltaY: dy, + }); + + return { content: [{ type: "text" as const, text: `Scrolled by (${dx}, ${dy}) at (${cx}, ${cy})` }] }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Scroll failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_screenshot + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_screenshot", + label: "Browser Screenshot", + description: + "Capture a PNG screenshot of the current page. Use this BEFORE interactions to find targets, and AFTER to verify results. This is the primary visual feedback mechanism.", + promptSnippet: "Capture a screenshot of the current page (PNG)", + promptGuidelines: [ + "Use browser_screenshot as your FIRST step when exploring a page — it shows you what's visible.", + "After every browser_click, browser_type, browser_scroll, or browser_navigate, capture a screenshot to VERIFY the action worked.", + "Screenshots are the fastest way to find click targets, notice hidden blockers, and decide next steps.", + "Never assume an action worked. Always re-screenshot to confirm.", + ], + parameters: Type.Object({ + fullPage: Type.Optional(Type.Boolean({ description: "Capture the full scrollable page (not just viewport). Default: false" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + const path = nextScreenshotPath(); + await daemon.captureScreenshot(path, params.fullPage === true); + return { + content: [{ type: "text" as const, text: `Screenshot saved: ${path}${params.fullPage ? " (full page)" : ""}` }], + details: { path }, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Screenshot failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + renderCall(_, theme) { + return new Text(`📸 Capturing screenshot...`, 0, 0); + }, + renderResult(result, _options, theme) { + const path = (result.details as { path?: string })?.path; + return new Text(path ? `📸 Screenshot: ${path}` : `📸 Screenshot captured`, 0, 0); + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_list_tabs + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_list_tabs", + label: "Browser List Tabs", + description: "List all open browser tabs.", + promptSnippet: "List all open browser tabs with URLs and titles", + promptGuidelines: [ + "Use browser_list_tabs to see what tabs are open before switching.", + "Use browser_switch_tab to switch to a specific tab by targetId.", + "Set includeChrome: true to also see Chrome internal pages (devtools, settings, etc).", + ], + parameters: Type.Object({ + includeChrome: Type.Optional(Type.Boolean({ description: "Include Chrome internal pages (chrome://, devtools://, etc). Default: true" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + const tabs = await daemon.listTabs(params.includeChrome !== false); + const listing = tabs + .map((t, i) => `[${i}] ${t.title || "(no title)"}\n URL: ${t.url}\n ID: ${t.targetId}`) + .join("\n\n"); + return { + content: [{ type: "text" as const, text: `${tabs.length} tab(s):\n\n${listing}` }], + details: { tabs }, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + renderResult(result, { expanded }, theme) { + const details = result.details as { + tabs?: TabInfo[]; + fullOutputPath?: string; + } | undefined; + + const tabCount = details?.tabs?.length ?? 0; + + if (expanded) { + // Expanded (CTRL+O): show full tab listing + const content = result.content[0]; + const raw = content?.type === "text" ? content.text : ""; + return new Text(raw || "(no tabs)", 0, 0); + } + + // Collapsed (default): compact count + return new Text( + theme ? theme.fg("dim", `📋 ${tabCount} tab${tabCount !== 1 ? "s" : ""}`) : `📋 ${tabCount} tab${tabCount !== 1 ? "s" : ""}`, + 0, + 0, + ); + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_current_tab + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_current_tab", + label: "Browser Current Tab", + description: "Get info about the currently active tab.", + promptSnippet: "Get the current active tab's URL, title, and ID", + promptGuidelines: [ + "Use browser_current_tab to confirm which tab you're controlling.", + ], + parameters: Type.Object({}), + async execute() { + try { + await daemon.ensureAlive(); + const tab = await daemon.currentTab(); + return { + content: [{ type: "text" as const, text: `Current tab: ${tab.title || "(no title)"}\nURL: ${tab.url}\nID: ${tab.targetId}` }], + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_switch_tab + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_switch_tab", + label: "Browser Switch Tab", + description: "Switch to a different browser tab by targetId (from browser_list_tabs).", + promptSnippet: "Switch to a browser tab by its targetId", + promptGuidelines: [ + "Use browser_list_tabs first to find the targetId, then browser_switch_tab to switch.", + ], + parameters: Type.Object({ + targetId: Type.String({ description: "Target ID of the tab to switch to (from browser_list_tabs)" }), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + await daemon.switchTab(params.targetId); + const tab = await daemon.currentTab(); + return { + content: [{ type: "text" as const, text: `Switched to: ${tab.title || "(no title)"}\nURL: ${tab.url}` }], + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Switch failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_new_tab + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_new_tab", + label: "Browser New Tab", + description: "Open a new browser tab and switch to it. Optionally navigate to a URL.", + promptSnippet: "Open a new browser tab, optionally navigate to a URL", + promptGuidelines: [ + "Use browser_new_tab for the first navigation in a task — it preserves the user's active tab.", + "Use browser_navigate for subsequent navigation in the same tab.", + ], + parameters: Type.Object({ + url: Type.Optional(Type.String({ description: "URL to navigate to in the new tab. Default: about:blank" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + const targetId = await daemon.newTab(params.url); + const msg = params.url + ? `New tab created and navigated to: ${params.url}\nTarget: ${targetId}` + : `New tab created: about:blank\nTarget: ${targetId}`; + return { content: [{ type: "text" as const, text: msg }] }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_execute_js + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_execute_js", + label: "Browser Execute JS", + description: + 'Execute JavaScript in the current page and return the result. Use for data extraction, DOM inspection, and page manipulation. Expressions with "return" are automatically wrapped in an IIFE.', + promptSnippet: "Execute JavaScript in the page and return the result", + promptGuidelines: [ + "Use browser_execute_js for extracting structured data from the page (text, attributes, JSON from embedded scripts).", + "Use browser_execute_js for DOM inspection when coordinates from screenshots aren't sufficient.", + "For bulk data extraction, prefer browser_http_get for API calls — it's 10-50x faster than browser DOM scraping.", + ], + parameters: Type.Object({ + expression: Type.String({ description: "JavaScript expression to evaluate. 'return' statements are auto-wrapped in IIFE." }), + targetId: Type.Optional(Type.String({ description: "Target ID of an iframe to evaluate in (optional)" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + const value = await daemon.evaluateJS(params.expression, params.targetId); + const raw = value === undefined ? "undefined" : JSON.stringify(value, null, 2); + const { text, fullOutputPath } = await applyTruncation(raw, "js"); + return { + content: [{ type: "text" as const, text }], + details: { valueLength: raw.length, fullOutputPath }, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `JS execution failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + renderResult(result, { expanded }, theme) { + const details = result.details as { + valueLength?: number; + fullOutputPath?: string; + } | undefined; + + const len = details?.valueLength ?? 0; + + if (expanded) { + // Expanded (CTRL+O): show the full JS result + const content = result.content[0]; + const raw = content?.type === "text" ? content.text : ""; + return new Text(raw || "undefined", 0, 0); + } + + // Collapsed (default): size + first-line preview + const content = result.content[0]; + const raw = content?.type === "text" ? content.text : ""; + const firstLine = raw.split("\n")[0] || ""; + const preview = + firstLine.length > 120 ? firstLine.slice(0, 117) + "..." : firstLine; + + const summary = `📜 JS result (${formatSize(len)})`; + return new Text( + theme + ? theme.fg("dim", summary) + "\n" + theme.fg("muted", preview) + : summary + "\n" + preview, + 0, + 0, + ); + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_http_get + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_http_get", + label: "Browser HTTP GET", + description: + "Make a direct HTTP GET request (no browser). Use for APIs and static pages — much faster than browser navigation for data retrieval.", + promptSnippet: "Make a direct HTTP GET request (outside browser, for APIs)", + promptGuidelines: [ + "Use browser_http_get for API calls and static page retrieval — it bypasses the browser entirely and is 10-50x faster.", + "Use browser_http_get with ThreadPoolExecutor for bulk data retrieval from multiple URLs.", + "Do NOT use browser_http_get for pages that require JavaScript rendering — use browser_navigate + browser_execute_js instead.", + ], + parameters: Type.Object({ + url: Type.String({ description: "URL to fetch" }), + headers: Type.Optional(Type.Record(Type.String(), Type.String(), { description: "Optional HTTP headers" })), + timeout: Type.Optional(Type.Number({ description: "Timeout in seconds. Default: 20" })), + }), + async execute(_id, params) { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), (params.timeout || 20) * 1000); + + const response = await fetch(params.url, { + headers: { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", + Accept: "text/html,application/json,*/*", + ...params.headers, + }, + signal: controller.signal, + }); + + clearTimeout(timer); + + const body = await response.text(); + const contentType = response.headers.get("content-type") || "unknown"; + + // Build header (always shown) and truncate body separately + const headerText = `HTTP ${response.status} ${response.statusText}\nContent-Type: ${contentType}\nLength: ${body.length} chars\n\n`; + const { text: truncatedBody, fullOutputPath, wasTruncated } = await applyTruncation( + body, + "http", + ); + + return { + content: [{ + type: "text" as const, + text: headerText + truncatedBody, + }], + details: { + status: response.status, + contentType, + length: body.length, + fullOutputPath, + wasTruncated, + }, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `HTTP request failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + renderResult(result, { expanded }, theme) { + const details = result.details as { + status?: number; + contentType?: string; + length?: number; + wasTruncated?: boolean; + fullOutputPath?: string; + } | undefined; + + const status = details?.status ?? 0; + const ct = details?.contentType ?? "unknown"; + const len = details?.length ?? 0; + const wasTruncated = details?.wasTruncated ?? false; + + if (expanded) { + // Expanded (CTRL+O): show full response body + const content = result.content[0]; + const raw = content?.type === "text" ? content.text : ""; + return new Text(raw || "(empty response)", 0, 0); + } + + // Collapsed (default): compact HTTP response summary + const shortCt = ct.split(";")[0] || ct; // drop charset etc. + const truncNote = wasTruncated ? " (truncated)" : ""; + const summary = `🌐 HTTP ${status}, ${shortCt}, ${formatSize(len)}${truncNote}`; + + return new Text( + theme + ? theme.fg(status >= 200 && status < 300 ? "success" : status >= 400 ? "error" : "warning", summary) + : summary, + 0, + 0, + ); + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_wait + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_wait", + label: "Browser Wait", + description: "Wait for a specified number of seconds.", + promptSnippet: "Wait for N seconds (default 1s)", + promptGuidelines: [ + "Use browser_wait for simple delays. Prefer browser_wait_for_load after navigation.", + ], + parameters: Type.Object({ + seconds: Type.Optional(Type.Number({ description: "Seconds to wait. Default: 1" })), + }), + async execute(_id, params) { + const sec = params.seconds || 1; + await new Promise((r) => setTimeout(r, sec * 1000)); + return { content: [{ type: "text" as const, text: `Waited ${sec}s.` }] }; + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_wait_for_load + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_wait_for_load", + label: "Browser Wait For Load", + description: + "Wait for the page to finish loading (document.readyState === 'complete'). Returns false on timeout. Note: SPAs may need additional waiting after this.", + promptSnippet: "Wait for document.readyState === 'complete'", + promptGuidelines: [ + "Use browser_wait_for_load after browser_navigate or browser_new_tab.", + "This checks document.readyState only. Single-page apps may still be loading data after this returns true.", + "For SPAs, use browser_execute_js to check for specific elements after browser_wait_for_load.", + ], + parameters: Type.Object({ + timeout: Type.Optional(Type.Number({ description: "Max seconds to wait. Default: 15" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + const timeout = params.timeout || 15; + const deadline = Date.now() + timeout * 1000; + + while (Date.now() < deadline) { + const state = await daemon.evaluateJS("document.readyState"); + if (state === "complete") { + return { content: [{ type: "text" as const, text: "Page loaded (readyState: complete)." }] }; + } + await new Promise((r) => setTimeout(r, 300)); + } + + return { + isError: true, + content: [{ type: "text" as const, text: `Page load timed out after ${timeout}s. Current state unknown.` }], + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Wait failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_handle_dialog + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_handle_dialog", + label: "Browser Handle Dialog", + description: + "Handle a JavaScript dialog (alert, confirm, prompt, beforeunload). Use when browser_page_info reports an open dialog.", + promptSnippet: "Accept or dismiss a JavaScript dialog (alert/confirm/prompt)", + promptGuidelines: [ + "Check browser_page_info first — if it returns a dialog, use browser_handle_dialog before any other browser actions.", + "JS dialogs freeze the page's JS thread, so no other interaction works until the dialog is handled.", + 'Use action: "accept" for alerts and confirms, "dismiss" to cancel. For prompts, provide promptText.', + ], + parameters: Type.Object({ + action: Type.String({ description: '"accept" or "dismiss"' }), + promptText: Type.Optional(Type.String({ description: 'Text to enter for prompt dialogs (only used with "accept")' })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + + if (params.action === "accept") { + await daemon.cdp("Page.handleJavaScriptDialog", { + accept: true, + ...(params.promptText !== undefined ? { promptText: params.promptText } : {}), + }); + } else { + await daemon.cdp("Page.handleJavaScriptDialog", { accept: false }); + } + + return { content: [{ type: "text" as const, text: `Dialog ${params.action}ed.` }] }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Dialog handling failed: ${err instanceof Error ? err.message : String(err)}` }], + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_open_urls + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_open_urls", + label: "Browser Open URLs", + description: + "Open multiple URLs in new browser tabs in parallel. Useful after web_search to open citation links for visual inspection. Creates real Chrome tabs with full JS rendering support — identical to browser_new_tab but for many URLs at once.", + promptSnippet: "Open multiple URLs in new tabs (parallel, full JS rendering)", + promptGuidelines: [ + "Use browser_open_urls to open multiple URLs in parallel — much faster than calling browser_new_tab repeatedly.", + "Use browser_open_urls after web_search to open citation links from the synthesized answer.", + "Each URL opens in a real Chrome tab with full JS rendering — same as browser_new_tab.", + "After opening, use browser_list_tabs to see all tabs, browser_switch_tab to switch, and browser_screenshot / browser_execute_js to interact.", + "Use browser_wait_for_load on a tab before extracting data from SPAs.", + ], + parameters: Type.Object({ + urls: Type.Array(Type.String(), { description: "Array of URLs to open in new tabs" }), + }), + async execute(_id, params, _signal, onUpdate) { + await daemon.ensureAlive(); + + const total = params.urls.length; + const results: Array<{ + url: string; + targetId: string; + ok: boolean; + error?: string; + }> = []; + + // ═══════════════════════════════════════════════════════════════════ + // Phase 1: Create all tabs in parallel. + // Target.createTarget is a browser-level CDP call — no session + // needed, no shared state mutated. Safe to parallelize. + // ═══════════════════════════════════════════════════════════════════ + const createPromises = params.urls.map(async (url: string) => { + try { + const result = (await daemon.cdp("Target.createTarget", { + url: "about:blank", + })) as { targetId: string }; + return { url, targetId: result.targetId, ok: true as const }; + } catch (err) { + return { + url, + targetId: "", + ok: false as const, + error: err instanceof Error ? err.message : String(err), + }; + } + }); + + const created = await Promise.all(createPromises); + + // ═══════════════════════════════════════════════════════════════════ + // Phase 2: Attach a CDP session to each tab and navigate. + // Each tab gets its own sessionId passed explicitly to cdp(), + // avoiding the shared daemon._sessionId. Safe to parallelize. + // ═══════════════════════════════════════════════════════════════════ + let completed = 0; + const attachAndNav = created + .filter((t) => t.ok) + .map(async (tab) => { + try { + // Attach to target to get a session for this tab + const attachResult = (await daemon.cdp( + "Target.attachToTarget", + { targetId: tab.targetId, flatten: true }, + )) as { sessionId: string }; + const sid = attachResult.sessionId; + + // Enable the Page domain so navigation events fire + await daemon.cdp("Page.enable", {}, sid); + + // Navigate to the URL using the tab's own session + await daemon.cdp("Page.navigate", { url: tab.url }, sid); + + return { ...tab, ok: true as const, targetId: tab.targetId }; + } catch (err) { + return { + ...tab, + ok: false as const, + error: err instanceof Error ? err.message : String(err), + }; + } finally { + completed++; + // Stream progress so the TUI shows live per-URL status + const okCount = created.filter((t) => t.ok).length; + onUpdate?.({ + content: [ + { + type: "text" as const, + text: `Opening URLs… ${completed}/${okCount} navigated`, + }, + ], + }); + } + }); + + // Also collect failures from Phase 1 + const failures = created.filter((t) => !t.ok); + + const settled = await Promise.all(attachAndNav); + results.push(...settled, ...failures); + + // ═══════════════════════════════════════════════════════════════════ + // Phase 3: Activate the last successfully opened tab so the user + // sees something meaningful in the foreground. + // ═══════════════════════════════════════════════════════════════════ + const lastOk = [...settled].reverse().find((r) => r.ok); + if (lastOk) { + try { + await daemon.cdp("Target.activateTarget", { + targetId: lastOk.targetId, + }); + } catch { + // best-effort — the tabs are still open + } + } + + // Build result text + const okTabs = results.filter((r) => r.ok); + const failedTabs = results.filter((r) => !r.ok); + + const lines: string[] = []; + if (okTabs.length > 0) { + lines.push( + `✅ ${okTabs.length}/${total} tabs opened:`, + ...okTabs.map( + (t, i) => ` [${i}] ${t.url} → ${t.targetId}`, + ), + ); + } + if (failedTabs.length > 0) { + lines.push( + `❌ ${failedTabs.length}/${total} failed:`, + ...failedTabs.map( + (t) => ` ${t.url}: ${t.error || "unknown error"}`, + ), + ); + } + + const isTotalFailure = okTabs.length === 0; + const fullOutput = lines.join("\n"); + const { text, fullOutputPath } = await applyTruncation(fullOutput, "urls"); + + return { + isError: isTotalFailure, + content: [{ type: "text" as const, text }], + details: { tabs: results, fullOutputPath }, + }; + }, + renderCall(args, _theme) { + const { urls } = args as { urls: string[] }; + return new Text( + `🌐 Opening ${urls.length} URL${urls.length !== 1 ? "s" : ""}…`, + 0, + 0, + ); + }, + renderResult(result, { expanded }, theme) { + const details = result.details as { + tabs?: Array<{ + url: string; + targetId: string; + ok: boolean; + error?: string; + }>; + fullOutputPath?: string; + } | undefined; + + const tabs = details?.tabs || []; + const ok = tabs.filter((t) => t.ok).length; + const fail = tabs.filter((t) => !t.ok).length; + const total = tabs.length; + + if (expanded) { + // Expanded (CTRL+O): show the full URL→targetId mapping + const content = result.content[0]; + const raw = content?.type === "text" ? content.text : ""; + // Strip the truncation footer for cleaner display, show it separately + const truncIdx = raw.indexOf("\n\n[Output truncated:"); + const displayText = truncIdx >= 0 ? raw.slice(0, truncIdx) : raw; + + let expandedText = displayText || "(no output)"; + if (details?.fullOutputPath && truncIdx < 0) { + // Truncated but we stripped the footer — add a compact note + expandedText += `\n\n${theme ? theme.fg("dim", `Full output: ${details.fullOutputPath}`) : `Full output: ${details.fullOutputPath}`}`; + } else if (details?.fullOutputPath) { + expandedText += `\n${theme ? theme.fg("dim", `Full output: ${details.fullOutputPath}`) : `Full output: ${details.fullOutputPath}`}`; + } + return new Text(expandedText, 0, 0); + } + + // Collapsed (default): compact summary + let text: string; + if (total === 0) { + text = "No URLs provided."; + } else if (fail === 0) { + text = `✅ ${ok} tab${ok !== 1 ? "s" : ""} opened`; + } else if (ok === 0) { + text = `❌ All ${fail} URL${fail !== 1 ? "s" : ""} failed`; + } else { + text = `⚠️ ${ok} opened, ${fail} failed`; + } + + return new Text( + theme ? theme.fg(fail === 0 ? "success" : ok === 0 ? "error" : "warning", text) : text, + 0, + 0, + ); + }, + }); +} |
