/** * Tool registrations for pi-browser-harness. * * Maps browser-harness daemon methods to pi tools with TypeBox schemas, * prompt snippets, guidelines, and TUI renderers. */ import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, truncateHead, withFileMutationQueue, } from "@mariozechner/pi-coding-agent"; import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; import { Text } from "@mariozechner/pi-tui"; import { Type } from "typebox"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { BrowserDaemon } from "./daemon"; import type { TabInfo } from "./protocol"; // ── AsyncFunction constructor ──────────────────────────────────────────────── const AsyncFunction = Object.getPrototypeOf(async () => {}).constructor as new (...args: string[]) => (...args: unknown[]) => Promise; // ── Key code map ───────────────────────────────────────────────────────────── const SPECIAL_KEYS = [ "Enter", "Tab", "Backspace", "Escape", "Delete", "ArrowLeft", "ArrowUp", "ArrowRight", "ArrowDown", "Home", "End", "PageUp", "PageDown", " ", ]; // ── Screenshot path ────────────────────────────────────────────────────────── let screenshotCounter = 0; function nextScreenshotPath(): string { screenshotCounter++; return join(tmpdir(), `pi-browser-screenshot-${Date.now()}-${screenshotCounter}.png`); } // ── Output truncation ──────────────────────────────────────────────────────── /** Track temp directories for cleanup on session shutdown. */ const tempDirs: string[] = []; /** Best-effort cleanup of all tracked temp directories. */ export async function cleanupTempDirs(): Promise { const dirs = tempDirs.splice(0); await Promise.all( dirs.map((d) => rm(d, { recursive: true, force: true }).catch(() => {})), ); } /** * Truncate output to pi's built-in limits (50KB / 2000 lines). * When truncated, writes the full output to a temp file so the LLM * can read it via the read tool and so CTRL+O expand can show it. */ async function applyTruncation( output: string, prefix: string, ): Promise<{ text: string; fullOutputPath?: string; wasTruncated: boolean; }> { const truncation = truncateHead(output, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES, }); if (!truncation.truncated) { return { text: truncation.content, wasTruncated: false }; } const tempDir = await mkdtemp(join(tmpdir(), `pi-bh-${prefix}-`)); tempDirs.push(tempDir); const tempFile = join(tempDir, "output.txt"); await withFileMutationQueue(tempFile, async () => { await writeFile(tempFile, output, "utf8"); }); const omittedBytes = truncation.totalBytes - truncation.outputBytes; let text = truncation.content; text += `\n\n[Output truncated: ${truncation.outputLines} of ${truncation.totalLines} lines`; text += ` (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}).`; text += ` ${formatSize(omittedBytes)} omitted. Full output: ${tempFile}]`; return { text, fullOutputPath: tempFile, wasTruncated: true }; } // ── Registration ───────────────────────────────────────────────────────────── export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { // ═══════════════════════════════════════════════════════════════════════════ // browser_upload_file // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ name: "browser_upload_file", label: "Browser Upload File", description: "Set files on a file input element via CDP DOM.setFileInputFiles. " + "Use this to upload files to forms without clicking the file picker dialog. " + "The path must be an absolute file path accessible from the machine running Chrome.", promptSnippet: "Upload a file to a file input element (bypasses file picker)", promptGuidelines: [ "Use browser_upload_file to set files on elements — much faster than clicking the file picker.", "The selector must match a file input element ().", "The filePath must be an absolute path on the machine where Chrome is running.", "To create a temp file first, use write or bash to create the file, then upload it.", ], parameters: Type.Object({ selector: Type.String({ description: "CSS selector for the file input element" }), filePath: Type.String({ description: "Absolute path to the file to upload" }), }), async execute(_id, params) { try { await daemon.ensureAlive(); await daemon.uploadFile(params.selector, params.filePath); return { content: [{ type: "text" as const, text: `Set file "${params.filePath}" on "${params.selector}".` }], details: undefined, }; } catch (err) { return { isError: true, content: [{ type: "text" as const, text: `Upload failed: ${err instanceof Error ? err.message : String(err)}` }], details: undefined, }; } }, }); // ═══════════════════════════════════════════════════════════════════════════ // ═══════════════════════════════════════════════════════════════════════════ // browser_execute_js // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ name: "browser_execute_js", label: "Browser Execute JS", description: 'Execute JavaScript in the current page and return the result. Use for data extraction, DOM inspection, and page manipulation. Expressions with "return" are automatically wrapped in an IIFE.', promptSnippet: "Execute JavaScript in the page and return the result", promptGuidelines: [ "Use browser_execute_js for extracting structured data from the page (text, attributes, JSON from embedded scripts).", "Use browser_execute_js for DOM inspection when coordinates from screenshots aren't sufficient.", "For bulk data extraction, prefer browser_http_get for API calls — it's 10-50x faster than browser DOM scraping.", "To execute JS inside an iframe, first find the iframe target via browser_list_tabs (includeChrome: true to see iframe targets), then pass targetId to browser_execute_js.", "Compositor clicks (browser_click) already pass through iframes — only use iframe JS when you need to read/change iframe DOM.", ], parameters: Type.Object({ expression: Type.String({ description: "JavaScript expression to evaluate. 'return' statements are auto-wrapped in IIFE." }), targetId: Type.Optional(Type.String({ description: "Target ID of an iframe to evaluate in (optional)" })), }), async execute(_id, params) { try { await daemon.ensureAlive(); const value = await daemon.evaluateJS(params.expression, params.targetId); const raw = value === undefined ? "undefined" : JSON.stringify(value, null, 2); const { text, fullOutputPath } = await applyTruncation(raw, "js"); return { content: [{ type: "text" as const, text }], details: { valueLength: raw.length, fullOutputPath }, }; } catch (err) { return { isError: true, content: [{ type: "text" as const, text: `JS execution failed: ${err instanceof Error ? err.message : String(err)}` }], details: undefined, }; } }, renderResult(result, { expanded }, theme) { const details = result.details as { valueLength?: number; fullOutputPath?: string; } | undefined; const len = details?.valueLength ?? 0; if (expanded) { // Expanded (CTRL+O): show the full JS result const content = result.content[0]; const raw = content?.type === "text" ? content.text : ""; return new Text(raw || "undefined", 0, 0); } // Collapsed (default): size + first-line preview const content = result.content[0]; const raw = content?.type === "text" ? content.text : ""; const firstLine = raw.split("\n")[0] || ""; const preview = firstLine.length > 120 ? firstLine.slice(0, 117) + "..." : firstLine; const summary = `📜 JS result (${formatSize(len)})`; return new Text( theme ? theme.fg("dim", summary) + "\n" + theme.fg("muted", preview) : summary + "\n" + preview, 0, 0, ); }, }); // ═══════════════════════════════════════════════════════════════════════════ // browser_http_get // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ name: "browser_http_get", label: "Browser HTTP GET", description: "Make a direct HTTP GET request (no browser). Use for APIs and static pages — much faster than browser navigation for data retrieval.", promptSnippet: "Make a direct HTTP GET request (outside browser, for APIs)", promptGuidelines: [ "Use browser_http_get for API calls and static page retrieval — it bypasses the browser entirely and is 10-50x faster.", "Use browser_http_get with ThreadPoolExecutor for bulk data retrieval from multiple URLs.", "Do NOT use browser_http_get for pages that require JavaScript rendering — use browser_navigate + browser_execute_js instead.", ], parameters: Type.Object({ url: Type.String({ description: "URL to fetch" }), headers: Type.Optional(Type.Record(Type.String(), Type.String(), { description: "Optional HTTP headers" })), timeout: Type.Optional(Type.Number({ description: "Timeout in seconds. Default: 20" })), }), async execute(_id, params) { try { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), (params.timeout || 20) * 1000); const response = await fetch(params.url, { headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", Accept: "text/html,application/json,*/*", ...params.headers, }, signal: controller.signal, }); clearTimeout(timer); const body = await response.text(); const contentType = response.headers.get("content-type") || "unknown"; // Build header (always shown) and truncate body separately const headerText = `HTTP ${response.status} ${response.statusText}\nContent-Type: ${contentType}\nLength: ${body.length} chars\n\n`; const { text: truncatedBody, fullOutputPath, wasTruncated } = await applyTruncation( body, "http", ); return { content: [{ type: "text" as const, text: headerText + truncatedBody, }], details: { status: response.status, contentType, length: body.length, fullOutputPath, wasTruncated, }, }; } catch (err) { return { isError: true, content: [{ type: "text" as const, text: `HTTP request failed: ${err instanceof Error ? err.message : String(err)}` }], details: undefined, }; } }, renderResult(result, { expanded }, theme) { const details = result.details as { status?: number; contentType?: string; length?: number; wasTruncated?: boolean; fullOutputPath?: string; } | undefined; const status = details?.status ?? 0; const ct = details?.contentType ?? "unknown"; const len = details?.length ?? 0; const wasTruncated = details?.wasTruncated ?? false; if (expanded) { // Expanded (CTRL+O): show full response body const content = result.content[0]; const raw = content?.type === "text" ? content.text : ""; return new Text(raw || "(empty response)", 0, 0); } // Collapsed (default): compact HTTP response summary const shortCt = ct.split(";")[0] || ct; // drop charset etc. const truncNote = wasTruncated ? " (truncated)" : ""; const summary = `🌐 HTTP ${status}, ${shortCt}, ${formatSize(len)}${truncNote}`; return new Text( theme ? theme.fg(status >= 200 && status < 300 ? "success" : status >= 400 ? "error" : "warning", summary) : summary, 0, 0, ); }, }); // ═══════════════════════════════════════════════════════════════════════════ // browser_download // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ name: "browser_download", label: "Browser Download", description: "Configure the download directory and disable the save-as prompt for this tab. " + "Call before triggering downloads to control where files are saved. " + "After the download is triggered, call again with eventsOnly: true to check for download progress events.", promptSnippet: "Configure browser download behavior (directory, no prompts)", promptGuidelines: [ "Use browser_download BEFORE clicking a download link or triggering a file save.", "Set downloadPath to an absolute directory path (e.g. /tmp/downloads).", "This disables the browser's save-as prompt, so files save automatically.", "To check if a download started, call again with eventsOnly: true and look for Browser.downloadProgress events.", ], parameters: Type.Object({ downloadPath: Type.Optional(Type.String({ description: "Absolute path to the download directory. Default: /tmp/browser-downloads" })), eventsOnly: Type.Optional(Type.Boolean({ description: "If true, only check for recent download progress events. Default: false" })), }), async execute(_id, params) { try { await daemon.ensureAlive(); if (params.eventsOnly) { // Drain events and filter for download progress const events = await daemon.drainEvents(); const downloads = events.filter( (e) => e.method === "Browser.downloadProgress" || e.method === "Browser.downloadWillBegin", ); if (downloads.length === 0) { return { content: [{ type: "text" as const, text: "No recent download events found." }], details: undefined, }; } const lines = downloads.map((e) => { const p = e.params as Record; return [ `Event: ${e.method}`, ` URL: ${p.url || "?"}`, ` State: ${p.state || "?"}`, p.receivedBytes !== undefined ? ` Received: ${p.receivedBytes} / ${p.totalBytes || "?"}` : "", ] .filter(Boolean) .join("\n"); }); return { content: [{ type: "text" as const, text: lines.join("\n\n") }], details: undefined, }; } const dlPath = params.downloadPath || "/tmp/browser-downloads"; await daemon.setDownloadBehavior(dlPath); return { content: [{ type: "text" as const, text: `Downloads will save to: ${dlPath} (save-as prompt disabled).` }], details: undefined, }; } catch (err) { return { isError: true, content: [{ type: "text" as const, text: `Download config failed: ${err instanceof Error ? err.message : String(err)}` }], details: undefined, }; } }, }); // ═══════════════════════════════════════════════════════════════════════════ // browser_viewport_resize // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ name: "browser_viewport_resize", label: "Browser Viewport Resize", description: "Resize the browser viewport. Useful for responsive testing or when a page " + "requires a specific viewport width. Adjusts both the visible viewport and " + "the device pixel ratio.", promptSnippet: "Resize the browser viewport (width, height, deviceScaleFactor)", promptGuidelines: [ "Use browser_viewport_resize for responsive testing or when a page needs specific dimensions.", "width and height are in CSS pixels (not device pixels).", "deviceScaleFactor defaults to 1. Use 2 for HiDPI/Retina emulation.", "Resizing may cause layout shifts — take a screenshot afterward to confirm.", ], parameters: Type.Object({ width: Type.Number({ description: "Viewport width in CSS pixels" }), height: Type.Number({ description: "Viewport height in CSS pixels" }), deviceScaleFactor: Type.Optional(Type.Number({ description: "Device pixel ratio. Default: 1" })), }), async execute(_id, params) { try { await daemon.ensureAlive(); await daemon.setViewportSize( params.width, params.height, params.deviceScaleFactor, ); return { content: [{ type: "text" as const, text: `Viewport resized to ${params.width}x${params.height}${params.deviceScaleFactor ? ` @${params.deviceScaleFactor}x` : ""}.`, }], details: undefined, }; } catch (err) { return { isError: true, content: [{ type: "text" as const, text: `Viewport resize failed: ${err instanceof Error ? err.message : String(err)}` }], details: undefined, }; } }, }); // ═══════════════════════════════════════════════════════════════════════════ // browser_drag_and_drop // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ name: "browser_drag_and_drop", label: "Browser Drag And Drop", description: "Perform a drag-and-drop operation from one viewport coordinate to another. " + "Use for kanban boards, file uploads via drag, reorderable lists, and " + "other drag-based UIs.", promptSnippet: "Drag from (startX, startY) and drop at (endX, endY)", promptGuidelines: [ "Use browser_drag_and_drop for drag-based interactions like moving cards, reordering items, or drag-to-upload.", "Capture a screenshot BEFORE to find the start and end coordinates.", "Capture a screenshot AFTER to verify the drag-and-drop worked.", "If the target site expects DOM-level drag events (not just compositor events), try browser_upload_file instead for file uploads.", ], parameters: Type.Object({ startX: Type.Number({ description: "X coordinate to start dragging from" }), startY: Type.Number({ description: "Y coordinate to start dragging from" }), endX: Type.Number({ description: "X coordinate to drop at" }), endY: Type.Number({ description: "Y coordinate to drop at" }), }), async execute(_id, params) { try { await daemon.ensureAlive(); await daemon.dragAndDrop( params.startX, params.startY, params.endX, params.endY, ); return { content: [{ type: "text" as const, text: `Dragged from (${params.startX}, ${params.startY}) to (${params.endX}, ${params.endY}).`, }], details: undefined, }; } catch (err) { return { isError: true, content: [{ type: "text" as const, text: `Drag-and-drop failed: ${err instanceof Error ? err.message : String(err)}` }], details: undefined, }; } }, }); // ═══════════════════════════════════════════════════════════════════════════ // browser_print_to_pdf // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ name: "browser_print_to_pdf", label: "Browser Print to PDF", description: "Print the current page to a PDF file via Page.printToPDF. " + "Saves the PDF to the specified path. Includes background colors and respects CSS page size.", promptSnippet: "Print the current page to a PDF file", promptGuidelines: [ "Use browser_print_to_pdf to save a page as PDF for archiving, sharing, or offline reading.", "The output path should be an absolute file path (e.g. /tmp/page.pdf).", "The page must be fully loaded before printing — use browser_wait_for_load first.", ], parameters: Type.Object({ outputPath: Type.Optional(Type.String({ description: "Absolute path for the PDF file. Default: /tmp/browser-print-.pdf" })), }), async execute(_id, params) { try { await daemon.ensureAlive(); const outputPath = params.outputPath || join(tmpdir(), `browser-print-${Date.now()}.pdf`); await daemon.printToPDF(outputPath); return { content: [{ type: "text" as const, text: `PDF saved: ${outputPath}` }], details: { path: outputPath }, }; } catch (err) { return { isError: true, content: [{ type: "text" as const, text: `Print to PDF failed: ${err instanceof Error ? err.message : String(err)}` }], details: undefined, }; } }, renderCall(_, theme) { return new Text(`📄 Printing to PDF...`, 0, 0); }, renderResult(result, _options, theme) { const path = (result.details as { path?: string })?.path; return new Text( path ? theme.fg("accent", `📄 PDF: ${path}`) : `📄 PDF saved`, 0, 0, ); }, }); // ═══════════════════════════════════════════════════════════════════════════ // browser_get_network_log // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ name: "browser_get_network_log", label: "Browser Get Network Log", description: "Get buffered network events (Network.requestWillBeSent, Network.responseReceived) " + "from the current page. Useful for inspecting XHR/fetch calls the page makes, " + "finding API endpoints, or debugging network issues.", promptSnippet: "Get buffered network request/response events", promptGuidelines: [ "Use browser_get_network_log to find API endpoints the page calls — often 10x faster than DOM scraping.", "Network events are buffered since the last call to this tool — call it after performing actions that trigger XHR/fetch.", "Filter by eventType to focus on specific events: 'Network.requestWillBeSent' (URLs, headers) or 'Network.responseReceived' (status, response headers).", "Combine with browser_http_get to replay captured API calls.", ], parameters: Type.Object({ eventTypes: Type.Optional( Type.Array(Type.String(), { description: "Network event types to filter. Default: ['Network.requestWillBeSent', 'Network.responseReceived']", }), ), limit: Type.Optional(Type.Number({ description: "Max events to return. Default: 50" })), }), async execute(_id, params) { try { await daemon.ensureAlive(); const events = await daemon.getNetworkLog( params.eventTypes, params.limit, ); if (events.length === 0) { return { content: [{ type: "text" as const, text: "No buffered network events. Perform actions that trigger XHR/fetch calls first." }], details: undefined, }; } const lines = events.map((e) => { const p = e.params as Record; if (e.method === "Network.requestWillBeSent") { const req = (p.request as Record) || {}; return [ `→ ${req.method || "GET"} ${req.url || p.documentURL || "?"}`, ` Type: ${p.type || "?"}`, ].join("\n"); } if (e.method === "Network.responseReceived") { const resp = (p.response as Record) || {}; return [ `← ${resp.status || "?"} ${resp.statusText || ""} ${resp.url || p.url || "?"}`, ` MIME: ${resp.mimeType || "?"}`, ].join("\n"); } return `${e.method}: ${JSON.stringify(p).slice(0, 200)}`; }); const raw = `${events.length} network event(s):\n\n${lines.join("\n\n")}`; const { text, fullOutputPath } = await applyTruncation(raw, "net"); return { content: [{ type: "text" as const, text }], details: { count: events.length, fullOutputPath }, }; } catch (err) { return { isError: true, content: [{ type: "text" as const, text: `Network log failed: ${err instanceof Error ? err.message : String(err)}` }], details: undefined, }; } }, }); // ═══════════════════════════════════════════════════════════════════════════ // browser_run_script // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ name: "browser_run_script", label: "Browser Run Script", description: "Execute a temporary JavaScript script file with access to the browser daemon " + "and Node.js APIs. Use this when the built-in browser_* tools are insufficient " + "for a multi-step workflow — write a script to disk, then run it with this tool. " + "The script receives these bindings in scope:\n" + " params — the arguments passed to this tool\n" + " daemon — the browser daemon (daemon.cdp(), daemon.evaluateJS(), daemon.getPageInfo(), etc.)\n" + " require — Node.js require() for builtins and installed packages\n" + " signal — AbortSignal for cancellation\n" + " onUpdate — progress callback: onUpdate({ content: [{ type: 'text', text: '...' }] })\n" + " ctx — ExtensionContext with cwd, sessionManager, ui, signal, etc.\n" + " console, fetch, JSON, Buffer, setTimeout, clearTimeout\n\n" + "The script MUST return { content: [{ type: 'text', text: '...' }], details?: {...} }.\n" + "For errors, throw: throw new Error('something went wrong').", promptSnippet: "Run a temporary script with browser daemon access (write script to disk first)", promptGuidelines: [ "Use write to create a temporary script file, then browser_run_script to execute it — no dynamic tool registration needed.", "Scripts are written to disk, making them auditable and re-runnable. The user can inspect them.", "The script has access to the browser daemon via `daemon` — use daemon.cdp(), daemon.evaluateJS(), daemon.getPageInfo(), etc.", "For pure data processing or file operations, use require() to access Node.js modules like fs, path, crypto.", "If the script only needs built-in browser_* tools, break those into separate tool calls before or after — do not try to invoke them from within the script.", ], parameters: Type.Object({ path: Type.String({ description: "Path to the temporary script file (.js or .mjs) to execute" }), params: Type.Optional(Type.Object({}, { additionalProperties: true, description: "Optional parameters to pass to the script as `params`" })), }), async execute(_id, p, signal, onUpdate, ctx) { const { path: scriptPath, params: scriptParams } = p as { path: string; params?: Record; }; // ── Read the script from disk ──────────────────────────────────── let source: string; try { source = await readFile(scriptPath, "utf8"); } catch (err) { const msg = err instanceof Error ? err.message : String(err); return { isError: true, content: [{ type: "text" as const, text: `Failed to read script: ${msg}\n\nPath: ${scriptPath}\nCheck that the file exists and is readable.`, }], details: { error: "read_failed", message: msg }, }; } if (source.trim().length === 0) { return { isError: true, content: [{ type: "text" as const, text: `Script is empty: ${scriptPath}` }], details: { error: "empty_script" }, }; } // ── Compile ───────────────────────────────────────────────────── let executeFn: (...args: unknown[]) => Promise; try { const wrapped = `"use strict";\n${source}`; executeFn = new AsyncFunction( "params", "daemon", "require", "signal", "onUpdate", "ctx", "console", "fetch", "JSON", "Buffer", "setTimeout", "clearTimeout", wrapped, ); } catch (err) { const msg = err instanceof Error ? err.message : String(err); return { isError: true, content: [{ type: "text" as const, text: `Syntax error in script: ${msg}\n\nFile: ${scriptPath}\nFix the JavaScript syntax and try again.`, }], details: { error: "syntax_error", message: msg }, }; } // ── Execute ───────────────────────────────────────────────────── try { const result = await executeFn( scriptParams ?? {}, daemon, require, signal, onUpdate ?? ((_update: unknown) => {}), ctx ?? { cwd: process.cwd() }, console, fetch, JSON, Buffer, setTimeout, clearTimeout, ); if (!result || typeof result !== "object" || !Array.isArray((result as any).content)) { return { isError: true, content: [{ type: "text" as const, text: `Script must return { content: [{ type: 'text', text: '...' }], details?: {...} }.\nGot: ${JSON.stringify(result)}\n\nFile: ${scriptPath}`, }], details: { error: "invalid_return" }, }; } return result as { content: Array<{ type: "text"; text: string }>; details: Record | undefined }; } catch (err) { const msg = err instanceof Error ? err.message : String(err); return { isError: true, content: [{ type: "text" as const, text: `Script execution failed: ${msg}\n\nFile: ${scriptPath}\nCheck for runtime errors, undefined variables, or broken require() calls.`, }], details: { error: "execution_failed", message: msg }, }; } }, }); }