diff options
| author | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-05-02 15:29:23 +0530 |
|---|---|---|
| committer | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-05-02 15:29:23 +0530 |
| commit | 6759b6fd0ab435aa1e9fdb4591bd828b7a861a33 (patch) | |
| tree | ca6122e48b3e488ac2e0dc8d66fe631aacb071de /src | |
| parent | 5b76aa3412ec0ad043c1e8ae23b00085a4100178 (diff) | |
feat: v0.2.1 — feature parity with browser-use/browser-harness
Adds the missing tools called out in docs/browser-harness-comparison.md:
upload_file, dispatch_key, download (configure dir), viewport_resize,
drag_and_drop, print_to_pdf, get_network_log, plus the iframe-target
helper. Updates promptGuidelines and the system prompt's tools table.
Tracked in todo.md.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Diffstat (limited to 'src')
| -rw-r--r-- | src/daemon.ts | 259 | ||||
| -rw-r--r-- | src/prompt.ts | 12 | ||||
| -rw-r--r-- | src/tools.ts | 459 |
3 files changed, 728 insertions, 2 deletions
diff --git a/src/daemon.ts b/src/daemon.ts index 717bd57..955e79b 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -552,6 +552,265 @@ export class BrowserDaemon { return events; } + /** Find an iframe target by URL substring. Use with evaluateJS(targetId). */ + async findIframeTarget(urlSubstr: string): Promise<string | null> { + const result = (await this.cdp("Target.getTargets")) as { + targetInfos: Array<{ targetId: string; type: string; url: string }>; + }; + for (const t of result.targetInfos) { + if (t.type === "iframe" && t.url?.includes(urlSubstr)) { + return t.targetId; + } + } + return null; + } + + /** Set files on a file input element via DOM.setFileInputFiles. */ + async uploadFile(selector: string, filePath: string): Promise<void> { + // First try CDP-level file input (DOM.setFileInputFiles) + try { + const doc = (await this.cdp("DOM.getDocument", { depth: -1 })) as { + root: { nodeId: number }; + }; + const qResult = (await this.cdp("DOM.querySelector", { + nodeId: doc.root.nodeId, + selector, + })) as { nodeId: number }; + + if (qResult.nodeId && qResult.nodeId !== 0) { + await this.cdp("DOM.setFileInputFiles", { + files: [filePath], + nodeId: qResult.nodeId, + }); + // Verify the file was actually set + const verify = await this.evaluateJS( + `document.querySelector(${JSON.stringify(selector)})?.files?.length || 0` + ); + if (verify && Number(verify) > 0) return; + } + } catch { + // CDP approach failed — fall through to JS fallback + } + + // Fallback: try JS-based approach for broader compatibility + // Read file content from Node.js and inject as a File object via JS + try { + const { readFile, stat } = await import("node:fs/promises"); + const { basename } = await import("node:path"); + const fileBuffer = await readFile(filePath); + const fileStat = await stat(filePath); + const fileName = basename(filePath); + const base64 = fileBuffer.toString("base64"); + const mimeType = fileName.endsWith(".png") ? "image/png" + : fileName.endsWith(".jpg") || fileName.endsWith(".jpeg") ? "image/jpeg" + : fileName.endsWith(".pdf") ? "application/pdf" + : fileName.endsWith(".json") ? "application/json" + : "text/plain"; + + const js = ` + (() => { + const input = document.querySelector(${JSON.stringify(selector)}); + if (!input || input.type !== 'file') { + throw new Error('File input not found for selector: ${selector.replace(/'/g, "\\'")}'); + } + const binaryStr = atob(${JSON.stringify(base64)}); + const bytes = new Uint8Array(binaryStr.length); + for (let i = 0; i < binaryStr.length; i++) bytes[i] = binaryStr.charCodeAt(i); + const file = new File([bytes], ${JSON.stringify(fileName)}, { type: ${JSON.stringify(mimeType)}, lastModified: ${fileStat.mtimeMs} }); + const dt = new DataTransfer(); + dt.items.add(file); + input.files = dt.files; + input.dispatchEvent(new Event('change', { bubbles: true })); + return input.files.length; + })() + `; + const fileCount = await this.evaluateJS(js); + if (!fileCount || Number(fileCount) === 0) { + throw new Error("File was not set on input (JS fallback also failed)"); + } + } catch (jsErr) { + throw new Error( + `Upload failed: ${jsErr instanceof Error ? jsErr.message : String(jsErr)}` + ); + } + } + + /** + * Configure download behavior: set download directory and disable the + * save-as prompt. Afterwards, drain Browser.downloadProgress events to + * confirm downloads started. + */ + async setDownloadBehavior(downloadPath: string): Promise<void> { + await this.cdp("Browser.setDownloadBehavior", { + behavior: "allow", + downloadPath, + eventsEnabled: true, + }); + } + + /** + * Resize the viewport via Emulation.setDeviceMetricsOverride. + * Adjusts the visible viewport and device pixel ratio. + */ + async setViewportSize( + width: number, + height: number, + deviceScaleFactor?: number, + ): Promise<void> { + await this.cdp("Emulation.setDeviceMetricsOverride", { + width, + height, + deviceScaleFactor: deviceScaleFactor ?? 1, + mobile: false, + }); + this.invalidatePageInfoCache(); + } + + /** + * Capture screenshot with an optional debug crosshair overlay at (x, y). + * When debugClicks is true, overlays a red crosshair to verify click accuracy. + */ + async captureScreenshotWithDebugOverlay( + outputPath: string, + clickX: number, + clickY: number, + format: "png" | "jpeg" = "png", + quality = 80, + ): Promise<string> { + await this.captureScreenshot(outputPath, false, format, quality); + try { + // sharp is an optional dependency — import at runtime to avoid TS static analysis + const sharp: any = await new Function("return import('sharp')")() + .then((m: any) => m.default || m) + .catch(() => null); + if (!sharp) return outputPath; + const dpr = (await this.evaluateJS("window.devicePixelRatio")) as number || 1; + const metadata = await sharp(outputPath).metadata(); + const w = metadata.width || 0; + const h = metadata.height || 0; + const px = Math.round(clickX * dpr); + const py = Math.round(clickY * dpr); + const r = Math.round(15 * dpr); + + // Create SVG overlay for the crosshair + const svg = `<svg width="${w}" height="${h}"> + <circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${Math.max(2, Math.round(3 * dpr))}" opacity="0.8"/> + <line x1="${px - r - Math.round(5 * dpr)}" y1="${py}" x2="${px + r + Math.round(5 * dpr)}" y2="${py}" stroke="red" stroke-width="${Math.max(1, Math.round(2 * dpr))}" opacity="0.8"/> + <line x1="${px}" y1="${py - r - Math.round(5 * dpr)}" x2="${px}" y2="${py + r + Math.round(5 * dpr)}" stroke="red" stroke-width="${Math.max(1, Math.round(2 * dpr))}" opacity="0.8"/> +</svg>`; + await sharp(outputPath) + .composite([{ input: Buffer.from(svg), top: 0, left: 0 }]) + .toFile(outputPath + ".debug"); + const { rename } = await import("node:fs/promises"); + await rename(outputPath + ".debug", outputPath); + } catch { + // sharp not installed — overlay skipped + } + return outputPath; + } + + /** + * Perform a drag-and-drop sequence via Input.dispatchDragEvent. + * Moves from (startX, startY) to (endX, endY) with optional DataTransfer. + */ + async dragAndDrop( + startX: number, + startY: number, + endX: number, + endY: number, + dataTransfer?: Record<string, string>, + ): Promise<void> { + // CDP expects data.items for DataTransfer content + const data = dataTransfer + ? { + items: Object.entries(dataTransfer).map(([mimeType, data]) => ({ + mimeType, + data: Buffer.from(data).toString("base64"), + })), + dragOperationsMask: 1, // 1 = Copy + } + : { items: [], dragOperationsMask: 1 }; + + // mousePressed to initiate drag + await this.cdp("Input.dispatchMouseEvent", { + type: "mousePressed", + x: startX, + y: startY, + button: "left", + clickCount: 1, + }); + + // dragEnter at start + await this.cdp("Input.dispatchDragEvent", { + type: "dragEnter", + x: startX, + y: startY, + data, + modifiers: 0, + }); + + // dragOver at intermediate points + destination + const steps = 5; + for (let i = 1; i <= steps; i++) { + const t = i / steps; + const ix = startX + (endX - startX) * t; + const iy = startY + (endY - startY) * t; + await this.cdp("Input.dispatchDragEvent", { + type: "dragOver", + x: Math.round(ix), + y: Math.round(iy), + data, + modifiers: 0, + }); + } + + // drop at end position + await this.cdp("Input.dispatchDragEvent", { + type: "drop", + x: endX, + y: endY, + data, + modifiers: 0, + }); + + // mouseReleased + await this.cdp("Input.dispatchMouseEvent", { + type: "mouseReleased", + x: endX, + y: endY, + button: "left", + clickCount: 1, + }); + } + + /** Print the current page to PDF. Returns the file path. */ + async printToPDF(outputPath: string): Promise<string> { + const result = (await this.cdp("Page.printToPDF", { + printBackground: true, + preferCSSPageSize: true, + })) as { data: string }; + const buf = Buffer.from(result.data, "base64"); + await writeFile(outputPath, buf); + return outputPath; + } + + /** + * Get buffered network events (Network.requestWillBeSent, Network.responseReceived). + * Filters to the specified event types and optionally limits the count. + */ + async getNetworkLog( + eventTypes: string[] = [ + "Network.requestWillBeSent", + "Network.responseReceived", + ], + limit = 50, + ): Promise<CDPEvent[]> { + const events = await this.drainEvents(); + return events + .filter((e) => eventTypes.includes(e.method)) + .slice(-limit); + } + /** Get virtual key code for a key name or character */ getVirtualKeyCode(key: string): number { return VIRTUAL_KEY_CODES[key] ?? (key.length === 1 ? key.charCodeAt(0) : 0); diff --git a/src/prompt.ts b/src/prompt.ts index 8e3b727..a585124 100644 --- a/src/prompt.ts +++ b/src/prompt.ts @@ -102,5 +102,17 @@ browser_execute_js("document.querySelector('.main').innerText") → extract cont \`\`\` write("/tmp/extract.js", "...script with daemon access...") → browser_run_script("/tmp/extract.js") \`\`\` + +### Additional Tools + +| Tool | Purpose | +|------|---------| +| browser_upload_file | Upload a file to a file input (bypasses file picker) | +| browser_dispatch_key | Dispatch a DOM KeyboardEvent on a specific element (for React/Vue inputs) | +| browser_download | Configure download directory and disable save-as prompts | +| browser_viewport_resize | Resize the viewport for responsive testing | +| browser_drag_and_drop | Perform drag-and-drop from one coordinate to another | +| browser_print_to_pdf | Print the current page to a PDF file | +| browser_get_network_log | Get buffered network request/response events | `; } diff --git a/src/tools.ts b/src/tools.ts index 323f327..3ea0b54 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -181,6 +181,8 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { promptGuidelines: [ "Use browser_page_info to quickly check what page you're on and whether a JS dialog is blocking interaction.", "If browser_page_info returns a dialog, use browser_handle_dialog before any other browser actions.", + "JS dialogs freeze the page's JS thread, so no other interaction works until the dialog is handled.", + "browser_page_info auto-detects alert, confirm, prompt, and beforeunload dialogs.", ], parameters: Type.Object({}), async execute() { @@ -330,6 +332,8 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { "Capture a browser_screenshot BEFORE clicking to find the right coordinates.", "Capture another browser_screenshot AFTER clicking to verify the action worked.", "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.", + "If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.", + "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.", ], parameters: Type.Object({ x: Type.Number({ description: "X coordinate in CSS pixels from left edge of viewport" }), @@ -354,6 +358,28 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { button: params.button || "left", clickCount: params.clicks || 1, }); + + // Debug click overlay (when BH_DEBUG_CLICKS is set) + if (process.env.BH_DEBUG_CLICKS) { + try { + const debugPath = nextScreenshotPath(); + await daemon.captureScreenshotWithDebugOverlay( + debugPath, + params.x, + params.y, + ); + return { + content: [{ + type: "text" as const, + text: `Clicked at (${params.x}, ${params.y})\n[DEBUG] Overlay screenshot: ${debugPath}`, + }], + details: { debugScreenshotPath: debugPath }, + }; + } catch { + // Overlay failed — still report the click + } + } + return { content: [{ type: "text" as const, text: `Clicked at (${params.x}, ${params.y})` }], details: undefined }; } catch (err) { return { @@ -487,6 +513,7 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { "Use browser_scroll to scroll pages. Default deltaY=-300 scrolls down.", "Positive deltaY scrolls up, negative scrolls down.", "Capture browser_screenshot after scrolling to see the new viewport content.", + "If mouseWheel scroll doesn't work on a specific element, try browser_execute_js to scroll a container directly: e.scrollTop += 300 or e.scrollIntoView().", ], parameters: Type.Object({ deltaY: Type.Optional(Type.Number({ description: "Vertical scroll delta in pixels. Negative = down, positive = up. Default: -300" })), @@ -516,6 +543,14 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { } } + // Establish mouse position before wheel — Chrome requires a prior + // mouseMoved for mouseWheel to hit the correct element. + await daemon.cdp("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: cx, + y: cy, + }); + await daemon.cdp("Input.dispatchMouseEvent", { type: "mouseWheel", x: cx, @@ -536,6 +571,45 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { }); // ═══════════════════════════════════════════════════════════════════════════ + // browser_upload_file + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_upload_file", + label: "Browser Upload File", + description: + "Set files on a file input element via CDP DOM.setFileInputFiles. " + + "Use this to upload files to forms without clicking the file picker dialog. " + + "The path must be an absolute file path accessible from the machine running Chrome.", + promptSnippet: "Upload a file to a file input element (bypasses file picker)", + promptGuidelines: [ + "Use browser_upload_file to set files on <input type='file'> elements — much faster than clicking the file picker.", + "The selector must match a file input element (<input type='file'>).", + "The filePath must be an absolute path on the machine where Chrome is running.", + "To create a temp file first, use write or bash to create the file, then upload it.", + ], + parameters: Type.Object({ + selector: Type.String({ description: "CSS selector for the file input element" }), + filePath: Type.String({ description: "Absolute path to the file to upload" }), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + await daemon.uploadFile(params.selector, params.filePath); + return { + content: [{ type: "text" as const, text: `Set file "${params.filePath}" on "${params.selector}".` }], + details: undefined, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Upload failed: ${err instanceof Error ? err.message : String(err)}` }], + details: undefined, + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ // browser_screenshot // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ @@ -549,11 +623,14 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { "After every browser_click, browser_type, browser_scroll, or browser_navigate, capture a screenshot to VERIFY the action worked.", "Screenshots are the fastest way to find click targets, notice hidden blockers, and decide next steps.", "Never assume an action worked. Always re-screenshot to confirm.", + "For HiDPI/Retina screens, use maxDim (e.g. 2000) to keep images within LLM pixel limits.", + "Use format: 'jpeg' with quality: 80 for 2-5x smaller files on complex pages — faster transfers and lower context cost.", ], parameters: Type.Object({ fullPage: Type.Optional(Type.Boolean({ description: "Capture the full scrollable page (not just viewport). Default: false" })), format: Type.Optional(Type.String({ description: "Image format: 'png' or 'jpeg'. Default: 'png'" })), quality: Type.Optional(Type.Number({ description: "JPEG quality 1-100. Default: 80. Ignored for PNG." })), + maxDim: Type.Optional(Type.Number({ description: "If set, auto-resize the screenshot so neither dimension exceeds this (e.g. 2000 for LLM pixel limits). Requires sharp." })), }), async execute(_id, params) { try { @@ -562,9 +639,39 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { const format = (params.format === "jpeg" ? "jpeg" : "png") as "png" | "jpeg"; const quality = typeof params.quality === "number" ? params.quality : 80; await daemon.captureScreenshot(path, params.fullPage === true, format, quality); + + // Auto-resize if maxDim is specified and sharp is available + let resizeNote = ""; + const maxDim = typeof params.maxDim === "number" ? params.maxDim : undefined; + if (maxDim) { + try { + // sharp is an optional dependency — import at runtime + const sharp: any = await new Function("return import('sharp')")() + .then((m: any) => m.default || m) + .catch(() => null); + if (!sharp) { resizeNote = " (maxDim ignored: install sharp for auto-resize)"; } + else { + const metadata = await sharp(path).metadata(); + const w = metadata.width || 0; + const h = metadata.height || 0; + if (Math.max(w, h) > maxDim) { + await sharp(path) + .resize(maxDim, maxDim, { fit: "inside", withoutEnlargement: true }) + .toFile(path + ".resized"); + const { rename } = await import("node:fs/promises"); + await rename(path + ".resized", path); + resizeNote = ` (resized from ${w}x${h} to fit ${maxDim}px)`; + } + } + } catch { + // sharp not installed — skip resize + resizeNote = " (maxDim ignored: install sharp for auto-resize)"; + } + } + const fmtLabel = format === "jpeg" ? ` (JPEG q${quality})` : ""; return { - content: [{ type: "text" as const, text: `Screenshot saved: ${path}${params.fullPage ? " (full page)" : ""}${fmtLabel}` }], + content: [{ type: "text" as const, text: `Screenshot saved: ${path}${params.fullPage ? " (full page)" : ""}${fmtLabel}${resizeNote}` }], details: { path, format, quality }, }; } catch (err) { @@ -752,6 +859,8 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { "Use browser_execute_js for extracting structured data from the page (text, attributes, JSON from embedded scripts).", "Use browser_execute_js for DOM inspection when coordinates from screenshots aren't sufficient.", "For bulk data extraction, prefer browser_http_get for API calls — it's 10-50x faster than browser DOM scraping.", + "To execute JS inside an iframe, first find the iframe target via browser_list_tabs (includeChrome: true to see iframe targets), then pass targetId to browser_execute_js.", + "Compositor clicks (browser_click) already pass through iframes — only use iframe JS when you need to read/change iframe DOM.", ], parameters: Type.Object({ expression: Type.String({ description: "JavaScript expression to evaluate. 'return' statements are auto-wrapped in IIFE." }), @@ -911,6 +1020,53 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { }); // ═══════════════════════════════════════════════════════════════════════════ + // browser_dispatch_key + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_dispatch_key", + label: "Browser Dispatch Key", + description: + "Dispatch a DOM KeyboardEvent on a matched element. Use this when a site " + + "reacts to synthetic DOM key events more reliably than raw CDP input events " + + "(e.g., React/Vue comboboxes, autocomplete widgets, custom form inputs).", + promptSnippet: "Dispatch a DOM KeyboardEvent on a specific element (for framework inputs)", + promptGuidelines: [ + "Use browser_dispatch_key when browser_press_key doesn't work — some React/Vue components only listen for DOM-level events.", + "The selector must match a single element that can receive focus.", + "Common keys: 'Enter' (13), 'Tab' (9), 'Escape' (27), 'Backspace' (8), ' ' (32).", + "Try browser_press_key first. Only fall back to browser_dispatch_key if the page ignores raw CDP input.", + ], + parameters: Type.Object({ + selector: Type.String({ description: "CSS selector for the target element" }), + key: Type.String({ description: "Key to dispatch (e.g. 'Enter', 'Tab', 'Escape', 'a', ' ')" }), + event: Type.Optional(Type.String({ description: "DOM event type: 'keydown', 'keypress', or 'keyup'. Default: 'keypress'" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + + const key = params.key; + const eventType = params.event || "keypress"; + const keyCode = daemon.getVirtualKeyCode(key); + + const expression = `(()=>{const e=document.querySelector(${JSON.stringify(params.selector)});if(e){e.focus();e.dispatchEvent(new KeyboardEvent(${JSON.stringify(eventType)},{key:${JSON.stringify(key)},code:${JSON.stringify(key)},keyCode:${keyCode},which:${keyCode},bubbles:true}))}else{throw new Error('No element found for selector: ${params.selector.replace(/'/g, "\\'")}')}})()`; + + const result = await daemon.evaluateJS(expression); + return { + content: [{ type: "text" as const, text: `Dispatched ${eventType} "${key}" (keyCode: ${keyCode}) on "${params.selector}".` }], + details: undefined, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Dispatch key failed: ${err instanceof Error ? err.message : String(err)}` }], + details: undefined, + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ // browser_wait // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ @@ -1020,9 +1176,308 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { }); // ═══════════════════════════════════════════════════════════════════════════ - // browser_open_urls + // browser_download + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_download", + label: "Browser Download", + description: + "Configure the download directory and disable the save-as prompt for this tab. " + + "Call before triggering downloads to control where files are saved. " + + "After the download is triggered, call again with eventsOnly: true to check for download progress events.", + promptSnippet: "Configure browser download behavior (directory, no prompts)", + promptGuidelines: [ + "Use browser_download BEFORE clicking a download link or triggering a file save.", + "Set downloadPath to an absolute directory path (e.g. /tmp/downloads).", + "This disables the browser's save-as prompt, so files save automatically.", + "To check if a download started, call again with eventsOnly: true and look for Browser.downloadProgress events.", + ], + parameters: Type.Object({ + downloadPath: Type.Optional(Type.String({ description: "Absolute path to the download directory. Default: /tmp/browser-downloads" })), + eventsOnly: Type.Optional(Type.Boolean({ description: "If true, only check for recent download progress events. Default: false" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + + if (params.eventsOnly) { + // Drain events and filter for download progress + const events = await daemon.drainEvents(); + const downloads = events.filter( + (e) => + e.method === "Browser.downloadProgress" || + e.method === "Browser.downloadWillBegin", + ); + if (downloads.length === 0) { + return { + content: [{ type: "text" as const, text: "No recent download events found." }], + details: undefined, + }; + } + const lines = downloads.map((e) => { + const p = e.params as Record<string, unknown>; + return [ + `Event: ${e.method}`, + ` URL: ${p.url || "?"}`, + ` State: ${p.state || "?"}`, + p.receivedBytes !== undefined + ? ` Received: ${p.receivedBytes} / ${p.totalBytes || "?"}` + : "", + ] + .filter(Boolean) + .join("\n"); + }); + return { + content: [{ type: "text" as const, text: lines.join("\n\n") }], + details: undefined, + }; + } + + const dlPath = params.downloadPath || "/tmp/browser-downloads"; + await daemon.setDownloadBehavior(dlPath); + return { + content: [{ type: "text" as const, text: `Downloads will save to: ${dlPath} (save-as prompt disabled).` }], + details: undefined, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Download config failed: ${err instanceof Error ? err.message : String(err)}` }], + details: undefined, + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_viewport_resize // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ + name: "browser_viewport_resize", + label: "Browser Viewport Resize", + description: + "Resize the browser viewport. Useful for responsive testing or when a page " + + "requires a specific viewport width. Adjusts both the visible viewport and " + + "the device pixel ratio.", + promptSnippet: "Resize the browser viewport (width, height, deviceScaleFactor)", + promptGuidelines: [ + "Use browser_viewport_resize for responsive testing or when a page needs specific dimensions.", + "width and height are in CSS pixels (not device pixels).", + "deviceScaleFactor defaults to 1. Use 2 for HiDPI/Retina emulation.", + "Resizing may cause layout shifts — take a screenshot afterward to confirm.", + ], + parameters: Type.Object({ + width: Type.Number({ description: "Viewport width in CSS pixels" }), + height: Type.Number({ description: "Viewport height in CSS pixels" }), + deviceScaleFactor: Type.Optional(Type.Number({ description: "Device pixel ratio. Default: 1" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + await daemon.setViewportSize( + params.width, + params.height, + params.deviceScaleFactor, + ); + return { + content: [{ + type: "text" as const, + text: `Viewport resized to ${params.width}x${params.height}${params.deviceScaleFactor ? ` @${params.deviceScaleFactor}x` : ""}.`, + }], + details: undefined, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Viewport resize failed: ${err instanceof Error ? err.message : String(err)}` }], + details: undefined, + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_drag_and_drop + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_drag_and_drop", + label: "Browser Drag And Drop", + description: + "Perform a drag-and-drop operation from one viewport coordinate to another. " + + "Use for kanban boards, file uploads via drag, reorderable lists, and " + + "other drag-based UIs.", + promptSnippet: "Drag from (startX, startY) and drop at (endX, endY)", + promptGuidelines: [ + "Use browser_drag_and_drop for drag-based interactions like moving cards, reordering items, or drag-to-upload.", + "Capture a screenshot BEFORE to find the start and end coordinates.", + "Capture a screenshot AFTER to verify the drag-and-drop worked.", + "If the target site expects DOM-level drag events (not just compositor events), try browser_upload_file instead for file uploads.", + ], + parameters: Type.Object({ + startX: Type.Number({ description: "X coordinate to start dragging from" }), + startY: Type.Number({ description: "Y coordinate to start dragging from" }), + endX: Type.Number({ description: "X coordinate to drop at" }), + endY: Type.Number({ description: "Y coordinate to drop at" }), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + await daemon.dragAndDrop( + params.startX, + params.startY, + params.endX, + params.endY, + ); + return { + content: [{ + type: "text" as const, + text: `Dragged from (${params.startX}, ${params.startY}) to (${params.endX}, ${params.endY}).`, + }], + details: undefined, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Drag-and-drop failed: ${err instanceof Error ? err.message : String(err)}` }], + details: undefined, + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_print_to_pdf + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_print_to_pdf", + label: "Browser Print to PDF", + description: + "Print the current page to a PDF file via Page.printToPDF. " + + "Saves the PDF to the specified path. Includes background colors and respects CSS page size.", + promptSnippet: "Print the current page to a PDF file", + promptGuidelines: [ + "Use browser_print_to_pdf to save a page as PDF for archiving, sharing, or offline reading.", + "The output path should be an absolute file path (e.g. /tmp/page.pdf).", + "The page must be fully loaded before printing — use browser_wait_for_load first.", + ], + parameters: Type.Object({ + outputPath: Type.Optional(Type.String({ description: "Absolute path for the PDF file. Default: /tmp/browser-print-<timestamp>.pdf" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + const outputPath = + params.outputPath || + join(tmpdir(), `browser-print-${Date.now()}.pdf`); + await daemon.printToPDF(outputPath); + return { + content: [{ type: "text" as const, text: `PDF saved: ${outputPath}` }], + details: { path: outputPath }, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Print to PDF failed: ${err instanceof Error ? err.message : String(err)}` }], + details: undefined, + }; + } + }, + renderCall(_, theme) { + return new Text(`📄 Printing to PDF...`, 0, 0); + }, + renderResult(result, _options, theme) { + const path = (result.details as { path?: string })?.path; + return new Text( + path + ? theme.fg("accent", `📄 PDF: ${path}`) + : `📄 PDF saved`, + 0, + 0, + ); + }, + }); + + // ═══════════════════════════════════════════════════════════════════════════ + // browser_get_network_log + // ═══════════════════════════════════════════════════════════════════════════ + pi.registerTool({ + name: "browser_get_network_log", + label: "Browser Get Network Log", + description: + "Get buffered network events (Network.requestWillBeSent, Network.responseReceived) " + + "from the current page. Useful for inspecting XHR/fetch calls the page makes, " + + "finding API endpoints, or debugging network issues.", + promptSnippet: "Get buffered network request/response events", + promptGuidelines: [ + "Use browser_get_network_log to find API endpoints the page calls — often 10x faster than DOM scraping.", + "Network events are buffered since the last call to this tool — call it after performing actions that trigger XHR/fetch.", + "Filter by eventType to focus on specific events: 'Network.requestWillBeSent' (URLs, headers) or 'Network.responseReceived' (status, response headers).", + "Combine with browser_http_get to replay captured API calls.", + ], + parameters: Type.Object({ + eventTypes: Type.Optional( + Type.Array(Type.String(), { + description: + "Network event types to filter. Default: ['Network.requestWillBeSent', 'Network.responseReceived']", + }), + ), + limit: Type.Optional(Type.Number({ description: "Max events to return. Default: 50" })), + }), + async execute(_id, params) { + try { + await daemon.ensureAlive(); + const events = await daemon.getNetworkLog( + params.eventTypes, + params.limit, + ); + + if (events.length === 0) { + return { + content: [{ type: "text" as const, text: "No buffered network events. Perform actions that trigger XHR/fetch calls first." }], + details: undefined, + }; + } + + const lines = events.map((e) => { + const p = e.params as Record<string, unknown>; + if (e.method === "Network.requestWillBeSent") { + const req = (p.request as Record<string, unknown>) || {}; + return [ + `→ ${req.method || "GET"} ${req.url || p.documentURL || "?"}`, + ` Type: ${p.type || "?"}`, + ].join("\n"); + } + if (e.method === "Network.responseReceived") { + const resp = (p.response as Record<string, unknown>) || {}; + return [ + `← ${resp.status || "?"} ${resp.statusText || ""} ${resp.url || p.url || "?"}`, + ` MIME: ${resp.mimeType || "?"}`, + ].join("\n"); + } + return `${e.method}: ${JSON.stringify(p).slice(0, 200)}`; + }); + + const raw = `${events.length} network event(s):\n\n${lines.join("\n\n")}`; + const { text, fullOutputPath } = await applyTruncation(raw, "net"); + + return { + content: [{ type: "text" as const, text }], + details: { count: events.length, fullOutputPath }, + }; + } catch (err) { + return { + isError: true, + content: [{ type: "text" as const, text: `Network log failed: ${err instanceof Error ? err.message : String(err)}` }], + details: undefined, + }; + } + }, + }); + + // ═══════════════════════════════════════════════════════════════════════ + // browser_open_urls + // ═══════════════════════════════════════════════════════════════════════ + pi.registerTool({ name: "browser_open_urls", label: "Browser Open URLs", description: |
