diff options
| author | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-05-02 16:20:09 +0530 |
|---|---|---|
| committer | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-05-02 16:20:09 +0530 |
| commit | 6708da54b1805756908dea84bf65fe73ab03426a (patch) | |
| tree | 95e144b83d69af9918321f2e3faf29c5a8bfad17 | |
| parent | 9ead6a0577b750bc3ae3f36a7e8bfe108e979925 (diff) | |
feat(domains): migrate screenshot, add typed sharp shim, restore click crosshair overlay
| -rw-r--r-- | src/domains/click.ts | 16 | ||||
| -rw-r--r-- | src/domains/screenshot.ts | 114 | ||||
| -rw-r--r-- | src/registry.ts | 2 | ||||
| -rw-r--r-- | src/tools.ts | 82 | ||||
| -rw-r--r-- | src/util/sharp-shim.ts | 49 |
5 files changed, 170 insertions, 93 deletions
diff --git a/src/domains/click.ts b/src/domains/click.ts index b3aa80a..c269dd8 100644 --- a/src/domains/click.ts +++ b/src/domains/click.ts @@ -1,11 +1,10 @@ -import { writeFile } from "node:fs/promises"; import { Type, type Static } from "typebox"; import { Text } from "@mariozechner/pi-tui"; import type { BrowserClient } from "../client"; import { Coords, MouseButton } from "../schemas/common"; -import { screenshotPath } from "../util/paths"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr } from "../util/tool"; +import { captureWithCrosshair } from "./screenshot"; const ClickArgs = Type.Object({ ...Coords.properties, @@ -58,16 +57,11 @@ export const clickTool = defineBrowserTool({ const clicked = await dispatchClick(client, args); if (!clicked.success) return clicked; if (process.env["BH_DEBUG_CLICKS"]) { - // Plain screenshot for now; the crosshair overlay returns in Task 14 - // (domains/screenshot.ts) once sharp-shim lands. - const path = screenshotPath(client.namespace, "png"); - const shot = await client.session().call("Page.captureScreenshot", { format: "png" }); - if (shot.success) { - const data = (shot.data as { data: string }).data; - await writeFile(path, Buffer.from(data, "base64")); + const debug = await captureWithCrosshair(client, { x: args.x, y: args.y }); + if (debug.success) { return ok({ - text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Screenshot: ${path}`, - details: { debugScreenshotPath: path, x: args.x, y: args.y }, + text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`, + details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y }, }); } } diff --git a/src/domains/screenshot.ts b/src/domains/screenshot.ts new file mode 100644 index 0000000..03b49f0 --- /dev/null +++ b/src/domains/screenshot.ts @@ -0,0 +1,114 @@ +import { writeFile, rename } from "node:fs/promises"; +import { Type } from "typebox"; +import type { BrowserClient } from "../client"; +import { type Result, err, ok } from "../util/result"; +import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; +import { screenshotPath } from "../util/paths"; +import { loadSharp } from "../util/sharp-shim"; +import { safeJs } from "../util/js-template"; + +const ScreenshotArgs = Type.Object({ + fullPage: Type.Optional(Type.Boolean({ default: false, description: "Capture beyond viewport" })), + format: Type.Optional(Type.Union([Type.Literal("png"), Type.Literal("jpeg")], { default: "png" })), + quality: Type.Optional(Type.Integer({ minimum: 1, maximum: 100, default: 80, description: "JPEG quality 1-100" })), + maxDim: Type.Optional(Type.Integer({ minimum: 100, maximum: 8000, description: "If max(w,h) exceeds this, resize via sharp." })), +}); + +type CaptureArgs = { + readonly fullPage?: boolean; + readonly format?: "png" | "jpeg"; + readonly quality?: number; +}; + +const captureBase = async ( + client: BrowserClient, + args: CaptureArgs, +): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => { + const format = args.format ?? "png"; + const quality = args.quality ?? 80; + const path = screenshotPath(client.namespace, format); + const params: Record<string, unknown> = { format, captureBeyondViewport: args.fullPage ?? false }; + if (format === "jpeg") params["quality"] = quality; + const r = await client.session().call("Page.captureScreenshot", params); + if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); + // Page.captureScreenshot returns { data: base64 } + const data = (r.data as { data: string }).data; + await writeFile(path, Buffer.from(data, "base64")); + return ok({ path, format }); +}; + +const resizeIfNeeded = async ( + path: string, + maxDim: number, +): Promise<{ readonly note: string }> => { + const load = await loadSharp(); + if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" }; + if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` }; + try { + const meta = await load.sharp(path).metadata(); + const w = meta.width ?? 0; + const h = meta.height ?? 0; + if (Math.max(w, h) <= maxDim) return { note: "" }; + const tmp = `${path}.resized`; + await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp); + await rename(tmp, path); + return { note: ` (resized to fit ${maxDim}px)` }; + } catch (e) { + return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` }; + } +}; + +export const screenshotTool = defineBrowserTool({ + name: "browser_screenshot", + label: "Browser Screenshot", + description: "Capture the current page as PNG or JPEG. JPEG is 2-5x smaller — prefer it for photo-heavy pages.", + promptSnippet: "Capture a screenshot of the current page", + promptGuidelines: [ + "Use BEFORE clicks/scrolls to find coordinates; AFTER to verify the action.", + "Pass format='jpeg' with a quality (60-90) for smaller files on photo-heavy pages.", + "Set maxDim if the page is huge and you want to fit under LLM image-size limits.", + ], + parameters: ScreenshotArgs, + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const cap = await captureBase(client, args); + if (!cap.success) return cap; + let note = ""; + if (args.maxDim !== undefined) { + const resized = await resizeIfNeeded(cap.data.path, args.maxDim); + note = resized.note; + } + return ok({ + text: `Screenshot saved: ${cap.data.path}${note}`, + details: { path: cap.data.path, format: cap.data.format, attached: false }, + }); + }, +}); + +export const captureWithCrosshair = async ( + client: BrowserClient, + args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number }, +): Promise<Result<{ readonly path: string }, ToolErr>> => { + const cap = await captureBase(client, args); + if (!cap.success) return cap; + const load = await loadSharp(); + if (load.kind !== "ok") return ok({ path: cap.data.path }); + try { + const dprR = await client.evaluateJs(safeJs`window.devicePixelRatio`); + const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1; + const meta = await load.sharp(cap.data.path).metadata(); + const w = meta.width ?? 0; + const h = meta.height ?? 0; + const px = Math.round(args.x * dpr); + const py = Math.round(args.y * dpr); + const r = Math.round(15 * dpr); + const stroke = Math.max(2, Math.round(3 * dpr)); + const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`; + const tmp = `${cap.data.path}.debug`; + await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp); + await rename(tmp, cap.data.path); + return ok({ path: cap.data.path }); + } catch { + // Crosshair overlay is best-effort; on failure return the plain screenshot. + return ok({ path: cap.data.path }); + } +}; diff --git a/src/registry.ts b/src/registry.ts index d3559ed..6e4acda 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -6,6 +6,7 @@ import { typeTool, pressKeyTool, dispatchKeyTool } from "./domains/keyboard"; import { pageInfoTool, waitTool, waitForLoadTool } from "./domains/page"; import { scrollTool } from "./domains/scroll"; import { handleDialogTool } from "./domains/dialog"; +import { screenshotTool } from "./domains/screenshot"; const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [ clickTool, @@ -17,6 +18,7 @@ const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [ waitTool, waitForLoadTool, handleDialogTool, + screenshotTool, ]; export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => { diff --git a/src/tools.ts b/src/tools.ts index be74a03..3ea9179 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -310,88 +310,6 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void { }); // ═══════════════════════════════════════════════════════════════════════════ - // browser_screenshot - // ═══════════════════════════════════════════════════════════════════════════ - pi.registerTool({ - name: "browser_screenshot", - label: "Browser Screenshot", - description: - "Capture a PNG or JPEG screenshot of the current page. Use this BEFORE interactions to find targets, and AFTER to verify results. JPEG (quality 80) is 2-5x smaller for complex pages — faster transfer and lower LLM context cost.", - promptSnippet: "Capture a screenshot of the current page (PNG or JPEG)", - promptGuidelines: [ - "Use browser_screenshot as your FIRST step when exploring a page — it shows you what's visible.", - "After every browser_click, browser_type, browser_scroll, or browser_navigate, capture a screenshot to VERIFY the action worked.", - "Screenshots are the fastest way to find click targets, notice hidden blockers, and decide next steps.", - "Never assume an action worked. Always re-screenshot to confirm.", - "For HiDPI/Retina screens, use maxDim (e.g. 2000) to keep images within LLM pixel limits.", - "Use format: 'jpeg' with quality: 80 for 2-5x smaller files on complex pages — faster transfers and lower context cost.", - ], - parameters: Type.Object({ - fullPage: Type.Optional(Type.Boolean({ description: "Capture the full scrollable page (not just viewport). Default: false" })), - format: Type.Optional(Type.String({ description: "Image format: 'png' or 'jpeg'. Default: 'png'" })), - quality: Type.Optional(Type.Number({ description: "JPEG quality 1-100. Default: 80. Ignored for PNG." })), - maxDim: Type.Optional(Type.Number({ description: "If set, auto-resize the screenshot so neither dimension exceeds this (e.g. 2000 for LLM pixel limits). Requires sharp." })), - }), - async execute(_id, params) { - try { - await daemon.ensureAlive(); - const path = nextScreenshotPath(); - const format = (params.format === "jpeg" ? "jpeg" : "png") as "png" | "jpeg"; - const quality = typeof params.quality === "number" ? params.quality : 80; - await daemon.captureScreenshot(path, params.fullPage === true, format, quality); - - // Auto-resize if maxDim is specified and sharp is available - let resizeNote = ""; - const maxDim = typeof params.maxDim === "number" ? params.maxDim : undefined; - if (maxDim) { - try { - // sharp is an optional dependency — import at runtime - const sharp: any = await new Function("return import('sharp')")() - .then((m: any) => m.default || m) - .catch(() => null); - if (!sharp) { resizeNote = " (maxDim ignored: install sharp for auto-resize)"; } - else { - const metadata = await sharp(path).metadata(); - const w = metadata.width || 0; - const h = metadata.height || 0; - if (Math.max(w, h) > maxDim) { - await sharp(path) - .resize(maxDim, maxDim, { fit: "inside", withoutEnlargement: true }) - .toFile(path + ".resized"); - const { rename } = await import("node:fs/promises"); - await rename(path + ".resized", path); - resizeNote = ` (resized from ${w}x${h} to fit ${maxDim}px)`; - } - } - } catch { - // sharp not installed — skip resize - resizeNote = " (maxDim ignored: install sharp for auto-resize)"; - } - } - - const fmtLabel = format === "jpeg" ? ` (JPEG q${quality})` : ""; - return { - content: [{ type: "text" as const, text: `Screenshot saved: ${path}${params.fullPage ? " (full page)" : ""}${fmtLabel}${resizeNote}` }], - details: { path, format, quality }, - }; - } catch (err) { - return { - isError: true, - content: [{ type: "text" as const, text: `Screenshot failed: ${err instanceof Error ? err.message : String(err)}` }], - details: undefined, - }; - } - }, - renderCall(_, theme) { - return new Text(`📸 Capturing screenshot...`, 0, 0); - }, - renderResult(result, _options, theme) { - const path = (result.details as { path?: string })?.path; - return new Text(path ? `📸 Screenshot: ${path}` : `📸 Screenshot captured`, 0, 0); - }, - }); - - // ═══════════════════════════════════════════════════════════════════════════ // browser_list_tabs // ═══════════════════════════════════════════════════════════════════════════ pi.registerTool({ diff --git a/src/util/sharp-shim.ts b/src/util/sharp-shim.ts new file mode 100644 index 0000000..7ea409d --- /dev/null +++ b/src/util/sharp-shim.ts @@ -0,0 +1,49 @@ +// Typed minimal shim for the optional `sharp` dependency. Sharp is a heavy +// native module not installed in every workspace; we import dynamically and +// distinguish "missing" (no install) from "error" (installed but threw). +// +// We declare only the surface we use. The dynamic import returns `unknown` +// at the boundary; the cast inside loadSharp() is the documented escape +// hatch — every other consumer sees the typed SharpFactory. + +export type SharpInstance = { + metadata(): Promise<{ width?: number; height?: number }>; + composite(layers: ReadonlyArray<{ input: Buffer; top: number; left: number }>): SharpInstance; + resize(width: number, height: number, opts?: { fit?: "inside" }): SharpInstance; + toFile(path: string): Promise<unknown>; +}; + +export type SharpFactory = (input: string | Buffer) => SharpInstance; + +export type SharpLoad = + | { readonly kind: "ok"; readonly sharp: SharpFactory } + | { readonly kind: "missing" } + | { readonly kind: "error"; readonly message: string }; + +export const loadSharp = async (): Promise<SharpLoad> => { + let mod: unknown; + try { + // sharp is an optional peer dependency; it may not be installed. + // @ts-expect-error sharp is an optional peer dependency not required at compile time + mod = await import("sharp"); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + if ( + msg.includes("Cannot find module") || + msg.includes("MODULE_NOT_FOUND") || + msg.includes("ERR_MODULE_NOT_FOUND") + ) { + return { kind: "missing" }; + } + return { kind: "error", message: msg }; + } + if (mod === null || mod === undefined) return { kind: "missing" }; + // The CommonJS sharp module's default export IS the factory. ESM-wrapped + // sharp puts it on .default. Probe both. + const m = mod as { default?: unknown }; + const candidate: unknown = typeof m.default === "function" ? m.default : mod; + if (typeof candidate !== "function") { + return { kind: "error", message: "sharp module did not expose a callable factory" }; + } + return { kind: "ok", sharp: candidate as SharpFactory }; +}; |
