aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authoramankumarsingh77 <amankumarsingh7702@gmal.com>2026-05-02 16:20:09 +0530
committeramankumarsingh77 <amankumarsingh7702@gmal.com>2026-05-02 16:20:09 +0530
commit6708da54b1805756908dea84bf65fe73ab03426a (patch)
tree95e144b83d69af9918321f2e3faf29c5a8bfad17
parent9ead6a0577b750bc3ae3f36a7e8bfe108e979925 (diff)
feat(domains): migrate screenshot, add typed sharp shim, restore click crosshair overlay
-rw-r--r--src/domains/click.ts16
-rw-r--r--src/domains/screenshot.ts114
-rw-r--r--src/registry.ts2
-rw-r--r--src/tools.ts82
-rw-r--r--src/util/sharp-shim.ts49
5 files changed, 170 insertions, 93 deletions
diff --git a/src/domains/click.ts b/src/domains/click.ts
index b3aa80a..c269dd8 100644
--- a/src/domains/click.ts
+++ b/src/domains/click.ts
@@ -1,11 +1,10 @@
-import { writeFile } from "node:fs/promises";
import { Type, type Static } from "typebox";
import { Text } from "@mariozechner/pi-tui";
import type { BrowserClient } from "../client";
import { Coords, MouseButton } from "../schemas/common";
-import { screenshotPath } from "../util/paths";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr } from "../util/tool";
+import { captureWithCrosshair } from "./screenshot";
const ClickArgs = Type.Object({
...Coords.properties,
@@ -58,16 +57,11 @@ export const clickTool = defineBrowserTool({
const clicked = await dispatchClick(client, args);
if (!clicked.success) return clicked;
if (process.env["BH_DEBUG_CLICKS"]) {
- // Plain screenshot for now; the crosshair overlay returns in Task 14
- // (domains/screenshot.ts) once sharp-shim lands.
- const path = screenshotPath(client.namespace, "png");
- const shot = await client.session().call("Page.captureScreenshot", { format: "png" });
- if (shot.success) {
- const data = (shot.data as { data: string }).data;
- await writeFile(path, Buffer.from(data, "base64"));
+ const debug = await captureWithCrosshair(client, { x: args.x, y: args.y });
+ if (debug.success) {
return ok({
- text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Screenshot: ${path}`,
- details: { debugScreenshotPath: path, x: args.x, y: args.y },
+ text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`,
+ details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y },
});
}
}
diff --git a/src/domains/screenshot.ts b/src/domains/screenshot.ts
new file mode 100644
index 0000000..03b49f0
--- /dev/null
+++ b/src/domains/screenshot.ts
@@ -0,0 +1,114 @@
+import { writeFile, rename } from "node:fs/promises";
+import { Type } from "typebox";
+import type { BrowserClient } from "../client";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { screenshotPath } from "../util/paths";
+import { loadSharp } from "../util/sharp-shim";
+import { safeJs } from "../util/js-template";
+
+const ScreenshotArgs = Type.Object({
+ fullPage: Type.Optional(Type.Boolean({ default: false, description: "Capture beyond viewport" })),
+ format: Type.Optional(Type.Union([Type.Literal("png"), Type.Literal("jpeg")], { default: "png" })),
+ quality: Type.Optional(Type.Integer({ minimum: 1, maximum: 100, default: 80, description: "JPEG quality 1-100" })),
+ maxDim: Type.Optional(Type.Integer({ minimum: 100, maximum: 8000, description: "If max(w,h) exceeds this, resize via sharp." })),
+});
+
+type CaptureArgs = {
+ readonly fullPage?: boolean;
+ readonly format?: "png" | "jpeg";
+ readonly quality?: number;
+};
+
+const captureBase = async (
+ client: BrowserClient,
+ args: CaptureArgs,
+): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => {
+ const format = args.format ?? "png";
+ const quality = args.quality ?? 80;
+ const path = screenshotPath(client.namespace, format);
+ const params: Record<string, unknown> = { format, captureBeyondViewport: args.fullPage ?? false };
+ if (format === "jpeg") params["quality"] = quality;
+ const r = await client.session().call("Page.captureScreenshot", params);
+ if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+ // Page.captureScreenshot returns { data: base64 }
+ const data = (r.data as { data: string }).data;
+ await writeFile(path, Buffer.from(data, "base64"));
+ return ok({ path, format });
+};
+
+const resizeIfNeeded = async (
+ path: string,
+ maxDim: number,
+): Promise<{ readonly note: string }> => {
+ const load = await loadSharp();
+ if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" };
+ if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` };
+ try {
+ const meta = await load.sharp(path).metadata();
+ const w = meta.width ?? 0;
+ const h = meta.height ?? 0;
+ if (Math.max(w, h) <= maxDim) return { note: "" };
+ const tmp = `${path}.resized`;
+ await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp);
+ await rename(tmp, path);
+ return { note: ` (resized to fit ${maxDim}px)` };
+ } catch (e) {
+ return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` };
+ }
+};
+
+export const screenshotTool = defineBrowserTool({
+ name: "browser_screenshot",
+ label: "Browser Screenshot",
+ description: "Capture the current page as PNG or JPEG. JPEG is 2-5x smaller — prefer it for photo-heavy pages.",
+ promptSnippet: "Capture a screenshot of the current page",
+ promptGuidelines: [
+ "Use BEFORE clicks/scrolls to find coordinates; AFTER to verify the action.",
+ "Pass format='jpeg' with a quality (60-90) for smaller files on photo-heavy pages.",
+ "Set maxDim if the page is huge and you want to fit under LLM image-size limits.",
+ ],
+ parameters: ScreenshotArgs,
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+ const cap = await captureBase(client, args);
+ if (!cap.success) return cap;
+ let note = "";
+ if (args.maxDim !== undefined) {
+ const resized = await resizeIfNeeded(cap.data.path, args.maxDim);
+ note = resized.note;
+ }
+ return ok({
+ text: `Screenshot saved: ${cap.data.path}${note}`,
+ details: { path: cap.data.path, format: cap.data.format, attached: false },
+ });
+ },
+});
+
+export const captureWithCrosshair = async (
+ client: BrowserClient,
+ args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number },
+): Promise<Result<{ readonly path: string }, ToolErr>> => {
+ const cap = await captureBase(client, args);
+ if (!cap.success) return cap;
+ const load = await loadSharp();
+ if (load.kind !== "ok") return ok({ path: cap.data.path });
+ try {
+ const dprR = await client.evaluateJs(safeJs`window.devicePixelRatio`);
+ const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1;
+ const meta = await load.sharp(cap.data.path).metadata();
+ const w = meta.width ?? 0;
+ const h = meta.height ?? 0;
+ const px = Math.round(args.x * dpr);
+ const py = Math.round(args.y * dpr);
+ const r = Math.round(15 * dpr);
+ const stroke = Math.max(2, Math.round(3 * dpr));
+ const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`;
+ const tmp = `${cap.data.path}.debug`;
+ await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp);
+ await rename(tmp, cap.data.path);
+ return ok({ path: cap.data.path });
+ } catch {
+ // Crosshair overlay is best-effort; on failure return the plain screenshot.
+ return ok({ path: cap.data.path });
+ }
+};
diff --git a/src/registry.ts b/src/registry.ts
index d3559ed..6e4acda 100644
--- a/src/registry.ts
+++ b/src/registry.ts
@@ -6,6 +6,7 @@ import { typeTool, pressKeyTool, dispatchKeyTool } from "./domains/keyboard";
import { pageInfoTool, waitTool, waitForLoadTool } from "./domains/page";
import { scrollTool } from "./domains/scroll";
import { handleDialogTool } from "./domains/dialog";
+import { screenshotTool } from "./domains/screenshot";
const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [
clickTool,
@@ -17,6 +18,7 @@ const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [
waitTool,
waitForLoadTool,
handleDialogTool,
+ screenshotTool,
];
export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => {
diff --git a/src/tools.ts b/src/tools.ts
index be74a03..3ea9179 100644
--- a/src/tools.ts
+++ b/src/tools.ts
@@ -310,88 +310,6 @@ export function registerTools(pi: ExtensionAPI, daemon: BrowserDaemon): void {
});
// ═══════════════════════════════════════════════════════════════════════════
- // browser_screenshot
- // ═══════════════════════════════════════════════════════════════════════════
- pi.registerTool({
- name: "browser_screenshot",
- label: "Browser Screenshot",
- description:
- "Capture a PNG or JPEG screenshot of the current page. Use this BEFORE interactions to find targets, and AFTER to verify results. JPEG (quality 80) is 2-5x smaller for complex pages — faster transfer and lower LLM context cost.",
- promptSnippet: "Capture a screenshot of the current page (PNG or JPEG)",
- promptGuidelines: [
- "Use browser_screenshot as your FIRST step when exploring a page — it shows you what's visible.",
- "After every browser_click, browser_type, browser_scroll, or browser_navigate, capture a screenshot to VERIFY the action worked.",
- "Screenshots are the fastest way to find click targets, notice hidden blockers, and decide next steps.",
- "Never assume an action worked. Always re-screenshot to confirm.",
- "For HiDPI/Retina screens, use maxDim (e.g. 2000) to keep images within LLM pixel limits.",
- "Use format: 'jpeg' with quality: 80 for 2-5x smaller files on complex pages — faster transfers and lower context cost.",
- ],
- parameters: Type.Object({
- fullPage: Type.Optional(Type.Boolean({ description: "Capture the full scrollable page (not just viewport). Default: false" })),
- format: Type.Optional(Type.String({ description: "Image format: 'png' or 'jpeg'. Default: 'png'" })),
- quality: Type.Optional(Type.Number({ description: "JPEG quality 1-100. Default: 80. Ignored for PNG." })),
- maxDim: Type.Optional(Type.Number({ description: "If set, auto-resize the screenshot so neither dimension exceeds this (e.g. 2000 for LLM pixel limits). Requires sharp." })),
- }),
- async execute(_id, params) {
- try {
- await daemon.ensureAlive();
- const path = nextScreenshotPath();
- const format = (params.format === "jpeg" ? "jpeg" : "png") as "png" | "jpeg";
- const quality = typeof params.quality === "number" ? params.quality : 80;
- await daemon.captureScreenshot(path, params.fullPage === true, format, quality);
-
- // Auto-resize if maxDim is specified and sharp is available
- let resizeNote = "";
- const maxDim = typeof params.maxDim === "number" ? params.maxDim : undefined;
- if (maxDim) {
- try {
- // sharp is an optional dependency — import at runtime
- const sharp: any = await new Function("return import('sharp')")()
- .then((m: any) => m.default || m)
- .catch(() => null);
- if (!sharp) { resizeNote = " (maxDim ignored: install sharp for auto-resize)"; }
- else {
- const metadata = await sharp(path).metadata();
- const w = metadata.width || 0;
- const h = metadata.height || 0;
- if (Math.max(w, h) > maxDim) {
- await sharp(path)
- .resize(maxDim, maxDim, { fit: "inside", withoutEnlargement: true })
- .toFile(path + ".resized");
- const { rename } = await import("node:fs/promises");
- await rename(path + ".resized", path);
- resizeNote = ` (resized from ${w}x${h} to fit ${maxDim}px)`;
- }
- }
- } catch {
- // sharp not installed — skip resize
- resizeNote = " (maxDim ignored: install sharp for auto-resize)";
- }
- }
-
- const fmtLabel = format === "jpeg" ? ` (JPEG q${quality})` : "";
- return {
- content: [{ type: "text" as const, text: `Screenshot saved: ${path}${params.fullPage ? " (full page)" : ""}${fmtLabel}${resizeNote}` }],
- details: { path, format, quality },
- };
- } catch (err) {
- return {
- isError: true,
- content: [{ type: "text" as const, text: `Screenshot failed: ${err instanceof Error ? err.message : String(err)}` }],
- details: undefined,
- };
- }
- },
- renderCall(_, theme) {
- return new Text(`📸 Capturing screenshot...`, 0, 0);
- },
- renderResult(result, _options, theme) {
- const path = (result.details as { path?: string })?.path;
- return new Text(path ? `📸 Screenshot: ${path}` : `📸 Screenshot captured`, 0, 0);
- },
- });
-
- // ═══════════════════════════════════════════════════════════════════════════
// browser_list_tabs
// ═══════════════════════════════════════════════════════════════════════════
pi.registerTool({
diff --git a/src/util/sharp-shim.ts b/src/util/sharp-shim.ts
new file mode 100644
index 0000000..7ea409d
--- /dev/null
+++ b/src/util/sharp-shim.ts
@@ -0,0 +1,49 @@
+// Typed minimal shim for the optional `sharp` dependency. Sharp is a heavy
+// native module not installed in every workspace; we import dynamically and
+// distinguish "missing" (no install) from "error" (installed but threw).
+//
+// We declare only the surface we use. The dynamic import returns `unknown`
+// at the boundary; the cast inside loadSharp() is the documented escape
+// hatch — every other consumer sees the typed SharpFactory.
+
+export type SharpInstance = {
+ metadata(): Promise<{ width?: number; height?: number }>;
+ composite(layers: ReadonlyArray<{ input: Buffer; top: number; left: number }>): SharpInstance;
+ resize(width: number, height: number, opts?: { fit?: "inside" }): SharpInstance;
+ toFile(path: string): Promise<unknown>;
+};
+
+export type SharpFactory = (input: string | Buffer) => SharpInstance;
+
+export type SharpLoad =
+ | { readonly kind: "ok"; readonly sharp: SharpFactory }
+ | { readonly kind: "missing" }
+ | { readonly kind: "error"; readonly message: string };
+
+export const loadSharp = async (): Promise<SharpLoad> => {
+ let mod: unknown;
+ try {
+ // sharp is an optional peer dependency; it may not be installed.
+ // @ts-expect-error sharp is an optional peer dependency not required at compile time
+ mod = await import("sharp");
+ } catch (e) {
+ const msg = e instanceof Error ? e.message : String(e);
+ if (
+ msg.includes("Cannot find module") ||
+ msg.includes("MODULE_NOT_FOUND") ||
+ msg.includes("ERR_MODULE_NOT_FOUND")
+ ) {
+ return { kind: "missing" };
+ }
+ return { kind: "error", message: msg };
+ }
+ if (mod === null || mod === undefined) return { kind: "missing" };
+ // The CommonJS sharp module's default export IS the factory. ESM-wrapped
+ // sharp puts it on .default. Probe both.
+ const m = mod as { default?: unknown };
+ const candidate: unknown = typeof m.default === "function" ? m.default : mod;
+ if (typeof candidate !== "function") {
+ return { kind: "error", message: "sharp module did not expose a callable factory" };
+ }
+ return { kind: "ok", sharp: candidate as SharpFactory };
+};