aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authoramankumarsingh77 <amankumarsingh7702@gmail.com>2026-07-28 22:00:33 +0530
committerAman Kumar <145339995+amankumarsingh77@users.noreply.github.com>2026-08-02 19:12:46 +0530
commit86045450c208bc51670dd9554e581f2a62b181c9 (patch)
tree6e1ee7617afd829549ade41e527f07155921a47e /src
parent632600ec0a0a68d9327e19bd419a1ae72047955f (diff)
refactor: helpers live in helper modules, not in tool files
form.ts exported resolveAndCall and detailsOf to form-batch.ts, snapshot.ts exported buildTree and collectInteractiveTargets to ref-resolve.ts, and screenshot.ts exported captureWithCrosshair to click.ts — so importing one tool pulled in another tool's whole module. The shared halves move out: ax-tree.ts (the AX tree model, its construction and the queries over it), element-call.ts (calling a function on a resolved element, and the PageResult it returns), screenshot-capture.ts (capture, resize, crosshair overlay). Each tool file now holds its tools and its own presentation, nothing else. NavOutcome stops being exported; nothing outside navigate.ts read it. target.ts stays a module rather than folding into click.ts as planned: it is already helper-only and has its own test suite, and moving a tested pure function into a tool file is the thing this commit removes. Having one consumer is not the defect. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Diffstat (limited to 'src')
-rw-r--r--src/domains/ax-tree.ts179
-rw-r--r--src/domains/click.ts2
-rw-r--r--src/domains/element-call.ts117
-rw-r--r--src/domains/form-batch.ts2
-rw-r--r--src/domains/form.ts117
-rw-r--r--src/domains/navigate.ts2
-rw-r--r--src/domains/ref-resolve.ts2
-rw-r--r--src/domains/screenshot-capture.ts80
-rw-r--r--src/domains/screenshot.ts79
-rw-r--r--src/domains/snapshot.ts189
10 files changed, 394 insertions, 375 deletions
diff --git a/src/domains/ax-tree.ts b/src/domains/ax-tree.ts
new file mode 100644
index 0000000..1329063
--- /dev/null
+++ b/src/domains/ax-tree.ts
@@ -0,0 +1,179 @@
+import type { AxNodeResult } from "../cdp/commands";
+import type { BrowserClient } from "../client";
+import type { Box } from "./box";
+import { cdpCall } from "./cdp-call";
+
+export type RawAxNode = AxNodeResult;
+type AxValue = NonNullable<RawAxNode["role"]>;
+
+export type SlimNode = {
+ role: string;
+ name?: string;
+ value?: string;
+ description?: string;
+ state?: string;
+ children: SlimNode[];
+ _backendId?: number;
+ ref?: string;
+ box?: Box;
+};
+
+export const INTERACTIVE_ROLES = new Set([
+ "button",
+ "link",
+ "textbox",
+ "checkbox",
+ "radio",
+ "combobox",
+ "menuitem",
+ "tab",
+ "switch",
+ "slider",
+ "searchbox",
+ "spinbutton",
+]);
+
+// The AX `value` is stale for freshly-typed or controlled inputs, so for these roles the live DOM property is read instead.
+export const VALUE_ROLES = new Set([
+ "textbox",
+ "searchbox",
+ "spinbutton",
+ "combobox",
+ "checkbox",
+ "radio",
+ "switch",
+ "slider",
+]);
+
+const stringOf = (v: AxValue | undefined): string | undefined => {
+ const x = v?.value;
+ if (typeof x === "string" && x !== "") return x;
+ if (typeof x === "number") return String(x);
+ return undefined;
+};
+
+const collectState = (props: RawAxNode["properties"]): string | undefined => {
+ if (!props || props.length === 0) return undefined;
+ const flags: string[] = [];
+ for (const p of props) {
+ const name = p.name;
+ if (!name) continue;
+ const raw = p.value?.value;
+ if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) {
+ flags.push(name);
+ }
+ if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`);
+ }
+ return flags.length > 0 ? flags.join(", ") : undefined;
+};
+
+export const buildTree = (
+ rawNodes: ReadonlyArray<RawAxNode>,
+ opts: { interestingOnly: boolean; maxNodes: number },
+): SlimNode[] => {
+ const byId = new Map<string, RawAxNode>();
+ for (const n of rawNodes) byId.set(n.nodeId, n);
+
+ const rootIds: string[] = [];
+ for (const n of rawNodes) {
+ if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId);
+ }
+
+ let budget = opts.maxNodes;
+
+ const slim = (node: RawAxNode): SlimNode | undefined => {
+ if (budget <= 0) return undefined;
+ if (opts.interestingOnly && node.ignored) {
+ // Chrome often ignores wrapper divs while their descendants remain meaningful, so recurse through an ignored node's children.
+ const out: SlimNode[] = [];
+ for (const cid of node.childIds ?? []) {
+ const child = byId.get(cid);
+ if (!child) continue;
+ const slimChild = slim(child);
+ if (slimChild) out.push(slimChild);
+ }
+ return out.length > 0 ? { role: "_hoist", children: out } : undefined;
+ }
+ budget--;
+ const role = stringOf(node.role) ?? "unknown";
+ const name = stringOf(node.name);
+ const value = stringOf(node.value);
+ const description = stringOf(node.description);
+ const state = collectState(node.properties);
+ const children: SlimNode[] = [];
+ for (const cid of node.childIds ?? []) {
+ const child = byId.get(cid);
+ if (!child) continue;
+ const slimChild = slim(child);
+ if (!slimChild) continue;
+ if (slimChild.role === "_hoist") children.push(...slimChild.children);
+ else children.push(slimChild);
+ }
+ const out: SlimNode = { role, children };
+ if (name !== undefined) out.name = name;
+ if (value !== undefined) out.value = value;
+ if (description !== undefined) out.description = description;
+ if (state !== undefined) out.state = state;
+ if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId;
+ return out;
+ };
+
+ const result: SlimNode[] = [];
+ for (const rid of rootIds) {
+ const root = byId.get(rid);
+ if (!root) continue;
+ const s = slim(root);
+ if (!s) continue;
+ if (s.role === "_hoist") result.push(...s.children);
+ else result.push(s);
+ }
+ return result;
+};
+
+export const countNodes = (nodes: ReadonlyArray<SlimNode>): number => {
+ let n = 0;
+ for (const node of nodes) n += 1 + countNodes(node.children);
+ return n;
+};
+
+export const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] =>
+ nodes.map((n) => {
+ const { _backendId: _bid, ...rest } = n;
+ return { ...rest, children: stripInternals(n.children) };
+ });
+
+export const collectInteractiveTargets = (
+ nodes: ReadonlyArray<SlimNode>,
+): Array<{ node: SlimNode; backendId: number }> => {
+ const out: Array<{ node: SlimNode; backendId: number }> = [];
+ const walk = (ns: ReadonlyArray<SlimNode>): void => {
+ for (const n of ns) {
+ if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) {
+ out.push({ node: n, backendId: n._backendId });
+ }
+ walk(n.children);
+ }
+ };
+ walk(nodes);
+ return out;
+};
+
+export const liveValue = async (client: BrowserClient, backendId: number): Promise<string | undefined> => {
+ const resolved = await cdpCall(client, "DOM.resolveNode", { backendNodeId: backendId });
+ if (!resolved.success) return undefined;
+ const objectId = resolved.data.object.objectId;
+ if (!objectId) return undefined;
+ const fn = `function () {
+ if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked";
+ if (typeof this.value === "string") return this.value;
+ return null;
+ }`;
+ const r = await cdpCall(client, "Runtime.callFunctionOn", {
+ objectId,
+ functionDeclaration: fn,
+ returnByValue: true,
+ });
+ if (!r.success) return undefined;
+ const value = r.data.result.value;
+ return typeof value === "string" ? value : undefined;
+};
diff --git a/src/domains/click.ts b/src/domains/click.ts
index 41c3024..79bb5ee 100644
--- a/src/domains/click.ts
+++ b/src/domains/click.ts
@@ -6,7 +6,7 @@ import { type Result, ok } from "../util/result";
import { debugClicksEnabled } from "../util/debug";
import { defineBrowserTool, type ToolErr } from "../util/tool";
import { cdpCall } from "./cdp-call";
-import { captureWithCrosshair } from "./screenshot";
+import { captureWithCrosshair } from "./screenshot-capture";
import { interactiveDiff } from "./ref-resolve";
import { resolveTarget } from "./target";
diff --git a/src/domains/element-call.ts b/src/domains/element-call.ts
new file mode 100644
index 0000000..0292e6f
--- /dev/null
+++ b/src/domains/element-call.ts
@@ -0,0 +1,117 @@
+import type { BrowserClient } from "../client";
+import { safeJs } from "../util/js-template";
+import { type Result, err, ok } from "../util/result";
+import type { ToolErr } from "../util/tool";
+import { cdpCall, evalJs } from "./cdp-call";
+import { resolveRefToObjectId } from "./ref-resolve";
+
+export type PageResult = {
+ readonly ok: boolean;
+ readonly reason?: string;
+ readonly kind?: string;
+ readonly value?: unknown;
+ readonly text?: string;
+ readonly checked?: boolean;
+ readonly changed?: boolean;
+ readonly tag?: string;
+ readonly options?: ReadonlyArray<{ value: string; text: string }>;
+};
+
+const isOption = (v: unknown): v is { readonly value: string; readonly text: string } =>
+ typeof v === "object" && v !== null
+ && "value" in v && typeof v.value === "string"
+ && "text" in v && typeof v.text === "string";
+
+export const isPageResult = (v: unknown): v is PageResult => {
+ if (typeof v !== "object" || v === null) return false;
+ if (!("ok" in v) || typeof v.ok !== "boolean") return false;
+ if ("reason" in v && typeof v.reason !== "string") return false;
+ if ("kind" in v && typeof v.kind !== "string") return false;
+ if ("text" in v && typeof v.text !== "string") return false;
+ if ("checked" in v && typeof v.checked !== "boolean") return false;
+ if ("changed" in v && typeof v.changed !== "boolean") return false;
+ if ("tag" in v && typeof v.tag !== "string") return false;
+ if ("options" in v && !(Array.isArray(v.options) && v.options.every(isOption))) return false;
+ return true;
+};
+
+export const runOnElement = async (
+ client: BrowserClient,
+ opts: {
+ ref?: string | undefined;
+ selector?: string | undefined;
+ fnBody: string;
+ arg: unknown;
+ notFound?: unknown;
+ },
+): Promise<Result<unknown, ToolErr>> => {
+ if (opts.ref !== undefined) {
+ const objectId = await resolveRefToObjectId(client, opts.ref);
+ if (!objectId.success) return objectId;
+ const r = await cdpCall(client, "Runtime.callFunctionOn", {
+ objectId: objectId.data,
+ functionDeclaration: `function (arg) { ${opts.fnBody} }`,
+ arguments: [{ value: opts.arg }],
+ returnByValue: true,
+ });
+ if (!r.success) return r;
+ if (r.data.exceptionDetails !== undefined) {
+ return err({
+ kind: "cdp_error",
+ message: `page function threw: ${r.data.exceptionDetails.text}`,
+ details: { ref: opts.ref },
+ });
+ }
+ return ok(r.data.result.value);
+ }
+ if (opts.selector === undefined) {
+ return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
+ }
+ // fnBody is harness-authored, not user input; only the selector and value are interpolated, and those go through safeJs.
+ const prelude = safeJs`
+ (() => {
+ const el = document.querySelector(${opts.selector});
+ if (!el) return ${opts.notFound ?? { status: "not_found" }};
+ const __arg = ${opts.arg};
+ return (function (arg) {`;
+ const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`;
+ const r = await evalJs(client, expr);
+ if (!r.success) return r;
+ return ok(r.data);
+};
+
+export const resolveAndCall = async (
+ client: BrowserClient,
+ ref: string,
+ functionDeclaration: string,
+ args: ReadonlyArray<unknown>,
+): Promise<Result<PageResult, ToolErr>> => {
+ const objectId = await resolveRefToObjectId(client, ref);
+ if (!objectId.success) return objectId;
+ const handle = objectId.data;
+ try {
+ const called = await cdpCall(client, "Runtime.callFunctionOn", {
+ objectId: handle,
+ functionDeclaration,
+ arguments: args.map((v) => ({ value: v })),
+ returnByValue: true,
+ awaitPromise: true,
+ });
+ if (!called.success) return err({ ...called.error, details: { ref } });
+ if (called.data.exceptionDetails !== undefined) {
+ return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(called.data.exceptionDetails)}`, details: { ref } });
+ }
+ const value = called.data.result.value;
+ if (!isPageResult(value)) {
+ return err({ kind: "internal", message: "page function returned no result object", details: { ref } });
+ }
+ return ok(value);
+ } finally {
+ await cdpCall(client, "Runtime.releaseObject", { objectId: handle });
+ }
+};
+
+export const detailsOf = (ref: string, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => {
+ const { ok: _ok, ...rest } = res;
+ return { ref, ...rest, ...(extra ?? {}) };
+};
diff --git a/src/domains/form-batch.ts b/src/domains/form-batch.ts
index 1d49cf6..567108b 100644
--- a/src/domains/form-batch.ts
+++ b/src/domains/form-batch.ts
@@ -2,7 +2,7 @@ import { Type } from "typebox";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
import { fillDeclaration } from "./fill-engine";
-import { detailsOf, resolveAndCall } from "./form";
+import { detailsOf, resolveAndCall } from "./element-call";
const FILL_FN = fillDeclaration({ rejectSelect: false, focusFirst: true });
diff --git a/src/domains/form.ts b/src/domains/form.ts
index 8af43c3..577eb5d 100644
--- a/src/domains/form.ts
+++ b/src/domains/form.ts
@@ -1,11 +1,9 @@
import { Type } from "typebox";
-import type { BrowserClient } from "../client";
-import { safeJs } from "../util/js-template";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
-import { cdpCall, evalJs } from "./cdp-call";
+import { detailsOf, isPageResult, type PageResult, resolveAndCall, runOnElement } from "./element-call";
import { fillBody } from "./fill-engine";
-import { interactiveDiff, resolveRefToObjectId } from "./ref-resolve";
+import { interactiveDiff } from "./ref-resolve";
const FillArgs = Type.Object({
ref: Type.Optional(
@@ -18,51 +16,6 @@ const FillArgs = Type.Object({
value: Type.String({ description: "Value to set in the field" }),
});
-const runOnElement = async (
- client: BrowserClient,
- opts: {
- ref?: string | undefined;
- selector?: string | undefined;
- fnBody: string;
- arg: unknown;
- notFound?: unknown;
- },
-): Promise<Result<unknown, ToolErr>> => {
- if (opts.ref !== undefined) {
- const objectId = await resolveRefToObjectId(client, opts.ref);
- if (!objectId.success) return objectId;
- const r = await cdpCall(client, "Runtime.callFunctionOn", {
- objectId: objectId.data,
- functionDeclaration: `function (arg) { ${opts.fnBody} }`,
- arguments: [{ value: opts.arg }],
- returnByValue: true,
- });
- if (!r.success) return r;
- if (r.data.exceptionDetails !== undefined) {
- return err({
- kind: "cdp_error",
- message: `page function threw: ${r.data.exceptionDetails.text}`,
- details: { ref: opts.ref },
- });
- }
- return ok(r.data.result.value);
- }
- if (opts.selector === undefined) {
- return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
- }
- // fnBody is harness-authored, not user input; only the selector and value are interpolated, and those go through safeJs.
- const prelude = safeJs`
- (() => {
- const el = document.querySelector(${opts.selector});
- if (!el) return ${opts.notFound ?? { status: "not_found" }};
- const __arg = ${opts.arg};
- return (function (arg) {`;
- const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`;
- const r = await evalJs(client, expr);
- if (!r.success) return r;
- return ok(r.data);
-};
-
const FILL_BODY = `const value = arg;\n${fillBody({ rejectSelect: true, focusFirst: false })}`;
const NOT_FOUND: PageResult = { ok: false, reason: "not_found" };
@@ -281,72 +234,6 @@ export const selectOptionTool = defineBrowserTool({
},
});
-export type PageResult = {
- readonly ok: boolean;
- readonly reason?: string;
- readonly kind?: string;
- readonly value?: unknown;
- readonly text?: string;
- readonly checked?: boolean;
- readonly changed?: boolean;
- readonly tag?: string;
- readonly options?: ReadonlyArray<{ value: string; text: string }>;
-};
-
-const isOption = (v: unknown): v is { readonly value: string; readonly text: string } =>
- typeof v === "object" && v !== null
- && "value" in v && typeof v.value === "string"
- && "text" in v && typeof v.text === "string";
-
-const isPageResult = (v: unknown): v is PageResult => {
- if (typeof v !== "object" || v === null) return false;
- if (!("ok" in v) || typeof v.ok !== "boolean") return false;
- if ("reason" in v && typeof v.reason !== "string") return false;
- if ("kind" in v && typeof v.kind !== "string") return false;
- if ("text" in v && typeof v.text !== "string") return false;
- if ("checked" in v && typeof v.checked !== "boolean") return false;
- if ("changed" in v && typeof v.changed !== "boolean") return false;
- if ("tag" in v && typeof v.tag !== "string") return false;
- if ("options" in v && !(Array.isArray(v.options) && v.options.every(isOption))) return false;
- return true;
-};
-
-export const resolveAndCall = async (
- client: BrowserClient,
- ref: string,
- functionDeclaration: string,
- args: ReadonlyArray<unknown>,
-): Promise<Result<PageResult, ToolErr>> => {
- const objectId = await resolveRefToObjectId(client, ref);
- if (!objectId.success) return objectId;
- const handle = objectId.data;
- try {
- const called = await cdpCall(client, "Runtime.callFunctionOn", {
- objectId: handle,
- functionDeclaration,
- arguments: args.map((v) => ({ value: v })),
- returnByValue: true,
- awaitPromise: true,
- });
- if (!called.success) return err({ ...called.error, details: { ref } });
- if (called.data.exceptionDetails !== undefined) {
- return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(called.data.exceptionDetails)}`, details: { ref } });
- }
- const value = called.data.result.value;
- if (!isPageResult(value)) {
- return err({ kind: "internal", message: "page function returned no result object", details: { ref } });
- }
- return ok(value);
- } finally {
- await cdpCall(client, "Runtime.releaseObject", { objectId: handle });
- }
-};
-
-export const detailsOf = (ref: string, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => {
- const { ok: _ok, ...rest } = res;
- return { ref, ...rest, ...(extra ?? {}) };
-};
-
const CHECK_FN = `
function(checked) {
const el = this;
diff --git a/src/domains/navigate.ts b/src/domains/navigate.ts
index 8afea6f..17e25a6 100644
--- a/src/domains/navigate.ts
+++ b/src/domains/navigate.ts
@@ -12,7 +12,7 @@ const NavigateArgs = Type.Object({
url: Type.String({ description: "Full URL to navigate to (e.g. https://github.com)" }),
});
-export type NavOutcome =
+type NavOutcome =
| { readonly kind: "in_place"; readonly targetId: string }
| { readonly kind: "new_tab_created"; readonly targetId: string; readonly reason: "no_tabs" };
diff --git a/src/domains/ref-resolve.ts b/src/domains/ref-resolve.ts
index 0f509a0..9eb335e 100644
--- a/src/domains/ref-resolve.ts
+++ b/src/domains/ref-resolve.ts
@@ -3,7 +3,7 @@ import { type Result, err, ok } from "../util/result";
import type { ToolErr } from "../util/tool";
import { type Box, boxOf } from "./box";
import { cdpCall } from "./cdp-call";
-import { buildTree, collectInteractiveTargets, type SlimNode } from "./snapshot";
+import { buildTree, collectInteractiveTargets, type SlimNode } from "./ax-tree";
const staleErr = (ref: string): ToolErr => ({
kind: "invalid_state",
diff --git a/src/domains/screenshot-capture.ts b/src/domains/screenshot-capture.ts
new file mode 100644
index 0000000..14bac06
--- /dev/null
+++ b/src/domains/screenshot-capture.ts
@@ -0,0 +1,80 @@
+import { rename, writeFile } from "node:fs/promises";
+import type { BrowserClient } from "../client";
+import { type Result, ok } from "../util/result";
+import type { ToolErr } from "../util/tool";
+import { screenshotPath } from "../util/paths";
+import { loadSharp } from "../util/sharp-shim";
+import { safeJs } from "../util/js-template";
+import { cdpCall, evalJs } from "./cdp-call";
+
+export type CaptureArgs = {
+ readonly fullPage?: boolean;
+ readonly format?: "png" | "jpeg";
+ readonly quality?: number;
+};
+
+export const captureBase = async (
+ client: BrowserClient,
+ args: CaptureArgs,
+): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => {
+ const format = args.format ?? "png";
+ const quality = args.quality ?? 80;
+ const path = screenshotPath(client.namespace, format);
+ const r = await cdpCall(client, "Page.captureScreenshot", {
+ format,
+ captureBeyondViewport: args.fullPage ?? false,
+ ...(format === "jpeg" ? { quality } : {}),
+ });
+ if (!r.success) return r;
+ await writeFile(path, Buffer.from(r.data.data, "base64"));
+ return ok({ path, format });
+};
+
+export const resizeIfNeeded = async (
+ path: string,
+ maxDim: number,
+): Promise<{ readonly note: string }> => {
+ const load = await loadSharp();
+ if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" };
+ if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` };
+ try {
+ const meta = await load.sharp(path).metadata();
+ const w = meta.width ?? 0;
+ const h = meta.height ?? 0;
+ if (Math.max(w, h) <= maxDim) return { note: "" };
+ const tmp = `${path}.resized`;
+ await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp);
+ await rename(tmp, path);
+ return { note: ` (resized to fit ${maxDim}px)` };
+ } catch (e) {
+ return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` };
+ }
+};
+
+export const captureWithCrosshair = async (
+ client: BrowserClient,
+ args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number },
+): Promise<Result<{ readonly path: string }, ToolErr>> => {
+ const cap = await captureBase(client, args);
+ if (!cap.success) return cap;
+ const load = await loadSharp();
+ if (load.kind !== "ok") return ok({ path: cap.data.path });
+ try {
+ const dprR = await evalJs(client, safeJs`window.devicePixelRatio`);
+ const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1;
+ const meta = await load.sharp(cap.data.path).metadata();
+ const w = meta.width ?? 0;
+ const h = meta.height ?? 0;
+ const px = Math.round(args.x * dpr);
+ const py = Math.round(args.y * dpr);
+ const r = Math.round(15 * dpr);
+ const stroke = Math.max(2, Math.round(3 * dpr));
+ const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`;
+ const tmp = `${cap.data.path}.debug`;
+ await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp);
+ await rename(tmp, cap.data.path);
+ return ok({ path: cap.data.path });
+ } catch {
+ return ok({ path: cap.data.path });
+ }
+};
diff --git a/src/domains/screenshot.ts b/src/domains/screenshot.ts
index 4b5732d..9f2587c 100644
--- a/src/domains/screenshot.ts
+++ b/src/domains/screenshot.ts
@@ -1,15 +1,10 @@
import { readFileSync } from "node:fs";
-import { writeFile, rename } from "node:fs/promises";
import { Type } from "typebox";
import { Image, type ImageTheme, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
-import type { BrowserClient } from "../client";
import { type Result, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
-import { screenshotPath } from "../util/paths";
-import { loadSharp } from "../util/sharp-shim";
import { asRecord, asString } from "../util/guards";
-import { safeJs } from "../util/js-template";
-import { cdpCall, evalJs } from "./cdp-call";
+import { captureBase, resizeIfNeeded } from "./screenshot-capture";
const ScreenshotArgs = Type.Object({
fullPage: Type.Optional(Type.Boolean({ default: false, description: "Capture beyond viewport" })),
@@ -18,50 +13,6 @@ const ScreenshotArgs = Type.Object({
maxDim: Type.Optional(Type.Integer({ minimum: 100, maximum: 8000, description: "If max(w,h) exceeds this, resize via sharp." })),
});
-type CaptureArgs = {
- readonly fullPage?: boolean;
- readonly format?: "png" | "jpeg";
- readonly quality?: number;
-};
-
-const captureBase = async (
- client: BrowserClient,
- args: CaptureArgs,
-): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => {
- const format = args.format ?? "png";
- const quality = args.quality ?? 80;
- const path = screenshotPath(client.namespace, format);
- const r = await cdpCall(client, "Page.captureScreenshot", {
- format,
- captureBeyondViewport: args.fullPage ?? false,
- ...(format === "jpeg" ? { quality } : {}),
- });
- if (!r.success) return r;
- await writeFile(path, Buffer.from(r.data.data, "base64"));
- return ok({ path, format });
-};
-
-const resizeIfNeeded = async (
- path: string,
- maxDim: number,
-): Promise<{ readonly note: string }> => {
- const load = await loadSharp();
- if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" };
- if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` };
- try {
- const meta = await load.sharp(path).metadata();
- const w = meta.width ?? 0;
- const h = meta.height ?? 0;
- if (Math.max(w, h) <= maxDim) return { note: "" };
- const tmp = `${path}.resized`;
- await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp);
- await rename(tmp, path);
- return { note: ` (resized to fit ${maxDim}px)` };
- } catch (e) {
- return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` };
- }
-};
-
export const screenshotTool = defineBrowserTool({
name: "browser_screenshot",
label: "Browser Screenshot",
@@ -123,31 +74,3 @@ export const screenshotTool = defineBrowserTool({
}
},
});
-
-export const captureWithCrosshair = async (
- client: BrowserClient,
- args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number },
-): Promise<Result<{ readonly path: string }, ToolErr>> => {
- const cap = await captureBase(client, args);
- if (!cap.success) return cap;
- const load = await loadSharp();
- if (load.kind !== "ok") return ok({ path: cap.data.path });
- try {
- const dprR = await evalJs(client, safeJs`window.devicePixelRatio`);
- const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1;
- const meta = await load.sharp(cap.data.path).metadata();
- const w = meta.width ?? 0;
- const h = meta.height ?? 0;
- const px = Math.round(args.x * dpr);
- const py = Math.round(args.y * dpr);
- const r = Math.round(15 * dpr);
- const stroke = Math.max(2, Math.round(3 * dpr));
- const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`;
- const tmp = `${cap.data.path}.debug`;
- await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp);
- await rename(tmp, cap.data.path);
- return ok({ path: cap.data.path });
- } catch {
- return ok({ path: cap.data.path });
- }
-};
diff --git a/src/domains/snapshot.ts b/src/domains/snapshot.ts
index 848ebb1..86c5fe3 100644
--- a/src/domains/snapshot.ts
+++ b/src/domains/snapshot.ts
@@ -2,10 +2,18 @@ import { readFileSync, writeFileSync } from "node:fs";
import { Type } from "typebox";
import { Container, Image, type ImageTheme, Markdown, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
-import type { BrowserClient } from "../client";
-import type { AxNodeResult } from "../cdp/commands";
-import { type Box, boxOf } from "./box";
+import { boxOf } from "./box";
import { cdpCall } from "./cdp-call";
+import {
+ buildTree,
+ collectInteractiveTargets,
+ countNodes,
+ INTERACTIVE_ROLES,
+ liveValue,
+ type SlimNode,
+ stripInternals,
+ VALUE_ROLES,
+} from "./ax-tree";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
import { applyTruncation } from "../util/truncate";
@@ -30,139 +38,6 @@ const SnapshotArgs = Type.Object({
),
});
-export type RawAxNode = AxNodeResult;
-type AxValue = NonNullable<RawAxNode["role"]>;
-
-export type SlimNode = {
- role: string;
- name?: string;
- value?: string;
- description?: string;
- state?: string;
- children: SlimNode[];
- _backendId?: number;
- ref?: string;
- box?: Box;
-};
-
-const INTERACTIVE_ROLES = new Set([
- "button",
- "link",
- "textbox",
- "checkbox",
- "radio",
- "combobox",
- "menuitem",
- "tab",
- "switch",
- "slider",
- "searchbox",
- "spinbutton",
-]);
-
-// The AX `value` is stale for freshly-typed or controlled inputs, so for these roles the live DOM property is read instead.
-const VALUE_ROLES = new Set([
- "textbox",
- "searchbox",
- "spinbutton",
- "combobox",
- "checkbox",
- "radio",
- "switch",
- "slider",
-]);
-
-const stringOf = (v: AxValue | undefined): string | undefined => {
- const x = v?.value;
- if (typeof x === "string" && x !== "") return x;
- if (typeof x === "number") return String(x);
- return undefined;
-};
-
-const collectState = (props: RawAxNode["properties"]): string | undefined => {
- if (!props || props.length === 0) return undefined;
- const flags: string[] = [];
- for (const p of props) {
- const name = p.name;
- if (!name) continue;
- const raw = p.value?.value;
- if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) {
- flags.push(name);
- }
- if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`);
- }
- return flags.length > 0 ? flags.join(", ") : undefined;
-};
-
-export const buildTree = (
- rawNodes: ReadonlyArray<RawAxNode>,
- opts: { interestingOnly: boolean; maxNodes: number },
-): SlimNode[] => {
- const byId = new Map<string, RawAxNode>();
- for (const n of rawNodes) byId.set(n.nodeId, n);
-
- const rootIds: string[] = [];
- for (const n of rawNodes) {
- if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId);
- }
-
- let budget = opts.maxNodes;
-
- const slim = (node: RawAxNode): SlimNode | undefined => {
- if (budget <= 0) return undefined;
- if (opts.interestingOnly && node.ignored) {
- // Chrome often ignores wrapper divs while their descendants remain meaningful, so recurse through an ignored node's children.
- const out: SlimNode[] = [];
- for (const cid of node.childIds ?? []) {
- const child = byId.get(cid);
- if (!child) continue;
- const slimChild = slim(child);
- if (slimChild) out.push(slimChild);
- }
- return out.length > 0 ? { role: "_hoist", children: out } : undefined;
- }
- budget--;
- const role = stringOf(node.role) ?? "unknown";
- const name = stringOf(node.name);
- const value = stringOf(node.value);
- const description = stringOf(node.description);
- const state = collectState(node.properties);
- const children: SlimNode[] = [];
- for (const cid of node.childIds ?? []) {
- const child = byId.get(cid);
- if (!child) continue;
- const slimChild = slim(child);
- if (!slimChild) continue;
- if (slimChild.role === "_hoist") children.push(...slimChild.children);
- else children.push(slimChild);
- }
- const out: SlimNode = { role, children };
- if (name !== undefined) out.name = name;
- if (value !== undefined) out.value = value;
- if (description !== undefined) out.description = description;
- if (state !== undefined) out.state = state;
- if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId;
- return out;
- };
-
- const result: SlimNode[] = [];
- for (const rid of rootIds) {
- const root = byId.get(rid);
- if (!root) continue;
- const s = slim(root);
- if (!s) continue;
- if (s.role === "_hoist") result.push(...s.children);
- else result.push(s);
- }
- return result;
-};
-
-const countNodes = (nodes: ReadonlyArray<SlimNode>): number => {
- let n = 0;
- for (const node of nodes) n += 1 + countNodes(node.children);
- return n;
-};
-
const summarize = (nodes: ReadonlyArray<SlimNode>): string => {
const counts = new Map<string, number>();
const walk = (ns: ReadonlyArray<SlimNode>): void => {
@@ -207,48 +82,6 @@ const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => {
return lines.join("\n");
};
-const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] =>
- nodes.map((n) => {
- const { _backendId: _bid, ...rest } = n;
- return { ...rest, children: stripInternals(n.children) };
- });
-
-export const collectInteractiveTargets = (
- nodes: ReadonlyArray<SlimNode>,
-): Array<{ node: SlimNode; backendId: number }> => {
- const out: Array<{ node: SlimNode; backendId: number }> = [];
- const walk = (ns: ReadonlyArray<SlimNode>): void => {
- for (const n of ns) {
- if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) {
- out.push({ node: n, backendId: n._backendId });
- }
- walk(n.children);
- }
- };
- walk(nodes);
- return out;
-};
-
-const liveValue = async (client: BrowserClient, backendId: number): Promise<string | undefined> => {
- const resolved = await cdpCall(client, "DOM.resolveNode", { backendNodeId: backendId });
- if (!resolved.success) return undefined;
- const objectId = resolved.data.object.objectId;
- if (!objectId) return undefined;
- const fn = `function () {
- if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked";
- if (typeof this.value === "string") return this.value;
- return null;
- }`;
- const r = await cdpCall(client, "Runtime.callFunctionOn", {
- objectId,
- functionDeclaration: fn,
- returnByValue: true,
- });
- if (!r.success) return undefined;
- const value = r.data.result.value;
- return typeof value === "string" ? value : undefined;
-};
-
type SnapshotDetails = {
nodeCount: number;
truncated: boolean;