diff options
| author | amankumarsingh77 <amankumarsingh7702@gmail.com> | 2026-07-28 22:00:33 +0530 |
|---|---|---|
| committer | Aman Kumar <145339995+amankumarsingh77@users.noreply.github.com> | 2026-08-02 19:12:46 +0530 |
| commit | 86045450c208bc51670dd9554e581f2a62b181c9 (patch) | |
| tree | 6e1ee7617afd829549ade41e527f07155921a47e /src | |
| parent | 632600ec0a0a68d9327e19bd419a1ae72047955f (diff) | |
refactor: helpers live in helper modules, not in tool files
form.ts exported resolveAndCall and detailsOf to form-batch.ts,
snapshot.ts exported buildTree and collectInteractiveTargets to
ref-resolve.ts, and screenshot.ts exported captureWithCrosshair to
click.ts — so importing one tool pulled in another tool's whole module.
The shared halves move out: ax-tree.ts (the AX tree model, its
construction and the queries over it), element-call.ts (calling a
function on a resolved element, and the PageResult it returns),
screenshot-capture.ts (capture, resize, crosshair overlay). Each tool
file now holds its tools and its own presentation, nothing else.
NavOutcome stops being exported; nothing outside navigate.ts read it.
target.ts stays a module rather than folding into click.ts as planned:
it is already helper-only and has its own test suite, and moving a tested
pure function into a tool file is the thing this commit removes. Having
one consumer is not the defect.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Diffstat (limited to 'src')
| -rw-r--r-- | src/domains/ax-tree.ts | 179 | ||||
| -rw-r--r-- | src/domains/click.ts | 2 | ||||
| -rw-r--r-- | src/domains/element-call.ts | 117 | ||||
| -rw-r--r-- | src/domains/form-batch.ts | 2 | ||||
| -rw-r--r-- | src/domains/form.ts | 117 | ||||
| -rw-r--r-- | src/domains/navigate.ts | 2 | ||||
| -rw-r--r-- | src/domains/ref-resolve.ts | 2 | ||||
| -rw-r--r-- | src/domains/screenshot-capture.ts | 80 | ||||
| -rw-r--r-- | src/domains/screenshot.ts | 79 | ||||
| -rw-r--r-- | src/domains/snapshot.ts | 189 |
10 files changed, 394 insertions, 375 deletions
diff --git a/src/domains/ax-tree.ts b/src/domains/ax-tree.ts new file mode 100644 index 0000000..1329063 --- /dev/null +++ b/src/domains/ax-tree.ts @@ -0,0 +1,179 @@ +import type { AxNodeResult } from "../cdp/commands"; +import type { BrowserClient } from "../client"; +import type { Box } from "./box"; +import { cdpCall } from "./cdp-call"; + +export type RawAxNode = AxNodeResult; +type AxValue = NonNullable<RawAxNode["role"]>; + +export type SlimNode = { + role: string; + name?: string; + value?: string; + description?: string; + state?: string; + children: SlimNode[]; + _backendId?: number; + ref?: string; + box?: Box; +}; + +export const INTERACTIVE_ROLES = new Set([ + "button", + "link", + "textbox", + "checkbox", + "radio", + "combobox", + "menuitem", + "tab", + "switch", + "slider", + "searchbox", + "spinbutton", +]); + +// The AX `value` is stale for freshly-typed or controlled inputs, so for these roles the live DOM property is read instead. +export const VALUE_ROLES = new Set([ + "textbox", + "searchbox", + "spinbutton", + "combobox", + "checkbox", + "radio", + "switch", + "slider", +]); + +const stringOf = (v: AxValue | undefined): string | undefined => { + const x = v?.value; + if (typeof x === "string" && x !== "") return x; + if (typeof x === "number") return String(x); + return undefined; +}; + +const collectState = (props: RawAxNode["properties"]): string | undefined => { + if (!props || props.length === 0) return undefined; + const flags: string[] = []; + for (const p of props) { + const name = p.name; + if (!name) continue; + const raw = p.value?.value; + if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) { + flags.push(name); + } + if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`); + } + return flags.length > 0 ? flags.join(", ") : undefined; +}; + +export const buildTree = ( + rawNodes: ReadonlyArray<RawAxNode>, + opts: { interestingOnly: boolean; maxNodes: number }, +): SlimNode[] => { + const byId = new Map<string, RawAxNode>(); + for (const n of rawNodes) byId.set(n.nodeId, n); + + const rootIds: string[] = []; + for (const n of rawNodes) { + if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId); + } + + let budget = opts.maxNodes; + + const slim = (node: RawAxNode): SlimNode | undefined => { + if (budget <= 0) return undefined; + if (opts.interestingOnly && node.ignored) { + // Chrome often ignores wrapper divs while their descendants remain meaningful, so recurse through an ignored node's children. + const out: SlimNode[] = []; + for (const cid of node.childIds ?? []) { + const child = byId.get(cid); + if (!child) continue; + const slimChild = slim(child); + if (slimChild) out.push(slimChild); + } + return out.length > 0 ? { role: "_hoist", children: out } : undefined; + } + budget--; + const role = stringOf(node.role) ?? "unknown"; + const name = stringOf(node.name); + const value = stringOf(node.value); + const description = stringOf(node.description); + const state = collectState(node.properties); + const children: SlimNode[] = []; + for (const cid of node.childIds ?? []) { + const child = byId.get(cid); + if (!child) continue; + const slimChild = slim(child); + if (!slimChild) continue; + if (slimChild.role === "_hoist") children.push(...slimChild.children); + else children.push(slimChild); + } + const out: SlimNode = { role, children }; + if (name !== undefined) out.name = name; + if (value !== undefined) out.value = value; + if (description !== undefined) out.description = description; + if (state !== undefined) out.state = state; + if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId; + return out; + }; + + const result: SlimNode[] = []; + for (const rid of rootIds) { + const root = byId.get(rid); + if (!root) continue; + const s = slim(root); + if (!s) continue; + if (s.role === "_hoist") result.push(...s.children); + else result.push(s); + } + return result; +}; + +export const countNodes = (nodes: ReadonlyArray<SlimNode>): number => { + let n = 0; + for (const node of nodes) n += 1 + countNodes(node.children); + return n; +}; + +export const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] => + nodes.map((n) => { + const { _backendId: _bid, ...rest } = n; + return { ...rest, children: stripInternals(n.children) }; + }); + +export const collectInteractiveTargets = ( + nodes: ReadonlyArray<SlimNode>, +): Array<{ node: SlimNode; backendId: number }> => { + const out: Array<{ node: SlimNode; backendId: number }> = []; + const walk = (ns: ReadonlyArray<SlimNode>): void => { + for (const n of ns) { + if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) { + out.push({ node: n, backendId: n._backendId }); + } + walk(n.children); + } + }; + walk(nodes); + return out; +}; + +export const liveValue = async (client: BrowserClient, backendId: number): Promise<string | undefined> => { + const resolved = await cdpCall(client, "DOM.resolveNode", { backendNodeId: backendId }); + if (!resolved.success) return undefined; + const objectId = resolved.data.object.objectId; + if (!objectId) return undefined; + const fn = `function () { + if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked"; + if (typeof this.value === "string") return this.value; + return null; + }`; + const r = await cdpCall(client, "Runtime.callFunctionOn", { + objectId, + functionDeclaration: fn, + returnByValue: true, + }); + if (!r.success) return undefined; + const value = r.data.result.value; + return typeof value === "string" ? value : undefined; +}; diff --git a/src/domains/click.ts b/src/domains/click.ts index 41c3024..79bb5ee 100644 --- a/src/domains/click.ts +++ b/src/domains/click.ts @@ -6,7 +6,7 @@ import { type Result, ok } from "../util/result"; import { debugClicksEnabled } from "../util/debug"; import { defineBrowserTool, type ToolErr } from "../util/tool"; import { cdpCall } from "./cdp-call"; -import { captureWithCrosshair } from "./screenshot"; +import { captureWithCrosshair } from "./screenshot-capture"; import { interactiveDiff } from "./ref-resolve"; import { resolveTarget } from "./target"; diff --git a/src/domains/element-call.ts b/src/domains/element-call.ts new file mode 100644 index 0000000..0292e6f --- /dev/null +++ b/src/domains/element-call.ts @@ -0,0 +1,117 @@ +import type { BrowserClient } from "../client"; +import { safeJs } from "../util/js-template"; +import { type Result, err, ok } from "../util/result"; +import type { ToolErr } from "../util/tool"; +import { cdpCall, evalJs } from "./cdp-call"; +import { resolveRefToObjectId } from "./ref-resolve"; + +export type PageResult = { + readonly ok: boolean; + readonly reason?: string; + readonly kind?: string; + readonly value?: unknown; + readonly text?: string; + readonly checked?: boolean; + readonly changed?: boolean; + readonly tag?: string; + readonly options?: ReadonlyArray<{ value: string; text: string }>; +}; + +const isOption = (v: unknown): v is { readonly value: string; readonly text: string } => + typeof v === "object" && v !== null + && "value" in v && typeof v.value === "string" + && "text" in v && typeof v.text === "string"; + +export const isPageResult = (v: unknown): v is PageResult => { + if (typeof v !== "object" || v === null) return false; + if (!("ok" in v) || typeof v.ok !== "boolean") return false; + if ("reason" in v && typeof v.reason !== "string") return false; + if ("kind" in v && typeof v.kind !== "string") return false; + if ("text" in v && typeof v.text !== "string") return false; + if ("checked" in v && typeof v.checked !== "boolean") return false; + if ("changed" in v && typeof v.changed !== "boolean") return false; + if ("tag" in v && typeof v.tag !== "string") return false; + if ("options" in v && !(Array.isArray(v.options) && v.options.every(isOption))) return false; + return true; +}; + +export const runOnElement = async ( + client: BrowserClient, + opts: { + ref?: string | undefined; + selector?: string | undefined; + fnBody: string; + arg: unknown; + notFound?: unknown; + }, +): Promise<Result<unknown, ToolErr>> => { + if (opts.ref !== undefined) { + const objectId = await resolveRefToObjectId(client, opts.ref); + if (!objectId.success) return objectId; + const r = await cdpCall(client, "Runtime.callFunctionOn", { + objectId: objectId.data, + functionDeclaration: `function (arg) { ${opts.fnBody} }`, + arguments: [{ value: opts.arg }], + returnByValue: true, + }); + if (!r.success) return r; + if (r.data.exceptionDetails !== undefined) { + return err({ + kind: "cdp_error", + message: `page function threw: ${r.data.exceptionDetails.text}`, + details: { ref: opts.ref }, + }); + } + return ok(r.data.result.value); + } + if (opts.selector === undefined) { + return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); + } + // fnBody is harness-authored, not user input; only the selector and value are interpolated, and those go through safeJs. + const prelude = safeJs` + (() => { + const el = document.querySelector(${opts.selector}); + if (!el) return ${opts.notFound ?? { status: "not_found" }}; + const __arg = ${opts.arg}; + return (function (arg) {`; + const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`; + const r = await evalJs(client, expr); + if (!r.success) return r; + return ok(r.data); +}; + +export const resolveAndCall = async ( + client: BrowserClient, + ref: string, + functionDeclaration: string, + args: ReadonlyArray<unknown>, +): Promise<Result<PageResult, ToolErr>> => { + const objectId = await resolveRefToObjectId(client, ref); + if (!objectId.success) return objectId; + const handle = objectId.data; + try { + const called = await cdpCall(client, "Runtime.callFunctionOn", { + objectId: handle, + functionDeclaration, + arguments: args.map((v) => ({ value: v })), + returnByValue: true, + awaitPromise: true, + }); + if (!called.success) return err({ ...called.error, details: { ref } }); + if (called.data.exceptionDetails !== undefined) { + return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(called.data.exceptionDetails)}`, details: { ref } }); + } + const value = called.data.result.value; + if (!isPageResult(value)) { + return err({ kind: "internal", message: "page function returned no result object", details: { ref } }); + } + return ok(value); + } finally { + await cdpCall(client, "Runtime.releaseObject", { objectId: handle }); + } +}; + +export const detailsOf = (ref: string, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => { + const { ok: _ok, ...rest } = res; + return { ref, ...rest, ...(extra ?? {}) }; +}; diff --git a/src/domains/form-batch.ts b/src/domains/form-batch.ts index 1d49cf6..567108b 100644 --- a/src/domains/form-batch.ts +++ b/src/domains/form-batch.ts @@ -2,7 +2,7 @@ import { Type } from "typebox"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { fillDeclaration } from "./fill-engine"; -import { detailsOf, resolveAndCall } from "./form"; +import { detailsOf, resolveAndCall } from "./element-call"; const FILL_FN = fillDeclaration({ rejectSelect: false, focusFirst: true }); diff --git a/src/domains/form.ts b/src/domains/form.ts index 8af43c3..577eb5d 100644 --- a/src/domains/form.ts +++ b/src/domains/form.ts @@ -1,11 +1,9 @@ import { Type } from "typebox"; -import type { BrowserClient } from "../client"; -import { safeJs } from "../util/js-template"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; -import { cdpCall, evalJs } from "./cdp-call"; +import { detailsOf, isPageResult, type PageResult, resolveAndCall, runOnElement } from "./element-call"; import { fillBody } from "./fill-engine"; -import { interactiveDiff, resolveRefToObjectId } from "./ref-resolve"; +import { interactiveDiff } from "./ref-resolve"; const FillArgs = Type.Object({ ref: Type.Optional( @@ -18,51 +16,6 @@ const FillArgs = Type.Object({ value: Type.String({ description: "Value to set in the field" }), }); -const runOnElement = async ( - client: BrowserClient, - opts: { - ref?: string | undefined; - selector?: string | undefined; - fnBody: string; - arg: unknown; - notFound?: unknown; - }, -): Promise<Result<unknown, ToolErr>> => { - if (opts.ref !== undefined) { - const objectId = await resolveRefToObjectId(client, opts.ref); - if (!objectId.success) return objectId; - const r = await cdpCall(client, "Runtime.callFunctionOn", { - objectId: objectId.data, - functionDeclaration: `function (arg) { ${opts.fnBody} }`, - arguments: [{ value: opts.arg }], - returnByValue: true, - }); - if (!r.success) return r; - if (r.data.exceptionDetails !== undefined) { - return err({ - kind: "cdp_error", - message: `page function threw: ${r.data.exceptionDetails.text}`, - details: { ref: opts.ref }, - }); - } - return ok(r.data.result.value); - } - if (opts.selector === undefined) { - return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." }); - } - // fnBody is harness-authored, not user input; only the selector and value are interpolated, and those go through safeJs. - const prelude = safeJs` - (() => { - const el = document.querySelector(${opts.selector}); - if (!el) return ${opts.notFound ?? { status: "not_found" }}; - const __arg = ${opts.arg}; - return (function (arg) {`; - const expr = `${prelude} ${opts.fnBody} }).call(el, __arg); })()`; - const r = await evalJs(client, expr); - if (!r.success) return r; - return ok(r.data); -}; - const FILL_BODY = `const value = arg;\n${fillBody({ rejectSelect: true, focusFirst: false })}`; const NOT_FOUND: PageResult = { ok: false, reason: "not_found" }; @@ -281,72 +234,6 @@ export const selectOptionTool = defineBrowserTool({ }, }); -export type PageResult = { - readonly ok: boolean; - readonly reason?: string; - readonly kind?: string; - readonly value?: unknown; - readonly text?: string; - readonly checked?: boolean; - readonly changed?: boolean; - readonly tag?: string; - readonly options?: ReadonlyArray<{ value: string; text: string }>; -}; - -const isOption = (v: unknown): v is { readonly value: string; readonly text: string } => - typeof v === "object" && v !== null - && "value" in v && typeof v.value === "string" - && "text" in v && typeof v.text === "string"; - -const isPageResult = (v: unknown): v is PageResult => { - if (typeof v !== "object" || v === null) return false; - if (!("ok" in v) || typeof v.ok !== "boolean") return false; - if ("reason" in v && typeof v.reason !== "string") return false; - if ("kind" in v && typeof v.kind !== "string") return false; - if ("text" in v && typeof v.text !== "string") return false; - if ("checked" in v && typeof v.checked !== "boolean") return false; - if ("changed" in v && typeof v.changed !== "boolean") return false; - if ("tag" in v && typeof v.tag !== "string") return false; - if ("options" in v && !(Array.isArray(v.options) && v.options.every(isOption))) return false; - return true; -}; - -export const resolveAndCall = async ( - client: BrowserClient, - ref: string, - functionDeclaration: string, - args: ReadonlyArray<unknown>, -): Promise<Result<PageResult, ToolErr>> => { - const objectId = await resolveRefToObjectId(client, ref); - if (!objectId.success) return objectId; - const handle = objectId.data; - try { - const called = await cdpCall(client, "Runtime.callFunctionOn", { - objectId: handle, - functionDeclaration, - arguments: args.map((v) => ({ value: v })), - returnByValue: true, - awaitPromise: true, - }); - if (!called.success) return err({ ...called.error, details: { ref } }); - if (called.data.exceptionDetails !== undefined) { - return err({ kind: "cdp_error", message: `page function threw: ${JSON.stringify(called.data.exceptionDetails)}`, details: { ref } }); - } - const value = called.data.result.value; - if (!isPageResult(value)) { - return err({ kind: "internal", message: "page function returned no result object", details: { ref } }); - } - return ok(value); - } finally { - await cdpCall(client, "Runtime.releaseObject", { objectId: handle }); - } -}; - -export const detailsOf = (ref: string, res: PageResult, extra?: Readonly<Record<string, unknown>>): Readonly<Record<string, unknown>> => { - const { ok: _ok, ...rest } = res; - return { ref, ...rest, ...(extra ?? {}) }; -}; - const CHECK_FN = ` function(checked) { const el = this; diff --git a/src/domains/navigate.ts b/src/domains/navigate.ts index 8afea6f..17e25a6 100644 --- a/src/domains/navigate.ts +++ b/src/domains/navigate.ts @@ -12,7 +12,7 @@ const NavigateArgs = Type.Object({ url: Type.String({ description: "Full URL to navigate to (e.g. https://github.com)" }), }); -export type NavOutcome = +type NavOutcome = | { readonly kind: "in_place"; readonly targetId: string } | { readonly kind: "new_tab_created"; readonly targetId: string; readonly reason: "no_tabs" }; diff --git a/src/domains/ref-resolve.ts b/src/domains/ref-resolve.ts index 0f509a0..9eb335e 100644 --- a/src/domains/ref-resolve.ts +++ b/src/domains/ref-resolve.ts @@ -3,7 +3,7 @@ import { type Result, err, ok } from "../util/result"; import type { ToolErr } from "../util/tool"; import { type Box, boxOf } from "./box"; import { cdpCall } from "./cdp-call"; -import { buildTree, collectInteractiveTargets, type SlimNode } from "./snapshot"; +import { buildTree, collectInteractiveTargets, type SlimNode } from "./ax-tree"; const staleErr = (ref: string): ToolErr => ({ kind: "invalid_state", diff --git a/src/domains/screenshot-capture.ts b/src/domains/screenshot-capture.ts new file mode 100644 index 0000000..14bac06 --- /dev/null +++ b/src/domains/screenshot-capture.ts @@ -0,0 +1,80 @@ +import { rename, writeFile } from "node:fs/promises"; +import type { BrowserClient } from "../client"; +import { type Result, ok } from "../util/result"; +import type { ToolErr } from "../util/tool"; +import { screenshotPath } from "../util/paths"; +import { loadSharp } from "../util/sharp-shim"; +import { safeJs } from "../util/js-template"; +import { cdpCall, evalJs } from "./cdp-call"; + +export type CaptureArgs = { + readonly fullPage?: boolean; + readonly format?: "png" | "jpeg"; + readonly quality?: number; +}; + +export const captureBase = async ( + client: BrowserClient, + args: CaptureArgs, +): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => { + const format = args.format ?? "png"; + const quality = args.quality ?? 80; + const path = screenshotPath(client.namespace, format); + const r = await cdpCall(client, "Page.captureScreenshot", { + format, + captureBeyondViewport: args.fullPage ?? false, + ...(format === "jpeg" ? { quality } : {}), + }); + if (!r.success) return r; + await writeFile(path, Buffer.from(r.data.data, "base64")); + return ok({ path, format }); +}; + +export const resizeIfNeeded = async ( + path: string, + maxDim: number, +): Promise<{ readonly note: string }> => { + const load = await loadSharp(); + if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" }; + if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` }; + try { + const meta = await load.sharp(path).metadata(); + const w = meta.width ?? 0; + const h = meta.height ?? 0; + if (Math.max(w, h) <= maxDim) return { note: "" }; + const tmp = `${path}.resized`; + await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp); + await rename(tmp, path); + return { note: ` (resized to fit ${maxDim}px)` }; + } catch (e) { + return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` }; + } +}; + +export const captureWithCrosshair = async ( + client: BrowserClient, + args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number }, +): Promise<Result<{ readonly path: string }, ToolErr>> => { + const cap = await captureBase(client, args); + if (!cap.success) return cap; + const load = await loadSharp(); + if (load.kind !== "ok") return ok({ path: cap.data.path }); + try { + const dprR = await evalJs(client, safeJs`window.devicePixelRatio`); + const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1; + const meta = await load.sharp(cap.data.path).metadata(); + const w = meta.width ?? 0; + const h = meta.height ?? 0; + const px = Math.round(args.x * dpr); + const py = Math.round(args.y * dpr); + const r = Math.round(15 * dpr); + const stroke = Math.max(2, Math.round(3 * dpr)); + const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`; + const tmp = `${cap.data.path}.debug`; + await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp); + await rename(tmp, cap.data.path); + return ok({ path: cap.data.path }); + } catch { + return ok({ path: cap.data.path }); + } +}; diff --git a/src/domains/screenshot.ts b/src/domains/screenshot.ts index 4b5732d..9f2587c 100644 --- a/src/domains/screenshot.ts +++ b/src/domains/screenshot.ts @@ -1,15 +1,10 @@ import { readFileSync } from "node:fs"; -import { writeFile, rename } from "node:fs/promises"; import { Type } from "typebox"; import { Image, type ImageTheme, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui"; -import type { BrowserClient } from "../client"; import { type Result, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; -import { screenshotPath } from "../util/paths"; -import { loadSharp } from "../util/sharp-shim"; import { asRecord, asString } from "../util/guards"; -import { safeJs } from "../util/js-template"; -import { cdpCall, evalJs } from "./cdp-call"; +import { captureBase, resizeIfNeeded } from "./screenshot-capture"; const ScreenshotArgs = Type.Object({ fullPage: Type.Optional(Type.Boolean({ default: false, description: "Capture beyond viewport" })), @@ -18,50 +13,6 @@ const ScreenshotArgs = Type.Object({ maxDim: Type.Optional(Type.Integer({ minimum: 100, maximum: 8000, description: "If max(w,h) exceeds this, resize via sharp." })), }); -type CaptureArgs = { - readonly fullPage?: boolean; - readonly format?: "png" | "jpeg"; - readonly quality?: number; -}; - -const captureBase = async ( - client: BrowserClient, - args: CaptureArgs, -): Promise<Result<{ readonly path: string; readonly format: "png" | "jpeg" }, ToolErr>> => { - const format = args.format ?? "png"; - const quality = args.quality ?? 80; - const path = screenshotPath(client.namespace, format); - const r = await cdpCall(client, "Page.captureScreenshot", { - format, - captureBeyondViewport: args.fullPage ?? false, - ...(format === "jpeg" ? { quality } : {}), - }); - if (!r.success) return r; - await writeFile(path, Buffer.from(r.data.data, "base64")); - return ok({ path, format }); -}; - -const resizeIfNeeded = async ( - path: string, - maxDim: number, -): Promise<{ readonly note: string }> => { - const load = await loadSharp(); - if (load.kind === "missing") return { note: " (maxDim ignored: install sharp for auto-resize)" }; - if (load.kind === "error") return { note: ` (maxDim ignored: sharp failed to load: ${load.message})` }; - try { - const meta = await load.sharp(path).metadata(); - const w = meta.width ?? 0; - const h = meta.height ?? 0; - if (Math.max(w, h) <= maxDim) return { note: "" }; - const tmp = `${path}.resized`; - await load.sharp(path).resize(maxDim, maxDim, { fit: "inside" }).toFile(tmp); - await rename(tmp, path); - return { note: ` (resized to fit ${maxDim}px)` }; - } catch (e) { - return { note: ` (maxDim ignored: sharp threw: ${e instanceof Error ? e.message : String(e)})` }; - } -}; - export const screenshotTool = defineBrowserTool({ name: "browser_screenshot", label: "Browser Screenshot", @@ -123,31 +74,3 @@ export const screenshotTool = defineBrowserTool({ } }, }); - -export const captureWithCrosshair = async ( - client: BrowserClient, - args: { readonly x: number; readonly y: number; readonly format?: "png" | "jpeg"; readonly quality?: number }, -): Promise<Result<{ readonly path: string }, ToolErr>> => { - const cap = await captureBase(client, args); - if (!cap.success) return cap; - const load = await loadSharp(); - if (load.kind !== "ok") return ok({ path: cap.data.path }); - try { - const dprR = await evalJs(client, safeJs`window.devicePixelRatio`); - const dpr = dprR.success && typeof dprR.data === "number" ? dprR.data : 1; - const meta = await load.sharp(cap.data.path).metadata(); - const w = meta.width ?? 0; - const h = meta.height ?? 0; - const px = Math.round(args.x * dpr); - const py = Math.round(args.y * dpr); - const r = Math.round(15 * dpr); - const stroke = Math.max(2, Math.round(3 * dpr)); - const svg = `<svg width="${w}" height="${h}"><circle cx="${px}" cy="${py}" r="${r}" fill="none" stroke="red" stroke-width="${stroke}" opacity="0.8"/><line x1="${px - r - 5}" y1="${py}" x2="${px + r + 5}" y2="${py}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/><line x1="${px}" y1="${py - r - 5}" x2="${px}" y2="${py + r + 5}" stroke="red" stroke-width="${Math.max(1, stroke - 1)}" opacity="0.8"/></svg>`; - const tmp = `${cap.data.path}.debug`; - await load.sharp(cap.data.path).composite([{ input: Buffer.from(svg), top: 0, left: 0 }]).toFile(tmp); - await rename(tmp, cap.data.path); - return ok({ path: cap.data.path }); - } catch { - return ok({ path: cap.data.path }); - } -}; diff --git a/src/domains/snapshot.ts b/src/domains/snapshot.ts index 848ebb1..86c5fe3 100644 --- a/src/domains/snapshot.ts +++ b/src/domains/snapshot.ts @@ -2,10 +2,18 @@ import { readFileSync, writeFileSync } from "node:fs"; import { Type } from "typebox"; import { Container, Image, type ImageTheme, Markdown, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui"; import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent"; -import type { BrowserClient } from "../client"; -import type { AxNodeResult } from "../cdp/commands"; -import { type Box, boxOf } from "./box"; +import { boxOf } from "./box"; import { cdpCall } from "./cdp-call"; +import { + buildTree, + collectInteractiveTargets, + countNodes, + INTERACTIVE_ROLES, + liveValue, + type SlimNode, + stripInternals, + VALUE_ROLES, +} from "./ax-tree"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { applyTruncation } from "../util/truncate"; @@ -30,139 +38,6 @@ const SnapshotArgs = Type.Object({ ), }); -export type RawAxNode = AxNodeResult; -type AxValue = NonNullable<RawAxNode["role"]>; - -export type SlimNode = { - role: string; - name?: string; - value?: string; - description?: string; - state?: string; - children: SlimNode[]; - _backendId?: number; - ref?: string; - box?: Box; -}; - -const INTERACTIVE_ROLES = new Set([ - "button", - "link", - "textbox", - "checkbox", - "radio", - "combobox", - "menuitem", - "tab", - "switch", - "slider", - "searchbox", - "spinbutton", -]); - -// The AX `value` is stale for freshly-typed or controlled inputs, so for these roles the live DOM property is read instead. -const VALUE_ROLES = new Set([ - "textbox", - "searchbox", - "spinbutton", - "combobox", - "checkbox", - "radio", - "switch", - "slider", -]); - -const stringOf = (v: AxValue | undefined): string | undefined => { - const x = v?.value; - if (typeof x === "string" && x !== "") return x; - if (typeof x === "number") return String(x); - return undefined; -}; - -const collectState = (props: RawAxNode["properties"]): string | undefined => { - if (!props || props.length === 0) return undefined; - const flags: string[] = []; - for (const p of props) { - const name = p.name; - if (!name) continue; - const raw = p.value?.value; - if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) { - flags.push(name); - } - if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`); - } - return flags.length > 0 ? flags.join(", ") : undefined; -}; - -export const buildTree = ( - rawNodes: ReadonlyArray<RawAxNode>, - opts: { interestingOnly: boolean; maxNodes: number }, -): SlimNode[] => { - const byId = new Map<string, RawAxNode>(); - for (const n of rawNodes) byId.set(n.nodeId, n); - - const rootIds: string[] = []; - for (const n of rawNodes) { - if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId); - } - - let budget = opts.maxNodes; - - const slim = (node: RawAxNode): SlimNode | undefined => { - if (budget <= 0) return undefined; - if (opts.interestingOnly && node.ignored) { - // Chrome often ignores wrapper divs while their descendants remain meaningful, so recurse through an ignored node's children. - const out: SlimNode[] = []; - for (const cid of node.childIds ?? []) { - const child = byId.get(cid); - if (!child) continue; - const slimChild = slim(child); - if (slimChild) out.push(slimChild); - } - return out.length > 0 ? { role: "_hoist", children: out } : undefined; - } - budget--; - const role = stringOf(node.role) ?? "unknown"; - const name = stringOf(node.name); - const value = stringOf(node.value); - const description = stringOf(node.description); - const state = collectState(node.properties); - const children: SlimNode[] = []; - for (const cid of node.childIds ?? []) { - const child = byId.get(cid); - if (!child) continue; - const slimChild = slim(child); - if (!slimChild) continue; - if (slimChild.role === "_hoist") children.push(...slimChild.children); - else children.push(slimChild); - } - const out: SlimNode = { role, children }; - if (name !== undefined) out.name = name; - if (value !== undefined) out.value = value; - if (description !== undefined) out.description = description; - if (state !== undefined) out.state = state; - if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId; - return out; - }; - - const result: SlimNode[] = []; - for (const rid of rootIds) { - const root = byId.get(rid); - if (!root) continue; - const s = slim(root); - if (!s) continue; - if (s.role === "_hoist") result.push(...s.children); - else result.push(s); - } - return result; -}; - -const countNodes = (nodes: ReadonlyArray<SlimNode>): number => { - let n = 0; - for (const node of nodes) n += 1 + countNodes(node.children); - return n; -}; - const summarize = (nodes: ReadonlyArray<SlimNode>): string => { const counts = new Map<string, number>(); const walk = (ns: ReadonlyArray<SlimNode>): void => { @@ -207,48 +82,6 @@ const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => { return lines.join("\n"); }; -const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] => - nodes.map((n) => { - const { _backendId: _bid, ...rest } = n; - return { ...rest, children: stripInternals(n.children) }; - }); - -export const collectInteractiveTargets = ( - nodes: ReadonlyArray<SlimNode>, -): Array<{ node: SlimNode; backendId: number }> => { - const out: Array<{ node: SlimNode; backendId: number }> = []; - const walk = (ns: ReadonlyArray<SlimNode>): void => { - for (const n of ns) { - if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) { - out.push({ node: n, backendId: n._backendId }); - } - walk(n.children); - } - }; - walk(nodes); - return out; -}; - -const liveValue = async (client: BrowserClient, backendId: number): Promise<string | undefined> => { - const resolved = await cdpCall(client, "DOM.resolveNode", { backendNodeId: backendId }); - if (!resolved.success) return undefined; - const objectId = resolved.data.object.objectId; - if (!objectId) return undefined; - const fn = `function () { - if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked"; - if (typeof this.value === "string") return this.value; - return null; - }`; - const r = await cdpCall(client, "Runtime.callFunctionOn", { - objectId, - functionDeclaration: fn, - returnByValue: true, - }); - if (!r.success) return undefined; - const value = r.data.result.value; - return typeof value === "string" ? value : undefined; -}; - type SnapshotDetails = { nodeCount: number; truncated: boolean; |
