aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authoramankumarsingh77 <amankumarsingh7702@gmal.com>2026-05-06 00:06:49 +0530
committeramankumarsingh77 <amankumarsingh7702@gmal.com>2026-05-06 00:06:49 +0530
commit589cd294931c01b3748779635d17a13dd240af19 (patch)
treeb635f9fa443cd120de905660fe0d2665184e9a8c /src
parent20ceaa1c6ade4950ed7d16caa5269ad2f4805db4 (diff)
release: v0.4.0 — accessibility snapshot, network capture, snapshot-first prompts
Adds two new tools and pivots the agent's default workflow away from screenshots toward structured page data. browser_snapshot — CDP accessibility tree (roles, names, states, hierarchy) with click coordinates @(x,y) attached to every interactive element via DOM.getBoxModel (parallel, 1.5s aggregate budget). Pass directly to browser_click — no screenshot round-trip. Optional includeScreenshot:true for visual confirmation. format:"json" returns a slim structure with full box per node. browser_network_requests — replaces the deprecated browser_get_network_log placeholder. Filters by urlPattern (substring or /regex/), method, status, resource type, recency. Optional response-body capture under a 5s aggregate budget with 50KB per-body cap. Buffer is page-scoped (cleared on tab switch), 500 records/tab, with a bufferOverflowed flag on drain. CDP plumbing: in-process aggregator in src/cdp/network-buffer.ts wired into the existing single-consumer event loop. Accessibility domain enabled alongside Page/DOM/Runtime/Network on every attach. Ctrl+O expand/collapse on three tools' renderResult — browser_snapshot, browser_network_requests, browser_execute_js. Compact-by-default views with keyHint("app.tools.expand") that respects user keybinding remaps. Tool prompts pivoted snapshot-first: browser_screenshot is documented as LAST RESORT for visual verification only; browser_snapshot is the default for understanding pages; browser_execute_js is the default for surgical reads. browser_click and browser_open_urls guidance follows. SKILL.md collapsed from ~342 to ~53 lines — keeping only what isn't already in tool prompts: the decision tree, connection rules, browser_run_script daemon bindings. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Diffstat (limited to 'src')
-rw-r--r--src/cdp/network-buffer.ts213
-rw-r--r--src/cdp/session.ts15
-rw-r--r--src/domains/click.ts11
-rw-r--r--src/domains/js.ts71
-rw-r--r--src/domains/navigate.ts2
-rw-r--r--src/domains/network.ts211
-rw-r--r--src/domains/screenshot.ts13
-rw-r--r--src/domains/snapshot.ts421
-rw-r--r--src/registry.ts6
9 files changed, 923 insertions, 40 deletions
diff --git a/src/cdp/network-buffer.ts b/src/cdp/network-buffer.ts
new file mode 100644
index 0000000..f841719
--- /dev/null
+++ b/src/cdp/network-buffer.ts
@@ -0,0 +1,213 @@
+/**
+ * In-memory aggregator for CDP Network.* events.
+ *
+ * The CDP event stream is consumed by a single internal loop in session.ts.
+ * That loop hands Network.* event params to this module, which joins them into
+ * one record per requestId (request → response → finished/failed). Tools then
+ * read records via drain(), filtered to what the caller asked for.
+ *
+ * Bounded by a fixed capacity (default 500): on overflow the oldest record is
+ * evicted FIFO. The overflow flag is reported once per drain so tools can warn
+ * the LLM that data may be missing.
+ *
+ * This module is pure (no CDP dependency) so the joining/filtering logic stays
+ * trivially testable in isolation.
+ */
+
+export type NetworkRecord = {
+ requestId: string;
+ method: string;
+ url: string;
+ status?: number;
+ statusText?: string;
+ type: string;
+ mimeType?: string;
+ /** Wall-clock ms when requestWillBeSent fired, in Date.now() units. */
+ requestStartedMs: number;
+ durationMs?: number;
+ requestBodySize: number;
+ responseBodySize?: number;
+ failed?: boolean;
+ errorText?: string;
+ /** Populated by the tool layer via Network.getResponseBody, never by the aggregator. */
+ body?: string | null;
+};
+
+export type NetworkFilter = {
+ /** Substring match by default. Wrap in slashes (`/foo/`) for regex. */
+ urlPattern?: string;
+ methodFilter?: string[];
+ statusFilter?: { min?: number; max?: number };
+ resourceTypes?: string[];
+ /** Only requests whose start is at most N ms ago. */
+ sinceMs?: number;
+ /** Cap on the returned records. */
+ limit?: number;
+};
+
+export type DrainResult = {
+ readonly records: NetworkRecord[];
+ /** Total matches before `limit` was applied. */
+ readonly total: number;
+ /** True if the ring buffer evicted at least one record since the last drain. */
+ readonly bufferOverflowed: boolean;
+};
+
+export type NetworkBuffer = {
+ ingestRequestWillBeSent(p: unknown): void;
+ ingestResponseReceived(p: unknown): void;
+ ingestLoadingFinished(p: unknown): void;
+ ingestLoadingFailed(p: unknown): void;
+ drain(filter: NetworkFilter): DrainResult;
+ clear(): void;
+};
+
+const compileUrlMatcher = (pattern: string): ((url: string) => boolean) => {
+ if (pattern.length >= 2 && pattern.startsWith("/") && pattern.endsWith("/")) {
+ try {
+ const re = new RegExp(pattern.slice(1, -1));
+ return (url) => re.test(url);
+ } catch {
+ // Fall through to substring match if the pattern is not a valid regex.
+ }
+ }
+ return (url) => url.includes(pattern);
+};
+
+export const createNetworkBuffer = (capacity = 500): NetworkBuffer => {
+ // Insertion-ordered map keyed by requestId. Map preserves insertion order in JS,
+ // so iteration gives us oldest-first eviction without a separate index.
+ const records = new Map<string, NetworkRecord>();
+ let overflowed = false;
+
+ const evictOldestIfFull = (): void => {
+ while (records.size >= capacity) {
+ const oldest = records.keys().next();
+ if (oldest.done) return;
+ records.delete(oldest.value);
+ overflowed = true;
+ }
+ };
+
+ return {
+ ingestRequestWillBeSent(p) {
+ // CDP boundary cast: Network.requestWillBeSent
+ const params = p as
+ | {
+ requestId?: string;
+ request?: {
+ url?: string;
+ method?: string;
+ postData?: string;
+ headers?: Record<string, string>;
+ };
+ type?: string;
+ timestamp?: number;
+ wallTime?: number;
+ }
+ | undefined;
+ const id = params?.requestId;
+ const url = params?.request?.url;
+ const method = params?.request?.method;
+ if (!id || !url || !method) return;
+ // Re-fired requestWillBeSent for the same id (e.g. redirect) — refresh in place.
+ records.delete(id);
+ evictOldestIfFull();
+ const postData = params.request?.postData;
+ records.set(id, {
+ requestId: id,
+ method,
+ url,
+ type: params.type ?? "Other",
+ requestStartedMs: Date.now(),
+ requestBodySize: postData ? Buffer.byteLength(postData, "utf8") : 0,
+ });
+ },
+
+ ingestResponseReceived(p) {
+ // CDP boundary cast: Network.responseReceived
+ const params = p as
+ | {
+ requestId?: string;
+ response?: {
+ status?: number;
+ statusText?: string;
+ mimeType?: string;
+ };
+ type?: string;
+ }
+ | undefined;
+ const id = params?.requestId;
+ if (!id) return;
+ const r = records.get(id);
+ if (!r) return;
+ if (params?.response?.status !== undefined) r.status = params.response.status;
+ if (params?.response?.statusText !== undefined && params.response.statusText !== "") r.statusText = params.response.statusText;
+ if (params?.response?.mimeType !== undefined) r.mimeType = params.response.mimeType;
+ // type is more accurate on responseReceived (CDP sometimes refines it)
+ if (params?.type !== undefined) r.type = params.type;
+ },
+
+ ingestLoadingFinished(p) {
+ // CDP boundary cast: Network.loadingFinished
+ const params = p as
+ | { requestId?: string; encodedDataLength?: number }
+ | undefined;
+ const id = params?.requestId;
+ if (!id) return;
+ const r = records.get(id);
+ if (!r) return;
+ if (params?.encodedDataLength !== undefined) r.responseBodySize = params.encodedDataLength;
+ r.durationMs = Date.now() - r.requestStartedMs;
+ },
+
+ ingestLoadingFailed(p) {
+ // CDP boundary cast: Network.loadingFailed
+ const params = p as
+ | { requestId?: string; errorText?: string; canceled?: boolean }
+ | undefined;
+ const id = params?.requestId;
+ if (!id) return;
+ const r = records.get(id);
+ if (!r) return;
+ r.failed = true;
+ if (params?.errorText !== undefined) r.errorText = params.errorText;
+ r.durationMs = Date.now() - r.requestStartedMs;
+ },
+
+ drain(filter) {
+ const matchUrl = filter.urlPattern !== undefined ? compileUrlMatcher(filter.urlPattern) : undefined;
+ const methods = filter.methodFilter ? new Set(filter.methodFilter.map((m) => m.toUpperCase())) : undefined;
+ const types = filter.resourceTypes ? new Set(filter.resourceTypes.map((t) => t.toLowerCase())) : undefined;
+ const minStatus = filter.statusFilter?.min;
+ const maxStatus = filter.statusFilter?.max;
+ const cutoff = filter.sinceMs !== undefined ? Date.now() - filter.sinceMs : undefined;
+
+ const matched: NetworkRecord[] = [];
+ for (const r of records.values()) {
+ if (matchUrl && !matchUrl(r.url)) continue;
+ if (methods && !methods.has(r.method.toUpperCase())) continue;
+ if (types && !types.has(r.type.toLowerCase())) continue;
+ if (cutoff !== undefined && r.requestStartedMs < cutoff) continue;
+ if (minStatus !== undefined && (r.status === undefined || r.status < minStatus)) continue;
+ if (maxStatus !== undefined && (r.status === undefined || r.status > maxStatus)) continue;
+ // Clone so callers can mutate (e.g. attach body) without polluting the buffer.
+ matched.push({ ...r });
+ }
+
+ const total = matched.length;
+ const limit = Math.min(filter.limit ?? 50, 500);
+ const limited = matched.slice(-limit); // most recent N
+
+ const bufferOverflowed = overflowed;
+ overflowed = false;
+
+ return { records: limited, total, bufferOverflowed };
+ },
+
+ clear() {
+ records.clear();
+ overflowed = false;
+ },
+ };
+};
diff --git a/src/cdp/session.ts b/src/cdp/session.ts
index 42f6107..89bdfdd 100644
--- a/src/cdp/session.ts
+++ b/src/cdp/session.ts
@@ -3,6 +3,7 @@ import type { CdpError } from "./errors";
import type { DialogInfo } from "./types";
import type { OwnershipRegistry } from "./ownership";
import type { CdpTransport } from "./transport";
+import { createNetworkBuffer, type DrainResult, type NetworkFilter } from "./network-buffer";
export type CdpSession = {
attachFirstPage(): Promise<Result<{ readonly targetId: string; readonly sessionId: string }, CdpError>>;
@@ -13,6 +14,7 @@ export type CdpSession = {
callBrowser(method: string, params?: Record<string, unknown>, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>;
takeDialog(): DialogInfo | null;
drainPageInfoInvalidations(): boolean;
+ drainNetworkBuffer(filter: NetworkFilter): DrainResult;
};
export const createCdpSession = (
@@ -23,6 +25,7 @@ export const createCdpSession = (
let targetId: string | null = null;
let dialog: DialogInfo | null = null;
let pageInfoDirty = false;
+ const networkBuffer = createNetworkBuffer();
let activeConsumer: Promise<void> = Promise.resolve();
@@ -48,6 +51,10 @@ export const createCdpSession = (
const params = ev.params as { targetId?: string } | undefined;
if (params?.targetId) ownership.remove(params.targetId);
}
+ if (ev.method === "Network.requestWillBeSent") networkBuffer.ingestRequestWillBeSent(ev.params);
+ else if (ev.method === "Network.responseReceived") networkBuffer.ingestResponseReceived(ev.params);
+ else if (ev.method === "Network.loadingFinished") networkBuffer.ingestLoadingFinished(ev.params);
+ else if (ev.method === "Network.loadingFailed") networkBuffer.ingestLoadingFailed(ev.params);
}
};
@@ -75,7 +82,7 @@ export const createCdpSession = (
// Switching to Promise.all over a single WS pipelines the round-trips and
// saves ~3× on tab-switch latency. Defer until session.ts has tests.
const enableDomains = async (sid: string): Promise<void> => {
- for (const d of ["Page", "DOM", "Runtime", "Network"]) {
+ for (const d of ["Page", "DOM", "Runtime", "Network", "Accessibility"]) {
await transport.request(`${d}.enable`, {}, { sessionId: sid });
}
};
@@ -143,6 +150,9 @@ export const createCdpSession = (
sessionId = a.sessionId;
targetId = tid;
pageInfoDirty = true;
+ // Buffer is page-scoped: on switch, the network history of the previous
+ // tab is no longer relevant to what the agent is now looking at.
+ networkBuffer.clear();
await enableDomains(a.sessionId);
return ok(undefined);
},
@@ -168,5 +178,8 @@ export const createCdpSession = (
pageInfoDirty = false;
return dirty;
},
+ drainNetworkBuffer(filter) {
+ return networkBuffer.drain(filter);
+ },
};
};
diff --git a/src/domains/click.ts b/src/domains/click.ts
index c269dd8..9946664 100644
--- a/src/domains/click.ts
+++ b/src/domains/click.ts
@@ -41,14 +41,15 @@ export const clickTool = defineBrowserTool({
name: "browser_click",
label: "Browser Click",
description:
- "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.",
+ "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Get coordinates from browser_snapshot's @(x,y) hints — do not screenshot to find click targets.",
promptSnippet: "Click at pixel coordinates (x, y) on the page",
promptGuidelines: [
- "Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).",
- "Capture a browser_screenshot BEFORE clicking to find the right coordinates.",
- "Capture another browser_screenshot AFTER clicking to verify the action worked.",
+ "Get (x, y) from browser_snapshot — the outline shows '@(x,y)' for every interactive element. NO screenshot needed.",
+ "If the target isn't an interactive role in the snapshot, fall back to browser_execute_js with `el.getBoundingClientRect()` — still NO screenshot.",
+ "Coordinates are viewport CSS pixels (not device pixels).",
+ "After clicking, verify with browser_snapshot (or browser_execute_js for a specific value). Only screenshot if you need to confirm visual rendering.",
"Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.",
- "If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.",
+ "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions (debug only).",
"For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.",
],
parameters: ClickArgs,
diff --git a/src/domains/js.ts b/src/domains/js.ts
index bd4c3e1..14862e8 100644
--- a/src/domains/js.ts
+++ b/src/domains/js.ts
@@ -3,10 +3,20 @@ import { resolve, isAbsolute } from "node:path";
import { tmpdir } from "node:os";
import { createRequire } from "node:module";
import { Type } from "typebox";
+import { Markdown, Text } from "@mariozechner/pi-tui";
+import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
import { applyTruncation } from "../util/truncate";
+const COMPACT_PREVIEW_BYTES = 120;
+
+const formatBytes = (n: number): string => {
+ if (n >= 1_048_576) return `${(n / 1_048_576).toFixed(1)} MB`;
+ if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`;
+ return `${n} B`;
+};
+
// AsyncFunction constructor — the documented Node.js mechanism for compiling
// arbitrary user-supplied source. Equivalent to `new Function` but produces an
// async function we can `await`. The full RCE risk is contained at the source
@@ -29,11 +39,15 @@ export const executeJsTool = defineBrowserTool({
name: "browser_execute_js",
label: "Browser Execute JS",
description:
- "Run JavaScript in the page (or a specific iframe target). `return X` gets auto-wrapped in an IIFE for convenience. Always use safeJs / JSON.stringify when interpolating untrusted strings into source.",
- promptSnippet: "Execute JS in the page (or iframe)",
+ "Surgical reads from the DOM. The cheapest, most precise way to extract a specific element's text, attribute, value, or geometry. Prefer over browser_screenshot for ANY data extraction or coordinate lookup. `return X` is auto-wrapped in an IIFE. Always use safeJs / JSON.stringify when interpolating untrusted strings.",
+ promptSnippet: "Execute JS in the page — surgical DOM reads (preferred over screenshot)",
promptGuidelines: [
+ "DEFAULT for extracting a specific value: `document.querySelector('.price').innerText`, `input.value`, `el.getAttribute('aria-label')`, etc.",
+ "For click coordinates not already in browser_snapshot: `JSON.stringify(document.querySelector('SELECTOR').getBoundingClientRect())` — beats a screenshot every time.",
+ "For 'is this element visible / enabled / checked / focused', a one-liner here beats a screenshot every time.",
+ "DO NOT call browser_screenshot to read a value or check element state. Use this tool.",
"`return foo` is auto-wrapped in an IIFE — both `foo` and `(() => foo)()` work.",
- "For iframes, get a targetId via browser_execute_js with `Object.values(document.querySelectorAll('iframe'))` then call browser_execute_js with targetId.",
+ "For iframes, first run `Object.values(document.querySelectorAll('iframe'))` to find the iframe, then pass its targetId.",
"Result must be JSON-serializable (Runtime.evaluate returnByValue=true).",
],
parameters: ExecuteJsArgs,
@@ -42,13 +56,58 @@ export const executeJsTool = defineBrowserTool({
if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
const valueStr = r.data === undefined ? "undefined" : JSON.stringify(r.data);
const truncated = await applyTruncation(valueStr, "js");
+ // If the value is JSON, pretty-print for the expanded TUI view. Falls back
+ // to the raw string if parse fails (e.g. value is "undefined" sentinel).
+ let pretty: string | undefined;
+ try {
+ const parsed = JSON.parse(valueStr);
+ pretty = JSON.stringify(parsed, null, 2);
+ } catch {
+ pretty = undefined;
+ }
return ok({
text: truncated.text,
- details: truncated.fullOutputPath !== undefined
- ? { valueLength: valueStr.length, fullOutputPath: truncated.fullOutputPath }
- : { valueLength: valueStr.length },
+ details: {
+ valueLength: valueStr.length,
+ // Full value for the expanded renderer. Capped at 200 KB so a giant
+ // dump doesn't bloat the conversation transcript on disk.
+ full: valueStr.length > 200_000 ? valueStr.slice(0, 200_000) : valueStr,
+ ...(pretty !== undefined ? { pretty: pretty.length > 200_000 ? pretty.slice(0, 200_000) : pretty } : {}),
+ ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
+ },
});
},
+
+ renderResult(result, expanded, theme) {
+ const details = result.details as
+ | { valueLength?: number; full?: string; pretty?: string; fullOutputPath?: string }
+ | undefined;
+ if (!details) return new Text(theme.fg("error", "execute_js: no details"), 0, 0);
+
+ const len = details.valueLength ?? 0;
+ const full = details.full ?? "";
+ const isJson = details.pretty !== undefined;
+
+ if (!expanded) {
+ const preview = full.length > COMPACT_PREVIEW_BYTES ? full.slice(0, COMPACT_PREVIEW_BYTES) + "…" : full;
+ const md = [
+ `**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`,
+ "```",
+ preview,
+ "```",
+ keyHint("app.tools.expand", "to expand"),
+ ].join("\n");
+ return new Markdown(md, 0, 0, getMarkdownTheme());
+ }
+
+ const body = details.pretty ?? full;
+ const fence = isJson ? "```json" : "```";
+ const tail = details.fullOutputPath
+ ? `\n\nFull value at \`${details.fullOutputPath}\` · ${keyHint("app.tools.expand", "to collapse")}`
+ : `\n\n${keyHint("app.tools.expand", "to collapse")}`;
+ const md = [`**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`, fence, body, "```", tail].join("\n");
+ return new Markdown(md, 0, 0, getMarkdownTheme());
+ },
});
const RunScriptArgs = Type.Object({
diff --git a/src/domains/navigate.ts b/src/domains/navigate.ts
index 6016122..7012d15 100644
--- a/src/domains/navigate.ts
+++ b/src/domains/navigate.ts
@@ -75,7 +75,7 @@ export const openUrlsTool = defineBrowserTool({
promptSnippet: "Open multiple URLs in new tabs (parallel)",
promptGuidelines: [
"Use after web_search to open citations in parallel.",
- "After opening, use browser_list_tabs / browser_switch_tab / browser_screenshot to interact.",
+ "After opening, use browser_list_tabs / browser_switch_tab to navigate, then browser_snapshot to inspect each tab.",
"Use browser_wait_for_load on a tab before extracting data from SPAs.",
],
parameters: OpenUrlsArgs,
diff --git a/src/domains/network.ts b/src/domains/network.ts
index 1377486..f13b1e7 100644
--- a/src/domains/network.ts
+++ b/src/domains/network.ts
@@ -1,7 +1,10 @@
import { Type } from "typebox";
+import { Markdown, Text } from "@mariozechner/pi-tui";
+import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
import { applyTruncation } from "../util/truncate";
+import type { NetworkRecord } from "../cdp/network-buffer";
const HttpGetArgs = Type.Object({
url: Type.String({ description: "URL to GET" }),
@@ -49,32 +52,200 @@ export const httpGetTool = defineBrowserTool({
},
});
-const NetLogArgs = Type.Object({
- eventTypes: Type.Optional(
- Type.Array(Type.String(), { description: "CDP event names to filter (e.g., ['Network.requestWillBeSent'])" }),
+const NetworkRequestsArgs = Type.Object({
+ urlPattern: Type.Optional(
+ Type.String({ description: "Substring match by default. Wrap in slashes (e.g. /api\\\\.example/) for regex." }),
+ ),
+ methodFilter: Type.Optional(
+ Type.Array(Type.String(), { description: 'HTTP methods to include, e.g. ["GET","POST"].' }),
+ ),
+ statusFilter: Type.Optional(
+ Type.Object(
+ {
+ min: Type.Optional(Type.Integer({ minimum: 100, maximum: 599 })),
+ max: Type.Optional(Type.Integer({ minimum: 100, maximum: 599 })),
+ },
+ { description: "Inclusive HTTP status range." },
+ ),
+ ),
+ resourceTypes: Type.Optional(
+ Type.Array(Type.String(), { description: 'CDP resource types: "Document","Stylesheet","Image","Media","Font","Script","TextTrack","XHR","Fetch","EventSource","WebSocket","Manifest","SignedExchange","Ping","CSPViolationReport","Preflight","Other".' }),
+ ),
+ sinceMs: Type.Optional(
+ Type.Integer({ minimum: 0, description: "Only requests started in the last N ms." }),
+ ),
+ limit: Type.Optional(
+ Type.Integer({ default: 50, minimum: 1, maximum: 500, description: "Max records to return." }),
+ ),
+ includeResponseBodies: Type.Optional(
+ Type.Boolean({ default: false, description: "Fetch response bodies via Network.getResponseBody. Costs one CDP call per matched request." }),
),
- limit: Type.Optional(Type.Integer({ default: 50, minimum: 1, maximum: 500 })),
});
-export const getNetworkLogTool = defineBrowserTool({
- name: "browser_get_network_log",
- label: "Browser Get Network Log",
+const BODY_BUDGET_MS = 5_000;
+const PER_BODY_CAP = 50_000;
+
+const renderNetworkMarkdown = (
+ records: ReadonlyArray<NetworkRecord>,
+ opts: { includeBodies: boolean; total: number; bufferOverflowed: boolean; expanded: boolean },
+): string => {
+ if (records.length === 0) {
+ if (opts.bufferOverflowed) return "_No matching requests. (Buffer overflowed since last drain — some events were dropped.)_";
+ return "_No matching requests captured yet._";
+ }
+
+ const failed = records.filter((r) => r.failed || (r.status !== undefined && r.status >= 500)).length;
+ const totalBytes = records.reduce((s, r) => s + (r.responseBodySize ?? 0), 0);
+ const formatBytes = (n: number): string => {
+ if (n >= 1_048_576) return `${(n / 1_048_576).toFixed(1)} MB`;
+ if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`;
+ return `${n} B`;
+ };
+ const showWindow = (() => {
+ const finished = records.filter((r) => r.durationMs !== undefined);
+ if (finished.length === 0) return "";
+ const span = Math.max(...records.map((r) => r.requestStartedMs)) - Math.min(...records.map((r) => r.requestStartedMs));
+ return ` · ${(span / 1000).toFixed(1)}s window`;
+ })();
+
+ const headerLines: string[] = [
+ `**${records.length} requests** · ${failed} failed · ${formatBytes(totalBytes)}${showWindow}`,
+ ];
+ if (opts.bufferOverflowed) headerLines.push("⚠ buffer overflowed since last drain — older events dropped");
+ if (opts.total > records.length) headerLines.push(`(${records.length} of ${opts.total} matches shown — increase \`limit\` for more)`);
+
+ const compactRows = (records.length > 5 && !opts.expanded ? records.slice(0, 5) : records).map((r) => {
+ const status = r.failed ? "ERR" : r.status !== undefined ? String(r.status) : "—";
+ const warn = r.failed || (r.status !== undefined && r.status >= 500) ? " ⚠" : "";
+ const ms = r.durationMs !== undefined ? `${r.durationMs} ms` : "—";
+ const url = r.url.length > 60 ? r.url.slice(0, 57) + "…" : r.url;
+ return `\`${r.method.padEnd(4)} ${status.padStart(3)}\` ${url} _${ms}_${warn}`;
+ });
+
+ const rows = compactRows.join("\n");
+
+ if (!opts.expanded) {
+ const more = records.length > 5 ? `\n … ${records.length - 5} more` : "";
+ return `${headerLines.join("\n")}\n\n${rows}${more}`;
+ }
+
+ // Expanded: full table + optional body sections.
+ const tableHeader = `| # | Method | Status | URL | Type | ms |\n|---|---|---|---|---|---|`;
+ const tableRows = records
+ .map((r, i) => {
+ const status = r.failed ? "ERR" : r.status !== undefined ? String(r.status) : "—";
+ const ms = r.durationMs !== undefined ? String(r.durationMs) : "—";
+ const url = r.url.length > 80 ? r.url.slice(0, 77) + "…" : r.url;
+ return `| ${i + 1} | ${r.method} | ${status} | \`${url}\` | ${r.type} | ${ms} |`;
+ })
+ .join("\n");
+
+ let bodySections = "";
+ if (opts.includeBodies) {
+ const sections: string[] = [];
+ records.forEach((r, i) => {
+ if (r.body === undefined) return;
+ const sizes = `${r.requestBodySize} B → ${r.responseBodySize ?? 0} B`;
+ const status = r.failed ? "failed" : r.status ?? "?";
+ const header = `── #${i + 1} ${r.method} ${r.url} (${status}, ${r.durationMs ?? "?"} ms, ${r.type}, ${sizes}) ──`;
+ const bodyText = r.body === null ? "_(body unavailable — already GC'd by Chrome)_" : "```\n" + r.body + "\n```";
+ sections.push(`${header}\n\n${bodyText}`);
+ });
+ if (sections.length > 0) bodySections = "\n\n" + sections.join("\n\n");
+ }
+
+ return [headerLines.join("\n"), "", tableHeader, tableRows, bodySections].filter((s) => s !== "").join("\n");
+};
+
+type NetworkDetails = {
+ total: number;
+ returned: number;
+ bufferOverflowed: boolean;
+ truncated: boolean;
+ fullOutputPath?: string;
+ requests: ReadonlyArray<NetworkRecord>;
+ includeBodies: boolean;
+};
+
+export const networkRequestsTool = defineBrowserTool({
+ name: "browser_network_requests",
+ label: "Browser Network Requests",
description:
- "Read buffered Network.* CDP events. NOTE: in v0.3 the event stream is consumed internally by the session manager; this tool is a placeholder. Use browser_execute_js with PerformanceObserver as a workaround.",
- promptSnippet: "Get buffered network events (DEPRECATED in v0.3 — use browser_execute_js with PerformanceObserver instead)",
+ "List recent network requests captured on the current tab since attach. Filter by URL pattern, method, status, resource type, or recency. Optionally include response bodies.",
+ promptSnippet: "Inspect recent network requests on the current page",
promptGuidelines: [
- "v0.3 routes CDP events through an internal AsyncIterable; a synchronous drain API isn't exposed yet.",
- "For now, use browser_execute_js with PerformanceObserver, performance.getEntries(), or fetch interceptors.",
+ "Buffer is reset on tab switch — only requests on the current tab since attach are visible.",
+ 'urlPattern is a substring; wrap in slashes for regex (e.g. "/\\\\.json$/").',
+ "includeResponseBodies:true costs an extra CDP call per matched request — use only when you need payloads.",
+ "If bufferOverflowed:true in the result, older events were dropped (capacity 500 records/tab).",
],
- parameters: NetLogArgs,
- async handler(args): Promise<Result<ToolOk, ToolErr>> {
- return ok({
- text: "browser_get_network_log: in v0.3 the CDP event stream is consumed by the session; use browser_execute_js with PerformanceObserver or performance.getEntries() to inspect network activity.",
- details: {
- deprecated: true,
- eventTypes: args.eventTypes ?? null,
- limit: args.limit ?? null,
- },
+ parameters: NetworkRequestsArgs,
+
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+ const session = client.session();
+ const drained = session.drainNetworkBuffer({
+ ...(args.urlPattern !== undefined ? { urlPattern: args.urlPattern } : {}),
+ ...(args.methodFilter !== undefined ? { methodFilter: args.methodFilter } : {}),
+ ...(args.statusFilter !== undefined ? { statusFilter: args.statusFilter } : {}),
+ ...(args.resourceTypes !== undefined ? { resourceTypes: args.resourceTypes } : {}),
+ ...(args.sinceMs !== undefined ? { sinceMs: args.sinceMs } : {}),
+ limit: args.limit ?? 50,
});
+
+ if (args.includeResponseBodies) {
+ const ac = new AbortController();
+ const timer = setTimeout(() => ac.abort(), BODY_BUDGET_MS);
+ const fills = drained.records.map(async (r) => {
+ if (ac.signal.aborted) return;
+ const bodyRes = await session.call("Network.getResponseBody", { requestId: r.requestId });
+ if (!bodyRes.success) {
+ r.body = null;
+ return;
+ }
+ const b = bodyRes.data as { body: string; base64Encoded?: boolean };
+ let body = b.base64Encoded ? Buffer.from(b.body, "base64").toString("utf8") : b.body;
+ if (body.length > PER_BODY_CAP) body = body.slice(0, PER_BODY_CAP) + "…";
+ r.body = body;
+ });
+ await Promise.allSettled(fills);
+ clearTimeout(timer);
+ }
+
+ const text = renderNetworkMarkdown(drained.records, {
+ includeBodies: args.includeResponseBodies ?? false,
+ total: drained.total,
+ bufferOverflowed: drained.bufferOverflowed,
+ expanded: true, // full text always returned to the LLM; renderResult handles compact view for humans
+ });
+ const trunc = await applyTruncation(text, "network");
+
+ const details: NetworkDetails = {
+ total: drained.total,
+ returned: drained.records.length,
+ bufferOverflowed: drained.bufferOverflowed,
+ truncated: trunc.wasTruncated,
+ requests: drained.records,
+ includeBodies: args.includeResponseBodies ?? false,
+ };
+ if (trunc.fullOutputPath !== undefined) details.fullOutputPath = trunc.fullOutputPath;
+
+ return ok({ text: trunc.text, details });
+ },
+
+ renderResult(result, expanded, theme) {
+ const details = result.details as NetworkDetails | undefined;
+ if (!details) return new Text(theme.fg("error", "network: no details"), 0, 0);
+
+ const md = renderNetworkMarkdown(details.requests, {
+ includeBodies: details.includeBodies && expanded,
+ total: details.total,
+ bufferOverflowed: details.bufferOverflowed,
+ expanded,
+ });
+ const tail = expanded
+ ? `\n\n${keyHint("app.tools.expand", "to collapse")}${details.fullOutputPath ? ` · full payload at \`${details.fullOutputPath}\`` : ""}`
+ : `\n\n${keyHint("app.tools.expand", "to expand")}`;
+
+ return new Markdown(md + tail, 0, 0, getMarkdownTheme());
},
});
diff --git a/src/domains/screenshot.ts b/src/domains/screenshot.ts
index 74042be..9d6313b 100644
--- a/src/domains/screenshot.ts
+++ b/src/domains/screenshot.ts
@@ -63,12 +63,15 @@ const resizeIfNeeded = async (
export const screenshotTool = defineBrowserTool({
name: "browser_screenshot",
label: "Browser Screenshot",
- description: "Capture the current page as PNG or JPEG. JPEG is 2-5x smaller — prefer it for photo-heavy pages.",
- promptSnippet: "Capture a screenshot of the current page",
+ description:
+ "Capture the page as a JPEG/PNG image. NOT a default exploration tool. Use ONLY when (a) you need to verify visual properties (layout, color, spacing, rendered text legibility), or (b) browser_snapshot and browser_execute_js cannot answer your question. For understanding what's on the page use browser_snapshot. For specific element values use browser_execute_js. For click coordinates use the @(x,y) hints in browser_snapshot's outline.",
+ promptSnippet: "Capture a screenshot — visual verification only",
promptGuidelines: [
- "Use BEFORE clicks/scrolls to find coordinates; AFTER to verify the action.",
- "Pass format='jpeg' with a quality (60-90) for smaller files on photo-heavy pages.",
- "Set maxDim if the page is huge and you want to fit under LLM image-size limits.",
+ "DO NOT use as a default exploration tool. browser_snapshot is the default for understanding page structure — far cheaper and more reliable.",
+ "DO NOT use to find click coordinates. browser_snapshot already includes @(x,y) for every interactive element. If a target isn't there, use browser_execute_js with element.getBoundingClientRect().",
+ "DO use to verify visual rendering after an action: did the modal animate in correctly, did the chart render, are colors right, did the layout reflow as expected.",
+ "DO use as a debugging fallback when browser_execute_js or browser_snapshot return surprising results and you suspect the rendered page differs from the DOM.",
+ "format='jpeg' with quality 60-90 keeps screenshots small for photo-heavy pages. Use maxDim if the page is huge.",
],
parameters: ScreenshotArgs,
async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
diff --git a/src/domains/snapshot.ts b/src/domains/snapshot.ts
new file mode 100644
index 0000000..39c493a
--- /dev/null
+++ b/src/domains/snapshot.ts
@@ -0,0 +1,421 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { Type } from "typebox";
+import { Container, Image, type ImageTheme, Markdown, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
+import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
+import { type Result, err, ok } from "../util/result";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { applyTruncation } from "../util/truncate";
+import { screenshotPath } from "../util/paths";
+
+const SnapshotArgs = Type.Object({
+ includeScreenshot: Type.Optional(
+ Type.Boolean({ default: false, description: "Also capture a JPEG screenshot of the current viewport." }),
+ ),
+ interestingOnly: Type.Optional(
+ Type.Boolean({ default: true, description: "Drop nodes the AX engine marked uninteresting (generic containers, inline text, etc.)." }),
+ ),
+ maxNodes: Type.Optional(
+ Type.Integer({ default: 1000, minimum: 1, maximum: 5000, description: "Cap on slim nodes returned." }),
+ ),
+ format: Type.Optional(
+ Type.Union(
+ [Type.Literal("outline"), Type.Literal("json")],
+ { default: "outline", description: "'outline' = indented markdown bullet tree; 'json' = raw slim structure." },
+ ),
+ ),
+});
+
+// Raw shapes from CDP Accessibility.getFullAXTree. Cast at this boundary only.
+type AxValue = { value?: unknown };
+type RawAxNode = {
+ nodeId: string;
+ parentId?: string;
+ childIds?: ReadonlyArray<string>;
+ ignored?: boolean;
+ role?: AxValue;
+ name?: AxValue;
+ value?: AxValue;
+ description?: AxValue;
+ properties?: ReadonlyArray<{ name?: string; value?: AxValue }>;
+ backendDOMNodeId?: number;
+};
+
+type Box = { x: number; y: number; width: number; height: number; cx: number; cy: number };
+
+type SlimNode = {
+ role: string;
+ name?: string;
+ value?: string;
+ description?: string;
+ state?: string;
+ children: SlimNode[];
+ /** Internal: kept for the post-build box fetch. Stripped before returning to the LLM. */
+ _backendId?: number;
+ /** Click target — center of the element's bounding box, in viewport CSS pixels. */
+ box?: Box;
+};
+
+const INTERACTIVE_ROLES = new Set([
+ "button",
+ "link",
+ "textbox",
+ "checkbox",
+ "radio",
+ "combobox",
+ "menuitem",
+ "tab",
+ "switch",
+ "slider",
+ "searchbox",
+ "spinbutton",
+]);
+
+const stringOf = (v: AxValue | undefined): string | undefined => {
+ const x = v?.value;
+ if (typeof x === "string" && x !== "") return x;
+ if (typeof x === "number") return String(x);
+ return undefined;
+};
+
+const collectState = (props: RawAxNode["properties"]): string | undefined => {
+ if (!props || props.length === 0) return undefined;
+ const flags: string[] = [];
+ for (const p of props) {
+ const name = p.name;
+ if (!name) continue;
+ const raw = p.value?.value;
+ // Boolean flags worth surfacing in the outline.
+ if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) {
+ flags.push(name);
+ }
+ if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`);
+ }
+ return flags.length > 0 ? flags.join(", ") : undefined;
+};
+
+const buildTree = (
+ rawNodes: ReadonlyArray<RawAxNode>,
+ opts: { interestingOnly: boolean; maxNodes: number },
+): SlimNode[] => {
+ // Index for parent/child lookups.
+ const byId = new Map<string, RawAxNode>();
+ for (const n of rawNodes) byId.set(n.nodeId, n);
+
+ // CDP doesn't mark a single root explicitly; the root nodes are the ones
+ // whose parentId isn't in the index (typically just one — RootWebArea).
+ const rootIds: string[] = [];
+ for (const n of rawNodes) {
+ if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId);
+ }
+
+ let budget = opts.maxNodes;
+
+ const slim = (node: RawAxNode): SlimNode | undefined => {
+ if (budget <= 0) return undefined;
+ if (opts.interestingOnly && node.ignored) {
+ // Skip the ignored node but recurse through its children — Chrome often
+ // ignores wrapper divs but the meaningful descendants are still there.
+ const out: SlimNode[] = [];
+ for (const cid of node.childIds ?? []) {
+ const child = byId.get(cid);
+ if (!child) continue;
+ const slimChild = slim(child);
+ if (slimChild) out.push(slimChild);
+ }
+ // Hoist children into the parent. We return undefined; the parent's
+ // recursion will see them by re-walking — but that double-counts. Instead
+ // we handle hoisting at the parent level. So here just signal "skip me"
+ // by returning undefined; the parent will hoist by checking ignored flag.
+ return out.length > 0 ? { role: "_hoist", children: out } : undefined;
+ }
+ budget--;
+ const role = stringOf(node.role) ?? "unknown";
+ const name = stringOf(node.name);
+ const value = stringOf(node.value);
+ const description = stringOf(node.description);
+ const state = collectState(node.properties);
+ const children: SlimNode[] = [];
+ for (const cid of node.childIds ?? []) {
+ const child = byId.get(cid);
+ if (!child) continue;
+ const slimChild = slim(child);
+ if (!slimChild) continue;
+ // Hoist any synthetic _hoist nodes' children into this level.
+ if (slimChild.role === "_hoist") children.push(...slimChild.children);
+ else children.push(slimChild);
+ }
+ const out: SlimNode = { role, children };
+ if (name !== undefined) out.name = name;
+ if (value !== undefined) out.value = value;
+ if (description !== undefined) out.description = description;
+ if (state !== undefined) out.state = state;
+ if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId;
+ return out;
+ };
+
+ const result: SlimNode[] = [];
+ for (const rid of rootIds) {
+ const root = byId.get(rid);
+ if (!root) continue;
+ const s = slim(root);
+ if (!s) continue;
+ if (s.role === "_hoist") result.push(...s.children);
+ else result.push(s);
+ }
+ return result;
+};
+
+const countNodes = (nodes: ReadonlyArray<SlimNode>): number => {
+ let n = 0;
+ for (const node of nodes) n += 1 + countNodes(node.children);
+ return n;
+};
+
+const summarize = (nodes: ReadonlyArray<SlimNode>): string => {
+ const counts = new Map<string, number>();
+ const walk = (ns: ReadonlyArray<SlimNode>): void => {
+ for (const n of ns) {
+ counts.set(n.role, (counts.get(n.role) ?? 0) + 1);
+ walk(n.children);
+ }
+ };
+ walk(nodes);
+ // Group landmarks (roles that are top-level structural).
+ const landmarkRoles = new Set(["banner", "navigation", "main", "complementary", "contentinfo", "search", "form", "region"]);
+ let landmarks = 0;
+ for (const [role, c] of counts) if (landmarkRoles.has(role)) landmarks += c;
+ const buttons = counts.get("button") ?? 0;
+ const inputs = (counts.get("textbox") ?? 0) + (counts.get("combobox") ?? 0) + (counts.get("checkbox") ?? 0);
+ const links = counts.get("link") ?? 0;
+ const parts: string[] = [];
+ if (landmarks > 0) parts.push(`${landmarks} landmark${landmarks === 1 ? "" : "s"}`);
+ if (buttons > 0) parts.push(`${buttons} button${buttons === 1 ? "" : "s"}`);
+ if (inputs > 0) parts.push(`${inputs} input${inputs === 1 ? "" : "s"}`);
+ if (links > 0) parts.push(`${links} link${links === 1 ? "" : "s"}`);
+ return parts.join(" · ");
+};
+
+const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => {
+ const lines: string[] = [];
+ const walk = (ns: ReadonlyArray<SlimNode>, depth: number): void => {
+ for (const n of ns) {
+ const indent = " ".repeat(depth);
+ let line = `${indent}- ${n.role}`;
+ if (n.name) line += ` "${n.name}"`;
+ if (n.value && n.value !== n.name) line += ` = ${JSON.stringify(n.value)}`;
+ if (n.state) line += ` (${n.state})`;
+ // Surface click coordinates for interactive nodes — agents can pass these
+ // straight to browser_click without a screenshot round-trip.
+ if (n.box && INTERACTIVE_ROLES.has(n.role)) {
+ line += ` @(${n.box.cx},${n.box.cy})`;
+ }
+ lines.push(line);
+ walk(n.children, depth + 1);
+ }
+ };
+ walk(nodes, 0);
+ return lines.join("\n");
+};
+
+/** Strip internal _backendId from a slim tree before returning to callers. */
+const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] =>
+ nodes.map((n) => {
+ const { _backendId: _bid, ...rest } = n;
+ return { ...rest, children: stripInternals(n.children) };
+ });
+
+/**
+ * Walk all slim nodes and collect (slimNode, backendId) pairs for interactive
+ * roles. Caller fetches DOM.getBoxModel for each and writes box back into the
+ * slim node by reference.
+ */
+const collectInteractiveTargets = (
+ nodes: ReadonlyArray<SlimNode>,
+): Array<{ node: SlimNode; backendId: number }> => {
+ const out: Array<{ node: SlimNode; backendId: number }> = [];
+ const walk = (ns: ReadonlyArray<SlimNode>): void => {
+ for (const n of ns) {
+ if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) {
+ out.push({ node: n, backendId: n._backendId });
+ }
+ walk(n.children);
+ }
+ };
+ walk(nodes);
+ return out;
+};
+
+type SnapshotDetails = {
+ nodeCount: number;
+ truncated: boolean;
+ fullOutputPath?: string;
+ screenshotPath?: string;
+ url: string;
+ title: string;
+};
+
+export const snapshotTool = defineBrowserTool({
+ name: "browser_snapshot",
+ label: "Browser Snapshot",
+ description:
+ "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy) plus click coordinates for every interactive element as @(x,y). Use this BEFORE deciding whether you need a screenshot — almost always sufficient on its own and far cheaper. Pair with browser_execute_js for surgical reads of specific element values.",
+ promptSnippet: "Get accessibility-tree snapshot of the current page (default for page inspection)",
+ promptGuidelines: [
+ "DEFAULT — use this whenever you need to know what's on a page, what's clickable, or how the page is structured.",
+ "Click coordinates come for free: every interactive element shows '@(x,y)' in the outline. Pass these straight to browser_click. NO screenshot round-trip needed.",
+ "DO NOT call browser_screenshot just to understand the page. This tool already gives you structure, labels, states, and click targets.",
+ "Pass includeScreenshot:true ONLY if you also need to verify visual rendering (rare).",
+ "format:'json' returns the raw slim structure (with `box` per node) for programmatic use; default 'outline' is human/LLM-readable.",
+ ],
+ parameters: SnapshotArgs,
+
+ async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+ const session = client.session();
+
+ // 1. AX tree
+ const axRes = await session.call("Accessibility.getFullAXTree", {});
+ if (!axRes.success) return err({ kind: "cdp_error", message: axRes.error.message });
+ const rawNodes = (axRes.data as { nodes: ReadonlyArray<RawAxNode> }).nodes;
+
+ // 2. Page url/title
+ const piRes = await client.pageInfo();
+ if (!piRes.success) return err({ kind: "cdp_error", message: piRes.error.message });
+ const pageUrl = "dialog" in piRes.data ? "" : piRes.data.url;
+ const pageTitle = "dialog" in piRes.data ? "" : piRes.data.title;
+
+ // 3. Slim + format
+ const slim = buildTree(rawNodes, {
+ interestingOnly: args.interestingOnly ?? true,
+ maxNodes: args.maxNodes ?? 1000,
+ });
+
+ // 4. Fetch bounding boxes for interactive nodes so the agent can click
+ // without a screenshot round-trip. Capped budget so a slow page can't
+ // wedge the call.
+ const targets = collectInteractiveTargets(slim);
+ if (targets.length > 0) {
+ const ac = new AbortController();
+ const timer = setTimeout(() => ac.abort(), 1_500);
+ await Promise.allSettled(
+ targets.map(async ({ node, backendId }) => {
+ if (ac.signal.aborted) return;
+ const r = await session.call("DOM.getBoxModel", { backendNodeId: backendId });
+ if (!r.success) return;
+ // CDP boundary cast: DOM.getBoxModel
+ const data = r.data as { model?: { content?: ReadonlyArray<number>; width?: number; height?: number } };
+ const content = data.model?.content;
+ if (!content || content.length < 8) return;
+ // content is [x1,y1, x2,y1, x2,y2, x1,y2] in viewport CSS pixels.
+ const x = content[0]!;
+ const y = content[1]!;
+ const width = data.model!.width ?? 0;
+ const height = data.model!.height ?? 0;
+ // Skip zero-size and off-viewport boxes — they can't be clicked.
+ if (width <= 0 || height <= 0 || x < 0 || y < 0) return;
+ node.box = {
+ x: Math.round(x),
+ y: Math.round(y),
+ width: Math.round(width),
+ height: Math.round(height),
+ cx: Math.round(x + width / 2),
+ cy: Math.round(y + height / 2),
+ };
+ }),
+ );
+ clearTimeout(timer);
+ }
+
+ const stripped = stripInternals(slim);
+ const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripped, null, 2);
+ const trunc = await applyTruncation(text, "snapshot");
+
+ // 4. Optional screenshot — JPEG q=80 to match screenshotTool's default.
+ let shotPath: string | undefined;
+ if (args.includeScreenshot) {
+ const shot = await session.call("Page.captureScreenshot", { format: "jpeg", quality: 80 });
+ if (shot.success) {
+ const data = (shot.data as { data: string }).data;
+ shotPath = screenshotPath(client.namespace, "jpeg");
+ writeFileSync(shotPath, Buffer.from(data, "base64"));
+ }
+ }
+
+ const details: SnapshotDetails = {
+ nodeCount: countNodes(slim),
+ truncated: trunc.wasTruncated,
+ url: pageUrl,
+ title: pageTitle,
+ };
+ if (trunc.fullOutputPath !== undefined) details.fullOutputPath = trunc.fullOutputPath;
+ if (shotPath !== undefined) details.screenshotPath = shotPath;
+
+ return ok({ text: trunc.text, details: { ...details, summary: summarize(slim) } });
+ },
+
+ renderResult(result, expanded, theme) {
+ const details = result.details as
+ | (SnapshotDetails & { summary?: string; ok?: boolean })
+ | undefined;
+ if (!details) return new Text(theme.fg("error", "snapshot: no details"), 0, 0);
+
+ const titleLine = details.title ? ` "${details.title}"` : "";
+ const summary = details.summary ?? "";
+ const screenshotNote = details.screenshotPath ? "screenshot attached" : "screenshot omitted";
+ const truncNote = details.truncated && details.fullOutputPath ? `\n\nFull tree at \`${details.fullOutputPath}\`` : "";
+
+ if (!expanded) {
+ const md = [
+ `**AX tree:** ${details.nodeCount} nodes · \`${details.url || "(no url)"}\``,
+ `${titleLine ? " " + titleLine : ""}`,
+ summary ? ` • ${summary}` : "",
+ ` ${keyHint("app.tools.expand", "to expand")} · ${screenshotNote}`,
+ ].filter((l) => l !== "").join("\n");
+ return new Markdown(md, 0, 0, getMarkdownTheme());
+ }
+
+ // Expanded view: read truncated text from result.content (the spilled file
+ // is too large to inline) — the same `text` we returned from handler is in
+ // result.content[0]. Fall back to a placeholder if shape is unexpected.
+ const content = result.content[0];
+ const treeText = content && content.type === "text" ? content.text : "";
+
+ const headerMd = [
+ `**AX tree:** ${details.nodeCount} nodes · \`${details.url || "(no url)"}\``,
+ titleLine ? ` ${titleLine}` : "",
+ "",
+ "```",
+ treeText,
+ "```",
+ truncNote,
+ "",
+ `${keyHint("app.tools.expand", "to collapse")}`,
+ ].filter((l) => l !== "").join("\n");
+
+ const treeBlock = new Markdown(headerMd, 0, 0, getMarkdownTheme());
+
+ if (!details.screenshotPath) return treeBlock;
+
+ // Mirror screenshotTool's width-clamp wrapper for the inline image.
+ try {
+ const buf = readFileSync(details.screenshotPath);
+ const b64 = buf.toString("base64");
+ const imageTheme: ImageTheme = { fallbackColor: (s: string) => theme.fg("dim", s) };
+ const image = new Image(b64, "image/jpeg", imageTheme, {
+ maxWidthCells: 80,
+ maxHeightCells: 24,
+ filename: details.screenshotPath,
+ });
+ const safeImage = {
+ invalidate: () => image.invalidate(),
+ render: (width: number) =>
+ image.render(width).map((line) => (visibleWidth(line) > width ? truncateToWidth(line, width) : line)),
+ };
+ const container = new Container();
+ container.addChild(treeBlock);
+ container.addChild(safeImage);
+ return container;
+ } catch {
+ return treeBlock;
+ }
+ },
+});
diff --git a/src/registry.ts b/src/registry.ts
index d205bf6..df912b1 100644
--- a/src/registry.ts
+++ b/src/registry.ts
@@ -13,7 +13,8 @@ import { listTabsTool, currentTabTool, switchTabTool, newTabTool, closeTabTool }
import { uploadFileTool, downloadTool, printToPdfTool } from "./domains/files";
import { viewportResizeTool } from "./domains/viewport";
import { dragAndDropTool } from "./domains/drag";
-import { httpGetTool, getNetworkLogTool } from "./domains/network";
+import { httpGetTool, networkRequestsTool } from "./domains/network";
+import { snapshotTool } from "./domains/snapshot";
import { executeJsTool, runScriptTool } from "./domains/js";
// Mutation tools that modify page / browser state are marked `serialized: true`
@@ -46,7 +47,8 @@ const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [
{ ...viewportResizeTool, serialized: true },
{ ...dragAndDropTool, serialized: true },
httpGetTool,
- getNetworkLogTool,
+ networkRequestsTool,
+ snapshotTool,
executeJsTool,
runScriptTool,
];