diff options
| author | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-05-06 00:06:49 +0530 |
|---|---|---|
| committer | amankumarsingh77 <amankumarsingh7702@gmal.com> | 2026-05-06 00:06:49 +0530 |
| commit | 589cd294931c01b3748779635d17a13dd240af19 (patch) | |
| tree | b635f9fa443cd120de905660fe0d2665184e9a8c /src | |
| parent | 20ceaa1c6ade4950ed7d16caa5269ad2f4805db4 (diff) | |
release: v0.4.0 — accessibility snapshot, network capture, snapshot-first prompts
Adds two new tools and pivots the agent's default workflow away from
screenshots toward structured page data.
browser_snapshot — CDP accessibility tree (roles, names, states, hierarchy)
with click coordinates @(x,y) attached to every interactive element via
DOM.getBoxModel (parallel, 1.5s aggregate budget). Pass directly to
browser_click — no screenshot round-trip. Optional includeScreenshot:true
for visual confirmation. format:"json" returns a slim structure with full
box per node.
browser_network_requests — replaces the deprecated browser_get_network_log
placeholder. Filters by urlPattern (substring or /regex/), method, status,
resource type, recency. Optional response-body capture under a 5s aggregate
budget with 50KB per-body cap. Buffer is page-scoped (cleared on tab switch),
500 records/tab, with a bufferOverflowed flag on drain.
CDP plumbing: in-process aggregator in src/cdp/network-buffer.ts wired into
the existing single-consumer event loop. Accessibility domain enabled
alongside Page/DOM/Runtime/Network on every attach.
Ctrl+O expand/collapse on three tools' renderResult — browser_snapshot,
browser_network_requests, browser_execute_js. Compact-by-default views with
keyHint("app.tools.expand") that respects user keybinding remaps.
Tool prompts pivoted snapshot-first: browser_screenshot is documented as
LAST RESORT for visual verification only; browser_snapshot is the default
for understanding pages; browser_execute_js is the default for surgical
reads. browser_click and browser_open_urls guidance follows.
SKILL.md collapsed from ~342 to ~53 lines — keeping only what isn't already
in tool prompts: the decision tree, connection rules, browser_run_script
daemon bindings.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Diffstat (limited to 'src')
| -rw-r--r-- | src/cdp/network-buffer.ts | 213 | ||||
| -rw-r--r-- | src/cdp/session.ts | 15 | ||||
| -rw-r--r-- | src/domains/click.ts | 11 | ||||
| -rw-r--r-- | src/domains/js.ts | 71 | ||||
| -rw-r--r-- | src/domains/navigate.ts | 2 | ||||
| -rw-r--r-- | src/domains/network.ts | 211 | ||||
| -rw-r--r-- | src/domains/screenshot.ts | 13 | ||||
| -rw-r--r-- | src/domains/snapshot.ts | 421 | ||||
| -rw-r--r-- | src/registry.ts | 6 |
9 files changed, 923 insertions, 40 deletions
diff --git a/src/cdp/network-buffer.ts b/src/cdp/network-buffer.ts new file mode 100644 index 0000000..f841719 --- /dev/null +++ b/src/cdp/network-buffer.ts @@ -0,0 +1,213 @@ +/** + * In-memory aggregator for CDP Network.* events. + * + * The CDP event stream is consumed by a single internal loop in session.ts. + * That loop hands Network.* event params to this module, which joins them into + * one record per requestId (request → response → finished/failed). Tools then + * read records via drain(), filtered to what the caller asked for. + * + * Bounded by a fixed capacity (default 500): on overflow the oldest record is + * evicted FIFO. The overflow flag is reported once per drain so tools can warn + * the LLM that data may be missing. + * + * This module is pure (no CDP dependency) so the joining/filtering logic stays + * trivially testable in isolation. + */ + +export type NetworkRecord = { + requestId: string; + method: string; + url: string; + status?: number; + statusText?: string; + type: string; + mimeType?: string; + /** Wall-clock ms when requestWillBeSent fired, in Date.now() units. */ + requestStartedMs: number; + durationMs?: number; + requestBodySize: number; + responseBodySize?: number; + failed?: boolean; + errorText?: string; + /** Populated by the tool layer via Network.getResponseBody, never by the aggregator. */ + body?: string | null; +}; + +export type NetworkFilter = { + /** Substring match by default. Wrap in slashes (`/foo/`) for regex. */ + urlPattern?: string; + methodFilter?: string[]; + statusFilter?: { min?: number; max?: number }; + resourceTypes?: string[]; + /** Only requests whose start is at most N ms ago. */ + sinceMs?: number; + /** Cap on the returned records. */ + limit?: number; +}; + +export type DrainResult = { + readonly records: NetworkRecord[]; + /** Total matches before `limit` was applied. */ + readonly total: number; + /** True if the ring buffer evicted at least one record since the last drain. */ + readonly bufferOverflowed: boolean; +}; + +export type NetworkBuffer = { + ingestRequestWillBeSent(p: unknown): void; + ingestResponseReceived(p: unknown): void; + ingestLoadingFinished(p: unknown): void; + ingestLoadingFailed(p: unknown): void; + drain(filter: NetworkFilter): DrainResult; + clear(): void; +}; + +const compileUrlMatcher = (pattern: string): ((url: string) => boolean) => { + if (pattern.length >= 2 && pattern.startsWith("/") && pattern.endsWith("/")) { + try { + const re = new RegExp(pattern.slice(1, -1)); + return (url) => re.test(url); + } catch { + // Fall through to substring match if the pattern is not a valid regex. + } + } + return (url) => url.includes(pattern); +}; + +export const createNetworkBuffer = (capacity = 500): NetworkBuffer => { + // Insertion-ordered map keyed by requestId. Map preserves insertion order in JS, + // so iteration gives us oldest-first eviction without a separate index. + const records = new Map<string, NetworkRecord>(); + let overflowed = false; + + const evictOldestIfFull = (): void => { + while (records.size >= capacity) { + const oldest = records.keys().next(); + if (oldest.done) return; + records.delete(oldest.value); + overflowed = true; + } + }; + + return { + ingestRequestWillBeSent(p) { + // CDP boundary cast: Network.requestWillBeSent + const params = p as + | { + requestId?: string; + request?: { + url?: string; + method?: string; + postData?: string; + headers?: Record<string, string>; + }; + type?: string; + timestamp?: number; + wallTime?: number; + } + | undefined; + const id = params?.requestId; + const url = params?.request?.url; + const method = params?.request?.method; + if (!id || !url || !method) return; + // Re-fired requestWillBeSent for the same id (e.g. redirect) — refresh in place. + records.delete(id); + evictOldestIfFull(); + const postData = params.request?.postData; + records.set(id, { + requestId: id, + method, + url, + type: params.type ?? "Other", + requestStartedMs: Date.now(), + requestBodySize: postData ? Buffer.byteLength(postData, "utf8") : 0, + }); + }, + + ingestResponseReceived(p) { + // CDP boundary cast: Network.responseReceived + const params = p as + | { + requestId?: string; + response?: { + status?: number; + statusText?: string; + mimeType?: string; + }; + type?: string; + } + | undefined; + const id = params?.requestId; + if (!id) return; + const r = records.get(id); + if (!r) return; + if (params?.response?.status !== undefined) r.status = params.response.status; + if (params?.response?.statusText !== undefined && params.response.statusText !== "") r.statusText = params.response.statusText; + if (params?.response?.mimeType !== undefined) r.mimeType = params.response.mimeType; + // type is more accurate on responseReceived (CDP sometimes refines it) + if (params?.type !== undefined) r.type = params.type; + }, + + ingestLoadingFinished(p) { + // CDP boundary cast: Network.loadingFinished + const params = p as + | { requestId?: string; encodedDataLength?: number } + | undefined; + const id = params?.requestId; + if (!id) return; + const r = records.get(id); + if (!r) return; + if (params?.encodedDataLength !== undefined) r.responseBodySize = params.encodedDataLength; + r.durationMs = Date.now() - r.requestStartedMs; + }, + + ingestLoadingFailed(p) { + // CDP boundary cast: Network.loadingFailed + const params = p as + | { requestId?: string; errorText?: string; canceled?: boolean } + | undefined; + const id = params?.requestId; + if (!id) return; + const r = records.get(id); + if (!r) return; + r.failed = true; + if (params?.errorText !== undefined) r.errorText = params.errorText; + r.durationMs = Date.now() - r.requestStartedMs; + }, + + drain(filter) { + const matchUrl = filter.urlPattern !== undefined ? compileUrlMatcher(filter.urlPattern) : undefined; + const methods = filter.methodFilter ? new Set(filter.methodFilter.map((m) => m.toUpperCase())) : undefined; + const types = filter.resourceTypes ? new Set(filter.resourceTypes.map((t) => t.toLowerCase())) : undefined; + const minStatus = filter.statusFilter?.min; + const maxStatus = filter.statusFilter?.max; + const cutoff = filter.sinceMs !== undefined ? Date.now() - filter.sinceMs : undefined; + + const matched: NetworkRecord[] = []; + for (const r of records.values()) { + if (matchUrl && !matchUrl(r.url)) continue; + if (methods && !methods.has(r.method.toUpperCase())) continue; + if (types && !types.has(r.type.toLowerCase())) continue; + if (cutoff !== undefined && r.requestStartedMs < cutoff) continue; + if (minStatus !== undefined && (r.status === undefined || r.status < minStatus)) continue; + if (maxStatus !== undefined && (r.status === undefined || r.status > maxStatus)) continue; + // Clone so callers can mutate (e.g. attach body) without polluting the buffer. + matched.push({ ...r }); + } + + const total = matched.length; + const limit = Math.min(filter.limit ?? 50, 500); + const limited = matched.slice(-limit); // most recent N + + const bufferOverflowed = overflowed; + overflowed = false; + + return { records: limited, total, bufferOverflowed }; + }, + + clear() { + records.clear(); + overflowed = false; + }, + }; +}; diff --git a/src/cdp/session.ts b/src/cdp/session.ts index 42f6107..89bdfdd 100644 --- a/src/cdp/session.ts +++ b/src/cdp/session.ts @@ -3,6 +3,7 @@ import type { CdpError } from "./errors"; import type { DialogInfo } from "./types"; import type { OwnershipRegistry } from "./ownership"; import type { CdpTransport } from "./transport"; +import { createNetworkBuffer, type DrainResult, type NetworkFilter } from "./network-buffer"; export type CdpSession = { attachFirstPage(): Promise<Result<{ readonly targetId: string; readonly sessionId: string }, CdpError>>; @@ -13,6 +14,7 @@ export type CdpSession = { callBrowser(method: string, params?: Record<string, unknown>, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>; takeDialog(): DialogInfo | null; drainPageInfoInvalidations(): boolean; + drainNetworkBuffer(filter: NetworkFilter): DrainResult; }; export const createCdpSession = ( @@ -23,6 +25,7 @@ export const createCdpSession = ( let targetId: string | null = null; let dialog: DialogInfo | null = null; let pageInfoDirty = false; + const networkBuffer = createNetworkBuffer(); let activeConsumer: Promise<void> = Promise.resolve(); @@ -48,6 +51,10 @@ export const createCdpSession = ( const params = ev.params as { targetId?: string } | undefined; if (params?.targetId) ownership.remove(params.targetId); } + if (ev.method === "Network.requestWillBeSent") networkBuffer.ingestRequestWillBeSent(ev.params); + else if (ev.method === "Network.responseReceived") networkBuffer.ingestResponseReceived(ev.params); + else if (ev.method === "Network.loadingFinished") networkBuffer.ingestLoadingFinished(ev.params); + else if (ev.method === "Network.loadingFailed") networkBuffer.ingestLoadingFailed(ev.params); } }; @@ -75,7 +82,7 @@ export const createCdpSession = ( // Switching to Promise.all over a single WS pipelines the round-trips and // saves ~3× on tab-switch latency. Defer until session.ts has tests. const enableDomains = async (sid: string): Promise<void> => { - for (const d of ["Page", "DOM", "Runtime", "Network"]) { + for (const d of ["Page", "DOM", "Runtime", "Network", "Accessibility"]) { await transport.request(`${d}.enable`, {}, { sessionId: sid }); } }; @@ -143,6 +150,9 @@ export const createCdpSession = ( sessionId = a.sessionId; targetId = tid; pageInfoDirty = true; + // Buffer is page-scoped: on switch, the network history of the previous + // tab is no longer relevant to what the agent is now looking at. + networkBuffer.clear(); await enableDomains(a.sessionId); return ok(undefined); }, @@ -168,5 +178,8 @@ export const createCdpSession = ( pageInfoDirty = false; return dirty; }, + drainNetworkBuffer(filter) { + return networkBuffer.drain(filter); + }, }; }; diff --git a/src/domains/click.ts b/src/domains/click.ts index c269dd8..9946664 100644 --- a/src/domains/click.ts +++ b/src/domains/click.ts @@ -41,14 +41,15 @@ export const clickTool = defineBrowserTool({ name: "browser_click", label: "Browser Click", description: - "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Use browser_screenshot first to find coordinates.", + "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Get coordinates from browser_snapshot's @(x,y) hints — do not screenshot to find click targets.", promptSnippet: "Click at pixel coordinates (x, y) on the page", promptGuidelines: [ - "Use browser_click for all clicks. Coordinates are viewport CSS pixels (not device pixels).", - "Capture a browser_screenshot BEFORE clicking to find the right coordinates.", - "Capture another browser_screenshot AFTER clicking to verify the action worked.", + "Get (x, y) from browser_snapshot — the outline shows '@(x,y)' for every interactive element. NO screenshot needed.", + "If the target isn't an interactive role in the snapshot, fall back to browser_execute_js with `el.getBoundingClientRect()` — still NO screenshot.", + "Coordinates are viewport CSS pixels (not device pixels).", + "After clicking, verify with browser_snapshot (or browser_execute_js for a specific value). Only screenshot if you need to confirm visual rendering.", "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.", - "If a click doesn't register, try setting BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions.", + "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions (debug only).", "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.", ], parameters: ClickArgs, diff --git a/src/domains/js.ts b/src/domains/js.ts index bd4c3e1..14862e8 100644 --- a/src/domains/js.ts +++ b/src/domains/js.ts @@ -3,10 +3,20 @@ import { resolve, isAbsolute } from "node:path"; import { tmpdir } from "node:os"; import { createRequire } from "node:module"; import { Type } from "typebox"; +import { Markdown, Text } from "@mariozechner/pi-tui"; +import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { applyTruncation } from "../util/truncate"; +const COMPACT_PREVIEW_BYTES = 120; + +const formatBytes = (n: number): string => { + if (n >= 1_048_576) return `${(n / 1_048_576).toFixed(1)} MB`; + if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${n} B`; +}; + // AsyncFunction constructor — the documented Node.js mechanism for compiling // arbitrary user-supplied source. Equivalent to `new Function` but produces an // async function we can `await`. The full RCE risk is contained at the source @@ -29,11 +39,15 @@ export const executeJsTool = defineBrowserTool({ name: "browser_execute_js", label: "Browser Execute JS", description: - "Run JavaScript in the page (or a specific iframe target). `return X` gets auto-wrapped in an IIFE for convenience. Always use safeJs / JSON.stringify when interpolating untrusted strings into source.", - promptSnippet: "Execute JS in the page (or iframe)", + "Surgical reads from the DOM. The cheapest, most precise way to extract a specific element's text, attribute, value, or geometry. Prefer over browser_screenshot for ANY data extraction or coordinate lookup. `return X` is auto-wrapped in an IIFE. Always use safeJs / JSON.stringify when interpolating untrusted strings.", + promptSnippet: "Execute JS in the page — surgical DOM reads (preferred over screenshot)", promptGuidelines: [ + "DEFAULT for extracting a specific value: `document.querySelector('.price').innerText`, `input.value`, `el.getAttribute('aria-label')`, etc.", + "For click coordinates not already in browser_snapshot: `JSON.stringify(document.querySelector('SELECTOR').getBoundingClientRect())` — beats a screenshot every time.", + "For 'is this element visible / enabled / checked / focused', a one-liner here beats a screenshot every time.", + "DO NOT call browser_screenshot to read a value or check element state. Use this tool.", "`return foo` is auto-wrapped in an IIFE — both `foo` and `(() => foo)()` work.", - "For iframes, get a targetId via browser_execute_js with `Object.values(document.querySelectorAll('iframe'))` then call browser_execute_js with targetId.", + "For iframes, first run `Object.values(document.querySelectorAll('iframe'))` to find the iframe, then pass its targetId.", "Result must be JSON-serializable (Runtime.evaluate returnByValue=true).", ], parameters: ExecuteJsArgs, @@ -42,13 +56,58 @@ export const executeJsTool = defineBrowserTool({ if (!r.success) return err({ kind: "cdp_error", message: r.error.message }); const valueStr = r.data === undefined ? "undefined" : JSON.stringify(r.data); const truncated = await applyTruncation(valueStr, "js"); + // If the value is JSON, pretty-print for the expanded TUI view. Falls back + // to the raw string if parse fails (e.g. value is "undefined" sentinel). + let pretty: string | undefined; + try { + const parsed = JSON.parse(valueStr); + pretty = JSON.stringify(parsed, null, 2); + } catch { + pretty = undefined; + } return ok({ text: truncated.text, - details: truncated.fullOutputPath !== undefined - ? { valueLength: valueStr.length, fullOutputPath: truncated.fullOutputPath } - : { valueLength: valueStr.length }, + details: { + valueLength: valueStr.length, + // Full value for the expanded renderer. Capped at 200 KB so a giant + // dump doesn't bloat the conversation transcript on disk. + full: valueStr.length > 200_000 ? valueStr.slice(0, 200_000) : valueStr, + ...(pretty !== undefined ? { pretty: pretty.length > 200_000 ? pretty.slice(0, 200_000) : pretty } : {}), + ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}), + }, }); }, + + renderResult(result, expanded, theme) { + const details = result.details as + | { valueLength?: number; full?: string; pretty?: string; fullOutputPath?: string } + | undefined; + if (!details) return new Text(theme.fg("error", "execute_js: no details"), 0, 0); + + const len = details.valueLength ?? 0; + const full = details.full ?? ""; + const isJson = details.pretty !== undefined; + + if (!expanded) { + const preview = full.length > COMPACT_PREVIEW_BYTES ? full.slice(0, COMPACT_PREVIEW_BYTES) + "…" : full; + const md = [ + `**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`, + "```", + preview, + "```", + keyHint("app.tools.expand", "to expand"), + ].join("\n"); + return new Markdown(md, 0, 0, getMarkdownTheme()); + } + + const body = details.pretty ?? full; + const fence = isJson ? "```json" : "```"; + const tail = details.fullOutputPath + ? `\n\nFull value at \`${details.fullOutputPath}\` · ${keyHint("app.tools.expand", "to collapse")}` + : `\n\n${keyHint("app.tools.expand", "to collapse")}`; + const md = [`**${formatBytes(len)}** ${isJson ? "JSON" : "value"}`, fence, body, "```", tail].join("\n"); + return new Markdown(md, 0, 0, getMarkdownTheme()); + }, }); const RunScriptArgs = Type.Object({ diff --git a/src/domains/navigate.ts b/src/domains/navigate.ts index 6016122..7012d15 100644 --- a/src/domains/navigate.ts +++ b/src/domains/navigate.ts @@ -75,7 +75,7 @@ export const openUrlsTool = defineBrowserTool({ promptSnippet: "Open multiple URLs in new tabs (parallel)", promptGuidelines: [ "Use after web_search to open citations in parallel.", - "After opening, use browser_list_tabs / browser_switch_tab / browser_screenshot to interact.", + "After opening, use browser_list_tabs / browser_switch_tab to navigate, then browser_snapshot to inspect each tab.", "Use browser_wait_for_load on a tab before extracting data from SPAs.", ], parameters: OpenUrlsArgs, diff --git a/src/domains/network.ts b/src/domains/network.ts index 1377486..f13b1e7 100644 --- a/src/domains/network.ts +++ b/src/domains/network.ts @@ -1,7 +1,10 @@ import { Type } from "typebox"; +import { Markdown, Text } from "@mariozechner/pi-tui"; +import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent"; import { type Result, err, ok } from "../util/result"; import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; import { applyTruncation } from "../util/truncate"; +import type { NetworkRecord } from "../cdp/network-buffer"; const HttpGetArgs = Type.Object({ url: Type.String({ description: "URL to GET" }), @@ -49,32 +52,200 @@ export const httpGetTool = defineBrowserTool({ }, }); -const NetLogArgs = Type.Object({ - eventTypes: Type.Optional( - Type.Array(Type.String(), { description: "CDP event names to filter (e.g., ['Network.requestWillBeSent'])" }), +const NetworkRequestsArgs = Type.Object({ + urlPattern: Type.Optional( + Type.String({ description: "Substring match by default. Wrap in slashes (e.g. /api\\\\.example/) for regex." }), + ), + methodFilter: Type.Optional( + Type.Array(Type.String(), { description: 'HTTP methods to include, e.g. ["GET","POST"].' }), + ), + statusFilter: Type.Optional( + Type.Object( + { + min: Type.Optional(Type.Integer({ minimum: 100, maximum: 599 })), + max: Type.Optional(Type.Integer({ minimum: 100, maximum: 599 })), + }, + { description: "Inclusive HTTP status range." }, + ), + ), + resourceTypes: Type.Optional( + Type.Array(Type.String(), { description: 'CDP resource types: "Document","Stylesheet","Image","Media","Font","Script","TextTrack","XHR","Fetch","EventSource","WebSocket","Manifest","SignedExchange","Ping","CSPViolationReport","Preflight","Other".' }), + ), + sinceMs: Type.Optional( + Type.Integer({ minimum: 0, description: "Only requests started in the last N ms." }), + ), + limit: Type.Optional( + Type.Integer({ default: 50, minimum: 1, maximum: 500, description: "Max records to return." }), + ), + includeResponseBodies: Type.Optional( + Type.Boolean({ default: false, description: "Fetch response bodies via Network.getResponseBody. Costs one CDP call per matched request." }), ), - limit: Type.Optional(Type.Integer({ default: 50, minimum: 1, maximum: 500 })), }); -export const getNetworkLogTool = defineBrowserTool({ - name: "browser_get_network_log", - label: "Browser Get Network Log", +const BODY_BUDGET_MS = 5_000; +const PER_BODY_CAP = 50_000; + +const renderNetworkMarkdown = ( + records: ReadonlyArray<NetworkRecord>, + opts: { includeBodies: boolean; total: number; bufferOverflowed: boolean; expanded: boolean }, +): string => { + if (records.length === 0) { + if (opts.bufferOverflowed) return "_No matching requests. (Buffer overflowed since last drain — some events were dropped.)_"; + return "_No matching requests captured yet._"; + } + + const failed = records.filter((r) => r.failed || (r.status !== undefined && r.status >= 500)).length; + const totalBytes = records.reduce((s, r) => s + (r.responseBodySize ?? 0), 0); + const formatBytes = (n: number): string => { + if (n >= 1_048_576) return `${(n / 1_048_576).toFixed(1)} MB`; + if (n >= 1024) return `${(n / 1024).toFixed(1)} KB`; + return `${n} B`; + }; + const showWindow = (() => { + const finished = records.filter((r) => r.durationMs !== undefined); + if (finished.length === 0) return ""; + const span = Math.max(...records.map((r) => r.requestStartedMs)) - Math.min(...records.map((r) => r.requestStartedMs)); + return ` · ${(span / 1000).toFixed(1)}s window`; + })(); + + const headerLines: string[] = [ + `**${records.length} requests** · ${failed} failed · ${formatBytes(totalBytes)}${showWindow}`, + ]; + if (opts.bufferOverflowed) headerLines.push("⚠ buffer overflowed since last drain — older events dropped"); + if (opts.total > records.length) headerLines.push(`(${records.length} of ${opts.total} matches shown — increase \`limit\` for more)`); + + const compactRows = (records.length > 5 && !opts.expanded ? records.slice(0, 5) : records).map((r) => { + const status = r.failed ? "ERR" : r.status !== undefined ? String(r.status) : "—"; + const warn = r.failed || (r.status !== undefined && r.status >= 500) ? " ⚠" : ""; + const ms = r.durationMs !== undefined ? `${r.durationMs} ms` : "—"; + const url = r.url.length > 60 ? r.url.slice(0, 57) + "…" : r.url; + return `\`${r.method.padEnd(4)} ${status.padStart(3)}\` ${url} _${ms}_${warn}`; + }); + + const rows = compactRows.join("\n"); + + if (!opts.expanded) { + const more = records.length > 5 ? `\n … ${records.length - 5} more` : ""; + return `${headerLines.join("\n")}\n\n${rows}${more}`; + } + + // Expanded: full table + optional body sections. + const tableHeader = `| # | Method | Status | URL | Type | ms |\n|---|---|---|---|---|---|`; + const tableRows = records + .map((r, i) => { + const status = r.failed ? "ERR" : r.status !== undefined ? String(r.status) : "—"; + const ms = r.durationMs !== undefined ? String(r.durationMs) : "—"; + const url = r.url.length > 80 ? r.url.slice(0, 77) + "…" : r.url; + return `| ${i + 1} | ${r.method} | ${status} | \`${url}\` | ${r.type} | ${ms} |`; + }) + .join("\n"); + + let bodySections = ""; + if (opts.includeBodies) { + const sections: string[] = []; + records.forEach((r, i) => { + if (r.body === undefined) return; + const sizes = `${r.requestBodySize} B → ${r.responseBodySize ?? 0} B`; + const status = r.failed ? "failed" : r.status ?? "?"; + const header = `── #${i + 1} ${r.method} ${r.url} (${status}, ${r.durationMs ?? "?"} ms, ${r.type}, ${sizes}) ──`; + const bodyText = r.body === null ? "_(body unavailable — already GC'd by Chrome)_" : "```\n" + r.body + "\n```"; + sections.push(`${header}\n\n${bodyText}`); + }); + if (sections.length > 0) bodySections = "\n\n" + sections.join("\n\n"); + } + + return [headerLines.join("\n"), "", tableHeader, tableRows, bodySections].filter((s) => s !== "").join("\n"); +}; + +type NetworkDetails = { + total: number; + returned: number; + bufferOverflowed: boolean; + truncated: boolean; + fullOutputPath?: string; + requests: ReadonlyArray<NetworkRecord>; + includeBodies: boolean; +}; + +export const networkRequestsTool = defineBrowserTool({ + name: "browser_network_requests", + label: "Browser Network Requests", description: - "Read buffered Network.* CDP events. NOTE: in v0.3 the event stream is consumed internally by the session manager; this tool is a placeholder. Use browser_execute_js with PerformanceObserver as a workaround.", - promptSnippet: "Get buffered network events (DEPRECATED in v0.3 — use browser_execute_js with PerformanceObserver instead)", + "List recent network requests captured on the current tab since attach. Filter by URL pattern, method, status, resource type, or recency. Optionally include response bodies.", + promptSnippet: "Inspect recent network requests on the current page", promptGuidelines: [ - "v0.3 routes CDP events through an internal AsyncIterable; a synchronous drain API isn't exposed yet.", - "For now, use browser_execute_js with PerformanceObserver, performance.getEntries(), or fetch interceptors.", + "Buffer is reset on tab switch — only requests on the current tab since attach are visible.", + 'urlPattern is a substring; wrap in slashes for regex (e.g. "/\\\\.json$/").', + "includeResponseBodies:true costs an extra CDP call per matched request — use only when you need payloads.", + "If bufferOverflowed:true in the result, older events were dropped (capacity 500 records/tab).", ], - parameters: NetLogArgs, - async handler(args): Promise<Result<ToolOk, ToolErr>> { - return ok({ - text: "browser_get_network_log: in v0.3 the CDP event stream is consumed by the session; use browser_execute_js with PerformanceObserver or performance.getEntries() to inspect network activity.", - details: { - deprecated: true, - eventTypes: args.eventTypes ?? null, - limit: args.limit ?? null, - }, + parameters: NetworkRequestsArgs, + + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const session = client.session(); + const drained = session.drainNetworkBuffer({ + ...(args.urlPattern !== undefined ? { urlPattern: args.urlPattern } : {}), + ...(args.methodFilter !== undefined ? { methodFilter: args.methodFilter } : {}), + ...(args.statusFilter !== undefined ? { statusFilter: args.statusFilter } : {}), + ...(args.resourceTypes !== undefined ? { resourceTypes: args.resourceTypes } : {}), + ...(args.sinceMs !== undefined ? { sinceMs: args.sinceMs } : {}), + limit: args.limit ?? 50, }); + + if (args.includeResponseBodies) { + const ac = new AbortController(); + const timer = setTimeout(() => ac.abort(), BODY_BUDGET_MS); + const fills = drained.records.map(async (r) => { + if (ac.signal.aborted) return; + const bodyRes = await session.call("Network.getResponseBody", { requestId: r.requestId }); + if (!bodyRes.success) { + r.body = null; + return; + } + const b = bodyRes.data as { body: string; base64Encoded?: boolean }; + let body = b.base64Encoded ? Buffer.from(b.body, "base64").toString("utf8") : b.body; + if (body.length > PER_BODY_CAP) body = body.slice(0, PER_BODY_CAP) + "…"; + r.body = body; + }); + await Promise.allSettled(fills); + clearTimeout(timer); + } + + const text = renderNetworkMarkdown(drained.records, { + includeBodies: args.includeResponseBodies ?? false, + total: drained.total, + bufferOverflowed: drained.bufferOverflowed, + expanded: true, // full text always returned to the LLM; renderResult handles compact view for humans + }); + const trunc = await applyTruncation(text, "network"); + + const details: NetworkDetails = { + total: drained.total, + returned: drained.records.length, + bufferOverflowed: drained.bufferOverflowed, + truncated: trunc.wasTruncated, + requests: drained.records, + includeBodies: args.includeResponseBodies ?? false, + }; + if (trunc.fullOutputPath !== undefined) details.fullOutputPath = trunc.fullOutputPath; + + return ok({ text: trunc.text, details }); + }, + + renderResult(result, expanded, theme) { + const details = result.details as NetworkDetails | undefined; + if (!details) return new Text(theme.fg("error", "network: no details"), 0, 0); + + const md = renderNetworkMarkdown(details.requests, { + includeBodies: details.includeBodies && expanded, + total: details.total, + bufferOverflowed: details.bufferOverflowed, + expanded, + }); + const tail = expanded + ? `\n\n${keyHint("app.tools.expand", "to collapse")}${details.fullOutputPath ? ` · full payload at \`${details.fullOutputPath}\`` : ""}` + : `\n\n${keyHint("app.tools.expand", "to expand")}`; + + return new Markdown(md + tail, 0, 0, getMarkdownTheme()); }, }); diff --git a/src/domains/screenshot.ts b/src/domains/screenshot.ts index 74042be..9d6313b 100644 --- a/src/domains/screenshot.ts +++ b/src/domains/screenshot.ts @@ -63,12 +63,15 @@ const resizeIfNeeded = async ( export const screenshotTool = defineBrowserTool({ name: "browser_screenshot", label: "Browser Screenshot", - description: "Capture the current page as PNG or JPEG. JPEG is 2-5x smaller — prefer it for photo-heavy pages.", - promptSnippet: "Capture a screenshot of the current page", + description: + "Capture the page as a JPEG/PNG image. NOT a default exploration tool. Use ONLY when (a) you need to verify visual properties (layout, color, spacing, rendered text legibility), or (b) browser_snapshot and browser_execute_js cannot answer your question. For understanding what's on the page use browser_snapshot. For specific element values use browser_execute_js. For click coordinates use the @(x,y) hints in browser_snapshot's outline.", + promptSnippet: "Capture a screenshot — visual verification only", promptGuidelines: [ - "Use BEFORE clicks/scrolls to find coordinates; AFTER to verify the action.", - "Pass format='jpeg' with a quality (60-90) for smaller files on photo-heavy pages.", - "Set maxDim if the page is huge and you want to fit under LLM image-size limits.", + "DO NOT use as a default exploration tool. browser_snapshot is the default for understanding page structure — far cheaper and more reliable.", + "DO NOT use to find click coordinates. browser_snapshot already includes @(x,y) for every interactive element. If a target isn't there, use browser_execute_js with element.getBoundingClientRect().", + "DO use to verify visual rendering after an action: did the modal animate in correctly, did the chart render, are colors right, did the layout reflow as expected.", + "DO use as a debugging fallback when browser_execute_js or browser_snapshot return surprising results and you suspect the rendered page differs from the DOM.", + "format='jpeg' with quality 60-90 keeps screenshots small for photo-heavy pages. Use maxDim if the page is huge.", ], parameters: ScreenshotArgs, async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { diff --git a/src/domains/snapshot.ts b/src/domains/snapshot.ts new file mode 100644 index 0000000..39c493a --- /dev/null +++ b/src/domains/snapshot.ts @@ -0,0 +1,421 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { Type } from "typebox"; +import { Container, Image, type ImageTheme, Markdown, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui"; +import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent"; +import { type Result, err, ok } from "../util/result"; +import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool"; +import { applyTruncation } from "../util/truncate"; +import { screenshotPath } from "../util/paths"; + +const SnapshotArgs = Type.Object({ + includeScreenshot: Type.Optional( + Type.Boolean({ default: false, description: "Also capture a JPEG screenshot of the current viewport." }), + ), + interestingOnly: Type.Optional( + Type.Boolean({ default: true, description: "Drop nodes the AX engine marked uninteresting (generic containers, inline text, etc.)." }), + ), + maxNodes: Type.Optional( + Type.Integer({ default: 1000, minimum: 1, maximum: 5000, description: "Cap on slim nodes returned." }), + ), + format: Type.Optional( + Type.Union( + [Type.Literal("outline"), Type.Literal("json")], + { default: "outline", description: "'outline' = indented markdown bullet tree; 'json' = raw slim structure." }, + ), + ), +}); + +// Raw shapes from CDP Accessibility.getFullAXTree. Cast at this boundary only. +type AxValue = { value?: unknown }; +type RawAxNode = { + nodeId: string; + parentId?: string; + childIds?: ReadonlyArray<string>; + ignored?: boolean; + role?: AxValue; + name?: AxValue; + value?: AxValue; + description?: AxValue; + properties?: ReadonlyArray<{ name?: string; value?: AxValue }>; + backendDOMNodeId?: number; +}; + +type Box = { x: number; y: number; width: number; height: number; cx: number; cy: number }; + +type SlimNode = { + role: string; + name?: string; + value?: string; + description?: string; + state?: string; + children: SlimNode[]; + /** Internal: kept for the post-build box fetch. Stripped before returning to the LLM. */ + _backendId?: number; + /** Click target — center of the element's bounding box, in viewport CSS pixels. */ + box?: Box; +}; + +const INTERACTIVE_ROLES = new Set([ + "button", + "link", + "textbox", + "checkbox", + "radio", + "combobox", + "menuitem", + "tab", + "switch", + "slider", + "searchbox", + "spinbutton", +]); + +const stringOf = (v: AxValue | undefined): string | undefined => { + const x = v?.value; + if (typeof x === "string" && x !== "") return x; + if (typeof x === "number") return String(x); + return undefined; +}; + +const collectState = (props: RawAxNode["properties"]): string | undefined => { + if (!props || props.length === 0) return undefined; + const flags: string[] = []; + for (const p of props) { + const name = p.name; + if (!name) continue; + const raw = p.value?.value; + // Boolean flags worth surfacing in the outline. + if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) { + flags.push(name); + } + if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`); + } + return flags.length > 0 ? flags.join(", ") : undefined; +}; + +const buildTree = ( + rawNodes: ReadonlyArray<RawAxNode>, + opts: { interestingOnly: boolean; maxNodes: number }, +): SlimNode[] => { + // Index for parent/child lookups. + const byId = new Map<string, RawAxNode>(); + for (const n of rawNodes) byId.set(n.nodeId, n); + + // CDP doesn't mark a single root explicitly; the root nodes are the ones + // whose parentId isn't in the index (typically just one — RootWebArea). + const rootIds: string[] = []; + for (const n of rawNodes) { + if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId); + } + + let budget = opts.maxNodes; + + const slim = (node: RawAxNode): SlimNode | undefined => { + if (budget <= 0) return undefined; + if (opts.interestingOnly && node.ignored) { + // Skip the ignored node but recurse through its children — Chrome often + // ignores wrapper divs but the meaningful descendants are still there. + const out: SlimNode[] = []; + for (const cid of node.childIds ?? []) { + const child = byId.get(cid); + if (!child) continue; + const slimChild = slim(child); + if (slimChild) out.push(slimChild); + } + // Hoist children into the parent. We return undefined; the parent's + // recursion will see them by re-walking — but that double-counts. Instead + // we handle hoisting at the parent level. So here just signal "skip me" + // by returning undefined; the parent will hoist by checking ignored flag. + return out.length > 0 ? { role: "_hoist", children: out } : undefined; + } + budget--; + const role = stringOf(node.role) ?? "unknown"; + const name = stringOf(node.name); + const value = stringOf(node.value); + const description = stringOf(node.description); + const state = collectState(node.properties); + const children: SlimNode[] = []; + for (const cid of node.childIds ?? []) { + const child = byId.get(cid); + if (!child) continue; + const slimChild = slim(child); + if (!slimChild) continue; + // Hoist any synthetic _hoist nodes' children into this level. + if (slimChild.role === "_hoist") children.push(...slimChild.children); + else children.push(slimChild); + } + const out: SlimNode = { role, children }; + if (name !== undefined) out.name = name; + if (value !== undefined) out.value = value; + if (description !== undefined) out.description = description; + if (state !== undefined) out.state = state; + if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId; + return out; + }; + + const result: SlimNode[] = []; + for (const rid of rootIds) { + const root = byId.get(rid); + if (!root) continue; + const s = slim(root); + if (!s) continue; + if (s.role === "_hoist") result.push(...s.children); + else result.push(s); + } + return result; +}; + +const countNodes = (nodes: ReadonlyArray<SlimNode>): number => { + let n = 0; + for (const node of nodes) n += 1 + countNodes(node.children); + return n; +}; + +const summarize = (nodes: ReadonlyArray<SlimNode>): string => { + const counts = new Map<string, number>(); + const walk = (ns: ReadonlyArray<SlimNode>): void => { + for (const n of ns) { + counts.set(n.role, (counts.get(n.role) ?? 0) + 1); + walk(n.children); + } + }; + walk(nodes); + // Group landmarks (roles that are top-level structural). + const landmarkRoles = new Set(["banner", "navigation", "main", "complementary", "contentinfo", "search", "form", "region"]); + let landmarks = 0; + for (const [role, c] of counts) if (landmarkRoles.has(role)) landmarks += c; + const buttons = counts.get("button") ?? 0; + const inputs = (counts.get("textbox") ?? 0) + (counts.get("combobox") ?? 0) + (counts.get("checkbox") ?? 0); + const links = counts.get("link") ?? 0; + const parts: string[] = []; + if (landmarks > 0) parts.push(`${landmarks} landmark${landmarks === 1 ? "" : "s"}`); + if (buttons > 0) parts.push(`${buttons} button${buttons === 1 ? "" : "s"}`); + if (inputs > 0) parts.push(`${inputs} input${inputs === 1 ? "" : "s"}`); + if (links > 0) parts.push(`${links} link${links === 1 ? "" : "s"}`); + return parts.join(" · "); +}; + +const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => { + const lines: string[] = []; + const walk = (ns: ReadonlyArray<SlimNode>, depth: number): void => { + for (const n of ns) { + const indent = " ".repeat(depth); + let line = `${indent}- ${n.role}`; + if (n.name) line += ` "${n.name}"`; + if (n.value && n.value !== n.name) line += ` = ${JSON.stringify(n.value)}`; + if (n.state) line += ` (${n.state})`; + // Surface click coordinates for interactive nodes — agents can pass these + // straight to browser_click without a screenshot round-trip. + if (n.box && INTERACTIVE_ROLES.has(n.role)) { + line += ` @(${n.box.cx},${n.box.cy})`; + } + lines.push(line); + walk(n.children, depth + 1); + } + }; + walk(nodes, 0); + return lines.join("\n"); +}; + +/** Strip internal _backendId from a slim tree before returning to callers. */ +const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] => + nodes.map((n) => { + const { _backendId: _bid, ...rest } = n; + return { ...rest, children: stripInternals(n.children) }; + }); + +/** + * Walk all slim nodes and collect (slimNode, backendId) pairs for interactive + * roles. Caller fetches DOM.getBoxModel for each and writes box back into the + * slim node by reference. + */ +const collectInteractiveTargets = ( + nodes: ReadonlyArray<SlimNode>, +): Array<{ node: SlimNode; backendId: number }> => { + const out: Array<{ node: SlimNode; backendId: number }> = []; + const walk = (ns: ReadonlyArray<SlimNode>): void => { + for (const n of ns) { + if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) { + out.push({ node: n, backendId: n._backendId }); + } + walk(n.children); + } + }; + walk(nodes); + return out; +}; + +type SnapshotDetails = { + nodeCount: number; + truncated: boolean; + fullOutputPath?: string; + screenshotPath?: string; + url: string; + title: string; +}; + +export const snapshotTool = defineBrowserTool({ + name: "browser_snapshot", + label: "Browser Snapshot", + description: + "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy) plus click coordinates for every interactive element as @(x,y). Use this BEFORE deciding whether you need a screenshot — almost always sufficient on its own and far cheaper. Pair with browser_execute_js for surgical reads of specific element values.", + promptSnippet: "Get accessibility-tree snapshot of the current page (default for page inspection)", + promptGuidelines: [ + "DEFAULT — use this whenever you need to know what's on a page, what's clickable, or how the page is structured.", + "Click coordinates come for free: every interactive element shows '@(x,y)' in the outline. Pass these straight to browser_click. NO screenshot round-trip needed.", + "DO NOT call browser_screenshot just to understand the page. This tool already gives you structure, labels, states, and click targets.", + "Pass includeScreenshot:true ONLY if you also need to verify visual rendering (rare).", + "format:'json' returns the raw slim structure (with `box` per node) for programmatic use; default 'outline' is human/LLM-readable.", + ], + parameters: SnapshotArgs, + + async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> { + const session = client.session(); + + // 1. AX tree + const axRes = await session.call("Accessibility.getFullAXTree", {}); + if (!axRes.success) return err({ kind: "cdp_error", message: axRes.error.message }); + const rawNodes = (axRes.data as { nodes: ReadonlyArray<RawAxNode> }).nodes; + + // 2. Page url/title + const piRes = await client.pageInfo(); + if (!piRes.success) return err({ kind: "cdp_error", message: piRes.error.message }); + const pageUrl = "dialog" in piRes.data ? "" : piRes.data.url; + const pageTitle = "dialog" in piRes.data ? "" : piRes.data.title; + + // 3. Slim + format + const slim = buildTree(rawNodes, { + interestingOnly: args.interestingOnly ?? true, + maxNodes: args.maxNodes ?? 1000, + }); + + // 4. Fetch bounding boxes for interactive nodes so the agent can click + // without a screenshot round-trip. Capped budget so a slow page can't + // wedge the call. + const targets = collectInteractiveTargets(slim); + if (targets.length > 0) { + const ac = new AbortController(); + const timer = setTimeout(() => ac.abort(), 1_500); + await Promise.allSettled( + targets.map(async ({ node, backendId }) => { + if (ac.signal.aborted) return; + const r = await session.call("DOM.getBoxModel", { backendNodeId: backendId }); + if (!r.success) return; + // CDP boundary cast: DOM.getBoxModel + const data = r.data as { model?: { content?: ReadonlyArray<number>; width?: number; height?: number } }; + const content = data.model?.content; + if (!content || content.length < 8) return; + // content is [x1,y1, x2,y1, x2,y2, x1,y2] in viewport CSS pixels. + const x = content[0]!; + const y = content[1]!; + const width = data.model!.width ?? 0; + const height = data.model!.height ?? 0; + // Skip zero-size and off-viewport boxes — they can't be clicked. + if (width <= 0 || height <= 0 || x < 0 || y < 0) return; + node.box = { + x: Math.round(x), + y: Math.round(y), + width: Math.round(width), + height: Math.round(height), + cx: Math.round(x + width / 2), + cy: Math.round(y + height / 2), + }; + }), + ); + clearTimeout(timer); + } + + const stripped = stripInternals(slim); + const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripped, null, 2); + const trunc = await applyTruncation(text, "snapshot"); + + // 4. Optional screenshot — JPEG q=80 to match screenshotTool's default. + let shotPath: string | undefined; + if (args.includeScreenshot) { + const shot = await session.call("Page.captureScreenshot", { format: "jpeg", quality: 80 }); + if (shot.success) { + const data = (shot.data as { data: string }).data; + shotPath = screenshotPath(client.namespace, "jpeg"); + writeFileSync(shotPath, Buffer.from(data, "base64")); + } + } + + const details: SnapshotDetails = { + nodeCount: countNodes(slim), + truncated: trunc.wasTruncated, + url: pageUrl, + title: pageTitle, + }; + if (trunc.fullOutputPath !== undefined) details.fullOutputPath = trunc.fullOutputPath; + if (shotPath !== undefined) details.screenshotPath = shotPath; + + return ok({ text: trunc.text, details: { ...details, summary: summarize(slim) } }); + }, + + renderResult(result, expanded, theme) { + const details = result.details as + | (SnapshotDetails & { summary?: string; ok?: boolean }) + | undefined; + if (!details) return new Text(theme.fg("error", "snapshot: no details"), 0, 0); + + const titleLine = details.title ? ` "${details.title}"` : ""; + const summary = details.summary ?? ""; + const screenshotNote = details.screenshotPath ? "screenshot attached" : "screenshot omitted"; + const truncNote = details.truncated && details.fullOutputPath ? `\n\nFull tree at \`${details.fullOutputPath}\`` : ""; + + if (!expanded) { + const md = [ + `**AX tree:** ${details.nodeCount} nodes · \`${details.url || "(no url)"}\``, + `${titleLine ? " " + titleLine : ""}`, + summary ? ` • ${summary}` : "", + ` ${keyHint("app.tools.expand", "to expand")} · ${screenshotNote}`, + ].filter((l) => l !== "").join("\n"); + return new Markdown(md, 0, 0, getMarkdownTheme()); + } + + // Expanded view: read truncated text from result.content (the spilled file + // is too large to inline) — the same `text` we returned from handler is in + // result.content[0]. Fall back to a placeholder if shape is unexpected. + const content = result.content[0]; + const treeText = content && content.type === "text" ? content.text : ""; + + const headerMd = [ + `**AX tree:** ${details.nodeCount} nodes · \`${details.url || "(no url)"}\``, + titleLine ? ` ${titleLine}` : "", + "", + "```", + treeText, + "```", + truncNote, + "", + `${keyHint("app.tools.expand", "to collapse")}`, + ].filter((l) => l !== "").join("\n"); + + const treeBlock = new Markdown(headerMd, 0, 0, getMarkdownTheme()); + + if (!details.screenshotPath) return treeBlock; + + // Mirror screenshotTool's width-clamp wrapper for the inline image. + try { + const buf = readFileSync(details.screenshotPath); + const b64 = buf.toString("base64"); + const imageTheme: ImageTheme = { fallbackColor: (s: string) => theme.fg("dim", s) }; + const image = new Image(b64, "image/jpeg", imageTheme, { + maxWidthCells: 80, + maxHeightCells: 24, + filename: details.screenshotPath, + }); + const safeImage = { + invalidate: () => image.invalidate(), + render: (width: number) => + image.render(width).map((line) => (visibleWidth(line) > width ? truncateToWidth(line, width) : line)), + }; + const container = new Container(); + container.addChild(treeBlock); + container.addChild(safeImage); + return container; + } catch { + return treeBlock; + } + }, +}); diff --git a/src/registry.ts b/src/registry.ts index d205bf6..df912b1 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -13,7 +13,8 @@ import { listTabsTool, currentTabTool, switchTabTool, newTabTool, closeTabTool } import { uploadFileTool, downloadTool, printToPdfTool } from "./domains/files"; import { viewportResizeTool } from "./domains/viewport"; import { dragAndDropTool } from "./domains/drag"; -import { httpGetTool, getNetworkLogTool } from "./domains/network"; +import { httpGetTool, networkRequestsTool } from "./domains/network"; +import { snapshotTool } from "./domains/snapshot"; import { executeJsTool, runScriptTool } from "./domains/js"; // Mutation tools that modify page / browser state are marked `serialized: true` @@ -46,7 +47,8 @@ const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [ { ...viewportResizeTool, serialized: true }, { ...dragAndDropTool, serialized: true }, httpGetTool, - getNetworkLogTool, + networkRequestsTool, + snapshotTool, executeJsTool, runScriptTool, ]; |
