aboutsummaryrefslogtreecommitdiff
path: root/src/domains/snapshot.ts
blob: 8665e0b501f40eaf141c3538151d645c42a6106a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
import { readFileSync, writeFileSync } from "node:fs";
import { Type } from "typebox";
import { Container, Image, type ImageTheme, Markdown, Text, truncateToWidth, visibleWidth } from "@mariozechner/pi-tui";
import { getMarkdownTheme, keyHint } from "@mariozechner/pi-coding-agent";
import type { CdpSession } from "../cdp/session";
import { type Result, err, ok } from "../util/result";
import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
import { applyTruncation } from "../util/truncate";
import { screenshotPath } from "../util/paths";

const SnapshotArgs = Type.Object({
  includeScreenshot: Type.Optional(
    Type.Boolean({ default: false, description: "Also capture a JPEG screenshot of the current viewport." }),
  ),
  interestingOnly: Type.Optional(
    Type.Boolean({ default: true, description: "Drop nodes the AX engine marked uninteresting (generic containers, inline text, etc.)." }),
  ),
  maxNodes: Type.Optional(
    Type.Integer({ default: 1000, minimum: 1, maximum: 5000, description: "Cap on slim nodes returned." }),
  ),
  format: Type.Optional(
    Type.Union(
      [Type.Literal("outline"), Type.Literal("json")],
      { default: "outline", description: "'outline' = indented markdown bullet tree; 'json' = raw slim structure." },
    ),
  ),
});

// Raw shapes from CDP Accessibility.getFullAXTree. Cast at this boundary only.
type AxValue = { value?: unknown };
export type RawAxNode = {
  nodeId: string;
  parentId?: string;
  childIds?: ReadonlyArray<string>;
  ignored?: boolean;
  role?: AxValue;
  name?: AxValue;
  value?: AxValue;
  description?: AxValue;
  properties?: ReadonlyArray<{ name?: string; value?: AxValue }>;
  backendDOMNodeId?: number;
};

type Box = { x: number; y: number; width: number; height: number; cx: number; cy: number };

export type SlimNode = {
  role: string;
  name?: string;
  value?: string;
  description?: string;
  state?: string;
  children: SlimNode[];
  /** Internal: kept for the post-build box fetch. Stripped before returning to the LLM. */
  _backendId?: number;
  /** Stable handle (e.g. "e12") for interaction tools. Survives re-renders via backendNodeId. */
  ref?: string;
  /** Click target — center of the element's bounding box, in viewport CSS pixels. */
  box?: Box;
};

const INTERACTIVE_ROLES = new Set([
  "button",
  "link",
  "textbox",
  "checkbox",
  "radio",
  "combobox",
  "menuitem",
  "tab",
  "switch",
  "slider",
  "searchbox",
  "spinbutton",
]);

// Roles whose live DOM value/checked state should override the (often stale)
// AX-tree value. The AX `value` is unreliable for freshly-typed or controlled
// inputs, so for these we read the real DOM property at snapshot time.
const VALUE_ROLES = new Set([
  "textbox",
  "searchbox",
  "spinbutton",
  "combobox",
  "checkbox",
  "radio",
  "switch",
  "slider",
]);

const stringOf = (v: AxValue | undefined): string | undefined => {
  const x = v?.value;
  if (typeof x === "string" && x !== "") return x;
  if (typeof x === "number") return String(x);
  return undefined;
};

const collectState = (props: RawAxNode["properties"]): string | undefined => {
  if (!props || props.length === 0) return undefined;
  const flags: string[] = [];
  for (const p of props) {
    const name = p.name;
    if (!name) continue;
    const raw = p.value?.value;
    // Boolean flags worth surfacing in the outline.
    if (raw === true && (name === "focused" || name === "required" || name === "disabled" || name === "checked" || name === "expanded" || name === "selected" || name === "pressed" || name === "modal")) {
      flags.push(name);
    }
    if (typeof raw === "number" && name === "level") flags.push(`level ${raw}`);
  }
  return flags.length > 0 ? flags.join(", ") : undefined;
};

export const buildTree = (
  rawNodes: ReadonlyArray<RawAxNode>,
  opts: { interestingOnly: boolean; maxNodes: number },
): SlimNode[] => {
  // Index for parent/child lookups.
  const byId = new Map<string, RawAxNode>();
  for (const n of rawNodes) byId.set(n.nodeId, n);

  // CDP doesn't mark a single root explicitly; the root nodes are the ones
  // whose parentId isn't in the index (typically just one — RootWebArea).
  const rootIds: string[] = [];
  for (const n of rawNodes) {
    if (!n.parentId || !byId.has(n.parentId)) rootIds.push(n.nodeId);
  }

  let budget = opts.maxNodes;

  const slim = (node: RawAxNode): SlimNode | undefined => {
    if (budget <= 0) return undefined;
    if (opts.interestingOnly && node.ignored) {
      // Skip the ignored node but recurse through its children — Chrome often
      // ignores wrapper divs but the meaningful descendants are still there.
      const out: SlimNode[] = [];
      for (const cid of node.childIds ?? []) {
        const child = byId.get(cid);
        if (!child) continue;
        const slimChild = slim(child);
        if (slimChild) out.push(slimChild);
      }
      // Hoist children into the parent. We return undefined; the parent's
      // recursion will see them by re-walking — but that double-counts. Instead
      // we handle hoisting at the parent level. So here just signal "skip me"
      // by returning undefined; the parent will hoist by checking ignored flag.
      return out.length > 0 ? { role: "_hoist", children: out } : undefined;
    }
    budget--;
    const role = stringOf(node.role) ?? "unknown";
    const name = stringOf(node.name);
    const value = stringOf(node.value);
    const description = stringOf(node.description);
    const state = collectState(node.properties);
    const children: SlimNode[] = [];
    for (const cid of node.childIds ?? []) {
      const child = byId.get(cid);
      if (!child) continue;
      const slimChild = slim(child);
      if (!slimChild) continue;
      // Hoist any synthetic _hoist nodes' children into this level.
      if (slimChild.role === "_hoist") children.push(...slimChild.children);
      else children.push(slimChild);
    }
    const out: SlimNode = { role, children };
    if (name !== undefined) out.name = name;
    if (value !== undefined) out.value = value;
    if (description !== undefined) out.description = description;
    if (state !== undefined) out.state = state;
    if (node.backendDOMNodeId !== undefined) out._backendId = node.backendDOMNodeId;
    return out;
  };

  const result: SlimNode[] = [];
  for (const rid of rootIds) {
    const root = byId.get(rid);
    if (!root) continue;
    const s = slim(root);
    if (!s) continue;
    if (s.role === "_hoist") result.push(...s.children);
    else result.push(s);
  }
  return result;
};

const countNodes = (nodes: ReadonlyArray<SlimNode>): number => {
  let n = 0;
  for (const node of nodes) n += 1 + countNodes(node.children);
  return n;
};

const summarize = (nodes: ReadonlyArray<SlimNode>): string => {
  const counts = new Map<string, number>();
  const walk = (ns: ReadonlyArray<SlimNode>): void => {
    for (const n of ns) {
      counts.set(n.role, (counts.get(n.role) ?? 0) + 1);
      walk(n.children);
    }
  };
  walk(nodes);
  // Group landmarks (roles that are top-level structural).
  const landmarkRoles = new Set(["banner", "navigation", "main", "complementary", "contentinfo", "search", "form", "region"]);
  let landmarks = 0;
  for (const [role, c] of counts) if (landmarkRoles.has(role)) landmarks += c;
  const buttons = counts.get("button") ?? 0;
  const inputs = (counts.get("textbox") ?? 0) + (counts.get("combobox") ?? 0) + (counts.get("checkbox") ?? 0);
  const links = counts.get("link") ?? 0;
  const parts: string[] = [];
  if (landmarks > 0) parts.push(`${landmarks} landmark${landmarks === 1 ? "" : "s"}`);
  if (buttons > 0) parts.push(`${buttons} button${buttons === 1 ? "" : "s"}`);
  if (inputs > 0) parts.push(`${inputs} input${inputs === 1 ? "" : "s"}`);
  if (links > 0) parts.push(`${links} link${links === 1 ? "" : "s"}`);
  return parts.join(" · ");
};

const renderOutline = (nodes: ReadonlyArray<SlimNode>): string => {
  const lines: string[] = [];
  const walk = (ns: ReadonlyArray<SlimNode>, depth: number): void => {
    for (const n of ns) {
      const indent = "  ".repeat(depth);
      let line = `${indent}- ${n.role}`;
      if (n.name) line += ` "${n.name}"`;
      if (n.value && n.value !== n.name) line += ` = ${JSON.stringify(n.value)}`;
      if (n.state) line += ` (${n.state})`;
      // Surface the stable ref + click coordinates for interactive nodes. Agents
      // pass `ref` to interaction tools (survives re-renders); @(x,y) is a fallback.
      if (n.ref) line += ` [${n.ref}]`;
      if (n.box && INTERACTIVE_ROLES.has(n.role)) {
        line += ` @(${n.box.cx},${n.box.cy})`;
      }
      lines.push(line);
      walk(n.children, depth + 1);
    }
  };
  walk(nodes, 0);
  return lines.join("\n");
};

/** Strip internal _backendId from a slim tree before returning to callers. */
const stripInternals = (nodes: ReadonlyArray<SlimNode>): SlimNode[] =>
  nodes.map((n) => {
    const { _backendId: _bid, ...rest } = n;
    return { ...rest, children: stripInternals(n.children) };
  });

/**
 * Walk all slim nodes and collect (slimNode, backendId) pairs for interactive
 * roles. Caller fetches DOM.getBoxModel for each and writes box back into the
 * slim node by reference.
 */
export const collectInteractiveTargets = (
  nodes: ReadonlyArray<SlimNode>,
): Array<{ node: SlimNode; backendId: number }> => {
  const out: Array<{ node: SlimNode; backendId: number }> = [];
  const walk = (ns: ReadonlyArray<SlimNode>): void => {
    for (const n of ns) {
      if (n._backendId !== undefined && INTERACTIVE_ROLES.has(n.role)) {
        out.push({ node: n, backendId: n._backendId });
      }
      walk(n.children);
    }
  };
  walk(nodes);
  return out;
};

/**
 * Read the live DOM value for a form control by backend node id. Resolves the
 * node to a JS object then reads the property that matters for its type:
 * `.value` for inputs/selects/textareas, `.checked` (→ "checked"/"unchecked")
 * for checkboxes/radios. Returns undefined on any failure so the caller falls
 * back to the AX value. Mirrors the box-fetch error handling: never throws.
 */
const liveValue = async (session: CdpSession, backendId: number): Promise<string | undefined> => {
  const resolved = await session.call("DOM.resolveNode", { backendNodeId: backendId });
  if (!resolved.success) return undefined;
  const objectId = (resolved.data as { object?: { objectId?: string } }).object?.objectId;
  if (!objectId) return undefined;
  const fn = `function () {
    if (this.type === "checkbox" || this.type === "radio") return this.checked ? "checked" : "unchecked";
    if (typeof this.value === "string") return this.value;
    return null;
  }`;
  const r = await session.call("Runtime.callFunctionOn", {
    objectId,
    functionDeclaration: fn,
    returnByValue: true,
  });
  if (!r.success) return undefined;
  const value = (r.data as { result?: { value?: unknown } }).result?.value;
  return typeof value === "string" ? value : undefined;
};

type SnapshotDetails = {
  nodeCount: number;
  truncated: boolean;
  fullOutputPath?: string;
  screenshotPath?: string;
  url: string;
  title: string;
};

export const snapshotTool = defineBrowserTool({
  name: "browser_snapshot",
  label: "Browser Snapshot",
  description:
    "DEFAULT tool for understanding what is on the page. Returns the structured accessibility tree (roles, names, states, hierarchy). Every interactive element gets a stable ref shown as [eN] plus click coordinates @(x,y). Pass the ref to browser_click/browser_fill/etc. — refs survive re-renders, coordinates don't. Use this BEFORE deciding whether you need a screenshot. Pair with browser_execute_js for surgical reads of specific element values.",
  promptSnippet: "Get accessibility-tree snapshot with stable element refs (default for page inspection)",
  promptGuidelines: [
    "DEFAULT — use this whenever you need to know what's on a page, what's clickable, or how the page is structured.",
    "Refs come for free: every interactive element shows '[eN]' in the outline. Pass eN as `ref` to browser_click/browser_fill/browser_select_option/browser_focus/browser_upload_file — refs survive re-renders, so prefer them over the '@(x,y)' coordinates (a fallback).",
    "DO NOT call browser_screenshot just to understand the page. This tool already gives you structure, labels, states, refs, and click targets.",
    "Re-run after a navigation or a major re-render to get fresh refs; a 'ref is stale' error from an interaction tool means you need a new snapshot.",
    "Pass includeScreenshot:true ONLY if you also need to verify visual rendering (rare).",
    "format:'json' returns the raw slim structure (with `ref` and `box` per node) for programmatic use; default 'outline' is human/LLM-readable.",
  ],
  parameters: SnapshotArgs,

  async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
    const session = client.session();

    // 1. AX tree
    const axRes = await session.call("Accessibility.getFullAXTree", {});
    if (!axRes.success) return err({ kind: "cdp_error", message: axRes.error.message });
    const rawNodes = (axRes.data as { nodes: ReadonlyArray<RawAxNode> }).nodes;

    // 2. Page url/title
    const piRes = await client.pageInfo();
    if (!piRes.success) return err({ kind: "cdp_error", message: piRes.error.message });
    const pageUrl = "dialog" in piRes.data ? "" : piRes.data.url;
    const pageTitle = "dialog" in piRes.data ? "" : piRes.data.title;

    // 3. Slim + format
    const slim = buildTree(rawNodes, {
      interestingOnly: args.interestingOnly ?? true,
      maxNodes: args.maxNodes ?? 1000,
    });

    // 4. Fetch bounding boxes for interactive nodes so the agent can click
    //    without a screenshot round-trip. Capped budget so a slow page can't
    //    wedge the call.
    const targets = collectInteractiveTargets(slim);

    // Assign stable refs (e1, e2, …) to every interactive target. Interaction
    // tools resolve these refs to backendNodeIds at call time, so they stay
    // valid across re-renders. The ref → backendNodeId map (and the per-ref
    // signature baseline for the post-mutation diff) is published below, after
    // the box/live-value fetch so the signature reflects the live DOM value.
    const refMap = new Map<string, number>();
    targets.forEach(({ node, backendId }, i) => {
      const ref = `e${i + 1}`;
      node.ref = ref;
      refMap.set(ref, backendId);
    });

    if (targets.length > 0) {
      const ac = new AbortController();
      const timer = setTimeout(() => ac.abort(), 1_500);
      await Promise.allSettled(
        targets.map(async ({ node, backendId }) => {
          if (ac.signal.aborted) return;
          const r = await session.call("DOM.getBoxModel", { backendNodeId: backendId });
          if (!r.success) return;
          // CDP boundary cast: DOM.getBoxModel
          const data = r.data as { model?: { content?: ReadonlyArray<number>; width?: number; height?: number } };
          const content = data.model?.content;
          if (!content || content.length < 8) return;
          // content is [x1,y1, x2,y1, x2,y2, x1,y2] in viewport CSS pixels.
          const x = content[0]!;
          const y = content[1]!;
          const width = data.model!.width ?? 0;
          const height = data.model!.height ?? 0;
          // Skip zero-size and off-viewport boxes — they can't be clicked.
          if (width <= 0 || height <= 0 || x < 0 || y < 0) return;
          node.box = {
            x: Math.round(x),
            y: Math.round(y),
            width: Math.round(width),
            height: Math.round(height),
            cx: Math.round(x + width / 2),
            cy: Math.round(y + height / 2),
          };
          // Override the AX value with the live DOM property for form controls.
          // The AX `value` is stale for freshly-typed / controlled inputs, so the
          // agent can't trust it as a read-back signal. Reading .value/.checked
          // from the actual element fixes that.
          if (ac.signal.aborted || !VALUE_ROLES.has(node.role)) return;
          const live = await liveValue(session, backendId);
          if (live !== undefined) node.value = live;
        }),
      );
      clearTimeout(timer);
    }

    // Publish refs to the active tab. The signature (role|name|value|state) is
    // the baseline the post-mutation auto-diff compares against, so it must use
    // the live values resolved above.
    const refSig = new Map<string, string>();
    for (const { node } of targets) {
      if (node.ref === undefined) continue;
      refSig.set(node.ref, `${node.role}|${node.name ?? ""}|${node.value ?? ""}|${node.state ?? ""}`);
    }
    session.setRefMap(refMap, refSig);

    const stripped = stripInternals(slim);
    const text = (args.format ?? "outline") === "outline" ? renderOutline(slim) : JSON.stringify(stripped, null, 2);
    const trunc = await applyTruncation(text, "snapshot");

    // 4. Optional screenshot — JPEG q=80 to match screenshotTool's default.
    let shotPath: string | undefined;
    if (args.includeScreenshot) {
      const shot = await session.call("Page.captureScreenshot", { format: "jpeg", quality: 80 });
      if (shot.success) {
        const data = (shot.data as { data: string }).data;
        shotPath = screenshotPath(client.namespace, "jpeg");
        writeFileSync(shotPath, Buffer.from(data, "base64"));
      }
    }

    const details: SnapshotDetails = {
      nodeCount: countNodes(slim),
      truncated: trunc.wasTruncated,
      url: pageUrl,
      title: pageTitle,
    };
    if (trunc.fullOutputPath !== undefined) details.fullOutputPath = trunc.fullOutputPath;
    if (shotPath !== undefined) details.screenshotPath = shotPath;

    return ok({ text: trunc.text, details: { ...details, summary: summarize(slim) } });
  },

  renderResult(result, expanded, theme) {
    const details = result.details as
      | (SnapshotDetails & { summary?: string; ok?: boolean })
      | undefined;
    if (!details) return new Text(theme.fg("error", "snapshot: no details"), 0, 0);

    const titleLine = details.title ? ` "${details.title}"` : "";
    const summary = details.summary ?? "";
    const screenshotNote = details.screenshotPath ? "screenshot attached" : "screenshot omitted";
    const truncNote = details.truncated && details.fullOutputPath ? `\n\nFull tree at \`${details.fullOutputPath}\`` : "";

    if (!expanded) {
      const md = [
        `**AX tree:** ${details.nodeCount} nodes · \`${details.url || "(no url)"}\``,
        `${titleLine ? "  " + titleLine : ""}`,
        summary ? `  • ${summary}` : "",
        `  ${keyHint("app.tools.expand", "to expand")} · ${screenshotNote}`,
      ].filter((l) => l !== "").join("\n");
      return new Markdown(md, 0, 0, getMarkdownTheme());
    }

    // Expanded view: read truncated text from result.content (the spilled file
    // is too large to inline) — the same `text` we returned from handler is in
    // result.content[0]. Fall back to a placeholder if shape is unexpected.
    const content = result.content[0];
    const treeText = content && content.type === "text" ? content.text : "";

    const headerMd = [
      `**AX tree:** ${details.nodeCount} nodes · \`${details.url || "(no url)"}\``,
      titleLine ? `  ${titleLine}` : "",
      "",
      "```",
      treeText,
      "```",
      truncNote,
      "",
      `${keyHint("app.tools.expand", "to collapse")}`,
    ].filter((l) => l !== "").join("\n");

    const treeBlock = new Markdown(headerMd, 0, 0, getMarkdownTheme());

    if (!details.screenshotPath) return treeBlock;

    // Mirror screenshotTool's width-clamp wrapper for the inline image.
    try {
      const buf = readFileSync(details.screenshotPath);
      const b64 = buf.toString("base64");
      const imageTheme: ImageTheme = { fallbackColor: (s: string) => theme.fg("dim", s) };
      const image = new Image(b64, "image/jpeg", imageTheme, {
        maxWidthCells: 80,
        maxHeightCells: 24,
        filename: details.screenshotPath,
      });
      const safeImage = {
        invalidate: () => image.invalidate(),
        render: (width: number) =>
          image.render(width).map((line) => (visibleWidth(line) > width ? truncateToWidth(line, width) : line)),
      };
      const container = new Container();
      container.addChild(treeBlock);
      container.addChild(safeImage);
      return container;
    } catch {
      return treeBlock;
    }
  },
});