feat: address elements by stable refs across re-renders (#8)

Interaction tools could only target elements by raw viewport coordinates
(browser_click) or model-guessed CSS selectors (browser_fill/focus/
select_option/upload). Neither survives a re-render: on React/SPA forms,
every save or edit reflows the page, so captured @(x,y) coordinates go
stale and guessed selectors miss — leaving the agent to fall back to
hand-rolled execute_js DOM walks and unnoticed failed saves.

Add a stable ref system, mirroring Playwright MCP / browser-use:

- snapshot assigns each interactive element a ref (e1, e2, …), shown as
  [eN] in the outline, and publishes a per-tab ref → backendNodeId map
- interaction tools accept an optional `ref` that re-resolves the element
  at call time (via DOM.resolveNode / getBoxModel), so it stays valid
  across re-renders; coordinates/selectors remain as fallbacks
- mutating tools append a compact diff of changed / newly-appeared (*) /
  removed interactive elements, so silent failed saves surface immediately
- stale refs return an actionable "re-snapshot" error instead of a miss

Ref map is replaced per snapshot (bounded memory); fill keeps the native
_valueTracker write path so React controlled inputs still update.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

1 files changed, 60 insertions, 24 deletions

diff --git a/src/domains/click.ts b/src/domains/click.ts
index 9946664..db340a8 100644
--- a/src/domains/click.ts
+++ b/src/domains/click.ts
@@ -1,13 +1,21 @@
-import { Type, type Static } from "typebox";
+import { Type } from "typebox";
 import { Text } from "@mariozechner/pi-tui";
 import type { BrowserClient } from "../client";
 import { Coords, MouseButton } from "../schemas/common";
 import { type Result, err, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr } from "../util/tool";
 import { captureWithCrosshair } from "./screenshot";
+import { interactiveDiff, resolveRefToBox } from "./ref-resolve";
 
 const ClickArgs = Type.Object({
-  ...Coords.properties,
+  ref: Type.Optional(
+    Type.String({
+      description:
+        "Stable element ref from browser_snapshot (e.g. 'e12'). PREFERRED over x/y — survives re-renders. When set, x/y are ignored.",
+    }),
+  ),
+  x: Type.Optional(Coords.properties.x),
+  y: Type.Optional(Coords.properties.y),
   button: Type.Optional(MouseButton),
   count: Type.Optional(
     Type.Integer({
@@ -18,20 +26,27 @@ const ClickArgs = Type.Object({
     }),
   ),
 });
-type ClickArgsT = Static<typeof ClickArgs>;
 
 const dispatchClick = async (
   client: BrowserClient,
-  args: ClickArgsT,
+  x: number,
+  y: number,
+  button: "left" | "right" | "middle",
+  count: number,
 ): Promise<Result<void, ToolErr>> => {
-  const button = args.button ?? "left";
-  const count = args.count ?? 1;
+  // Move the pointer to the target first. Some focus/hover handlers (and React
+  // synthetic events) only fire when a mousemove precedes the press, so without
+  // it a click can land without focusing the element.
+  const moved = await client.session().call("Input.dispatchMouseEvent", {
+    type: "mouseMoved", x, y,
+  });
+  if (!moved.success) return err({ kind: "cdp_error", message: moved.error.message });
   const pressed = await client.session().call("Input.dispatchMouseEvent", {
-    type: "mousePressed", x: args.x, y: args.y, button, clickCount: count,
+    type: "mousePressed", x, y, button, clickCount: count,
   });
   if (!pressed.success) return err({ kind: "cdp_error", message: pressed.error.message });
   const released = await client.session().call("Input.dispatchMouseEvent", {
-    type: "mouseReleased", x: args.x, y: args.y, button, clickCount: count,
+    type: "mouseReleased", x, y, button, clickCount: count,
   });
   if (!released.success) return err({ kind: "cdp_error", message: released.error.message });
   return ok(undefined);
@@ -41,34 +56,55 @@ export const clickTool = defineBrowserTool({
   name: "browser_click",
   label: "Browser Click",
   description:
-    "Click at viewport CSS-pixel coordinates. Compositor-level click works through iframes, shadow DOM, and cross-origin content. Get coordinates from browser_snapshot's @(x,y) hints — do not screenshot to find click targets.",
-  promptSnippet: "Click at pixel coordinates (x, y) on the page",
+    "Click an element. PREFERRED: pass `ref` (e.g. 'e12') from browser_snapshot — it re-resolves the element's position at click time, so it works even after the page re-renders and moves things. Fallback: pass viewport CSS-pixel `x`/`y`. Compositor-level click works through iframes, shadow DOM, and cross-origin content. After clicking, a compact diff of page changes is appended.",
+  promptSnippet: "Click an element by ref (preferred) or pixel coordinates",
   promptGuidelines: [
-    "Get (x, y) from browser_snapshot — the outline shows '@(x,y)' for every interactive element. NO screenshot needed.",
-    "If the target isn't an interactive role in the snapshot, fall back to browser_execute_js with `el.getBoundingClientRect()` — still NO screenshot.",
-    "Coordinates are viewport CSS pixels (not device pixels).",
-    "After clicking, verify with browser_snapshot (or browser_execute_js for a specific value). Only screenshot if you need to confirm visual rendering.",
-    "Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content — no selector needed.",
-    "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots showing exact click positions (debug only).",
-    "For React/Vue components that don't respond to clicks, try browser_dispatch_key to send DOM-level events.",
+    "PREFER `ref` from browser_snapshot (the outline shows '[eN]' for every interactive element) — it survives re-renders, unlike coordinates which go stale after a save/edit reflows the page.",
+    "Fallback: pass (x, y) from the snapshot's '@(x,y)' hint when there's no ref.",
+    "A 'ref is stale' error means the page changed — re-run browser_snapshot to get fresh refs.",
+    "After clicking, read the appended page-changes diff to confirm the action landed before moving on (no separate snapshot needed for a quick check).",
+    "Coordinates are viewport CSS pixels (not device pixels). Compositor-level clicks pass through iframes, shadow DOM, and cross-origin content.",
+    "If a click doesn't register, set BH_DEBUG_CLICKS=1 to get annotated screenshots (debug only). For React/Vue components ignoring clicks, try browser_dispatch_key.",
   ],
   parameters: ClickArgs,
-  renderCall: (a) => new Text(`🖱️ Click at (${a.x}, ${a.y})`, 0, 0),
+  renderCall: (a) => new Text(`🖱️ Click ${a.ref ? `[${a.ref}]` : `(${a.x}, ${a.y})`}`, 0, 0),
   async handler(args, { client }) {
-    const clicked = await dispatchClick(client, args);
+    const button = args.button ?? "left";
+    const count = args.count ?? 1;
+
+    // Resolve the click point: ref re-resolves to the element's CURRENT box
+    // (so re-renders don't matter); otherwise use the literal x/y.
+    let x: number;
+    let y: number;
+    if (args.ref !== undefined) {
+      const box = await resolveRefToBox(client, args.ref);
+      if (!box.success) return box;
+      x = box.data.cx;
+      y = box.data.cy;
+    } else if (args.x !== undefined && args.y !== undefined) {
+      x = args.x;
+      y = args.y;
+    } else {
+      return err({ kind: "invalid_state", message: "Provide either `ref` or both `x` and `y`." });
+    }
+
+    const clicked = await dispatchClick(client, x, y, button, count);
     if (!clicked.success) return clicked;
+
+    const target = args.ref !== undefined ? `[${args.ref}] (${x}, ${y})` : `(${x}, ${y})`;
+    const diff = await interactiveDiff(client);
     if (process.env["BH_DEBUG_CLICKS"]) {
-      const debug = await captureWithCrosshair(client, { x: args.x, y: args.y });
+      const debug = await captureWithCrosshair(client, { x, y });
       if (debug.success) {
         return ok({
-          text: `Clicked at (${args.x}, ${args.y})\n[DEBUG] Overlay screenshot: ${debug.data.path}`,
-          details: { debugScreenshotPath: debug.data.path, x: args.x, y: args.y },
+          text: `Clicked at ${target}\n[DEBUG] Overlay screenshot: ${debug.data.path}${diff}`,
+          details: { debugScreenshotPath: debug.data.path, x, y, ref: args.ref },
         });
       }
     }
     return ok({
-      text: `Clicked at (${args.x}, ${args.y})`,
-      details: { x: args.x, y: args.y },
+      text: `Clicked at ${target}${diff}`,
+      details: { x, y, ref: args.ref },
     });
   },
 });