feat: address elements by stable refs across re-renders (#8)

Interaction tools could only target elements by raw viewport coordinates
(browser_click) or model-guessed CSS selectors (browser_fill/focus/
select_option/upload). Neither survives a re-render: on React/SPA forms,
every save or edit reflows the page, so captured @(x,y) coordinates go
stale and guessed selectors miss — leaving the agent to fall back to
hand-rolled execute_js DOM walks and unnoticed failed saves.

Add a stable ref system, mirroring Playwright MCP / browser-use:

- snapshot assigns each interactive element a ref (e1, e2, …), shown as
  [eN] in the outline, and publishes a per-tab ref → backendNodeId map
- interaction tools accept an optional `ref` that re-resolves the element
  at call time (via DOM.resolveNode / getBoxModel), so it stays valid
  across re-renders; coordinates/selectors remain as fallbacks
- mutating tools append a compact diff of changed / newly-appeared (*) /
  removed interactive elements, so silent failed saves surface immediately
- stale refs return an actionable "re-snapshot" error instead of a miss

Ref map is replaced per snapshot (bounded memory); fill keeps the native
_valueTracker write path so React controlled inputs still update.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

1 files changed, 61 insertions, 5 deletions

diff --git a/src/domains/keyboard.ts b/src/domains/keyboard.ts
index c39814e..69a7396 100644
--- a/src/domains/keyboard.ts
+++ b/src/domains/keyboard.ts
@@ -3,6 +3,7 @@ import { safeJs } from "../util/js-template";
 import { virtualKeyCode } from "../util/keycodes";
 import { type Result, err, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
+import { resolveRefToObjectId } from "./ref-resolve";
 
 const TypeArgs = Type.Object({
   text: Type.String({ description: "Text to type" }),
@@ -19,6 +20,29 @@ export const typeTool = defineBrowserTool({
   ],
   parameters: TypeArgs,
   async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
+    // Guard against silent text loss: Input.insertText succeeds regardless of
+    // whether an editable element is focused, so without this check typed text
+    // can vanish into the void with no error. Fail loudly with an actionable
+    // message instead. IIFE starts with "(" so evaluateJs returns it directly.
+    const focused = await client.evaluateJs(safeJs`
+      (() => {
+        const el = document.activeElement;
+        const ok = !!el && el !== document.body &&
+          (el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.isContentEditable);
+        return { ok, tag: el ? el.tagName : null };
+      })()
+    `);
+    if (focused.success) {
+      const f = focused.data as { ok: boolean; tag: string | null };
+      if (!f.ok) {
+        return err({
+          kind: "invalid_state",
+          message:
+            "No editable element is focused — typed text would be lost. Click the field with browser_click first, or use browser_fill with a selector.",
+          details: { activeElement: f.tag },
+        });
+      }
+    }
     const r = await client.session().call("Input.insertText", { text: args.text });
     if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
     return ok({ text: `Typed: "${args.text}"` });
@@ -67,6 +91,14 @@ export const pressKeyTool = defineBrowserTool({
     };
     const down = await client.session().call("Input.dispatchKeyEvent", downParams);
     if (!down.success) return err({ kind: "cdp_error", message: down.error.message });
+    // For a printable character with no command modifier (Ctrl/Meta/Alt = 1|2|4),
+    // emit a `char` event so the page receives keypress/textInput and the
+    // character is actually inserted. Shift (8) is allowed (capitals/symbols).
+    const hasCommandModifier = (modifiers & (1 | 2 | 4)) !== 0;
+    if (isChar && !hasCommandModifier) {
+      const charEv = await client.session().call("Input.dispatchKeyEvent", { type: "char", text: k, key: k });
+      if (!charEv.success) return err({ kind: "cdp_error", message: charEv.error.message });
+    }
     const up = await client.session().call("Input.dispatchKeyEvent", { type: "keyUp", key: k, code: k, modifiers });
     if (!up.success) return err({ kind: "cdp_error", message: up.error.message });
     return ok({ text: `Pressed: ${k}${modifiers ? ` (modifiers=${modifiers})` : ""}` });
@@ -74,7 +106,10 @@ export const pressKeyTool = defineBrowserTool({
 });
 
 const DispatchKeyArgs = Type.Object({
-  selector: Type.String({ description: "CSS selector of the target element" }),
+  ref: Type.Optional(
+    Type.String({ description: "Stable element ref from browser_snapshot (e.g. 'e7'). PREFERRED over selector." }),
+  ),
+  selector: Type.Optional(Type.String({ description: "CSS selector of the target element (fallback when no ref)" })),
   key: Type.String({ description: "Key value (e.g., 'Enter', 'a')" }),
   eventType: Type.Optional(
     Type.Union(
@@ -88,17 +123,38 @@ export const dispatchKeyTool = defineBrowserTool({
   name: "browser_dispatch_key",
   label: "Browser Dispatch Key",
   description:
-    "Dispatch a DOM KeyboardEvent on a specific element via JS injection. Use for React/Vue components that listen to synthetic events more reliably than CDP input.",
-  promptSnippet: "Dispatch a DOM KeyboardEvent on a specific element",
+    "Dispatch a DOM KeyboardEvent on a specific element via JS injection. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`. Use for React/Vue components that listen to synthetic events more reliably than CDP input.",
+  promptSnippet: "Dispatch a DOM KeyboardEvent on an element by ref (preferred) or selector",
   promptGuidelines: [
-    "Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text into inputs (use browser_type or browser_press_key for actual typing).",
+    "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders.",
+    "Dispatches a synthetic DOM KeyboardEvent — for React/Vue synthetic event listeners. Does NOT insert text (use browser_type or browser_press_key for actual typing).",
     "Try browser_press_key first; only use browser_dispatch_key when the page ignores raw CDP key events.",
-    "The selector must match exactly one or more elements; zero matches is reported as an error.",
     "eventType defaults to 'keydown'.",
   ],
   parameters: DispatchKeyArgs,
   async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
     const eventType = args.eventType ?? "keydown";
+    const target = args.ref ?? args.selector ?? "";
+    // Ref path: dispatch on the single resolved node. Selector path: dispatch on
+    // all matches (preserves the original multi-match behavior).
+    if (args.ref !== undefined) {
+      const objectId = await resolveRefToObjectId(client, args.ref);
+      if (!objectId.success) return objectId;
+      const r = await client.session().call("Runtime.callFunctionOn", {
+        objectId: objectId.data,
+        functionDeclaration: `function (type, key) { this.dispatchEvent(new KeyboardEvent(type, { key, bubbles: true, cancelable: true })); return 1; }`,
+        arguments: [{ value: eventType }, { value: args.key }],
+        returnByValue: true,
+      });
+      if (!r.success) return err({ kind: "cdp_error", message: r.error.message });
+      return ok({
+        text: `Dispatched ${eventType}('${args.key}') on ${target}`,
+        details: { matched: 1, ref: args.ref },
+      });
+    }
+    if (args.selector === undefined) {
+      return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
+    }
     const expr = safeJs`
       (() => {
         const els = document.querySelectorAll(${args.selector});