From 0b32e88f84f36a759d7191e8bdfd09a44789fa95 Mon Sep 17 00:00:00 2001 From: Aman Kumar <145339995+amankumarsingh77@users.noreply.github.com> Date: Mon, 29 Jun 2026 00:02:54 +0530 Subject: feat: address elements by stable refs across re-renders (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interaction tools could only target elements by raw viewport coordinates (browser_click) or model-guessed CSS selectors (browser_fill/focus/ select_option/upload). Neither survives a re-render: on React/SPA forms, every save or edit reflows the page, so captured @(x,y) coordinates go stale and guessed selectors miss — leaving the agent to fall back to hand-rolled execute_js DOM walks and unnoticed failed saves. Add a stable ref system, mirroring Playwright MCP / browser-use: - snapshot assigns each interactive element a ref (e1, e2, …), shown as [eN] in the outline, and publishes a per-tab ref → backendNodeId map - interaction tools accept an optional `ref` that re-resolves the element at call time (via DOM.resolveNode / getBoxModel), so it stays valid across re-renders; coordinates/selectors remain as fallbacks - mutating tools append a compact diff of changed / newly-appeared (*) / removed interactive elements, so silent failed saves surface immediately - stale refs return an actionable "re-snapshot" error instead of a miss Ref map is replaced per snapshot (bounded memory); fill keeps the native _valueTracker write path so React controlled inputs still update. Co-authored-by: Claude Opus 4.8 --- src/registry.ts | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/registry.ts') diff --git a/src/registry.ts b/src/registry.ts index fcdb66a..d09f1cc 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -3,6 +3,7 @@ import type { BrowserClient } from "./client"; import { type AnyBrowserToolDefinition, registerBrowserTool } from "./util/tool"; import { clickTool } from "./domains/click"; import { typeTool, pressKeyTool, dispatchKeyTool } from "./domains/keyboard"; +import { fillTool, selectOptionTool, focusTool } from "./domains/form"; import { pageInfoTool, waitTool, waitForLoadTool } from "./domains/page"; import { scrollTool } from "./domains/scroll"; import { handleDialogTool } from "./domains/dialog"; @@ -27,6 +28,9 @@ const TOOLS: ReadonlyArray = [ setupTool, { ...clickTool, serialized: true }, { ...typeTool, serialized: true }, + { ...fillTool, serialized: true }, + { ...selectOptionTool, serialized: true }, + { ...focusTool, serialized: true }, { ...pressKeyTool, serialized: true }, { ...dispatchKeyTool, serialized: true }, { ...scrollTool, serialized: true }, -- cgit v1.2.3