From 0b32e88f84f36a759d7191e8bdfd09a44789fa95 Mon Sep 17 00:00:00 2001
From: Aman Kumar <145339995+amankumarsingh77@users.noreply.github.com>
Date: Mon, 29 Jun 2026 00:02:54 +0530
Subject: feat: address elements by stable refs across re-renders (#8)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Interaction tools could only target elements by raw viewport coordinates
(browser_click) or model-guessed CSS selectors (browser_fill/focus/
select_option/upload). Neither survives a re-render: on React/SPA forms,
every save or edit reflows the page, so captured @(x,y) coordinates go
stale and guessed selectors miss — leaving the agent to fall back to
hand-rolled execute_js DOM walks and unnoticed failed saves.

Add a stable ref system, mirroring Playwright MCP / browser-use:

- snapshot assigns each interactive element a ref (e1, e2, …), shown as
  [eN] in the outline, and publishes a per-tab ref → backendNodeId map
- interaction tools accept an optional `ref` that re-resolves the element
  at call time (via DOM.resolveNode / getBoxModel), so it stays valid
  across re-renders; coordinates/selectors remain as fallbacks
- mutating tools append a compact diff of changed / newly-appeared (*) /
  removed interactive elements, so silent failed saves surface immediately
- stale refs return an actionable "re-snapshot" error instead of a miss

Ref map is replaced per snapshot (bounded memory); fill keeps the native
_valueTracker write path so React controlled inputs still update.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/cdp/session.ts | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

(limited to 'src/cdp')

diff --git a/src/cdp/session.ts b/src/cdp/session.ts
index ac19456..e090858 100644
--- a/src/cdp/session.ts
+++ b/src/cdp/session.ts
@@ -17,12 +17,25 @@ type TabSession = {
   pageInfoDirty: boolean;
   networkBuffer: ReturnType<typeof createNetworkBuffer>;
   consoleBuffer: ReturnType<typeof createConsoleBuffer>;
+  // Stable element refs (e1, e2, …) → CDP backendNodeId, from the latest
+  // snapshot of this tab. Replaced wholesale on every snapshot — old refs are
+  // stale anyway, so this stays bounded (~50–60 KB worst case).
+  refMap: Map<string, number>;
+  // Per-ref signature ("role|name|value") from the latest snapshot, used as the
+  // baseline for the post-mutation auto-diff. Replaced alongside refMap.
+  refSig: Map<string, string>;
 };
 
 export type CdpSession = {
   attachFirstPage(): Promise<Result<{ readonly targetId: string; readonly sessionId: string }, CdpError>>;
   switchTo(targetId: string): Promise<Result<void, CdpError>>;
   current(): { readonly sessionId: string; readonly targetId: string } | null;
+  /** Replace the active tab's ref map and ref signatures (from a fresh snapshot). */
+  setRefMap(refMap: Map<string, number>, refSig: Map<string, string>): void;
+  /** Resolve a ref (e.g. "e12") to its backendNodeId on the active tab, or undefined if unknown/stale. */
+  resolveRef(ref: string): number | undefined;
+  /** The active tab's prior ref signatures — baseline for the post-mutation diff. */
+  refSignatures(): ReadonlyMap<string, string>;
   call(method: string, params?: Record<string, unknown>, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>;
   callOnTarget(method: string, params: Record<string, unknown>, sessionId: string, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>;
   callBrowser(method: string, params?: Record<string, unknown>, opts?: { timeoutMs?: number }): Promise<Result<unknown, CdpError>>;
@@ -211,6 +224,8 @@ export const createCdpSession = (
         pageInfoDirty: false,
         networkBuffer: createNetworkBuffer(),
         consoleBuffer: createConsoleBuffer(),
+        refMap: new Map(),
+        refSig: new Map(),
       });
       sessionIdToTargetId.set(a.sessionId, pickTargetId);
       return ok({ targetId: pickTargetId, sessionId: a.sessionId });
@@ -230,6 +245,8 @@ export const createCdpSession = (
         pageInfoDirty: true,
         networkBuffer: createNetworkBuffer(),
         consoleBuffer: createConsoleBuffer(),
+        refMap: new Map(),
+        refSig: new Map(),
       };
       if (existing) {
         // Each Target.attachToTarget produces a new sessionId — update it.
@@ -249,6 +266,20 @@ export const createCdpSession = (
     current() {
       return sessionId && targetId ? { sessionId, targetId } : null;
     },
+    setRefMap(refMap, refSig) {
+      const tab = targetId ? tabs.get(targetId) : undefined;
+      if (!tab) return;
+      tab.refMap = refMap;
+      tab.refSig = refSig;
+    },
+    resolveRef(ref) {
+      const tab = targetId ? tabs.get(targetId) : undefined;
+      return tab?.refMap.get(ref);
+    },
+    refSignatures() {
+      const tab = targetId ? tabs.get(targetId) : undefined;
+      return tab?.refSig ?? new Map();
+    },
     call(method, params = {}, opts = {}) {
       return transport.request(method, params, { sessionId, ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}) });
     },
-- 
cgit v1.2.3