From 0b32e88f84f36a759d7191e8bdfd09a44789fa95 Mon Sep 17 00:00:00 2001
From: Aman Kumar <145339995+amankumarsingh77@users.noreply.github.com>
Date: Mon, 29 Jun 2026 00:02:54 +0530
Subject: feat: address elements by stable refs across re-renders (#8)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Interaction tools could only target elements by raw viewport coordinates
(browser_click) or model-guessed CSS selectors (browser_fill/focus/
select_option/upload). Neither survives a re-render: on React/SPA forms,
every save or edit reflows the page, so captured @(x,y) coordinates go
stale and guessed selectors miss — leaving the agent to fall back to
hand-rolled execute_js DOM walks and unnoticed failed saves.

Add a stable ref system, mirroring Playwright MCP / browser-use:

- snapshot assigns each interactive element a ref (e1, e2, …), shown as
  [eN] in the outline, and publishes a per-tab ref → backendNodeId map
- interaction tools accept an optional `ref` that re-resolves the element
  at call time (via DOM.resolveNode / getBoxModel), so it stays valid
  across re-renders; coordinates/selectors remain as fallbacks
- mutating tools append a compact diff of changed / newly-appeared (*) /
  removed interactive elements, so silent failed saves surface immediately
- stale refs return an actionable "re-snapshot" error instead of a miss

Ref map is replaced per snapshot (bounded memory); fill keeps the native
_valueTracker write path so React controlled inputs still update.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/domains/files.ts | 37 +++++++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

(limited to 'src/domains/files.ts')

diff --git a/src/domains/files.ts b/src/domains/files.ts
index 91f8761..d68d236 100644
--- a/src/domains/files.ts
+++ b/src/domains/files.ts
@@ -6,12 +6,28 @@ import { type Result, err, ok } from "../util/result";
 import { defineBrowserTool, type ToolErr, type ToolOk } from "../util/tool";
 import { safeJs } from "../util/js-template";
 import { pdfPath } from "../util/paths";
+import { resolveRefToBackendId } from "./ref-resolve";
 
 const UploadArgs = Type.Object({
-  selector: Type.String({ description: "CSS selector of the file <input>" }),
+  ref: Type.Optional(
+    Type.String({ description: "Stable ref of the file <input> from browser_snapshot. PREFERRED over selector." }),
+  ),
+  selector: Type.Optional(Type.String({ description: "CSS selector of the file <input> (fallback when no ref)" })),
   filePath: Type.String({ description: "Absolute path to the file to upload" }),
 });
 
+// Ref path: set files directly on the resolved backendNodeId via CDP — simplest
+// and most reliable, no selector needed.
+const tryRefUpload = async (
+  client: BrowserClient,
+  backendId: number,
+  filePath: string,
+): Promise<Result<void, ToolErr>> => {
+  const set = await client.session().call("DOM.setFileInputFiles", { files: [filePath], backendNodeId: backendId });
+  if (!set.success) return err({ kind: "cdp_error", message: set.error.message });
+  return ok(undefined);
+};
+
 const verifyReadable = async (filePath: string): Promise<Result<void, ToolErr>> => {
   try {
     await access(filePath, constants.R_OK);
@@ -89,16 +105,29 @@ const jsFallbackUpload = async (
 export const uploadFileTool = defineBrowserTool({
   name: "browser_upload_file",
   label: "Browser Upload File",
-  description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages.",
-  promptSnippet: "Upload a file to a file input",
+  description: "Set files on a file <input> via CDP, with a JS-DataTransfer fallback for stubborn pages. PREFERRED: pass `ref` from browser_snapshot; fallback: a CSS `selector`.",
+  promptSnippet: "Upload a file to a file input by ref (preferred) or selector",
   promptGuidelines: [
+    "PREFER `ref` from browser_snapshot over a CSS selector — survives re-renders. (File inputs are often hidden; snapshot still surfaces them.)",
     "File path must be absolute and readable.",
-    "Selector must match a file input.",
   ],
   parameters: UploadArgs,
   async handler(args, { client }): Promise<Result<ToolOk, ToolErr>> {
     const readable = await verifyReadable(args.filePath);
     if (!readable.success) return readable;
+
+    // Ref path: resolve to backendNodeId and set files directly via CDP.
+    if (args.ref !== undefined) {
+      const backendId = resolveRefToBackendId(client, args.ref);
+      if (!backendId.success) return backendId;
+      const up = await tryRefUpload(client, backendId.data, args.filePath);
+      if (!up.success) return up;
+      return ok({ text: `Uploaded ${args.filePath} via CDP (ref ${args.ref})`, details: { mode: "cdp", ref: args.ref, filePath: args.filePath } });
+    }
+
+    if (args.selector === undefined) {
+      return err({ kind: "invalid_state", message: "Provide either `ref` or `selector`." });
+    }
     const cdp = await tryCdpUpload(client, args.selector, args.filePath);
     if (cdp.success) {
       return ok({
-- 
cgit v1.2.3