aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTianren Dong <tianrendong.dtr@gmail.com>2026-06-03 20:57:27 -0400
committerTianren Dong <tianrendong.dtr@gmail.com>2026-06-06 14:07:55 -0400
commit341e2b3cebe41a4a14b00d83a84208971e0df7f5 (patch)
tree8c1657ee2850247f4b31ab8a2f6605e37b490fe2
parentfa33b0797755b146b175f4aa792c8d0c22ba781f (diff)
0.15.36: richer chrome_snapshot + chrome_find/chrome_inspect
- chrome_snapshot returns a concise observation (layout, hints, actions, forms, page map, query matches, diff) with mode/query/maxTextChars params - new chrome_find (query -> ranked uids) and chrome_inspect (deep element context, with page.inspect fallback) - includeSnapshot on click/type/fill/key now embeds the formatted snapshot - snapshot/inspect logic shipped as eval-free packaged snapshot_injected.js, sharing window.__PI_CHROME_STATE__ (el- uid scheme) with existing input helpers
-rw-r--r--CHANGELOG.md8
-rw-r--r--README.md2
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/manifest.json2
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/service_worker.js192
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js677
-rw-r--r--extensions/chrome-profile-bridge/index.ts228
-rw-r--r--package.json2
7 files changed, 996 insertions, 115 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 60e185e..d0f8d50 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,14 @@
All notable user-facing changes to `pi-chrome`.
+## 0.15.36 — 2026-06-03
+
+- **Richer page observation.** `chrome_snapshot` now returns a concise, agent-friendly observation — structural layout/context, page hints, visible actions, form fields, a page map, query matches, and a diff of changes since the previous snapshot — instead of a raw JSON dump. New `mode` (`auto`/`interactive`/`forms`/`pageMap`/`text`/`changes`/`full`), `query`, and `maxTextChars` parameters let the agent zoom in instead of dumping the whole page.
+- **New `chrome_find` tool.** Find elements, regions, or text by natural-language query (`'merge button'`, `'email error'`) and get ranked matches with stable uids and coordinates. Thin wrapper around `chrome_snapshot({ query })`.
+- **New `chrome_inspect` tool.** Inspect one snapshot uid/selector deeply: nearby text, nearby actions, form context, ancestors, and a suggested click target. Falls back to a focused snapshot if the loaded extension predates `page.inspect`.
+- **`includeSnapshot` now embeds the formatted snapshot.** `chrome_click`/`chrome_type`/`chrome_fill`/`chrome_key` with `includeSnapshot=true` append the fresh concise snapshot to the tool text so the agent can verify in one round trip.
+- **Snapshot logic moved to a packaged `snapshot_injected.js`.** The MAIN-world snapshot/inspect implementation ships as an eval-free packaged script, shared `window.__PI_CHROME_STATE__` (same `el-` uid scheme) with the existing input helpers.
+
## 0.15.34 — 2026-06-01
- **Every tab Pi uses now joins the session group.** Previously only `chrome_tab new`/`group` created/used the `Pi Session: <name-or-id>` group; tabs Pi drove via `page.*` actions (navigate, click, type, snapshot, screenshot, etc.) on the existing/active tab stayed ungrouped. Now any ungrouped tab Pi interacts with is pulled into this session's group, so the user can see exactly which tabs Pi is driving. Tabs already in a group (the user's or another session's) are left untouched.
diff --git a/README.md b/README.md
index b98e993..3aea586 100644
--- a/README.md
+++ b/README.md
@@ -140,7 +140,7 @@ Agents can verify page state immediately instead of blindly retrying.
| Category | Tools |
| --------------- | ---------------------------------------------------------------------------------------------- |
| **Tabs** | `chrome_tab` (list/new/activate/close/version), `chrome_launch` |
-| **Inspect** | `chrome_snapshot` (uids + selectors + text + viewport), `chrome_screenshot`, `chrome_evaluate` |
+| **Inspect** | `chrome_snapshot` (concise observation: layout, actions, forms, page map, query matches, diff + stable uids), `chrome_find` (query → ranked uids), `chrome_inspect` (deep single-element context), `chrome_screenshot`, `chrome_evaluate` |
| **Navigate** | `chrome_navigate` (with optional `initScript` at `document_start`), `chrome_wait_for` |
| **Interact** | `chrome_click`, `chrome_type`, `chrome_fill`, `chrome_key`, `chrome_hover` |
| **Gesture** | `chrome_drag` (Chrome pointer drag), `chrome_scroll` (wheel + momentum), `chrome_tap` (touch) |
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json
index 877bed5..7ee1a3c 100644
--- a/extensions/chrome-profile-bridge/browser-extension/manifest.json
+++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json
@@ -1,7 +1,7 @@
{
"manifest_version": 3,
"name": "Pi Chrome Connector",
- "version": "0.15.35",
+ "version": "0.15.36",
"description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.",
"permissions": [
"tabs",
diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
index 9a51ad7..c66e23a 100644
--- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js
+++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
@@ -833,12 +833,9 @@ async function dispatch(action, params) {
return { closed: tab.id };
}
case "page.snapshot":
- return executeInTab(params, snapshotPage, [
- params.maxElements || 80,
- params.containingText ?? null,
- params.roleFilter ?? null,
- params.nearUid ?? null,
- ]);
+ return snapshotInTab(params);
+ case "page.inspect":
+ return inspectInTab(params);
case "page.evaluate":
return evaluateInTab(params);
case "page.click":
@@ -1120,12 +1117,95 @@ async function evaluateInTab(params) {
async function withOptionalSnapshot(params, actionFn) {
const result = await actionFn(params);
if (params.includeSnapshot) {
- const snapshot = await executeInTab({ ...params, foreground: false }, snapshotPage, [params.maxElements || 80, null, null, null]);
+ const snapshot = await snapshotInTab({ ...params, foreground: false });
return { result, snapshot };
}
return result;
}
+// Snapshot/inspect run from a packaged MAIN-world script (snapshot_injected.js) injected via
+// chrome.scripting.executeScript({ files }). That file is free of eval/new Function, so it works
+// on strict-CSP pages, and it installs globalThis.__piChromeSnapshotPage / __piChromeInspectTarget.
+// It shares window.__PI_CHROME_STATE__ (same el- uid scheme) with the CDP-injected input helpers.
+async function snapshotInTab(params) {
+ const tab = await getTabByParams(params);
+ if (params.foreground) await bringToFront(tab);
+ const args = [
+ params.maxElements || 80,
+ params.containingText ?? null,
+ params.roleFilter ?? null,
+ params.nearUid ?? null,
+ params.mode || "auto",
+ params.query ?? null,
+ params.maxTextChars ?? null,
+ ];
+ await chrome.scripting.executeScript({
+ target: { tabId: tab.id, frameIds: [0] },
+ world: "MAIN",
+ files: ["snapshot_injected.js"],
+ });
+ const results = await chrome.scripting.executeScript({
+ target: { tabId: tab.id, frameIds: [0] },
+ world: "MAIN",
+ func: async (invocationArgs) => {
+ try {
+ const snapshotPage = globalThis.__piChromeSnapshotPage;
+ if (typeof snapshotPage !== "function") throw new Error("snapshot_injected.js did not install __piChromeSnapshotPage");
+ return { ok: true, value: await snapshotPage(...invocationArgs) };
+ } catch (error) {
+ return { ok: false, error: error?.stack || error?.message || String(error) };
+ }
+ },
+ args: [args],
+ });
+ const first = results?.[0];
+ if (first?.error) {
+ const message = typeof first.error === "string" ? first.error : (first.error.message || JSON.stringify(first.error));
+ throw new Error(message);
+ }
+ const envelope = first?.result;
+ if (envelope && typeof envelope === "object" && envelope.ok === false) {
+ throw new Error(envelope.error || "Chrome snapshot script failed");
+ }
+ return envelope?.value;
+}
+
+async function inspectInTab(params) {
+ if (!params.uid && !params.selector) throw new Error("chrome_inspect requires uid or selector");
+ const tab = await getTabByParams(params);
+ if (params.foreground) await bringToFront(tab);
+ const args = [params.uid ?? null, params.selector ?? null, params.scrollIntoView === true];
+ await chrome.scripting.executeScript({
+ target: { tabId: tab.id, frameIds: [0] },
+ world: "MAIN",
+ files: ["snapshot_injected.js"],
+ });
+ const results = await chrome.scripting.executeScript({
+ target: { tabId: tab.id, frameIds: [0] },
+ world: "MAIN",
+ func: async (invocationArgs) => {
+ try {
+ const inspectTarget = globalThis.__piChromeInspectTarget;
+ if (typeof inspectTarget !== "function") throw new Error("snapshot_injected.js did not install __piChromeInspectTarget");
+ return { ok: true, value: await inspectTarget(...invocationArgs) };
+ } catch (error) {
+ return { ok: false, error: error?.stack || error?.message || String(error) };
+ }
+ },
+ args: [args],
+ });
+ const first = results?.[0];
+ if (first?.error) {
+ const message = typeof first.error === "string" ? first.error : (first.error.message || JSON.stringify(first.error));
+ throw new Error(message);
+ }
+ const envelope = first?.result;
+ if (envelope && typeof envelope === "object" && envelope.ok === false) {
+ throw new Error(envelope.error || "Chrome inspect script failed");
+ }
+ return envelope?.value;
+}
+
// One-shot init script registry, scoped per tab. The source is registered with CDP
// Page.addScriptToEvaluateOnNewDocument, which runs it at document_start in the page's MAIN
// world and is NOT subject to page CSP (the old func:(code)=>new Function(code) path was
@@ -1642,104 +1722,6 @@ function installEarlyCapture() {
state.instrumentationInstalled = true;
}
-function snapshotPage(maxElements, containingText, roleFilter, nearUid) {
- installPiChromeInstrumentation();
- const unique = (selector) => {
- try { return document.querySelectorAll(selector).length === 1; } catch { return false; }
- };
- const cssEscape = (value) => (window.CSS && CSS.escape) ? CSS.escape(value) : String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&");
- const selectorFor = (element) => {
- if (element.id && unique("#" + cssEscape(element.id))) return "#" + cssEscape(element.id);
- const attr = ["aria-label", "name", "placeholder", "data-testid", "role"].find((name) => element.getAttribute(name));
- if (attr) {
- const candidate = element.tagName.toLowerCase() + "[" + attr + "=" + JSON.stringify(element.getAttribute(attr)) + "]";
- if (unique(candidate)) return candidate;
- }
- const parts = [];
- let current = element;
- while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 5) {
- let part = current.tagName.toLowerCase();
- if (current.classList.length > 0) part += "." + Array.from(current.classList).slice(0, 2).map(cssEscape).join(".");
- const siblings = Array.from(current.parentElement?.children ?? []).filter((sibling) => sibling.tagName === current.tagName);
- if (siblings.length > 1) part += ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")";
- parts.unshift(part);
- const candidate = parts.join(" > ");
- if (unique(candidate)) return candidate;
- current = current.parentElement;
- }
- return parts.join(" > ");
- };
- const visible = (element) => isElementVisible(element);
- const labelFor = (element) => (
- element.getAttribute("aria-label") ||
- element.getAttribute("title") ||
- element.getAttribute("placeholder") ||
- element.innerText ||
- element.value ||
- element.textContent ||
- ""
- ).trim().replace(/\s+/g, " ").slice(0, 160);
- let candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [role="menuitem"], [role="tab"], [role="checkbox"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'));
- if (containingText) {
- const needle = String(containingText).toLowerCase();
- candidates = candidates.filter((element) => labelFor(element).toLowerCase().includes(needle));
- }
- if (roleFilter) {
- const wanted = String(roleFilter).toLowerCase();
- candidates = candidates.filter((element) => {
- const role = (element.getAttribute("role") || element.tagName).toLowerCase();
- return role === wanted;
- });
- }
- let near;
- if (nearUid) {
- const state = getPiChromeState();
- near = state.elements[nearUid];
- }
- if (near) {
- const nearRect = near.getBoundingClientRect();
- const cx = nearRect.left + nearRect.width / 2;
- const cy = nearRect.top + nearRect.height / 2;
- candidates.sort((a, b) => {
- const ra = a.getBoundingClientRect();
- const rb = b.getBoundingClientRect();
- const da = Math.hypot(ra.left + ra.width / 2 - cx, ra.top + ra.height / 2 - cy);
- const db = Math.hypot(rb.left + rb.width / 2 - cx, rb.top + rb.height / 2 - cy);
- return da - db;
- });
- }
- const elements = candidates.filter(visible).slice(0, maxElements).map((element, index) => {
- const rect = element.getBoundingClientRect();
- const style = getComputedStyle(element);
- const cx = rect.left + rect.width / 2;
- const cy = rect.top + rect.height / 2;
- const occluded = occluderAt(cx, cy, element);
- return {
- index,
- uid: rememberElement(element),
- tag: element.tagName.toLowerCase(),
- selector: selectorFor(element),
- label: labelFor(element),
- href: element.href || undefined,
- type: element.getAttribute("type") || undefined,
- role: element.getAttribute("role") || undefined,
- disabled: Boolean(element.disabled || element.getAttribute("aria-disabled") === "true"),
- inert: Boolean(element.closest?.("[inert]")),
- pointerEvents: style.pointerEvents,
- occluded: occluded || undefined,
- rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
- };
- });
- return {
- title: document.title,
- url: location.href,
- viewport: { width: innerWidth, height: innerHeight, scrollX, scrollY },
- text: document.body ? document.body.innerText.replace(/\s+\n/g, "\n").trim().slice(0, 30000) : "",
- elements,
- filter: { containingText: containingText || undefined, roleFilter: roleFilter || undefined, nearUid: nearUid || undefined },
- };
-}
-
function probePage() {
// Sanity probe used by /chrome-doctor. Returns evidence that MAIN-world execution works.
return {
diff --git a/extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js b/extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js
new file mode 100644
index 0000000..ceee8ce
--- /dev/null
+++ b/extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js
@@ -0,0 +1,677 @@
+// Static MAIN-world snapshot implementation injected by the MV3 service worker.
+// Keep this file free of eval/new Function so `chrome_snapshot` works on strict-CSP pages.
+(() => {
+ function getPiChromeState() {
+ const state = window.__PI_CHROME_STATE__ || {
+ nextElementUid: 1,
+ elements: {},
+ console: [],
+ network: [],
+ nextRequestId: 1,
+ instrumentationInstalled: false,
+ lastSnapshotDigest: null,
+ };
+ window.__PI_CHROME_STATE__ = state;
+ return state;
+ }
+
+ function rememberElement(element) {
+ const state = getPiChromeState();
+ if (!element.__piChromeUid) element.__piChromeUid = "el-" + state.nextElementUid++;
+ state.elements[element.__piChromeUid] = element;
+ return element.__piChromeUid;
+ }
+
+ function isElementVisible(element) {
+ if (!element || !element.getBoundingClientRect) return false;
+ const style = getComputedStyle(element);
+ if (style.visibility === "hidden" || style.display === "none") return false;
+ const rect = element.getBoundingClientRect();
+ if (rect.width === 0 || rect.height === 0) return false;
+ if (rect.bottom < 0 || rect.right < 0) return false;
+ if (rect.top > innerHeight || rect.left > innerWidth) return false;
+ return true;
+ }
+
+ function occluderAt(x, y, expected) {
+ const top = document.elementFromPoint(x, y);
+ if (!top || top === expected) return null;
+ if (expected && expected.contains(top)) return null;
+ if (top.contains(expected)) return null;
+ return {
+ tag: top.tagName.toLowerCase(),
+ id: top.id || undefined,
+ className: typeof top.className === "string" ? top.className : undefined,
+ };
+ }
+
+ function textOf(element, max) {
+ return (element?.innerText || element?.textContent || "").replace(/\s+/g, " ").trim().slice(0, max || 500);
+ }
+
+ function accessibleLabel(element) {
+ if (!element) return "";
+ const labelledBy = element.getAttribute("aria-labelledby");
+ if (labelledBy) {
+ const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.innerText || "").join(" ").trim();
+ if (text) return text;
+ }
+ const id = element.id;
+ if (id) {
+ try {
+ const label = document.querySelector(`label[for="${cssEscape(id)}"]`);
+ if (label?.innerText) return label.innerText;
+ } catch {}
+ }
+ const wrappingLabel = element.closest?.("label");
+ return (
+ element.getAttribute("aria-label") ||
+ element.getAttribute("title") ||
+ element.getAttribute("placeholder") ||
+ wrappingLabel?.innerText ||
+ element.innerText ||
+ element.textContent ||
+ ""
+ ).trim().replace(/\s+/g, " ").slice(0, 180);
+ }
+
+ function cssEscape(value) {
+ return (window.CSS && CSS.escape) ? CSS.escape(value) : String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&");
+ }
+
+ function roleOf(element) {
+ const explicit = element.getAttribute("role");
+ if (explicit) return explicit.toLowerCase();
+ const tag = element.tagName.toLowerCase();
+ const type = (element.getAttribute("type") || "").toLowerCase();
+ if (tag === "a" && element.href) return "link";
+ if (tag === "button" || type === "button" || type === "submit" || type === "reset") return "button";
+ if (tag === "textarea") return "textbox";
+ if (tag === "select") return "combobox";
+ if (tag === "input") {
+ if (["checkbox", "radio", "range", "search", "email", "password", "tel", "url", "number"].includes(type)) return type === "checkbox" || type === "radio" || type === "range" ? type : "textbox";
+ return "textbox";
+ }
+ if (element.isContentEditable) return "textbox";
+ if (tag.match(/^h[1-6]$/)) return "heading";
+ return tag;
+ }
+
+ function isSensitiveField(element) {
+ if (!element) return false;
+ const tag = element.tagName?.toLowerCase?.() || "";
+ if (!/^(input|textarea|select)$/.test(tag) && !element.isContentEditable) return false;
+ const type = (element.getAttribute("type") || "").toLowerCase();
+ if (["password"].includes(type)) return true;
+ const haystack = [
+ type,
+ element.getAttribute("name"),
+ element.id,
+ element.getAttribute("autocomplete"),
+ element.getAttribute("aria-label"),
+ element.getAttribute("placeholder"),
+ element.getAttribute("data-testid"),
+ ].filter(Boolean).join(" ").toLowerCase();
+ return /password|passwd|\bpwd\b|secret|token|bearer|api[-_ ]?key|access[-_ ]?key|auth[-_ ]?code|one[-_ ]?time|otp|2fa|mfa|verification[-_ ]?code|recovery[-_ ]?code|credit[-_ ]?card|card[-_ ]?number|cc-number|cc-csc|cvc|cvv|security[-_ ]?code|ssn|social[-_ ]?security/.test(haystack);
+ }
+
+ function installPiChromeInstrumentation() {
+ const state = getPiChromeState();
+ if (state.instrumentationInstalled) return;
+ state.instrumentationInstalled = true;
+ const pushConsole = (level, args) => {
+ state.console.push({
+ id: state.console.length + 1,
+ level,
+ timestamp: Date.now(),
+ url: location.href,
+ args: Array.from(args).map((arg) => {
+ try {
+ if (typeof arg === "string") return arg;
+ if (arg instanceof Error) return { name: arg.name, message: arg.message, stack: arg.stack };
+ return JSON.parse(JSON.stringify(arg));
+ } catch {
+ return String(arg);
+ }
+ }),
+ });
+ if (state.console.length > 500) state.console.splice(0, state.console.length - 500);
+ };
+ for (const level of ["debug", "log", "info", "warn", "error"]) {
+ const original = console[level];
+ if (typeof original !== "function" || original.__piChromeWrapped) continue;
+ const wrapped = function(...args) {
+ pushConsole(level, args);
+ return original.apply(this, args);
+ };
+ wrapped.__piChromeWrapped = true;
+ console[level] = wrapped;
+ }
+ window.addEventListener("error", (event) => pushConsole("pageerror", [event.message, event.filename + ":" + event.lineno + ":" + event.colno]));
+ window.addEventListener("unhandledrejection", (event) => pushConsole("unhandledrejection", [event.reason]));
+
+ const record = (entry) => {
+ state.network.push(entry);
+ if (state.network.length > 1000) state.network.splice(0, state.network.length - 1000);
+ return entry;
+ };
+ if (window.fetch && !window.fetch.__piChromeWrapped) {
+ const originalFetch = window.fetch.bind(window);
+ const wrappedFetch = async (...args) => {
+ const id = "req-" + state.nextRequestId++;
+ const startedAt = Date.now();
+ const input = args[0];
+ const init = args[1] || {};
+ const url = typeof input === "string" ? input : input?.url;
+ const method = (init.method || input?.method || "GET").toUpperCase();
+ const entry = record({ id, type: "fetch", method, url: String(url || ""), startedAt, pageUrl: location.href, status: "pending" });
+ try {
+ const response = await originalFetch(...args);
+ entry.status = response.status;
+ entry.statusText = response.statusText;
+ entry.ok = response.ok;
+ entry.responseUrl = response.url;
+ entry.durationMs = Date.now() - startedAt;
+ entry.responseHeaders = Array.from(response.headers.entries());
+ entry.responseBodyOmitted = "response body capture is disabled by default";
+ return response;
+ } catch (error) {
+ entry.error = error?.message || String(error);
+ entry.durationMs = Date.now() - startedAt;
+ throw error;
+ }
+ };
+ wrappedFetch.__piChromeWrapped = true;
+ window.fetch = wrappedFetch;
+ }
+ if (window.XMLHttpRequest && !XMLHttpRequest.prototype.open.__piChromeWrapped) {
+ const originalOpen = XMLHttpRequest.prototype.open;
+ const originalSend = XMLHttpRequest.prototype.send;
+ XMLHttpRequest.prototype.open = function(method, url, ...rest) {
+ this.__piChromeRequest = { method: String(method || "GET").toUpperCase(), url: String(url || "") };
+ return originalOpen.call(this, method, url, ...rest);
+ };
+ XMLHttpRequest.prototype.open.__piChromeWrapped = true;
+ XMLHttpRequest.prototype.send = function(body) {
+ const id = "req-" + state.nextRequestId++;
+ const startedAt = Date.now();
+ const info = this.__piChromeRequest || {};
+ const entry = record({ id, type: "xhr", method: info.method || "GET", url: info.url || "", startedAt, pageUrl: location.href, status: "pending" });
+ this.addEventListener("loadend", () => {
+ entry.status = this.status;
+ entry.statusText = this.statusText;
+ entry.responseUrl = this.responseURL;
+ entry.durationMs = Date.now() - startedAt;
+ try { entry.responseHeadersText = this.getAllResponseHeaders(); } catch {}
+ entry.responseBodyOmitted = "response body capture is disabled by default";
+ });
+ this.addEventListener("error", () => { entry.error = "XMLHttpRequest error"; entry.durationMs = Date.now() - startedAt; });
+ return originalSend.call(this, body);
+ };
+ }
+ }
+
+ function hashString(text) {
+ let h = 0;
+ for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) | 0;
+ return h;
+ }
+
+ function selectorFor(element) {
+ const unique = (selector) => {
+ try { return document.querySelectorAll(selector).length === 1; } catch { return false; }
+ };
+ if (element.id && unique("#" + cssEscape(element.id))) return "#" + cssEscape(element.id);
+ const attr = ["aria-label", "name", "placeholder", "data-testid", "role"].find((name) => element.getAttribute(name));
+ if (attr) {
+ const candidate = element.tagName.toLowerCase() + "[" + attr + "=" + JSON.stringify(element.getAttribute(attr)) + "]";
+ if (unique(candidate)) return candidate;
+ }
+ const parts = [];
+ let current = element;
+ while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 5) {
+ let part = current.tagName.toLowerCase();
+ if (current.classList.length > 0) part += "." + Array.from(current.classList).slice(0, 2).map(cssEscape).join(".");
+ const siblings = Array.from(current.parentElement?.children ?? []).filter((sibling) => sibling.tagName === current.tagName);
+ if (siblings.length > 1) part += ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")";
+ parts.unshift(part);
+ const candidate = parts.join(" > ");
+ if (unique(candidate)) return candidate;
+ current = current.parentElement;
+ }
+ return parts.join(" > ");
+ }
+
+ function directHeadingText(element) {
+ const labelledBy = element.getAttribute?.("aria-labelledby");
+ if (labelledBy) {
+ const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.innerText || "").join(" ").replace(/\s+/g, " ").trim();
+ if (text) return text.slice(0, 180);
+ }
+ const aria = element.getAttribute?.("aria-label");
+ if (aria) return aria.trim().slice(0, 180);
+ const heading = Array.from(element.querySelectorAll?.("h1,h2,h3,h4,[role='heading']") || []).find(isElementVisible);
+ if (heading) return textOf(heading, 180);
+ return "";
+ }
+
+ function meaningfulContainerFor(element) {
+ let current = element.parentElement;
+ let fallback = current;
+ let depth = 0;
+ while (current && current !== document.body && depth++ < 8) {
+ if (!isElementVisible(current)) { current = current.parentElement; continue; }
+ const tag = current.tagName.toLowerCase();
+ const role = (current.getAttribute("role") || "").toLowerCase();
+ const cls = typeof current.className === "string" ? current.className : "";
+ const id = current.id || "";
+ const named = Boolean(current.getAttribute("aria-label") || current.getAttribute("aria-labelledby") || directHeadingText(current));
+ const semantic = /^(form|dialog|section|article|nav|header|main|aside|footer|li|tr|td|fieldset)$/.test(tag) ||
+ /^(dialog|alertdialog|region|group|listitem|row|cell|tabpanel|menu|toolbar|navigation|main|banner|contentinfo|complementary)$/.test(role);
+ const classHint = /card|panel|pane|modal|dialog|section|content|container|toolbar|menu|list|item|row|cell|header|footer|sidebar|drawer|popover|dropdown/i.test(`${id} ${cls}`);
+ const rect = current.getBoundingClientRect();
+ const childActions = current.querySelectorAll?.('a, button, input, textarea, select, [role="button"], [role="link"], [tabindex]:not([tabindex="-1"])').length || 0;
+ if ((semantic || classHint || named) && rect.width > 20 && rect.height > 20 && childActions <= 80) return current;
+ if (!fallback && rect.width > 20 && rect.height > 20) fallback = current;
+ current = current.parentElement;
+ }
+ return fallback || document.body;
+ }
+
+ function contextForElement(element) {
+ const container = meaningfulContainerFor(element);
+ if (!container || container === document.body || container === element) return undefined;
+ return {
+ uid: rememberElement(container),
+ tag: container.tagName.toLowerCase(),
+ role: roleOf(container),
+ label: directHeadingText(container) || accessibleLabel(container) || textOf(container, 140),
+ rect: rectSummary(container),
+ };
+ }
+
+ function summarizeElement(element, index) {
+ const rect = element.getBoundingClientRect();
+ const style = getComputedStyle(element);
+ const cx = rect.left + rect.width / 2;
+ const cy = rect.top + rect.height / 2;
+ const occluded = occluderAt(cx, cy, element);
+ const role = roleOf(element);
+ const disabled = Boolean(element.disabled || element.getAttribute("aria-disabled") === "true");
+ const rawValue = "value" in element && typeof element.value === "string" ? element.value : undefined;
+ const sensitive = isSensitiveField(element);
+ const value = rawValue && !sensitive ? rawValue.slice(0, 120) : undefined;
+ const checked = "checked" in element ? Boolean(element.checked) : undefined;
+ return {
+ index,
+ uid: rememberElement(element),
+ tag: element.tagName.toLowerCase(),
+ role,
+ selector: selectorFor(element),
+ label: accessibleLabel(element),
+ href: element.href || undefined,
+ type: element.getAttribute("type") || undefined,
+ value: value || undefined,
+ hasValue: rawValue ? rawValue.length > 0 : undefined,
+ valueLength: rawValue && sensitive ? rawValue.length : undefined,
+ valueRedacted: sensitive && rawValue ? true : undefined,
+ checked,
+ disabled,
+ inert: Boolean(element.closest?.("[inert]")),
+ pointerEvents: style.pointerEvents,
+ occluded: occluded || undefined,
+ context: contextForElement(element),
+ rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
+ };
+ }
+
+ function isInViewport(element) {
+ const rect = element.getBoundingClientRect();
+ return rect.bottom >= 0 && rect.right >= 0 && rect.top <= innerHeight && rect.left <= innerWidth;
+ }
+
+ function formSummaries() {
+ const fields = Array.from(document.querySelectorAll('input, textarea, select, [contenteditable="true"]'))
+ .filter(isElementVisible)
+ .slice(0, 80)
+ .map((element, index) => ({
+ ...summarizeElement(element, index),
+ required: Boolean(element.required || element.getAttribute("aria-required") === "true"),
+ invalid: Boolean(element.matches?.(":invalid") || element.getAttribute("aria-invalid") === "true"),
+ autocomplete: element.getAttribute("autocomplete") || undefined,
+ }));
+ const submits = Array.from(document.querySelectorAll('button, input[type="submit"], [role="button"]'))
+ .filter(isElementVisible)
+ .filter((element) => /submit|save|continue|next|send|sign in|log in|create|update|done/i.test(accessibleLabel(element) + " " + (element.getAttribute("type") || "")))
+ .slice(0, 30)
+ .map((element, index) => summarizeElement(element, index));
+ return { fields, submits };
+ }
+
+ function pageMap() {
+ const landmarkSelectors = [
+ ["header", 'header, [role="banner"]'],
+ ["nav", 'nav, [role="navigation"]'],
+ ["main", 'main, [role="main"]'],
+ ["aside", 'aside, [role="complementary"]'],
+ ["footer", 'footer, [role="contentinfo"]'],
+ ["dialog", 'dialog, [role="dialog"], [aria-modal="true"]'],
+ ["form", "form"],
+ ];
+ const regions = [];
+ for (const [kind, selector] of landmarkSelectors) {
+ for (const element of Array.from(document.querySelectorAll(selector)).filter(isElementVisible).slice(0, 12)) {
+ const headings = Array.from(element.querySelectorAll("h1,h2,h3,[role='heading']")).filter(isElementVisible).slice(0, 6).map((h) => textOf(h, 120));
+ const actions = Array.from(element.querySelectorAll('a, button, input, textarea, select, [role="button"], [role="link"], [tabindex]:not([tabindex="-1"])')).filter(isElementVisible).slice(0, 8).map((a) => {
+ const summary = summarizeElement(a, 0);
+ return { uid: summary.uid, role: summary.role, label: summary.label || summary.selector, disabled: summary.disabled || undefined };
+ });
+ regions.push({ kind, uid: rememberElement(element), label: accessibleLabel(element) || headings[0] || textOf(element, 100), headings, actions });
+ }
+ }
+ const headings = Array.from(document.querySelectorAll("h1,h2,h3,[role='heading']")).filter(isElementVisible).slice(0, 30).map((element) => ({
+ uid: rememberElement(element),
+ level: Number(element.tagName?.slice(1)) || Number(element.getAttribute("aria-level")) || undefined,
+ text: textOf(element, 180),
+ }));
+ return { regions, headings };
+ }
+
+ function layoutSections(elements, forms) {
+ const byUid = new Map();
+ const addToSection = (summary, kind) => {
+ const source = getPiChromeState().elements[summary.uid];
+ const container = source ? meaningfulContainerFor(source) : null;
+ if (!container || container === document.body) return;
+ const uid = rememberElement(container);
+ let section = byUid.get(uid);
+ if (!section) {
+ const rect = rectSummary(container);
+ section = {
+ uid,
+ tag: container.tagName.toLowerCase(),
+ role: roleOf(container),
+ label: directHeadingText(container) || accessibleLabel(container) || textOf(container, 160),
+ text: textOf(container, 260),
+ rect,
+ actions: [],
+ fields: [],
+ };
+ byUid.set(uid, section);
+ }
+ const item = { uid: summary.uid, role: summary.role, label: summary.label || summary.selector, disabled: summary.disabled || undefined };
+ if (kind === "field") section.fields.push(item);
+ else section.actions.push(item);
+ };
+ for (const el of (elements || []).slice(0, 80)) addToSection(el, ["textbox", "checkbox", "radio", "combobox"].includes(el.role) ? "field" : "action");
+ for (const field of (forms?.fields || []).slice(0, 80)) addToSection(field, "field");
+ const sections = Array.from(byUid.values())
+ .filter((section) => section.actions.length || section.fields.length)
+ .sort((a, b) => a.rect.y - b.rect.y || a.rect.x - b.rect.x)
+ .slice(0, 18);
+ for (const section of sections) {
+ section.actions = section.actions.slice(0, 10);
+ section.fields = section.fields.slice(0, 10);
+ }
+ return sections;
+ }
+
+ function tokenScore(haystack, query) {
+ if (!query) return 0;
+ const hay = String(haystack || "").toLowerCase();
+ const tokens = String(query).toLowerCase().split(/\W+/).filter(Boolean);
+ if (!tokens.length) return 0;
+ let score = 0;
+ for (const token of tokens) {
+ if (hay.includes(token)) score += token.length <= 2 ? 1 : 3;
+ }
+ if (hay.includes(String(query).toLowerCase())) score += 8;
+ return score;
+ }
+
+ function queryMatches(query, elements, map) {
+ if (!query) return [];
+ const candidates = [];
+ for (const element of elements) {
+ const hay = [element.role, element.label, element.selector, element.type, element.href].filter(Boolean).join(" ");
+ const score = tokenScore(hay, query);
+ if (score > 0) candidates.push({ score, kind: "element", ...element });
+ }
+ const textNodes = [];
+ for (const block of Array.from(document.querySelectorAll("h1,h2,h3,h4,p,li,td,th,label,summary,[role='alert']")).filter(isElementVisible).slice(0, 300)) {
+ const text = textOf(block, 300);
+ const score = tokenScore(text, query);
+ if (score > 0) textNodes.push({ score, kind: "text", uid: rememberElement(block), tag: block.tagName.toLowerCase(), role: roleOf(block), text, rect: rectSummary(block) });
+ }
+ for (const region of map.regions || []) {
+ const score = tokenScore([region.kind, region.label, ...(region.headings || [])].join(" "), query);
+ if (score > 0) candidates.push({ score, kind: "region", ...region });
+ }
+ return candidates.concat(textNodes).sort((a, b) => b.score - a.score).slice(0, 20);
+ }
+
+ function rectSummary(element) {
+ const rect = element.getBoundingClientRect();
+ return { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) };
+ }
+
+ function activeElementSummary() {
+ const el = document.activeElement;
+ if (!el || el === document.body || el === document.documentElement) return null;
+ return summarizeElement(el, 0);
+ }
+
+ function modalSummary() {
+ const selectors = 'dialog[open], [role="dialog"], [aria-modal="true"], [role="alertdialog"]';
+ const modal = Array.from(document.querySelectorAll(selectors)).find(isElementVisible);
+ if (!modal) return null;
+ return { uid: rememberElement(modal), tag: modal.tagName.toLowerCase(), role: roleOf(modal), label: accessibleLabel(modal) || textOf(modal, 180), rect: rectSummary(modal) };
+ }
+
+ function digestFor(snapshot) {
+ return {
+ url: snapshot.url,
+ title: snapshot.title,
+ textHash: hashString(snapshot.text || ""),
+ focusedUid: snapshot.focused?.uid || null,
+ modalUid: snapshot.modal?.uid || null,
+ labels: (snapshot.elements || []).slice(0, 50).map((el) => ({ uid: el.uid, role: el.role, label: el.label, disabled: el.disabled, value: el.value, checked: el.checked })),
+ };
+ }
+
+ function diffSnapshot(previous, current) {
+ if (!previous) return { firstSnapshot: true };
+ const changes = [];
+ if (previous.url !== current.url) changes.push({ kind: "url", before: previous.url, after: current.url });
+ if (previous.title !== current.title) changes.push({ kind: "title", before: previous.title, after: current.title });
+ if (previous.textHash !== current.textHash) changes.push({ kind: "textChanged" });
+ if (previous.focusedUid !== current.focusedUid) changes.push({ kind: "focus", before: previous.focusedUid, after: current.focusedUid });
+ if (previous.modalUid !== current.modalUid) changes.push({ kind: "modal", before: previous.modalUid, after: current.modalUid });
+ const prevByUid = new Map((previous.labels || []).map((x) => [x.uid, x]));
+ const curByUid = new Map((current.labels || []).map((x) => [x.uid, x]));
+ const added = [];
+ const removed = [];
+ const updated = [];
+ for (const cur of current.labels || []) {
+ const prev = prevByUid.get(cur.uid);
+ if (!prev) added.push(cur);
+ else if (prev.label !== cur.label || prev.disabled !== cur.disabled || prev.value !== cur.value || prev.checked !== cur.checked) updated.push({ uid: cur.uid, before: prev, after: cur });
+ }
+ for (const prev of previous.labels || []) {
+ if (!curByUid.has(prev.uid)) removed.push(prev);
+ }
+ return { changes, added: added.slice(0, 12), removed: removed.slice(0, 12), updated: updated.slice(0, 12) };
+ }
+
+ function visibleTextSnippets(maxChars) {
+ const snippets = [];
+ const blocks = Array.from(document.querySelectorAll("h1,h2,h3,h4,p,li,td,th,label,summary,[role='alert']")).filter(isElementVisible);
+ let used = 0;
+ for (const block of blocks) {
+ if (!isInViewport(block) && snippets.length > 12) continue;
+ const text = textOf(block, 500);
+ if (!text || snippets.some((s) => s.text === text)) continue;
+ const next = { uid: rememberElement(block), tag: block.tagName.toLowerCase(), text, rect: rectSummary(block) };
+ snippets.push(next);
+ used += text.length;
+ if (used >= maxChars || snippets.length >= 40) break;
+ }
+ return snippets;
+ }
+
+ function snapshotPage(maxElements, containingText, roleFilter, nearUid, mode, query, maxTextChars) {
+ installPiChromeInstrumentation();
+ mode = ["auto", "interactive", "forms", "pageMap", "text", "changes", "full"].includes(mode) ? mode : "auto";
+ const fullTextLimit = Number(maxTextChars || (mode === "full" ? 30000 : mode === "text" ? 18000 : 6000));
+ let candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [role="menuitem"], [role="tab"], [role="checkbox"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'));
+ if (containingText) {
+ const needle = String(containingText).toLowerCase();
+ candidates = candidates.filter((element) => accessibleLabel(element).toLowerCase().includes(needle));
+ }
+ if (roleFilter) {
+ const wanted = String(roleFilter).toLowerCase();
+ candidates = candidates.filter((element) => roleOf(element) === wanted || element.tagName.toLowerCase() === wanted);
+ }
+ let near;
+ if (nearUid) near = getPiChromeState().elements[nearUid];
+ if (near) {
+ const nearRect = near.getBoundingClientRect();
+ const cx = nearRect.left + nearRect.width / 2;
+ const cy = nearRect.top + nearRect.height / 2;
+ candidates.sort((a, b) => {
+ const ra = a.getBoundingClientRect();
+ const rb = b.getBoundingClientRect();
+ const da = Math.hypot(ra.left + ra.width / 2 - cx, ra.top + ra.height / 2 - cy);
+ const db = Math.hypot(rb.left + rb.width / 2 - cx, rb.top + rb.height / 2 - cy);
+ return da - db;
+ });
+ } else {
+ candidates.sort((a, b) => {
+ const ar = a.getBoundingClientRect();
+ const br = b.getBoundingClientRect();
+ const avis = isInViewport(a) ? 0 : 1;
+ const bvis = isInViewport(b) ? 0 : 1;
+ return avis - bvis || ar.top - br.top || ar.left - br.left;
+ });
+ }
+ const visibleCandidates = candidates.filter(isElementVisible);
+ const elements = visibleCandidates.slice(0, maxElements).map((element, index) => summarizeElement(element, index));
+ const queryElements = query
+ ? visibleCandidates.slice(0, Math.max(maxElements, 500)).map((element, index) => summarizeElement(element, index))
+ : elements;
+ const map = pageMap();
+ const forms = formSummaries();
+ const layout = layoutSections(elements, forms);
+ const focused = activeElementSummary();
+ const modal = modalSummary();
+ const bodyText = document.body ? document.body.innerText.replace(/\s+\n/g, "\n").trim() : "";
+ const text = bodyText.slice(0, fullTextLimit);
+ const snapshot = {
+ title: document.title,
+ url: location.href,
+ mode,
+ query: query || undefined,
+ viewport: { width: innerWidth, height: innerHeight, scrollX, scrollY },
+ summary: {
+ visibleText: textOf(document.body, 500),
+ visibleInteractiveCount: elements.filter((el) => el.rect.y >= 0 && el.rect.y <= innerHeight).length,
+ totalInteractiveSampled: elements.length,
+ totalInteractiveVisible: visibleCandidates.length,
+ focused: focused ? { uid: focused.uid, role: focused.role, label: focused.label } : undefined,
+ modal: modal ? { uid: modal.uid, label: modal.label } : undefined,
+ hints: [],
+ },
+ focused: focused || undefined,
+ modal: modal || undefined,
+ text,
+ textTruncated: bodyText.length > text.length,
+ textSnippets: visibleTextSnippets(mode === "text" ? 12000 : 3000),
+ elements,
+ forms,
+ layout,
+ pageMap: map,
+ matches: queryMatches(query, queryElements, map),
+ filter: { containingText: containingText || undefined, roleFilter: roleFilter || undefined, nearUid: nearUid || undefined },
+ };
+ if (snapshot.modal) snapshot.summary.hints.push("A modal/dialog is visible; interact with it before the underlying page.");
+ const disabledImportant = elements.find((el) => el.disabled && /submit|save|merge|continue|next|send|approve|login|sign in/i.test(el.label || ""));
+ if (disabledImportant) snapshot.summary.hints.push(`${disabledImportant.uid} '${disabledImportant.label}' is disabled.`);
+ const occluded = elements.find((el) => el.occluded);
+ if (occluded) snapshot.summary.hints.push(`${occluded.uid} '${occluded.label || occluded.role}' appears occluded by ${occluded.occluded.tag}.`);
+
+ const state = getPiChromeState();
+ const currentDigest = digestFor(snapshot);
+ snapshot.diff = diffSnapshot(state.lastSnapshotDigest, currentDigest);
+ state.lastSnapshotDigest = currentDigest;
+
+ if (mode === "interactive") {
+ delete snapshot.text;
+ delete snapshot.textSnippets;
+ delete snapshot.pageMap;
+ } else if (mode === "forms") {
+ delete snapshot.text;
+ delete snapshot.textSnippets;
+ snapshot.elements = elements.filter((el) => ["textbox", "checkbox", "radio", "combobox", "button"].includes(el.role));
+ } else if (mode === "pageMap") {
+ delete snapshot.text;
+ delete snapshot.textSnippets;
+ snapshot.elements = elements.slice(0, 20);
+ } else if (mode === "changes") {
+ delete snapshot.text;
+ delete snapshot.textSnippets;
+ delete snapshot.elements;
+ delete snapshot.forms;
+ delete snapshot.layout;
+ delete snapshot.pageMap;
+ } else if (mode === "text") {
+ snapshot.elements = elements.slice(0, 20);
+ } else if (mode !== "full") {
+ snapshot.elements = elements.slice(0, Math.min(maxElements, 40));
+ snapshot.text = text.slice(0, Math.min(text.length, 6000));
+ }
+ return snapshot;
+ }
+
+ function inspectTarget(uid, selector, shouldScrollIntoView) {
+ installPiChromeInstrumentation();
+ const state = getPiChromeState();
+ let element = null;
+ if (uid) element = state.elements[uid];
+ if (!element && selector) element = document.querySelector(selector);
+ if (!element || !element.isConnected) throw new Error(uid ? `No live element for uid: ${uid}. Take a fresh chrome_snapshot.` : `No element matches selector: ${selector}`);
+ if (shouldScrollIntoView) element.scrollIntoView?.({ block: "center", inline: "center", behavior: "instant" });
+ const summary = summarizeElement(element, 0);
+ const ancestors = [];
+ let current = element.parentElement;
+ while (current && current !== document.body && ancestors.length < 6) {
+ ancestors.push({ uid: rememberElement(current), tag: current.tagName.toLowerCase(), role: roleOf(current), label: accessibleLabel(current) || textOf(current, 100), selector: selectorFor(current) });
+ current = current.parentElement;
+ }
+ const container = element.closest?.('form, dialog, [role="dialog"], [aria-modal="true"], section, article, main, aside') || element.parentElement || document.body;
+ const nearbyText = Array.from(container.querySelectorAll("h1,h2,h3,h4,p,li,label,[role='alert']"))
+ .filter(isElementVisible)
+ .slice(0, 24)
+ .map((node) => ({ uid: rememberElement(node), tag: node.tagName.toLowerCase(), text: textOf(node, 240), rect: rectSummary(node) }))
+ .filter((entry) => entry.text);
+ const nearbyActions = Array.from(container.querySelectorAll('a, button, input, textarea, select, [role="button"], [role="link"], [tabindex]:not([tabindex="-1"])'))
+ .filter(isElementVisible)
+ .slice(0, 30)
+ .map((node, index) => summarizeElement(node, index));
+ const form = element.closest?.("form");
+ const formContext = form ? {
+ uid: rememberElement(form),
+ label: accessibleLabel(form) || textOf(form, 160),
+ fields: Array.from(form.querySelectorAll('input, textarea, select, [contenteditable="true"]')).filter(isElementVisible).slice(0, 30).map((node, index) => summarizeElement(node, index)),
+ actions: Array.from(form.querySelectorAll('button, input[type="submit"], [role="button"]')).filter(isElementVisible).slice(0, 12).map((node, index) => summarizeElement(node, index)),
+ } : undefined;
+ const rect = element.getBoundingClientRect();
+ const center = { x: Math.round(rect.left + rect.width / 2), y: Math.round(rect.top + rect.height / 2) };
+ const clickSuggestion = summary.disabled || summary.inert || summary.pointerEvents === "none"
+ ? undefined
+ : { uid: summary.uid, x: center.x, y: center.y };
+ return { target: summary, ancestors, nearbyText, nearbyActions, formContext, clickSuggestion };
+ }
+
+ globalThis.__piChromeSnapshotPage = snapshotPage;
+ globalThis.__piChromeInspectTarget = inspectTarget;
+})();
diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts
index bed1302..ef89e23 100644
--- a/extensions/chrome-profile-bridge/index.ts
+++ b/extensions/chrome-profile-bridge/index.ts
@@ -74,6 +74,137 @@ function safeJson(value: unknown): string {
return JSON.stringify(value, null, 2);
}
+const snapshotModeValues = ["auto", "interactive", "forms", "pageMap", "text", "changes", "full"] as const;
+
+function compactLine(value: unknown, max = 140): string {
+ const text = String(value ?? "").replace(/\s+/g, " ").trim();
+ return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+}
+
+function rectText(rect: any): string {
+ if (!rect) return "?";
+ return `${rect.x},${rect.y} ${rect.width}x${rect.height}`;
+}
+
+function formatChromeSnapshot(snapshot: any): string {
+ if (!snapshot || typeof snapshot !== "object") return safeJson(snapshot);
+ if (snapshot.mode === "full") return truncateText(safeJson(snapshot));
+ const lines: string[] = [];
+ lines.push(`# Chrome snapshot${snapshot.mode ? ` (${snapshot.mode})` : ""}`);
+ lines.push(`${snapshot.title || "(untitled)"}`);
+ if (snapshot.url) lines.push(`${snapshot.url}`);
+ if (snapshot.viewport) lines.push(`viewport=${snapshot.viewport.width}x${snapshot.viewport.height} scroll=${snapshot.viewport.scrollX || 0},${snapshot.viewport.scrollY || 0}`);
+ if (snapshot.summary?.modal) lines.push(`modal: ${snapshot.summary.modal.uid} ${compactLine(snapshot.summary.modal.label)}`);
+ if (snapshot.summary?.focused) lines.push(`focused: ${snapshot.summary.focused.uid} ${snapshot.summary.focused.role || ""} ${compactLine(snapshot.summary.focused.label)}`);
+ if (Array.isArray(snapshot.summary?.hints) && snapshot.summary.hints.length) {
+ lines.push("\n## Hints");
+ for (const hint of snapshot.summary.hints.slice(0, 6)) lines.push(`- ${hint}`);
+ }
+ if (snapshot.diff && !snapshot.diff.firstSnapshot) {
+ const changed = [
+ ...(snapshot.diff.changes || []).map((c: any) => c.kind === "textChanged" ? "text changed" : `${c.kind}: ${compactLine(c.before, 50)} → ${compactLine(c.after, 50)}`),
+ ...(snapshot.diff.added || []).slice(0, 4).map((e: any) => `added ${e.uid} ${e.role || ""} ${compactLine(e.label)}`),
+ ...(snapshot.diff.updated || []).slice(0, 4).map((u: any) => `updated ${u.uid} ${compactLine(u.after?.label || u.before?.label)}`),
+ ];
+ if (changed.length) {
+ lines.push("\n## Changed since last snapshot");
+ for (const item of changed.slice(0, 10)) lines.push(`- ${item}`);
+ }
+ }
+ if (Array.isArray(snapshot.matches) && snapshot.matches.length) {
+ lines.push(`\n## Matches for "${snapshot.query}"`);
+ for (const match of snapshot.matches.slice(0, 12)) {
+ if (match.kind === "text") lines.push(`- ${match.uid} text ${compactLine(match.text)} @ ${rectText(match.rect)}`);
+ else if (match.kind === "region") lines.push(`- ${match.uid} region ${compactLine(match.label)} headings=${(match.headings || []).map((h: string) => compactLine(h, 50)).join(" | ")}`);
+ else lines.push(`- ${match.uid} ${match.role || match.tag || "element"}${match.disabled ? " disabled" : ""} ${compactLine(match.label || match.selector)} @ ${rectText(match.rect)}`);
+ }
+ }
+ if (snapshot.mode === "pageMap" && snapshot.pageMap) {
+ lines.push("\n## Page map");
+ for (const region of (snapshot.pageMap.regions || []).slice(0, 18)) {
+ lines.push(`- ${region.uid} ${region.kind}: ${compactLine(region.label)}`);
+ for (const action of (region.actions || []).slice(0, 5)) lines.push(` - ${action.uid} ${action.role || ""}${action.disabled ? " disabled" : ""} ${compactLine(action.label)}`);
+ }
+ if (snapshot.pageMap.headings?.length) {
+ lines.push("\nHeadings:");
+ for (const h of snapshot.pageMap.headings.slice(0, 20)) lines.push(`- ${h.uid} h${h.level || ""} ${compactLine(h.text)}`);
+ }
+ }
+ if (Array.isArray(snapshot.layout) && snapshot.layout.length && snapshot.mode !== "changes") {
+ lines.push("\n## Layout / context");
+ for (const section of snapshot.layout.slice(0, snapshot.mode === "pageMap" ? 18 : 8)) {
+ const bits = [`${section.uid}`, section.role || section.tag, compactLine(section.label || section.text || "(unnamed section)", 110), `@ ${rectText(section.rect)}`];
+ lines.push(`- ${bits.filter(Boolean).join(" ")}`);
+ const fieldLabels = (section.fields || []).slice(0, 4).map((f: any) => `${f.uid} ${compactLine(f.label || f.role, 40)}`);
+ const actionLabels = (section.actions || []).slice(0, 5).map((a: any) => `${a.uid}${a.disabled ? " disabled" : ""} ${compactLine(a.label || a.role, 40)}`);
+ if (fieldLabels.length) lines.push(` fields: ${fieldLabels.join("; ")}`);
+ if (actionLabels.length) lines.push(` actions: ${actionLabels.join("; ")}`);
+ }
+ }
+ if ((snapshot.mode === "forms" || snapshot.forms?.fields?.length) && snapshot.mode !== "pageMap") {
+ const fields = snapshot.forms?.fields || [];
+ const submits = snapshot.forms?.submits || [];
+ if (fields.length || submits.length) lines.push("\n## Forms");
+ for (const field of fields.slice(0, snapshot.mode === "forms" ? 40 : 12)) {
+ const bits = [field.uid, field.role || field.tag, field.required ? "required" : "", field.invalid ? "invalid" : "", field.disabled ? "disabled" : "", compactLine(field.label || field.selector, 90)];
+ if (field.value) bits.push(`value=${compactLine(field.value, 50)}`);
+ else if (field.valueRedacted) bits.push("value=[redacted]");
+ lines.push(`- ${bits.filter(Boolean).join(" ")} @ ${rectText(field.rect)}`);
+ }
+ for (const submit of submits.slice(0, 8)) lines.push(`- ${submit.uid} submit/action${submit.disabled ? " disabled" : ""} ${compactLine(submit.label || submit.selector)} @ ${rectText(submit.rect)}`);
+ }
+ if (Array.isArray(snapshot.elements) && snapshot.mode !== "pageMap") {
+ lines.push("\n## Visible actions");
+ for (const el of snapshot.elements.slice(0, snapshot.mode === "interactive" ? 60 : 25)) {
+ const flags = [el.disabled ? "disabled" : "", el.occluded ? `occluded-by-${el.occluded.tag}` : ""].filter(Boolean).join(",");
+ const context = el.context?.label ? ` in ${el.context.uid} ${compactLine(el.context.label, 60)}` : "";
+ lines.push(`- ${el.uid} ${el.role || el.tag}${flags ? ` [${flags}]` : ""} ${compactLine(el.label || el.selector)}${context} @ ${rectText(el.rect)}`);
+ }
+ if (snapshot.elements.length > (snapshot.mode === "interactive" ? 60 : 25)) lines.push(`- … ${snapshot.elements.length - (snapshot.mode === "interactive" ? 60 : 25)} more; retry with maxElements or mode=interactive`);
+ }
+ if ((snapshot.mode === "text" || snapshot.mode === "auto") && Array.isArray(snapshot.textSnippets) && snapshot.textSnippets.length) {
+ lines.push("\n## Text snippets");
+ for (const snip of snapshot.textSnippets.slice(0, snapshot.mode === "text" ? 40 : 14)) lines.push(`- ${snip.uid} ${compactLine(snip.text, snapshot.mode === "text" ? 240 : 160)}`);
+ if (snapshot.textTruncated) lines.push("- … page text truncated; retry with mode=text or maxTextChars for more");
+ }
+ lines.push("\nTip: use chrome_snapshot({query:'...', mode:'interactive|forms|pageMap|text|changes|full'}) or nearUid to zoom in.");
+ return truncateText(lines.join("\n"));
+}
+
+function formatIncludedSnapshotText(raw: unknown, text: string): string {
+ const snapshot = raw && typeof raw === "object" ? (raw as { snapshot?: unknown }).snapshot : undefined;
+ return snapshot ? `${text}\n\n${formatChromeSnapshot(snapshot)}` : text;
+}
+
+function formatChromeInspect(inspect: any): string {
+ if (!inspect || typeof inspect !== "object") return safeJson(inspect);
+ const t = inspect.target || {};
+ const lines: string[] = [];
+ lines.push(`# Chrome inspect ${t.uid || ""}`.trim());
+ lines.push(`${t.role || t.tag || "element"}${t.disabled ? " disabled" : ""}${t.occluded ? ` occluded-by-${t.occluded.tag}` : ""} ${compactLine(t.label || t.selector)}`);
+ if (t.selector) lines.push(`selector: ${t.selector}`);
+ if (t.rect) lines.push(`rect: ${rectText(t.rect)}`);
+ if (inspect.clickSuggestion) lines.push(`suggested click: chrome_click({ uid: "${inspect.clickSuggestion.uid}" }) or x=${inspect.clickSuggestion.x}, y=${inspect.clickSuggestion.y}`);
+ if (Array.isArray(inspect.nearbyText) && inspect.nearbyText.length) {
+ lines.push("\n## Nearby text");
+ for (const item of inspect.nearbyText.slice(0, 12)) lines.push(`- ${item.uid} ${compactLine(item.text, 180)}`);
+ }
+ if (inspect.formContext) {
+ lines.push("\n## Form context");
+ for (const field of (inspect.formContext.fields || []).slice(0, 20)) lines.push(`- ${field.uid} ${field.role || field.tag}${field.disabled ? " disabled" : ""} ${compactLine(field.label || field.selector)}${field.value ? ` value=${compactLine(field.value, 60)}` : field.valueRedacted ? " value=[redacted]" : ""}`);
+ for (const action of (inspect.formContext.actions || []).slice(0, 10)) lines.push(`- ${action.uid} action${action.disabled ? " disabled" : ""} ${compactLine(action.label || action.selector)}`);
+ }
+ if (Array.isArray(inspect.nearbyActions) && inspect.nearbyActions.length) {
+ lines.push("\n## Nearby actions");
+ for (const action of inspect.nearbyActions.slice(0, 18)) lines.push(`- ${action.uid} ${action.role || action.tag}${action.disabled ? " disabled" : ""} ${compactLine(action.label || action.selector)} @ ${rectText(action.rect)}`);
+ }
+ if (Array.isArray(inspect.ancestors) && inspect.ancestors.length) {
+ lines.push("\n## Ancestors");
+ for (const a of inspect.ancestors.slice(0, 6)) lines.push(`- ${a.uid} ${a.role || a.tag} ${compactLine(a.label || a.selector, 120)}`);
+ }
+ return truncateText(lines.join("\n"));
+}
+
function extensionRoot(): string {
// Resolve relative to this extension file, not ctx.cwd. ctx.cwd can temporarily be
// an attachment/clipboard path when Pi is handling pasted images.
@@ -1109,13 +1240,16 @@ Usage rules:
name: "chrome_snapshot",
label: "Chrome Snapshot",
description:
- "Inspect a page in the user's existing Chrome profile: title, URL, visible body text, viewport, and clickable/focusable elements with stable uids plus CSS selectors. Runs in the background by default; pass background=false to bring Chrome to the foreground so the user can watch.",
- promptSnippet: "Inspect the current Chrome page and get CSS selectors for browser automation.",
+ "Inspect a page in the user's existing Chrome profile. Default output is a concise, agent-friendly observation with structural layout/context, stable uids, visible actions, form fields, page hints, and changes since the previous snapshot. Use mode/query/nearUid to zoom instead of dumping the whole page. Runs in the background by default; pass background=false to bring Chrome to the foreground so the user can watch.",
+ promptSnippet: "Observe the current Chrome page: concise summary, structural layout, visible actions, forms, page map, query matches, and stable uids.",
parameters: Type.Object({
targetId: Type.Optional(Type.String()),
urlIncludes: Type.Optional(Type.String()),
titleIncludes: Type.Optional(Type.String()),
maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS })),
+ mode: Type.Optional(StringEnum(snapshotModeValues)),
+ query: Type.Optional(Type.String({ description: "Find/rank elements, regions, and text matching this phrase, e.g. 'merge button', 'email error', 'approve PR'." })),
+ maxTextChars: Type.Optional(Type.Number({ description: "Max body text chars included in the underlying snapshot. Defaults are smaller for concise modes." })),
containingText: Type.Optional(Type.String({ description: "Only return elements whose label/text contains this string (case-insensitive). Useful when the page has many controls." })),
roleFilter: Type.Optional(Type.String({ description: "Only return elements matching this ARIA role or tag name (case-insensitive). e.g. 'button', 'link', 'textbox'." })),
nearUid: Type.Optional(Type.String({ description: "Sort elements by proximity to this snapshot uid. Useful for finding controls near a known anchor." })),
@@ -1132,7 +1266,84 @@ Usage rules:
DEFAULT_TIMEOUT_MS,
signal,
);
- return { content: [{ type: "text", text: truncateText(safeJson(snapshot)) }], details: { snapshot } };
+ return { content: [{ type: "text", text: formatChromeSnapshot(snapshot) }], details: { snapshot } };
+ },
+ });
+
+ pi.registerTool({
+ name: "chrome_find",
+ label: "Chrome Find",
+ description:
+ "Find elements, page regions, or text on the current Chrome page by query. Returns ranked matches with stable uids and coordinates. This is a focused wrapper around chrome_snapshot({ query }).",
+ promptSnippet: "Find matching controls/text/regions in Chrome by natural-language query and return stable uids.",
+ parameters: Type.Object({
+ query: Type.String({ description: "What to find, e.g. 'merge button', 'email error', 'approve PR', 'search box'." }),
+ mode: Type.Optional(StringEnum(snapshotModeValues)),
+ maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS })),
+ targetId: Type.Optional(Type.String()),
+ urlIncludes: Type.Optional(Type.String()),
+ titleIncludes: Type.Optional(Type.String()),
+ background: Type.Optional(
+ Type.Boolean({ description: "If true (the default), run silently in the background without focusing Chrome; pass false so Chrome focuses + the tab activates and the user can watch." }),
+ ),
+ host: Type.Optional(Type.String()),
+ port: Type.Optional(Type.Number()),
+ }),
+ async execute(_id, params, signal): Promise<ToolTextResult> {
+ const snapshot = await authorizedBridgeSend(
+ "page.snapshot",
+ withBackground({ ...params, mode: params.mode || "auto", maxElements: params.maxElements ?? MAX_ELEMENTS }),
+ DEFAULT_TIMEOUT_MS,
+ signal,
+ );
+ return { content: [{ type: "text", text: formatChromeSnapshot(snapshot) }], details: { snapshot } };
+ },
+ });
+
+ pi.registerTool({
+ name: "chrome_inspect",
+ label: "Chrome Inspect Element",
+ description:
+ "Inspect one snapshot uid or selector deeply: nearby text, nearby actions, form context, ancestors, and suggested click target. Use after chrome_snapshot/chrome_find when you need context around one element.",
+ promptSnippet: "Inspect a Chrome snapshot uid deeply for nearby text, form context, and suggested actions.",
+ parameters: Type.Object({
+ uid: Type.Optional(Type.String({ description: "Stable element uid from chrome_snapshot/chrome_find." })),
+ selector: Type.Optional(Type.String({ description: "CSS selector if uid is unavailable." })),
+ scrollIntoView: Type.Optional(Type.Boolean({ description: "If true, scroll the target into view before inspecting. Default false to avoid changing page state." })),
+ targetId: Type.Optional(Type.String()),
+ urlIncludes: Type.Optional(Type.String()),
+ titleIncludes: Type.Optional(Type.String()),
+ background: Type.Optional(
+ Type.Boolean({ description: "If true (the default), run silently in the background without focusing Chrome; pass false so Chrome focuses + the tab activates and the user can watch." }),
+ ),
+ host: Type.Optional(Type.String()),
+ port: Type.Optional(Type.Number()),
+ }),
+ async execute(_id, params, signal): Promise<ToolTextResult> {
+ try {
+ const inspect = await authorizedBridgeSend("page.inspect", withBackground(params), DEFAULT_TIMEOUT_MS, signal);
+ return { content: [{ type: "text", text: formatChromeInspect(inspect) }], details: { inspect } };
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ if (!/Unknown action: page\.inspect/i.test(message)) throw error;
+ // Compatibility fallback for a loaded Chrome extension service worker that has not
+ // been reloaded since chrome_inspect was added. It is less rich than page.inspect,
+ // but still gives useful nearby candidates instead of failing the workflow.
+ const snapshot = await authorizedBridgeSend(
+ "page.snapshot",
+ withBackground({
+ ...params,
+ mode: "interactive",
+ maxElements: MAX_ELEMENTS,
+ nearUid: params.uid,
+ query: params.selector,
+ }),
+ DEFAULT_TIMEOUT_MS,
+ signal,
+ );
+ const text = `chrome_inspect fallback: loaded Chrome extension does not yet support page.inspect; reload it at chrome://extensions for deep inspect.\n\n${formatChromeSnapshot(snapshot)}`;
+ return { content: [{ type: "text", text }], details: { snapshot, fallback: "page.snapshot" } };
+ }
},
});
@@ -1219,7 +1430,7 @@ Usage rules:
const summary = summarizeActionResult(result);
const target = params.uid ?? params.selector ?? `${params.x},${params.y}`;
const text = summary ? `Clicked ${target} — ${summary}` : `Clicked ${target}`;
- return { content: [{ type: "text", text }], details: { result: raw as Json } };
+ return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } };
},
});
@@ -1251,7 +1462,8 @@ Usage rules:
const summary = summarizeActionResult(result);
const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : "";
const base = `Typed ${params.text.length} character(s)${into}.`;
- return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } };
+ const text = summary ? `${base} (${summary})` : base;
+ return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } };
},
});
@@ -1283,7 +1495,8 @@ Usage rules:
const summary = summarizeActionResult(result);
const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : "";
const base = `Filled ${params.text.length} character(s)${into}.`;
- return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } };
+ const text = summary ? `${base} (${summary})` : base;
+ return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } };
},
});
@@ -1317,7 +1530,8 @@ Usage rules:
const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json;
const summary = summarizeActionResult(result);
const base = `Pressed ${params.key}.`;
- return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } };
+ const text = summary ? `${base} (${summary})` : base;
+ return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } };
},
});
diff --git a/package.json b/package.json
index d2a2b18..3279144 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "pi-chrome",
- "version": "0.15.35",
+ "version": "0.15.36",
"scripts": {
"test": "node test-suite/unit/csp-eval.test.mjs",
"version": "node scripts/sync-manifest-version.js",