aboutsummaryrefslogtreecommitdiff
path: root/extensions
diff options
context:
space:
mode:
authorTianren Dong <tianrendong.dtr@gmail.com>2026-06-03 20:57:27 -0400
committerTianren Dong <tianrendong.dtr@gmail.com>2026-06-06 14:07:55 -0400
commit341e2b3cebe41a4a14b00d83a84208971e0df7f5 (patch)
tree8c1657ee2850247f4b31ab8a2f6605e37b490fe2 /extensions
parentfa33b0797755b146b175f4aa792c8d0c22ba781f (diff)
0.15.36: richer chrome_snapshot + chrome_find/chrome_inspect
- chrome_snapshot returns a concise observation (layout, hints, actions, forms, page map, query matches, diff) with mode/query/maxTextChars params - new chrome_find (query -> ranked uids) and chrome_inspect (deep element context, with page.inspect fallback) - includeSnapshot on click/type/fill/key now embeds the formatted snapshot - snapshot/inspect logic shipped as eval-free packaged snapshot_injected.js, sharing window.__PI_CHROME_STATE__ (el- uid scheme) with existing input helpers
Diffstat (limited to 'extensions')
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/manifest.json2
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/service_worker.js192
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js677
-rw-r--r--extensions/chrome-profile-bridge/index.ts228
4 files changed, 986 insertions, 113 deletions
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json
index 877bed5..7ee1a3c 100644
--- a/extensions/chrome-profile-bridge/browser-extension/manifest.json
+++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json
@@ -1,7 +1,7 @@
{
"manifest_version": 3,
"name": "Pi Chrome Connector",
- "version": "0.15.35",
+ "version": "0.15.36",
"description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.",
"permissions": [
"tabs",
diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
index 9a51ad7..c66e23a 100644
--- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js
+++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
@@ -833,12 +833,9 @@ async function dispatch(action, params) {
return { closed: tab.id };
}
case "page.snapshot":
- return executeInTab(params, snapshotPage, [
- params.maxElements || 80,
- params.containingText ?? null,
- params.roleFilter ?? null,
- params.nearUid ?? null,
- ]);
+ return snapshotInTab(params);
+ case "page.inspect":
+ return inspectInTab(params);
case "page.evaluate":
return evaluateInTab(params);
case "page.click":
@@ -1120,12 +1117,95 @@ async function evaluateInTab(params) {
async function withOptionalSnapshot(params, actionFn) {
const result = await actionFn(params);
if (params.includeSnapshot) {
- const snapshot = await executeInTab({ ...params, foreground: false }, snapshotPage, [params.maxElements || 80, null, null, null]);
+ const snapshot = await snapshotInTab({ ...params, foreground: false });
return { result, snapshot };
}
return result;
}
+// Snapshot/inspect run from a packaged MAIN-world script (snapshot_injected.js) injected via
+// chrome.scripting.executeScript({ files }). That file is free of eval/new Function, so it works
+// on strict-CSP pages, and it installs globalThis.__piChromeSnapshotPage / __piChromeInspectTarget.
+// It shares window.__PI_CHROME_STATE__ (same el- uid scheme) with the CDP-injected input helpers.
+async function snapshotInTab(params) {
+ const tab = await getTabByParams(params);
+ if (params.foreground) await bringToFront(tab);
+ const args = [
+ params.maxElements || 80,
+ params.containingText ?? null,
+ params.roleFilter ?? null,
+ params.nearUid ?? null,
+ params.mode || "auto",
+ params.query ?? null,
+ params.maxTextChars ?? null,
+ ];
+ await chrome.scripting.executeScript({
+ target: { tabId: tab.id, frameIds: [0] },
+ world: "MAIN",
+ files: ["snapshot_injected.js"],
+ });
+ const results = await chrome.scripting.executeScript({
+ target: { tabId: tab.id, frameIds: [0] },
+ world: "MAIN",
+ func: async (invocationArgs) => {
+ try {
+ const snapshotPage = globalThis.__piChromeSnapshotPage;
+ if (typeof snapshotPage !== "function") throw new Error("snapshot_injected.js did not install __piChromeSnapshotPage");
+ return { ok: true, value: await snapshotPage(...invocationArgs) };
+ } catch (error) {
+ return { ok: false, error: error?.stack || error?.message || String(error) };
+ }
+ },
+ args: [args],
+ });
+ const first = results?.[0];
+ if (first?.error) {
+ const message = typeof first.error === "string" ? first.error : (first.error.message || JSON.stringify(first.error));
+ throw new Error(message);
+ }
+ const envelope = first?.result;
+ if (envelope && typeof envelope === "object" && envelope.ok === false) {
+ throw new Error(envelope.error || "Chrome snapshot script failed");
+ }
+ return envelope?.value;
+}
+
+async function inspectInTab(params) {
+ if (!params.uid && !params.selector) throw new Error("chrome_inspect requires uid or selector");
+ const tab = await getTabByParams(params);
+ if (params.foreground) await bringToFront(tab);
+ const args = [params.uid ?? null, params.selector ?? null, params.scrollIntoView === true];
+ await chrome.scripting.executeScript({
+ target: { tabId: tab.id, frameIds: [0] },
+ world: "MAIN",
+ files: ["snapshot_injected.js"],
+ });
+ const results = await chrome.scripting.executeScript({
+ target: { tabId: tab.id, frameIds: [0] },
+ world: "MAIN",
+ func: async (invocationArgs) => {
+ try {
+ const inspectTarget = globalThis.__piChromeInspectTarget;
+ if (typeof inspectTarget !== "function") throw new Error("snapshot_injected.js did not install __piChromeInspectTarget");
+ return { ok: true, value: await inspectTarget(...invocationArgs) };
+ } catch (error) {
+ return { ok: false, error: error?.stack || error?.message || String(error) };
+ }
+ },
+ args: [args],
+ });
+ const first = results?.[0];
+ if (first?.error) {
+ const message = typeof first.error === "string" ? first.error : (first.error.message || JSON.stringify(first.error));
+ throw new Error(message);
+ }
+ const envelope = first?.result;
+ if (envelope && typeof envelope === "object" && envelope.ok === false) {
+ throw new Error(envelope.error || "Chrome inspect script failed");
+ }
+ return envelope?.value;
+}
+
// One-shot init script registry, scoped per tab. The source is registered with CDP
// Page.addScriptToEvaluateOnNewDocument, which runs it at document_start in the page's MAIN
// world and is NOT subject to page CSP (the old func:(code)=>new Function(code) path was
@@ -1642,104 +1722,6 @@ function installEarlyCapture() {
state.instrumentationInstalled = true;
}
-function snapshotPage(maxElements, containingText, roleFilter, nearUid) {
- installPiChromeInstrumentation();
- const unique = (selector) => {
- try { return document.querySelectorAll(selector).length === 1; } catch { return false; }
- };
- const cssEscape = (value) => (window.CSS && CSS.escape) ? CSS.escape(value) : String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&");
- const selectorFor = (element) => {
- if (element.id && unique("#" + cssEscape(element.id))) return "#" + cssEscape(element.id);
- const attr = ["aria-label", "name", "placeholder", "data-testid", "role"].find((name) => element.getAttribute(name));
- if (attr) {
- const candidate = element.tagName.toLowerCase() + "[" + attr + "=" + JSON.stringify(element.getAttribute(attr)) + "]";
- if (unique(candidate)) return candidate;
- }
- const parts = [];
- let current = element;
- while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 5) {
- let part = current.tagName.toLowerCase();
- if (current.classList.length > 0) part += "." + Array.from(current.classList).slice(0, 2).map(cssEscape).join(".");
- const siblings = Array.from(current.parentElement?.children ?? []).filter((sibling) => sibling.tagName === current.tagName);
- if (siblings.length > 1) part += ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")";
- parts.unshift(part);
- const candidate = parts.join(" > ");
- if (unique(candidate)) return candidate;
- current = current.parentElement;
- }
- return parts.join(" > ");
- };
- const visible = (element) => isElementVisible(element);
- const labelFor = (element) => (
- element.getAttribute("aria-label") ||
- element.getAttribute("title") ||
- element.getAttribute("placeholder") ||
- element.innerText ||
- element.value ||
- element.textContent ||
- ""
- ).trim().replace(/\s+/g, " ").slice(0, 160);
- let candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [role="menuitem"], [role="tab"], [role="checkbox"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'));
- if (containingText) {
- const needle = String(containingText).toLowerCase();
- candidates = candidates.filter((element) => labelFor(element).toLowerCase().includes(needle));
- }
- if (roleFilter) {
- const wanted = String(roleFilter).toLowerCase();
- candidates = candidates.filter((element) => {
- const role = (element.getAttribute("role") || element.tagName).toLowerCase();
- return role === wanted;
- });
- }
- let near;
- if (nearUid) {
- const state = getPiChromeState();
- near = state.elements[nearUid];
- }
- if (near) {
- const nearRect = near.getBoundingClientRect();
- const cx = nearRect.left + nearRect.width / 2;
- const cy = nearRect.top + nearRect.height / 2;
- candidates.sort((a, b) => {
- const ra = a.getBoundingClientRect();
- const rb = b.getBoundingClientRect();
- const da = Math.hypot(ra.left + ra.width / 2 - cx, ra.top + ra.height / 2 - cy);
- const db = Math.hypot(rb.left + rb.width / 2 - cx, rb.top + rb.height / 2 - cy);
- return da - db;
- });
- }
- const elements = candidates.filter(visible).slice(0, maxElements).map((element, index) => {
- const rect = element.getBoundingClientRect();
- const style = getComputedStyle(element);
- const cx = rect.left + rect.width / 2;
- const cy = rect.top + rect.height / 2;
- const occluded = occluderAt(cx, cy, element);
- return {
- index,
- uid: rememberElement(element),
- tag: element.tagName.toLowerCase(),
- selector: selectorFor(element),
- label: labelFor(element),
- href: element.href || undefined,
- type: element.getAttribute("type") || undefined,
- role: element.getAttribute("role") || undefined,
- disabled: Boolean(element.disabled || element.getAttribute("aria-disabled") === "true"),
- inert: Boolean(element.closest?.("[inert]")),
- pointerEvents: style.pointerEvents,
- occluded: occluded || undefined,
- rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
- };
- });
- return {
- title: document.title,
- url: location.href,
- viewport: { width: innerWidth, height: innerHeight, scrollX, scrollY },
- text: document.body ? document.body.innerText.replace(/\s+\n/g, "\n").trim().slice(0, 30000) : "",
- elements,
- filter: { containingText: containingText || undefined, roleFilter: roleFilter || undefined, nearUid: nearUid || undefined },
- };
-}
-
function probePage() {
// Sanity probe used by /chrome-doctor. Returns evidence that MAIN-world execution works.
return {
diff --git a/extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js b/extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js
new file mode 100644
index 0000000..ceee8ce
--- /dev/null
+++ b/extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js
@@ -0,0 +1,677 @@
+// Static MAIN-world snapshot implementation injected by the MV3 service worker.
+// Keep this file free of eval/new Function so `chrome_snapshot` works on strict-CSP pages.
+(() => {
+ function getPiChromeState() {
+ const state = window.__PI_CHROME_STATE__ || {
+ nextElementUid: 1,
+ elements: {},
+ console: [],
+ network: [],
+ nextRequestId: 1,
+ instrumentationInstalled: false,
+ lastSnapshotDigest: null,
+ };
+ window.__PI_CHROME_STATE__ = state;
+ return state;
+ }
+
+ function rememberElement(element) {
+ const state = getPiChromeState();
+ if (!element.__piChromeUid) element.__piChromeUid = "el-" + state.nextElementUid++;
+ state.elements[element.__piChromeUid] = element;
+ return element.__piChromeUid;
+ }
+
+ function isElementVisible(element) {
+ if (!element || !element.getBoundingClientRect) return false;
+ const style = getComputedStyle(element);
+ if (style.visibility === "hidden" || style.display === "none") return false;
+ const rect = element.getBoundingClientRect();
+ if (rect.width === 0 || rect.height === 0) return false;
+ if (rect.bottom < 0 || rect.right < 0) return false;
+ if (rect.top > innerHeight || rect.left > innerWidth) return false;
+ return true;
+ }
+
+ function occluderAt(x, y, expected) {
+ const top = document.elementFromPoint(x, y);
+ if (!top || top === expected) return null;
+ if (expected && expected.contains(top)) return null;
+ if (top.contains(expected)) return null;
+ return {
+ tag: top.tagName.toLowerCase(),
+ id: top.id || undefined,
+ className: typeof top.className === "string" ? top.className : undefined,
+ };
+ }
+
+ function textOf(element, max) {
+ return (element?.innerText || element?.textContent || "").replace(/\s+/g, " ").trim().slice(0, max || 500);
+ }
+
+ function accessibleLabel(element) {
+ if (!element) return "";
+ const labelledBy = element.getAttribute("aria-labelledby");
+ if (labelledBy) {
+ const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.innerText || "").join(" ").trim();
+ if (text) return text;
+ }
+ const id = element.id;
+ if (id) {
+ try {
+ const label = document.querySelector(`label[for="${cssEscape(id)}"]`);
+ if (label?.innerText) return label.innerText;
+ } catch {}
+ }
+ const wrappingLabel = element.closest?.("label");
+ return (
+ element.getAttribute("aria-label") ||
+ element.getAttribute("title") ||
+ element.getAttribute("placeholder") ||
+ wrappingLabel?.innerText ||
+ element.innerText ||
+ element.textContent ||
+ ""
+ ).trim().replace(/\s+/g, " ").slice(0, 180);
+ }
+
+ function cssEscape(value) {
+ return (window.CSS && CSS.escape) ? CSS.escape(value) : String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&");
+ }
+
+ function roleOf(element) {
+ const explicit = element.getAttribute("role");
+ if (explicit) return explicit.toLowerCase();
+ const tag = element.tagName.toLowerCase();
+ const type = (element.getAttribute("type") || "").toLowerCase();
+ if (tag === "a" && element.href) return "link";
+ if (tag === "button" || type === "button" || type === "submit" || type === "reset") return "button";
+ if (tag === "textarea") return "textbox";
+ if (tag === "select") return "combobox";
+ if (tag === "input") {
+ if (["checkbox", "radio", "range", "search", "email", "password", "tel", "url", "number"].includes(type)) return type === "checkbox" || type === "radio" || type === "range" ? type : "textbox";
+ return "textbox";
+ }
+ if (element.isContentEditable) return "textbox";
+ if (tag.match(/^h[1-6]$/)) return "heading";
+ return tag;
+ }
+
+ function isSensitiveField(element) {
+ if (!element) return false;
+ const tag = element.tagName?.toLowerCase?.() || "";
+ if (!/^(input|textarea|select)$/.test(tag) && !element.isContentEditable) return false;
+ const type = (element.getAttribute("type") || "").toLowerCase();
+ if (["password"].includes(type)) return true;
+ const haystack = [
+ type,
+ element.getAttribute("name"),
+ element.id,
+ element.getAttribute("autocomplete"),
+ element.getAttribute("aria-label"),
+ element.getAttribute("placeholder"),
+ element.getAttribute("data-testid"),
+ ].filter(Boolean).join(" ").toLowerCase();
+ return /password|passwd|\bpwd\b|secret|token|bearer|api[-_ ]?key|access[-_ ]?key|auth[-_ ]?code|one[-_ ]?time|otp|2fa|mfa|verification[-_ ]?code|recovery[-_ ]?code|credit[-_ ]?card|card[-_ ]?number|cc-number|cc-csc|cvc|cvv|security[-_ ]?code|ssn|social[-_ ]?security/.test(haystack);
+ }
+
+ function installPiChromeInstrumentation() {
+ const state = getPiChromeState();
+ if (state.instrumentationInstalled) return;
+ state.instrumentationInstalled = true;
+ const pushConsole = (level, args) => {
+ state.console.push({
+ id: state.console.length + 1,
+ level,
+ timestamp: Date.now(),
+ url: location.href,
+ args: Array.from(args).map((arg) => {
+ try {
+ if (typeof arg === "string") return arg;
+ if (arg instanceof Error) return { name: arg.name, message: arg.message, stack: arg.stack };
+ return JSON.parse(JSON.stringify(arg));
+ } catch {
+ return String(arg);
+ }
+ }),
+ });
+ if (state.console.length > 500) state.console.splice(0, state.console.length - 500);
+ };
+ for (const level of ["debug", "log", "info", "warn", "error"]) {
+ const original = console[level];
+ if (typeof original !== "function" || original.__piChromeWrapped) continue;
+ const wrapped = function(...args) {
+ pushConsole(level, args);
+ return original.apply(this, args);
+ };
+ wrapped.__piChromeWrapped = true;
+ console[level] = wrapped;
+ }
+ window.addEventListener("error", (event) => pushConsole("pageerror", [event.message, event.filename + ":" + event.lineno + ":" + event.colno]));
+ window.addEventListener("unhandledrejection", (event) => pushConsole("unhandledrejection", [event.reason]));
+
+ const record = (entry) => {
+ state.network.push(entry);
+ if (state.network.length > 1000) state.network.splice(0, state.network.length - 1000);
+ return entry;
+ };
+ if (window.fetch && !window.fetch.__piChromeWrapped) {
+ const originalFetch = window.fetch.bind(window);
+ const wrappedFetch = async (...args) => {
+ const id = "req-" + state.nextRequestId++;
+ const startedAt = Date.now();
+ const input = args[0];
+ const init = args[1] || {};
+ const url = typeof input === "string" ? input : input?.url;
+ const method = (init.method || input?.method || "GET").toUpperCase();
+ const entry = record({ id, type: "fetch", method, url: String(url || ""), startedAt, pageUrl: location.href, status: "pending" });
+ try {
+ const response = await originalFetch(...args);
+ entry.status = response.status;
+ entry.statusText = response.statusText;
+ entry.ok = response.ok;
+ entry.responseUrl = response.url;
+ entry.durationMs = Date.now() - startedAt;
+ entry.responseHeaders = Array.from(response.headers.entries());
+ entry.responseBodyOmitted = "response body capture is disabled by default";
+ return response;
+ } catch (error) {
+ entry.error = error?.message || String(error);
+ entry.durationMs = Date.now() - startedAt;
+ throw error;
+ }
+ };
+ wrappedFetch.__piChromeWrapped = true;
+ window.fetch = wrappedFetch;
+ }
+ if (window.XMLHttpRequest && !XMLHttpRequest.prototype.open.__piChromeWrapped) {
+ const originalOpen = XMLHttpRequest.prototype.open;
+ const originalSend = XMLHttpRequest.prototype.send;
+ XMLHttpRequest.prototype.open = function(method, url, ...rest) {
+ this.__piChromeRequest = { method: String(method || "GET").toUpperCase(), url: String(url || "") };
+ return originalOpen.call(this, method, url, ...rest);
+ };
+ XMLHttpRequest.prototype.open.__piChromeWrapped = true;
+ XMLHttpRequest.prototype.send = function(body) {
+ const id = "req-" + state.nextRequestId++;
+ const startedAt = Date.now();
+ const info = this.__piChromeRequest || {};
+ const entry = record({ id, type: "xhr", method: info.method || "GET", url: info.url || "", startedAt, pageUrl: location.href, status: "pending" });
+ this.addEventListener("loadend", () => {
+ entry.status = this.status;
+ entry.statusText = this.statusText;
+ entry.responseUrl = this.responseURL;
+ entry.durationMs = Date.now() - startedAt;
+ try { entry.responseHeadersText = this.getAllResponseHeaders(); } catch {}
+ entry.responseBodyOmitted = "response body capture is disabled by default";
+ });
+ this.addEventListener("error", () => { entry.error = "XMLHttpRequest error"; entry.durationMs = Date.now() - startedAt; });
+ return originalSend.call(this, body);
+ };
+ }
+ }
+
+ function hashString(text) {
+ let h = 0;
+ for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) | 0;
+ return h;
+ }
+
+ function selectorFor(element) {
+ const unique = (selector) => {
+ try { return document.querySelectorAll(selector).length === 1; } catch { return false; }
+ };
+ if (element.id && unique("#" + cssEscape(element.id))) return "#" + cssEscape(element.id);
+ const attr = ["aria-label", "name", "placeholder", "data-testid", "role"].find((name) => element.getAttribute(name));
+ if (attr) {
+ const candidate = element.tagName.toLowerCase() + "[" + attr + "=" + JSON.stringify(element.getAttribute(attr)) + "]";
+ if (unique(candidate)) return candidate;
+ }
+ const parts = [];
+ let current = element;
+ while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 5) {
+ let part = current.tagName.toLowerCase();
+ if (current.classList.length > 0) part += "." + Array.from(current.classList).slice(0, 2).map(cssEscape).join(".");
+ const siblings = Array.from(current.parentElement?.children ?? []).filter((sibling) => sibling.tagName === current.tagName);
+ if (siblings.length > 1) part += ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")";
+ parts.unshift(part);
+ const candidate = parts.join(" > ");
+ if (unique(candidate)) return candidate;
+ current = current.parentElement;
+ }
+ return parts.join(" > ");
+ }
+
+ function directHeadingText(element) {
+ const labelledBy = element.getAttribute?.("aria-labelledby");
+ if (labelledBy) {
+ const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.innerText || "").join(" ").replace(/\s+/g, " ").trim();
+ if (text) return text.slice(0, 180);
+ }
+ const aria = element.getAttribute?.("aria-label");
+ if (aria) return aria.trim().slice(0, 180);
+ const heading = Array.from(element.querySelectorAll?.("h1,h2,h3,h4,[role='heading']") || []).find(isElementVisible);
+ if (heading) return textOf(heading, 180);
+ return "";
+ }
+
+ function meaningfulContainerFor(element) {
+ let current = element.parentElement;
+ let fallback = current;
+ let depth = 0;
+ while (current && current !== document.body && depth++ < 8) {
+ if (!isElementVisible(current)) { current = current.parentElement; continue; }
+ const tag = current.tagName.toLowerCase();
+ const role = (current.getAttribute("role") || "").toLowerCase();
+ const cls = typeof current.className === "string" ? current.className : "";
+ const id = current.id || "";
+ const named = Boolean(current.getAttribute("aria-label") || current.getAttribute("aria-labelledby") || directHeadingText(current));
+ const semantic = /^(form|dialog|section|article|nav|header|main|aside|footer|li|tr|td|fieldset)$/.test(tag) ||
+ /^(dialog|alertdialog|region|group|listitem|row|cell|tabpanel|menu|toolbar|navigation|main|banner|contentinfo|complementary)$/.test(role);
+ const classHint = /card|panel|pane|modal|dialog|section|content|container|toolbar|menu|list|item|row|cell|header|footer|sidebar|drawer|popover|dropdown/i.test(`${id} ${cls}`);
+ const rect = current.getBoundingClientRect();
+ const childActions = current.querySelectorAll?.('a, button, input, textarea, select, [role="button"], [role="link"], [tabindex]:not([tabindex="-1"])').length || 0;
+ if ((semantic || classHint || named) && rect.width > 20 && rect.height > 20 && childActions <= 80) return current;
+ if (!fallback && rect.width > 20 && rect.height > 20) fallback = current;
+ current = current.parentElement;
+ }
+ return fallback || document.body;
+ }
+
+ function contextForElement(element) {
+ const container = meaningfulContainerFor(element);
+ if (!container || container === document.body || container === element) return undefined;
+ return {
+ uid: rememberElement(container),
+ tag: container.tagName.toLowerCase(),
+ role: roleOf(container),
+ label: directHeadingText(container) || accessibleLabel(container) || textOf(container, 140),
+ rect: rectSummary(container),
+ };
+ }
+
+ function summarizeElement(element, index) {
+ const rect = element.getBoundingClientRect();
+ const style = getComputedStyle(element);
+ const cx = rect.left + rect.width / 2;
+ const cy = rect.top + rect.height / 2;
+ const occluded = occluderAt(cx, cy, element);
+ const role = roleOf(element);
+ const disabled = Boolean(element.disabled || element.getAttribute("aria-disabled") === "true");
+ const rawValue = "value" in element && typeof element.value === "string" ? element.value : undefined;
+ const sensitive = isSensitiveField(element);
+ const value = rawValue && !sensitive ? rawValue.slice(0, 120) : undefined;
+ const checked = "checked" in element ? Boolean(element.checked) : undefined;
+ return {
+ index,
+ uid: rememberElement(element),
+ tag: element.tagName.toLowerCase(),
+ role,
+ selector: selectorFor(element),
+ label: accessibleLabel(element),
+ href: element.href || undefined,
+ type: element.getAttribute("type") || undefined,
+ value: value || undefined,
+ hasValue: rawValue ? rawValue.length > 0 : undefined,
+ valueLength: rawValue && sensitive ? rawValue.length : undefined,
+ valueRedacted: sensitive && rawValue ? true : undefined,
+ checked,
+ disabled,
+ inert: Boolean(element.closest?.("[inert]")),
+ pointerEvents: style.pointerEvents,
+ occluded: occluded || undefined,
+ context: contextForElement(element),
+ rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
+ };
+ }
+
+ function isInViewport(element) {
+ const rect = element.getBoundingClientRect();
+ return rect.bottom >= 0 && rect.right >= 0 && rect.top <= innerHeight && rect.left <= innerWidth;
+ }
+
+ function formSummaries() {
+ const fields = Array.from(document.querySelectorAll('input, textarea, select, [contenteditable="true"]'))
+ .filter(isElementVisible)
+ .slice(0, 80)
+ .map((element, index) => ({
+ ...summarizeElement(element, index),
+ required: Boolean(element.required || element.getAttribute("aria-required") === "true"),
+ invalid: Boolean(element.matches?.(":invalid") || element.getAttribute("aria-invalid") === "true"),
+ autocomplete: element.getAttribute("autocomplete") || undefined,
+ }));
+ const submits = Array.from(document.querySelectorAll('button, input[type="submit"], [role="button"]'))
+ .filter(isElementVisible)
+ .filter((element) => /submit|save|continue|next|send|sign in|log in|create|update|done/i.test(accessibleLabel(element) + " " + (element.getAttribute("type") || "")))
+ .slice(0, 30)
+ .map((element, index) => summarizeElement(element, index));
+ return { fields, submits };
+ }
+
+ function pageMap() {
+ const landmarkSelectors = [
+ ["header", 'header, [role="banner"]'],
+ ["nav", 'nav, [role="navigation"]'],
+ ["main", 'main, [role="main"]'],
+ ["aside", 'aside, [role="complementary"]'],
+ ["footer", 'footer, [role="contentinfo"]'],
+ ["dialog", 'dialog, [role="dialog"], [aria-modal="true"]'],
+ ["form", "form"],
+ ];
+ const regions = [];
+ for (const [kind, selector] of landmarkSelectors) {
+ for (const element of Array.from(document.querySelectorAll(selector)).filter(isElementVisible).slice(0, 12)) {
+ const headings = Array.from(element.querySelectorAll("h1,h2,h3,[role='heading']")).filter(isElementVisible).slice(0, 6).map((h) => textOf(h, 120));
+ const actions = Array.from(element.querySelectorAll('a, button, input, textarea, select, [role="button"], [role="link"], [tabindex]:not([tabindex="-1"])')).filter(isElementVisible).slice(0, 8).map((a) => {
+ const summary = summarizeElement(a, 0);
+ return { uid: summary.uid, role: summary.role, label: summary.label || summary.selector, disabled: summary.disabled || undefined };
+ });
+ regions.push({ kind, uid: rememberElement(element), label: accessibleLabel(element) || headings[0] || textOf(element, 100), headings, actions });
+ }
+ }
+ const headings = Array.from(document.querySelectorAll("h1,h2,h3,[role='heading']")).filter(isElementVisible).slice(0, 30).map((element) => ({
+ uid: rememberElement(element),
+ level: Number(element.tagName?.slice(1)) || Number(element.getAttribute("aria-level")) || undefined,
+ text: textOf(element, 180),
+ }));
+ return { regions, headings };
+ }
+
+ function layoutSections(elements, forms) {
+ const byUid = new Map();
+ const addToSection = (summary, kind) => {
+ const source = getPiChromeState().elements[summary.uid];
+ const container = source ? meaningfulContainerFor(source) : null;
+ if (!container || container === document.body) return;
+ const uid = rememberElement(container);
+ let section = byUid.get(uid);
+ if (!section) {
+ const rect = rectSummary(container);
+ section = {
+ uid,
+ tag: container.tagName.toLowerCase(),
+ role: roleOf(container),
+ label: directHeadingText(container) || accessibleLabel(container) || textOf(container, 160),
+ text: textOf(container, 260),
+ rect,
+ actions: [],
+ fields: [],
+ };
+ byUid.set(uid, section);
+ }
+ const item = { uid: summary.uid, role: summary.role, label: summary.label || summary.selector, disabled: summary.disabled || undefined };
+ if (kind === "field") section.fields.push(item);
+ else section.actions.push(item);
+ };
+ for (const el of (elements || []).slice(0, 80)) addToSection(el, ["textbox", "checkbox", "radio", "combobox"].includes(el.role) ? "field" : "action");
+ for (const field of (forms?.fields || []).slice(0, 80)) addToSection(field, "field");
+ const sections = Array.from(byUid.values())
+ .filter((section) => section.actions.length || section.fields.length)
+ .sort((a, b) => a.rect.y - b.rect.y || a.rect.x - b.rect.x)
+ .slice(0, 18);
+ for (const section of sections) {
+ section.actions = section.actions.slice(0, 10);
+ section.fields = section.fields.slice(0, 10);
+ }
+ return sections;
+ }
+
+ function tokenScore(haystack, query) {
+ if (!query) return 0;
+ const hay = String(haystack || "").toLowerCase();
+ const tokens = String(query).toLowerCase().split(/\W+/).filter(Boolean);
+ if (!tokens.length) return 0;
+ let score = 0;
+ for (const token of tokens) {
+ if (hay.includes(token)) score += token.length <= 2 ? 1 : 3;
+ }
+ if (hay.includes(String(query).toLowerCase())) score += 8;
+ return score;
+ }
+
+ function queryMatches(query, elements, map) {
+ if (!query) return [];
+ const candidates = [];
+ for (const element of elements) {
+ const hay = [element.role, element.label, element.selector, element.type, element.href].filter(Boolean).join(" ");
+ const score = tokenScore(hay, query);
+ if (score > 0) candidates.push({ score, kind: "element", ...element });
+ }
+ const textNodes = [];
+ for (const block of Array.from(document.querySelectorAll("h1,h2,h3,h4,p,li,td,th,label,summary,[role='alert']")).filter(isElementVisible).slice(0, 300)) {
+ const text = textOf(block, 300);
+ const score = tokenScore(text, query);
+ if (score > 0) textNodes.push({ score, kind: "text", uid: rememberElement(block), tag: block.tagName.toLowerCase(), role: roleOf(block), text, rect: rectSummary(block) });
+ }
+ for (const region of map.regions || []) {
+ const score = tokenScore([region.kind, region.label, ...(region.headings || [])].join(" "), query);
+ if (score > 0) candidates.push({ score, kind: "region", ...region });
+ }
+ return candidates.concat(textNodes).sort((a, b) => b.score - a.score).slice(0, 20);
+ }
+
+ function rectSummary(element) {
+ const rect = element.getBoundingClientRect();
+ return { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) };
+ }
+
+ function activeElementSummary() {
+ const el = document.activeElement;
+ if (!el || el === document.body || el === document.documentElement) return null;
+ return summarizeElement(el, 0);
+ }
+
+ function modalSummary() {
+ const selectors = 'dialog[open], [role="dialog"], [aria-modal="true"], [role="alertdialog"]';
+ const modal = Array.from(document.querySelectorAll(selectors)).find(isElementVisible);
+ if (!modal) return null;
+ return { uid: rememberElement(modal), tag: modal.tagName.toLowerCase(), role: roleOf(modal), label: accessibleLabel(modal) || textOf(modal, 180), rect: rectSummary(modal) };
+ }
+
+ function digestFor(snapshot) {
+ return {
+ url: snapshot.url,
+ title: snapshot.title,
+ textHash: hashString(snapshot.text || ""),
+ focusedUid: snapshot.focused?.uid || null,
+ modalUid: snapshot.modal?.uid || null,
+ labels: (snapshot.elements || []).slice(0, 50).map((el) => ({ uid: el.uid, role: el.role, label: el.label, disabled: el.disabled, value: el.value, checked: el.checked })),
+ };
+ }
+
+ function diffSnapshot(previous, current) {
+ if (!previous) return { firstSnapshot: true };
+ const changes = [];
+ if (previous.url !== current.url) changes.push({ kind: "url", before: previous.url, after: current.url });
+ if (previous.title !== current.title) changes.push({ kind: "title", before: previous.title, after: current.title });
+ if (previous.textHash !== current.textHash) changes.push({ kind: "textChanged" });
+ if (previous.focusedUid !== current.focusedUid) changes.push({ kind: "focus", before: previous.focusedUid, after: current.focusedUid });
+ if (previous.modalUid !== current.modalUid) changes.push({ kind: "modal", before: previous.modalUid, after: current.modalUid });
+ const prevByUid = new Map((previous.labels || []).map((x) => [x.uid, x]));
+ const curByUid = new Map((current.labels || []).map((x) => [x.uid, x]));
+ const added = [];
+ const removed = [];
+ const updated = [];
+ for (const cur of current.labels || []) {
+ const prev = prevByUid.get(cur.uid);
+ if (!prev) added.push(cur);
+ else if (prev.label !== cur.label || prev.disabled !== cur.disabled || prev.value !== cur.value || prev.checked !== cur.checked) updated.push({ uid: cur.uid, before: prev, after: cur });
+ }
+ for (const prev of previous.labels || []) {
+ if (!curByUid.has(prev.uid)) removed.push(prev);
+ }
+ return { changes, added: added.slice(0, 12), removed: removed.slice(0, 12), updated: updated.slice(0, 12) };
+ }
+
+ function visibleTextSnippets(maxChars) {
+ const snippets = [];
+ const blocks = Array.from(document.querySelectorAll("h1,h2,h3,h4,p,li,td,th,label,summary,[role='alert']")).filter(isElementVisible);
+ let used = 0;
+ for (const block of blocks) {
+ if (!isInViewport(block) && snippets.length > 12) continue;
+ const text = textOf(block, 500);
+ if (!text || snippets.some((s) => s.text === text)) continue;
+ const next = { uid: rememberElement(block), tag: block.tagName.toLowerCase(), text, rect: rectSummary(block) };
+ snippets.push(next);
+ used += text.length;
+ if (used >= maxChars || snippets.length >= 40) break;
+ }
+ return snippets;
+ }
+
+ function snapshotPage(maxElements, containingText, roleFilter, nearUid, mode, query, maxTextChars) {
+ installPiChromeInstrumentation();
+ mode = ["auto", "interactive", "forms", "pageMap", "text", "changes", "full"].includes(mode) ? mode : "auto";
+ const fullTextLimit = Number(maxTextChars || (mode === "full" ? 30000 : mode === "text" ? 18000 : 6000));
+ let candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [role="menuitem"], [role="tab"], [role="checkbox"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'));
+ if (containingText) {
+ const needle = String(containingText).toLowerCase();
+ candidates = candidates.filter((element) => accessibleLabel(element).toLowerCase().includes(needle));
+ }
+ if (roleFilter) {
+ const wanted = String(roleFilter).toLowerCase();
+ candidates = candidates.filter((element) => roleOf(element) === wanted || element.tagName.toLowerCase() === wanted);
+ }
+ let near;
+ if (nearUid) near = getPiChromeState().elements[nearUid];
+ if (near) {
+ const nearRect = near.getBoundingClientRect();
+ const cx = nearRect.left + nearRect.width / 2;
+ const cy = nearRect.top + nearRect.height / 2;
+ candidates.sort((a, b) => {
+ const ra = a.getBoundingClientRect();
+ const rb = b.getBoundingClientRect();
+ const da = Math.hypot(ra.left + ra.width / 2 - cx, ra.top + ra.height / 2 - cy);
+ const db = Math.hypot(rb.left + rb.width / 2 - cx, rb.top + rb.height / 2 - cy);
+ return da - db;
+ });
+ } else {
+ candidates.sort((a, b) => {
+ const ar = a.getBoundingClientRect();
+ const br = b.getBoundingClientRect();
+ const avis = isInViewport(a) ? 0 : 1;
+ const bvis = isInViewport(b) ? 0 : 1;
+ return avis - bvis || ar.top - br.top || ar.left - br.left;
+ });
+ }
+ const visibleCandidates = candidates.filter(isElementVisible);
+ const elements = visibleCandidates.slice(0, maxElements).map((element, index) => summarizeElement(element, index));
+ const queryElements = query
+ ? visibleCandidates.slice(0, Math.max(maxElements, 500)).map((element, index) => summarizeElement(element, index))
+ : elements;
+ const map = pageMap();
+ const forms = formSummaries();
+ const layout = layoutSections(elements, forms);
+ const focused = activeElementSummary();
+ const modal = modalSummary();
+ const bodyText = document.body ? document.body.innerText.replace(/\s+\n/g, "\n").trim() : "";
+ const text = bodyText.slice(0, fullTextLimit);
+ const snapshot = {
+ title: document.title,
+ url: location.href,
+ mode,
+ query: query || undefined,
+ viewport: { width: innerWidth, height: innerHeight, scrollX, scrollY },
+ summary: {
+ visibleText: textOf(document.body, 500),
+ visibleInteractiveCount: elements.filter((el) => el.rect.y >= 0 && el.rect.y <= innerHeight).length,
+ totalInteractiveSampled: elements.length,
+ totalInteractiveVisible: visibleCandidates.length,
+ focused: focused ? { uid: focused.uid, role: focused.role, label: focused.label } : undefined,
+ modal: modal ? { uid: modal.uid, label: modal.label } : undefined,
+ hints: [],
+ },
+ focused: focused || undefined,
+ modal: modal || undefined,
+ text,
+ textTruncated: bodyText.length > text.length,
+ textSnippets: visibleTextSnippets(mode === "text" ? 12000 : 3000),
+ elements,
+ forms,
+ layout,
+ pageMap: map,
+ matches: queryMatches(query, queryElements, map),
+ filter: { containingText: containingText || undefined, roleFilter: roleFilter || undefined, nearUid: nearUid || undefined },
+ };
+ if (snapshot.modal) snapshot.summary.hints.push("A modal/dialog is visible; interact with it before the underlying page.");
+ const disabledImportant = elements.find((el) => el.disabled && /submit|save|merge|continue|next|send|approve|login|sign in/i.test(el.label || ""));
+ if (disabledImportant) snapshot.summary.hints.push(`${disabledImportant.uid} '${disabledImportant.label}' is disabled.`);
+ const occluded = elements.find((el) => el.occluded);
+ if (occluded) snapshot.summary.hints.push(`${occluded.uid} '${occluded.label || occluded.role}' appears occluded by ${occluded.occluded.tag}.`);
+
+ const state = getPiChromeState();
+ const currentDigest = digestFor(snapshot);
+ snapshot.diff = diffSnapshot(state.lastSnapshotDigest, currentDigest);
+ state.lastSnapshotDigest = currentDigest;
+
+ if (mode === "interactive") {
+ delete snapshot.text;
+ delete snapshot.textSnippets;
+ delete snapshot.pageMap;
+ } else if (mode === "forms") {
+ delete snapshot.text;
+ delete snapshot.textSnippets;
+ snapshot.elements = elements.filter((el) => ["textbox", "checkbox", "radio", "combobox", "button"].includes(el.role));
+ } else if (mode === "pageMap") {
+ delete snapshot.text;
+ delete snapshot.textSnippets;
+ snapshot.elements = elements.slice(0, 20);
+ } else if (mode === "changes") {
+ delete snapshot.text;
+ delete snapshot.textSnippets;
+ delete snapshot.elements;
+ delete snapshot.forms;
+ delete snapshot.layout;
+ delete snapshot.pageMap;
+ } else if (mode === "text") {
+ snapshot.elements = elements.slice(0, 20);
+ } else if (mode !== "full") {
+ snapshot.elements = elements.slice(0, Math.min(maxElements, 40));
+ snapshot.text = text.slice(0, Math.min(text.length, 6000));
+ }
+ return snapshot;
+ }
+
+ function inspectTarget(uid, selector, shouldScrollIntoView) {
+ installPiChromeInstrumentation();
+ const state = getPiChromeState();
+ let element = null;
+ if (uid) element = state.elements[uid];
+ if (!element && selector) element = document.querySelector(selector);
+ if (!element || !element.isConnected) throw new Error(uid ? `No live element for uid: ${uid}. Take a fresh chrome_snapshot.` : `No element matches selector: ${selector}`);
+ if (shouldScrollIntoView) element.scrollIntoView?.({ block: "center", inline: "center", behavior: "instant" });
+ const summary = summarizeElement(element, 0);
+ const ancestors = [];
+ let current = element.parentElement;
+ while (current && current !== document.body && ancestors.length < 6) {
+ ancestors.push({ uid: rememberElement(current), tag: current.tagName.toLowerCase(), role: roleOf(current), label: accessibleLabel(current) || textOf(current, 100), selector: selectorFor(current) });
+ current = current.parentElement;
+ }
+ const container = element.closest?.('form, dialog, [role="dialog"], [aria-modal="true"], section, article, main, aside') || element.parentElement || document.body;
+ const nearbyText = Array.from(container.querySelectorAll("h1,h2,h3,h4,p,li,label,[role='alert']"))
+ .filter(isElementVisible)
+ .slice(0, 24)
+ .map((node) => ({ uid: rememberElement(node), tag: node.tagName.toLowerCase(), text: textOf(node, 240), rect: rectSummary(node) }))
+ .filter((entry) => entry.text);
+ const nearbyActions = Array.from(container.querySelectorAll('a, button, input, textarea, select, [role="button"], [role="link"], [tabindex]:not([tabindex="-1"])'))
+ .filter(isElementVisible)
+ .slice(0, 30)
+ .map((node, index) => summarizeElement(node, index));
+ const form = element.closest?.("form");
+ const formContext = form ? {
+ uid: rememberElement(form),
+ label: accessibleLabel(form) || textOf(form, 160),
+ fields: Array.from(form.querySelectorAll('input, textarea, select, [contenteditable="true"]')).filter(isElementVisible).slice(0, 30).map((node, index) => summarizeElement(node, index)),
+ actions: Array.from(form.querySelectorAll('button, input[type="submit"], [role="button"]')).filter(isElementVisible).slice(0, 12).map((node, index) => summarizeElement(node, index)),
+ } : undefined;
+ const rect = element.getBoundingClientRect();
+ const center = { x: Math.round(rect.left + rect.width / 2), y: Math.round(rect.top + rect.height / 2) };
+ const clickSuggestion = summary.disabled || summary.inert || summary.pointerEvents === "none"
+ ? undefined
+ : { uid: summary.uid, x: center.x, y: center.y };
+ return { target: summary, ancestors, nearbyText, nearbyActions, formContext, clickSuggestion };
+ }
+
+ globalThis.__piChromeSnapshotPage = snapshotPage;
+ globalThis.__piChromeInspectTarget = inspectTarget;
+})();
diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts
index bed1302..ef89e23 100644
--- a/extensions/chrome-profile-bridge/index.ts
+++ b/extensions/chrome-profile-bridge/index.ts
@@ -74,6 +74,137 @@ function safeJson(value: unknown): string {
return JSON.stringify(value, null, 2);
}
+const snapshotModeValues = ["auto", "interactive", "forms", "pageMap", "text", "changes", "full"] as const;
+
+function compactLine(value: unknown, max = 140): string {
+ const text = String(value ?? "").replace(/\s+/g, " ").trim();
+ return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+}
+
+function rectText(rect: any): string {
+ if (!rect) return "?";
+ return `${rect.x},${rect.y} ${rect.width}x${rect.height}`;
+}
+
+function formatChromeSnapshot(snapshot: any): string {
+ if (!snapshot || typeof snapshot !== "object") return safeJson(snapshot);
+ if (snapshot.mode === "full") return truncateText(safeJson(snapshot));
+ const lines: string[] = [];
+ lines.push(`# Chrome snapshot${snapshot.mode ? ` (${snapshot.mode})` : ""}`);
+ lines.push(`${snapshot.title || "(untitled)"}`);
+ if (snapshot.url) lines.push(`${snapshot.url}`);
+ if (snapshot.viewport) lines.push(`viewport=${snapshot.viewport.width}x${snapshot.viewport.height} scroll=${snapshot.viewport.scrollX || 0},${snapshot.viewport.scrollY || 0}`);
+ if (snapshot.summary?.modal) lines.push(`modal: ${snapshot.summary.modal.uid} ${compactLine(snapshot.summary.modal.label)}`);
+ if (snapshot.summary?.focused) lines.push(`focused: ${snapshot.summary.focused.uid} ${snapshot.summary.focused.role || ""} ${compactLine(snapshot.summary.focused.label)}`);
+ if (Array.isArray(snapshot.summary?.hints) && snapshot.summary.hints.length) {
+ lines.push("\n## Hints");
+ for (const hint of snapshot.summary.hints.slice(0, 6)) lines.push(`- ${hint}`);
+ }
+ if (snapshot.diff && !snapshot.diff.firstSnapshot) {
+ const changed = [
+ ...(snapshot.diff.changes || []).map((c: any) => c.kind === "textChanged" ? "text changed" : `${c.kind}: ${compactLine(c.before, 50)} → ${compactLine(c.after, 50)}`),
+ ...(snapshot.diff.added || []).slice(0, 4).map((e: any) => `added ${e.uid} ${e.role || ""} ${compactLine(e.label)}`),
+ ...(snapshot.diff.updated || []).slice(0, 4).map((u: any) => `updated ${u.uid} ${compactLine(u.after?.label || u.before?.label)}`),
+ ];
+ if (changed.length) {
+ lines.push("\n## Changed since last snapshot");
+ for (const item of changed.slice(0, 10)) lines.push(`- ${item}`);
+ }
+ }
+ if (Array.isArray(snapshot.matches) && snapshot.matches.length) {
+ lines.push(`\n## Matches for "${snapshot.query}"`);
+ for (const match of snapshot.matches.slice(0, 12)) {
+ if (match.kind === "text") lines.push(`- ${match.uid} text ${compactLine(match.text)} @ ${rectText(match.rect)}`);
+ else if (match.kind === "region") lines.push(`- ${match.uid} region ${compactLine(match.label)} headings=${(match.headings || []).map((h: string) => compactLine(h, 50)).join(" | ")}`);
+ else lines.push(`- ${match.uid} ${match.role || match.tag || "element"}${match.disabled ? " disabled" : ""} ${compactLine(match.label || match.selector)} @ ${rectText(match.rect)}`);
+ }
+ }
+ if (snapshot.mode === "pageMap" && snapshot.pageMap) {
+ lines.push("\n## Page map");
+ for (const region of (snapshot.pageMap.regions || []).slice(0, 18)) {
+ lines.push(`- ${region.uid} ${region.kind}: ${compactLine(region.label)}`);
+ for (const action of (region.actions || []).slice(0, 5)) lines.push(` - ${action.uid} ${action.role || ""}${action.disabled ? " disabled" : ""} ${compactLine(action.label)}`);
+ }
+ if (snapshot.pageMap.headings?.length) {
+ lines.push("\nHeadings:");
+ for (const h of snapshot.pageMap.headings.slice(0, 20)) lines.push(`- ${h.uid} h${h.level || ""} ${compactLine(h.text)}`);
+ }
+ }
+ if (Array.isArray(snapshot.layout) && snapshot.layout.length && snapshot.mode !== "changes") {
+ lines.push("\n## Layout / context");
+ for (const section of snapshot.layout.slice(0, snapshot.mode === "pageMap" ? 18 : 8)) {
+ const bits = [`${section.uid}`, section.role || section.tag, compactLine(section.label || section.text || "(unnamed section)", 110), `@ ${rectText(section.rect)}`];
+ lines.push(`- ${bits.filter(Boolean).join(" ")}`);
+ const fieldLabels = (section.fields || []).slice(0, 4).map((f: any) => `${f.uid} ${compactLine(f.label || f.role, 40)}`);
+ const actionLabels = (section.actions || []).slice(0, 5).map((a: any) => `${a.uid}${a.disabled ? " disabled" : ""} ${compactLine(a.label || a.role, 40)}`);
+ if (fieldLabels.length) lines.push(` fields: ${fieldLabels.join("; ")}`);
+ if (actionLabels.length) lines.push(` actions: ${actionLabels.join("; ")}`);
+ }
+ }
+ if ((snapshot.mode === "forms" || snapshot.forms?.fields?.length) && snapshot.mode !== "pageMap") {
+ const fields = snapshot.forms?.fields || [];
+ const submits = snapshot.forms?.submits || [];
+ if (fields.length || submits.length) lines.push("\n## Forms");
+ for (const field of fields.slice(0, snapshot.mode === "forms" ? 40 : 12)) {
+ const bits = [field.uid, field.role || field.tag, field.required ? "required" : "", field.invalid ? "invalid" : "", field.disabled ? "disabled" : "", compactLine(field.label || field.selector, 90)];
+ if (field.value) bits.push(`value=${compactLine(field.value, 50)}`);
+ else if (field.valueRedacted) bits.push("value=[redacted]");
+ lines.push(`- ${bits.filter(Boolean).join(" ")} @ ${rectText(field.rect)}`);
+ }
+ for (const submit of submits.slice(0, 8)) lines.push(`- ${submit.uid} submit/action${submit.disabled ? " disabled" : ""} ${compactLine(submit.label || submit.selector)} @ ${rectText(submit.rect)}`);
+ }
+ if (Array.isArray(snapshot.elements) && snapshot.mode !== "pageMap") {
+ lines.push("\n## Visible actions");
+ for (const el of snapshot.elements.slice(0, snapshot.mode === "interactive" ? 60 : 25)) {
+ const flags = [el.disabled ? "disabled" : "", el.occluded ? `occluded-by-${el.occluded.tag}` : ""].filter(Boolean).join(",");
+ const context = el.context?.label ? ` in ${el.context.uid} ${compactLine(el.context.label, 60)}` : "";
+ lines.push(`- ${el.uid} ${el.role || el.tag}${flags ? ` [${flags}]` : ""} ${compactLine(el.label || el.selector)}${context} @ ${rectText(el.rect)}`);
+ }
+ if (snapshot.elements.length > (snapshot.mode === "interactive" ? 60 : 25)) lines.push(`- … ${snapshot.elements.length - (snapshot.mode === "interactive" ? 60 : 25)} more; retry with maxElements or mode=interactive`);
+ }
+ if ((snapshot.mode === "text" || snapshot.mode === "auto") && Array.isArray(snapshot.textSnippets) && snapshot.textSnippets.length) {
+ lines.push("\n## Text snippets");
+ for (const snip of snapshot.textSnippets.slice(0, snapshot.mode === "text" ? 40 : 14)) lines.push(`- ${snip.uid} ${compactLine(snip.text, snapshot.mode === "text" ? 240 : 160)}`);
+ if (snapshot.textTruncated) lines.push("- … page text truncated; retry with mode=text or maxTextChars for more");
+ }
+ lines.push("\nTip: use chrome_snapshot({query:'...', mode:'interactive|forms|pageMap|text|changes|full'}) or nearUid to zoom in.");
+ return truncateText(lines.join("\n"));
+}
+
+function formatIncludedSnapshotText(raw: unknown, text: string): string {
+ const snapshot = raw && typeof raw === "object" ? (raw as { snapshot?: unknown }).snapshot : undefined;
+ return snapshot ? `${text}\n\n${formatChromeSnapshot(snapshot)}` : text;
+}
+
+function formatChromeInspect(inspect: any): string {
+ if (!inspect || typeof inspect !== "object") return safeJson(inspect);
+ const t = inspect.target || {};
+ const lines: string[] = [];
+ lines.push(`# Chrome inspect ${t.uid || ""}`.trim());
+ lines.push(`${t.role || t.tag || "element"}${t.disabled ? " disabled" : ""}${t.occluded ? ` occluded-by-${t.occluded.tag}` : ""} ${compactLine(t.label || t.selector)}`);
+ if (t.selector) lines.push(`selector: ${t.selector}`);
+ if (t.rect) lines.push(`rect: ${rectText(t.rect)}`);
+ if (inspect.clickSuggestion) lines.push(`suggested click: chrome_click({ uid: "${inspect.clickSuggestion.uid}" }) or x=${inspect.clickSuggestion.x}, y=${inspect.clickSuggestion.y}`);
+ if (Array.isArray(inspect.nearbyText) && inspect.nearbyText.length) {
+ lines.push("\n## Nearby text");
+ for (const item of inspect.nearbyText.slice(0, 12)) lines.push(`- ${item.uid} ${compactLine(item.text, 180)}`);
+ }
+ if (inspect.formContext) {
+ lines.push("\n## Form context");
+ for (const field of (inspect.formContext.fields || []).slice(0, 20)) lines.push(`- ${field.uid} ${field.role || field.tag}${field.disabled ? " disabled" : ""} ${compactLine(field.label || field.selector)}${field.value ? ` value=${compactLine(field.value, 60)}` : field.valueRedacted ? " value=[redacted]" : ""}`);
+ for (const action of (inspect.formContext.actions || []).slice(0, 10)) lines.push(`- ${action.uid} action${action.disabled ? " disabled" : ""} ${compactLine(action.label || action.selector)}`);
+ }
+ if (Array.isArray(inspect.nearbyActions) && inspect.nearbyActions.length) {
+ lines.push("\n## Nearby actions");
+ for (const action of inspect.nearbyActions.slice(0, 18)) lines.push(`- ${action.uid} ${action.role || action.tag}${action.disabled ? " disabled" : ""} ${compactLine(action.label || action.selector)} @ ${rectText(action.rect)}`);
+ }
+ if (Array.isArray(inspect.ancestors) && inspect.ancestors.length) {
+ lines.push("\n## Ancestors");
+ for (const a of inspect.ancestors.slice(0, 6)) lines.push(`- ${a.uid} ${a.role || a.tag} ${compactLine(a.label || a.selector, 120)}`);
+ }
+ return truncateText(lines.join("\n"));
+}
+
function extensionRoot(): string {
// Resolve relative to this extension file, not ctx.cwd. ctx.cwd can temporarily be
// an attachment/clipboard path when Pi is handling pasted images.
@@ -1109,13 +1240,16 @@ Usage rules:
name: "chrome_snapshot",
label: "Chrome Snapshot",
description:
- "Inspect a page in the user's existing Chrome profile: title, URL, visible body text, viewport, and clickable/focusable elements with stable uids plus CSS selectors. Runs in the background by default; pass background=false to bring Chrome to the foreground so the user can watch.",
- promptSnippet: "Inspect the current Chrome page and get CSS selectors for browser automation.",
+ "Inspect a page in the user's existing Chrome profile. Default output is a concise, agent-friendly observation with structural layout/context, stable uids, visible actions, form fields, page hints, and changes since the previous snapshot. Use mode/query/nearUid to zoom instead of dumping the whole page. Runs in the background by default; pass background=false to bring Chrome to the foreground so the user can watch.",
+ promptSnippet: "Observe the current Chrome page: concise summary, structural layout, visible actions, forms, page map, query matches, and stable uids.",
parameters: Type.Object({
targetId: Type.Optional(Type.String()),
urlIncludes: Type.Optional(Type.String()),
titleIncludes: Type.Optional(Type.String()),
maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS })),
+ mode: Type.Optional(StringEnum(snapshotModeValues)),
+ query: Type.Optional(Type.String({ description: "Find/rank elements, regions, and text matching this phrase, e.g. 'merge button', 'email error', 'approve PR'." })),
+ maxTextChars: Type.Optional(Type.Number({ description: "Max body text chars included in the underlying snapshot. Defaults are smaller for concise modes." })),
containingText: Type.Optional(Type.String({ description: "Only return elements whose label/text contains this string (case-insensitive). Useful when the page has many controls." })),
roleFilter: Type.Optional(Type.String({ description: "Only return elements matching this ARIA role or tag name (case-insensitive). e.g. 'button', 'link', 'textbox'." })),
nearUid: Type.Optional(Type.String({ description: "Sort elements by proximity to this snapshot uid. Useful for finding controls near a known anchor." })),
@@ -1132,7 +1266,84 @@ Usage rules:
DEFAULT_TIMEOUT_MS,
signal,
);
- return { content: [{ type: "text", text: truncateText(safeJson(snapshot)) }], details: { snapshot } };
+ return { content: [{ type: "text", text: formatChromeSnapshot(snapshot) }], details: { snapshot } };
+ },
+ });
+
+ pi.registerTool({
+ name: "chrome_find",
+ label: "Chrome Find",
+ description:
+ "Find elements, page regions, or text on the current Chrome page by query. Returns ranked matches with stable uids and coordinates. This is a focused wrapper around chrome_snapshot({ query }).",
+ promptSnippet: "Find matching controls/text/regions in Chrome by natural-language query and return stable uids.",
+ parameters: Type.Object({
+ query: Type.String({ description: "What to find, e.g. 'merge button', 'email error', 'approve PR', 'search box'." }),
+ mode: Type.Optional(StringEnum(snapshotModeValues)),
+ maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS })),
+ targetId: Type.Optional(Type.String()),
+ urlIncludes: Type.Optional(Type.String()),
+ titleIncludes: Type.Optional(Type.String()),
+ background: Type.Optional(
+ Type.Boolean({ description: "If true (the default), run silently in the background without focusing Chrome; pass false so Chrome focuses + the tab activates and the user can watch." }),
+ ),
+ host: Type.Optional(Type.String()),
+ port: Type.Optional(Type.Number()),
+ }),
+ async execute(_id, params, signal): Promise<ToolTextResult> {
+ const snapshot = await authorizedBridgeSend(
+ "page.snapshot",
+ withBackground({ ...params, mode: params.mode || "auto", maxElements: params.maxElements ?? MAX_ELEMENTS }),
+ DEFAULT_TIMEOUT_MS,
+ signal,
+ );
+ return { content: [{ type: "text", text: formatChromeSnapshot(snapshot) }], details: { snapshot } };
+ },
+ });
+
+ pi.registerTool({
+ name: "chrome_inspect",
+ label: "Chrome Inspect Element",
+ description:
+ "Inspect one snapshot uid or selector deeply: nearby text, nearby actions, form context, ancestors, and suggested click target. Use after chrome_snapshot/chrome_find when you need context around one element.",
+ promptSnippet: "Inspect a Chrome snapshot uid deeply for nearby text, form context, and suggested actions.",
+ parameters: Type.Object({
+ uid: Type.Optional(Type.String({ description: "Stable element uid from chrome_snapshot/chrome_find." })),
+ selector: Type.Optional(Type.String({ description: "CSS selector if uid is unavailable." })),
+ scrollIntoView: Type.Optional(Type.Boolean({ description: "If true, scroll the target into view before inspecting. Default false to avoid changing page state." })),
+ targetId: Type.Optional(Type.String()),
+ urlIncludes: Type.Optional(Type.String()),
+ titleIncludes: Type.Optional(Type.String()),
+ background: Type.Optional(
+ Type.Boolean({ description: "If true (the default), run silently in the background without focusing Chrome; pass false so Chrome focuses + the tab activates and the user can watch." }),
+ ),
+ host: Type.Optional(Type.String()),
+ port: Type.Optional(Type.Number()),
+ }),
+ async execute(_id, params, signal): Promise<ToolTextResult> {
+ try {
+ const inspect = await authorizedBridgeSend("page.inspect", withBackground(params), DEFAULT_TIMEOUT_MS, signal);
+ return { content: [{ type: "text", text: formatChromeInspect(inspect) }], details: { inspect } };
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ if (!/Unknown action: page\.inspect/i.test(message)) throw error;
+ // Compatibility fallback for a loaded Chrome extension service worker that has not
+ // been reloaded since chrome_inspect was added. It is less rich than page.inspect,
+ // but still gives useful nearby candidates instead of failing the workflow.
+ const snapshot = await authorizedBridgeSend(
+ "page.snapshot",
+ withBackground({
+ ...params,
+ mode: "interactive",
+ maxElements: MAX_ELEMENTS,
+ nearUid: params.uid,
+ query: params.selector,
+ }),
+ DEFAULT_TIMEOUT_MS,
+ signal,
+ );
+ const text = `chrome_inspect fallback: loaded Chrome extension does not yet support page.inspect; reload it at chrome://extensions for deep inspect.\n\n${formatChromeSnapshot(snapshot)}`;
+ return { content: [{ type: "text", text }], details: { snapshot, fallback: "page.snapshot" } };
+ }
},
});
@@ -1219,7 +1430,7 @@ Usage rules:
const summary = summarizeActionResult(result);
const target = params.uid ?? params.selector ?? `${params.x},${params.y}`;
const text = summary ? `Clicked ${target} — ${summary}` : `Clicked ${target}`;
- return { content: [{ type: "text", text }], details: { result: raw as Json } };
+ return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } };
},
});
@@ -1251,7 +1462,8 @@ Usage rules:
const summary = summarizeActionResult(result);
const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : "";
const base = `Typed ${params.text.length} character(s)${into}.`;
- return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } };
+ const text = summary ? `${base} (${summary})` : base;
+ return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } };
},
});
@@ -1283,7 +1495,8 @@ Usage rules:
const summary = summarizeActionResult(result);
const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : "";
const base = `Filled ${params.text.length} character(s)${into}.`;
- return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } };
+ const text = summary ? `${base} (${summary})` : base;
+ return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } };
},
});
@@ -1317,7 +1530,8 @@ Usage rules:
const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json;
const summary = summarizeActionResult(result);
const base = `Pressed ${params.key}.`;
- return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } };
+ const text = summary ? `${base} (${summary})` : base;
+ return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } };
},
});