diff options
| author | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-06-03 20:57:27 -0400 |
|---|---|---|
| committer | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-06-06 14:07:55 -0400 |
| commit | 341e2b3cebe41a4a14b00d83a84208971e0df7f5 (patch) | |
| tree | 8c1657ee2850247f4b31ab8a2f6605e37b490fe2 /extensions | |
| parent | fa33b0797755b146b175f4aa792c8d0c22ba781f (diff) | |
0.15.36: richer chrome_snapshot + chrome_find/chrome_inspect
- chrome_snapshot returns a concise observation (layout, hints, actions, forms,
page map, query matches, diff) with mode/query/maxTextChars params
- new chrome_find (query -> ranked uids) and chrome_inspect (deep element context,
with page.inspect fallback)
- includeSnapshot on click/type/fill/key now embeds the formatted snapshot
- snapshot/inspect logic shipped as eval-free packaged snapshot_injected.js,
sharing window.__PI_CHROME_STATE__ (el- uid scheme) with existing input helpers
Diffstat (limited to 'extensions')
4 files changed, 986 insertions, 113 deletions
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json index 877bed5..7ee1a3c 100644 --- a/extensions/chrome-profile-bridge/browser-extension/manifest.json +++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Pi Chrome Connector", - "version": "0.15.35", + "version": "0.15.36", "description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.", "permissions": [ "tabs", diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js index 9a51ad7..c66e23a 100644 --- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js +++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js @@ -833,12 +833,9 @@ async function dispatch(action, params) { return { closed: tab.id }; } case "page.snapshot": - return executeInTab(params, snapshotPage, [ - params.maxElements || 80, - params.containingText ?? null, - params.roleFilter ?? null, - params.nearUid ?? null, - ]); + return snapshotInTab(params); + case "page.inspect": + return inspectInTab(params); case "page.evaluate": return evaluateInTab(params); case "page.click": @@ -1120,12 +1117,95 @@ async function evaluateInTab(params) { async function withOptionalSnapshot(params, actionFn) { const result = await actionFn(params); if (params.includeSnapshot) { - const snapshot = await executeInTab({ ...params, foreground: false }, snapshotPage, [params.maxElements || 80, null, null, null]); + const snapshot = await snapshotInTab({ ...params, foreground: false }); return { result, snapshot }; } return result; } +// Snapshot/inspect run from a packaged MAIN-world script (snapshot_injected.js) injected via +// chrome.scripting.executeScript({ files }). That file is free of eval/new Function, so it works +// on strict-CSP pages, and it installs globalThis.__piChromeSnapshotPage / __piChromeInspectTarget. +// It shares window.__PI_CHROME_STATE__ (same el- uid scheme) with the CDP-injected input helpers. +async function snapshotInTab(params) { + const tab = await getTabByParams(params); + if (params.foreground) await bringToFront(tab); + const args = [ + params.maxElements || 80, + params.containingText ?? null, + params.roleFilter ?? null, + params.nearUid ?? null, + params.mode || "auto", + params.query ?? null, + params.maxTextChars ?? null, + ]; + await chrome.scripting.executeScript({ + target: { tabId: tab.id, frameIds: [0] }, + world: "MAIN", + files: ["snapshot_injected.js"], + }); + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id, frameIds: [0] }, + world: "MAIN", + func: async (invocationArgs) => { + try { + const snapshotPage = globalThis.__piChromeSnapshotPage; + if (typeof snapshotPage !== "function") throw new Error("snapshot_injected.js did not install __piChromeSnapshotPage"); + return { ok: true, value: await snapshotPage(...invocationArgs) }; + } catch (error) { + return { ok: false, error: error?.stack || error?.message || String(error) }; + } + }, + args: [args], + }); + const first = results?.[0]; + if (first?.error) { + const message = typeof first.error === "string" ? first.error : (first.error.message || JSON.stringify(first.error)); + throw new Error(message); + } + const envelope = first?.result; + if (envelope && typeof envelope === "object" && envelope.ok === false) { + throw new Error(envelope.error || "Chrome snapshot script failed"); + } + return envelope?.value; +} + +async function inspectInTab(params) { + if (!params.uid && !params.selector) throw new Error("chrome_inspect requires uid or selector"); + const tab = await getTabByParams(params); + if (params.foreground) await bringToFront(tab); + const args = [params.uid ?? null, params.selector ?? null, params.scrollIntoView === true]; + await chrome.scripting.executeScript({ + target: { tabId: tab.id, frameIds: [0] }, + world: "MAIN", + files: ["snapshot_injected.js"], + }); + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id, frameIds: [0] }, + world: "MAIN", + func: async (invocationArgs) => { + try { + const inspectTarget = globalThis.__piChromeInspectTarget; + if (typeof inspectTarget !== "function") throw new Error("snapshot_injected.js did not install __piChromeInspectTarget"); + return { ok: true, value: await inspectTarget(...invocationArgs) }; + } catch (error) { + return { ok: false, error: error?.stack || error?.message || String(error) }; + } + }, + args: [args], + }); + const first = results?.[0]; + if (first?.error) { + const message = typeof first.error === "string" ? first.error : (first.error.message || JSON.stringify(first.error)); + throw new Error(message); + } + const envelope = first?.result; + if (envelope && typeof envelope === "object" && envelope.ok === false) { + throw new Error(envelope.error || "Chrome inspect script failed"); + } + return envelope?.value; +} + // One-shot init script registry, scoped per tab. The source is registered with CDP // Page.addScriptToEvaluateOnNewDocument, which runs it at document_start in the page's MAIN // world and is NOT subject to page CSP (the old func:(code)=>new Function(code) path was @@ -1642,104 +1722,6 @@ function installEarlyCapture() { state.instrumentationInstalled = true; } -function snapshotPage(maxElements, containingText, roleFilter, nearUid) { - installPiChromeInstrumentation(); - const unique = (selector) => { - try { return document.querySelectorAll(selector).length === 1; } catch { return false; } - }; - const cssEscape = (value) => (window.CSS && CSS.escape) ? CSS.escape(value) : String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); - const selectorFor = (element) => { - if (element.id && unique("#" + cssEscape(element.id))) return "#" + cssEscape(element.id); - const attr = ["aria-label", "name", "placeholder", "data-testid", "role"].find((name) => element.getAttribute(name)); - if (attr) { - const candidate = element.tagName.toLowerCase() + "[" + attr + "=" + JSON.stringify(element.getAttribute(attr)) + "]"; - if (unique(candidate)) return candidate; - } - const parts = []; - let current = element; - while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 5) { - let part = current.tagName.toLowerCase(); - if (current.classList.length > 0) part += "." + Array.from(current.classList).slice(0, 2).map(cssEscape).join("."); - const siblings = Array.from(current.parentElement?.children ?? []).filter((sibling) => sibling.tagName === current.tagName); - if (siblings.length > 1) part += ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")"; - parts.unshift(part); - const candidate = parts.join(" > "); - if (unique(candidate)) return candidate; - current = current.parentElement; - } - return parts.join(" > "); - }; - const visible = (element) => isElementVisible(element); - const labelFor = (element) => ( - element.getAttribute("aria-label") || - element.getAttribute("title") || - element.getAttribute("placeholder") || - element.innerText || - element.value || - element.textContent || - "" - ).trim().replace(/\s+/g, " ").slice(0, 160); - let candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [role="menuitem"], [role="tab"], [role="checkbox"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])')); - if (containingText) { - const needle = String(containingText).toLowerCase(); - candidates = candidates.filter((element) => labelFor(element).toLowerCase().includes(needle)); - } - if (roleFilter) { - const wanted = String(roleFilter).toLowerCase(); - candidates = candidates.filter((element) => { - const role = (element.getAttribute("role") || element.tagName).toLowerCase(); - return role === wanted; - }); - } - let near; - if (nearUid) { - const state = getPiChromeState(); - near = state.elements[nearUid]; - } - if (near) { - const nearRect = near.getBoundingClientRect(); - const cx = nearRect.left + nearRect.width / 2; - const cy = nearRect.top + nearRect.height / 2; - candidates.sort((a, b) => { - const ra = a.getBoundingClientRect(); - const rb = b.getBoundingClientRect(); - const da = Math.hypot(ra.left + ra.width / 2 - cx, ra.top + ra.height / 2 - cy); - const db = Math.hypot(rb.left + rb.width / 2 - cx, rb.top + rb.height / 2 - cy); - return da - db; - }); - } - const elements = candidates.filter(visible).slice(0, maxElements).map((element, index) => { - const rect = element.getBoundingClientRect(); - const style = getComputedStyle(element); - const cx = rect.left + rect.width / 2; - const cy = rect.top + rect.height / 2; - const occluded = occluderAt(cx, cy, element); - return { - index, - uid: rememberElement(element), - tag: element.tagName.toLowerCase(), - selector: selectorFor(element), - label: labelFor(element), - href: element.href || undefined, - type: element.getAttribute("type") || undefined, - role: element.getAttribute("role") || undefined, - disabled: Boolean(element.disabled || element.getAttribute("aria-disabled") === "true"), - inert: Boolean(element.closest?.("[inert]")), - pointerEvents: style.pointerEvents, - occluded: occluded || undefined, - rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, - }; - }); - return { - title: document.title, - url: location.href, - viewport: { width: innerWidth, height: innerHeight, scrollX, scrollY }, - text: document.body ? document.body.innerText.replace(/\s+\n/g, "\n").trim().slice(0, 30000) : "", - elements, - filter: { containingText: containingText || undefined, roleFilter: roleFilter || undefined, nearUid: nearUid || undefined }, - }; -} - function probePage() { // Sanity probe used by /chrome-doctor. Returns evidence that MAIN-world execution works. return { diff --git a/extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js b/extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js new file mode 100644 index 0000000..ceee8ce --- /dev/null +++ b/extensions/chrome-profile-bridge/browser-extension/snapshot_injected.js @@ -0,0 +1,677 @@ +// Static MAIN-world snapshot implementation injected by the MV3 service worker. +// Keep this file free of eval/new Function so `chrome_snapshot` works on strict-CSP pages. +(() => { + function getPiChromeState() { + const state = window.__PI_CHROME_STATE__ || { + nextElementUid: 1, + elements: {}, + console: [], + network: [], + nextRequestId: 1, + instrumentationInstalled: false, + lastSnapshotDigest: null, + }; + window.__PI_CHROME_STATE__ = state; + return state; + } + + function rememberElement(element) { + const state = getPiChromeState(); + if (!element.__piChromeUid) element.__piChromeUid = "el-" + state.nextElementUid++; + state.elements[element.__piChromeUid] = element; + return element.__piChromeUid; + } + + function isElementVisible(element) { + if (!element || !element.getBoundingClientRect) return false; + const style = getComputedStyle(element); + if (style.visibility === "hidden" || style.display === "none") return false; + const rect = element.getBoundingClientRect(); + if (rect.width === 0 || rect.height === 0) return false; + if (rect.bottom < 0 || rect.right < 0) return false; + if (rect.top > innerHeight || rect.left > innerWidth) return false; + return true; + } + + function occluderAt(x, y, expected) { + const top = document.elementFromPoint(x, y); + if (!top || top === expected) return null; + if (expected && expected.contains(top)) return null; + if (top.contains(expected)) return null; + return { + tag: top.tagName.toLowerCase(), + id: top.id || undefined, + className: typeof top.className === "string" ? top.className : undefined, + }; + } + + function textOf(element, max) { + return (element?.innerText || element?.textContent || "").replace(/\s+/g, " ").trim().slice(0, max || 500); + } + + function accessibleLabel(element) { + if (!element) return ""; + const labelledBy = element.getAttribute("aria-labelledby"); + if (labelledBy) { + const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.innerText || "").join(" ").trim(); + if (text) return text; + } + const id = element.id; + if (id) { + try { + const label = document.querySelector(`label[for="${cssEscape(id)}"]`); + if (label?.innerText) return label.innerText; + } catch {} + } + const wrappingLabel = element.closest?.("label"); + return ( + element.getAttribute("aria-label") || + element.getAttribute("title") || + element.getAttribute("placeholder") || + wrappingLabel?.innerText || + element.innerText || + element.textContent || + "" + ).trim().replace(/\s+/g, " ").slice(0, 180); + } + + function cssEscape(value) { + return (window.CSS && CSS.escape) ? CSS.escape(value) : String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); + } + + function roleOf(element) { + const explicit = element.getAttribute("role"); + if (explicit) return explicit.toLowerCase(); + const tag = element.tagName.toLowerCase(); + const type = (element.getAttribute("type") || "").toLowerCase(); + if (tag === "a" && element.href) return "link"; + if (tag === "button" || type === "button" || type === "submit" || type === "reset") return "button"; + if (tag === "textarea") return "textbox"; + if (tag === "select") return "combobox"; + if (tag === "input") { + if (["checkbox", "radio", "range", "search", "email", "password", "tel", "url", "number"].includes(type)) return type === "checkbox" || type === "radio" || type === "range" ? type : "textbox"; + return "textbox"; + } + if (element.isContentEditable) return "textbox"; + if (tag.match(/^h[1-6]$/)) return "heading"; + return tag; + } + + function isSensitiveField(element) { + if (!element) return false; + const tag = element.tagName?.toLowerCase?.() || ""; + if (!/^(input|textarea|select)$/.test(tag) && !element.isContentEditable) return false; + const type = (element.getAttribute("type") || "").toLowerCase(); + if (["password"].includes(type)) return true; + const haystack = [ + type, + element.getAttribute("name"), + element.id, + element.getAttribute("autocomplete"), + element.getAttribute("aria-label"), + element.getAttribute("placeholder"), + element.getAttribute("data-testid"), + ].filter(Boolean).join(" ").toLowerCase(); + return /password|passwd|\bpwd\b|secret|token|bearer|api[-_ ]?key|access[-_ ]?key|auth[-_ ]?code|one[-_ ]?time|otp|2fa|mfa|verification[-_ ]?code|recovery[-_ ]?code|credit[-_ ]?card|card[-_ ]?number|cc-number|cc-csc|cvc|cvv|security[-_ ]?code|ssn|social[-_ ]?security/.test(haystack); + } + + function installPiChromeInstrumentation() { + const state = getPiChromeState(); + if (state.instrumentationInstalled) return; + state.instrumentationInstalled = true; + const pushConsole = (level, args) => { + state.console.push({ + id: state.console.length + 1, + level, + timestamp: Date.now(), + url: location.href, + args: Array.from(args).map((arg) => { + try { + if (typeof arg === "string") return arg; + if (arg instanceof Error) return { name: arg.name, message: arg.message, stack: arg.stack }; + return JSON.parse(JSON.stringify(arg)); + } catch { + return String(arg); + } + }), + }); + if (state.console.length > 500) state.console.splice(0, state.console.length - 500); + }; + for (const level of ["debug", "log", "info", "warn", "error"]) { + const original = console[level]; + if (typeof original !== "function" || original.__piChromeWrapped) continue; + const wrapped = function(...args) { + pushConsole(level, args); + return original.apply(this, args); + }; + wrapped.__piChromeWrapped = true; + console[level] = wrapped; + } + window.addEventListener("error", (event) => pushConsole("pageerror", [event.message, event.filename + ":" + event.lineno + ":" + event.colno])); + window.addEventListener("unhandledrejection", (event) => pushConsole("unhandledrejection", [event.reason])); + + const record = (entry) => { + state.network.push(entry); + if (state.network.length > 1000) state.network.splice(0, state.network.length - 1000); + return entry; + }; + if (window.fetch && !window.fetch.__piChromeWrapped) { + const originalFetch = window.fetch.bind(window); + const wrappedFetch = async (...args) => { + const id = "req-" + state.nextRequestId++; + const startedAt = Date.now(); + const input = args[0]; + const init = args[1] || {}; + const url = typeof input === "string" ? input : input?.url; + const method = (init.method || input?.method || "GET").toUpperCase(); + const entry = record({ id, type: "fetch", method, url: String(url || ""), startedAt, pageUrl: location.href, status: "pending" }); + try { + const response = await originalFetch(...args); + entry.status = response.status; + entry.statusText = response.statusText; + entry.ok = response.ok; + entry.responseUrl = response.url; + entry.durationMs = Date.now() - startedAt; + entry.responseHeaders = Array.from(response.headers.entries()); + entry.responseBodyOmitted = "response body capture is disabled by default"; + return response; + } catch (error) { + entry.error = error?.message || String(error); + entry.durationMs = Date.now() - startedAt; + throw error; + } + }; + wrappedFetch.__piChromeWrapped = true; + window.fetch = wrappedFetch; + } + if (window.XMLHttpRequest && !XMLHttpRequest.prototype.open.__piChromeWrapped) { + const originalOpen = XMLHttpRequest.prototype.open; + const originalSend = XMLHttpRequest.prototype.send; + XMLHttpRequest.prototype.open = function(method, url, ...rest) { + this.__piChromeRequest = { method: String(method || "GET").toUpperCase(), url: String(url || "") }; + return originalOpen.call(this, method, url, ...rest); + }; + XMLHttpRequest.prototype.open.__piChromeWrapped = true; + XMLHttpRequest.prototype.send = function(body) { + const id = "req-" + state.nextRequestId++; + const startedAt = Date.now(); + const info = this.__piChromeRequest || {}; + const entry = record({ id, type: "xhr", method: info.method || "GET", url: info.url || "", startedAt, pageUrl: location.href, status: "pending" }); + this.addEventListener("loadend", () => { + entry.status = this.status; + entry.statusText = this.statusText; + entry.responseUrl = this.responseURL; + entry.durationMs = Date.now() - startedAt; + try { entry.responseHeadersText = this.getAllResponseHeaders(); } catch {} + entry.responseBodyOmitted = "response body capture is disabled by default"; + }); + this.addEventListener("error", () => { entry.error = "XMLHttpRequest error"; entry.durationMs = Date.now() - startedAt; }); + return originalSend.call(this, body); + }; + } + } + + function hashString(text) { + let h = 0; + for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) | 0; + return h; + } + + function selectorFor(element) { + const unique = (selector) => { + try { return document.querySelectorAll(selector).length === 1; } catch { return false; } + }; + if (element.id && unique("#" + cssEscape(element.id))) return "#" + cssEscape(element.id); + const attr = ["aria-label", "name", "placeholder", "data-testid", "role"].find((name) => element.getAttribute(name)); + if (attr) { + const candidate = element.tagName.toLowerCase() + "[" + attr + "=" + JSON.stringify(element.getAttribute(attr)) + "]"; + if (unique(candidate)) return candidate; + } + const parts = []; + let current = element; + while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 5) { + let part = current.tagName.toLowerCase(); + if (current.classList.length > 0) part += "." + Array.from(current.classList).slice(0, 2).map(cssEscape).join("."); + const siblings = Array.from(current.parentElement?.children ?? []).filter((sibling) => sibling.tagName === current.tagName); + if (siblings.length > 1) part += ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")"; + parts.unshift(part); + const candidate = parts.join(" > "); + if (unique(candidate)) return candidate; + current = current.parentElement; + } + return parts.join(" > "); + } + + function directHeadingText(element) { + const labelledBy = element.getAttribute?.("aria-labelledby"); + if (labelledBy) { + const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.innerText || "").join(" ").replace(/\s+/g, " ").trim(); + if (text) return text.slice(0, 180); + } + const aria = element.getAttribute?.("aria-label"); + if (aria) return aria.trim().slice(0, 180); + const heading = Array.from(element.querySelectorAll?.("h1,h2,h3,h4,[role='heading']") || []).find(isElementVisible); + if (heading) return textOf(heading, 180); + return ""; + } + + function meaningfulContainerFor(element) { + let current = element.parentElement; + let fallback = current; + let depth = 0; + while (current && current !== document.body && depth++ < 8) { + if (!isElementVisible(current)) { current = current.parentElement; continue; } + const tag = current.tagName.toLowerCase(); + const role = (current.getAttribute("role") || "").toLowerCase(); + const cls = typeof current.className === "string" ? current.className : ""; + const id = current.id || ""; + const named = Boolean(current.getAttribute("aria-label") || current.getAttribute("aria-labelledby") || directHeadingText(current)); + const semantic = /^(form|dialog|section|article|nav|header|main|aside|footer|li|tr|td|fieldset)$/.test(tag) || + /^(dialog|alertdialog|region|group|listitem|row|cell|tabpanel|menu|toolbar|navigation|main|banner|contentinfo|complementary)$/.test(role); + const classHint = /card|panel|pane|modal|dialog|section|content|container|toolbar|menu|list|item|row|cell|header|footer|sidebar|drawer|popover|dropdown/i.test(`${id} ${cls}`); + const rect = current.getBoundingClientRect(); + const childActions = current.querySelectorAll?.('a, button, input, textarea, select, [role="button"], [role="link"], [tabindex]:not([tabindex="-1"])').length || 0; + if ((semantic || classHint || named) && rect.width > 20 && rect.height > 20 && childActions <= 80) return current; + if (!fallback && rect.width > 20 && rect.height > 20) fallback = current; + current = current.parentElement; + } + return fallback || document.body; + } + + function contextForElement(element) { + const container = meaningfulContainerFor(element); + if (!container || container === document.body || container === element) return undefined; + return { + uid: rememberElement(container), + tag: container.tagName.toLowerCase(), + role: roleOf(container), + label: directHeadingText(container) || accessibleLabel(container) || textOf(container, 140), + rect: rectSummary(container), + }; + } + + function summarizeElement(element, index) { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + const cx = rect.left + rect.width / 2; + const cy = rect.top + rect.height / 2; + const occluded = occluderAt(cx, cy, element); + const role = roleOf(element); + const disabled = Boolean(element.disabled || element.getAttribute("aria-disabled") === "true"); + const rawValue = "value" in element && typeof element.value === "string" ? element.value : undefined; + const sensitive = isSensitiveField(element); + const value = rawValue && !sensitive ? rawValue.slice(0, 120) : undefined; + const checked = "checked" in element ? Boolean(element.checked) : undefined; + return { + index, + uid: rememberElement(element), + tag: element.tagName.toLowerCase(), + role, + selector: selectorFor(element), + label: accessibleLabel(element), + href: element.href || undefined, + type: element.getAttribute("type") || undefined, + value: value || undefined, + hasValue: rawValue ? rawValue.length > 0 : undefined, + valueLength: rawValue && sensitive ? rawValue.length : undefined, + valueRedacted: sensitive && rawValue ? true : undefined, + checked, + disabled, + inert: Boolean(element.closest?.("[inert]")), + pointerEvents: style.pointerEvents, + occluded: occluded || undefined, + context: contextForElement(element), + rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, + }; + } + + function isInViewport(element) { + const rect = element.getBoundingClientRect(); + return rect.bottom >= 0 && rect.right >= 0 && rect.top <= innerHeight && rect.left <= innerWidth; + } + + function formSummaries() { + const fields = Array.from(document.querySelectorAll('input, textarea, select, [contenteditable="true"]')) + .filter(isElementVisible) + .slice(0, 80) + .map((element, index) => ({ + ...summarizeElement(element, index), + required: Boolean(element.required || element.getAttribute("aria-required") === "true"), + invalid: Boolean(element.matches?.(":invalid") || element.getAttribute("aria-invalid") === "true"), + autocomplete: element.getAttribute("autocomplete") || undefined, + })); + const submits = Array.from(document.querySelectorAll('button, input[type="submit"], [role="button"]')) + .filter(isElementVisible) + .filter((element) => /submit|save|continue|next|send|sign in|log in|create|update|done/i.test(accessibleLabel(element) + " " + (element.getAttribute("type") || ""))) + .slice(0, 30) + .map((element, index) => summarizeElement(element, index)); + return { fields, submits }; + } + + function pageMap() { + const landmarkSelectors = [ + ["header", 'header, [role="banner"]'], + ["nav", 'nav, [role="navigation"]'], + ["main", 'main, [role="main"]'], + ["aside", 'aside, [role="complementary"]'], + ["footer", 'footer, [role="contentinfo"]'], + ["dialog", 'dialog, [role="dialog"], [aria-modal="true"]'], + ["form", "form"], + ]; + const regions = []; + for (const [kind, selector] of landmarkSelectors) { + for (const element of Array.from(document.querySelectorAll(selector)).filter(isElementVisible).slice(0, 12)) { + const headings = Array.from(element.querySelectorAll("h1,h2,h3,[role='heading']")).filter(isElementVisible).slice(0, 6).map((h) => textOf(h, 120)); + const actions = Array.from(element.querySelectorAll('a, button, input, textarea, select, [role="button"], [role="link"], [tabindex]:not([tabindex="-1"])')).filter(isElementVisible).slice(0, 8).map((a) => { + const summary = summarizeElement(a, 0); + return { uid: summary.uid, role: summary.role, label: summary.label || summary.selector, disabled: summary.disabled || undefined }; + }); + regions.push({ kind, uid: rememberElement(element), label: accessibleLabel(element) || headings[0] || textOf(element, 100), headings, actions }); + } + } + const headings = Array.from(document.querySelectorAll("h1,h2,h3,[role='heading']")).filter(isElementVisible).slice(0, 30).map((element) => ({ + uid: rememberElement(element), + level: Number(element.tagName?.slice(1)) || Number(element.getAttribute("aria-level")) || undefined, + text: textOf(element, 180), + })); + return { regions, headings }; + } + + function layoutSections(elements, forms) { + const byUid = new Map(); + const addToSection = (summary, kind) => { + const source = getPiChromeState().elements[summary.uid]; + const container = source ? meaningfulContainerFor(source) : null; + if (!container || container === document.body) return; + const uid = rememberElement(container); + let section = byUid.get(uid); + if (!section) { + const rect = rectSummary(container); + section = { + uid, + tag: container.tagName.toLowerCase(), + role: roleOf(container), + label: directHeadingText(container) || accessibleLabel(container) || textOf(container, 160), + text: textOf(container, 260), + rect, + actions: [], + fields: [], + }; + byUid.set(uid, section); + } + const item = { uid: summary.uid, role: summary.role, label: summary.label || summary.selector, disabled: summary.disabled || undefined }; + if (kind === "field") section.fields.push(item); + else section.actions.push(item); + }; + for (const el of (elements || []).slice(0, 80)) addToSection(el, ["textbox", "checkbox", "radio", "combobox"].includes(el.role) ? "field" : "action"); + for (const field of (forms?.fields || []).slice(0, 80)) addToSection(field, "field"); + const sections = Array.from(byUid.values()) + .filter((section) => section.actions.length || section.fields.length) + .sort((a, b) => a.rect.y - b.rect.y || a.rect.x - b.rect.x) + .slice(0, 18); + for (const section of sections) { + section.actions = section.actions.slice(0, 10); + section.fields = section.fields.slice(0, 10); + } + return sections; + } + + function tokenScore(haystack, query) { + if (!query) return 0; + const hay = String(haystack || "").toLowerCase(); + const tokens = String(query).toLowerCase().split(/\W+/).filter(Boolean); + if (!tokens.length) return 0; + let score = 0; + for (const token of tokens) { + if (hay.includes(token)) score += token.length <= 2 ? 1 : 3; + } + if (hay.includes(String(query).toLowerCase())) score += 8; + return score; + } + + function queryMatches(query, elements, map) { + if (!query) return []; + const candidates = []; + for (const element of elements) { + const hay = [element.role, element.label, element.selector, element.type, element.href].filter(Boolean).join(" "); + const score = tokenScore(hay, query); + if (score > 0) candidates.push({ score, kind: "element", ...element }); + } + const textNodes = []; + for (const block of Array.from(document.querySelectorAll("h1,h2,h3,h4,p,li,td,th,label,summary,[role='alert']")).filter(isElementVisible).slice(0, 300)) { + const text = textOf(block, 300); + const score = tokenScore(text, query); + if (score > 0) textNodes.push({ score, kind: "text", uid: rememberElement(block), tag: block.tagName.toLowerCase(), role: roleOf(block), text, rect: rectSummary(block) }); + } + for (const region of map.regions || []) { + const score = tokenScore([region.kind, region.label, ...(region.headings || [])].join(" "), query); + if (score > 0) candidates.push({ score, kind: "region", ...region }); + } + return candidates.concat(textNodes).sort((a, b) => b.score - a.score).slice(0, 20); + } + + function rectSummary(element) { + const rect = element.getBoundingClientRect(); + return { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }; + } + + function activeElementSummary() { + const el = document.activeElement; + if (!el || el === document.body || el === document.documentElement) return null; + return summarizeElement(el, 0); + } + + function modalSummary() { + const selectors = 'dialog[open], [role="dialog"], [aria-modal="true"], [role="alertdialog"]'; + const modal = Array.from(document.querySelectorAll(selectors)).find(isElementVisible); + if (!modal) return null; + return { uid: rememberElement(modal), tag: modal.tagName.toLowerCase(), role: roleOf(modal), label: accessibleLabel(modal) || textOf(modal, 180), rect: rectSummary(modal) }; + } + + function digestFor(snapshot) { + return { + url: snapshot.url, + title: snapshot.title, + textHash: hashString(snapshot.text || ""), + focusedUid: snapshot.focused?.uid || null, + modalUid: snapshot.modal?.uid || null, + labels: (snapshot.elements || []).slice(0, 50).map((el) => ({ uid: el.uid, role: el.role, label: el.label, disabled: el.disabled, value: el.value, checked: el.checked })), + }; + } + + function diffSnapshot(previous, current) { + if (!previous) return { firstSnapshot: true }; + const changes = []; + if (previous.url !== current.url) changes.push({ kind: "url", before: previous.url, after: current.url }); + if (previous.title !== current.title) changes.push({ kind: "title", before: previous.title, after: current.title }); + if (previous.textHash !== current.textHash) changes.push({ kind: "textChanged" }); + if (previous.focusedUid !== current.focusedUid) changes.push({ kind: "focus", before: previous.focusedUid, after: current.focusedUid }); + if (previous.modalUid !== current.modalUid) changes.push({ kind: "modal", before: previous.modalUid, after: current.modalUid }); + const prevByUid = new Map((previous.labels || []).map((x) => [x.uid, x])); + const curByUid = new Map((current.labels || []).map((x) => [x.uid, x])); + const added = []; + const removed = []; + const updated = []; + for (const cur of current.labels || []) { + const prev = prevByUid.get(cur.uid); + if (!prev) added.push(cur); + else if (prev.label !== cur.label || prev.disabled !== cur.disabled || prev.value !== cur.value || prev.checked !== cur.checked) updated.push({ uid: cur.uid, before: prev, after: cur }); + } + for (const prev of previous.labels || []) { + if (!curByUid.has(prev.uid)) removed.push(prev); + } + return { changes, added: added.slice(0, 12), removed: removed.slice(0, 12), updated: updated.slice(0, 12) }; + } + + function visibleTextSnippets(maxChars) { + const snippets = []; + const blocks = Array.from(document.querySelectorAll("h1,h2,h3,h4,p,li,td,th,label,summary,[role='alert']")).filter(isElementVisible); + let used = 0; + for (const block of blocks) { + if (!isInViewport(block) && snippets.length > 12) continue; + const text = textOf(block, 500); + if (!text || snippets.some((s) => s.text === text)) continue; + const next = { uid: rememberElement(block), tag: block.tagName.toLowerCase(), text, rect: rectSummary(block) }; + snippets.push(next); + used += text.length; + if (used >= maxChars || snippets.length >= 40) break; + } + return snippets; + } + + function snapshotPage(maxElements, containingText, roleFilter, nearUid, mode, query, maxTextChars) { + installPiChromeInstrumentation(); + mode = ["auto", "interactive", "forms", "pageMap", "text", "changes", "full"].includes(mode) ? mode : "auto"; + const fullTextLimit = Number(maxTextChars || (mode === "full" ? 30000 : mode === "text" ? 18000 : 6000)); + let candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [role="menuitem"], [role="tab"], [role="checkbox"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])')); + if (containingText) { + const needle = String(containingText).toLowerCase(); + candidates = candidates.filter((element) => accessibleLabel(element).toLowerCase().includes(needle)); + } + if (roleFilter) { + const wanted = String(roleFilter).toLowerCase(); + candidates = candidates.filter((element) => roleOf(element) === wanted || element.tagName.toLowerCase() === wanted); + } + let near; + if (nearUid) near = getPiChromeState().elements[nearUid]; + if (near) { + const nearRect = near.getBoundingClientRect(); + const cx = nearRect.left + nearRect.width / 2; + const cy = nearRect.top + nearRect.height / 2; + candidates.sort((a, b) => { + const ra = a.getBoundingClientRect(); + const rb = b.getBoundingClientRect(); + const da = Math.hypot(ra.left + ra.width / 2 - cx, ra.top + ra.height / 2 - cy); + const db = Math.hypot(rb.left + rb.width / 2 - cx, rb.top + rb.height / 2 - cy); + return da - db; + }); + } else { + candidates.sort((a, b) => { + const ar = a.getBoundingClientRect(); + const br = b.getBoundingClientRect(); + const avis = isInViewport(a) ? 0 : 1; + const bvis = isInViewport(b) ? 0 : 1; + return avis - bvis || ar.top - br.top || ar.left - br.left; + }); + } + const visibleCandidates = candidates.filter(isElementVisible); + const elements = visibleCandidates.slice(0, maxElements).map((element, index) => summarizeElement(element, index)); + const queryElements = query + ? visibleCandidates.slice(0, Math.max(maxElements, 500)).map((element, index) => summarizeElement(element, index)) + : elements; + const map = pageMap(); + const forms = formSummaries(); + const layout = layoutSections(elements, forms); + const focused = activeElementSummary(); + const modal = modalSummary(); + const bodyText = document.body ? document.body.innerText.replace(/\s+\n/g, "\n").trim() : ""; + const text = bodyText.slice(0, fullTextLimit); + const snapshot = { + title: document.title, + url: location.href, + mode, + query: query || undefined, + viewport: { width: innerWidth, height: innerHeight, scrollX, scrollY }, + summary: { + visibleText: textOf(document.body, 500), + visibleInteractiveCount: elements.filter((el) => el.rect.y >= 0 && el.rect.y <= innerHeight).length, + totalInteractiveSampled: elements.length, + totalInteractiveVisible: visibleCandidates.length, + focused: focused ? { uid: focused.uid, role: focused.role, label: focused.label } : undefined, + modal: modal ? { uid: modal.uid, label: modal.label } : undefined, + hints: [], + }, + focused: focused || undefined, + modal: modal || undefined, + text, + textTruncated: bodyText.length > text.length, + textSnippets: visibleTextSnippets(mode === "text" ? 12000 : 3000), + elements, + forms, + layout, + pageMap: map, + matches: queryMatches(query, queryElements, map), + filter: { containingText: containingText || undefined, roleFilter: roleFilter || undefined, nearUid: nearUid || undefined }, + }; + if (snapshot.modal) snapshot.summary.hints.push("A modal/dialog is visible; interact with it before the underlying page."); + const disabledImportant = elements.find((el) => el.disabled && /submit|save|merge|continue|next|send|approve|login|sign in/i.test(el.label || "")); + if (disabledImportant) snapshot.summary.hints.push(`${disabledImportant.uid} '${disabledImportant.label}' is disabled.`); + const occluded = elements.find((el) => el.occluded); + if (occluded) snapshot.summary.hints.push(`${occluded.uid} '${occluded.label || occluded.role}' appears occluded by ${occluded.occluded.tag}.`); + + const state = getPiChromeState(); + const currentDigest = digestFor(snapshot); + snapshot.diff = diffSnapshot(state.lastSnapshotDigest, currentDigest); + state.lastSnapshotDigest = currentDigest; + + if (mode === "interactive") { + delete snapshot.text; + delete snapshot.textSnippets; + delete snapshot.pageMap; + } else if (mode === "forms") { + delete snapshot.text; + delete snapshot.textSnippets; + snapshot.elements = elements.filter((el) => ["textbox", "checkbox", "radio", "combobox", "button"].includes(el.role)); + } else if (mode === "pageMap") { + delete snapshot.text; + delete snapshot.textSnippets; + snapshot.elements = elements.slice(0, 20); + } else if (mode === "changes") { + delete snapshot.text; + delete snapshot.textSnippets; + delete snapshot.elements; + delete snapshot.forms; + delete snapshot.layout; + delete snapshot.pageMap; + } else if (mode === "text") { + snapshot.elements = elements.slice(0, 20); + } else if (mode !== "full") { + snapshot.elements = elements.slice(0, Math.min(maxElements, 40)); + snapshot.text = text.slice(0, Math.min(text.length, 6000)); + } + return snapshot; + } + + function inspectTarget(uid, selector, shouldScrollIntoView) { + installPiChromeInstrumentation(); + const state = getPiChromeState(); + let element = null; + if (uid) element = state.elements[uid]; + if (!element && selector) element = document.querySelector(selector); + if (!element || !element.isConnected) throw new Error(uid ? `No live element for uid: ${uid}. Take a fresh chrome_snapshot.` : `No element matches selector: ${selector}`); + if (shouldScrollIntoView) element.scrollIntoView?.({ block: "center", inline: "center", behavior: "instant" }); + const summary = summarizeElement(element, 0); + const ancestors = []; + let current = element.parentElement; + while (current && current !== document.body && ancestors.length < 6) { + ancestors.push({ uid: rememberElement(current), tag: current.tagName.toLowerCase(), role: roleOf(current), label: accessibleLabel(current) || textOf(current, 100), selector: selectorFor(current) }); + current = current.parentElement; + } + const container = element.closest?.('form, dialog, [role="dialog"], [aria-modal="true"], section, article, main, aside') || element.parentElement || document.body; + const nearbyText = Array.from(container.querySelectorAll("h1,h2,h3,h4,p,li,label,[role='alert']")) + .filter(isElementVisible) + .slice(0, 24) + .map((node) => ({ uid: rememberElement(node), tag: node.tagName.toLowerCase(), text: textOf(node, 240), rect: rectSummary(node) })) + .filter((entry) => entry.text); + const nearbyActions = Array.from(container.querySelectorAll('a, button, input, textarea, select, [role="button"], [role="link"], [tabindex]:not([tabindex="-1"])')) + .filter(isElementVisible) + .slice(0, 30) + .map((node, index) => summarizeElement(node, index)); + const form = element.closest?.("form"); + const formContext = form ? { + uid: rememberElement(form), + label: accessibleLabel(form) || textOf(form, 160), + fields: Array.from(form.querySelectorAll('input, textarea, select, [contenteditable="true"]')).filter(isElementVisible).slice(0, 30).map((node, index) => summarizeElement(node, index)), + actions: Array.from(form.querySelectorAll('button, input[type="submit"], [role="button"]')).filter(isElementVisible).slice(0, 12).map((node, index) => summarizeElement(node, index)), + } : undefined; + const rect = element.getBoundingClientRect(); + const center = { x: Math.round(rect.left + rect.width / 2), y: Math.round(rect.top + rect.height / 2) }; + const clickSuggestion = summary.disabled || summary.inert || summary.pointerEvents === "none" + ? undefined + : { uid: summary.uid, x: center.x, y: center.y }; + return { target: summary, ancestors, nearbyText, nearbyActions, formContext, clickSuggestion }; + } + + globalThis.__piChromeSnapshotPage = snapshotPage; + globalThis.__piChromeInspectTarget = inspectTarget; +})(); diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts index bed1302..ef89e23 100644 --- a/extensions/chrome-profile-bridge/index.ts +++ b/extensions/chrome-profile-bridge/index.ts @@ -74,6 +74,137 @@ function safeJson(value: unknown): string { return JSON.stringify(value, null, 2); } +const snapshotModeValues = ["auto", "interactive", "forms", "pageMap", "text", "changes", "full"] as const; + +function compactLine(value: unknown, max = 140): string { + const text = String(value ?? "").replace(/\s+/g, " ").trim(); + return text.length > max ? `${text.slice(0, max - 1)}…` : text; +} + +function rectText(rect: any): string { + if (!rect) return "?"; + return `${rect.x},${rect.y} ${rect.width}x${rect.height}`; +} + +function formatChromeSnapshot(snapshot: any): string { + if (!snapshot || typeof snapshot !== "object") return safeJson(snapshot); + if (snapshot.mode === "full") return truncateText(safeJson(snapshot)); + const lines: string[] = []; + lines.push(`# Chrome snapshot${snapshot.mode ? ` (${snapshot.mode})` : ""}`); + lines.push(`${snapshot.title || "(untitled)"}`); + if (snapshot.url) lines.push(`${snapshot.url}`); + if (snapshot.viewport) lines.push(`viewport=${snapshot.viewport.width}x${snapshot.viewport.height} scroll=${snapshot.viewport.scrollX || 0},${snapshot.viewport.scrollY || 0}`); + if (snapshot.summary?.modal) lines.push(`modal: ${snapshot.summary.modal.uid} ${compactLine(snapshot.summary.modal.label)}`); + if (snapshot.summary?.focused) lines.push(`focused: ${snapshot.summary.focused.uid} ${snapshot.summary.focused.role || ""} ${compactLine(snapshot.summary.focused.label)}`); + if (Array.isArray(snapshot.summary?.hints) && snapshot.summary.hints.length) { + lines.push("\n## Hints"); + for (const hint of snapshot.summary.hints.slice(0, 6)) lines.push(`- ${hint}`); + } + if (snapshot.diff && !snapshot.diff.firstSnapshot) { + const changed = [ + ...(snapshot.diff.changes || []).map((c: any) => c.kind === "textChanged" ? "text changed" : `${c.kind}: ${compactLine(c.before, 50)} → ${compactLine(c.after, 50)}`), + ...(snapshot.diff.added || []).slice(0, 4).map((e: any) => `added ${e.uid} ${e.role || ""} ${compactLine(e.label)}`), + ...(snapshot.diff.updated || []).slice(0, 4).map((u: any) => `updated ${u.uid} ${compactLine(u.after?.label || u.before?.label)}`), + ]; + if (changed.length) { + lines.push("\n## Changed since last snapshot"); + for (const item of changed.slice(0, 10)) lines.push(`- ${item}`); + } + } + if (Array.isArray(snapshot.matches) && snapshot.matches.length) { + lines.push(`\n## Matches for "${snapshot.query}"`); + for (const match of snapshot.matches.slice(0, 12)) { + if (match.kind === "text") lines.push(`- ${match.uid} text ${compactLine(match.text)} @ ${rectText(match.rect)}`); + else if (match.kind === "region") lines.push(`- ${match.uid} region ${compactLine(match.label)} headings=${(match.headings || []).map((h: string) => compactLine(h, 50)).join(" | ")}`); + else lines.push(`- ${match.uid} ${match.role || match.tag || "element"}${match.disabled ? " disabled" : ""} ${compactLine(match.label || match.selector)} @ ${rectText(match.rect)}`); + } + } + if (snapshot.mode === "pageMap" && snapshot.pageMap) { + lines.push("\n## Page map"); + for (const region of (snapshot.pageMap.regions || []).slice(0, 18)) { + lines.push(`- ${region.uid} ${region.kind}: ${compactLine(region.label)}`); + for (const action of (region.actions || []).slice(0, 5)) lines.push(` - ${action.uid} ${action.role || ""}${action.disabled ? " disabled" : ""} ${compactLine(action.label)}`); + } + if (snapshot.pageMap.headings?.length) { + lines.push("\nHeadings:"); + for (const h of snapshot.pageMap.headings.slice(0, 20)) lines.push(`- ${h.uid} h${h.level || ""} ${compactLine(h.text)}`); + } + } + if (Array.isArray(snapshot.layout) && snapshot.layout.length && snapshot.mode !== "changes") { + lines.push("\n## Layout / context"); + for (const section of snapshot.layout.slice(0, snapshot.mode === "pageMap" ? 18 : 8)) { + const bits = [`${section.uid}`, section.role || section.tag, compactLine(section.label || section.text || "(unnamed section)", 110), `@ ${rectText(section.rect)}`]; + lines.push(`- ${bits.filter(Boolean).join(" ")}`); + const fieldLabels = (section.fields || []).slice(0, 4).map((f: any) => `${f.uid} ${compactLine(f.label || f.role, 40)}`); + const actionLabels = (section.actions || []).slice(0, 5).map((a: any) => `${a.uid}${a.disabled ? " disabled" : ""} ${compactLine(a.label || a.role, 40)}`); + if (fieldLabels.length) lines.push(` fields: ${fieldLabels.join("; ")}`); + if (actionLabels.length) lines.push(` actions: ${actionLabels.join("; ")}`); + } + } + if ((snapshot.mode === "forms" || snapshot.forms?.fields?.length) && snapshot.mode !== "pageMap") { + const fields = snapshot.forms?.fields || []; + const submits = snapshot.forms?.submits || []; + if (fields.length || submits.length) lines.push("\n## Forms"); + for (const field of fields.slice(0, snapshot.mode === "forms" ? 40 : 12)) { + const bits = [field.uid, field.role || field.tag, field.required ? "required" : "", field.invalid ? "invalid" : "", field.disabled ? "disabled" : "", compactLine(field.label || field.selector, 90)]; + if (field.value) bits.push(`value=${compactLine(field.value, 50)}`); + else if (field.valueRedacted) bits.push("value=[redacted]"); + lines.push(`- ${bits.filter(Boolean).join(" ")} @ ${rectText(field.rect)}`); + } + for (const submit of submits.slice(0, 8)) lines.push(`- ${submit.uid} submit/action${submit.disabled ? " disabled" : ""} ${compactLine(submit.label || submit.selector)} @ ${rectText(submit.rect)}`); + } + if (Array.isArray(snapshot.elements) && snapshot.mode !== "pageMap") { + lines.push("\n## Visible actions"); + for (const el of snapshot.elements.slice(0, snapshot.mode === "interactive" ? 60 : 25)) { + const flags = [el.disabled ? "disabled" : "", el.occluded ? `occluded-by-${el.occluded.tag}` : ""].filter(Boolean).join(","); + const context = el.context?.label ? ` in ${el.context.uid} ${compactLine(el.context.label, 60)}` : ""; + lines.push(`- ${el.uid} ${el.role || el.tag}${flags ? ` [${flags}]` : ""} ${compactLine(el.label || el.selector)}${context} @ ${rectText(el.rect)}`); + } + if (snapshot.elements.length > (snapshot.mode === "interactive" ? 60 : 25)) lines.push(`- … ${snapshot.elements.length - (snapshot.mode === "interactive" ? 60 : 25)} more; retry with maxElements or mode=interactive`); + } + if ((snapshot.mode === "text" || snapshot.mode === "auto") && Array.isArray(snapshot.textSnippets) && snapshot.textSnippets.length) { + lines.push("\n## Text snippets"); + for (const snip of snapshot.textSnippets.slice(0, snapshot.mode === "text" ? 40 : 14)) lines.push(`- ${snip.uid} ${compactLine(snip.text, snapshot.mode === "text" ? 240 : 160)}`); + if (snapshot.textTruncated) lines.push("- … page text truncated; retry with mode=text or maxTextChars for more"); + } + lines.push("\nTip: use chrome_snapshot({query:'...', mode:'interactive|forms|pageMap|text|changes|full'}) or nearUid to zoom in."); + return truncateText(lines.join("\n")); +} + +function formatIncludedSnapshotText(raw: unknown, text: string): string { + const snapshot = raw && typeof raw === "object" ? (raw as { snapshot?: unknown }).snapshot : undefined; + return snapshot ? `${text}\n\n${formatChromeSnapshot(snapshot)}` : text; +} + +function formatChromeInspect(inspect: any): string { + if (!inspect || typeof inspect !== "object") return safeJson(inspect); + const t = inspect.target || {}; + const lines: string[] = []; + lines.push(`# Chrome inspect ${t.uid || ""}`.trim()); + lines.push(`${t.role || t.tag || "element"}${t.disabled ? " disabled" : ""}${t.occluded ? ` occluded-by-${t.occluded.tag}` : ""} ${compactLine(t.label || t.selector)}`); + if (t.selector) lines.push(`selector: ${t.selector}`); + if (t.rect) lines.push(`rect: ${rectText(t.rect)}`); + if (inspect.clickSuggestion) lines.push(`suggested click: chrome_click({ uid: "${inspect.clickSuggestion.uid}" }) or x=${inspect.clickSuggestion.x}, y=${inspect.clickSuggestion.y}`); + if (Array.isArray(inspect.nearbyText) && inspect.nearbyText.length) { + lines.push("\n## Nearby text"); + for (const item of inspect.nearbyText.slice(0, 12)) lines.push(`- ${item.uid} ${compactLine(item.text, 180)}`); + } + if (inspect.formContext) { + lines.push("\n## Form context"); + for (const field of (inspect.formContext.fields || []).slice(0, 20)) lines.push(`- ${field.uid} ${field.role || field.tag}${field.disabled ? " disabled" : ""} ${compactLine(field.label || field.selector)}${field.value ? ` value=${compactLine(field.value, 60)}` : field.valueRedacted ? " value=[redacted]" : ""}`); + for (const action of (inspect.formContext.actions || []).slice(0, 10)) lines.push(`- ${action.uid} action${action.disabled ? " disabled" : ""} ${compactLine(action.label || action.selector)}`); + } + if (Array.isArray(inspect.nearbyActions) && inspect.nearbyActions.length) { + lines.push("\n## Nearby actions"); + for (const action of inspect.nearbyActions.slice(0, 18)) lines.push(`- ${action.uid} ${action.role || action.tag}${action.disabled ? " disabled" : ""} ${compactLine(action.label || action.selector)} @ ${rectText(action.rect)}`); + } + if (Array.isArray(inspect.ancestors) && inspect.ancestors.length) { + lines.push("\n## Ancestors"); + for (const a of inspect.ancestors.slice(0, 6)) lines.push(`- ${a.uid} ${a.role || a.tag} ${compactLine(a.label || a.selector, 120)}`); + } + return truncateText(lines.join("\n")); +} + function extensionRoot(): string { // Resolve relative to this extension file, not ctx.cwd. ctx.cwd can temporarily be // an attachment/clipboard path when Pi is handling pasted images. @@ -1109,13 +1240,16 @@ Usage rules: name: "chrome_snapshot", label: "Chrome Snapshot", description: - "Inspect a page in the user's existing Chrome profile: title, URL, visible body text, viewport, and clickable/focusable elements with stable uids plus CSS selectors. Runs in the background by default; pass background=false to bring Chrome to the foreground so the user can watch.", - promptSnippet: "Inspect the current Chrome page and get CSS selectors for browser automation.", + "Inspect a page in the user's existing Chrome profile. Default output is a concise, agent-friendly observation with structural layout/context, stable uids, visible actions, form fields, page hints, and changes since the previous snapshot. Use mode/query/nearUid to zoom instead of dumping the whole page. Runs in the background by default; pass background=false to bring Chrome to the foreground so the user can watch.", + promptSnippet: "Observe the current Chrome page: concise summary, structural layout, visible actions, forms, page map, query matches, and stable uids.", parameters: Type.Object({ targetId: Type.Optional(Type.String()), urlIncludes: Type.Optional(Type.String()), titleIncludes: Type.Optional(Type.String()), maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS })), + mode: Type.Optional(StringEnum(snapshotModeValues)), + query: Type.Optional(Type.String({ description: "Find/rank elements, regions, and text matching this phrase, e.g. 'merge button', 'email error', 'approve PR'." })), + maxTextChars: Type.Optional(Type.Number({ description: "Max body text chars included in the underlying snapshot. Defaults are smaller for concise modes." })), containingText: Type.Optional(Type.String({ description: "Only return elements whose label/text contains this string (case-insensitive). Useful when the page has many controls." })), roleFilter: Type.Optional(Type.String({ description: "Only return elements matching this ARIA role or tag name (case-insensitive). e.g. 'button', 'link', 'textbox'." })), nearUid: Type.Optional(Type.String({ description: "Sort elements by proximity to this snapshot uid. Useful for finding controls near a known anchor." })), @@ -1132,7 +1266,84 @@ Usage rules: DEFAULT_TIMEOUT_MS, signal, ); - return { content: [{ type: "text", text: truncateText(safeJson(snapshot)) }], details: { snapshot } }; + return { content: [{ type: "text", text: formatChromeSnapshot(snapshot) }], details: { snapshot } }; + }, + }); + + pi.registerTool({ + name: "chrome_find", + label: "Chrome Find", + description: + "Find elements, page regions, or text on the current Chrome page by query. Returns ranked matches with stable uids and coordinates. This is a focused wrapper around chrome_snapshot({ query }).", + promptSnippet: "Find matching controls/text/regions in Chrome by natural-language query and return stable uids.", + parameters: Type.Object({ + query: Type.String({ description: "What to find, e.g. 'merge button', 'email error', 'approve PR', 'search box'." }), + mode: Type.Optional(StringEnum(snapshotModeValues)), + maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional( + Type.Boolean({ description: "If true (the default), run silently in the background without focusing Chrome; pass false so Chrome focuses + the tab activates and the user can watch." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params, signal): Promise<ToolTextResult> { + const snapshot = await authorizedBridgeSend( + "page.snapshot", + withBackground({ ...params, mode: params.mode || "auto", maxElements: params.maxElements ?? MAX_ELEMENTS }), + DEFAULT_TIMEOUT_MS, + signal, + ); + return { content: [{ type: "text", text: formatChromeSnapshot(snapshot) }], details: { snapshot } }; + }, + }); + + pi.registerTool({ + name: "chrome_inspect", + label: "Chrome Inspect Element", + description: + "Inspect one snapshot uid or selector deeply: nearby text, nearby actions, form context, ancestors, and suggested click target. Use after chrome_snapshot/chrome_find when you need context around one element.", + promptSnippet: "Inspect a Chrome snapshot uid deeply for nearby text, form context, and suggested actions.", + parameters: Type.Object({ + uid: Type.Optional(Type.String({ description: "Stable element uid from chrome_snapshot/chrome_find." })), + selector: Type.Optional(Type.String({ description: "CSS selector if uid is unavailable." })), + scrollIntoView: Type.Optional(Type.Boolean({ description: "If true, scroll the target into view before inspecting. Default false to avoid changing page state." })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional( + Type.Boolean({ description: "If true (the default), run silently in the background without focusing Chrome; pass false so Chrome focuses + the tab activates and the user can watch." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params, signal): Promise<ToolTextResult> { + try { + const inspect = await authorizedBridgeSend("page.inspect", withBackground(params), DEFAULT_TIMEOUT_MS, signal); + return { content: [{ type: "text", text: formatChromeInspect(inspect) }], details: { inspect } }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!/Unknown action: page\.inspect/i.test(message)) throw error; + // Compatibility fallback for a loaded Chrome extension service worker that has not + // been reloaded since chrome_inspect was added. It is less rich than page.inspect, + // but still gives useful nearby candidates instead of failing the workflow. + const snapshot = await authorizedBridgeSend( + "page.snapshot", + withBackground({ + ...params, + mode: "interactive", + maxElements: MAX_ELEMENTS, + nearUid: params.uid, + query: params.selector, + }), + DEFAULT_TIMEOUT_MS, + signal, + ); + const text = `chrome_inspect fallback: loaded Chrome extension does not yet support page.inspect; reload it at chrome://extensions for deep inspect.\n\n${formatChromeSnapshot(snapshot)}`; + return { content: [{ type: "text", text }], details: { snapshot, fallback: "page.snapshot" } }; + } }, }); @@ -1219,7 +1430,7 @@ Usage rules: const summary = summarizeActionResult(result); const target = params.uid ?? params.selector ?? `${params.x},${params.y}`; const text = summary ? `Clicked ${target} — ${summary}` : `Clicked ${target}`; - return { content: [{ type: "text", text }], details: { result: raw as Json } }; + return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } }; }, }); @@ -1251,7 +1462,8 @@ Usage rules: const summary = summarizeActionResult(result); const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""; const base = `Typed ${params.text.length} character(s)${into}.`; - return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } }; + const text = summary ? `${base} (${summary})` : base; + return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } }; }, }); @@ -1283,7 +1495,8 @@ Usage rules: const summary = summarizeActionResult(result); const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""; const base = `Filled ${params.text.length} character(s)${into}.`; - return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } }; + const text = summary ? `${base} (${summary})` : base; + return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } }; }, }); @@ -1317,7 +1530,8 @@ Usage rules: const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json; const summary = summarizeActionResult(result); const base = `Pressed ${params.key}.`; - return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } }; + const text = summary ? `${base} (${summary})` : base; + return { content: [{ type: "text", text: formatIncludedSnapshotText(raw, text) }], details: { result: raw as Json } }; }, }); |
