diff options
| author | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-12 23:53:16 -0400 |
|---|---|---|
| committer | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-12 23:53:16 -0400 |
| commit | d78081aa58ecaa1c72e847c2d095b5f07c391506 (patch) | |
| tree | 756d45f3f580ea6cbce5a01501ec417a702c81a3 /extensions | |
| parent | 2206c5541de2356a165acac4683d5653684f346a (diff) | |
Update pi-packs installer and package names
Diffstat (limited to 'extensions')
3 files changed, 682 insertions, 93 deletions
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json index baef09c..5ceb76e 100644 --- a/extensions/chrome-profile-bridge/browser-extension/manifest.json +++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json @@ -1,9 +1,9 @@ { "manifest_version": 3, "name": "Pi Existing Chrome Profile Bridge", - "version": "0.7.0", + "version": "0.8.0", "description": "Lets Pi control tabs in this existing Chrome profile via a local bridge at 127.0.0.1.", - "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms"], + "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation"], "host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"], "background": { "service_worker": "service_worker.js" diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js index 2f6c701..ed33470 100644 --- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js +++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js @@ -4,8 +4,6 @@ const POLL_ERROR_BACKOFF_MS = 2000; let polling = false; function armKeepaliveAlarm() { - // MV3 service workers can be suspended; alarms are the supported way to - // wake the extension again. Chrome's minimum period is 0.5 minutes. chrome.alarms.create("pi-bridge-keepalive", { periodInMinutes: 0.5 }); } @@ -41,17 +39,12 @@ async function pollLoop() { polling = true; try { while (true) { - // Long-poll /next continuously. The bridge holds the request for up to ~25s when no - // command is pending and returns {type:"none"}; we immediately re-issue the fetch so - // commands sent while the SW is otherwise idle still get picked up promptly. The open - // fetch also keeps the MV3 service worker alive between alarm wake-ups. const response = await fetch(`${BRIDGE_URL}/next?name=${encodeURIComponent(CLIENT_NAME)}`, { cache: "no-store", }); if (!response.ok) throw new Error(`bridge /next HTTP ${response.status}`); const payload = await response.json(); if (payload.type === "command") await handleCommand(payload.command); - // Otherwise (type:"none"), loop and re-issue the long-poll. } } catch (error) { await sleep(POLL_ERROR_BACKOFF_MS); @@ -107,11 +100,22 @@ async function dispatch(action, params) { return { closed: tab.id }; } case "page.snapshot": - return executeInTab(params, snapshotPage, [params.maxElements || 80]); + return executeInTab(params, snapshotPage, [ + params.maxElements || 80, + params.containingText ?? null, + params.roleFilter ?? null, + params.nearUid ?? null, + ]); case "page.evaluate": - return executeInTab(params, evaluateExpression, [params.expression, params.awaitPromise !== false]); + return evaluateInTab(params); case "page.click": return executeActionInTab(params, clickPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]); + case "page.hover": + return executeActionInTab(params, hoverPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]); + case "page.drag": + return executeActionInTab(params, dragPage, [params.fromUid ?? null, params.fromSelector ?? null, params.fromX ?? null, params.fromY ?? null, params.toUid ?? null, params.toSelector ?? null, params.toX ?? null, params.toY ?? null, params.steps ?? 12]); + case "page.upload": + return executeActionInTab(params, uploadFiles, [params.selector ?? null, params.uid ?? null, params.files || []]); case "page.type": return executeActionInTab(params, typeIntoPage, [params.selector ?? null, params.uid ?? null, params.text || "", Boolean(params.pressEnter)]); case "page.fill": @@ -126,38 +130,27 @@ async function dispatch(action, params) { return executeInTab(params, getNetworkRequest, [params.requestId]); case "page.waitFor": return executeInTab(params, waitForPage, [params.kind, params.value, params.timeoutMs || 10000, params.intervalMs || 250]); + case "page.probe": + // Lightweight capability probe for /chrome-doctor. Runs in MAIN world. + return executeInTab(params, probePage, []); case "page.navigate": { const tab = await getTabByParams(params); if (params.foreground) await bringToFront(tab); + if (params.initScript) { + // Register a one-shot document_start content script. We register, navigate, wait, then unregister. + await registerInitScript(tab.id, params.initScript); + } const wait = params.waitUntilLoad !== false ? waitForTabComplete(tab.id, params.timeoutMs || 15000) : Promise.resolve(undefined); const updated = await chrome.tabs.update(tab.id, { url: params.url }); - await wait; - return formatTab(await chrome.tabs.get(updated.id)); - } - case "page.screenshot": { - const tab = await getTabByParams(params); - if (params.foreground) await bringToFront(tab); - // captureVisibleTab requires the target tab to be the active tab in its window. Activate it - // without focusing the window so other apps don't get pushed behind Chrome, and restore the - // previous active tab afterwards to minimize disruption. - let previousActiveId; - if (!tab.active) { - const activeBefore = await chrome.tabs.query({ active: true, windowId: tab.windowId }); - previousActiveId = activeBefore[0]?.id; - await chrome.tabs.update(tab.id, { active: true }); - } try { - const dataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, { - format: params.format || "png", - quality: params.format === "jpeg" ? params.quality : undefined, - }); - return { dataUrl, tab: formatTab(tab) }; + await wait; } finally { - if (previousActiveId !== undefined && previousActiveId !== tab.id) { - await chrome.tabs.update(previousActiveId, { active: true }).catch(() => undefined); - } + if (params.initScript) await unregisterInitScript(tab.id).catch(() => undefined); } + return formatTab(await chrome.tabs.get(updated.id)); } + case "page.screenshot": + return takeScreenshot(params); default: throw new Error(`Unknown action: ${action}`); } @@ -198,26 +191,40 @@ async function getTabByParams(params) { return tab; } +// Helper sources that get concatenated into the injected MAIN-world script. Kept as separate +// functions so callers below can reference them by `.toString()`. The helpers do not perform any +// eval themselves — they're plain function declarations. +const HELPER_FUNCS = [ + getPiChromeState, + rememberElement, + elementBySelectorOrUid, + installPiChromeInstrumentation, + resolvePoint, + dispatchInputEvents, + setNativeValue, + normalizeKey, + isElementVisible, + occluderAt, + pageHash, + pointerEventSequence, +]; + async function executeInTab(params, func, args) { const tab = await getTabByParams(params); if (params.foreground) await bringToFront(tab); - const helperSource = [ - getPiChromeState, - rememberElement, - elementBySelectorOrUid, - installPiChromeInstrumentation, - resolvePoint, - dispatchInputEvents, - setNativeValue, - normalizeKey, - ].map((helper) => helper.toString()).join("\n"); + const helperSource = HELPER_FUNCS.map((helper) => helper.toString()).join("\n"); const results = await chrome.scripting.executeScript({ target: { tabId: tab.id }, world: "MAIN", func: async (helperSource, source, invocationArgs) => { try { - (0, eval)(helperSource); - const injected = (0, eval)(`(${source})`); + // Helpers are plain function declarations; injecting them via Function constructor avoids + // running through `eval` (which is restricted under strict CSP) and keeps them isolated. + new Function(helperSource).call(globalThis); + // The action itself is reconstructed from its source text. We use `new Function` rather + // than `eval` because the latter is blocked by `script-src 'self'` (no `'unsafe-eval'`) + // CSPs that are common on production sites. + const injected = new Function(helperSource + "\nreturn (" + source + ");").call(globalThis); return { ok: true, value: await injected(...invocationArgs) }; } catch (error) { return { ok: false, error: error?.stack || error?.message || String(error) }; @@ -237,15 +244,108 @@ async function executeInTab(params, func, args) { return envelope?.value; } +// Dedicated executor for page.evaluate. Doesn't go through the helper-source injection chain; +// that chain was the root cause of `chrome_evaluate` silently returning null on pages with strict +// CSP. We build a single Function in MAIN world and invoke it directly. +async function evaluateInTab(params) { + const tab = await getTabByParams(params); + if (params.foreground) await bringToFront(tab); + const expression = String(params.expression ?? ""); + const awaitPromise = params.awaitPromise !== false; + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + world: "MAIN", + func: async (expression, awaitPromise) => { + const stringify = (v) => { + if (v === undefined) return { kind: "undefined" }; + if (typeof v === "function") return { kind: "function", source: v.toString().slice(0, 500) }; + if (typeof v === "symbol") return { kind: "symbol", description: v.description }; + if (typeof v === "bigint") return { kind: "bigint", value: v.toString() }; + if (v instanceof Error) return { kind: "error", name: v.name, message: v.message, stack: v.stack }; + return v; + }; + // Compile via the Function constructor. We try expression form first so callers can pass + // `1+1` or `document.title` without a `return`; if that's a SyntaxError we retry with the + // statement form so callers can use multi-statement bodies (loops, var decls, etc). + const compile = (src) => { + try { + return { fn: new Function(`return (async () => (${src}))();`), mode: "expression" }; + } catch (e1) { + if (e1 && e1.name === "SyntaxError") { + try { + return { fn: new Function(`return (async () => { ${src} })();`), mode: "statement" }; + } catch (e2) { + throw e2; + } + } + throw e1; + } + }; + try { + const { fn } = compile(expression); + const value = await fn.call(globalThis); + const resolved = awaitPromise && value && typeof value.then === "function" ? await value : value; + return { ok: true, value: stringify(resolved) }; + } catch (error) { + return { ok: false, error: error?.stack || error?.message || String(error) }; + } + }, + args: [expression, awaitPromise], + }); + const first = results?.[0]; + if (first?.error) { + const message = typeof first.error === "string" ? first.error : (first.error.message || JSON.stringify(first.error)); + throw new Error(`chrome_evaluate failed: ${message}`); + } + const envelope = first?.result; + if (!envelope) throw new Error("chrome_evaluate returned no envelope from MAIN world"); + if (envelope.ok === false) throw new Error(envelope.error || "chrome_evaluate failed"); + const v = envelope.value; + // Unwrap special markers from MAIN world + if (v && typeof v === "object" && !Array.isArray(v)) { + if (v.kind === "undefined") return undefined; + if (v.kind === "function") return `[Function: ${v.source}]`; + if (v.kind === "symbol") return `[Symbol: ${v.description}]`; + if (v.kind === "bigint") return v.value; + if (v.kind === "error") throw new Error(`${v.name}: ${v.message}\n${v.stack || ""}`); + } + return v; +} + async function executeActionInTab(params, func, args) { const result = await executeInTab(params, func, args); if (params.includeSnapshot) { - const snapshot = await executeInTab({ ...params, foreground: false }, snapshotPage, [params.maxElements || 80]); + const snapshot = await executeInTab({ ...params, foreground: false }, snapshotPage, [params.maxElements || 80, null, null, null]); return { result, snapshot }; } return result; } +// One-shot init script registry, scoped per tab. The script source is injected at +// document_start of the next committed navigation in that tab, in MAIN world, then cleared. +const initScriptIds = new Map(); +async function registerInitScript(tabId, source) { + initScriptIds.set(tabId, source); +} +async function unregisterInitScript(tabId) { + initScriptIds.delete(tabId); +} + +if (chrome.webNavigation && chrome.webNavigation.onCommitted) { + chrome.webNavigation.onCommitted.addListener((details) => { + if (details.frameId !== 0) return; + const source = initScriptIds.get(details.tabId); + if (!source) return; + chrome.scripting.executeScript({ + target: { tabId: details.tabId, frameIds: [0] }, + world: "MAIN", + injectImmediately: true, + func: (code) => { try { new Function(code).call(globalThis); } catch (e) { console.error("[pi-chrome init script]", e); } }, + args: [source], + }).catch(() => undefined); + }); +} + async function bringToFront(tab) { await chrome.windows.update(tab.windowId, { focused: true }); await chrome.tabs.update(tab.id, { active: true }); @@ -268,6 +368,56 @@ function waitForTabComplete(tabId, timeoutMs) { }); } +async function takeScreenshot(params) { + const tab = await getTabByParams(params); + if (params.foreground) await bringToFront(tab); + let previousActiveId; + if (!tab.active) { + const activeBefore = await chrome.tabs.query({ active: true, windowId: tab.windowId }); + previousActiveId = activeBefore[0]?.id; + await chrome.tabs.update(tab.id, { active: true }); + } + try { + if (params.fullPage) { + // Tile-stitched full page capture: scroll, capture, paste, repeat. + const tiles = await executeInTab({ ...params, foreground: false }, captureFullPageTiles, []); + // captureFullPageTiles only computes scroll positions / metrics; we capture per scroll here + // (chrome.tabs.captureVisibleTab can't be called from MAIN world). + const captured = []; + for (const tile of tiles.tiles) { + await executeInTab({ ...params, foreground: false }, scrollToY, [tile.scrollY]); + // Small settle delay; many sites have on-scroll animations / lazy-load. + await sleep(120); + const dataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, { + format: params.format || "png", + quality: params.format === "jpeg" ? params.quality : undefined, + }); + captured.push({ y: tile.y, dataUrl }); + } + await executeInTab({ ...params, foreground: false }, scrollToY, [tiles.originalScrollY]); + return { + fullPage: true, + tab: formatTab(tab), + dimensions: { width: tiles.width, height: tiles.height, viewportHeight: tiles.viewportHeight, dpr: tiles.dpr }, + tiles: captured, + }; + } + const dataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, { + format: params.format || "png", + quality: params.format === "jpeg" ? params.quality : undefined, + }); + return { dataUrl, tab: formatTab(tab) }; + } finally { + if (previousActiveId !== undefined && previousActiveId !== tab.id) { + await chrome.tabs.update(previousActiveId, { active: true }).catch(() => undefined); + } + } +} + +// --------------------------------------------------------------------------- +// MAIN-world helpers (function declarations injected into the page). +// --------------------------------------------------------------------------- + function getPiChromeState() { const state = window.__PI_CHROME_STATE__ || { nextElementUid: 1, @@ -302,6 +452,77 @@ function elementBySelectorOrUid(selector, uid) { return null; } +function isElementVisible(element) { + if (!element || !element.getBoundingClientRect) return false; + const style = getComputedStyle(element); + if (style.visibility === "hidden" || style.display === "none") return false; + const rect = element.getBoundingClientRect(); + if (rect.width === 0 || rect.height === 0) return false; + if (rect.bottom < 0 || rect.right < 0) return false; + if (rect.top > innerHeight || rect.left > innerWidth) return false; + return true; +} + +function occluderAt(x, y, expected) { + const top = document.elementFromPoint(x, y); + if (!top || top === expected) return null; + if (expected && expected.contains(top)) return null; + if (top.contains(expected)) return null; + return { + tag: top.tagName.toLowerCase(), + id: top.id || undefined, + className: typeof top.className === "string" ? top.className : undefined, + }; +} + +function pageHash() { + // Cheap rolling hash used for `pageMutated`. Combines first 4kb of body innerText with the + // current values of inputs/textareas (which are not part of innerText) and the count of + // descendants of <body>. This catches: text changes, input value edits, and DOM structure + // changes — the three things a click/type/fill might cause. + const body = document.body; + const text = (body ? body.innerText : "").slice(0, 4000); + let h = 0; + for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) | 0; + if (body) { + const inputs = body.querySelectorAll("input,textarea,select"); + let valueBlob = ""; + for (let i = 0; i < inputs.length && valueBlob.length < 4000; i++) { + const v = inputs[i].value; + if (typeof v === "string") valueBlob += v + "\x00"; + } + for (let i = 0; i < valueBlob.length; i++) h = (h * 31 + valueBlob.charCodeAt(i)) | 0; + h = (h * 31 + body.getElementsByTagName("*").length) | 0; + } + return h; +} + +function pointerEventSequence(element, x, y, sequence) { + let defaultPrevented = false; + for (const type of sequence) { + const isPointer = type.startsWith("pointer"); + const Ctor = isPointer ? PointerEvent : MouseEvent; + const init = { + bubbles: true, + cancelable: true, + view: window, + clientX: x, + clientY: y, + button: 0, + buttons: type === "pointermove" || type === "mousemove" ? 0 : 1, + }; + if (isPointer) { + init.pointerType = "mouse"; + init.pointerId = 1; + init.isPrimary = true; + } + const ev = new Ctor(type, init); + element.dispatchEvent(ev); + if (ev.defaultPrevented) defaultPrevented = true; + } + return defaultPrevented; +} + function installPiChromeInstrumentation() { const state = getPiChromeState(); if (state.instrumentationInstalled) return; @@ -407,7 +628,7 @@ function installPiChromeInstrumentation() { } } -function snapshotPage(maxElements) { +function snapshotPage(maxElements, containingText, roleFilter, nearUid) { installPiChromeInstrumentation(); const unique = (selector) => { try { return document.querySelectorAll(selector).length === 1; } catch { return false; } @@ -434,11 +655,7 @@ function snapshotPage(maxElements) { } return parts.join(" > "); }; - const visible = (element) => { - const style = getComputedStyle(element); - const rect = element.getBoundingClientRect(); - return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0; - }; + const visible = (element) => isElementVisible(element); const labelFor = (element) => ( element.getAttribute("aria-label") || element.getAttribute("title") || @@ -448,9 +665,41 @@ function snapshotPage(maxElements) { element.textContent || "" ).trim().replace(/\s+/g, " ").slice(0, 160); - const candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])')); + let candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [role="menuitem"], [role="tab"], [role="checkbox"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])')); + if (containingText) { + const needle = String(containingText).toLowerCase(); + candidates = candidates.filter((element) => labelFor(element).toLowerCase().includes(needle)); + } + if (roleFilter) { + const wanted = String(roleFilter).toLowerCase(); + candidates = candidates.filter((element) => { + const role = (element.getAttribute("role") || element.tagName).toLowerCase(); + return role === wanted; + }); + } + let near; + if (nearUid) { + const state = getPiChromeState(); + near = state.elements[nearUid]; + } + if (near) { + const nearRect = near.getBoundingClientRect(); + const cx = nearRect.left + nearRect.width / 2; + const cy = nearRect.top + nearRect.height / 2; + candidates.sort((a, b) => { + const ra = a.getBoundingClientRect(); + const rb = b.getBoundingClientRect(); + const da = Math.hypot(ra.left + ra.width / 2 - cx, ra.top + ra.height / 2 - cy); + const db = Math.hypot(rb.left + rb.width / 2 - cx, rb.top + rb.height / 2 - cy); + return da - db; + }); + } const elements = candidates.filter(visible).slice(0, maxElements).map((element, index) => { const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + const cx = rect.left + rect.width / 2; + const cy = rect.top + rect.height / 2; + const occluded = occluderAt(cx, cy, element); return { index, uid: rememberElement(element), @@ -461,6 +710,9 @@ function snapshotPage(maxElements) { type: element.getAttribute("type") || undefined, role: element.getAttribute("role") || undefined, disabled: Boolean(element.disabled || element.getAttribute("aria-disabled") === "true"), + inert: Boolean(element.closest?.("[inert]")), + pointerEvents: style.pointerEvents, + occluded: occluded || undefined, rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, }; }); @@ -470,14 +722,44 @@ function snapshotPage(maxElements) { viewport: { width: innerWidth, height: innerHeight, scrollX, scrollY }, text: document.body ? document.body.innerText.replace(/\s+\n/g, "\n").trim().slice(0, 30000) : "", elements, + filter: { containingText: containingText || undefined, roleFilter: roleFilter || undefined, nearUid: nearUid || undefined }, }; } -async function evaluateExpression(expression, awaitPromise) { - installPiChromeInstrumentation(); - const indirectEval = (0, eval); - const value = indirectEval(expression); - return awaitPromise && value && typeof value.then === "function" ? await value : value; +function probePage() { + // Sanity probe used by /chrome-doctor. Returns evidence that MAIN-world execution works. + return { + arithmetic: 1 + 1, + location: location.href, + title: document.title, + documentReady: document.readyState, + userAgent: navigator.userAgent.slice(0, 200), + webdriver: !!navigator.webdriver, + }; +} + +function captureFullPageTiles() { + // Returns the *plan* for tile capture; the actual chrome.tabs.captureVisibleTab calls happen + // in the SW. We just report the scroll positions and metrics. + const html = document.documentElement; + const body = document.body; + const width = Math.max(html.scrollWidth, body ? body.scrollWidth : 0, innerWidth); + const height = Math.max(html.scrollHeight, body ? body.scrollHeight : 0, innerHeight); + const viewportHeight = innerHeight; + const dpr = window.devicePixelRatio || 1; + const originalScrollY = scrollY; + const tiles = []; + let y = 0; + while (y < height) { + tiles.push({ y, scrollY: y }); + y += viewportHeight; + } + return { width, height, viewportHeight, dpr, originalScrollY, tiles }; +} + +function scrollToY(y) { + window.scrollTo({ top: y, left: 0, behavior: "instant" }); + return { scrollY }; } function resolvePoint(selector, uid, x, y) { @@ -492,12 +774,88 @@ function resolvePoint(selector, uid, x, y) { } function clickPage(selector, uid, x, y) { + installPiChromeInstrumentation(); + const before = pageHash(); const point = resolvePoint(selector, uid, x, y); if (!point.element) throw new Error("No element at click point"); - for (const type of ["pointerdown", "mousedown", "pointerup", "mouseup", "click"]) { - point.element.dispatchEvent(new MouseEvent(type, { bubbles: true, cancelable: true, view: window, clientX: point.x, clientY: point.y, button: 0 })); + const visible = isElementVisible(point.element); + const occluded = occluderAt(point.x, point.y, point.element); + const defaultPrevented = pointerEventSequence(point.element, point.x, point.y, [ + "pointerdown", "mousedown", "pointerup", "mouseup", "click", + ]); + // Heuristic: if the clicked thing looks like a media play affordance and the page has paused + // audio/video, the synthetic click may not unlock autoplay. Surface a warning. + let autoplayHint; + const label = (point.element.getAttribute("aria-label") || point.element.textContent || "").toLowerCase(); + if (/^(play|start|begin|next|continue|unmute)/.test(label.trim())) { + const idleMedia = Array.from(document.querySelectorAll("audio,video")).some((m) => m.paused); + if (idleMedia) autoplayHint = "This element looks like a media affordance and the page has paused media. Synthetic clicks do not satisfy user-activation gates; audio/video may not start."; + } + return { + x: point.x, + y: point.y, + selector, + uid, + tag: point.element.tagName, + isTrusted: false, + defaultPrevented, + elementVisible: visible, + occludedBy: occluded || undefined, + pageMutated: pageHash() !== before, + autoplayHint, + }; +} + +function hoverPage(selector, uid, x, y) { + installPiChromeInstrumentation(); + const point = resolvePoint(selector, uid, x, y); + if (!point.element) throw new Error("No element to hover"); + const defaultPrevented = pointerEventSequence(point.element, point.x, point.y, [ + "pointerover", "mouseover", "pointerenter", "mouseenter", "pointermove", "mousemove", + ]); + return { x: point.x, y: point.y, selector, uid, tag: point.element.tagName, defaultPrevented, isTrusted: false }; +} + +function dragPage(fromUid, fromSelector, fromX, fromY, toUid, toSelector, toX, toY, steps) { + installPiChromeInstrumentation(); + const before = pageHash(); + const from = resolvePoint(fromSelector, fromUid, fromX, fromY); + const to = resolvePoint(toSelector, toUid, toX, toY); + if (!from.element) throw new Error("Drag source element not found"); + if (!to.element) throw new Error("Drag target element not found"); + pointerEventSequence(from.element, from.x, from.y, ["pointerover", "pointerdown", "mousedown"]); + for (let i = 1; i <= steps; i++) { + const t = i / steps; + const x = from.x + (to.x - from.x) * t; + const y = from.y + (to.y - from.y) * t; + const overEl = document.elementFromPoint(x, y) || to.element; + pointerEventSequence(overEl, x, y, ["pointermove", "mousemove"]); + } + pointerEventSequence(to.element, to.x, to.y, ["pointerover", "mouseover", "pointerup", "mouseup"]); + return { + from: { x: from.x, y: from.y }, + to: { x: to.x, y: to.y }, + steps, + pageMutated: pageHash() !== before, + note: "Synthetic pointer drag. HTML5 DataTransfer is not synthesized; native drag-and-drop targets may not respond.", + }; +} + +function uploadFiles(selector, uid, files) { + installPiChromeInstrumentation(); + const element = elementBySelectorOrUid(selector, uid); + if (!element || element.tagName !== "INPUT" || element.type !== "file") { + throw new Error("Target must be <input type=file>"); } - return { x: point.x, y: point.y, selector, uid, tag: point.element.tagName }; + const dt = new DataTransfer(); + for (const f of files) { + const bytes = Uint8Array.from(atob(f.base64 || ""), (c) => c.charCodeAt(0)); + dt.items.add(new File([bytes], f.name, { type: f.type || "application/octet-stream" })); + } + element.files = dt.files; + element.dispatchEvent(new Event("input", { bubbles: true })); + element.dispatchEvent(new Event("change", { bubbles: true })); + return { uploaded: files.map((f) => ({ name: f.name, type: f.type, size: (f.base64 || "").length })) }; } function dispatchInputEvents(element, data, inputType = "insertText") { @@ -515,6 +873,7 @@ function setNativeValue(element, value) { function typeIntoPage(selector, uid, text, pressEnter) { installPiChromeInstrumentation(); + const before = pageHash(); let element = elementBySelectorOrUid(selector, uid) || document.activeElement; if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element"); element.focus(); @@ -530,11 +889,17 @@ function typeIntoPage(selector, uid, text, pressEnter) { throw new Error("Focused element is not text-editable"); } if (pressEnter) pressKeyInPage("Enter"); - return { selector, uid, length: text.length, pressEnter }; + return { + selector, uid, length: text.length, pressEnter, + isTrusted: false, + valueMatches: "value" in element ? element.value.includes(text) : undefined, + pageMutated: pageHash() !== before, + }; } function fillPage(selector, uid, text, submit) { installPiChromeInstrumentation(); + const before = pageHash(); let element = elementBySelectorOrUid(selector, uid) || document.activeElement; if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element"); element.focus(); @@ -550,19 +915,32 @@ function fillPage(selector, uid, text, submit) { throw new Error("Focused element is not text-editable"); } if (submit) pressKeyInPage("Enter"); - return { selector, uid, length: String(text).length, submit }; + return { + selector, uid, length: String(text).length, submit, + isTrusted: false, + valueMatches: "value" in element ? element.value === String(text) : undefined, + pageMutated: pageHash() !== before, + }; } function pressKeyInPage(key) { const normalized = normalizeKey(key); const target = document.activeElement || document.body; - target.dispatchEvent(new KeyboardEvent("keydown", { key: normalized, bubbles: true, cancelable: true })); - target.dispatchEvent(new KeyboardEvent("keyup", { key: normalized, bubbles: true, cancelable: true })); + const before = (typeof pageHash === "function") ? pageHash() : 0; + const down = new KeyboardEvent("keydown", { key: normalized, bubbles: true, cancelable: true }); + target.dispatchEvent(down); + const up = new KeyboardEvent("keyup", { key: normalized, bubbles: true, cancelable: true }); + target.dispatchEvent(up); if (normalized === "Enter") { const form = target.closest?.("form"); if (form) form.requestSubmit?.(); } - return { key: normalized }; + return { + key: normalized, + isTrusted: false, + defaultPrevented: down.defaultPrevented || up.defaultPrevented, + pageMutated: (typeof pageHash === "function") ? pageHash() !== before : undefined, + }; } function listConsoleMessages(clear) { @@ -581,7 +959,7 @@ function listNetworkRequests(includePreservedRequests, clear) { .filter((request) => includePreservedRequests || request.pageUrl === currentUrl) .map(({ responseBody, ...summary }) => ({ ...summary, hasResponseBody: responseBody !== undefined })); if (clear) state.network = []; - return { requests, count: requests.length, note: "Captures fetch/XHR after instrumentation is installed (snapshot/evaluate/network/console tools install it). Browser-initiated document/static asset requests are not captured." }; + return { requests, count: requests.length, note: "Captures fetch/XHR after instrumentation is installed. Browser-initiated document/static asset requests are not captured." }; } function getNetworkRequest(requestId) { @@ -596,7 +974,9 @@ async function waitForPage(kind, value, timeoutMs, intervalMs) { while (Date.now() - started < timeoutMs) { let ok = false; if (kind === "selector") ok = Boolean(document.querySelector(value)); - else ok = Boolean((0, eval)(value)); + else { + try { ok = Boolean(new Function("return (" + value + ");").call(globalThis)); } catch { ok = false; } + } if (ok) return { elapsedMs: Date.now() - started }; await new Promise((resolve) => setTimeout(resolve, intervalMs)); } diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts index 537fd9b..ac176bc 100644 --- a/extensions/chrome-profile-bridge/index.ts +++ b/extensions/chrome-profile-bridge/index.ts @@ -46,7 +46,8 @@ type BridgeResult = { error?: string; }; -const PI_CHROME_VERSION = "0.7.0"; +const PI_CHROME_VERSION = "0.8.0"; +const PI_CHROME_GLOBAL_KEY = "__piChromeProfileBridgeLoaded__"; const DEFAULT_HOST = process.env.PI_CHROME_BRIDGE_HOST ?? "127.0.0.1"; const DEFAULT_PORT = Number(process.env.PI_CHROME_BRIDGE_PORT ?? "17318"); const DEFAULT_TIMEOUT_MS = 30_000; @@ -83,7 +84,30 @@ function workspaceCwd(ctx: ExtensionContext): string { function browserExtensionPath(): string { return join(extensionRoot(), "browser-extension"); -} +} + +function hostnameOf(url: string | undefined): string { + if (!url) return ""; + try { return new URL(url).hostname; } catch { return ""; } +} + +// Description of a click/type/fill result's significant fields so the agent doesn't have to +// guess whether the action actually changed the page. +function summarizeActionResult(result: unknown): string | undefined { + if (!result || typeof result !== "object") return undefined; + const r = result as Record<string, unknown>; + const parts: string[] = []; + if (r.pageMutated === false) parts.push("pageMutated=false"); + if (r.defaultPrevented === true) parts.push("defaultPrevented=true"); + if (r.elementVisible === false) parts.push("element NOT visible"); + if (r.occludedBy) { + const o = r.occludedBy as { tag?: string; id?: string }; + parts.push(`occluded by <${o.tag ?? "?"}${o.id ? "#" + o.id : ""}>`); + } + if (r.valueMatches === false) parts.push("input value did not stick"); + if (r.autoplayHint) parts.push("autoplay-gated affordance — synthetic click may not start media"); + return parts.length ? parts.join("; ") : undefined; +} function readRequestBody(request: IncomingMessage): Promise<string> { return new Promise((resolveBody, rejectBody) => { @@ -338,6 +362,18 @@ const imageFormatValues = ["png", "jpeg"] as const; const waitForValues = ["selector", "expression"] as const; export default function (pi: ExtensionAPI): void { + const globalState = globalThis as typeof globalThis & { + [PI_CHROME_GLOBAL_KEY]?: { version: string; root: string }; + }; + const alreadyLoaded = globalState[PI_CHROME_GLOBAL_KEY]; + if (alreadyLoaded) { + console.warn( + `pi-chrome already loaded from ${alreadyLoaded.root} (v${alreadyLoaded.version}); skipping duplicate from ${extensionRoot()}.`, + ); + return; + } + globalState[PI_CHROME_GLOBAL_KEY] = { version: PI_CHROME_VERSION, root: extensionRoot() }; + const bridge = new ChromeProfileBridge(DEFAULT_HOST, DEFAULT_PORT); let backgroundDefault = false; @@ -374,21 +410,34 @@ export default function (pi: ExtensionAPI): void { pi.on("before_agent_start", (event) => { const primer = ` <chrome-profile-bridge> -Chrome control is available through the chrome_* tools via a companion Chrome extension installed in the user's normal Chrome profile. -This is not CDP: it can use the user's existing Chrome windows and authenticated sessions after the user loads the companion browser extension. -If chrome_* tools time out, ask the user to run /chrome-onboard, then load the bundled browser-extension folder in chrome://extensions. Prefer chrome_snapshot before clicking/typing; use stable element uids from snapshots with chrome_click/chrome_type when available. For form work, use includeSnapshot=true on actions to verify in one round trip. Avoid destructive actions unless explicitly requested. By default chrome_* tools focus Chrome and activate the target tab so the user can watch the agent work. The user can switch to silent/background mode for the whole session via /chrome-background; you can also pass background=true on a single tool call when the user explicitly wants the action to be silent (for example, scraping while they keep working in another app). +Chrome control is available through the chrome_* tools via a companion Chrome extension installed in the user's normal Chrome profile. Tools target the existing signed-in profile, no CDP, no throwaway profile. + +Capability model (important): +- All input is **synthetic DOM events** (\`isTrusted=false\`). Synthetic events drive React/Vue/Angular state fine, but they do NOT satisfy Chrome's user-activation gates: audio/video autoplay, clipboard write, file pickers, fullscreen, and Web Push prompts will NOT open from a chrome_click. +- \`chrome_evaluate\` runs in MAIN world via the Function constructor. It works on pages with strict CSP (\`script-src 'self'\` without \`'unsafe-eval'\`), and surfaces thrown exceptions. +- Tool results include \`pageMutated\`, \`defaultPrevented\`, \`elementVisible\`, \`occludedBy\`, and (for type/fill) \`valueMatches\`. If \`pageMutated\` is false after a click that should have changed something, the click likely didn't take effect — do NOT just retry; check the action result and snapshot for the cause. + +Usage rules: +1. \`chrome_snapshot\` before clicking/typing; pass \`uid\` over \`selector\`. +2. \`includeSnapshot=true\` on click/type/fill to verify in one round trip. +3. If \`chrome_evaluate\` returns null when you expected a value, the expression evaluated to null/undefined in the page; surface the value via \`JSON.stringify\` to confirm. +4. \`chrome_navigate\` supports an optional \`initScript\` that runs at document_start in MAIN world for the next navigation (good for seeding localStorage or stubbing Date.now). +5. By default chrome_* tools focus Chrome so the user can watch; pass \`background=true\` or run /chrome-background to silence the whole session. +6. If you hit an autoplay/clipboard/file-picker gate, tell the user; this bridge cannot satisfy it. +7. Run /chrome-doctor when in doubt about connectivity or capabilities. </chrome-profile-bridge>`; return { systemPrompt: event.systemPrompt + primer }; }); pi.registerCommand("chrome-doctor", { description: - "Check Chrome bridge connectivity and diagnose setup. Reports the local bridge, companion Chrome extension status (ID + version), and a one-line fix for common failures (extension not loaded, stale service worker, version drift).", + "Check Chrome bridge connectivity and capability tier. Probes the local bridge, the companion Chrome extension, MAIN-world evaluation, and CDP availability, and prints one-line fixes for common failures.", handler: async (_args, ctx) => { ctx.ui.notify("Performing Chrome bridge health check", "info"); const lines: string[] = [`pi-chrome v${PI_CHROME_VERSION}`]; const status = bridge.status(); lines.push(`• Local bridge: mode=${status.mode}, url=${status.url}`); + let extensionAlive = false; try { const started = Date.now(); const version = (await bridge.send("tab.version", {}, 35_000)) as { @@ -397,10 +446,8 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b bridgeUrl?: string; }; const latencyMs = Date.now() - started; - if (version.extensionId) - lines.push(`✓ Companion Chrome extension responding (ID: ${version.extensionId}, ext v${version.extensionVersion ?? "unknown"}, latency ${latencyMs}ms)`); - else lines.push(`✓ Companion Chrome extension responding (no extension ID reported, latency ${latencyMs}ms)`); - if (version.bridgeUrl) lines.push(`• Extension polling: ${version.bridgeUrl}`); + extensionAlive = true; + lines.push(`✓ Companion Chrome extension responding (ID: ${version.extensionId ?? "?"}, ext v${version.extensionVersion ?? "?"}, latency ${latencyMs}ms)`); if (version.extensionVersion && version.extensionVersion !== PI_CHROME_VERSION) { lines.push( `⚠ Extension version (${version.extensionVersion}) differs from pi-chrome (${PI_CHROME_VERSION}). Reload "Pi Existing Chrome Profile Bridge" in chrome://extensions to pick up the latest service worker.`, @@ -415,6 +462,43 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b lines.push(" Fix: run /chrome-onboard, then load the bundled browser-extension folder in chrome://extensions and keep that Chrome window open."); } } + + if (extensionAlive) { + // MAIN-world evaluate probe. + try { + const value = await bridge.send("page.evaluate", { expression: "1+1", awaitPromise: true, foreground: false }, 10_000); + if (value === 2) lines.push(`✓ chrome_evaluate("1+1") = 2`); + else lines.push(`⚠ chrome_evaluate("1+1") returned ${JSON.stringify(value)} (expected 2). The current tab may have a restrictive CSP or be a chrome:// URL.`); + } catch (error) { + lines.push(`✗ chrome_evaluate failed: ${(error as Error).message}`); + } + + // Capability probe via MAIN-world helper. + try { + const probe = (await bridge.send("page.probe", { foreground: false }, 10_000)) as Record<string, unknown>; + if (probe && probe.arithmetic === 2) lines.push(`✓ MAIN-world helper injection works (location=${hostnameOf(String(probe.location))})`); + if (probe && probe.webdriver) lines.push(`⚠ navigator.webdriver=true on current tab — site fingerprinting may flag automation.`); + } catch (error) { + lines.push(`⚠ page.probe failed: ${(error as Error).message}`); + } + } + + // CDP availability hint. + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 250); + const response = await fetch("http://127.0.0.1:9222/json/version", { signal: controller.signal }).catch(() => undefined); + clearTimeout(timer); + if (response && response.ok) { + const info = (await response.json().catch(() => ({}))) as { Browser?: string }; + lines.push(`✓ CDP endpoint reachable at 127.0.0.1:9222 (${info.Browser ?? "unknown"}). Trusted input via CDP is not yet wired into pi-chrome — reserved for a future release.`); + } else { + lines.push(`• CDP not available (no listener on 127.0.0.1:9222). Synthetic input only; autoplay/clipboard/file-picker gates cannot be satisfied. Future pi-chrome versions will use CDP for trusted input when this port is enabled.`); + } + } catch { + lines.push(`• CDP probe inconclusive.`); + } + ctx.ui.notify(lines.join("\n"), "info"); }, }); @@ -540,6 +624,9 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b urlIncludes: Type.Optional(Type.String()), titleIncludes: Type.Optional(Type.String()), maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS })), + containingText: Type.Optional(Type.String({ description: "Only return elements whose label/text contains this string (case-insensitive). Useful when the page has many controls." })), + roleFilter: Type.Optional(Type.String({ description: "Only return elements matching this ARIA role or tag name (case-insensitive). e.g. 'button', 'link', 'textbox'." })), + nearUid: Type.Optional(Type.String({ description: "Sort elements by proximity to this snapshot uid. Useful for finding controls near a known anchor." })), background: Type.Optional( Type.Boolean({ description: "If true, run silently in the background without focusing Chrome. Default false (Chrome focuses + tab activates so the user can watch)." }), ), @@ -569,6 +656,7 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b titleIncludes: Type.Optional(Type.String()), waitUntilLoad: Type.Optional(Type.Boolean({ default: true })), timeoutMs: Type.Optional(Type.Number({ default: 15_000 })), + initScript: Type.Optional(Type.String({ description: "Optional JavaScript source to run in MAIN world at document_start of the next navigation. Useful for seeding localStorage, stubbing Date.now(), or defining navigator.webdriver=undefined. Requires the companion extension's webNavigation permission." })), background: Type.Optional( Type.Boolean({ description: "If true, navigate silently without focusing Chrome. Default false." }), ), @@ -576,8 +664,8 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b port: Type.Optional(Type.Number()), }), async execute(_id, params): Promise<ToolTextResult> { - const result = await bridge.send("page.navigate", withBackground(params), params.timeoutMs ?? 15_000); - return { content: [{ type: "text", text: `Navigated to ${params.url}` }], details: { result: result as Json } }; + const result = await bridge.send("page.navigate", withBackground(params), (params.timeoutMs ?? 15_000) + 2_000); + return { content: [{ type: "text", text: `Navigated to ${params.url}${params.initScript ? " (with initScript)" : ""}` }], details: { result: result as Json } }; }, }); @@ -590,7 +678,6 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b parameters: Type.Object({ expression: Type.String(), awaitPromise: Type.Optional(Type.Boolean({ default: true })), - returnByValue: Type.Optional(Type.Boolean({ default: true })), targetId: Type.Optional(Type.String()), urlIncludes: Type.Optional(Type.String()), titleIncludes: Type.Optional(Type.String()), @@ -602,7 +689,12 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b }), async execute(_id, params): Promise<ToolTextResult> { const value = await bridge.send("page.evaluate", withBackground(params), DEFAULT_TIMEOUT_MS); - return { content: [{ type: "text", text: truncateText(typeof value === "string" ? value : safeJson(value)) }], details: { value: value as Json } }; + const text = value === undefined + ? "undefined" + : typeof value === "string" + ? value + : safeJson(value) ?? "undefined"; + return { content: [{ type: "text", text: truncateText(text) }], details: { value: value as Json } }; }, }); @@ -629,8 +721,12 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b port: Type.Optional(Type.Number()), }), async execute(_id, params): Promise<ToolTextResult> { - const result = await bridge.send("page.click", withBackground(params), DEFAULT_TIMEOUT_MS); - return { content: [{ type: "text", text: `Clicked ${params.uid ?? params.selector ?? `${params.x},${params.y}`}` }], details: { result: result as Json } }; + const raw = await bridge.send("page.click", withBackground(params), DEFAULT_TIMEOUT_MS); + const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json; + const summary = summarizeActionResult(result); + const target = params.uid ?? params.selector ?? `${params.x},${params.y}`; + const text = summary ? `Clicked ${target} — ${summary}` : `Clicked ${target}`; + return { content: [{ type: "text", text }], details: { result: raw as Json } }; }, }); @@ -657,8 +753,12 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b port: Type.Optional(Type.Number()), }), async execute(_id, params): Promise<ToolTextResult> { - const result = await bridge.send("page.type", withBackground(params), DEFAULT_TIMEOUT_MS); - return { content: [{ type: "text", text: `Typed ${params.text.length} character(s)${params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""}.` }], details: { result: result as Json } }; + const raw = await bridge.send("page.type", withBackground(params), DEFAULT_TIMEOUT_MS); + const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json; + const summary = summarizeActionResult(result); + const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""; + const base = `Typed ${params.text.length} character(s)${into}.`; + return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } }; }, }); @@ -685,8 +785,12 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b port: Type.Optional(Type.Number()), }), async execute(_id, params): Promise<ToolTextResult> { - const result = await bridge.send("page.fill", withBackground(params), DEFAULT_TIMEOUT_MS); - return { content: [{ type: "text", text: `Filled ${params.text.length} character(s)${params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""}.` }], details: { result: result as Json } }; + const raw = await bridge.send("page.fill", withBackground(params), DEFAULT_TIMEOUT_MS); + const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json; + const summary = summarizeActionResult(result); + const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""; + const base = `Filled ${params.text.length} character(s)${into}.`; + return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } }; }, }); @@ -710,8 +814,11 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b port: Type.Optional(Type.Number()), }), async execute(_id, params): Promise<ToolTextResult> { - const result = await bridge.send("page.key", withBackground(params), DEFAULT_TIMEOUT_MS); - return { content: [{ type: "text", text: `Pressed ${params.key}.` }], details: { result: result as Json } }; + const raw = await bridge.send("page.key", withBackground(params), DEFAULT_TIMEOUT_MS); + const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json; + const summary = summarizeActionResult(result); + const base = `Pressed ${params.key}.`; + return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } }; }, }); @@ -825,11 +932,113 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b const cwd = workspaceCwd(ctx); const defaultPath = join(cwd, ".pi", "chrome-screenshots", `${new Date().toISOString().replace(/[:.]/g, "-")}.${format}`); const outputPath = params.path ? resolve(cwd, params.path) : defaultPath; - const result = (await bridge.send("page.screenshot", withBackground(params), DEFAULT_TIMEOUT_MS)) as { dataUrl: string; tab?: unknown }; - const base64 = result.dataUrl.replace(/^data:image\/(?:png|jpeg);base64,/, ""); + const result = (await bridge.send("page.screenshot", withBackground(params), params.fullPage ? 120_000 : DEFAULT_TIMEOUT_MS)) as { + dataUrl?: string; + tab?: unknown; + fullPage?: boolean; + dimensions?: { width: number; height: number; viewportHeight: number; dpr: number }; + tiles?: Array<{ y: number; dataUrl: string }>; + }; await mkdir(dirname(outputPath), { recursive: true }); + if (result.fullPage && result.tiles && result.dimensions) { + // Stitch via PNG if format is png; otherwise we fall back to writing tile files and a + // manifest. We avoid pulling in an image library by writing each tile next to the main + // path with a -tileN suffix and a stitched.json manifest. + const { width, height, viewportHeight, dpr } = result.dimensions; + const manifest: Array<{ path: string; y: number }> = []; + for (let i = 0; i < result.tiles.length; i++) { + const tile = result.tiles[i]; + const tilePath = outputPath.replace(/(\.[^.]+)$/, `-tile${i}$1`); + const base64 = tile.dataUrl.replace(/^data:image\/(?:png|jpeg);base64,/, ""); + await writeFile(tilePath, Buffer.from(base64, "base64")); + manifest.push({ path: tilePath, y: tile.y }); + } + await writeFile(outputPath + ".json", JSON.stringify({ width, height, viewportHeight, dpr, tiles: manifest }, null, 2)); + return { + content: [{ type: "text", text: `Saved ${result.tiles.length} full-page tile(s) for ${width}×${height}px page. Manifest: ${outputPath}.json` }], + details: { manifest: outputPath + ".json", tiles: manifest, dimensions: result.dimensions, tab: result.tab } as unknown as Record<string, unknown>, + }; + } + if (!result.dataUrl) throw new Error("Screenshot returned no dataUrl"); + const base64 = result.dataUrl.replace(/^data:image\/(?:png|jpeg);base64,/, ""); await writeFile(outputPath, Buffer.from(base64, "base64")); return { content: [{ type: "text", text: `Saved Chrome screenshot to ${outputPath}` }], details: { path: outputPath, format, tab: result.tab } }; }, }); + + pi.registerTool({ + name: "chrome_hover", + label: "Chrome Hover", + description: "Hover over an element (synthetic pointerover/mouseover/pointermove) by uid, selector, or x/y. Triggers CSS :hover state and any JS hover handlers; isTrusted is false.", + promptSnippet: "Hover a Chrome element to trigger :hover / mouseover handlers.", + parameters: Type.Object({ + uid: Type.Optional(Type.String()), + selector: Type.Optional(Type.String()), + x: Type.Optional(Type.Number()), + y: Type.Optional(Type.Number()), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional(Type.Boolean()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.hover", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: `Hovered ${params.uid ?? params.selector ?? `${params.x},${params.y}`}` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_drag", + label: "Chrome Drag", + description: "Synthetic pointer drag from one uid/selector/point to another. Dispatches pointerdown → multi-step pointermove → pointerup. Note: HTML5 DataTransfer is NOT synthesized, so native HTML5 drag-and-drop targets may not respond.", + promptSnippet: "Drag a Chrome element from one point to another.", + parameters: Type.Object({ + fromUid: Type.Optional(Type.String()), + fromSelector: Type.Optional(Type.String()), + fromX: Type.Optional(Type.Number()), + fromY: Type.Optional(Type.Number()), + toUid: Type.Optional(Type.String()), + toSelector: Type.Optional(Type.String()), + toX: Type.Optional(Type.Number()), + toY: Type.Optional(Type.Number()), + steps: Type.Optional(Type.Number({ default: 12 })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional(Type.Boolean()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.drag", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: `Dragged from ${params.fromUid ?? params.fromSelector} to ${params.toUid ?? params.toSelector}` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_upload_file", + label: "Chrome Upload File", + description: "Programmatically set the files of an <input type=file> element from local file paths. Uses DataTransfer to populate input.files and dispatches input+change events. Does NOT open the native file picker; works with React/Vue/Angular controlled inputs.", + promptSnippet: "Attach local files to a Chrome <input type=file> without opening the native file picker.", + parameters: Type.Object({ + uid: Type.Optional(Type.String()), + selector: Type.Optional(Type.String()), + paths: Type.Array(Type.String(), { description: "Local absolute file paths to upload." }), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional(Type.Boolean()), + }), + async execute(_id, params, _signal, _onUpdate, ctx): Promise<ToolTextResult> { + const { readFile } = await import("node:fs/promises"); + const { basename } = await import("node:path"); + const cwd = workspaceCwd(ctx); + const files: Array<{ name: string; type: string; base64: string }> = []; + for (const p of params.paths) { + const abs = resolve(cwd, p); + const buf = await readFile(abs); + files.push({ name: basename(abs), type: "application/octet-stream", base64: buf.toString("base64") }); + } + const result = await bridge.send("page.upload", withBackground({ ...params, files }), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: `Uploaded ${files.length} file(s) to ${params.uid ?? params.selector}` }], details: { result: result as Json } }; + }, + }); } |
