diff options
| author | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-08 14:15:53 -0400 |
|---|---|---|
| committer | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-08 14:15:53 -0400 |
| commit | 2206c5541de2356a165acac4683d5653684f346a (patch) | |
| tree | df33f0532be66e0799c8aac23189d59f05f3574d | |
Track pi-chrome package source
| -rw-r--r-- | README.md | 166 | ||||
| -rw-r--r-- | extensions/chrome-profile-bridge/browser-extension/manifest.json | 14 | ||||
| -rw-r--r-- | extensions/chrome-profile-bridge/browser-extension/service_worker.js | 619 | ||||
| -rw-r--r-- | extensions/chrome-profile-bridge/index.ts | 835 | ||||
| -rw-r--r-- | package.json | 31 |
5 files changed, 1665 insertions, 0 deletions
diff --git a/README.md b/README.md new file mode 100644 index 0000000..684144d --- /dev/null +++ b/README.md @@ -0,0 +1,166 @@ +# pi-chrome + +[](https://www.npmjs.com/package/pi-chrome) + +Control the Chrome profile you already use from Pi. + +`pi-chrome` gives Pi agents browser tools for your **real Chrome windows, tabs, and authenticated sessions**. It uses a companion Chrome extension instead of the Chrome DevTools Protocol (CDP), so it does not launch a throwaway debug browser profile and does not require re-signing into the apps you already have open. + +Multiple Pi sessions can use Chrome at the same time. The first Pi session starts the local bridge; later sessions automatically detect that bridge and submit their Chrome commands through it. + +## Why try it? + +- **Uses your existing Chrome profile** — works with the Chrome windows/tabs you are already using, including logged-in GitHub, admin dashboards, local apps, and internal tools. +- **Watch your authenticated Chrome work** — by default, `chrome_*` tool calls focus Chrome and activate the target tab so you can see the agent inspect, navigate, click, and type in real time. Switch to silent/background mode for the whole session with `/chrome-background`, or pass `background: true` on a single tool call when you want quiet. +- **Full browser automation toolkit for Pi** — list/create/activate/close tabs, snapshot pages with usable CSS selectors, navigate, evaluate JavaScript, click, type, press keys, wait for page state, and capture screenshots. +- **Built-in setup and agent guidance** — `/chrome-onboard` walks users through installing the companion extension, `/chrome-doctor` checks connectivity and version drift, screenshots save to disk, and the prompt primer tells agents to inspect with `chrome_snapshot` before acting and avoid destructive actions unless explicitly requested. + +## Install + +```bash +pi install npm:pi-chrome +``` + +For local development from a checkout: + +```bash +pi install ./pi-chrome +``` + +### Why an unpacked Chrome extension? + +`pi-chrome` cannot ship through the Chrome Web Store: a Web Store extension is not allowed to talk to a local bridge controlled by another tool. Instead it ships as a small, MIT-licensed unpacked extension in `extensions/chrome-profile-bridge/browser-extension/` — read the source before loading. `/chrome-doctor` reports the loaded extension version and warns when it drifts from the installed `pi-chrome`. + +## First-time setup + +In Pi, run: + +```text +/chrome-onboard +``` + +Pi first shows setup instructions and waits for confirmation. Press Enter to continue. On macOS it will: + +- open `chrome://extensions` +- reveal the bundled `browser-extension` folder in Finder +- copy the extension folder path to your clipboard + +Then in Chrome: + +1. Enable **Developer mode**. +2. Click **Load unpacked**. +3. Select the revealed/copied `browser-extension` folder. +4. Return to Pi and run: + +```text +/chrome-doctor +``` + +Expected output: + +```text +Performing Chrome bridge health check +pi-chrome v<version> +• Local bridge: mode=server, url=http://127.0.0.1:17318 +✓ Companion Chrome extension responding (ID: <chrome-extension-id>, ext v<version>) +``` + +## Background mode + +By default, `chrome_*` tools focus Chrome and activate the target tab so you can watch the agent work — great for demos, pair-driving, debugging, and first-time confidence that things are happening. + +When you want quiet (planner / audit / worker sessions running alongside your editor), turn background mode on for the whole Pi session: + +```text +/chrome-background # toggle +/chrome-background on # explicit +/chrome-background off # explicit +``` + +For a single tool call, the agent can pass `background: true` directly. The per-call value always wins over the session toggle. + +## Quick demo prompts + +After setup, try one of these in Pi: + +Silent inspection (no Chrome interruption): + +```text +Inspect my active GitHub tab with chrome_snapshot using background:true and summarize the PR state without focusing Chrome. +``` + +Existing authenticated tab: + +```text +Use chrome_tab list to find my existing GitHub tab, chrome_snapshot it, then summarize the visible PR state. Do not click anything yet. +``` + +Local web app repro with screenshot: + +```text +Use chrome_tab list to find my localhost app, inspect it with chrome_snapshot, navigate through the bug repro flow, and save a screenshot when you reach the broken state. +``` + +## Recipes + +Copy-paste these into Pi after setup. Each one uses tabs you already have open and accounts you are already signed into. + +- **PR triage:** "Use chrome_tab list to find my GitHub notifications tab, snapshot it, and summarize PRs needing my review." +- **Linear standup:** "Open my Linear current cycle in the active tab, snapshot it, and write me a 5-bullet standup." +- **Bug repro with evidence:** "Open the staging app I'm already signed into, reproduce <bug>, and save a screenshot of each step under ./repro/." +- **Form auto-fill (no submit):** "Open <vendor> portal, fill the new-vendor form from this JSON, but stop before submit." +- **Admin cross-check:** "Across my Stripe / Postmark / our admin tabs, find any user where state disagrees." +- **Local dev visual diff:** "Snapshot localhost:3000 and the staging URL of the same page; tell me what's visually different." +- **Auth-only data pull:** "Open my analytics dashboard tab and chrome_evaluate to extract today's KPIs from the page state." + +Screenshots save under `.pi/chrome-screenshots/` by default, which composes nicely with PR demo workflows. + +## Diagnostics + +- `/chrome-doctor` — single command that checks connectivity and reports the loaded Chrome extension ID + version, plus a one-line fix for common setup failures (extension not loaded, bridge owner stale after `pi update`, version mismatch between pi-chrome and the loaded Chrome extension). + +If the Chrome extension you have loaded is older than `pi-chrome` on disk, `/chrome-doctor` will tell you to reload it from `chrome://extensions`. + +## Compose with + +- **pi-qq** — ask side questions about what the agent saw in Chrome without polluting the main transcript: `/qq summarize what the active GitHub tab shows`. +- **trifecta-footer** — watch context pressure as the agent scrapes large pages; the footer's red threshold is a clean signal to `/qq` for a recap before context overflows. +- **PR demo skills** (such as `ios-pr-agent` / `ios-demo-record` workflows) — `chrome_screenshot` writes to `.pi/chrome-screenshots/` so you can attach images to PR descriptions or demo bundles. + +## Tools + +The package registers these Pi tools: + +- `chrome_launch` — setup/help entry point; opens a URL if the bridge is connected +- `chrome_tab` — list, create, activate, close, or inspect tabs +- `chrome_snapshot` — inspect title, URL, visible text, viewport, and clickable/focusable selectors +- `chrome_navigate` — navigate an existing tab +- `chrome_evaluate` — evaluate JavaScript in a tab +- `chrome_click` — click by selector or coordinates +- `chrome_type` — type text, optionally focusing a selector first +- `chrome_key` — send keyboard keys +- `chrome_wait_for` — wait for a selector or expression +- `chrome_screenshot` — capture viewport screenshots to disk + +These tools are especially useful for authenticated web app debugging, repro flows, admin workflows, visual checks, and inspecting local development pages without rebuilding login state. + +<details> +<summary><strong>How it works (technical details)</strong></summary> + +Pi starts a local bridge on `127.0.0.1:17318`. The companion Chrome extension, installed in your normal Chrome profile, polls that local bridge for commands and executes them using Chrome extension APIs. + +If another Pi session is already running the bridge, additional Pi sessions automatically act as clients of that shared bridge. This lets planner/audit/worker sessions all use the same authenticated Chrome profile concurrently without fighting over the port. + +This is intentionally different from CDP-based tools: the browser extension lives inside the profile you already use, so Pi can interact with existing tabs and authenticated page state. + +</details> + +## Security model + +The companion Chrome extension runs in the Chrome profile where you install it and has broad tab/scripting permissions. Only install it from a package source you trust. + +The Pi side listens on `127.0.0.1:17318` by default. Override before starting Pi with: + +```bash +PI_CHROME_BRIDGE_PORT=17319 pi +``` diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json new file mode 100644 index 0000000..baef09c --- /dev/null +++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json @@ -0,0 +1,14 @@ +{ + "manifest_version": 3, + "name": "Pi Existing Chrome Profile Bridge", + "version": "0.7.0", + "description": "Lets Pi control tabs in this existing Chrome profile via a local bridge at 127.0.0.1.", + "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms"], + "host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"], + "background": { + "service_worker": "service_worker.js" + }, + "action": { + "default_title": "Pi Chrome Bridge" + } +} diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js new file mode 100644 index 0000000..2f6c701 --- /dev/null +++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js @@ -0,0 +1,619 @@ +const BRIDGE_URL = "http://127.0.0.1:17318"; +const CLIENT_NAME = `Pi Chrome Bridge ${chrome.runtime.id}`; +const POLL_ERROR_BACKOFF_MS = 2000; +let polling = false; + +function armKeepaliveAlarm() { + // MV3 service workers can be suspended; alarms are the supported way to + // wake the extension again. Chrome's minimum period is 0.5 minutes. + chrome.alarms.create("pi-bridge-keepalive", { periodInMinutes: 0.5 }); +} + +chrome.runtime.onInstalled.addListener(() => { + chrome.action.setBadgeText({ text: "pi" }); + chrome.action.setBadgeBackgroundColor({ color: "#4f46e5" }); + armKeepaliveAlarm(); + void pollLoop(); +}); + +chrome.runtime.onStartup.addListener(() => { + armKeepaliveAlarm(); + void pollLoop(); +}); + +chrome.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === "pi-bridge-keepalive") void pollLoop(); +}); + +chrome.action.onClicked.addListener(() => { + armKeepaliveAlarm(); + void pollLoop(); +}); + +armKeepaliveAlarm(); + +setInterval(() => { + void pollLoop(); +}, 1000); + +async function pollLoop() { + if (polling) return; + polling = true; + try { + while (true) { + // Long-poll /next continuously. The bridge holds the request for up to ~25s when no + // command is pending and returns {type:"none"}; we immediately re-issue the fetch so + // commands sent while the SW is otherwise idle still get picked up promptly. The open + // fetch also keeps the MV3 service worker alive between alarm wake-ups. + const response = await fetch(`${BRIDGE_URL}/next?name=${encodeURIComponent(CLIENT_NAME)}`, { + cache: "no-store", + }); + if (!response.ok) throw new Error(`bridge /next HTTP ${response.status}`); + const payload = await response.json(); + if (payload.type === "command") await handleCommand(payload.command); + // Otherwise (type:"none"), loop and re-issue the long-poll. + } + } catch (error) { + await sleep(POLL_ERROR_BACKOFF_MS); + } finally { + polling = false; + } +} + +async function handleCommand(command) { + try { + const result = await dispatch(command.action, command.params ?? {}); + await postResult({ id: command.id, ok: true, result }); + } catch (error) { + await postResult({ id: command.id, ok: false, error: error?.message ?? String(error) }); + } +} + +async function postResult(result) { + await fetch(`${BRIDGE_URL}/result`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(result), + }); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function dispatch(action, params) { + switch (action) { + case "tab.version": + return { + extensionId: chrome.runtime.id, + extensionVersion: chrome.runtime.getManifest().version, + bridgeUrl: BRIDGE_URL, + userAgent: navigator.userAgent, + }; + case "tab.list": + return (await chrome.tabs.query({})).map(formatTab); + case "tab.new": { + const tab = await chrome.tabs.create({ url: params.url || "about:blank", active: true }); + return formatTab(tab); + } + case "tab.activate": { + const tab = await getTabByParams(params); + await chrome.windows.update(tab.windowId, { focused: true }); + return formatTab(await chrome.tabs.update(tab.id, { active: true })); + } + case "tab.close": { + const tab = await getTabByParams(params); + await chrome.tabs.remove(tab.id); + return { closed: tab.id }; + } + case "page.snapshot": + return executeInTab(params, snapshotPage, [params.maxElements || 80]); + case "page.evaluate": + return executeInTab(params, evaluateExpression, [params.expression, params.awaitPromise !== false]); + case "page.click": + return executeActionInTab(params, clickPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]); + case "page.type": + return executeActionInTab(params, typeIntoPage, [params.selector ?? null, params.uid ?? null, params.text || "", Boolean(params.pressEnter)]); + case "page.fill": + return executeActionInTab(params, fillPage, [params.selector ?? null, params.uid ?? null, params.text || "", params.submit === true]); + case "page.key": + return executeActionInTab(params, pressKeyInPage, [params.key]); + case "page.console.list": + return executeInTab(params, listConsoleMessages, [params.clear === true]); + case "page.network.list": + return executeInTab(params, listNetworkRequests, [params.includePreservedRequests === true, params.clear === true]); + case "page.network.get": + return executeInTab(params, getNetworkRequest, [params.requestId]); + case "page.waitFor": + return executeInTab(params, waitForPage, [params.kind, params.value, params.timeoutMs || 10000, params.intervalMs || 250]); + case "page.navigate": { + const tab = await getTabByParams(params); + if (params.foreground) await bringToFront(tab); + const wait = params.waitUntilLoad !== false ? waitForTabComplete(tab.id, params.timeoutMs || 15000) : Promise.resolve(undefined); + const updated = await chrome.tabs.update(tab.id, { url: params.url }); + await wait; + return formatTab(await chrome.tabs.get(updated.id)); + } + case "page.screenshot": { + const tab = await getTabByParams(params); + if (params.foreground) await bringToFront(tab); + // captureVisibleTab requires the target tab to be the active tab in its window. Activate it + // without focusing the window so other apps don't get pushed behind Chrome, and restore the + // previous active tab afterwards to minimize disruption. + let previousActiveId; + if (!tab.active) { + const activeBefore = await chrome.tabs.query({ active: true, windowId: tab.windowId }); + previousActiveId = activeBefore[0]?.id; + await chrome.tabs.update(tab.id, { active: true }); + } + try { + const dataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, { + format: params.format || "png", + quality: params.format === "jpeg" ? params.quality : undefined, + }); + return { dataUrl, tab: formatTab(tab) }; + } finally { + if (previousActiveId !== undefined && previousActiveId !== tab.id) { + await chrome.tabs.update(previousActiveId, { active: true }).catch(() => undefined); + } + } + } + default: + throw new Error(`Unknown action: ${action}`); + } +} + +function formatTab(tab) { + return { + id: tab.id, + windowId: tab.windowId, + active: tab.active, + highlighted: tab.highlighted, + title: tab.title || "", + url: tab.url || "", + status: tab.status, + pinned: tab.pinned, + incognito: tab.incognito, + }; +} + +async function getTabByParams(params) { + const tabs = await chrome.tabs.query({}); + let tab; + if (params.targetId !== undefined) { + const id = Number(params.targetId); + tab = tabs.find((candidate) => candidate.id === id); + } else if (params.urlIncludes) { + tab = tabs.find((candidate) => (candidate.url || "").includes(params.urlIncludes)); + } else if (params.titleIncludes) { + tab = tabs.find((candidate) => (candidate.title || "").includes(params.titleIncludes)); + } else { + const active = await chrome.tabs.query({ active: true, lastFocusedWindow: true }); + tab = active[0] || tabs.find((candidate) => candidate.active) || tabs[0]; + } + if (!tab?.id) throw new Error("No matching Chrome tab found"); + if ((tab.url || "").startsWith("chrome://") || (tab.url || "").startsWith("chrome-extension://")) { + throw new Error(`Chrome blocks extension automation on protected URL: ${tab.url}`); + } + return tab; +} + +async function executeInTab(params, func, args) { + const tab = await getTabByParams(params); + if (params.foreground) await bringToFront(tab); + const helperSource = [ + getPiChromeState, + rememberElement, + elementBySelectorOrUid, + installPiChromeInstrumentation, + resolvePoint, + dispatchInputEvents, + setNativeValue, + normalizeKey, + ].map((helper) => helper.toString()).join("\n"); + const results = await chrome.scripting.executeScript({ + target: { tabId: tab.id }, + world: "MAIN", + func: async (helperSource, source, invocationArgs) => { + try { + (0, eval)(helperSource); + const injected = (0, eval)(`(${source})`); + return { ok: true, value: await injected(...invocationArgs) }; + } catch (error) { + return { ok: false, error: error?.stack || error?.message || String(error) }; + } + }, + args: [helperSource, func.toString(), args], + }); + const first = results?.[0]; + if (first?.error) { + const message = typeof first.error === "string" ? first.error : (first.error.message || JSON.stringify(first.error)); + throw new Error(message); + } + const envelope = first?.result; + if (envelope && typeof envelope === "object" && envelope.ok === false) { + throw new Error(envelope.error || "Chrome page script failed"); + } + return envelope?.value; +} + +async function executeActionInTab(params, func, args) { + const result = await executeInTab(params, func, args); + if (params.includeSnapshot) { + const snapshot = await executeInTab({ ...params, foreground: false }, snapshotPage, [params.maxElements || 80]); + return { result, snapshot }; + } + return result; +} + +async function bringToFront(tab) { + await chrome.windows.update(tab.windowId, { focused: true }); + await chrome.tabs.update(tab.id, { active: true }); +} + +function waitForTabComplete(tabId, timeoutMs) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + chrome.tabs.onUpdated.removeListener(listener); + reject(new Error(`Timed out after ${timeoutMs}ms waiting for tab ${tabId} to load`)); + }, timeoutMs); + const listener = (updatedTabId, changeInfo) => { + if (updatedTabId === tabId && changeInfo.status === "complete") { + clearTimeout(timer); + chrome.tabs.onUpdated.removeListener(listener); + resolve(true); + } + }; + chrome.tabs.onUpdated.addListener(listener); + }); +} + +function getPiChromeState() { + const state = window.__PI_CHROME_STATE__ || { + nextElementUid: 1, + elements: {}, + console: [], + network: [], + nextRequestId: 1, + instrumentationInstalled: false, + }; + window.__PI_CHROME_STATE__ = state; + return state; +} + +function rememberElement(element) { + const state = getPiChromeState(); + if (!element.__piChromeUid) element.__piChromeUid = "el-" + state.nextElementUid++; + state.elements[element.__piChromeUid] = element; + return element.__piChromeUid; +} + +function elementBySelectorOrUid(selector, uid) { + if (uid) { + const element = getPiChromeState().elements[uid]; + if (!element || !element.isConnected) throw new Error(`No live element for uid: ${uid}. Take a fresh chrome_snapshot.`); + return element; + } + if (selector) { + const element = document.querySelector(selector); + if (!element) throw new Error(`No element matches selector: ${selector}`); + return element; + } + return null; +} + +function installPiChromeInstrumentation() { + const state = getPiChromeState(); + if (state.instrumentationInstalled) return; + state.instrumentationInstalled = true; + const pushConsole = (level, args) => { + state.console.push({ + id: state.console.length + 1, + level, + timestamp: Date.now(), + url: location.href, + args: Array.from(args).map((arg) => { + try { + if (typeof arg === "string") return arg; + if (arg instanceof Error) return { name: arg.name, message: arg.message, stack: arg.stack }; + return JSON.parse(JSON.stringify(arg)); + } catch { + return String(arg); + } + }), + }); + if (state.console.length > 500) state.console.splice(0, state.console.length - 500); + }; + for (const level of ["debug", "log", "info", "warn", "error"]){ + const original = console[level]; + if (typeof original !== "function" || original.__piChromeWrapped) continue; + const wrapped = function(...args) { + pushConsole(level, args); + return original.apply(this, args); + }; + wrapped.__piChromeWrapped = true; + console[level] = wrapped; + } + window.addEventListener("error", (event) => pushConsole("pageerror", [event.message, event.filename + ":" + event.lineno + ":" + event.colno])); + window.addEventListener("unhandledrejection", (event) => pushConsole("unhandledrejection", [event.reason])); + + const trimBody = (text) => typeof text === "string" && text.length > 200000 ? text.slice(0, 200000) + `\n[truncated ${text.length - 200000} chars]` : text; + const record = (entry) => { + state.network.push(entry); + if (state.network.length > 1000) state.network.splice(0, state.network.length - 1000); + return entry; + }; + if (window.fetch && !window.fetch.__piChromeWrapped) { + const originalFetch = window.fetch.bind(window); + const wrappedFetch = async (...args) => { + const id = "req-" + state.nextRequestId++; + const startedAt = Date.now(); + const input = args[0]; + const init = args[1] || {}; + const url = typeof input === "string" ? input : input?.url; + const method = (init.method || input?.method || "GET").toUpperCase(); + const entry = record({ id, type: "fetch", method, url: String(url || ""), startedAt, pageUrl: location.href, status: "pending" }); + try { + const response = await originalFetch(...args); + entry.status = response.status; + entry.statusText = response.statusText; + entry.ok = response.ok; + entry.responseUrl = response.url; + entry.durationMs = Date.now() - startedAt; + entry.responseHeaders = Array.from(response.headers.entries()); + response.clone().text().then((text) => { + entry.responseBody = trimBody(text); + entry.responseBodyTruncated = typeof text === "string" && text.length > 200000; + }).catch((error) => { entry.responseBodyError = error?.message || String(error); }); + return response; + } catch (error) { + entry.error = error?.message || String(error); + entry.durationMs = Date.now() - startedAt; + throw error; + } + }; + wrappedFetch.__piChromeWrapped = true; + window.fetch = wrappedFetch; + } + if (window.XMLHttpRequest && !XMLHttpRequest.prototype.open.__piChromeWrapped) { + const originalOpen = XMLHttpRequest.prototype.open; + const originalSend = XMLHttpRequest.prototype.send; + XMLHttpRequest.prototype.open = function(method, url, ...rest) { + this.__piChromeRequest = { method: String(method || "GET").toUpperCase(), url: String(url || "") }; + return originalOpen.call(this, method, url, ...rest); + }; + XMLHttpRequest.prototype.open.__piChromeWrapped = true; + XMLHttpRequest.prototype.send = function(body) { + const id = "req-" + state.nextRequestId++; + const startedAt = Date.now(); + const info = this.__piChromeRequest || {}; + const entry = record({ id, type: "xhr", method: info.method || "GET", url: info.url || "", startedAt, pageUrl: location.href, status: "pending" }); + this.addEventListener("loadend", () => { + entry.status = this.status; + entry.statusText = this.statusText; + entry.responseUrl = this.responseURL; + entry.durationMs = Date.now() - startedAt; + try { entry.responseHeadersText = this.getAllResponseHeaders(); } catch {} + try { + if (typeof this.responseText === "string") { + entry.responseBody = trimBody(this.responseText); + entry.responseBodyTruncated = this.responseText.length > 200000; + } + } catch (error) { entry.responseBodyError = error?.message || String(error); } + }); + this.addEventListener("error", () => { entry.error = "XMLHttpRequest error"; entry.durationMs = Date.now() - startedAt; }); + return originalSend.call(this, body); + }; + } +} + +function snapshotPage(maxElements) { + installPiChromeInstrumentation(); + const unique = (selector) => { + try { return document.querySelectorAll(selector).length === 1; } catch { return false; } + }; + const cssEscape = (value) => (window.CSS && CSS.escape) ? CSS.escape(value) : String(value).replace(/[^a-zA-Z0-9_-]/g, "\\$&"); + const selectorFor = (element) => { + if (element.id && unique("#" + cssEscape(element.id))) return "#" + cssEscape(element.id); + const attr = ["aria-label", "name", "placeholder", "data-testid", "role"].find((name) => element.getAttribute(name)); + if (attr) { + const candidate = element.tagName.toLowerCase() + "[" + attr + "=" + JSON.stringify(element.getAttribute(attr)) + "]"; + if (unique(candidate)) return candidate; + } + const parts = []; + let current = element; + while (current && current.nodeType === Node.ELEMENT_NODE && parts.length < 5) { + let part = current.tagName.toLowerCase(); + if (current.classList.length > 0) part += "." + Array.from(current.classList).slice(0, 2).map(cssEscape).join("."); + const siblings = Array.from(current.parentElement?.children ?? []).filter((sibling) => sibling.tagName === current.tagName); + if (siblings.length > 1) part += ":nth-of-type(" + (siblings.indexOf(current) + 1) + ")"; + parts.unshift(part); + const candidate = parts.join(" > "); + if (unique(candidate)) return candidate; + current = current.parentElement; + } + return parts.join(" > "); + }; + const visible = (element) => { + const style = getComputedStyle(element); + const rect = element.getBoundingClientRect(); + return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0; + }; + const labelFor = (element) => ( + element.getAttribute("aria-label") || + element.getAttribute("title") || + element.getAttribute("placeholder") || + element.innerText || + element.value || + element.textContent || + "" + ).trim().replace(/\s+/g, " ").slice(0, 160); + const candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])')); + const elements = candidates.filter(visible).slice(0, maxElements).map((element, index) => { + const rect = element.getBoundingClientRect(); + return { + index, + uid: rememberElement(element), + tag: element.tagName.toLowerCase(), + selector: selectorFor(element), + label: labelFor(element), + href: element.href || undefined, + type: element.getAttribute("type") || undefined, + role: element.getAttribute("role") || undefined, + disabled: Boolean(element.disabled || element.getAttribute("aria-disabled") === "true"), + rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, + }; + }); + return { + title: document.title, + url: location.href, + viewport: { width: innerWidth, height: innerHeight, scrollX, scrollY }, + text: document.body ? document.body.innerText.replace(/\s+\n/g, "\n").trim().slice(0, 30000) : "", + elements, + }; +} + +async function evaluateExpression(expression, awaitPromise) { + installPiChromeInstrumentation(); + const indirectEval = (0, eval); + const value = indirectEval(expression); + return awaitPromise && value && typeof value.then === "function" ? await value : value; +} + +function resolvePoint(selector, uid, x, y) { + const element = elementBySelectorOrUid(selector, uid); + if (element) { + element.scrollIntoView({ block: "center", inline: "center", behavior: "instant" }); + const rect = element.getBoundingClientRect(); + return { element, x: rect.left + rect.width / 2, y: rect.top + rect.height / 2, rect }; + } + if (typeof x !== "number" || typeof y !== "number") throw new Error("Provide selector, uid, or x/y"); + return { element: document.elementFromPoint(x, y), x, y, rect: undefined }; +} + +function clickPage(selector, uid, x, y) { + const point = resolvePoint(selector, uid, x, y); + if (!point.element) throw new Error("No element at click point"); + for (const type of ["pointerdown", "mousedown", "pointerup", "mouseup", "click"]) { + point.element.dispatchEvent(new MouseEvent(type, { bubbles: true, cancelable: true, view: window, clientX: point.x, clientY: point.y, button: 0 })); + } + return { x: point.x, y: point.y, selector, uid, tag: point.element.tagName }; +} + +function dispatchInputEvents(element, data, inputType = "insertText") { + element.dispatchEvent(new InputEvent("beforeinput", { bubbles: true, cancelable: true, inputType, data })); + element.dispatchEvent(new InputEvent("input", { bubbles: true, inputType, data })); + element.dispatchEvent(new Event("change", { bubbles: true })); +} + +function setNativeValue(element, value) { + const prototype = element instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype; + const descriptor = Object.getOwnPropertyDescriptor(prototype, "value"); + if (descriptor?.set) descriptor.set.call(element, value); + else element.value = value; +} + +function typeIntoPage(selector, uid, text, pressEnter) { + installPiChromeInstrumentation(); + let element = elementBySelectorOrUid(selector, uid) || document.activeElement; + if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element"); + element.focus(); + if (element.isContentEditable) { + document.execCommand("insertText", false, text); + } else if ("value" in element) { + const start = element.selectionStart ?? element.value.length; + const end = element.selectionEnd ?? element.value.length; + setNativeValue(element, element.value.slice(0, start) + text + element.value.slice(end)); + element.selectionStart = element.selectionEnd = start + text.length; + dispatchInputEvents(element, text, "insertText"); + } else { + throw new Error("Focused element is not text-editable"); + } + if (pressEnter) pressKeyInPage("Enter"); + return { selector, uid, length: text.length, pressEnter }; +} + +function fillPage(selector, uid, text, submit) { + installPiChromeInstrumentation(); + let element = elementBySelectorOrUid(selector, uid) || document.activeElement; + if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element"); + element.focus(); + if (element.isContentEditable) { + element.textContent = ""; + document.execCommand("insertText", false, text); + } else if ("value" in element) { + setNativeValue(element, text); + const length = String(text).length; + try { element.selectionStart = element.selectionEnd = length; } catch {} + dispatchInputEvents(element, text, "insertReplacementText"); + } else { + throw new Error("Focused element is not text-editable"); + } + if (submit) pressKeyInPage("Enter"); + return { selector, uid, length: String(text).length, submit }; +} + +function pressKeyInPage(key) { + const normalized = normalizeKey(key); + const target = document.activeElement || document.body; + target.dispatchEvent(new KeyboardEvent("keydown", { key: normalized, bubbles: true, cancelable: true })); + target.dispatchEvent(new KeyboardEvent("keyup", { key: normalized, bubbles: true, cancelable: true })); + if (normalized === "Enter") { + const form = target.closest?.("form"); + if (form) form.requestSubmit?.(); + } + return { key: normalized }; +} + +function listConsoleMessages(clear) { + installPiChromeInstrumentation(); + const state = getPiChromeState(); + const messages = state.console.slice(); + if (clear) state.console = []; + return { messages, count: messages.length }; +} + +function listNetworkRequests(includePreservedRequests, clear) { + installPiChromeInstrumentation(); + const state = getPiChromeState(); + const currentUrl = location.href; + const requests = state.network + .filter((request) => includePreservedRequests || request.pageUrl === currentUrl) + .map(({ responseBody, ...summary }) => ({ ...summary, hasResponseBody: responseBody !== undefined })); + if (clear) state.network = []; + return { requests, count: requests.length, note: "Captures fetch/XHR after instrumentation is installed (snapshot/evaluate/network/console tools install it). Browser-initiated document/static asset requests are not captured." }; +} + +function getNetworkRequest(requestId) { + installPiChromeInstrumentation(); + const request = getPiChromeState().network.find((entry) => entry.id === requestId); + if (!request) throw new Error(`No network request with id ${requestId}`); + return request; +} + +async function waitForPage(kind, value, timeoutMs, intervalMs) { + const started = Date.now(); + while (Date.now() - started < timeoutMs) { + let ok = false; + if (kind === "selector") ok = Boolean(document.querySelector(value)); + else ok = Boolean((0, eval)(value)); + if (ok) return { elapsedMs: Date.now() - started }; + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + throw new Error(`Timed out after ${timeoutMs}ms waiting for ${kind}: ${value}`); +} + +function normalizeKey(key) { + const table = { + enter: "Enter", + escape: "Escape", + tab: "Tab", + backspace: "Backspace", + delete: "Delete", + arrowup: "ArrowUp", + arrowdown: "ArrowDown", + arrowleft: "ArrowLeft", + arrowright: "ArrowRight", + }; + return table[String(key).toLowerCase()] || key; +} diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts new file mode 100644 index 0000000..537fd9b --- /dev/null +++ b/extensions/chrome-profile-bridge/index.ts @@ -0,0 +1,835 @@ +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { StringEnum } from "@earendil-works/pi-ai"; +import { Type } from "typebox"; +import { existsSync, statSync } from "node:fs"; +import { mkdir, writeFile } from "node:fs/promises"; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { dirname, join, resolve } from "node:path"; + +/** + * Existing-profile Chrome bridge for pi. + * + * This is intentionally not a Chrome DevTools Protocol integration. CDP cannot attach to + * already-running normal Chrome windows and recent Chrome builds block default-profile + * remote debugging. Instead, install the companion Chrome extension from the + * browser-extension folder bundled next to this Pi extension. + * + * The companion extension runs inside the user's real Chrome profile and polls this local + * pi extension for commands. That gives pi access to the user's existing tabs/authenticated + * profile, subject to the browser extension permissions the user grants. + */ + +type Json = null | boolean | number | string | Json[] | { [key: string]: Json }; + +type ToolTextResult = { + content: Array<{ type: "text"; text: string }>; + details?: Record<string, unknown>; +}; + +type BridgeCommand = { + id: string; + action: string; + params: Record<string, unknown>; +}; + +type PendingCommand = { + command: BridgeCommand; + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; +}; + +type BridgeResult = { + id: string; + ok: boolean; + result?: unknown; + error?: string; +}; + +const PI_CHROME_VERSION = "0.7.0"; +const DEFAULT_HOST = process.env.PI_CHROME_BRIDGE_HOST ?? "127.0.0.1"; +const DEFAULT_PORT = Number(process.env.PI_CHROME_BRIDGE_PORT ?? "17318"); +const DEFAULT_TIMEOUT_MS = 30_000; +const MAX_TEXT_CHARS = 30_000; +const MAX_ELEMENTS = 80; + +function truncateText(text: string, maxChars = MAX_TEXT_CHARS): string { + if (text.length <= maxChars) return text; + return `${text.slice(0, maxChars)}\n\n[truncated ${text.length - maxChars} characters]`; +} + +function safeJson(value: unknown): string { + return JSON.stringify(value, null, 2); +} + +function extensionRoot(): string { + // Resolve relative to this extension file, not ctx.cwd. ctx.cwd can temporarily be + // an attachment/clipboard path when Pi is handling pasted images. + if (typeof __dirname === "string") return __dirname; + return process.cwd(); +} + +function workspaceCwd(ctx: ExtensionContext): string { + for (const candidate of [ctx.cwd, process.cwd()]) { + if (!candidate) continue; + try { + if (existsSync(candidate) && statSync(candidate).isDirectory()) return candidate; + } catch { + // try next candidate + } + } + return process.cwd(); +} + +function browserExtensionPath(): string { + return join(extensionRoot(), "browser-extension"); +} + +function readRequestBody(request: IncomingMessage): Promise<string> { + return new Promise((resolveBody, rejectBody) => { + const chunks: Buffer[] = []; + request.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + request.on("end", () => resolveBody(Buffer.concat(chunks).toString("utf8"))); + request.on("error", rejectBody); + }); +} + +function sendJson(response: ServerResponse, status: number, body: unknown): void { + response.writeHead(status, { + "content-type": "application/json; charset=utf-8", + "access-control-allow-origin": "*", + "access-control-allow-methods": "GET,POST,OPTIONS", + "access-control-allow-headers": "content-type", + "cache-control": "no-store", + }); + response.end(JSON.stringify(body)); +} + +class ChromeProfileBridge { + private server: Server | undefined; + private pending = new Map<string, PendingCommand>(); + private queue: BridgeCommand[] = []; + private waiters: Array<(command: BridgeCommand | undefined) => void> = []; + private lastSeenAt: number | undefined; + private clientName: string | undefined; + private mode: "server" | "client" | undefined; + + constructor( + private readonly host: string, + private readonly port: number, + ) {} + + get url(): string { + return `http://${this.host}:${this.port}`; + } + + get connected(): boolean { + // MV3 service workers can pause between polls/alarms. Treat a recent poll as + // connected without sending a synthetic command; real chrome_* tool calls are + // the authoritative end-to-end health check. + return this.lastSeenAt !== undefined && Date.now() - this.lastSeenAt < 5 * 60_000; + } + + status(): Record<string, unknown> { + return { + url: this.url, + mode: this.mode ?? "starting", + connected: this.connected, + lastSeenAt: this.lastSeenAt, + clientName: this.clientName, + queuedCommands: this.queue.length, + pendingCommands: this.pending.size, + }; + } + + async start(): Promise<void> { + if (this.server || this.mode === "client") return; + this.server = createServer((request, response) => { + void this.handle(request, response).catch((error) => { + sendJson(response, 500, { error: (error as Error).message }); + }); + }); + try { + await new Promise<void>((resolveStart, rejectStart) => { + this.server!.once("error", rejectStart); + this.server!.listen(this.port, this.host, () => { + this.server!.off("error", rejectStart); + resolveStart(); + }); + }); + this.mode = "server"; + } catch (error) { + this.server.close(); + this.server = undefined; + if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error; + // Another Pi session already owns the bridge port. Use it as the shared + // machine-local broker so multiple Pi sessions can control Chrome at once. + this.mode = "client"; + } + } + + stop(): void { + if (this.mode === "client") { + this.mode = undefined; + return; + } + for (const pending of this.pending.values()) { + clearTimeout(pending.timer); + pending.reject(new Error("Chrome profile bridge stopped")); + } + this.pending.clear(); + this.queue = []; + for (const waiter of this.waiters) waiter(undefined); + this.waiters = []; + this.server?.close(); + this.server = undefined; + this.mode = undefined; + } + + send(action: string, params: Record<string, unknown>, timeoutMs = DEFAULT_TIMEOUT_MS): Promise<unknown> { + if (this.mode === "client") return this.sendViaOwner(action, params, timeoutMs); + return this.sendLocal(action, params, timeoutMs); + } + + private sendLocal(action: string, params: Record<string, unknown>, timeoutMs = DEFAULT_TIMEOUT_MS): Promise<unknown> { + const id = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + const command = { id, action, params }; + return new Promise((resolveCommand, rejectCommand) => { + const timer = setTimeout(() => { + this.pending.delete(id); + this.queue = this.queue.filter((queued) => queued.id !== id); + rejectCommand( + new Error( + `Timed out waiting for Chrome extension after ${timeoutMs}ms. Run /chrome-onboard, then load the bundled browser-extension folder in your normal Chrome profile.`, + ), + ); + }, timeoutMs); + this.pending.set(id, { command, resolve: resolveCommand, reject: rejectCommand, timer }); + this.enqueue(command); + }); + } + + private async sendViaOwner(action: string, params: Record<string, unknown>, timeoutMs: number): Promise<unknown> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs + 2_000); + try { + const response = await fetch(`${this.url}/command`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ action, params, timeoutMs }), + signal: controller.signal, + }); + const payload = (await response.json().catch(() => ({}))) as { ok?: boolean; result?: unknown; error?: string }; + if (response.status === 404) { + throw new Error( + "A running Pi session owns the Chrome bridge but is using an older pi-chrome without multi-session support. Restart that Pi session after `pi update`, then retry.", + ); + } + if (!response.ok || !payload.ok) throw new Error(payload.error ?? `Chrome bridge owner HTTP ${response.status}`); + return payload.result; + } catch (error) { + if ((error as Error).name === "AbortError") { + throw new Error(`Timed out waiting for shared Chrome bridge owner after ${timeoutMs}ms`); + } + throw error; + } finally { + clearTimeout(timer); + } + } + + private enqueue(command: BridgeCommand): void { + const waiter = this.waiters.shift(); + if (waiter) waiter(command); + else this.queue.push(command); + } + + private async handle(request: IncomingMessage, response: ServerResponse): Promise<void> { + if (request.method === "OPTIONS") { + sendJson(response, 200, { ok: true }); + return; + } + const url = new URL(request.url ?? "/", this.url); + if (request.method === "GET" && url.pathname === "/status") { + sendJson(response, 200, this.status()); + return; + } + if (request.method === "POST" && url.pathname === "/command") { + const body = JSON.parse(await readRequestBody(request)) as { + action?: string; + params?: Record<string, unknown>; + timeoutMs?: number; + }; + if (!body.action) { + sendJson(response, 400, { ok: false, error: "Missing command action" }); + return; + } + try { + const result = await this.sendLocal(body.action, body.params ?? {}, body.timeoutMs ?? DEFAULT_TIMEOUT_MS); + sendJson(response, 200, { ok: true, result }); + } catch (error) { + sendJson(response, 504, { ok: false, error: (error as Error).message }); + } + return; + } + if (request.method === "GET" && url.pathname === "/next") { + this.lastSeenAt = Date.now(); + this.clientName = url.searchParams.get("name") ?? undefined; + let aborted = false; + let activeWaiter: ((command: BridgeCommand | undefined) => void) | undefined; + request.once("close", () => { + aborted = true; + if (activeWaiter) this.waiters = this.waiters.filter((entry) => entry !== activeWaiter); + }); + let command = this.queue.shift(); + if (!command) { + command = await this.waitForCommand(25_000, (waiter) => { + activeWaiter = waiter; + }); + } + if (aborted) { + // Long-poll connection died before we could deliver. Requeue any command we pulled + // so the next live /next picks it up instead of dropping it on the floor. + if (command) this.queue.unshift(command); + return; + } + sendJson(response, 200, command ? { type: "command", command } : { type: "none" }); + return; + } + if (request.method === "POST" && url.pathname === "/result") { + this.lastSeenAt = Date.now(); + const result = JSON.parse(await readRequestBody(request)) as BridgeResult; + const pending = this.pending.get(result.id); + if (!pending) { + sendJson(response, 404, { ok: false, error: "unknown command id" }); + return; + } + clearTimeout(pending.timer); + this.pending.delete(result.id); + if (result.ok) pending.resolve(result.result); + else pending.reject(new Error(result.error ?? "Chrome extension command failed")); + sendJson(response, 200, { ok: true }); + return; + } + sendJson(response, 404, { error: "not found" }); + } + + private waitForCommand( + timeoutMs: number, + registerWaiter?: (waiter: (command: BridgeCommand | undefined) => void) => void, + ): Promise<BridgeCommand | undefined> { + return new Promise((resolveWait) => { + let settled = false; + const waiter = (command: BridgeCommand | undefined) => { + if (settled) return; + settled = true; + clearTimeout(timer); + this.waiters = this.waiters.filter((entry) => entry !== waiter); + resolveWait(command); + }; + const timer = setTimeout(() => waiter(undefined), timeoutMs); + this.waiters.push(waiter); + registerWaiter?.(waiter); + }); + } +} + +const tabActionValues = ["list", "new", "activate", "close", "version"] as const; +const imageFormatValues = ["png", "jpeg"] as const; +const waitForValues = ["selector", "expression"] as const; + +export default function (pi: ExtensionAPI): void { + const bridge = new ChromeProfileBridge(DEFAULT_HOST, DEFAULT_PORT); + let backgroundDefault = false; + + // Translate the public `background` parameter (default false = visible/foreground) into the + // service worker's wire-level `foreground` flag, accepting legacy `foreground` as a fallback. + const withBackground = <T extends Record<string, unknown>>(params: T): T => { + const typed = params as { background?: boolean; foreground?: boolean }; + const explicit = + typed.background !== undefined + ? typed.background + : typed.foreground !== undefined + ? !typed.foreground + : undefined; + const background = explicit ?? backgroundDefault; + return { ...params, foreground: !background } as T; + }; + + pi.on("session_start", async (_event, ctx) => { + await bridge.start(); + const status = bridge.status(); + ctx.ui.setStatus("chrome", `Chrome bridge :${DEFAULT_PORT}`); + ctx.ui.notify( + status.mode === "client" + ? `Chrome profile bridge connected to shared bridge at ${bridge.url}.` + : `Chrome profile bridge listening at ${bridge.url}. Run /chrome-onboard to load the bundled browser extension in your normal Chrome profile.`, + "info", + ); + }); + + pi.on("session_shutdown", () => { + bridge.stop(); + }); + + pi.on("before_agent_start", (event) => { + const primer = ` +<chrome-profile-bridge> +Chrome control is available through the chrome_* tools via a companion Chrome extension installed in the user's normal Chrome profile. +This is not CDP: it can use the user's existing Chrome windows and authenticated sessions after the user loads the companion browser extension. +If chrome_* tools time out, ask the user to run /chrome-onboard, then load the bundled browser-extension folder in chrome://extensions. Prefer chrome_snapshot before clicking/typing; use stable element uids from snapshots with chrome_click/chrome_type when available. For form work, use includeSnapshot=true on actions to verify in one round trip. Avoid destructive actions unless explicitly requested. By default chrome_* tools focus Chrome and activate the target tab so the user can watch the agent work. The user can switch to silent/background mode for the whole session via /chrome-background; you can also pass background=true on a single tool call when the user explicitly wants the action to be silent (for example, scraping while they keep working in another app). +</chrome-profile-bridge>`; + return { systemPrompt: event.systemPrompt + primer }; + }); + + pi.registerCommand("chrome-doctor", { + description: + "Check Chrome bridge connectivity and diagnose setup. Reports the local bridge, companion Chrome extension status (ID + version), and a one-line fix for common failures (extension not loaded, stale service worker, version drift).", + handler: async (_args, ctx) => { + ctx.ui.notify("Performing Chrome bridge health check", "info"); + const lines: string[] = [`pi-chrome v${PI_CHROME_VERSION}`]; + const status = bridge.status(); + lines.push(`• Local bridge: mode=${status.mode}, url=${status.url}`); + try { + const started = Date.now(); + const version = (await bridge.send("tab.version", {}, 35_000)) as { + extensionId?: string; + extensionVersion?: string; + bridgeUrl?: string; + }; + const latencyMs = Date.now() - started; + if (version.extensionId) + lines.push(`✓ Companion Chrome extension responding (ID: ${version.extensionId}, ext v${version.extensionVersion ?? "unknown"}, latency ${latencyMs}ms)`); + else lines.push(`✓ Companion Chrome extension responding (no extension ID reported, latency ${latencyMs}ms)`); + if (version.bridgeUrl) lines.push(`• Extension polling: ${version.bridgeUrl}`); + if (version.extensionVersion && version.extensionVersion !== PI_CHROME_VERSION) { + lines.push( + `⚠ Extension version (${version.extensionVersion}) differs from pi-chrome (${PI_CHROME_VERSION}). Reload "Pi Existing Chrome Profile Bridge" in chrome://extensions to pick up the latest service worker.`, + ); + } + } catch (error) { + const message = (error as Error).message; + lines.push(`✗ Companion Chrome extension not responding: ${message}`); + if (message.includes("older pi-chrome without multi-session")) { + lines.push(" Fix: restart the Pi session that owns the bridge (it was started on an older pi-chrome)."); + } else { + lines.push(" Fix: run /chrome-onboard, then load the bundled browser-extension folder in chrome://extensions and keep that Chrome window open."); + } + } + ctx.ui.notify(lines.join("\n"), "info"); + }, + }); + + pi.registerCommand("chrome-background", { + description: + "Toggle silent/background mode for chrome_* tools. Background ON: chrome_* tools act silently — your editor/terminal keeps focus, Chrome does not pop up, your workflow is not interrupted. Background OFF (default): Chrome focuses and activates the target tab so you can watch the agent work, useful for demos, pair-driving, and debugging — tradeoff: Chrome pops up and steals focus. Pass `on` / `off` to set explicitly, or no argument to toggle.", + getArgumentCompletions: (prefix) => { + const items = [ + { value: "on", label: "on", description: "Run chrome_* actions silently without focusing Chrome" }, + { value: "off", label: "off", description: "Bring Chrome to the foreground for chrome_* actions (default)" }, + ]; + const lowered = prefix.toLowerCase(); + const matches = items.filter((item) => item.value.startsWith(lowered)); + return matches.length > 0 ? matches : null; + }, + handler: async (args, ctx) => { + const arg = (args || "").trim().toLowerCase(); + if (arg === "on" || arg === "true" || arg === "1") backgroundDefault = true; + else if (arg === "off" || arg === "false" || arg === "0") backgroundDefault = false; + else backgroundDefault = !backgroundDefault; + ctx.ui.notify( + backgroundDefault + ? "Chrome background mode ON. chrome_* tools will run silently. Your current app keeps focus." + : "Chrome background mode OFF. chrome_* tools will focus Chrome and activate the target tab so you can watch the agent work.", + "info", + ); + }, + }); + + pi.registerCommand("chrome-onboard", { + description: "Guide Chrome extension setup for the existing-profile bridge", + handler: async (_args, ctx) => { + const extensionPath = browserExtensionPath(); + const proceed = await ctx.ui.confirm( + "Chrome bridge setup", + `This will open chrome://extensions and reveal the extension folder in Finder.\n\nAfter the windows open: enable Developer mode → Load unpacked → select:\n${extensionPath}\n\nPress Enter to continue, or Esc to cancel.`, + ); + if (!proceed) { + ctx.ui.notify("Chrome bridge setup cancelled", "info"); + return; + } + if (process.platform === "darwin") { + await pi.exec("open", ["-a", "Google Chrome", "chrome://extensions"], { cwd: workspaceCwd(ctx), timeout: 5_000 }).catch(() => undefined); + await pi.exec("open", ["-R", extensionPath], { cwd: workspaceCwd(ctx), timeout: 5_000 }).catch(() => undefined); + await pi.exec("sh", ["-lc", `printf %s ${JSON.stringify(extensionPath)} | pbcopy`], { cwd: workspaceCwd(ctx), timeout: 5_000 }).catch(() => undefined); + } + ctx.ui.notify( + "Chrome bridge setup opened. The extension path has been copied to your clipboard. After loading it, run /chrome-doctor.", + "info", + ); + }, + }); + + pi.registerTool({ + name: "chrome_launch", + label: "Chrome Bridge Setup", + description: + "Start/check the local bridge used by the companion Chrome extension. This does not launch a separate Chrome profile; install the unpacked Chrome extension in your existing Chrome profile to connect.", + promptSnippet: "Show instructions for connecting Pi to the user's existing Chrome profile via the companion extension.", + parameters: Type.Object({ + port: Type.Optional(Type.Number({ description: "Ignored unless PI_CHROME_BRIDGE_PORT is set before Pi starts." })), + url: Type.Optional(Type.String({ description: "Optional URL to open in the existing Chrome profile after the extension is connected." })), + userDataDir: Type.Optional(Type.String({ description: "Ignored. This bridge intentionally uses the user's existing Chrome profile through the companion extension." })), + useDefaultProfile: Type.Optional(Type.Boolean({ description: "Ignored; existing-profile access comes from the companion Chrome extension." })), + headless: Type.Optional(Type.Boolean({ description: "Ignored." })), + }), + async execute(_id, params, _signal, _onUpdate, ctx): Promise<ToolTextResult> { + if (params.url && bridge.connected) { + const result = await bridge.send("tab.new", { url: params.url }, DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: `Chrome bridge connected; opened ${params.url}` }], details: { status: bridge.status(), result } }; + } + return { + content: [ + { + type: "text", + text: + `Chrome profile bridge is listening at ${bridge.url}.\n\n` + + `To connect your existing Chrome profile:\n` + + `1. Open chrome://extensions in the Chrome profile you normally use.\n` + + `2. Enable Developer mode.\n` + + `3. Click “Load unpacked”.\n` + + `4. Select: ${browserExtensionPath()}\n\n` + + `Status: ${bridge.connected ? "connected" : "waiting for extension"}.`, + }, + ], + details: { status: bridge.status(), extensionPath: browserExtensionPath() }, + }; + }, + }); + + pi.registerTool({ + name: "chrome_tab", + label: "Chrome Tab", + description: "List, create, activate, close, or inspect tabs in the user's existing Chrome profile via the companion extension.", + promptSnippet: "List/open/activate/close existing Chrome tabs through the companion extension.", + parameters: Type.Object({ + action: StringEnum(tabActionValues), + url: Type.Optional(Type.String({ description: "URL for action=new." })), + targetId: Type.Optional(Type.String({ description: "Chrome tab id for activate/close." })), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send(`tab.${params.action}`, params, DEFAULT_TIMEOUT_MS); + if (params.action === "list") { + const tabs = result as Array<{ id: number; title: string; url: string; active: boolean; windowId: number }>; + const text = tabs.map((tab) => `${tab.id}\t${tab.active ? "*" : " "}\t${tab.title || "(untitled)"}\t${tab.url}`).join("\n") || "No tabs."; + return { content: [{ type: "text", text }], details: { tabs } }; + } + return { content: [{ type: "text", text: safeJson(result) }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_snapshot", + label: "Chrome Snapshot", + description: + "Inspect a page in the user's existing Chrome profile: title, URL, visible body text, viewport, and clickable/focusable elements with stable uids plus CSS selectors. Brings Chrome to the foreground by default so the user can watch; pass background=true to inspect silently.", + promptSnippet: "Inspect the current Chrome page and get CSS selectors for browser automation.", + parameters: Type.Object({ + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS })), + background: Type.Optional( + Type.Boolean({ description: "If true, run silently in the background without focusing Chrome. Default false (Chrome focuses + tab activates so the user can watch)." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const snapshot = await bridge.send( + "page.snapshot", + withBackground({ ...params, maxElements: params.maxElements ?? MAX_ELEMENTS }), + DEFAULT_TIMEOUT_MS, + ); + return { content: [{ type: "text", text: truncateText(safeJson(snapshot)) }], details: { snapshot } }; + }, + }); + + pi.registerTool({ + name: "chrome_navigate", + label: "Chrome Navigate", + description: + "Navigate an existing Chrome tab to a URL via the companion extension. By default focuses Chrome and activates the tab so the user can watch; pass background=true to navigate silently. Optionally waits for load completion.", + promptSnippet: "Navigate a Chrome tab in the user's existing profile.", + parameters: Type.Object({ + url: Type.String(), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + waitUntilLoad: Type.Optional(Type.Boolean({ default: true })), + timeoutMs: Type.Optional(Type.Number({ default: 15_000 })), + background: Type.Optional( + Type.Boolean({ description: "If true, navigate silently without focusing Chrome. Default false." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.navigate", withBackground(params), params.timeoutMs ?? 15_000); + return { content: [{ type: "text", text: `Navigated to ${params.url}` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_evaluate", + label: "Chrome Evaluate", + description: + "Evaluate JavaScript in an existing Chrome tab through the companion extension. Runs in the page context and returns JSON-serializable values when possible. By default focuses Chrome and activates the tab; pass background=true to evaluate silently.", + promptSnippet: "Evaluate JavaScript in the active Chrome tab through the companion extension.", + parameters: Type.Object({ + expression: Type.String(), + awaitPromise: Type.Optional(Type.Boolean({ default: true })), + returnByValue: Type.Optional(Type.Boolean({ default: true })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional( + Type.Boolean({ description: "If true, evaluate silently without focusing Chrome. Default false." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const value = await bridge.send("page.evaluate", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: truncateText(typeof value === "string" ? value : safeJson(value)) }], details: { value: value as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_click", + label: "Chrome Click", + description: + "Click a snapshot uid, CSS selector, or viewport coordinate in an existing Chrome tab through the companion extension. The click is dispatched as a synthetic DOM event; by default Chrome is focused so the user can watch, pass background=true to click silently. Pass includeSnapshot=true to return a fresh snapshot after the click.", + promptSnippet: "Click page elements in Chrome by snapshot uid, selector, or viewport coordinate.", + parameters: Type.Object({ + uid: Type.Optional(Type.String({ description: "Stable element uid from chrome_snapshot. Prefer uid over selector after taking a snapshot." })), + selector: Type.Optional(Type.String({ description: "CSS selector to click. Prefer uid from chrome_snapshot when available." })), + x: Type.Optional(Type.Number({ description: "Viewport x coordinate if uid/selector is omitted." })), + y: Type.Optional(Type.Number({ description: "Viewport y coordinate if uid/selector is omitted." })), + includeSnapshot: Type.Optional(Type.Boolean({ description: "If true, include a fresh chrome_snapshot result after the click." })), + maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS, description: "Max elements in the included snapshot." })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional( + Type.Boolean({ description: "If true, click silently without focusing Chrome. Default false." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.click", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: `Clicked ${params.uid ?? params.selector ?? `${params.x},${params.y}`}` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_type", + label: "Chrome Type", + description: + "Focus an optional snapshot uid or CSS selector, then type text into an existing Chrome tab through the companion extension. By default focuses Chrome and activates the tab so the user can watch; pass background=true to type silently. Pass includeSnapshot=true to return a fresh snapshot after typing.", + promptSnippet: "Type text into Chrome, optionally focusing a snapshot uid or selector first.", + parameters: Type.Object({ + text: Type.String(), + uid: Type.Optional(Type.String({ description: "Stable element uid from chrome_snapshot." })), + selector: Type.Optional(Type.String({ description: "CSS selector to focus before typing." })), + includeSnapshot: Type.Optional(Type.Boolean({ description: "If true, include a fresh chrome_snapshot result after typing." })), + maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS, description: "Max elements in the included snapshot." })), + pressEnter: Type.Optional(Type.Boolean()), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional( + Type.Boolean({ description: "If true, type silently without focusing Chrome. Default false." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.type", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: `Typed ${params.text.length} character(s)${params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""}.` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_fill", + label: "Chrome Fill", + description: + "Set the full value of a text input, textarea, or contenteditable element using framework-aware native value setters and input/change events. Accepts a snapshot uid or CSS selector. Pass includeSnapshot=true to verify after filling.", + promptSnippet: "Fill a Chrome form field by snapshot uid or selector, optionally returning a fresh snapshot.", + parameters: Type.Object({ + text: Type.String(), + uid: Type.Optional(Type.String({ description: "Stable element uid from chrome_snapshot." })), + selector: Type.Optional(Type.String({ description: "CSS selector to fill if uid is omitted." })), + submit: Type.Optional(Type.Boolean({ description: "If true, press Enter after filling." })), + includeSnapshot: Type.Optional(Type.Boolean({ description: "If true, include a fresh chrome_snapshot result after filling." })), + maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS, description: "Max elements in the included snapshot." })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional( + Type.Boolean({ description: "If true, fill silently without focusing Chrome. Default false." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.fill", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: `Filled ${params.text.length} character(s)${params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""}.` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_key", + label: "Chrome Key", + description: + "Send a keyboard key to an existing Chrome tab (Enter, Escape, Tab, Backspace, Delete, ArrowUp/Down/Left/Right, or one character). By default focuses Chrome and activates the tab so the user can watch; pass background=true to send the key silently. Pass includeSnapshot=true to verify after the keypress.", + promptSnippet: "Press keys in Chrome through the companion extension.", + parameters: Type.Object({ + key: Type.String(), + includeSnapshot: Type.Optional(Type.Boolean({ description: "If true, include a fresh chrome_snapshot result after the keypress." })), + maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS, description: "Max elements in the included snapshot." })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional( + Type.Boolean({ description: "If true, send the key silently without focusing Chrome. Default false." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.key", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: `Pressed ${params.key}.` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_wait_for", + label: "Chrome Wait For", + description: "Poll an existing Chrome tab until a selector exists or a JavaScript expression returns truthy.", + promptSnippet: "Wait for page state in Chrome before further automation.", + parameters: Type.Object({ + kind: StringEnum(waitForValues), + value: Type.String({ description: "CSS selector when kind=selector; JavaScript expression when kind=expression." }), + timeoutMs: Type.Optional(Type.Number({ default: 10_000 })), + intervalMs: Type.Optional(Type.Number({ default: 250 })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.waitFor", params, (params.timeoutMs ?? 10_000) + 2_000); + return { content: [{ type: "text", text: `Observed ${params.kind}: ${params.value}` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_list_console_messages", + label: "Chrome Console Messages", + description: + "List console messages captured in the page by the companion extension. Capture starts after any chrome_snapshot, chrome_evaluate, chrome_list_console_messages, or chrome_list_network_requests call installs page instrumentation.", + promptSnippet: "List captured console messages from the active Chrome page.", + parameters: Type.Object({ + clear: Type.Optional(Type.Boolean({ description: "Clear the captured console log after reading." })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional(Type.Boolean({ description: "If true, run silently without focusing Chrome. Default false." })), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.console.list", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: truncateText(safeJson(result)) }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_list_network_requests", + label: "Chrome Network Requests", + description: + "List fetch/XMLHttpRequest activity captured in the page by the companion extension. Capture starts after instrumentation is installed by snapshot/evaluate/network/console tools; browser document/static asset requests are not captured. Use includePreservedRequests=true to keep requests from earlier same-tab navigations that were captured before navigation.", + promptSnippet: "List captured XHR/fetch requests from the active Chrome page before doing DOM-heavy debugging.", + parameters: Type.Object({ + includePreservedRequests: Type.Optional(Type.Boolean({ description: "Include captured requests from earlier locations in the same tab/session." })), + clear: Type.Optional(Type.Boolean({ description: "Clear the captured request log after reading." })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional(Type.Boolean({ description: "If true, run silently without focusing Chrome. Default false." })), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.network.list", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: truncateText(safeJson(result)) }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_get_network_request", + label: "Chrome Network Request", + description: "Retrieve one captured fetch/XMLHttpRequest entry, including response body when available, by requestId from chrome_list_network_requests.", + promptSnippet: "Fetch captured request details and response body by requestId.", + parameters: Type.Object({ + requestId: Type.String({ description: "Request id returned by chrome_list_network_requests." }), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional(Type.Boolean({ description: "If true, run silently without focusing Chrome. Default false." })), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.network.get", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: truncateText(safeJson(result)) }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ + name: "chrome_screenshot", + label: "Chrome Screenshot", + description: + "Capture a screenshot of an existing Chrome tab via the companion extension and save it to disk. Chrome's extension screenshot API requires the target tab to be the active tab in its window. By default Chrome is focused and the tab activates so the user can watch; pass background=true to capture silently (the tab is briefly activated within its window for the capture, then the previous active tab is restored).", + promptSnippet: "Capture Chrome screenshots and save them under .pi/chrome-screenshots by default.", + parameters: Type.Object({ + path: Type.Optional(Type.String({ description: "Output path. Defaults to .pi/chrome-screenshots/<timestamp>.<format>." })), + format: Type.Optional(StringEnum(imageFormatValues)), + quality: Type.Optional(Type.Number({ description: "JPEG quality 0-100." })), + fullPage: Type.Optional(Type.Boolean({ description: "Not supported by the extension bridge yet; viewport screenshots are captured." })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional( + Type.Boolean({ description: "If true, capture silently without focusing the Chrome window (the target tab is briefly activated within its window for the capture, then restored). Default false." }), + ), + host: Type.Optional(Type.String()), + port: Type.Optional(Type.Number()), + }), + async execute(_id, params, _signal, _onUpdate, ctx: ExtensionContext): Promise<ToolTextResult> { + const format = params.format ?? "png"; + const cwd = workspaceCwd(ctx); + const defaultPath = join(cwd, ".pi", "chrome-screenshots", `${new Date().toISOString().replace(/[:.]/g, "-")}.${format}`); + const outputPath = params.path ? resolve(cwd, params.path) : defaultPath; + const result = (await bridge.send("page.screenshot", withBackground(params), DEFAULT_TIMEOUT_MS)) as { dataUrl: string; tab?: unknown }; + const base64 = result.dataUrl.replace(/^data:image\/(?:png|jpeg);base64,/, ""); + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, Buffer.from(base64, "base64")); + return { content: [{ type: "text", text: `Saved Chrome screenshot to ${outputPath}` }], details: { path: outputPath, format, tab: result.tab } }; + }, + }); +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..1438b37 --- /dev/null +++ b/package.json @@ -0,0 +1,31 @@ +{ + "name": "pi-chrome", + "version": "0.7.0", + "description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).", + "keywords": [ + "pi-package", + "pi-extension", + "chrome", + "browser", + "automation", + "authenticated-session", + "real-profile", + "web-debugging" + ], + "license": "MIT", + "type": "commonjs", + "files": [ + "extensions", + "README.md" + ], + "pi": { + "extensions": [ + "./extensions/chrome-profile-bridge/index.ts" + ] + }, + "peerDependencies": { + "@earendil-works/pi-ai": "*", + "@earendil-works/pi-coding-agent": "*", + "typebox": "*" + } +} |
