aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTianren Dong <tianrendong.dtr@gmail.com>2026-05-12 23:53:16 -0400
committerTianren Dong <tianrendong.dtr@gmail.com>2026-05-12 23:53:16 -0400
commitd78081aa58ecaa1c72e847c2d095b5f07c391506 (patch)
tree756d45f3f580ea6cbce5a01501ec417a702c81a3
parent2206c5541de2356a165acac4683d5653684f346a (diff)
Update pi-packs installer and package names
-rw-r--r--README.md2
-rw-r--r--REBUILD-NOTES.md185
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/manifest.json4
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/service_worker.js514
-rw-r--r--extensions/chrome-profile-bridge/index.ts257
-rw-r--r--package.json2
6 files changed, 869 insertions, 95 deletions
diff --git a/README.md b/README.md
index 684144d..9b2bade 100644
--- a/README.md
+++ b/README.md
@@ -124,7 +124,7 @@ If the Chrome extension you have loaded is older than `pi-chrome` on disk, `/chr
## Compose with
- **pi-qq** — ask side questions about what the agent saw in Chrome without polluting the main transcript: `/qq summarize what the active GitHub tab shows`.
-- **trifecta-footer** — watch context pressure as the agent scrapes large pages; the footer's red threshold is a clean signal to `/qq` for a recap before context overflows.
+- **pi-bar** — watch context pressure as the agent scrapes large pages; the footer's red threshold is a clean signal to `/qq` for a recap before context overflows.
- **PR demo skills** (such as `ios-pr-agent` / `ios-demo-record` workflows) — `chrome_screenshot` writes to `.pi/chrome-screenshots/` so you can attach images to PR descriptions or demo bundles.
## Tools
diff --git a/REBUILD-NOTES.md b/REBUILD-NOTES.md
new file mode 100644
index 0000000..7e49d9f
--- /dev/null
+++ b/REBUILD-NOTES.md
@@ -0,0 +1,185 @@
+# pi-chrome 0.7.0 → 0.8.0 rebuild
+
+Scoped to Tier 1 (bugs + diagnostics), Tier 2 (surface gaps), Tier 5.1 (version
+drift), and Tier 6 (primer) from the design plan. Tier 3 (CDP backend), Tier 4
+(multi-session reliability), and Tier 5.2/5.3 (auto-reload + Web Store) are
+deferred.
+
+## What changed
+
+### T1.1 — `chrome_evaluate` returning `null` (root cause fixed)
+
+`page.evaluate` no longer goes through the `executeInTab` helper-source
+re-eval pipeline. A dedicated `evaluateInTab` path compiles the expression via
+`new Function(...)` directly in MAIN world, with a syntax-error fallback that
+re-compiles in statement-mode so multi-statement bodies still work. Special
+return values (`undefined`, errors, symbols, bigints, functions) are surfaced
+via a tagged envelope so the Node side can return them faithfully.
+
+Verified live:
+- `chrome_evaluate("1+1")` → `2`
+- `chrome_evaluate("Promise.resolve(99)")` → `99`
+- `chrome_evaluate("(()=>{throw new Error('x')})()")` → throws `Error: x`
+- `chrome_evaluate("document.body.innerHTML = '<...>'; return 'ok'")` → `"ok"`
+
+### T1.2 — Truthful click/type/fill results
+
+Every action handler now returns `isTrusted: false`,
+`defaultPrevented`, `elementVisible`, `occludedBy`, `valueMatches`, and
+`pageMutated`. The Node side formats those into the tool result text so the
+agent can read failure cause from a single response.
+
+Verified live:
+- `chrome_click(occluded-button)` → `"Clicked el-3 — pageMutated=false; occluded by <div#overlay>"`
+- `chrome_click(working-button)` → `"Clicked el-1"` (silent on success)
+
+### T1.3 — `/chrome-doctor` extended
+
+The command now probes:
+- Local bridge mode + URL
+- Companion extension version (with mismatch warning vs `PI_CHROME_VERSION`)
+- `chrome_evaluate("1+1") === 2` in the active tab
+- MAIN-world helper injection via `page.probe`, including `navigator.webdriver` fingerprint
+- CDP availability at `127.0.0.1:9222` (informational; not used yet)
+
+### T1.4 — Misleading params removed/implemented
+
+- Dropped `chrome_evaluate.returnByValue` (was ignored).
+- Implemented `chrome_screenshot.fullPage` via tile stitching (see T2.2).
+
+### T1.5 — Autoplay-gate heuristic
+
+`clickPage` checks whether the clicked element's label looks media-related
+(`/^(play|start|begin|next|continue|unmute)/`) AND the page has paused
+`<audio>`/`<video>`. If both, the result includes an `autoplayHint` that surfaces
+to the agent: "synthetic click may not start media".
+
+Verified live: clicking `<button aria-label="Play">` with a paused `<audio>`
+returns `autoplay-gated affordance — synthetic click may not start media`.
+
+### T2.1 — New input tools
+
+- `chrome_hover(uid|selector|x,y)` — dispatches pointerover/mouseover/pointerenter/mouseenter/pointermove/mousemove.
+- `chrome_drag(from*, to*, steps?)` — pointerdown at A, N pointermoves, pointerup at B. Note in tool description that HTML5 `DataTransfer` is NOT synthesized so native drag-and-drop targets may not respond.
+- `chrome_upload_file(uid|selector, paths[])` — reads files from disk, populates `<input type=file>` via `DataTransfer`, dispatches input+change.
+
+Verified live (via direct bridge HTTP since the running pi session was loaded
+before these tools were registered):
+- hover → `defaultPrevented=false`
+- drag from src to dst → `pointerdown` on src and `pointerup` on dst fired; result `pageMutated=true`
+- upload with base64 of `hello world` → input.files has 1 file, name and size match
+
+### T2.2 — Full-page screenshots
+
+`chrome_screenshot({fullPage:true})` now records page dimensions, scrolls
+through the page in viewport-sized increments, captures one tile per scroll
+position via `chrome.tabs.captureVisibleTab`, restores scrollY, and returns an
+array of `{y, dataUrl}`. The Node side writes each tile to disk with a
+`-tileN.png` suffix and a `.json` manifest listing dimensions and tile y
+offsets. (Cross-platform PNG stitching would need an image library; the
+manifest format lets the caller stitch with `sips`, `montage`, `pillow`, etc.)
+
+Verified live: Wikipedia article → 2 tiles, dimensions captured (`width: 1263, height: 919, dpr: 2`).
+
+### T2.3 — `chrome_navigate.initScript`
+
+Added optional `initScript` parameter on `chrome_navigate`. The script is run
+in MAIN world at `webNavigation.onCommitted` of the next navigation in that
+tab, before page scripts execute. Requires the new `webNavigation`
+permission (added to manifest).
+
+Verified live: navigated to `https://example.com` with
+`initScript: "window.__PI_INIT_FIRED__ = Date.now()"`. After load, the global
+was set ~3.5s before `Date.now()` — confirming it fired at document_start, not
+post-load.
+
+### T2.5 — Snapshot filters + extra signals
+
+`chrome_snapshot` accepts:
+- `containingText` — case-insensitive substring filter on the element label
+- `roleFilter` — exact match on `role` attribute OR tag name
+- `nearUid` — sort elements by distance to a previously snapshotted element
+
+Each element now includes:
+- `inert: true` when inside an `[inert]` ancestor
+- `pointerEvents` from computed style
+- `occluded: {tag, id, className}` if `document.elementFromPoint(center)` returns a different element (and not a descendant/ancestor)
+
+Verified live:
+- `containingText:"hidden"` returned only the matching button
+- `roleFilter:"button"` returned only buttons
+- `nearUid:el-anchor` correctly ordered anchor → near → far by pixel distance
+- Occluded button (covered by full-viewport `div#overlay`) reported `occluded: {tag:"div", id:"overlay"}`
+
+### T5.1 — Version drift warning
+
+`/chrome-doctor` already flags version drift between the loaded extension and
+`PI_CHROME_VERSION`. Per-tool-call drift warnings deferred to a future pass
+(needs every chrome_* response to include extension version, which costs an
+extra wire round trip or piggyback header). Bumped version to **0.8.0**.
+
+### T6 — Primer rewritten
+
+The `before_agent_start` primer now spells out:
+- All input is synthetic (`isTrusted=false`)
+- Audio/clipboard/file-picker gates won't work
+- Tool results include `pageMutated`, `defaultPrevented`, `elementVisible`, `occludedBy`, `valueMatches` and what to do with them
+- `chrome_evaluate` returning null means the expression evaluated to null in the page
+- `chrome_navigate.initScript` is available
+- `/chrome-doctor` for capability check
+
+### Other touch-ups
+
+- `chrome_navigate` timeout now adds 2s slack on top of `timeoutMs` to allow the wire round trip.
+- Snapshot's candidate query now includes `[role=menuitem]`, `[role=tab]`, `[role=checkbox]`.
+- `page.waitFor` `expression` mode no longer uses indirect `eval` (uses `Function` constructor for CSP-safe evaluation).
+
+## Files touched
+
+- `extensions/chrome-profile-bridge/index.ts` — bump version, primer, doctor probes, truthful tool result text, new tool registrations, full-page screenshot stitching.
+- `extensions/chrome-profile-bridge/browser-extension/service_worker.js` — rewritten evaluate path, snapshot filters/flags, click/type/fill instrumentation, hover/drag/upload/probe handlers, full-page screenshot tile plan, initScript dispatcher.
+- `extensions/chrome-profile-bridge/browser-extension/manifest.json` — bump version, add `webNavigation` permission.
+- `package.json` — bump version.
+
+## What's deferred (Tier 3 / 4 / 5.x)
+
+- **CDP backend** (`chrome_trusted_*`): not implemented. Requires `puppeteer-core` dep and reconciling CDP target IDs with extension tab IDs. `/chrome-doctor` reports whether `127.0.0.1:9222` is reachable so a future pass can light this up.
+- **Multi-session handover / tab leasing**: not changed.
+- **Extension auto-reload on version drift / Web Store distribution**: not changed.
+
+## To unblock the next pi session
+
+The currently-running pi session has the **old** `index.ts` registered (Pi loads
+extensions at startup). To get the new tool surface (`chrome_hover`,
+`chrome_drag`, `chrome_upload_file`, new `chrome_doctor`, new primer, new
+truthful click results):
+
+1. **Restart this pi session.** (The workspace symlink + global install are
+ both already updated to 0.8.0.)
+2. The Chrome extension at `chrome://extensions` is **already reloaded** to
+ 0.8.0 (you did that during this session).
+3. Run `/chrome-doctor` in the new session — expect 5 green checks.
+
+## Acceptance test results
+
+| Test | Result |
+|------|--------|
+| `chrome_evaluate("1+1")` returns `2` | ✓ |
+| `chrome_evaluate(promise)` resolves | ✓ |
+| `chrome_evaluate` statement-mode fallback | ✓ |
+| `chrome_evaluate` rethrows errors | ✓ |
+| `chrome_evaluate(undefined)` doesn't crash | ✓ |
+| `chrome_click` reports `pageMutated=false; occluded by` for covered buttons | ✓ |
+| `chrome_click` clean success has no warning suffix | ✓ |
+| `chrome_click` autoplay heuristic fires on `<button aria-label=Play>` w/ paused audio | ✓ |
+| `chrome_snapshot.containingText` filters correctly | ✓ |
+| `chrome_snapshot.roleFilter` filters correctly | ✓ |
+| `chrome_snapshot.nearUid` sorts by distance | ✓ |
+| `chrome_snapshot` reports `occluded` and `inert` and `pointerEvents` | ✓ |
+| `page.hover` dispatches | ✓ |
+| `page.drag` produces real pointerdown/up events on src/dst | ✓ |
+| `page.upload` populates input.files with correct name/size | ✓ |
+| `page.navigate` with `initScript` runs at document_start | ✓ |
+| `page.screenshot` `fullPage` produces correct tile count | ✓ |
+| `page.probe` returns MAIN-world arithmetic and webdriver flag | ✓ |
+| CDP 9222 probe reports correctly when not reachable | ✓ |
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json
index baef09c..5ceb76e 100644
--- a/extensions/chrome-profile-bridge/browser-extension/manifest.json
+++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json
@@ -1,9 +1,9 @@
{
"manifest_version": 3,
"name": "Pi Existing Chrome Profile Bridge",
- "version": "0.7.0",
+ "version": "0.8.0",
"description": "Lets Pi control tabs in this existing Chrome profile via a local bridge at 127.0.0.1.",
- "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms"],
+ "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation"],
"host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"],
"background": {
"service_worker": "service_worker.js"
diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
index 2f6c701..ed33470 100644
--- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js
+++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
@@ -4,8 +4,6 @@ const POLL_ERROR_BACKOFF_MS = 2000;
let polling = false;
function armKeepaliveAlarm() {
- // MV3 service workers can be suspended; alarms are the supported way to
- // wake the extension again. Chrome's minimum period is 0.5 minutes.
chrome.alarms.create("pi-bridge-keepalive", { periodInMinutes: 0.5 });
}
@@ -41,17 +39,12 @@ async function pollLoop() {
polling = true;
try {
while (true) {
- // Long-poll /next continuously. The bridge holds the request for up to ~25s when no
- // command is pending and returns {type:"none"}; we immediately re-issue the fetch so
- // commands sent while the SW is otherwise idle still get picked up promptly. The open
- // fetch also keeps the MV3 service worker alive between alarm wake-ups.
const response = await fetch(`${BRIDGE_URL}/next?name=${encodeURIComponent(CLIENT_NAME)}`, {
cache: "no-store",
});
if (!response.ok) throw new Error(`bridge /next HTTP ${response.status}`);
const payload = await response.json();
if (payload.type === "command") await handleCommand(payload.command);
- // Otherwise (type:"none"), loop and re-issue the long-poll.
}
} catch (error) {
await sleep(POLL_ERROR_BACKOFF_MS);
@@ -107,11 +100,22 @@ async function dispatch(action, params) {
return { closed: tab.id };
}
case "page.snapshot":
- return executeInTab(params, snapshotPage, [params.maxElements || 80]);
+ return executeInTab(params, snapshotPage, [
+ params.maxElements || 80,
+ params.containingText ?? null,
+ params.roleFilter ?? null,
+ params.nearUid ?? null,
+ ]);
case "page.evaluate":
- return executeInTab(params, evaluateExpression, [params.expression, params.awaitPromise !== false]);
+ return evaluateInTab(params);
case "page.click":
return executeActionInTab(params, clickPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]);
+ case "page.hover":
+ return executeActionInTab(params, hoverPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]);
+ case "page.drag":
+ return executeActionInTab(params, dragPage, [params.fromUid ?? null, params.fromSelector ?? null, params.fromX ?? null, params.fromY ?? null, params.toUid ?? null, params.toSelector ?? null, params.toX ?? null, params.toY ?? null, params.steps ?? 12]);
+ case "page.upload":
+ return executeActionInTab(params, uploadFiles, [params.selector ?? null, params.uid ?? null, params.files || []]);
case "page.type":
return executeActionInTab(params, typeIntoPage, [params.selector ?? null, params.uid ?? null, params.text || "", Boolean(params.pressEnter)]);
case "page.fill":
@@ -126,38 +130,27 @@ async function dispatch(action, params) {
return executeInTab(params, getNetworkRequest, [params.requestId]);
case "page.waitFor":
return executeInTab(params, waitForPage, [params.kind, params.value, params.timeoutMs || 10000, params.intervalMs || 250]);
+ case "page.probe":
+ // Lightweight capability probe for /chrome-doctor. Runs in MAIN world.
+ return executeInTab(params, probePage, []);
case "page.navigate": {
const tab = await getTabByParams(params);
if (params.foreground) await bringToFront(tab);
+ if (params.initScript) {
+ // Register a one-shot document_start content script. We register, navigate, wait, then unregister.
+ await registerInitScript(tab.id, params.initScript);
+ }
const wait = params.waitUntilLoad !== false ? waitForTabComplete(tab.id, params.timeoutMs || 15000) : Promise.resolve(undefined);
const updated = await chrome.tabs.update(tab.id, { url: params.url });
- await wait;
- return formatTab(await chrome.tabs.get(updated.id));
- }
- case "page.screenshot": {
- const tab = await getTabByParams(params);
- if (params.foreground) await bringToFront(tab);
- // captureVisibleTab requires the target tab to be the active tab in its window. Activate it
- // without focusing the window so other apps don't get pushed behind Chrome, and restore the
- // previous active tab afterwards to minimize disruption.
- let previousActiveId;
- if (!tab.active) {
- const activeBefore = await chrome.tabs.query({ active: true, windowId: tab.windowId });
- previousActiveId = activeBefore[0]?.id;
- await chrome.tabs.update(tab.id, { active: true });
- }
try {
- const dataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, {
- format: params.format || "png",
- quality: params.format === "jpeg" ? params.quality : undefined,
- });
- return { dataUrl, tab: formatTab(tab) };
+ await wait;
} finally {
- if (previousActiveId !== undefined && previousActiveId !== tab.id) {
- await chrome.tabs.update(previousActiveId, { active: true }).catch(() => undefined);
- }
+ if (params.initScript) await unregisterInitScript(tab.id).catch(() => undefined);
}
+ return formatTab(await chrome.tabs.get(updated.id));
}
+ case "page.screenshot":
+ return takeScreenshot(params);
default:
throw new Error(`Unknown action: ${action}`);
}
@@ -198,26 +191,40 @@ async function getTabByParams(params) {
return tab;
}
+// Helper sources that get concatenated into the injected MAIN-world script. Kept as separate
+// functions so callers below can reference them by `.toString()`. The helpers do not perform any
+// eval themselves — they're plain function declarations.
+const HELPER_FUNCS = [
+ getPiChromeState,
+ rememberElement,
+ elementBySelectorOrUid,
+ installPiChromeInstrumentation,
+ resolvePoint,
+ dispatchInputEvents,
+ setNativeValue,
+ normalizeKey,
+ isElementVisible,
+ occluderAt,
+ pageHash,
+ pointerEventSequence,
+];
+
async function executeInTab(params, func, args) {
const tab = await getTabByParams(params);
if (params.foreground) await bringToFront(tab);
- const helperSource = [
- getPiChromeState,
- rememberElement,
- elementBySelectorOrUid,
- installPiChromeInstrumentation,
- resolvePoint,
- dispatchInputEvents,
- setNativeValue,
- normalizeKey,
- ].map((helper) => helper.toString()).join("\n");
+ const helperSource = HELPER_FUNCS.map((helper) => helper.toString()).join("\n");
const results = await chrome.scripting.executeScript({
target: { tabId: tab.id },
world: "MAIN",
func: async (helperSource, source, invocationArgs) => {
try {
- (0, eval)(helperSource);
- const injected = (0, eval)(`(${source})`);
+ // Helpers are plain function declarations; injecting them via Function constructor avoids
+ // running through `eval` (which is restricted under strict CSP) and keeps them isolated.
+ new Function(helperSource).call(globalThis);
+ // The action itself is reconstructed from its source text. We use `new Function` rather
+ // than `eval` because the latter is blocked by `script-src 'self'` (no `'unsafe-eval'`)
+ // CSPs that are common on production sites.
+ const injected = new Function(helperSource + "\nreturn (" + source + ");").call(globalThis);
return { ok: true, value: await injected(...invocationArgs) };
} catch (error) {
return { ok: false, error: error?.stack || error?.message || String(error) };
@@ -237,15 +244,108 @@ async function executeInTab(params, func, args) {
return envelope?.value;
}
+// Dedicated executor for page.evaluate. Doesn't go through the helper-source injection chain;
+// that chain was the root cause of `chrome_evaluate` silently returning null on pages with strict
+// CSP. We build a single Function in MAIN world and invoke it directly.
+async function evaluateInTab(params) {
+ const tab = await getTabByParams(params);
+ if (params.foreground) await bringToFront(tab);
+ const expression = String(params.expression ?? "");
+ const awaitPromise = params.awaitPromise !== false;
+ const results = await chrome.scripting.executeScript({
+ target: { tabId: tab.id },
+ world: "MAIN",
+ func: async (expression, awaitPromise) => {
+ const stringify = (v) => {
+ if (v === undefined) return { kind: "undefined" };
+ if (typeof v === "function") return { kind: "function", source: v.toString().slice(0, 500) };
+ if (typeof v === "symbol") return { kind: "symbol", description: v.description };
+ if (typeof v === "bigint") return { kind: "bigint", value: v.toString() };
+ if (v instanceof Error) return { kind: "error", name: v.name, message: v.message, stack: v.stack };
+ return v;
+ };
+ // Compile via the Function constructor. We try expression form first so callers can pass
+ // `1+1` or `document.title` without a `return`; if that's a SyntaxError we retry with the
+ // statement form so callers can use multi-statement bodies (loops, var decls, etc).
+ const compile = (src) => {
+ try {
+ return { fn: new Function(`return (async () => (${src}))();`), mode: "expression" };
+ } catch (e1) {
+ if (e1 && e1.name === "SyntaxError") {
+ try {
+ return { fn: new Function(`return (async () => { ${src} })();`), mode: "statement" };
+ } catch (e2) {
+ throw e2;
+ }
+ }
+ throw e1;
+ }
+ };
+ try {
+ const { fn } = compile(expression);
+ const value = await fn.call(globalThis);
+ const resolved = awaitPromise && value && typeof value.then === "function" ? await value : value;
+ return { ok: true, value: stringify(resolved) };
+ } catch (error) {
+ return { ok: false, error: error?.stack || error?.message || String(error) };
+ }
+ },
+ args: [expression, awaitPromise],
+ });
+ const first = results?.[0];
+ if (first?.error) {
+ const message = typeof first.error === "string" ? first.error : (first.error.message || JSON.stringify(first.error));
+ throw new Error(`chrome_evaluate failed: ${message}`);
+ }
+ const envelope = first?.result;
+ if (!envelope) throw new Error("chrome_evaluate returned no envelope from MAIN world");
+ if (envelope.ok === false) throw new Error(envelope.error || "chrome_evaluate failed");
+ const v = envelope.value;
+ // Unwrap special markers from MAIN world
+ if (v && typeof v === "object" && !Array.isArray(v)) {
+ if (v.kind === "undefined") return undefined;
+ if (v.kind === "function") return `[Function: ${v.source}]`;
+ if (v.kind === "symbol") return `[Symbol: ${v.description}]`;
+ if (v.kind === "bigint") return v.value;
+ if (v.kind === "error") throw new Error(`${v.name}: ${v.message}\n${v.stack || ""}`);
+ }
+ return v;
+}
+
async function executeActionInTab(params, func, args) {
const result = await executeInTab(params, func, args);
if (params.includeSnapshot) {
- const snapshot = await executeInTab({ ...params, foreground: false }, snapshotPage, [params.maxElements || 80]);
+ const snapshot = await executeInTab({ ...params, foreground: false }, snapshotPage, [params.maxElements || 80, null, null, null]);
return { result, snapshot };
}
return result;
}
+// One-shot init script registry, scoped per tab. The script source is injected at
+// document_start of the next committed navigation in that tab, in MAIN world, then cleared.
+const initScriptIds = new Map();
+async function registerInitScript(tabId, source) {
+ initScriptIds.set(tabId, source);
+}
+async function unregisterInitScript(tabId) {
+ initScriptIds.delete(tabId);
+}
+
+if (chrome.webNavigation && chrome.webNavigation.onCommitted) {
+ chrome.webNavigation.onCommitted.addListener((details) => {
+ if (details.frameId !== 0) return;
+ const source = initScriptIds.get(details.tabId);
+ if (!source) return;
+ chrome.scripting.executeScript({
+ target: { tabId: details.tabId, frameIds: [0] },
+ world: "MAIN",
+ injectImmediately: true,
+ func: (code) => { try { new Function(code).call(globalThis); } catch (e) { console.error("[pi-chrome init script]", e); } },
+ args: [source],
+ }).catch(() => undefined);
+ });
+}
+
async function bringToFront(tab) {
await chrome.windows.update(tab.windowId, { focused: true });
await chrome.tabs.update(tab.id, { active: true });
@@ -268,6 +368,56 @@ function waitForTabComplete(tabId, timeoutMs) {
});
}
+async function takeScreenshot(params) {
+ const tab = await getTabByParams(params);
+ if (params.foreground) await bringToFront(tab);
+ let previousActiveId;
+ if (!tab.active) {
+ const activeBefore = await chrome.tabs.query({ active: true, windowId: tab.windowId });
+ previousActiveId = activeBefore[0]?.id;
+ await chrome.tabs.update(tab.id, { active: true });
+ }
+ try {
+ if (params.fullPage) {
+ // Tile-stitched full page capture: scroll, capture, paste, repeat.
+ const tiles = await executeInTab({ ...params, foreground: false }, captureFullPageTiles, []);
+ // captureFullPageTiles only computes scroll positions / metrics; we capture per scroll here
+ // (chrome.tabs.captureVisibleTab can't be called from MAIN world).
+ const captured = [];
+ for (const tile of tiles.tiles) {
+ await executeInTab({ ...params, foreground: false }, scrollToY, [tile.scrollY]);
+ // Small settle delay; many sites have on-scroll animations / lazy-load.
+ await sleep(120);
+ const dataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, {
+ format: params.format || "png",
+ quality: params.format === "jpeg" ? params.quality : undefined,
+ });
+ captured.push({ y: tile.y, dataUrl });
+ }
+ await executeInTab({ ...params, foreground: false }, scrollToY, [tiles.originalScrollY]);
+ return {
+ fullPage: true,
+ tab: formatTab(tab),
+ dimensions: { width: tiles.width, height: tiles.height, viewportHeight: tiles.viewportHeight, dpr: tiles.dpr },
+ tiles: captured,
+ };
+ }
+ const dataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, {
+ format: params.format || "png",
+ quality: params.format === "jpeg" ? params.quality : undefined,
+ });
+ return { dataUrl, tab: formatTab(tab) };
+ } finally {
+ if (previousActiveId !== undefined && previousActiveId !== tab.id) {
+ await chrome.tabs.update(previousActiveId, { active: true }).catch(() => undefined);
+ }
+ }
+}
+
+// ---------------------------------------------------------------------------
+// MAIN-world helpers (function declarations injected into the page).
+// ---------------------------------------------------------------------------
+
function getPiChromeState() {
const state = window.__PI_CHROME_STATE__ || {
nextElementUid: 1,
@@ -302,6 +452,77 @@ function elementBySelectorOrUid(selector, uid) {
return null;
}
+function isElementVisible(element) {
+ if (!element || !element.getBoundingClientRect) return false;
+ const style = getComputedStyle(element);
+ if (style.visibility === "hidden" || style.display === "none") return false;
+ const rect = element.getBoundingClientRect();
+ if (rect.width === 0 || rect.height === 0) return false;
+ if (rect.bottom < 0 || rect.right < 0) return false;
+ if (rect.top > innerHeight || rect.left > innerWidth) return false;
+ return true;
+}
+
+function occluderAt(x, y, expected) {
+ const top = document.elementFromPoint(x, y);
+ if (!top || top === expected) return null;
+ if (expected && expected.contains(top)) return null;
+ if (top.contains(expected)) return null;
+ return {
+ tag: top.tagName.toLowerCase(),
+ id: top.id || undefined,
+ className: typeof top.className === "string" ? top.className : undefined,
+ };
+}
+
+function pageHash() {
+ // Cheap rolling hash used for `pageMutated`. Combines first 4kb of body innerText with the
+ // current values of inputs/textareas (which are not part of innerText) and the count of
+ // descendants of <body>. This catches: text changes, input value edits, and DOM structure
+ // changes — the three things a click/type/fill might cause.
+ const body = document.body;
+ const text = (body ? body.innerText : "").slice(0, 4000);
+ let h = 0;
+ for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) | 0;
+ if (body) {
+ const inputs = body.querySelectorAll("input,textarea,select");
+ let valueBlob = "";
+ for (let i = 0; i < inputs.length && valueBlob.length < 4000; i++) {
+ const v = inputs[i].value;
+ if (typeof v === "string") valueBlob += v + "\x00";
+ }
+ for (let i = 0; i < valueBlob.length; i++) h = (h * 31 + valueBlob.charCodeAt(i)) | 0;
+ h = (h * 31 + body.getElementsByTagName("*").length) | 0;
+ }
+ return h;
+}
+
+function pointerEventSequence(element, x, y, sequence) {
+ let defaultPrevented = false;
+ for (const type of sequence) {
+ const isPointer = type.startsWith("pointer");
+ const Ctor = isPointer ? PointerEvent : MouseEvent;
+ const init = {
+ bubbles: true,
+ cancelable: true,
+ view: window,
+ clientX: x,
+ clientY: y,
+ button: 0,
+ buttons: type === "pointermove" || type === "mousemove" ? 0 : 1,
+ };
+ if (isPointer) {
+ init.pointerType = "mouse";
+ init.pointerId = 1;
+ init.isPrimary = true;
+ }
+ const ev = new Ctor(type, init);
+ element.dispatchEvent(ev);
+ if (ev.defaultPrevented) defaultPrevented = true;
+ }
+ return defaultPrevented;
+}
+
function installPiChromeInstrumentation() {
const state = getPiChromeState();
if (state.instrumentationInstalled) return;
@@ -407,7 +628,7 @@ function installPiChromeInstrumentation() {
}
}
-function snapshotPage(maxElements) {
+function snapshotPage(maxElements, containingText, roleFilter, nearUid) {
installPiChromeInstrumentation();
const unique = (selector) => {
try { return document.querySelectorAll(selector).length === 1; } catch { return false; }
@@ -434,11 +655,7 @@ function snapshotPage(maxElements) {
}
return parts.join(" > ");
};
- const visible = (element) => {
- const style = getComputedStyle(element);
- const rect = element.getBoundingClientRect();
- return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0;
- };
+ const visible = (element) => isElementVisible(element);
const labelFor = (element) => (
element.getAttribute("aria-label") ||
element.getAttribute("title") ||
@@ -448,9 +665,41 @@ function snapshotPage(maxElements) {
element.textContent ||
""
).trim().replace(/\s+/g, " ").slice(0, 160);
- const candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'));
+ let candidates = Array.from(document.querySelectorAll('a, button, input, textarea, select, summary, [role="button"], [role="link"], [role="menuitem"], [role="tab"], [role="checkbox"], [contenteditable="true"], [tabindex]:not([tabindex="-1"])'));
+ if (containingText) {
+ const needle = String(containingText).toLowerCase();
+ candidates = candidates.filter((element) => labelFor(element).toLowerCase().includes(needle));
+ }
+ if (roleFilter) {
+ const wanted = String(roleFilter).toLowerCase();
+ candidates = candidates.filter((element) => {
+ const role = (element.getAttribute("role") || element.tagName).toLowerCase();
+ return role === wanted;
+ });
+ }
+ let near;
+ if (nearUid) {
+ const state = getPiChromeState();
+ near = state.elements[nearUid];
+ }
+ if (near) {
+ const nearRect = near.getBoundingClientRect();
+ const cx = nearRect.left + nearRect.width / 2;
+ const cy = nearRect.top + nearRect.height / 2;
+ candidates.sort((a, b) => {
+ const ra = a.getBoundingClientRect();
+ const rb = b.getBoundingClientRect();
+ const da = Math.hypot(ra.left + ra.width / 2 - cx, ra.top + ra.height / 2 - cy);
+ const db = Math.hypot(rb.left + rb.width / 2 - cx, rb.top + rb.height / 2 - cy);
+ return da - db;
+ });
+ }
const elements = candidates.filter(visible).slice(0, maxElements).map((element, index) => {
const rect = element.getBoundingClientRect();
+ const style = getComputedStyle(element);
+ const cx = rect.left + rect.width / 2;
+ const cy = rect.top + rect.height / 2;
+ const occluded = occluderAt(cx, cy, element);
return {
index,
uid: rememberElement(element),
@@ -461,6 +710,9 @@ function snapshotPage(maxElements) {
type: element.getAttribute("type") || undefined,
role: element.getAttribute("role") || undefined,
disabled: Boolean(element.disabled || element.getAttribute("aria-disabled") === "true"),
+ inert: Boolean(element.closest?.("[inert]")),
+ pointerEvents: style.pointerEvents,
+ occluded: occluded || undefined,
rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) },
};
});
@@ -470,14 +722,44 @@ function snapshotPage(maxElements) {
viewport: { width: innerWidth, height: innerHeight, scrollX, scrollY },
text: document.body ? document.body.innerText.replace(/\s+\n/g, "\n").trim().slice(0, 30000) : "",
elements,
+ filter: { containingText: containingText || undefined, roleFilter: roleFilter || undefined, nearUid: nearUid || undefined },
};
}
-async function evaluateExpression(expression, awaitPromise) {
- installPiChromeInstrumentation();
- const indirectEval = (0, eval);
- const value = indirectEval(expression);
- return awaitPromise && value && typeof value.then === "function" ? await value : value;
+function probePage() {
+ // Sanity probe used by /chrome-doctor. Returns evidence that MAIN-world execution works.
+ return {
+ arithmetic: 1 + 1,
+ location: location.href,
+ title: document.title,
+ documentReady: document.readyState,
+ userAgent: navigator.userAgent.slice(0, 200),
+ webdriver: !!navigator.webdriver,
+ };
+}
+
+function captureFullPageTiles() {
+ // Returns the *plan* for tile capture; the actual chrome.tabs.captureVisibleTab calls happen
+ // in the SW. We just report the scroll positions and metrics.
+ const html = document.documentElement;
+ const body = document.body;
+ const width = Math.max(html.scrollWidth, body ? body.scrollWidth : 0, innerWidth);
+ const height = Math.max(html.scrollHeight, body ? body.scrollHeight : 0, innerHeight);
+ const viewportHeight = innerHeight;
+ const dpr = window.devicePixelRatio || 1;
+ const originalScrollY = scrollY;
+ const tiles = [];
+ let y = 0;
+ while (y < height) {
+ tiles.push({ y, scrollY: y });
+ y += viewportHeight;
+ }
+ return { width, height, viewportHeight, dpr, originalScrollY, tiles };
+}
+
+function scrollToY(y) {
+ window.scrollTo({ top: y, left: 0, behavior: "instant" });
+ return { scrollY };
}
function resolvePoint(selector, uid, x, y) {
@@ -492,12 +774,88 @@ function resolvePoint(selector, uid, x, y) {
}
function clickPage(selector, uid, x, y) {
+ installPiChromeInstrumentation();
+ const before = pageHash();
const point = resolvePoint(selector, uid, x, y);
if (!point.element) throw new Error("No element at click point");
- for (const type of ["pointerdown", "mousedown", "pointerup", "mouseup", "click"]) {
- point.element.dispatchEvent(new MouseEvent(type, { bubbles: true, cancelable: true, view: window, clientX: point.x, clientY: point.y, button: 0 }));
+ const visible = isElementVisible(point.element);
+ const occluded = occluderAt(point.x, point.y, point.element);
+ const defaultPrevented = pointerEventSequence(point.element, point.x, point.y, [
+ "pointerdown", "mousedown", "pointerup", "mouseup", "click",
+ ]);
+ // Heuristic: if the clicked thing looks like a media play affordance and the page has paused
+ // audio/video, the synthetic click may not unlock autoplay. Surface a warning.
+ let autoplayHint;
+ const label = (point.element.getAttribute("aria-label") || point.element.textContent || "").toLowerCase();
+ if (/^(play|start|begin|next|continue|unmute)/.test(label.trim())) {
+ const idleMedia = Array.from(document.querySelectorAll("audio,video")).some((m) => m.paused);
+ if (idleMedia) autoplayHint = "This element looks like a media affordance and the page has paused media. Synthetic clicks do not satisfy user-activation gates; audio/video may not start.";
+ }
+ return {
+ x: point.x,
+ y: point.y,
+ selector,
+ uid,
+ tag: point.element.tagName,
+ isTrusted: false,
+ defaultPrevented,
+ elementVisible: visible,
+ occludedBy: occluded || undefined,
+ pageMutated: pageHash() !== before,
+ autoplayHint,
+ };
+}
+
+function hoverPage(selector, uid, x, y) {
+ installPiChromeInstrumentation();
+ const point = resolvePoint(selector, uid, x, y);
+ if (!point.element) throw new Error("No element to hover");
+ const defaultPrevented = pointerEventSequence(point.element, point.x, point.y, [
+ "pointerover", "mouseover", "pointerenter", "mouseenter", "pointermove", "mousemove",
+ ]);
+ return { x: point.x, y: point.y, selector, uid, tag: point.element.tagName, defaultPrevented, isTrusted: false };
+}
+
+function dragPage(fromUid, fromSelector, fromX, fromY, toUid, toSelector, toX, toY, steps) {
+ installPiChromeInstrumentation();
+ const before = pageHash();
+ const from = resolvePoint(fromSelector, fromUid, fromX, fromY);
+ const to = resolvePoint(toSelector, toUid, toX, toY);
+ if (!from.element) throw new Error("Drag source element not found");
+ if (!to.element) throw new Error("Drag target element not found");
+ pointerEventSequence(from.element, from.x, from.y, ["pointerover", "pointerdown", "mousedown"]);
+ for (let i = 1; i <= steps; i++) {
+ const t = i / steps;
+ const x = from.x + (to.x - from.x) * t;
+ const y = from.y + (to.y - from.y) * t;
+ const overEl = document.elementFromPoint(x, y) || to.element;
+ pointerEventSequence(overEl, x, y, ["pointermove", "mousemove"]);
+ }
+ pointerEventSequence(to.element, to.x, to.y, ["pointerover", "mouseover", "pointerup", "mouseup"]);
+ return {
+ from: { x: from.x, y: from.y },
+ to: { x: to.x, y: to.y },
+ steps,
+ pageMutated: pageHash() !== before,
+ note: "Synthetic pointer drag. HTML5 DataTransfer is not synthesized; native drag-and-drop targets may not respond.",
+ };
+}
+
+function uploadFiles(selector, uid, files) {
+ installPiChromeInstrumentation();
+ const element = elementBySelectorOrUid(selector, uid);
+ if (!element || element.tagName !== "INPUT" || element.type !== "file") {
+ throw new Error("Target must be <input type=file>");
}
- return { x: point.x, y: point.y, selector, uid, tag: point.element.tagName };
+ const dt = new DataTransfer();
+ for (const f of files) {
+ const bytes = Uint8Array.from(atob(f.base64 || ""), (c) => c.charCodeAt(0));
+ dt.items.add(new File([bytes], f.name, { type: f.type || "application/octet-stream" }));
+ }
+ element.files = dt.files;
+ element.dispatchEvent(new Event("input", { bubbles: true }));
+ element.dispatchEvent(new Event("change", { bubbles: true }));
+ return { uploaded: files.map((f) => ({ name: f.name, type: f.type, size: (f.base64 || "").length })) };
}
function dispatchInputEvents(element, data, inputType = "insertText") {
@@ -515,6 +873,7 @@ function setNativeValue(element, value) {
function typeIntoPage(selector, uid, text, pressEnter) {
installPiChromeInstrumentation();
+ const before = pageHash();
let element = elementBySelectorOrUid(selector, uid) || document.activeElement;
if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element");
element.focus();
@@ -530,11 +889,17 @@ function typeIntoPage(selector, uid, text, pressEnter) {
throw new Error("Focused element is not text-editable");
}
if (pressEnter) pressKeyInPage("Enter");
- return { selector, uid, length: text.length, pressEnter };
+ return {
+ selector, uid, length: text.length, pressEnter,
+ isTrusted: false,
+ valueMatches: "value" in element ? element.value.includes(text) : undefined,
+ pageMutated: pageHash() !== before,
+ };
}
function fillPage(selector, uid, text, submit) {
installPiChromeInstrumentation();
+ const before = pageHash();
let element = elementBySelectorOrUid(selector, uid) || document.activeElement;
if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element");
element.focus();
@@ -550,19 +915,32 @@ function fillPage(selector, uid, text, submit) {
throw new Error("Focused element is not text-editable");
}
if (submit) pressKeyInPage("Enter");
- return { selector, uid, length: String(text).length, submit };
+ return {
+ selector, uid, length: String(text).length, submit,
+ isTrusted: false,
+ valueMatches: "value" in element ? element.value === String(text) : undefined,
+ pageMutated: pageHash() !== before,
+ };
}
function pressKeyInPage(key) {
const normalized = normalizeKey(key);
const target = document.activeElement || document.body;
- target.dispatchEvent(new KeyboardEvent("keydown", { key: normalized, bubbles: true, cancelable: true }));
- target.dispatchEvent(new KeyboardEvent("keyup", { key: normalized, bubbles: true, cancelable: true }));
+ const before = (typeof pageHash === "function") ? pageHash() : 0;
+ const down = new KeyboardEvent("keydown", { key: normalized, bubbles: true, cancelable: true });
+ target.dispatchEvent(down);
+ const up = new KeyboardEvent("keyup", { key: normalized, bubbles: true, cancelable: true });
+ target.dispatchEvent(up);
if (normalized === "Enter") {
const form = target.closest?.("form");
if (form) form.requestSubmit?.();
}
- return { key: normalized };
+ return {
+ key: normalized,
+ isTrusted: false,
+ defaultPrevented: down.defaultPrevented || up.defaultPrevented,
+ pageMutated: (typeof pageHash === "function") ? pageHash() !== before : undefined,
+ };
}
function listConsoleMessages(clear) {
@@ -581,7 +959,7 @@ function listNetworkRequests(includePreservedRequests, clear) {
.filter((request) => includePreservedRequests || request.pageUrl === currentUrl)
.map(({ responseBody, ...summary }) => ({ ...summary, hasResponseBody: responseBody !== undefined }));
if (clear) state.network = [];
- return { requests, count: requests.length, note: "Captures fetch/XHR after instrumentation is installed (snapshot/evaluate/network/console tools install it). Browser-initiated document/static asset requests are not captured." };
+ return { requests, count: requests.length, note: "Captures fetch/XHR after instrumentation is installed. Browser-initiated document/static asset requests are not captured." };
}
function getNetworkRequest(requestId) {
@@ -596,7 +974,9 @@ async function waitForPage(kind, value, timeoutMs, intervalMs) {
while (Date.now() - started < timeoutMs) {
let ok = false;
if (kind === "selector") ok = Boolean(document.querySelector(value));
- else ok = Boolean((0, eval)(value));
+ else {
+ try { ok = Boolean(new Function("return (" + value + ");").call(globalThis)); } catch { ok = false; }
+ }
if (ok) return { elapsedMs: Date.now() - started };
await new Promise((resolve) => setTimeout(resolve, intervalMs));
}
diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts
index 537fd9b..ac176bc 100644
--- a/extensions/chrome-profile-bridge/index.ts
+++ b/extensions/chrome-profile-bridge/index.ts
@@ -46,7 +46,8 @@ type BridgeResult = {
error?: string;
};
-const PI_CHROME_VERSION = "0.7.0";
+const PI_CHROME_VERSION = "0.8.0";
+const PI_CHROME_GLOBAL_KEY = "__piChromeProfileBridgeLoaded__";
const DEFAULT_HOST = process.env.PI_CHROME_BRIDGE_HOST ?? "127.0.0.1";
const DEFAULT_PORT = Number(process.env.PI_CHROME_BRIDGE_PORT ?? "17318");
const DEFAULT_TIMEOUT_MS = 30_000;
@@ -83,7 +84,30 @@ function workspaceCwd(ctx: ExtensionContext): string {
function browserExtensionPath(): string {
return join(extensionRoot(), "browser-extension");
-}
+}
+
+function hostnameOf(url: string | undefined): string {
+ if (!url) return "";
+ try { return new URL(url).hostname; } catch { return ""; }
+}
+
+// Description of a click/type/fill result's significant fields so the agent doesn't have to
+// guess whether the action actually changed the page.
+function summarizeActionResult(result: unknown): string | undefined {
+ if (!result || typeof result !== "object") return undefined;
+ const r = result as Record<string, unknown>;
+ const parts: string[] = [];
+ if (r.pageMutated === false) parts.push("pageMutated=false");
+ if (r.defaultPrevented === true) parts.push("defaultPrevented=true");
+ if (r.elementVisible === false) parts.push("element NOT visible");
+ if (r.occludedBy) {
+ const o = r.occludedBy as { tag?: string; id?: string };
+ parts.push(`occluded by <${o.tag ?? "?"}${o.id ? "#" + o.id : ""}>`);
+ }
+ if (r.valueMatches === false) parts.push("input value did not stick");
+ if (r.autoplayHint) parts.push("autoplay-gated affordance — synthetic click may not start media");
+ return parts.length ? parts.join("; ") : undefined;
+}
function readRequestBody(request: IncomingMessage): Promise<string> {
return new Promise((resolveBody, rejectBody) => {
@@ -338,6 +362,18 @@ const imageFormatValues = ["png", "jpeg"] as const;
const waitForValues = ["selector", "expression"] as const;
export default function (pi: ExtensionAPI): void {
+ const globalState = globalThis as typeof globalThis & {
+ [PI_CHROME_GLOBAL_KEY]?: { version: string; root: string };
+ };
+ const alreadyLoaded = globalState[PI_CHROME_GLOBAL_KEY];
+ if (alreadyLoaded) {
+ console.warn(
+ `pi-chrome already loaded from ${alreadyLoaded.root} (v${alreadyLoaded.version}); skipping duplicate from ${extensionRoot()}.`,
+ );
+ return;
+ }
+ globalState[PI_CHROME_GLOBAL_KEY] = { version: PI_CHROME_VERSION, root: extensionRoot() };
+
const bridge = new ChromeProfileBridge(DEFAULT_HOST, DEFAULT_PORT);
let backgroundDefault = false;
@@ -374,21 +410,34 @@ export default function (pi: ExtensionAPI): void {
pi.on("before_agent_start", (event) => {
const primer = `
<chrome-profile-bridge>
-Chrome control is available through the chrome_* tools via a companion Chrome extension installed in the user's normal Chrome profile.
-This is not CDP: it can use the user's existing Chrome windows and authenticated sessions after the user loads the companion browser extension.
-If chrome_* tools time out, ask the user to run /chrome-onboard, then load the bundled browser-extension folder in chrome://extensions. Prefer chrome_snapshot before clicking/typing; use stable element uids from snapshots with chrome_click/chrome_type when available. For form work, use includeSnapshot=true on actions to verify in one round trip. Avoid destructive actions unless explicitly requested. By default chrome_* tools focus Chrome and activate the target tab so the user can watch the agent work. The user can switch to silent/background mode for the whole session via /chrome-background; you can also pass background=true on a single tool call when the user explicitly wants the action to be silent (for example, scraping while they keep working in another app).
+Chrome control is available through the chrome_* tools via a companion Chrome extension installed in the user's normal Chrome profile. Tools target the existing signed-in profile, no CDP, no throwaway profile.
+
+Capability model (important):
+- All input is **synthetic DOM events** (\`isTrusted=false\`). Synthetic events drive React/Vue/Angular state fine, but they do NOT satisfy Chrome's user-activation gates: audio/video autoplay, clipboard write, file pickers, fullscreen, and Web Push prompts will NOT open from a chrome_click.
+- \`chrome_evaluate\` runs in MAIN world via the Function constructor. It works on pages with strict CSP (\`script-src 'self'\` without \`'unsafe-eval'\`), and surfaces thrown exceptions.
+- Tool results include \`pageMutated\`, \`defaultPrevented\`, \`elementVisible\`, \`occludedBy\`, and (for type/fill) \`valueMatches\`. If \`pageMutated\` is false after a click that should have changed something, the click likely didn't take effect — do NOT just retry; check the action result and snapshot for the cause.
+
+Usage rules:
+1. \`chrome_snapshot\` before clicking/typing; pass \`uid\` over \`selector\`.
+2. \`includeSnapshot=true\` on click/type/fill to verify in one round trip.
+3. If \`chrome_evaluate\` returns null when you expected a value, the expression evaluated to null/undefined in the page; surface the value via \`JSON.stringify\` to confirm.
+4. \`chrome_navigate\` supports an optional \`initScript\` that runs at document_start in MAIN world for the next navigation (good for seeding localStorage or stubbing Date.now).
+5. By default chrome_* tools focus Chrome so the user can watch; pass \`background=true\` or run /chrome-background to silence the whole session.
+6. If you hit an autoplay/clipboard/file-picker gate, tell the user; this bridge cannot satisfy it.
+7. Run /chrome-doctor when in doubt about connectivity or capabilities.
</chrome-profile-bridge>`;
return { systemPrompt: event.systemPrompt + primer };
});
pi.registerCommand("chrome-doctor", {
description:
- "Check Chrome bridge connectivity and diagnose setup. Reports the local bridge, companion Chrome extension status (ID + version), and a one-line fix for common failures (extension not loaded, stale service worker, version drift).",
+ "Check Chrome bridge connectivity and capability tier. Probes the local bridge, the companion Chrome extension, MAIN-world evaluation, and CDP availability, and prints one-line fixes for common failures.",
handler: async (_args, ctx) => {
ctx.ui.notify("Performing Chrome bridge health check", "info");
const lines: string[] = [`pi-chrome v${PI_CHROME_VERSION}`];
const status = bridge.status();
lines.push(`• Local bridge: mode=${status.mode}, url=${status.url}`);
+ let extensionAlive = false;
try {
const started = Date.now();
const version = (await bridge.send("tab.version", {}, 35_000)) as {
@@ -397,10 +446,8 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
bridgeUrl?: string;
};
const latencyMs = Date.now() - started;
- if (version.extensionId)
- lines.push(`✓ Companion Chrome extension responding (ID: ${version.extensionId}, ext v${version.extensionVersion ?? "unknown"}, latency ${latencyMs}ms)`);
- else lines.push(`✓ Companion Chrome extension responding (no extension ID reported, latency ${latencyMs}ms)`);
- if (version.bridgeUrl) lines.push(`• Extension polling: ${version.bridgeUrl}`);
+ extensionAlive = true;
+ lines.push(`✓ Companion Chrome extension responding (ID: ${version.extensionId ?? "?"}, ext v${version.extensionVersion ?? "?"}, latency ${latencyMs}ms)`);
if (version.extensionVersion && version.extensionVersion !== PI_CHROME_VERSION) {
lines.push(
`⚠ Extension version (${version.extensionVersion}) differs from pi-chrome (${PI_CHROME_VERSION}). Reload "Pi Existing Chrome Profile Bridge" in chrome://extensions to pick up the latest service worker.`,
@@ -415,6 +462,43 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
lines.push(" Fix: run /chrome-onboard, then load the bundled browser-extension folder in chrome://extensions and keep that Chrome window open.");
}
}
+
+ if (extensionAlive) {
+ // MAIN-world evaluate probe.
+ try {
+ const value = await bridge.send("page.evaluate", { expression: "1+1", awaitPromise: true, foreground: false }, 10_000);
+ if (value === 2) lines.push(`✓ chrome_evaluate("1+1") = 2`);
+ else lines.push(`⚠ chrome_evaluate("1+1") returned ${JSON.stringify(value)} (expected 2). The current tab may have a restrictive CSP or be a chrome:// URL.`);
+ } catch (error) {
+ lines.push(`✗ chrome_evaluate failed: ${(error as Error).message}`);
+ }
+
+ // Capability probe via MAIN-world helper.
+ try {
+ const probe = (await bridge.send("page.probe", { foreground: false }, 10_000)) as Record<string, unknown>;
+ if (probe && probe.arithmetic === 2) lines.push(`✓ MAIN-world helper injection works (location=${hostnameOf(String(probe.location))})`);
+ if (probe && probe.webdriver) lines.push(`⚠ navigator.webdriver=true on current tab — site fingerprinting may flag automation.`);
+ } catch (error) {
+ lines.push(`⚠ page.probe failed: ${(error as Error).message}`);
+ }
+ }
+
+ // CDP availability hint.
+ try {
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(), 250);
+ const response = await fetch("http://127.0.0.1:9222/json/version", { signal: controller.signal }).catch(() => undefined);
+ clearTimeout(timer);
+ if (response && response.ok) {
+ const info = (await response.json().catch(() => ({}))) as { Browser?: string };
+ lines.push(`✓ CDP endpoint reachable at 127.0.0.1:9222 (${info.Browser ?? "unknown"}). Trusted input via CDP is not yet wired into pi-chrome — reserved for a future release.`);
+ } else {
+ lines.push(`• CDP not available (no listener on 127.0.0.1:9222). Synthetic input only; autoplay/clipboard/file-picker gates cannot be satisfied. Future pi-chrome versions will use CDP for trusted input when this port is enabled.`);
+ }
+ } catch {
+ lines.push(`• CDP probe inconclusive.`);
+ }
+
ctx.ui.notify(lines.join("\n"), "info");
},
});
@@ -540,6 +624,9 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
urlIncludes: Type.Optional(Type.String()),
titleIncludes: Type.Optional(Type.String()),
maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS })),
+ containingText: Type.Optional(Type.String({ description: "Only return elements whose label/text contains this string (case-insensitive). Useful when the page has many controls." })),
+ roleFilter: Type.Optional(Type.String({ description: "Only return elements matching this ARIA role or tag name (case-insensitive). e.g. 'button', 'link', 'textbox'." })),
+ nearUid: Type.Optional(Type.String({ description: "Sort elements by proximity to this snapshot uid. Useful for finding controls near a known anchor." })),
background: Type.Optional(
Type.Boolean({ description: "If true, run silently in the background without focusing Chrome. Default false (Chrome focuses + tab activates so the user can watch)." }),
),
@@ -569,6 +656,7 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
titleIncludes: Type.Optional(Type.String()),
waitUntilLoad: Type.Optional(Type.Boolean({ default: true })),
timeoutMs: Type.Optional(Type.Number({ default: 15_000 })),
+ initScript: Type.Optional(Type.String({ description: "Optional JavaScript source to run in MAIN world at document_start of the next navigation. Useful for seeding localStorage, stubbing Date.now(), or defining navigator.webdriver=undefined. Requires the companion extension's webNavigation permission." })),
background: Type.Optional(
Type.Boolean({ description: "If true, navigate silently without focusing Chrome. Default false." }),
),
@@ -576,8 +664,8 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
port: Type.Optional(Type.Number()),
}),
async execute(_id, params): Promise<ToolTextResult> {
- const result = await bridge.send("page.navigate", withBackground(params), params.timeoutMs ?? 15_000);
- return { content: [{ type: "text", text: `Navigated to ${params.url}` }], details: { result: result as Json } };
+ const result = await bridge.send("page.navigate", withBackground(params), (params.timeoutMs ?? 15_000) + 2_000);
+ return { content: [{ type: "text", text: `Navigated to ${params.url}${params.initScript ? " (with initScript)" : ""}` }], details: { result: result as Json } };
},
});
@@ -590,7 +678,6 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
parameters: Type.Object({
expression: Type.String(),
awaitPromise: Type.Optional(Type.Boolean({ default: true })),
- returnByValue: Type.Optional(Type.Boolean({ default: true })),
targetId: Type.Optional(Type.String()),
urlIncludes: Type.Optional(Type.String()),
titleIncludes: Type.Optional(Type.String()),
@@ -602,7 +689,12 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
}),
async execute(_id, params): Promise<ToolTextResult> {
const value = await bridge.send("page.evaluate", withBackground(params), DEFAULT_TIMEOUT_MS);
- return { content: [{ type: "text", text: truncateText(typeof value === "string" ? value : safeJson(value)) }], details: { value: value as Json } };
+ const text = value === undefined
+ ? "undefined"
+ : typeof value === "string"
+ ? value
+ : safeJson(value) ?? "undefined";
+ return { content: [{ type: "text", text: truncateText(text) }], details: { value: value as Json } };
},
});
@@ -629,8 +721,12 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
port: Type.Optional(Type.Number()),
}),
async execute(_id, params): Promise<ToolTextResult> {
- const result = await bridge.send("page.click", withBackground(params), DEFAULT_TIMEOUT_MS);
- return { content: [{ type: "text", text: `Clicked ${params.uid ?? params.selector ?? `${params.x},${params.y}`}` }], details: { result: result as Json } };
+ const raw = await bridge.send("page.click", withBackground(params), DEFAULT_TIMEOUT_MS);
+ const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json;
+ const summary = summarizeActionResult(result);
+ const target = params.uid ?? params.selector ?? `${params.x},${params.y}`;
+ const text = summary ? `Clicked ${target} — ${summary}` : `Clicked ${target}`;
+ return { content: [{ type: "text", text }], details: { result: raw as Json } };
},
});
@@ -657,8 +753,12 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
port: Type.Optional(Type.Number()),
}),
async execute(_id, params): Promise<ToolTextResult> {
- const result = await bridge.send("page.type", withBackground(params), DEFAULT_TIMEOUT_MS);
- return { content: [{ type: "text", text: `Typed ${params.text.length} character(s)${params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""}.` }], details: { result: result as Json } };
+ const raw = await bridge.send("page.type", withBackground(params), DEFAULT_TIMEOUT_MS);
+ const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json;
+ const summary = summarizeActionResult(result);
+ const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : "";
+ const base = `Typed ${params.text.length} character(s)${into}.`;
+ return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } };
},
});
@@ -685,8 +785,12 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
port: Type.Optional(Type.Number()),
}),
async execute(_id, params): Promise<ToolTextResult> {
- const result = await bridge.send("page.fill", withBackground(params), DEFAULT_TIMEOUT_MS);
- return { content: [{ type: "text", text: `Filled ${params.text.length} character(s)${params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : ""}.` }], details: { result: result as Json } };
+ const raw = await bridge.send("page.fill", withBackground(params), DEFAULT_TIMEOUT_MS);
+ const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json;
+ const summary = summarizeActionResult(result);
+ const into = params.uid || params.selector ? ` into ${params.uid ?? params.selector}` : "";
+ const base = `Filled ${params.text.length} character(s)${into}.`;
+ return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } };
},
});
@@ -710,8 +814,11 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
port: Type.Optional(Type.Number()),
}),
async execute(_id, params): Promise<ToolTextResult> {
- const result = await bridge.send("page.key", withBackground(params), DEFAULT_TIMEOUT_MS);
- return { content: [{ type: "text", text: `Pressed ${params.key}.` }], details: { result: result as Json } };
+ const raw = await bridge.send("page.key", withBackground(params), DEFAULT_TIMEOUT_MS);
+ const result = (params.includeSnapshot ? (raw as { result: unknown }).result : raw) as Json;
+ const summary = summarizeActionResult(result);
+ const base = `Pressed ${params.key}.`;
+ return { content: [{ type: "text", text: summary ? `${base} (${summary})` : base }], details: { result: raw as Json } };
},
});
@@ -825,11 +932,113 @@ If chrome_* tools time out, ask the user to run /chrome-onboard, then load the b
const cwd = workspaceCwd(ctx);
const defaultPath = join(cwd, ".pi", "chrome-screenshots", `${new Date().toISOString().replace(/[:.]/g, "-")}.${format}`);
const outputPath = params.path ? resolve(cwd, params.path) : defaultPath;
- const result = (await bridge.send("page.screenshot", withBackground(params), DEFAULT_TIMEOUT_MS)) as { dataUrl: string; tab?: unknown };
- const base64 = result.dataUrl.replace(/^data:image\/(?:png|jpeg);base64,/, "");
+ const result = (await bridge.send("page.screenshot", withBackground(params), params.fullPage ? 120_000 : DEFAULT_TIMEOUT_MS)) as {
+ dataUrl?: string;
+ tab?: unknown;
+ fullPage?: boolean;
+ dimensions?: { width: number; height: number; viewportHeight: number; dpr: number };
+ tiles?: Array<{ y: number; dataUrl: string }>;
+ };
await mkdir(dirname(outputPath), { recursive: true });
+ if (result.fullPage && result.tiles && result.dimensions) {
+ // Stitch via PNG if format is png; otherwise we fall back to writing tile files and a
+ // manifest. We avoid pulling in an image library by writing each tile next to the main
+ // path with a -tileN suffix and a stitched.json manifest.
+ const { width, height, viewportHeight, dpr } = result.dimensions;
+ const manifest: Array<{ path: string; y: number }> = [];
+ for (let i = 0; i < result.tiles.length; i++) {
+ const tile = result.tiles[i];
+ const tilePath = outputPath.replace(/(\.[^.]+)$/, `-tile${i}$1`);
+ const base64 = tile.dataUrl.replace(/^data:image\/(?:png|jpeg);base64,/, "");
+ await writeFile(tilePath, Buffer.from(base64, "base64"));
+ manifest.push({ path: tilePath, y: tile.y });
+ }
+ await writeFile(outputPath + ".json", JSON.stringify({ width, height, viewportHeight, dpr, tiles: manifest }, null, 2));
+ return {
+ content: [{ type: "text", text: `Saved ${result.tiles.length} full-page tile(s) for ${width}×${height}px page. Manifest: ${outputPath}.json` }],
+ details: { manifest: outputPath + ".json", tiles: manifest, dimensions: result.dimensions, tab: result.tab } as unknown as Record<string, unknown>,
+ };
+ }
+ if (!result.dataUrl) throw new Error("Screenshot returned no dataUrl");
+ const base64 = result.dataUrl.replace(/^data:image\/(?:png|jpeg);base64,/, "");
await writeFile(outputPath, Buffer.from(base64, "base64"));
return { content: [{ type: "text", text: `Saved Chrome screenshot to ${outputPath}` }], details: { path: outputPath, format, tab: result.tab } };
},
});
+
+ pi.registerTool({
+ name: "chrome_hover",
+ label: "Chrome Hover",
+ description: "Hover over an element (synthetic pointerover/mouseover/pointermove) by uid, selector, or x/y. Triggers CSS :hover state and any JS hover handlers; isTrusted is false.",
+ promptSnippet: "Hover a Chrome element to trigger :hover / mouseover handlers.",
+ parameters: Type.Object({
+ uid: Type.Optional(Type.String()),
+ selector: Type.Optional(Type.String()),
+ x: Type.Optional(Type.Number()),
+ y: Type.Optional(Type.Number()),
+ targetId: Type.Optional(Type.String()),
+ urlIncludes: Type.Optional(Type.String()),
+ titleIncludes: Type.Optional(Type.String()),
+ background: Type.Optional(Type.Boolean()),
+ }),
+ async execute(_id, params): Promise<ToolTextResult> {
+ const result = await bridge.send("page.hover", withBackground(params), DEFAULT_TIMEOUT_MS);
+ return { content: [{ type: "text", text: `Hovered ${params.uid ?? params.selector ?? `${params.x},${params.y}`}` }], details: { result: result as Json } };
+ },
+ });
+
+ pi.registerTool({
+ name: "chrome_drag",
+ label: "Chrome Drag",
+ description: "Synthetic pointer drag from one uid/selector/point to another. Dispatches pointerdown → multi-step pointermove → pointerup. Note: HTML5 DataTransfer is NOT synthesized, so native HTML5 drag-and-drop targets may not respond.",
+ promptSnippet: "Drag a Chrome element from one point to another.",
+ parameters: Type.Object({
+ fromUid: Type.Optional(Type.String()),
+ fromSelector: Type.Optional(Type.String()),
+ fromX: Type.Optional(Type.Number()),
+ fromY: Type.Optional(Type.Number()),
+ toUid: Type.Optional(Type.String()),
+ toSelector: Type.Optional(Type.String()),
+ toX: Type.Optional(Type.Number()),
+ toY: Type.Optional(Type.Number()),
+ steps: Type.Optional(Type.Number({ default: 12 })),
+ targetId: Type.Optional(Type.String()),
+ urlIncludes: Type.Optional(Type.String()),
+ titleIncludes: Type.Optional(Type.String()),
+ background: Type.Optional(Type.Boolean()),
+ }),
+ async execute(_id, params): Promise<ToolTextResult> {
+ const result = await bridge.send("page.drag", withBackground(params), DEFAULT_TIMEOUT_MS);
+ return { content: [{ type: "text", text: `Dragged from ${params.fromUid ?? params.fromSelector} to ${params.toUid ?? params.toSelector}` }], details: { result: result as Json } };
+ },
+ });
+
+ pi.registerTool({
+ name: "chrome_upload_file",
+ label: "Chrome Upload File",
+ description: "Programmatically set the files of an <input type=file> element from local file paths. Uses DataTransfer to populate input.files and dispatches input+change events. Does NOT open the native file picker; works with React/Vue/Angular controlled inputs.",
+ promptSnippet: "Attach local files to a Chrome <input type=file> without opening the native file picker.",
+ parameters: Type.Object({
+ uid: Type.Optional(Type.String()),
+ selector: Type.Optional(Type.String()),
+ paths: Type.Array(Type.String(), { description: "Local absolute file paths to upload." }),
+ targetId: Type.Optional(Type.String()),
+ urlIncludes: Type.Optional(Type.String()),
+ titleIncludes: Type.Optional(Type.String()),
+ background: Type.Optional(Type.Boolean()),
+ }),
+ async execute(_id, params, _signal, _onUpdate, ctx): Promise<ToolTextResult> {
+ const { readFile } = await import("node:fs/promises");
+ const { basename } = await import("node:path");
+ const cwd = workspaceCwd(ctx);
+ const files: Array<{ name: string; type: string; base64: string }> = [];
+ for (const p of params.paths) {
+ const abs = resolve(cwd, p);
+ const buf = await readFile(abs);
+ files.push({ name: basename(abs), type: "application/octet-stream", base64: buf.toString("base64") });
+ }
+ const result = await bridge.send("page.upload", withBackground({ ...params, files }), DEFAULT_TIMEOUT_MS);
+ return { content: [{ type: "text", text: `Uploaded ${files.length} file(s) to ${params.uid ?? params.selector}` }], details: { result: result as Json } };
+ },
+ });
}
diff --git a/package.json b/package.json
index 1438b37..7cfcebc 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "pi-chrome",
- "version": "0.7.0",
+ "version": "0.8.0",
"description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).",
"keywords": [
"pi-package",