From 26ab0d36e45b1b65f667d4743eb30e83389ff57f Mon Sep 17 00:00:00 2001 From: Tianren Dong Date: Wed, 13 May 2026 02:35:01 -0400 Subject: pi-chrome 0.14.2: recover from foreign-extension input overlays When a different Chrome extension (password manager, autofill popup, etc.) anchors an overlay to the focused input, Chrome's CDP refuses our Input.dispatchMouseEvent with 'Cannot access a chrome-extension:// URL of different extension'. The CDP target list shows the foreign chrome-extension:// URL as type:'other' attached to the same tab. New behaviour in cdp() wrapper: 1. On that specific error for any Input.* call, snapshot CDP targets to identify the offending extension id and log it (foreign-ext- detected event added to ring buffer). 2. Send Esc via Input.dispatchKeyEvent (key dispatch routes by focus, not coordinates, so it bypasses the overlay) to dismiss the popup. 3. Retry the original Input.* call once. 4. If retry still fails, throw a clear error naming the foreign extension id and listing three concrete workarounds: disable that extension on the page, use Tab-to-focus instead of clicking, or run /chrome quiet off. --- .../browser-extension/manifest.json | 2 +- .../browser-extension/service_worker.js | 60 ++ extensions/chrome-profile-bridge/index.ts | 5 +- package.json | 11 +- test-suite/README.md | 184 ++-- test-suite/_lib.js | 22 +- test-suite/_style.css | 27 + test-suite/challenges/10-user-activation.html | 16 +- test-suite/challenges/12-fingerprint.html | 6 +- .../challenges/23-stack-trace-fingerprint.html | 8 +- test-suite/challenges/29-shadow-dom-controls.html | 44 + test-suite/challenges/30-iframe-targeting.html | 44 + test-suite/challenges/31-file-upload.html | 30 + .../challenges/32-keyboard-tab-navigation.html | 61 ++ .../challenges/33-network-console-capture.html | 33 + test-suite/fixtures/pi-chrome-upload.txt | 1 + test-suite/index.html | 211 +++-- test-suite/manifest.json | 952 +++++++++++++++++++++ test-suite/manifest.schema.json | 79 ++ test-suite/notes/bypass-ideas.md | 13 +- 20 files changed, 1654 insertions(+), 155 deletions(-) create mode 100644 test-suite/challenges/29-shadow-dom-controls.html create mode 100644 test-suite/challenges/30-iframe-targeting.html create mode 100644 test-suite/challenges/31-file-upload.html create mode 100644 test-suite/challenges/32-keyboard-tab-navigation.html create mode 100644 test-suite/challenges/33-network-console-capture.html create mode 100644 test-suite/fixtures/pi-chrome-upload.txt create mode 100644 test-suite/manifest.json create mode 100644 test-suite/manifest.schema.json diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json index aa7bfae..f0f8a6a 100644 --- a/extensions/chrome-profile-bridge/browser-extension/manifest.json +++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Pi Chrome Connector", - "version": "0.14.1", + "version": "0.14.2", "description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.", "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation", "debugger"], "host_permissions": ["", "http://127.0.0.1:17318/*"], diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js index d256f1c..36e5b20 100644 --- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js +++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js @@ -185,12 +185,72 @@ function cdpRaw(tabId, method, params) { // chrome.debugger.attach can stay cached in attachedTabs even after Chrome killed // the session (tab nav, devtools opened/closed, etc). Recover by detaching the // stale entry and re-attaching, then retry the command once. +// Find foreign chrome-extension targets currently anchored to the tab. Password managers, +// autofill helpers, and other input-attached extensions create type:"other" CDP targets +// whose URL is chrome-extension:///... When that target is in focus, CDP refuses +// our Input.dispatchMouseEvent calls with "Cannot access a chrome-extension:// URL of +// different extension" — surfacing a cryptic error to the user. +async function findForeignExtensionTargets() { + try { + const targets = await new Promise((resolve) => chrome.debugger.getTargets((t) => resolve(t || []))); + return targets.filter((t) => { + const url = String(t.url || ""); + if (!url.startsWith("chrome-extension://")) return false; + if (t.extensionId === chrome.runtime.id) return false; + return true; + }); + } catch { + return []; + } +} + +function extractForeignExtId(targets) { + for (const t of targets) { + if (t.extensionId && t.extensionId !== chrome.runtime.id) return t.extensionId; + const m = String(t.url || "").match(/chrome-extension:\/\/([a-p]+)\//); + if (m && m[1] !== chrome.runtime.id) return m[1]; + } + return null; +} + +async function dismissOverlayViaEscape(tabId) { + // Esc routes through key dispatcher (target-by-focus), not by mouse coordinates, so it + // works even when a foreign chrome-extension popup is intercepting pointer events. + try { + await cdpRaw(tabId, "Input.dispatchKeyEvent", { type: "keyDown", key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 }); + await cdpRaw(tabId, "Input.dispatchKeyEvent", { type: "keyUp", key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 }); + await sleep(120); + } catch {} +} + async function cdp(tabId, method, params) { try { return await cdpRaw(tabId, method, params); } catch (error) { const msg = String(error?.message || error); const isStale = /Debugger is not attached|Detached while|Target closed|No tab with id/i.test(msg); + const isForeignExtBlock = /Cannot access a chrome-extension:\/\/ URL of different extension/i.test(msg); + if (isForeignExtBlock && /Input\./.test(method)) { + // Foreign chrome-extension popup (autofill, password manager) is hijacking input. + // Try once: dismiss via Esc, then retry. + const before = await findForeignExtensionTargets(); + recordAttachEvent({ kind: "foreign-ext-detected", tabId, method, foreignExtId: extractForeignExtId(before), targetCount: before.length }); + await dismissOverlayViaEscape(tabId); + try { + return await cdpRaw(tabId, method, params); + } catch (retryErr) { + const retryMsg = String(retryErr?.message || retryErr); + if (/Cannot access a chrome-extension:\/\/ URL of different extension/i.test(retryMsg)) { + const after = await findForeignExtensionTargets(); + const id = extractForeignExtId(after) || extractForeignExtId(before) || "unknown"; + throw new Error( + `Another Chrome extension (${id}) has an input overlay on this page (e.g. a password manager / autofill popup). \n` + + `pi-chrome tried to dismiss it with Escape but it reappeared. Disable that extension on this page, focus the field via Tab instead of clicking, or run /chrome quiet off so the agent uses synthetic input here.`, + ); + } + throw retryErr; + } + } if (!isStale) throw error; attachedTabs.delete(tabId); await chrome.debugger.attach({ tabId }, CDP_VERSION).catch(() => undefined); diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts index 2fa51b0..2923040 100644 --- a/extensions/chrome-profile-bridge/index.ts +++ b/extensions/chrome-profile-bridge/index.ts @@ -1,6 +1,5 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { getSettingsListTheme } from "@earendil-works/pi-coding-agent"; -import { StringEnum } from "@earendil-works/pi-ai"; import { Container, type SettingItem, SettingsList, Text } from "@earendil-works/pi-tui"; import { Type } from "typebox"; import { existsSync, readFileSync, statSync } from "node:fs"; @@ -381,6 +380,10 @@ const tabActionValues = ["list", "new", "activate", "close", "version"] as const const imageFormatValues = ["png", "jpeg"] as const; const waitForValues = ["selector", "expression"] as const; +function StringEnum(values: T) { + return Type.Union(values.map((value) => Type.Literal(value)) as [ReturnType, ...ReturnType[]]); +} + export default function (pi: ExtensionAPI): void { const globalState = globalThis as typeof globalThis & { [PI_CHROME_GLOBAL_KEY]?: { version: string; root: string }; diff --git a/package.json b/package.json index e8354ae..dcaaa0e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-chrome", - "version": "0.14.1", + "version": "0.14.2", "description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).", "keywords": [ "pi-package", @@ -24,8 +24,15 @@ ] }, "peerDependencies": { - "@earendil-works/pi-ai": "*", "@earendil-works/pi-coding-agent": "*", "typebox": "*" + }, + "peerDependenciesMeta": { + "@earendil-works/pi-coding-agent": { + "optional": true + }, + "typebox": { + "optional": true + } } } diff --git a/test-suite/README.md b/test-suite/README.md index 241976c..bb8becb 100644 --- a/test-suite/README.md +++ b/test-suite/README.md @@ -1,8 +1,13 @@ -# pi-chrome anti-automation test suite +# pi-chrome browser-control benchmark -Static pages that each block a different automated-interaction signal. Drive them -with the `chrome_*` tools to discover where the companion-extension's synthetic -events still look like a bot, and to verify any humanization fixes. +Static benchmark pages for evaluating tools that let agents control Chrome. The +suite covers two related questions: + +1. **Can the agent complete normal browser work?** Forms, scroll containers, + contenteditable, files, frames, Shadow DOM, network/console inspection. +2. **Does the interaction look like a human/browser action when sites care?** + `isTrusted`, user activation, pointer paths, key cadence, native controls, + drag/drop, touch, paste, and scroll momentum. ## Run @@ -14,61 +19,120 @@ python3 -m http.server 8765 Each challenge page exposes: -- `window.__challenge` — short id -- `window.__verdict` — `"PENDING" | "PASS" | "FAIL"` -- `window.__reason` — array of strings, why it failed (or how it passed) +- `window.__challenge` — id +- `window.__verdict` — `"PENDING" | "PASS" | "FAIL" | "SKIP" | "WARN"` +- `window.__reason` — array of reasons - `window.__events` — raw event log for forensics -`PASS` means the page believes a real human did the thing. -`FAIL` means the page caught the automation. - -## Aggregate runner - -`index.html` lists every challenge, loads each one in an iframe, lets you (or -the agent) drive the interaction, and tallies verdicts. - -Recommended agent flow: - -1. `chrome_navigate` to `http://127.0.0.1:8765/`. -2. For each challenge link, open it, run the listed interaction with the - `chrome_*` tools, then read `window.__verdict` / `window.__reason` via - `chrome_evaluate`. -3. Compare against the manual baseline (do the same interaction by hand and - confirm `PASS`). - -## Challenges - -| id | file | what it blocks | -|----|------|----------------| -| `is-trusted-click` | `challenges/01-is-trusted-click.html` | click handler ignores `event.isTrusted === false` | -| `is-trusted-keyboard` | `challenges/02-is-trusted-keyboard.html` | input rejects if `keydown.isTrusted` false | -| `webdriver-flag` | `challenges/03-webdriver-flag.html` | `navigator.webdriver` truthy / Chrome runtime quirks | -| `mouse-entropy` | `challenges/04-mouse-entropy.html` | requires organic mousemove path before click | -| `event-timing` | `challenges/05-event-timing.html` | rejects when pointerdown→pointerup gap < N ms or always-equal | -| `click-coordinates` | `challenges/06-click-coordinates.html` | rejects clicks that land on exact element center | -| `pointer-properties` | `challenges/07-pointer-properties.html` | demands pressure / non-zero movementX/Y / pointerId variety | -| `keyboard-cadence` | `challenges/08-keyboard-cadence.html` | rejects same-tick `keydown`/`keyup` and missing per-char events | -| `composition-input` | `challenges/09-composition-input.html` | listens for native `keydown` + `keypress` + `input` sequence | -| `user-activation` | `challenges/10-user-activation.html` | feature gates: clipboard.writeText, fullscreen | -| `honeypot` | `challenges/11-honeypot.html` | hidden field must stay empty | -| `fingerprint` | `challenges/12-fingerprint.html` | UA-CH, languages, plugins, permissions consistency | -| `focus-order` | `challenges/13-focus-order.html` | requires `pointerdown`-driven focus, not direct `.focus()` | -| `wheel-scroll` | `challenges/14-wheel-scroll.html` | rejects scrollTop jumps without `wheel`/`scroll` events | -| `drag-drop-datatransfer` | `challenges/15-drag-drop-datatransfer.html` | requires full HTML5 drag cycle with populated `DataTransfer` | -| `contenteditable-selection` | `challenges/16-contenteditable-selection.html` | requires `selectionchange` + monotonic caret advance per keystroke | -| `paste-clipboard` | `challenges/17-paste-clipboard.html` | demands real OS paste with `clipboardData` + `inputType=insertFromPaste` | -| `native-select` | `challenges/18-native-select.html` | rejects `