diff options
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | extensions/chrome-profile-bridge/browser-extension/manifest.json | 8 | ||||
| -rw-r--r-- | extensions/chrome-profile-bridge/browser-extension/service_worker.js | 86 | ||||
| -rw-r--r-- | extensions/chrome-profile-bridge/index.ts | 25 | ||||
| -rw-r--r-- | package.json | 2 |
5 files changed, 97 insertions, 26 deletions
@@ -69,7 +69,7 @@ pi-chrome v<version> By default, `chrome_*` clicks and keystrokes are **synthetic** DOM events (`event.isTrusted === false`). They drive React/Vue/Angular state correctly but **do not** satisfy Chrome's user-activation gates: clipboard write, fullscreen, file picker, and autoplay all need a real user gesture. -pi-chrome can optionally route input through `chrome.debugger` (CDP `Input.dispatchMouseEvent` / `Input.dispatchKeyEvent`) so each event arrives as `isTrusted=true`, satisfies user-activation, and bypasses site bot-detection that filters synthetic events. The tradeoff: Chrome pins a yellow *"Pi Existing Chrome Profile Bridge started debugging this browser"* banner to the top of any debugged tab. +pi-chrome can optionally route input through `chrome.debugger` (CDP `Input.dispatchMouseEvent` / `Input.dispatchKeyEvent`) so each event arrives as `isTrusted=true`, satisfies user-activation, and bypasses site bot-detection that filters synthetic events. The tradeoff: Chrome pins a yellow *"Pi Chrome Connector started debugging this browser"* banner to the top of any debugged tab. Usage: diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json index dcff745..77177ab 100644 --- a/extensions/chrome-profile-bridge/browser-extension/manifest.json +++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json @@ -1,14 +1,14 @@ { "manifest_version": 3, - "name": "Pi Existing Chrome Profile Bridge", - "version": "0.10.2", - "description": "Lets Pi control tabs in this existing Chrome profile via a local bridge at 127.0.0.1.", + "name": "Pi Chrome Connector", + "version": "0.11.0", + "description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.", "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation", "debugger"], "host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"], "background": { "service_worker": "service_worker.js" }, "action": { - "default_title": "Pi Chrome Bridge" + "default_title": "Pi Chrome Connector" } } diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js index bd00b1e..ddfef56 100644 --- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js +++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js @@ -1,12 +1,12 @@ const BRIDGE_URL = "http://127.0.0.1:17318"; -const CLIENT_NAME = `Pi Chrome Bridge ${chrome.runtime.id}`; +const CLIENT_NAME = `Pi Chrome Connector ${chrome.runtime.id}`; const POLL_ERROR_BACKOFF_MS = 2000; let polling = false; // =================== Trusted-input (CDP) layer =================== // Tracks which tabs we have attached chrome.debugger to, plus session-level mode. const attachedTabs = new Map(); // tabId -> { detachAt: number, pointer: {x,y} } -let TRUSTED_MODE = "off"; // "off" | "on" | "auto" +let TRUSTED_MODE = "auto"; // "off" | "on" | "auto" (default: smart retry only) const TRUSTED_IDLE_DETACH_MS = 15_000; const CDP_VERSION = "1.3"; @@ -35,6 +35,31 @@ function trustedStatus() { }; } +// Auto-upgrade: if synthetic result carries suggestTrusted=true, the bridge mode is "auto" +// (default) or "on", and the caller didn't explicitly opt out, retry once with trusted CDP +// path. Surfaces both results so callers can see what happened. +async function maybeUpgradeToTrusted(kind, params, syntheticResult, trustedFn) { + if (!syntheticResult || !syntheticResult.suggestTrusted) return syntheticResult; + if (params && params.trusted === false) return syntheticResult; + if (TRUSTED_MODE === "off") return syntheticResult; + if (!chrome.debugger) return syntheticResult; + try { + const trustedResult = await trustedFn(); + return { + ...trustedResult, + autoRetried: true, + autoRetryReason: syntheticResult.suggestReason || `${kind} produced no mutation`, + syntheticAttempt: { pageMutated: syntheticResult.pageMutated, suggestReason: syntheticResult.suggestReason }, + }; + } catch (error) { + return { + ...syntheticResult, + autoRetryAttempted: true, + autoRetryError: error?.message || String(error), + }; + } +} + async function attachDebugger(tabId) { if (!chrome.debugger) throw new Error("chrome.debugger API unavailable; reload the extension to grant the new permission"); if (attachedTabs.has(tabId)) { @@ -485,9 +510,11 @@ async function dispatch(action, params) { ]); case "page.evaluate": return evaluateInTab(params); - case "page.click": + case "page.click": { if (await wantsTrusted(params)) return trustedClick(params); - return executeActionInTab(params, clickPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]); + const synth = await executeActionInTab(params, clickPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]); + return await maybeUpgradeToTrusted("click", params, synth, () => trustedClick(params)); + } case "page.hover": if (await wantsTrusted(params)) return trustedHover(params); return executeActionInTab(params, hoverPage, [params.selector ?? null, params.uid ?? null, params.x ?? null, params.y ?? null]); @@ -496,9 +523,11 @@ async function dispatch(action, params) { return executeActionInTab(params, dragPage, [params.fromUid ?? null, params.fromSelector ?? null, params.fromX ?? null, params.fromY ?? null, params.toUid ?? null, params.toSelector ?? null, params.toX ?? null, params.toY ?? null, params.steps ?? 12]); case "page.upload": return executeActionInTab(params, uploadFiles, [params.selector ?? null, params.uid ?? null, params.files || []]); - case "page.type": + case "page.type": { if (await wantsTrusted(params)) return trustedType(params); - return executeActionInTab(params, typeIntoPage, [params.selector ?? null, params.uid ?? null, params.text || "", Boolean(params.pressEnter)]); + const synth = await executeActionInTab(params, typeIntoPage, [params.selector ?? null, params.uid ?? null, params.text || "", Boolean(params.pressEnter)]); + return await maybeUpgradeToTrusted("type", params, synth, () => trustedType(params)); + } case "page.fill": if (await wantsTrusted(params)) return trustedFill(params); return executeActionInTab(params, fillPage, [params.selector ?? null, params.uid ?? null, params.text || "", params.submit === true]); @@ -1262,23 +1291,46 @@ async function clickPage(selector, uid, x, y) { // Heuristic: if the clicked thing looks like a media play affordance and the page has paused // audio/video, the synthetic click may not unlock autoplay. Surface a warning. let autoplayHint; - const label = (point.element.getAttribute("aria-label") || point.element.textContent || "").toLowerCase(); - if (/^(play|start|begin|next|continue|unmute)/.test(label.trim())) { + const labelRaw = (point.element.getAttribute("aria-label") || point.element.textContent || "").trim(); + const label = labelRaw.toLowerCase(); + if (/^(play|start|begin|next|continue|unmute)/.test(label)) { const idleMedia = Array.from(document.querySelectorAll("audio,video")).some((m) => m.paused); if (idleMedia) autoplayHint = "This element looks like a media affordance and the page has paused media. Synthetic clicks do not satisfy user-activation gates; audio/video may not start."; } + const pageMutated = pageHash() !== before; + // Smart-auto retry hint: only set when synthetic produced no observable change AND the + // element looks gated, OR the page just emitted a user-activation rejection. The dispatcher + // uses this to decide whether to retry with trusted mode. + let suggestTrusted = false; + let suggestReason; + if (!pageMutated) { + if (autoplayHint) { suggestTrusted = true; suggestReason = "play/media affordance + idle media"; } + else if (/copy(\s|$)|paste|share|download|fullscreen|sign in with|continue with|allow|enable/i.test(label)) { + suggestTrusted = true; suggestReason = `label '${labelRaw.slice(0, 40)}' looks gated`; + } else { + // Inspect recent console errors for activation-gate rejections. + const recent = (state.console || []).slice(-8); + const hit = recent.find((e) => /NotAllowedError|Document is not focused|requires transient activation|gesture is required/.test( + (e.args || []).map((a) => typeof a === "string" ? a : (a && a.message) || JSON.stringify(a)).join(" ") + )); + if (hit) { suggestTrusted = true; suggestReason = "recent console error indicates user-activation gate"; } + } + } return { x: point.x, y: point.y, selector, uid, tag: point.element.tagName, + label: labelRaw.slice(0, 80) || undefined, isTrusted: false, defaultPrevented, elementVisible: visible, occludedBy: occluded || undefined, - pageMutated: pageHash() !== before, + pageMutated, autoplayHint, + suggestTrusted: suggestTrusted || undefined, + suggestReason, }; } @@ -1526,15 +1578,27 @@ async function typeIntoPage(selector, uid, text, pressEnter) { const before = pageHash(); let element = elementBySelectorOrUid(selector, uid) || document.activeElement; if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element"); + const initialValue = "value" in element ? element.value : (element.isContentEditable ? element.textContent : null); element.focus(); if (!(element.isContentEditable || "value" in element)) throw new Error("Focused element is not text-editable"); for (const ch of Array.from(text)) await typeCharacter(element, ch); if (pressEnter) pressKeyInPage("Enter"); + const finalValue = "value" in element ? element.value : element.textContent; + const valueMatches = "value" in element ? element.value.includes(text) : (element.textContent || "").includes(text); + const pageMutated = pageHash() !== before; + // Smart-auto retry hint when typing didn't land at all (e.g., editor blocks synthetic input). + let suggestTrusted = false, suggestReason; + if (text.length > 0 && initialValue === finalValue) { + suggestTrusted = true; + suggestReason = "value did not change — editor likely rejects synthetic input"; + } return { selector, uid, length: text.length, pressEnter, isTrusted: false, - valueMatches: "value" in element ? element.value.includes(text) : undefined, - pageMutated: pageHash() !== before, + valueMatches, + pageMutated, + suggestTrusted: suggestTrusted || undefined, + suggestReason, }; } diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts index f8bcb79..5eee401 100644 --- a/extensions/chrome-profile-bridge/index.ts +++ b/extensions/chrome-profile-bridge/index.ts @@ -471,7 +471,7 @@ Usage rules: lines.push( `✗ EXTENSION VERSION MISMATCH: companion extension is v${version.extensionVersion}, but pi-chrome is v${PI_CHROME_VERSION}.`, ` All chrome_* tools will run with the OLD extension code until this is fixed.`, - ` Fix: open chrome://extensions and click reload on "Pi Existing Chrome Profile Bridge".`, + ` Fix: open chrome://extensions and click reload on "Pi Chrome Connector".`, ` (Future version drifts will self-heal: the extension now polls pi-chrome's expected version and reloads itself.)`, ); } else { @@ -518,7 +518,14 @@ Usage rules: permissionGranted?: boolean; }; if (status.permissionGranted) { - lines.push(`✓ Trusted-input mode available via chrome.debugger (current: ${status.mode ?? "off"}${status.attachedTabs && status.attachedTabs.length ? `; attached to tab ${status.attachedTabs.join(",")}` : ""}). Pass trusted=true on chrome_click/type/etc, or run /chrome-trusted on, to satisfy isTrusted + user-activation gates.`); + const attached = status.attachedTabs && status.attachedTabs.length ? `; attached to tab ${status.attachedTabs.join(",")}` : ""; + const note = + status.mode === "auto" + ? " — smart-retry enabled: synthetic input runs first; if a click/type produced no page change AND the target looks gated, the call is automatically re-run with trusted CDP (yellow debugger banner appears only for that retry)." + : status.mode === "on" + ? " — every chrome_* call goes through CDP; the yellow debugger banner is visible while attached." + : " — synthetic events only; pass trusted=true on chrome_click/type/etc, or switch to auto/on with /chrome-trusted, when isTrusted or user-activation gates matter."; + lines.push(`✓ Trusted-input mode available via chrome.debugger (current: ${status.mode ?? "off"}${attached}).${note}`); } else { lines.push(`⚠ chrome.debugger API unavailable. The extension is missing the "debugger" permission — reload the extension in chrome://extensions and accept the new permission prompt.`); } @@ -560,7 +567,7 @@ Usage rules: if (!status.permissionGranted) { ctx.ui.notify( - "chrome.debugger API unavailable — the extension is missing the 'debugger' permission. Open chrome://extensions, reload 'Pi Existing Chrome Profile Bridge', and accept the new permission prompt.", + "chrome.debugger API unavailable — the extension is missing the 'debugger' permission. Open chrome://extensions, reload 'Pi Chrome Connector', and accept the new permission prompt.", "warning", ); return; @@ -577,9 +584,9 @@ Usage rules: if (!target) { // Interactive picker. Show current mode + tradeoffs in each label. const options = [ - `on${current === "on" ? " (current)" : ""} — all chrome_* tools via CDP; yellow debugger banner appears`, - `off${current === "off" ? " (current)" : ""} — synthetic DOM events only (default)`, - `auto${current === "auto" ? " (current)" : ""} — CDP only when a tool passes trusted=true`, + `auto${current === "auto" ? " (current)" : ""} — default; synthetic first, retry with CDP only when a call looks gated`, + `off${current === "off" ? " (current)" : ""} — synthetic DOM events only; never auto-retry`, + `on${current === "on" ? " (current)" : ""} — every chrome_* call goes through CDP (yellow debugger banner permanently visible)`, `status — print current mode and any attached tabs\u2026`, ]; const picked = await ctx.ui.select( @@ -610,7 +617,7 @@ Usage rules: if (target === "on" && current === "off") { const ok = await ctx.ui.confirm( "Turn on trusted-input mode?", - "All chrome_* tools will dispatch through chrome.debugger (CDP). Events will arrive as isTrusted=true and satisfy user-activation gates (clipboard, fullscreen, autoplay, file picker).\n\nChrome will pin a yellow 'Pi Existing Chrome Profile Bridge started debugging this browser' banner to the top of any debugged tab while attached. Clicking 'Cancel' on that banner detaches the debugger.", + "All chrome_* tools will dispatch through chrome.debugger (CDP). Events will arrive as isTrusted=true and satisfy user-activation gates (clipboard, fullscreen, autoplay, file picker).\n\nChrome will pin a yellow 'Pi Chrome Connector started debugging this browser' banner to the top of any debugged tab while attached. Clicking 'Cancel' on that banner detaches the debugger.", ); if (!ok) { ctx.ui.notify("Trusted-input mode unchanged.", "info"); @@ -835,7 +842,7 @@ Usage rules: name: "chrome_click", label: "Chrome Click", description: - "Click a snapshot uid, CSS selector, or viewport coordinate in an existing Chrome tab through the companion extension. Defaults to synthetic DOM events (isTrusted=false). Pass trusted=true (or run /chrome-trusted on) to route through chrome.debugger so events arrive as browser-trusted and satisfy user-activation gates — Chrome shows a yellow 'started debugging' banner while attached. Pass includeSnapshot=true to return a fresh snapshot after the click.", + "Click a snapshot uid, CSS selector, or viewport coordinate. Default 'auto' mode runs synthetic DOM events first and silently retries with trusted CDP only when the click looks gated (no page change + affordance label matches play/copy/share/sign-in/etc, or a recent NotAllowedError). The yellow 'started debugging' banner appears only when the retry actually happens. Pass trusted=true to force CDP for this call (banner appears immediately). Pass trusted=false to skip retry. Pass includeSnapshot=true to return a fresh snapshot after the click.", promptSnippet: "Click page elements in Chrome by snapshot uid, selector, or viewport coordinate.", parameters: Type.Object({ uid: Type.Optional(Type.String({ description: "Stable element uid from chrome_snapshot. Prefer uid over selector after taking a snapshot." })), @@ -868,7 +875,7 @@ Usage rules: name: "chrome_type", label: "Chrome Type", description: - "Focus an optional snapshot uid or CSS selector, then type text into an existing Chrome tab. Defaults to synthetic per-character keydown/beforeinput/input/keyup sequence. Pass trusted=true (or run /chrome-trusted on) to route through chrome.debugger so each keystroke is browser-trusted (isTrusted=true). Pass includeSnapshot=true to return a fresh snapshot after typing.", + "Focus an optional snapshot uid or CSS selector, then type text. Default 'auto' mode runs synthetic per-character keydown/beforeinput/input/keyup first; if the input value doesn't change at all (editor rejected synthetic input) the call is silently retried through chrome.debugger so each keystroke is browser-trusted (isTrusted=true). Pass trusted=true to force CDP for this call. Pass trusted=false to skip retry. Pass includeSnapshot=true to return a fresh snapshot after typing.", promptSnippet: "Type text into Chrome, optionally focusing a snapshot uid or selector first.", parameters: Type.Object({ text: Type.String(), diff --git a/package.json b/package.json index a9092bd..a0d0b3a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-chrome", - "version": "0.10.2", + "version": "0.11.0", "description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).", "keywords": [ "pi-package", |