diff options
| author | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-13 01:48:36 -0400 |
|---|---|---|
| committer | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-13 01:48:36 -0400 |
| commit | 066f50b9cdb60ad3d4f61ba0598ac9683d8ae446 (patch) | |
| tree | bdd4372b365651a7ebc36bc6920ac2e0ed44c4ae | |
| parent | 07cf31df3cd47d18d8a067c45a667632fb3b8925 (diff) | |
pi-chrome 0.11.1: chrome_tap (CDP touch), smoother trustedScroll, cdp auto-recover
- chrome_tap tool: real browser-trusted touchstart/touchend via
Input.dispatchTouchEvent for sites that gate on TouchEvent (mobile-
first PWAs, swipe carousels). Always uses CDP; banner appears.
- trustedScroll: per-step delta capped at ~80px, n=6..60 steps with a
bell-shaped weight curve, 22-52ms sleep between events so IntersectionObserver and rAF samples get gradient updates (fixes
intersection-visibility / smooth-scroll tests).
- cdp() wrapper auto-detects stale debugger sessions (tab nav,
DevTools toggle, manual cancel) and re-attaches+retries once
instead of throwing 'Debugger is not attached'. Fixes intermittent
failures during automated test sweeps.
| -rw-r--r-- | extensions/chrome-profile-bridge/browser-extension/manifest.json | 2 | ||||
| -rw-r--r-- | extensions/chrome-profile-bridge/browser-extension/service_worker.js | 58 | ||||
| -rw-r--r-- | extensions/chrome-profile-bridge/index.ts | 23 | ||||
| -rw-r--r-- | package.json | 2 |
4 files changed, 78 insertions, 7 deletions
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json index 77177ab..8ce8392 100644 --- a/extensions/chrome-profile-bridge/browser-extension/manifest.json +++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Pi Chrome Connector", - "version": "0.11.0", + "version": "0.11.1", "description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.", "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation", "debugger"], "host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"], diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js index ddfef56..444fdbc 100644 --- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js +++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js @@ -103,7 +103,7 @@ setInterval(() => { } }, 5000); -function cdp(tabId, method, params) { +function cdpRaw(tabId, method, params) { return new Promise((resolve, reject) => { chrome.debugger.sendCommand({ tabId }, method, params || {}, (result) => { if (chrome.runtime.lastError) reject(new Error(`${method}: ${chrome.runtime.lastError.message}`)); @@ -112,6 +112,24 @@ function cdp(tabId, method, params) { }); } +// Wraps cdpRaw with one auto-recover on detached/closed sessions: +// chrome.debugger.attach can stay cached in attachedTabs even after Chrome killed +// the session (tab nav, devtools opened/closed, etc). Recover by detaching the +// stale entry and re-attaching, then retry the command once. +async function cdp(tabId, method, params) { + try { + return await cdpRaw(tabId, method, params); + } catch (error) { + const msg = String(error?.message || error); + const isStale = /Debugger is not attached|Detached while|Target closed|No tab with id/i.test(msg); + if (!isStale) throw error; + attachedTabs.delete(tabId); + await chrome.debugger.attach({ tabId }, CDP_VERSION).catch(() => undefined); + attachedTabs.set(tabId, { detachAt: Date.now() + TRUSTED_IDLE_DETACH_MS, pointer: { x: 120 + Math.random() * 200, y: 80 + Math.random() * 120 } }); + return cdpRaw(tabId, method, params); + } +} + // Resolve target -> {x, y, rect} in viewport coords by running tiny script in tab. async function resolveTargetInTab(tabId, params) { const results = await chrome.scripting.executeScript({ @@ -349,20 +367,48 @@ async function trustedScroll(params) { const x = resolved.rect ? resolved.rect.left + Math.min(resolved.rect.width, 800) / 2 : resolved.x; const y = resolved.rect ? resolved.rect.top + Math.min(resolved.rect.height, 600) / 2 : resolved.y; const totalY = params.deltaY || 0, totalX = params.deltaX || 0; - const n = Math.max(3, Math.min(20, params.steps || Math.ceil(Math.abs(totalY) / 120))); - // momentum-shaped front-loaded weights - const w = []; for (let i = 1; i <= n; i++) w.push(1 / i); + // Cap per-step delta so IntersectionObserver / scroll-driven animations get gradient samples. + // Real wheel notches ~50-120px; we aim ~40-90. + const MAX_STEP = 80; + const minStepsByDelta = Math.ceil(Math.max(Math.abs(totalY), Math.abs(totalX)) / MAX_STEP); + const n = Math.max(6, Math.min(60, params.steps || Math.max(minStepsByDelta, 12))); + // Momentum shape: ramp-up, plateau, decay. Each weight peaked mid-sequence then tapers. + const w = []; + for (let i = 0; i < n; i++) { + const t = i / Math.max(1, n - 1); + // bell-ish curve biased earlier so motion starts strong then decays (like a trackpad flick). + const base = Math.sin(Math.min(1, t * 1.4) * Math.PI); + const decay = Math.pow(1 - t * 0.6, 2); + w.push(0.2 + base * 0.8 * decay); + } const sumW = w.reduce((a, b) => a + b, 0); for (let i = 0; i < n; i++) { const dy = totalY * (w[i] / sumW), dx = totalX * (w[i] / sumW); await cdp(tab.id, "Input.dispatchMouseEvent", { type: "mouseWheel", x, y, deltaX: dx, deltaY: dy, pointerType: "mouse", }); - await sleep(rng(14, 32)); + // Sleep ~one frame+ so IntersectionObserver / rAF samples can run between events. + await sleep(rng(22, 52)); } return { trusted: true, deltaX: totalX, deltaY: totalY, steps: n }; } +async function trustedTap(params) { + const tab = await getTabByParams(params); + if (params.foreground) await bringToFront(tab); + await attachDebugger(tab.id); + const resolved = (params.selector || params.uid || (typeof params.x === "number" && typeof params.y === "number")) + ? await resolveTargetInTab(tab.id, params) + : null; + if (!resolved || !resolved.found) throw new Error("trusted.tap: target not found"); + const point = resolved.rect ? pickInsideRect(resolved.rect) : { x: resolved.x, y: resolved.y }; + const tp = { x: point.x, y: point.y, radiusX: 8, radiusY: 8, rotationAngle: 0, force: 0.5, id: 1 }; + await cdp(tab.id, "Input.dispatchTouchEvent", { type: "touchStart", touchPoints: [tp] }); + await sleep(rng(40, 110)); + await cdp(tab.id, "Input.dispatchTouchEvent", { type: "touchEnd", touchPoints: [] }); + return { trusted: true, x: point.x, y: point.y, tag: resolved.tag }; +} + async function trustedDrag(params) { const tab = await getTabByParams(params); if (params.foreground) await bringToFront(tab); @@ -537,6 +583,8 @@ async function dispatch(action, params) { case "page.scroll": if (await wantsTrusted(params)) return trustedScroll(params); return executeActionInTab(params, scrollPage, [params.selector ?? null, params.uid ?? null, params.deltaY ?? 0, params.deltaX ?? 0, params.steps ?? null]); + case "page.tap": + return trustedTap(params); case "trusted.mode": return setTrustedMode(params.mode); case "trusted.status": diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts index 5eee401..fc1b6c5 100644 --- a/extensions/chrome-profile-bridge/index.ts +++ b/extensions/chrome-profile-bridge/index.ts @@ -1166,6 +1166,29 @@ Usage rules: }); pi.registerTool({ + name: "chrome_tap", + label: "Chrome Tap (Touch)", + description: + "Dispatch a real browser-trusted touchstart/touchend tap via chrome.debugger (CDP Input.dispatchTouchEvent). Use for sites that gate on TouchEvent rather than MouseEvent (mobile-first PWAs, swipe carousels). Always uses the trusted CDP path — the yellow debugger banner appears.", + promptSnippet: "Tap (real touch) a Chrome element by snapshot uid, selector, or coordinate.", + parameters: Type.Object({ + uid: Type.Optional(Type.String()), + selector: Type.Optional(Type.String()), + x: Type.Optional(Type.Number()), + y: Type.Optional(Type.Number()), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional(Type.Boolean()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.tap", withBackground(params), DEFAULT_TIMEOUT_MS); + const target = params.uid ?? params.selector ?? `${params.x},${params.y}`; + return { content: [{ type: "text", text: `Tapped ${target} (touch)` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ name: "chrome_scroll", label: "Chrome Scroll", description: "Scroll the page or a specific scrollable element by dispatching real wheel events with momentum-shaped deltas, then applying the scroll. Positive deltaY scrolls down. Pass uid/selector to scroll within a container, otherwise the document scrolls.", diff --git a/package.json b/package.json index a0d0b3a..87a8357 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-chrome", - "version": "0.11.0", + "version": "0.11.1", "description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).", "keywords": [ "pi-package", |
