aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTianren Dong <tianrendong.dtr@gmail.com>2026-05-13 01:48:36 -0400
committerTianren Dong <tianrendong.dtr@gmail.com>2026-05-13 01:48:36 -0400
commit066f50b9cdb60ad3d4f61ba0598ac9683d8ae446 (patch)
treebdd4372b365651a7ebc36bc6920ac2e0ed44c4ae
parent07cf31df3cd47d18d8a067c45a667632fb3b8925 (diff)
pi-chrome 0.11.1: chrome_tap (CDP touch), smoother trustedScroll, cdp auto-recover
- chrome_tap tool: real browser-trusted touchstart/touchend via Input.dispatchTouchEvent for sites that gate on TouchEvent (mobile- first PWAs, swipe carousels). Always uses CDP; banner appears. - trustedScroll: per-step delta capped at ~80px, n=6..60 steps with a bell-shaped weight curve, 22-52ms sleep between events so IntersectionObserver and rAF samples get gradient updates (fixes intersection-visibility / smooth-scroll tests). - cdp() wrapper auto-detects stale debugger sessions (tab nav, DevTools toggle, manual cancel) and re-attaches+retries once instead of throwing 'Debugger is not attached'. Fixes intermittent failures during automated test sweeps.
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/manifest.json2
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/service_worker.js58
-rw-r--r--extensions/chrome-profile-bridge/index.ts23
-rw-r--r--package.json2
4 files changed, 78 insertions, 7 deletions
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json
index 77177ab..8ce8392 100644
--- a/extensions/chrome-profile-bridge/browser-extension/manifest.json
+++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json
@@ -1,7 +1,7 @@
{
"manifest_version": 3,
"name": "Pi Chrome Connector",
- "version": "0.11.0",
+ "version": "0.11.1",
"description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.",
"permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation", "debugger"],
"host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"],
diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
index ddfef56..444fdbc 100644
--- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js
+++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
@@ -103,7 +103,7 @@ setInterval(() => {
}
}, 5000);
-function cdp(tabId, method, params) {
+function cdpRaw(tabId, method, params) {
return new Promise((resolve, reject) => {
chrome.debugger.sendCommand({ tabId }, method, params || {}, (result) => {
if (chrome.runtime.lastError) reject(new Error(`${method}: ${chrome.runtime.lastError.message}`));
@@ -112,6 +112,24 @@ function cdp(tabId, method, params) {
});
}
+// Wraps cdpRaw with one auto-recover on detached/closed sessions:
+// chrome.debugger.attach can stay cached in attachedTabs even after Chrome killed
+// the session (tab nav, devtools opened/closed, etc). Recover by detaching the
+// stale entry and re-attaching, then retry the command once.
+async function cdp(tabId, method, params) {
+ try {
+ return await cdpRaw(tabId, method, params);
+ } catch (error) {
+ const msg = String(error?.message || error);
+ const isStale = /Debugger is not attached|Detached while|Target closed|No tab with id/i.test(msg);
+ if (!isStale) throw error;
+ attachedTabs.delete(tabId);
+ await chrome.debugger.attach({ tabId }, CDP_VERSION).catch(() => undefined);
+ attachedTabs.set(tabId, { detachAt: Date.now() + TRUSTED_IDLE_DETACH_MS, pointer: { x: 120 + Math.random() * 200, y: 80 + Math.random() * 120 } });
+ return cdpRaw(tabId, method, params);
+ }
+}
+
// Resolve target -> {x, y, rect} in viewport coords by running tiny script in tab.
async function resolveTargetInTab(tabId, params) {
const results = await chrome.scripting.executeScript({
@@ -349,20 +367,48 @@ async function trustedScroll(params) {
const x = resolved.rect ? resolved.rect.left + Math.min(resolved.rect.width, 800) / 2 : resolved.x;
const y = resolved.rect ? resolved.rect.top + Math.min(resolved.rect.height, 600) / 2 : resolved.y;
const totalY = params.deltaY || 0, totalX = params.deltaX || 0;
- const n = Math.max(3, Math.min(20, params.steps || Math.ceil(Math.abs(totalY) / 120)));
- // momentum-shaped front-loaded weights
- const w = []; for (let i = 1; i <= n; i++) w.push(1 / i);
+ // Cap per-step delta so IntersectionObserver / scroll-driven animations get gradient samples.
+ // Real wheel notches ~50-120px; we aim ~40-90.
+ const MAX_STEP = 80;
+ const minStepsByDelta = Math.ceil(Math.max(Math.abs(totalY), Math.abs(totalX)) / MAX_STEP);
+ const n = Math.max(6, Math.min(60, params.steps || Math.max(minStepsByDelta, 12)));
+ // Momentum shape: ramp-up, plateau, decay. Each weight peaked mid-sequence then tapers.
+ const w = [];
+ for (let i = 0; i < n; i++) {
+ const t = i / Math.max(1, n - 1);
+ // bell-ish curve biased earlier so motion starts strong then decays (like a trackpad flick).
+ const base = Math.sin(Math.min(1, t * 1.4) * Math.PI);
+ const decay = Math.pow(1 - t * 0.6, 2);
+ w.push(0.2 + base * 0.8 * decay);
+ }
const sumW = w.reduce((a, b) => a + b, 0);
for (let i = 0; i < n; i++) {
const dy = totalY * (w[i] / sumW), dx = totalX * (w[i] / sumW);
await cdp(tab.id, "Input.dispatchMouseEvent", {
type: "mouseWheel", x, y, deltaX: dx, deltaY: dy, pointerType: "mouse",
});
- await sleep(rng(14, 32));
+ // Sleep ~one frame+ so IntersectionObserver / rAF samples can run between events.
+ await sleep(rng(22, 52));
}
return { trusted: true, deltaX: totalX, deltaY: totalY, steps: n };
}
+async function trustedTap(params) {
+ const tab = await getTabByParams(params);
+ if (params.foreground) await bringToFront(tab);
+ await attachDebugger(tab.id);
+ const resolved = (params.selector || params.uid || (typeof params.x === "number" && typeof params.y === "number"))
+ ? await resolveTargetInTab(tab.id, params)
+ : null;
+ if (!resolved || !resolved.found) throw new Error("trusted.tap: target not found");
+ const point = resolved.rect ? pickInsideRect(resolved.rect) : { x: resolved.x, y: resolved.y };
+ const tp = { x: point.x, y: point.y, radiusX: 8, radiusY: 8, rotationAngle: 0, force: 0.5, id: 1 };
+ await cdp(tab.id, "Input.dispatchTouchEvent", { type: "touchStart", touchPoints: [tp] });
+ await sleep(rng(40, 110));
+ await cdp(tab.id, "Input.dispatchTouchEvent", { type: "touchEnd", touchPoints: [] });
+ return { trusted: true, x: point.x, y: point.y, tag: resolved.tag };
+}
+
async function trustedDrag(params) {
const tab = await getTabByParams(params);
if (params.foreground) await bringToFront(tab);
@@ -537,6 +583,8 @@ async function dispatch(action, params) {
case "page.scroll":
if (await wantsTrusted(params)) return trustedScroll(params);
return executeActionInTab(params, scrollPage, [params.selector ?? null, params.uid ?? null, params.deltaY ?? 0, params.deltaX ?? 0, params.steps ?? null]);
+ case "page.tap":
+ return trustedTap(params);
case "trusted.mode":
return setTrustedMode(params.mode);
case "trusted.status":
diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts
index 5eee401..fc1b6c5 100644
--- a/extensions/chrome-profile-bridge/index.ts
+++ b/extensions/chrome-profile-bridge/index.ts
@@ -1166,6 +1166,29 @@ Usage rules:
});
pi.registerTool({
+ name: "chrome_tap",
+ label: "Chrome Tap (Touch)",
+ description:
+ "Dispatch a real browser-trusted touchstart/touchend tap via chrome.debugger (CDP Input.dispatchTouchEvent). Use for sites that gate on TouchEvent rather than MouseEvent (mobile-first PWAs, swipe carousels). Always uses the trusted CDP path — the yellow debugger banner appears.",
+ promptSnippet: "Tap (real touch) a Chrome element by snapshot uid, selector, or coordinate.",
+ parameters: Type.Object({
+ uid: Type.Optional(Type.String()),
+ selector: Type.Optional(Type.String()),
+ x: Type.Optional(Type.Number()),
+ y: Type.Optional(Type.Number()),
+ targetId: Type.Optional(Type.String()),
+ urlIncludes: Type.Optional(Type.String()),
+ titleIncludes: Type.Optional(Type.String()),
+ background: Type.Optional(Type.Boolean()),
+ }),
+ async execute(_id, params): Promise<ToolTextResult> {
+ const result = await bridge.send("page.tap", withBackground(params), DEFAULT_TIMEOUT_MS);
+ const target = params.uid ?? params.selector ?? `${params.x},${params.y}`;
+ return { content: [{ type: "text", text: `Tapped ${target} (touch)` }], details: { result: result as Json } };
+ },
+ });
+
+ pi.registerTool({
name: "chrome_scroll",
label: "Chrome Scroll",
description: "Scroll the page or a specific scrollable element by dispatching real wheel events with momentum-shaped deltas, then applying the scroll. Positive deltaY scrolls down. Pass uid/selector to scroll within a container, otherwise the document scrolls.",
diff --git a/package.json b/package.json
index a0d0b3a..87a8357 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "pi-chrome",
- "version": "0.11.0",
+ "version": "0.11.1",
"description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).",
"keywords": [
"pi-package",