diff options
| author | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-13 01:05:46 -0400 |
|---|---|---|
| committer | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-13 01:05:46 -0400 |
| commit | 33110e803fcacc31dc2859dbc0d470b008556798 (patch) | |
| tree | bb30dad365139c558f9d9651cb8bca375320cb8e /extensions | |
| parent | d78081aa58ecaa1c72e847c2d095b5f07c391506 (diff) | |
pi-chrome: humanize synthetic input + anti-automation test suite
Bump 0.8.0 → 0.9.1.
Companion extension (service_worker.js):
- humanise pointer paths: easing Bezier-ish interpolation w/ wobble, 12–42
steps, 4–18 ms gaps; pointer position persisted across calls
- click point picked inside element rect, not dead center
- pointer events fill movementX/Y, screenX/Y, pressure (0.5 on down),
pointerId, width/height, tilt, composed
- click sequence: humanMoveTo → pointerdown → focus → 45–140 ms hold →
pointerup → mouseup → click
- per-character typing: keydown → keypress → beforeinput → value mutate
(or execCommand for contenteditable) → input → keyup, with 25–95 ms
hold + 35–140 ms inter-key; auto Shift chord for uppercase/symbols
- pressKeyInPage emits same cadence + handles printable insert / Backspace
- hoverPage: humanised path + 80–220 ms dwell
- dragPage: full HTML5 sequence (dragstart/drag/dragenter/dragover/
dragleave/drop/dragend) with shared DataTransfer + pointer continuity
- new scrollPage: wheel events with front-loaded momentum-shaped deltas
- self-reload on version drift: SW reads x-pi-chrome-version header,
reloads when older than pi-chrome
Pi side (index.ts):
- new chrome_scroll tool
- PI_CHROME_VERSION read from package.json (and re-read on every /next
so future bumps take effect without pi restart)
- /chrome-doctor escalates version mismatch to top-line ✗ error and
skips downstream capability probes when stale
- /next response carries x-pi-chrome-version header for SW self-heal
Test suite (test-suite/):
- 28 challenge pages each catching a different synthetic-event tell:
isTrusted, webdriver/fingerprint, mouse entropy/timing/coords,
pointer pressure/continuity, per-char cadence + framework invariants,
user activation, honeypots, focus order, wheel scroll, drag+
DataTransfer, contenteditable+Selection, paste/clipboard, native
select, hover dwell, React valueTracker, keyboard modifiers, touch
events, stack-trace fingerprint, viewport-edge clicks, mousemove
rate, scroll momentum, IntersectionObserver visibility
- shared _lib.js exposes window.__verdict + writes localStorage so
index.html dashboard aggregates results
- notes/bypass-ideas.md documents fixes still needed per challenge
Diffstat (limited to 'extensions')
3 files changed, 417 insertions, 63 deletions
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json index 5ceb76e..9fb49f0 100644 --- a/extensions/chrome-profile-bridge/browser-extension/manifest.json +++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Pi Existing Chrome Profile Bridge", - "version": "0.8.0", + "version": "0.9.1", "description": "Lets Pi control tabs in this existing Chrome profile via a local bridge at 127.0.0.1.", "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation"], "host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"], diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js index ed33470..60293c4 100644 --- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js +++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js @@ -43,6 +43,13 @@ async function pollLoop() { cache: "no-store", }); if (!response.ok) throw new Error(`bridge /next HTTP ${response.status}`); + const expected = response.headers.get("x-pi-chrome-version"); + const ours = chrome.runtime.getManifest().version; + if (expected && expected !== ours && isVersionOlder(ours, expected)) { + console.warn(`[pi-chrome] extension v${ours} behind pi-chrome v${expected}; reloading extension`); + try { chrome.runtime.reload(); } catch {} + return; + } const payload = await response.json(); if (payload.type === "command") await handleCommand(payload.command); } @@ -74,6 +81,18 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } +function isVersionOlder(a, b) { + const pa = String(a).split(".").map((n) => parseInt(n, 10) || 0); + const pb = String(b).split(".").map((n) => parseInt(n, 10) || 0); + const n = Math.max(pa.length, pb.length); + for (let i = 0; i < n; i++) { + const x = pa[i] ?? 0, y = pb[i] ?? 0; + if (x < y) return true; + if (x > y) return false; + } + return false; +} + async function dispatch(action, params) { switch (action) { case "tab.version": @@ -122,6 +141,8 @@ async function dispatch(action, params) { return executeActionInTab(params, fillPage, [params.selector ?? null, params.uid ?? null, params.text || "", params.submit === true]); case "page.key": return executeActionInTab(params, pressKeyInPage, [params.key]); + case "page.scroll": + return executeActionInTab(params, scrollPage, [params.selector ?? null, params.uid ?? null, params.deltaY ?? 0, params.deltaX ?? 0, params.steps ?? null]); case "page.console.list": return executeInTab(params, listConsoleMessages, [params.clear === true]); case "page.network.list": @@ -207,6 +228,15 @@ const HELPER_FUNCS = [ occluderAt, pageHash, pointerEventSequence, + sleepPage, + rand, + dispatchPointerLikeEvent, + humanMoveTo, + humanClickPoint, + printableKeyCode, + dispatchKeyEvent, + typeCharacter, + scrollPage, ]; async function executeInTab(params, func, args) { @@ -497,32 +527,94 @@ function pageHash() { return h; } +function sleepPage(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function rand(min, max) { + return min + Math.random() * (max - min); +} + +function dispatchPointerLikeEvent(element, type, x, y, prevX, prevY, opts = {}) { + const isPointer = type.startsWith("pointer"); + const Ctor = isPointer ? PointerEvent : MouseEvent; + const isMove = type === "pointermove" || type === "mousemove"; + const isUpOrClick = type === "pointerup" || type === "mouseup" || type === "click"; + const init = { + bubbles: true, + cancelable: true, + view: window, + clientX: x, + clientY: y, + screenX: x + (window.screenX || 0), + screenY: y + (window.screenY || 0), + movementX: Number.isFinite(prevX) ? x - prevX : 0, + movementY: Number.isFinite(prevY) ? y - prevY : 0, + button: 0, + buttons: isMove || isUpOrClick ? 0 : 1, + }; + if (isPointer) { + init.pointerType = "mouse"; + init.pointerId = 1; + init.isPrimary = true; + init.width = 1; + init.height = 1; + init.pressure = opts.pressure ?? (type === "pointerdown" ? 0.5 : 0); + init.tangentialPressure = 0; + init.tiltX = 0; + init.tiltY = 0; + } + const ev = new Ctor(type, init); + element.dispatchEvent(ev); + return ev.defaultPrevented; +} + function pointerEventSequence(element, x, y, sequence) { let defaultPrevented = false; + const state = getPiChromeState(); + const prevX = state.pointer?.x; + const prevY = state.pointer?.y; for (const type of sequence) { - const isPointer = type.startsWith("pointer"); - const Ctor = isPointer ? PointerEvent : MouseEvent; - const init = { - bubbles: true, - cancelable: true, - view: window, - clientX: x, - clientY: y, - button: 0, - buttons: type === "pointermove" || type === "mousemove" ? 0 : 1, - }; - if (isPointer) { - init.pointerType = "mouse"; - init.pointerId = 1; - init.isPrimary = true; - } - const ev = new Ctor(type, init); - element.dispatchEvent(ev); - if (ev.defaultPrevented) defaultPrevented = true; + defaultPrevented = dispatchPointerLikeEvent(element, type, x, y, prevX, prevY) || defaultPrevented; } + state.pointer = { x, y, t: performance.now() }; return defaultPrevented; } +async function humanMoveTo(x, y, steps) { + const state = getPiChromeState(); + const startX = Number.isFinite(state.pointer?.x) ? state.pointer.x : rand(12, Math.max(24, innerWidth - 12)); + const startY = Number.isFinite(state.pointer?.y) ? state.pointer.y : rand(12, Math.max(24, innerHeight - 12)); + const n = steps || Math.max(12, Math.min(42, Math.round(Math.hypot(x - startX, y - startY) / 18))); + let prevX = startX, prevY = startY; + let defaultPrevented = false; + for (let i = 1; i <= n; i++) { + const t = i / n; + const ease = t * t * (3 - 2 * t); + const wobble = Math.sin(t * Math.PI) * 8; + const px = startX + (x - startX) * ease + rand(-wobble, wobble); + const py = startY + (y - startY) * ease + rand(-wobble, wobble); + const el = document.elementFromPoint(px, py) || document.body || document.documentElement; + defaultPrevented = dispatchPointerLikeEvent(el, "pointermove", px, py, prevX, prevY) || defaultPrevented; + defaultPrevented = dispatchPointerLikeEvent(el, "mousemove", px, py, prevX, prevY) || defaultPrevented; + prevX = px; prevY = py; + await sleepPage(rand(4, 18)); + } + state.pointer = { x, y, t: performance.now() }; + return defaultPrevented; +} + +function humanClickPoint(point) { + if (!point.rect) return { x: point.x, y: point.y }; + const rect = point.rect; + const insetX = Math.min(rect.width * 0.35, Math.max(2, rect.width / 2 - 1)); + const insetY = Math.min(rect.height * 0.35, Math.max(2, rect.height / 2 - 1)); + return { + x: rect.left + rect.width / 2 + rand(-insetX, insetX), + y: rect.top + rect.height / 2 + rand(-insetY, insetY), + }; +} + function installPiChromeInstrumentation() { const state = getPiChromeState(); if (state.instrumentationInstalled) return; @@ -773,16 +865,31 @@ function resolvePoint(selector, uid, x, y) { return { element: document.elementFromPoint(x, y), x, y, rect: undefined }; } -function clickPage(selector, uid, x, y) { +async function clickPage(selector, uid, x, y) { installPiChromeInstrumentation(); const before = pageHash(); const point = resolvePoint(selector, uid, x, y); if (!point.element) throw new Error("No element at click point"); + const clickPoint = humanClickPoint(point); + point.x = clickPoint.x; + point.y = clickPoint.y; + point.element = document.elementFromPoint(point.x, point.y) || point.element; const visible = isElementVisible(point.element); const occluded = occluderAt(point.x, point.y, point.element); - const defaultPrevented = pointerEventSequence(point.element, point.x, point.y, [ - "pointerdown", "mousedown", "pointerup", "mouseup", "click", - ]); + let defaultPrevented = await humanMoveTo(point.x, point.y); + const state = getPiChromeState(); + const prevX = state.pointer?.x; + const prevY = state.pointer?.y; + defaultPrevented = dispatchPointerLikeEvent(point.element, "pointerdown", point.x, point.y, prevX, prevY, { pressure: 0.5 }) || defaultPrevented; + defaultPrevented = dispatchPointerLikeEvent(point.element, "mousedown", point.x, point.y, prevX, prevY) || defaultPrevented; + if (typeof point.element.focus === "function" && /^(A|BUTTON|INPUT|TEXTAREA|SELECT|SUMMARY)$/.test(point.element.tagName)) { + try { point.element.focus({ preventScroll: true }); } catch { try { point.element.focus(); } catch {} } + } + await sleepPage(rand(45, 140)); + defaultPrevented = dispatchPointerLikeEvent(point.element, "pointerup", point.x, point.y, prevX, prevY) || defaultPrevented; + defaultPrevented = dispatchPointerLikeEvent(point.element, "mouseup", point.x, point.y, prevX, prevY) || defaultPrevented; + defaultPrevented = dispatchPointerLikeEvent(point.element, "click", point.x, point.y, prevX, prevY) || defaultPrevented; + state.pointer = { x: point.x, y: point.y, t: performance.now() }; // Heuristic: if the clicked thing looks like a media play affordance and the page has paused // audio/video, the synthetic click may not unlock autoplay. Surface a warning. let autoplayHint; @@ -806,38 +913,136 @@ function clickPage(selector, uid, x, y) { }; } -function hoverPage(selector, uid, x, y) { +async function hoverPage(selector, uid, x, y) { installPiChromeInstrumentation(); const point = resolvePoint(selector, uid, x, y); if (!point.element) throw new Error("No element to hover"); - const defaultPrevented = pointerEventSequence(point.element, point.x, point.y, [ - "pointerover", "mouseover", "pointerenter", "mouseenter", "pointermove", "mousemove", - ]); + await humanMoveTo(point.x, point.y); + const state = getPiChromeState(); + const prevX = state.pointer?.x, prevY = state.pointer?.y; + let defaultPrevented = false; + for (const type of ["pointerover", "mouseover", "pointerenter", "mouseenter"]) { + defaultPrevented = dispatchPointerLikeEvent(point.element, type, point.x, point.y, prevX, prevY) || defaultPrevented; + } + // Small dwell so hover-intent handlers fire. + await sleepPage(rand(80, 220)); return { x: point.x, y: point.y, selector, uid, tag: point.element.tagName, defaultPrevented, isTrusted: false }; } -function dragPage(fromUid, fromSelector, fromX, fromY, toUid, toSelector, toX, toY, steps) { +async function dragPage(fromUid, fromSelector, fromX, fromY, toUid, toSelector, toX, toY, steps) { installPiChromeInstrumentation(); const before = pageHash(); const from = resolvePoint(fromSelector, fromUid, fromX, fromY); const to = resolvePoint(toSelector, toUid, toX, toY); if (!from.element) throw new Error("Drag source element not found"); if (!to.element) throw new Error("Drag target element not found"); - pointerEventSequence(from.element, from.x, from.y, ["pointerover", "pointerdown", "mousedown"]); - for (let i = 1; i <= steps; i++) { - const t = i / steps; - const x = from.x + (to.x - from.x) * t; - const y = from.y + (to.y - from.y) * t; + // Move to source. + await humanMoveTo(from.x, from.y); + const state = getPiChromeState(); + let prevX = state.pointer?.x, prevY = state.pointer?.y; + // Build a shared DataTransfer so HTML5 drag-and-drop handlers can populate / read it. + const dt = new DataTransfer(); + const dragInit = (type, target, x, y) => { + const ev = new DragEvent(type, { + bubbles: true, cancelable: true, composed: true, + clientX: x, clientY: y, + screenX: x + (window.screenX || 0), screenY: y + (window.screenY || 0), + button: 0, buttons: 1, view: window, + dataTransfer: dt, + }); + target.dispatchEvent(ev); + return ev; + }; + dispatchPointerLikeEvent(from.element, "pointerover", from.x, from.y, prevX, prevY); + dispatchPointerLikeEvent(from.element, "pointerdown", from.x, from.y, prevX, prevY, { pressure: 0.5 }); + dispatchPointerLikeEvent(from.element, "mousedown", from.x, from.y, prevX, prevY); + await sleepPage(rand(40, 110)); + dragInit("dragstart", from.element, from.x, from.y); + dragInit("drag", from.element, from.x, from.y); + let lastOver = from.element; + const n = steps || 18; + for (let i = 1; i <= n; i++) { + const t = i / n; + const ease = t * t * (3 - 2 * t); + const wobble = Math.sin(t * Math.PI) * 6; + const x = from.x + (to.x - from.x) * ease + rand(-wobble, wobble); + const y = from.y + (to.y - from.y) * ease + rand(-wobble, wobble); const overEl = document.elementFromPoint(x, y) || to.element; - pointerEventSequence(overEl, x, y, ["pointermove", "mousemove"]); + dispatchPointerLikeEvent(overEl, "pointermove", x, y, prevX, prevY); + dispatchPointerLikeEvent(overEl, "mousemove", x, y, prevX, prevY); + if (overEl !== lastOver) { + dragInit("dragleave", lastOver, x, y); + dragInit("dragenter", overEl, x, y); + lastOver = overEl; + } + dragInit("dragover", overEl, x, y); + dragInit("drag", from.element, x, y); + prevX = x; prevY = y; + await sleepPage(rand(8, 26)); } - pointerEventSequence(to.element, to.x, to.y, ["pointerover", "mouseover", "pointerup", "mouseup"]); + dispatchPointerLikeEvent(to.element, "pointerover", to.x, to.y, prevX, prevY); + dispatchPointerLikeEvent(to.element, "mouseover", to.x, to.y, prevX, prevY); + dragInit("drop", to.element, to.x, to.y); + dragInit("dragend", from.element, to.x, to.y); + dispatchPointerLikeEvent(to.element, "pointerup", to.x, to.y, prevX, prevY); + dispatchPointerLikeEvent(to.element, "mouseup", to.x, to.y, prevX, prevY); + state.pointer = { x: to.x, y: to.y, t: performance.now() }; return { from: { x: from.x, y: from.y }, to: { x: to.x, y: to.y }, - steps, + steps: n, + pageMutated: pageHash() !== before, + note: "Synthetic drag with HTML5 DragEvent + shared DataTransfer. isTrusted is still false.", + }; +} + +async function scrollPage(selector, uid, deltaY, deltaX, steps) { + installPiChromeInstrumentation(); + const before = pageHash(); + let target; + if (selector || uid) { + target = elementBySelectorOrUid(selector, uid); + } else { + target = document.scrollingElement || document.documentElement || document.body; + } + if (!target) throw new Error("No scroll target"); + const rect = target.getBoundingClientRect ? target.getBoundingClientRect() : { left: 0, top: 0, width: innerWidth, height: innerHeight }; + const cx = Math.max(0, Math.min(innerWidth - 1, rect.left + Math.min(rect.width, innerWidth) / 2)); + const cy = Math.max(0, Math.min(innerHeight - 1, rect.top + Math.min(rect.height, innerHeight) / 2)); + const n = Math.max(3, Math.min(40, steps || Math.max(3, Math.ceil(Math.abs(deltaY || 0) / 100)))); + // Front-loaded wheel deltas, momentum-style. + const totalY = deltaY || 0; + const totalX = deltaX || 0; + const weights = []; + for (let i = 1; i <= n; i++) weights.push(1 / i); + const sumW = weights.reduce((a, b) => a + b, 0); + let movedY = 0, movedX = 0; + for (let i = 0; i < n; i++) { + const dy = totalY * (weights[i] / sumW); + const dx = totalX * (weights[i] / sumW); + const ev = new WheelEvent("wheel", { + bubbles: true, cancelable: true, composed: true, view: window, + clientX: cx, clientY: cy, + deltaX: dx, deltaY: dy, deltaMode: 0, + }); + target.dispatchEvent(ev); + if (!ev.defaultPrevented) { + // Apply scroll ourselves; mirrors what the browser would do. + if (target === document.scrollingElement || target === document.documentElement || target === document.body) { + window.scrollBy({ left: dx, top: dy, behavior: "instant" }); + } else { + target.scrollTop += dy; + target.scrollLeft += dx; + } + } + movedY += dy; movedX += dx; + await sleepPage(rand(12, 28)); + } + return { + deltaX: movedX, deltaY: movedY, steps: n, + scrollTop: target.scrollTop, scrollLeft: target.scrollLeft, pageMutated: pageHash() !== before, - note: "Synthetic pointer drag. HTML5 DataTransfer is not synthesized; native drag-and-drop targets may not respond.", + isTrusted: false, }; } @@ -871,23 +1076,90 @@ function setNativeValue(element, value) { else element.value = value; } -function typeIntoPage(selector, uid, text, pressEnter) { - installPiChromeInstrumentation(); - const before = pageHash(); - let element = elementBySelectorOrUid(selector, uid) || document.activeElement; - if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element"); - element.focus(); +function printableKeyCode(ch) { + if (ch === " ") return 32; + const upper = ch.toUpperCase(); + if (/^[A-Z]$/.test(upper)) return upper.charCodeAt(0); + if (/^[0-9]$/.test(ch)) return ch.charCodeAt(0); + return ch.charCodeAt(0) || 0; +} + +function dispatchKeyEvent(element, type, key, mods = {}) { + const code = key.length === 1 && /^[a-z]$/i.test(key) ? `Key${key.toUpperCase()}` : + key.length === 1 && /^[0-9]$/.test(key) ? `Digit${key}` : + key === " " ? "Space" : key; + const SPECIAL = { Enter: 13, Tab: 9, Backspace: 8, Delete: 46, Escape: 27, + ArrowLeft: 37, ArrowUp: 38, ArrowRight: 39, ArrowDown: 40, " ": 32, Shift: 16, Control: 17, Alt: 18, Meta: 91 }; + const keyCode = key.length === 1 ? printableKeyCode(key) : (SPECIAL[key] ?? 0); + const ev = new KeyboardEvent(type, { + key, + code, + keyCode, + which: keyCode, + charCode: type === "keypress" && key.length === 1 ? key.charCodeAt(0) : 0, + shiftKey: !!mods.shiftKey, + ctrlKey: !!mods.ctrlKey, + altKey: !!mods.altKey, + metaKey: !!mods.metaKey, + bubbles: true, + cancelable: true, + composed: true, + view: window, + }); + element.dispatchEvent(ev); + return ev; +} + +async function typeCharacter(element, ch) { + const needShift = ch.length === 1 && (/^[A-Z]$/.test(ch) || "~!@#$%^&*()_+{}|:\"<>?".includes(ch)); + if (needShift) { + dispatchKeyEvent(element, "keydown", "Shift", { shiftKey: true }); + await sleepPage(rand(8, 24)); + } + const mods = { shiftKey: needShift }; + const down = dispatchKeyEvent(element, "keydown", ch, mods); + if (down.defaultPrevented) { + if (needShift) dispatchKeyEvent(element, "keyup", "Shift", { shiftKey: false }); + return { defaultPrevented: true }; + } + if (ch.length === 1) dispatchKeyEvent(element, "keypress", ch, mods); + if (element.isContentEditable) { - document.execCommand("insertText", false, text); + // execCommand("insertText") fires its own beforeinput + input. Don't double-dispatch. + document.execCommand("insertText", false, ch); } else if ("value" in element) { const start = element.selectionStart ?? element.value.length; const end = element.selectionEnd ?? element.value.length; - setNativeValue(element, element.value.slice(0, start) + text + element.value.slice(end)); - element.selectionStart = element.selectionEnd = start + text.length; - dispatchInputEvents(element, text, "insertText"); + const next = element.value.slice(0, start) + ch + element.value.slice(end); + const before = new InputEvent("beforeinput", { bubbles: true, cancelable: true, inputType: "insertText", data: ch }); + element.dispatchEvent(before); + if (!before.defaultPrevented) { + setNativeValue(element, next); + try { element.selectionStart = element.selectionEnd = start + ch.length; } catch {} + element.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "insertText", data: ch })); + } } else { throw new Error("Focused element is not text-editable"); } + + await sleepPage(rand(25, 95)); + dispatchKeyEvent(element, "keyup", ch, mods); + if (needShift) { + await sleepPage(rand(5, 18)); + dispatchKeyEvent(element, "keyup", "Shift", { shiftKey: false }); + } + await sleepPage(rand(35, 140)); + return { defaultPrevented: false }; +} + +async function typeIntoPage(selector, uid, text, pressEnter) { + installPiChromeInstrumentation(); + const before = pageHash(); + let element = elementBySelectorOrUid(selector, uid) || document.activeElement; + if (!element) throw new Error(selector || uid ? `No element for ${selector || uid}` : "No active element"); + element.focus(); + if (!(element.isContentEditable || "value" in element)) throw new Error("Focused element is not text-editable"); + for (const ch of Array.from(text)) await typeCharacter(element, ch); if (pressEnter) pressKeyInPage("Enter"); return { selector, uid, length: text.length, pressEnter, @@ -923,14 +1195,48 @@ function fillPage(selector, uid, text, submit) { }; } -function pressKeyInPage(key) { +async function pressKeyInPage(key) { const normalized = normalizeKey(key); const target = document.activeElement || document.body; - const before = (typeof pageHash === "function") ? pageHash() : 0; - const down = new KeyboardEvent("keydown", { key: normalized, bubbles: true, cancelable: true }); - target.dispatchEvent(down); - const up = new KeyboardEvent("keyup", { key: normalized, bubbles: true, cancelable: true }); - target.dispatchEvent(up); + const before = pageHash(); + const down = dispatchKeyEvent(target, "keydown", normalized); + if (normalized.length === 1) dispatchKeyEvent(target, "keypress", normalized); + // Character insertion for printable keys when focus is in an editable. + if (normalized.length === 1 && !down.defaultPrevented && (target.isContentEditable || ("value" in target && (target.tagName === "INPUT" || target.tagName === "TEXTAREA")))) { + if (target.isContentEditable) { + const bi = new InputEvent("beforeinput", { bubbles: true, cancelable: true, inputType: "insertText", data: normalized }); + target.dispatchEvent(bi); + if (!bi.defaultPrevented) { + document.execCommand("insertText", false, normalized); + target.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "insertText", data: normalized })); + } + } else { + const start = target.selectionStart ?? target.value.length; + const end = target.selectionEnd ?? target.value.length; + const bi = new InputEvent("beforeinput", { bubbles: true, cancelable: true, inputType: "insertText", data: normalized }); + target.dispatchEvent(bi); + if (!bi.defaultPrevented) { + setNativeValue(target, target.value.slice(0, start) + normalized + target.value.slice(end)); + try { target.selectionStart = target.selectionEnd = start + 1; } catch {} + target.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "insertText", data: normalized })); + } + } + } else if (normalized === "Backspace" && "value" in target) { + const start = target.selectionStart ?? target.value.length; + const end = target.selectionEnd ?? target.value.length; + if (start > 0 || end > start) { + const from = start === end ? start - 1 : start; + const bi = new InputEvent("beforeinput", { bubbles: true, cancelable: true, inputType: "deleteContentBackward" }); + target.dispatchEvent(bi); + if (!bi.defaultPrevented) { + setNativeValue(target, target.value.slice(0, from) + target.value.slice(end)); + try { target.selectionStart = target.selectionEnd = from; } catch {} + target.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "deleteContentBackward" })); + } + } + } + await sleepPage(rand(25, 95)); + const up = dispatchKeyEvent(target, "keyup", normalized); if (normalized === "Enter") { const form = target.closest?.("form"); if (form) form.requestSubmit?.(); @@ -939,7 +1245,7 @@ function pressKeyInPage(key) { key: normalized, isTrusted: false, defaultPrevented: down.defaultPrevented || up.defaultPrevented, - pageMutated: (typeof pageHash === "function") ? pageHash() !== before : undefined, + pageMutated: pageHash() !== before, }; } diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts index ac176bc..8117d78 100644 --- a/extensions/chrome-profile-bridge/index.ts +++ b/extensions/chrome-profile-bridge/index.ts @@ -1,7 +1,7 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { StringEnum } from "@earendil-works/pi-ai"; import { Type } from "typebox"; -import { existsSync, statSync } from "node:fs"; +import { existsSync, readFileSync, statSync } from "node:fs"; import { mkdir, writeFile } from "node:fs/promises"; import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; import { dirname, join, resolve } from "node:path"; @@ -46,7 +46,15 @@ type BridgeResult = { error?: string; }; -const PI_CHROME_VERSION = "0.8.0"; +const PI_CHROME_PKG_PATH = resolve(__dirname, "..", "..", "package.json"); +function readPiChromeVersion(): string { + try { + const pkg = JSON.parse(readFileSync(PI_CHROME_PKG_PATH, "utf8")) as { version?: string }; + if (pkg.version) return pkg.version; + } catch {} + return "0.0.0-dev"; +} +const PI_CHROME_VERSION = readPiChromeVersion(); const PI_CHROME_GLOBAL_KEY = "__piChromeProfileBridgeLoaded__"; const DEFAULT_HOST = process.env.PI_CHROME_BRIDGE_HOST ?? "127.0.0.1"; const DEFAULT_PORT = Number(process.env.PI_CHROME_BRIDGE_PORT ?? "17318"); @@ -118,13 +126,14 @@ function readRequestBody(request: IncomingMessage): Promise<string> { }); } -function sendJson(response: ServerResponse, status: number, body: unknown): void { +function sendJson(response: ServerResponse, status: number, body: unknown, extraHeaders?: Record<string, string>): void { response.writeHead(status, { "content-type": "application/json; charset=utf-8", "access-control-allow-origin": "*", "access-control-allow-methods": "GET,POST,OPTIONS", "access-control-allow-headers": "content-type", "cache-control": "no-store", + ...(extraHeaders ?? {}), }); response.end(JSON.stringify(body)); } @@ -316,7 +325,16 @@ class ChromeProfileBridge { if (command) this.queue.unshift(command); return; } - sendJson(response, 200, command ? { type: "command", command } : { type: "none" }); + // Re-read version on every /next so bumping package.json takes effect without pi restart. + const currentVersion = readPiChromeVersion(); + sendJson( + response, + 200, + command + ? { type: "command", command, expectedExtensionVersion: currentVersion } + : { type: "none", expectedExtensionVersion: currentVersion }, + { "x-pi-chrome-version": currentVersion }, + ); return; } if (request.method === "POST" && url.pathname === "/result") { @@ -438,6 +456,7 @@ Usage rules: const status = bridge.status(); lines.push(`• Local bridge: mode=${status.mode}, url=${status.url}`); let extensionAlive = false; + let versionMismatch = false; try { const started = Date.now(); const version = (await bridge.send("tab.version", {}, 35_000)) as { @@ -447,11 +466,16 @@ Usage rules: }; const latencyMs = Date.now() - started; extensionAlive = true; - lines.push(`✓ Companion Chrome extension responding (ID: ${version.extensionId ?? "?"}, ext v${version.extensionVersion ?? "?"}, latency ${latencyMs}ms)`); if (version.extensionVersion && version.extensionVersion !== PI_CHROME_VERSION) { + versionMismatch = true; lines.push( - `⚠ Extension version (${version.extensionVersion}) differs from pi-chrome (${PI_CHROME_VERSION}). Reload "Pi Existing Chrome Profile Bridge" in chrome://extensions to pick up the latest service worker.`, + `✗ EXTENSION VERSION MISMATCH: companion extension is v${version.extensionVersion}, but pi-chrome is v${PI_CHROME_VERSION}.`, + ` All chrome_* tools will run with the OLD extension code until this is fixed.`, + ` Fix: open chrome://extensions and click reload on "Pi Existing Chrome Profile Bridge".`, + ` (Future version drifts will self-heal: the extension now polls pi-chrome's expected version and reloads itself.)`, ); + } else { + lines.push(`✓ Companion Chrome extension responding (ID: ${version.extensionId ?? "?"}, ext v${version.extensionVersion ?? "?"}, latency ${latencyMs}ms)`); } } catch (error) { const message = (error as Error).message; @@ -463,7 +487,7 @@ Usage rules: } } - if (extensionAlive) { + if (extensionAlive && !versionMismatch) { // MAIN-world evaluate probe. try { const value = await bridge.send("page.evaluate", { expression: "1+1", awaitPromise: true, foreground: false }, 10_000); @@ -481,6 +505,8 @@ Usage rules: } catch (error) { lines.push(`⚠ page.probe failed: ${(error as Error).message}`); } + } else if (versionMismatch) { + lines.push(`… Skipped MAIN-world capability checks because the loaded extension is stale.`); } // CDP availability hint. @@ -990,7 +1016,7 @@ Usage rules: pi.registerTool({ name: "chrome_drag", label: "Chrome Drag", - description: "Synthetic pointer drag from one uid/selector/point to another. Dispatches pointerdown → multi-step pointermove → pointerup. Note: HTML5 DataTransfer is NOT synthesized, so native HTML5 drag-and-drop targets may not respond.", + description: "Synthetic drag from one uid/selector/point to another. Dispatches pointerdown → humanised pointermove path → dragstart/drag/dragenter/dragover/dragleave/drop/dragend with a shared HTML5 DataTransfer, then pointerup. isTrusted=false.", promptSnippet: "Drag a Chrome element from one point to another.", parameters: Type.Object({ fromUid: Type.Optional(Type.String()), @@ -1014,6 +1040,28 @@ Usage rules: }); pi.registerTool({ + name: "chrome_scroll", + label: "Chrome Scroll", + description: "Scroll the page or a specific scrollable element by dispatching real wheel events with momentum-shaped deltas, then applying the scroll. Positive deltaY scrolls down. Pass uid/selector to scroll within a container, otherwise the document scrolls.", + promptSnippet: "Scroll a Chrome page or container via wheel events (not raw scrollTop).", + parameters: Type.Object({ + uid: Type.Optional(Type.String()), + selector: Type.Optional(Type.String()), + deltaY: Type.Optional(Type.Number({ description: "Pixels to scroll vertically. Positive = down." })), + deltaX: Type.Optional(Type.Number({ description: "Pixels to scroll horizontally. Positive = right." })), + steps: Type.Optional(Type.Number({ description: "Number of wheel events to dispatch. Defaults to ceil(|deltaY|/100)." })), + targetId: Type.Optional(Type.String()), + urlIncludes: Type.Optional(Type.String()), + titleIncludes: Type.Optional(Type.String()), + background: Type.Optional(Type.Boolean()), + }), + async execute(_id, params): Promise<ToolTextResult> { + const result = await bridge.send("page.scroll", withBackground(params), DEFAULT_TIMEOUT_MS); + return { content: [{ type: "text", text: `Scrolled dy=${params.deltaY ?? 0} dx=${params.deltaX ?? 0}` }], details: { result: result as Json } }; + }, + }); + + pi.registerTool({ name: "chrome_upload_file", label: "Chrome Upload File", description: "Programmatically set the files of an <input type=file> element from local file paths. Uses DataTransfer to populate input.files and dispatches input+change events. Does NOT open the native file picker; works with React/Vue/Angular controlled inputs.", |
