diff options
20 files changed, 1654 insertions, 155 deletions
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json index aa7bfae..f0f8a6a 100644 --- a/extensions/chrome-profile-bridge/browser-extension/manifest.json +++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Pi Chrome Connector", - "version": "0.14.1", + "version": "0.14.2", "description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.", "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation", "debugger"], "host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"], diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js index d256f1c..36e5b20 100644 --- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js +++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js @@ -185,12 +185,72 @@ function cdpRaw(tabId, method, params) { // chrome.debugger.attach can stay cached in attachedTabs even after Chrome killed // the session (tab nav, devtools opened/closed, etc). Recover by detaching the // stale entry and re-attaching, then retry the command once. +// Find foreign chrome-extension targets currently anchored to the tab. Password managers, +// autofill helpers, and other input-attached extensions create type:"other" CDP targets +// whose URL is chrome-extension://<otherId>/... When that target is in focus, CDP refuses +// our Input.dispatchMouseEvent calls with "Cannot access a chrome-extension:// URL of +// different extension" — surfacing a cryptic error to the user. +async function findForeignExtensionTargets() { + try { + const targets = await new Promise((resolve) => chrome.debugger.getTargets((t) => resolve(t || []))); + return targets.filter((t) => { + const url = String(t.url || ""); + if (!url.startsWith("chrome-extension://")) return false; + if (t.extensionId === chrome.runtime.id) return false; + return true; + }); + } catch { + return []; + } +} + +function extractForeignExtId(targets) { + for (const t of targets) { + if (t.extensionId && t.extensionId !== chrome.runtime.id) return t.extensionId; + const m = String(t.url || "").match(/chrome-extension:\/\/([a-p]+)\//); + if (m && m[1] !== chrome.runtime.id) return m[1]; + } + return null; +} + +async function dismissOverlayViaEscape(tabId) { + // Esc routes through key dispatcher (target-by-focus), not by mouse coordinates, so it + // works even when a foreign chrome-extension popup is intercepting pointer events. + try { + await cdpRaw(tabId, "Input.dispatchKeyEvent", { type: "keyDown", key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 }); + await cdpRaw(tabId, "Input.dispatchKeyEvent", { type: "keyUp", key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 }); + await sleep(120); + } catch {} +} + async function cdp(tabId, method, params) { try { return await cdpRaw(tabId, method, params); } catch (error) { const msg = String(error?.message || error); const isStale = /Debugger is not attached|Detached while|Target closed|No tab with id/i.test(msg); + const isForeignExtBlock = /Cannot access a chrome-extension:\/\/ URL of different extension/i.test(msg); + if (isForeignExtBlock && /Input\./.test(method)) { + // Foreign chrome-extension popup (autofill, password manager) is hijacking input. + // Try once: dismiss via Esc, then retry. + const before = await findForeignExtensionTargets(); + recordAttachEvent({ kind: "foreign-ext-detected", tabId, method, foreignExtId: extractForeignExtId(before), targetCount: before.length }); + await dismissOverlayViaEscape(tabId); + try { + return await cdpRaw(tabId, method, params); + } catch (retryErr) { + const retryMsg = String(retryErr?.message || retryErr); + if (/Cannot access a chrome-extension:\/\/ URL of different extension/i.test(retryMsg)) { + const after = await findForeignExtensionTargets(); + const id = extractForeignExtId(after) || extractForeignExtId(before) || "unknown"; + throw new Error( + `Another Chrome extension (${id}) has an input overlay on this page (e.g. a password manager / autofill popup). \n` + + `pi-chrome tried to dismiss it with Escape but it reappeared. Disable that extension on this page, focus the field via Tab instead of clicking, or run /chrome quiet off so the agent uses synthetic input here.`, + ); + } + throw retryErr; + } + } if (!isStale) throw error; attachedTabs.delete(tabId); await chrome.debugger.attach({ tabId }, CDP_VERSION).catch(() => undefined); diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts index 2fa51b0..2923040 100644 --- a/extensions/chrome-profile-bridge/index.ts +++ b/extensions/chrome-profile-bridge/index.ts @@ -1,6 +1,5 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { getSettingsListTheme } from "@earendil-works/pi-coding-agent"; -import { StringEnum } from "@earendil-works/pi-ai"; import { Container, type SettingItem, SettingsList, Text } from "@earendil-works/pi-tui"; import { Type } from "typebox"; import { existsSync, readFileSync, statSync } from "node:fs"; @@ -381,6 +380,10 @@ const tabActionValues = ["list", "new", "activate", "close", "version"] as const const imageFormatValues = ["png", "jpeg"] as const; const waitForValues = ["selector", "expression"] as const; +function StringEnum<T extends readonly [string, ...string[]]>(values: T) { + return Type.Union(values.map((value) => Type.Literal(value)) as [ReturnType<typeof Type.Literal>, ...ReturnType<typeof Type.Literal>[]]); +} + export default function (pi: ExtensionAPI): void { const globalState = globalThis as typeof globalThis & { [PI_CHROME_GLOBAL_KEY]?: { version: string; root: string }; diff --git a/package.json b/package.json index e8354ae..dcaaa0e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-chrome", - "version": "0.14.1", + "version": "0.14.2", "description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).", "keywords": [ "pi-package", @@ -24,8 +24,15 @@ ] }, "peerDependencies": { - "@earendil-works/pi-ai": "*", "@earendil-works/pi-coding-agent": "*", "typebox": "*" + }, + "peerDependenciesMeta": { + "@earendil-works/pi-coding-agent": { + "optional": true + }, + "typebox": { + "optional": true + } } } diff --git a/test-suite/README.md b/test-suite/README.md index 241976c..bb8becb 100644 --- a/test-suite/README.md +++ b/test-suite/README.md @@ -1,8 +1,13 @@ -# pi-chrome anti-automation test suite +# pi-chrome browser-control benchmark -Static pages that each block a different automated-interaction signal. Drive them -with the `chrome_*` tools to discover where the companion-extension's synthetic -events still look like a bot, and to verify any humanization fixes. +Static benchmark pages for evaluating tools that let agents control Chrome. The +suite covers two related questions: + +1. **Can the agent complete normal browser work?** Forms, scroll containers, + contenteditable, files, frames, Shadow DOM, network/console inspection. +2. **Does the interaction look like a human/browser action when sites care?** + `isTrusted`, user activation, pointer paths, key cadence, native controls, + drag/drop, touch, paste, and scroll momentum. ## Run @@ -14,61 +19,120 @@ python3 -m http.server 8765 Each challenge page exposes: -- `window.__challenge` — short id -- `window.__verdict` — `"PENDING" | "PASS" | "FAIL"` -- `window.__reason` — array of strings, why it failed (or how it passed) +- `window.__challenge` — id +- `window.__verdict` — `"PENDING" | "PASS" | "FAIL" | "SKIP" | "WARN"` +- `window.__reason` — array of reasons - `window.__events` — raw event log for forensics -`PASS` means the page believes a real human did the thing. -`FAIL` means the page caught the automation. - -## Aggregate runner - -`index.html` lists every challenge, loads each one in an iframe, lets you (or -the agent) drive the interaction, and tallies verdicts. - -Recommended agent flow: - -1. `chrome_navigate` to `http://127.0.0.1:8765/`. -2. For each challenge link, open it, run the listed interaction with the - `chrome_*` tools, then read `window.__verdict` / `window.__reason` via - `chrome_evaluate`. -3. Compare against the manual baseline (do the same interaction by hand and - confirm `PASS`). - -## Challenges - -| id | file | what it blocks | -|----|------|----------------| -| `is-trusted-click` | `challenges/01-is-trusted-click.html` | click handler ignores `event.isTrusted === false` | -| `is-trusted-keyboard` | `challenges/02-is-trusted-keyboard.html` | input rejects if `keydown.isTrusted` false | -| `webdriver-flag` | `challenges/03-webdriver-flag.html` | `navigator.webdriver` truthy / Chrome runtime quirks | -| `mouse-entropy` | `challenges/04-mouse-entropy.html` | requires organic mousemove path before click | -| `event-timing` | `challenges/05-event-timing.html` | rejects when pointerdown→pointerup gap < N ms or always-equal | -| `click-coordinates` | `challenges/06-click-coordinates.html` | rejects clicks that land on exact element center | -| `pointer-properties` | `challenges/07-pointer-properties.html` | demands pressure / non-zero movementX/Y / pointerId variety | -| `keyboard-cadence` | `challenges/08-keyboard-cadence.html` | rejects same-tick `keydown`/`keyup` and missing per-char events | -| `composition-input` | `challenges/09-composition-input.html` | listens for native `keydown` + `keypress` + `input` sequence | -| `user-activation` | `challenges/10-user-activation.html` | feature gates: clipboard.writeText, fullscreen | -| `honeypot` | `challenges/11-honeypot.html` | hidden field must stay empty | -| `fingerprint` | `challenges/12-fingerprint.html` | UA-CH, languages, plugins, permissions consistency | -| `focus-order` | `challenges/13-focus-order.html` | requires `pointerdown`-driven focus, not direct `.focus()` | -| `wheel-scroll` | `challenges/14-wheel-scroll.html` | rejects scrollTop jumps without `wheel`/`scroll` events | -| `drag-drop-datatransfer` | `challenges/15-drag-drop-datatransfer.html` | requires full HTML5 drag cycle with populated `DataTransfer` | -| `contenteditable-selection` | `challenges/16-contenteditable-selection.html` | requires `selectionchange` + monotonic caret advance per keystroke | -| `paste-clipboard` | `challenges/17-paste-clipboard.html` | demands real OS paste with `clipboardData` + `inputType=insertFromPaste` | -| `native-select` | `challenges/18-native-select.html` | rejects `<option>` clicks / `.value=` assignment; needs trusted picker change | -| `hover-dwell` | `challenges/19-hover-dwell.html` | reveal gated on ≥600ms hover with pointermove activity, latency after reveal | -| `react-value-tracker` | `challenges/20-react-value-tracker.html` | input must mutate via native `value` setter so React's `_valueTracker` is stale | -| `keyboard-modifiers` | `challenges/21-keyboard-modifiers.html` | Shift+a chord: modifier ordering, `shiftKey`, `code=KeyA`, `getModifierState` | -| `touch-events` | `challenges/22-touch-events.html` | real `TouchEvent`+`Touch` (radius, force) — synthetic pointerType=touch fails | -| `stack-trace-fingerprint` | `challenges/23-stack-trace-fingerprint.html` | inspects handler call stack for extension/eval/Runtime.evaluate frames | -| `viewport-edge-clicks` | `challenges/24-viewport-edge-clicks.html` | rejects pointer events with clientX/Y outside viewport or (0,0) default | -| `pointer-continuity` | `challenges/25-pointer-continuity.html` | two clicks far apart need bridging pointermoves with mid-span samples | -| `mousemove-rate` | `challenges/26-mousemove-rate.html` | mousemove Δt distribution must match ~60–250Hz with jitter | -| `scroll-momentum` | `challenges/27-scroll-momentum.html` | wheel ΔY trace must show peak + decaying momentum tail ≥100ms | -| `intersection-visibility` | `challenges/28-intersection-visibility.html` | IntersectionObserver gradient + rAF scroll frames — no teleport | - -## What each `PASS` would require in `service_worker.js` - -See `notes/bypass-ideas.md`. +`manifest.json` is the source of truth for benchmark metadata: category, goal, +expected result per mode, prerequisites, flake risk, manual baseline status, and +canonical tool recipe. `manifest.schema.json` documents the manifest shape. +Recipes express tool intent; runners may need to adapt descriptive selectors +(e.g. shadow/iframe notation) and expand path placeholders like `$PWD`. + +## Modes / expected outcomes + +The same page can have different expected results depending on tool capability: + +- `synthetic` — DOM-dispatched events / framework-aware setters. Fast and quiet. +- `trusted` — browser-trusted input, usually via `chrome.debugger`/CDP. Can show + Chrome's debugging banner. +- `manual` — human baseline in same browser/profile. + +Expected values in `manifest.json`: + +- `PASS` / `FAIL` — deterministic target for that mode. +- `CONDITIONAL` — depends on browser policy, OS, device capability, permissions, + or an unreleased tool primitive. Inspect `prerequisites`, `notes`, and + `flakeRisk`. + +Manual baselines are tracked separately with `manualBaseline`. `unverified` +means the manual expectation is a target, not a recorded contract. + +## Recommended agent flow + +1. Navigate to dashboard: + `http://127.0.0.1:8765/`. +2. Pick mode (`synthetic`, `trusted`, or `manual`) and clear local verdicts. +3. For each manifest row: + - `chrome_navigate` to `http://127.0.0.1:8765/<file>`. + - `chrome_snapshot` before acting; prefer snapshot `uid` over raw selector. + - Execute the listed `recipe`, adapting descriptive frame/shadow selectors to + whatever selectors/uids the tool exposes. + - Read: + ```js + JSON.stringify({ + v: window.__verdict, + r: window.__reason, + e: window.__events?.slice(-20) + }) + ``` +4. Return to dashboard and compare actual verdicts with expected values. +5. Copy JSON report from dashboard for PRs or regression notes. + +## Challenge categories + +- `trusted-input` — browser-trusted click/key events. +- `pointer-humanization` — paths, coordinates, movement continuity/rate. +- `keyboard` / `focus-keyboard` — typing fidelity, modifiers, Tab flows. +- `activation-gates` — clipboard/fullscreen/user activation. +- `scroll` / `scroll-visibility` — wheel events, momentum, IntersectionObserver. +- `drag-drop` — HTML5 drag/drop + `DataTransfer`. +- `clipboard` — OS/browser paste path. +- `native-controls` — controls that should use browser UI/keyboard semantics. +- `frameworks` / `editing` — React-style value tracking, contenteditable. +- `dom-complexity` / `frames` — Shadow DOM and iframe targeting. +- `files` — file attachment to `<input type=file>`. +- `observability` — console/network capture tools. +- `fingerprint` — environment and stack fingerprint probes. +- `agent-safety` — hidden honeypots and safe target selection. + +## Current challenge inventory + +The dashboard renders this from `manifest.json`. In brief: + +1. trusted click +2. trusted keyboard +3. webdriver/runtime flags +4. mouse entropy before click +5. click timing +6. click coordinate variation +7. pointer event properties +8. keyboard cadence +9. beforeinput/input order +10. user activation gates +11. honeypot safety +12. fingerprint consistency +13. focus order +14. wheel scroll +15. drag/drop `DataTransfer` +16. contenteditable selection +17. paste clipboard +18. native select +19. hover dwell +20. React value tracker +21. keyboard modifiers +22. touch events +23. stack trace fingerprint +24. viewport click coordinates +25. pointer continuity +26. mousemove rate +27. scroll momentum +28. intersection visibility +29. Shadow DOM controls +30. iframe targeting +31. file upload +32. keyboard Tab navigation +33. network/console capture + +## Design notes + +- A failure is useful only when compared to expected mode. Example: synthetic + `isTrusted` failing is expected and validates that the test detects quiet DOM + events. +- Some tests are capability-gated. Example: touch tests should be `SKIP`/manual + conditional on non-touch hardware. +- Fingerprint tests should warn before blocking. Real Chrome profiles can use + software WebGL in VMs, remote desktops, or policy-constrained environments. +- `notes/bypass-ideas.md` is historical guidance for older synthetic-only + versions. Prefer `manifest.json` for current expected outcomes. diff --git a/test-suite/_lib.js b/test-suite/_lib.js index c6d121c..95d6393 100644 --- a/test-suite/_lib.js +++ b/test-suite/_lib.js @@ -17,7 +17,9 @@ el.dataset.verdict = state.verdict; el.style.background = state.verdict === "PASS" ? "#1f7a1f" : - state.verdict === "FAIL" ? "#a11" : "#444"; + state.verdict === "FAIL" ? "#a11" : + state.verdict === "SKIP" ? "#76520b" : + state.verdict === "WARN" ? "#6b5d00" : "#444"; el.style.color = "#fff"; const r = document.getElementById("__reason"); if (r) r.textContent = state.reason.join("\n"); @@ -64,6 +66,22 @@ document.title = `[FAIL] ${state.id}`; persist(); render(); }, + skip(...reasons) { + if (state.verdict === "FAIL" || state.verdict === "PASS") return; + state.verdict = "SKIP"; + state.reason.push(...reasons.map((r) => "↷ " + r)); + window.__verdict = state.verdict; + document.title = `[SKIP] ${state.id}`; + persist(); render(); + }, + warn(...reasons) { + if (state.verdict === "FAIL" || state.verdict === "PASS") return; + state.verdict = "WARN"; + state.reason.push(...reasons.map((r) => "! " + r)); + window.__verdict = state.verdict; + document.title = `[WARN] ${state.id}`; + persist(); render(); + }, log, state, }; @@ -72,7 +90,7 @@ try { localStorage.setItem( "pi-chrome-suite:" + state.id, - JSON.stringify({ verdict: state.verdict, reason: state.reason, ts: Date.now() }) + JSON.stringify({ id: state.id, verdict: state.verdict, reason: state.reason, events: state.events.slice(-50), ts: Date.now() }) ); } catch {} } diff --git a/test-suite/_style.css b/test-suite/_style.css index a90bc85..f8af1b4 100644 --- a/test-suite/_style.css +++ b/test-suite/_style.css @@ -2,3 +2,30 @@ body { font: 14px system-ui, sans-serif; margin: 0; background: #1a1a1a; color: main { padding: 24px; max-width: 720px; } code { background:#222;padding:1px 4px;border-radius:3px } a { color: #6cf; } +button, input, select { font: 14px system-ui, sans-serif; } +button { padding: 7px 11px; border-radius: 6px; border: 1px solid #555; background: #262626; color: #eee; cursor:pointer; } +button:hover { background:#333; } +input, select { padding: 6px 8px; border-radius: 6px; border: 1px solid #555; background:#111; color:#eee; } +table { border-collapse: collapse; width: 100%; font: 13px ui-monospace, SFMono-Regular, Menlo, monospace; } +th, td { border-bottom: 1px solid #333; padding: 8px; vertical-align: top; text-align:left; } +th { position: sticky; top: 0; background:#151515; z-index:1; } +tr.ok { background: rgba(31,122,31,.08); } +tr.mismatch { background: rgba(170,17,17,.12); } +tr.conditional { background: rgba(118,82,11,.12); } +.panel { background:#202020; border:1px solid #333; border-radius:10px; padding:12px; margin:16px 0; } +.controls { display:flex; flex-wrap:wrap; gap:10px; align-items:center; } +.hint, .notes, .reason { color:#aaa; font-size:12px; margin-top:4px; } +.summary { display:flex; flex-wrap:wrap; gap:8px; margin: 14px 0; } +.pill { display:inline-block; border-radius:999px; padding:2px 8px; font-size:12px; font-weight:700; background:#444; color:#fff; } +.pass { background:#1f7a1f; } +.fail { background:#a11; } +.skip, .conditional { background:#76520b; } +.warn { background:#6b5d00; } +.pending { background:#444; } +.expected { background:#2b4b6b; } +.risk { color:#ffd479; } +.code, pre { background:#111; color:#eee; padding:12px; border-radius:6px; overflow:auto; } +.copy-fallback { display:none; width:100%; min-height:180px; margin-top:10px; font:12px ui-monospace, SFMono-Regular, Menlo, monospace; background:#111; color:#eee; border:1px solid #555; border-radius:6px; padding:10px; } +.copy-fallback[data-copied="true"] { border-color:#1f7a1f; } +.copy-fallback[data-copied="false"] { border-color:#a11; } +details summary { cursor:pointer; color:#9cf; } diff --git a/test-suite/challenges/10-user-activation.html b/test-suite/challenges/10-user-activation.html index 1a12e39..6e3b9c1 100644 --- a/test-suite/challenges/10-user-activation.html +++ b/test-suite/challenges/10-user-activation.html @@ -22,12 +22,18 @@ document.getElementById("go").addEventListener("click", async () => { catch (e) { clip = "err:" + e.name; } try { await document.documentElement.requestFullscreen(); fs = "ok"; document.exitFullscreen?.(); } catch (e) { fs = "err:" + e.name; } - const bad = []; - if (!wasActive && !hadBeenActive) bad.push("userActivation.isActive/hasBeenActive both false (synthetic click)"); - if (clip.startsWith("err")) bad.push("clipboard.writeText " + clip); - if (fs.startsWith("err")) bad.push("requestFullscreen " + fs); Challenge.log("gates", { clip, fs }); - if (bad.length) Challenge.fail(...bad); + if (!wasActive && !hadBeenActive) { + return Challenge.fail("userActivation.isActive/hasBeenActive both false (synthetic click)"); + } + const okCount = [clip, fs].filter(x => x === "ok").length; + if (okCount === 0) { + return Challenge.skip(`activation present, but gated APIs blocked by environment/policy: clipboard=${clip}; fullscreen=${fs}`); + } + const warnings = []; + if (clip.startsWith("err")) warnings.push("clipboard.writeText " + clip); + if (fs.startsWith("err")) warnings.push("requestFullscreen " + fs); + if (warnings.length) Challenge.pass(`activation active; ${okCount}/2 gates succeeded; ${warnings.join("; ")}`); else Challenge.pass(`activation active; clipboard=${clip}; fullscreen=${fs}`); }); </script> diff --git a/test-suite/challenges/12-fingerprint.html b/test-suite/challenges/12-fingerprint.html index dbc8a25..5acf2ce 100644 --- a/test-suite/challenges/12-fingerprint.html +++ b/test-suite/challenges/12-fingerprint.html @@ -36,7 +36,7 @@ Challenge.init({ id: "fingerprint", instructions: "no interaction" }); const vendor = dbg ? gl.getParameter(dbg.UNMASKED_VENDOR_WEBGL) : gl.getParameter(gl.VENDOR); const renderer = dbg ? gl.getParameter(dbg.UNMASKED_RENDERER_WEBGL) : gl.getParameter(gl.RENDERER); const swiftshader = /SwiftShader|llvmpipe|Software/i.test(renderer); - checks.push({ name: "webgl-not-software", bad: swiftshader, note: `${vendor} / ${renderer}` }); + checks.push({ name: "webgl-not-software", bad: false, warn: swiftshader, note: `${vendor} / ${renderer}${swiftshader ? " (software renderer; warning in VM/remote contexts)" : ""}` }); } catch (e) { checks.push({ name: "webgl-not-software", bad: false, note: "n/a " + e.message }); } @@ -48,9 +48,11 @@ Challenge.init({ id: "fingerprint", instructions: "no interaction" }); checks.push({ name: "webdriver-false", bad: navigator.webdriver === true, note: String(navigator.webdriver) }); const rep = document.getElementById("rep"); - rep.textContent = checks.map(c => `${c.bad ? "✗" : "✓"} ${c.name} ${c.note}`).join("\n"); + rep.textContent = checks.map(c => `${c.bad ? "✗" : c.warn ? "!" : "✓"} ${c.name} ${c.note}`).join("\n"); const failed = checks.filter(c => c.bad); + const warned = checks.filter(c => c.warn); if (failed.length) Challenge.fail(...failed.map(c => `${c.name}: ${c.note}`)); + else if (warned.length) Challenge.warn(...warned.map(c => `${c.name}: ${c.note}`)); else Challenge.pass("fingerprint consistent with real Chrome"); })(); </script> diff --git a/test-suite/challenges/23-stack-trace-fingerprint.html b/test-suite/challenges/23-stack-trace-fingerprint.html index 4c1772b..902f95a 100644 --- a/test-suite/challenges/23-stack-trace-fingerprint.html +++ b/test-suite/challenges/23-stack-trace-fingerprint.html @@ -34,13 +34,11 @@ function inspectStack(stack) { if (!stack) return ["empty stack"]; const hits = SUSPICIOUS.filter(r => r.test(stack)).map(r => r.source); const lines = stack.split("\n").filter(l => l.trim().startsWith("at ")); - // A natural inline-handler stack has ≥2 frames within this document. - const docFrames = lines.filter(l => l.includes(location.pathname.split("/").pop())); - const anonOnly = lines.length > 0 && lines.every(l => /<anonymous>/.test(l)); + // Do not fail on generic <anonymous> frames alone: inline scripts, extensions, + // and browser versions vary here. This test should catch concrete automation + // tells, not punish legitimate stack formatting differences. const reasons = []; if (hits.length) reasons.push("suspicious frames: " + hits.join(",")); - if (anonOnly) reasons.push("all stack frames anonymous"); - if (lines.length && docFrames.length === 0) reasons.push("no frame from this document"); return reasons; } diff --git a/test-suite/challenges/29-shadow-dom-controls.html b/test-suite/challenges/29-shadow-dom-controls.html new file mode 100644 index 0000000..bb2355c --- /dev/null +++ b/test-suite/challenges/29-shadow-dom-controls.html @@ -0,0 +1,44 @@ +<!doctype html> +<meta charset="utf-8"> +<title>29 shadow DOM controls</title> +<link rel="stylesheet" href="../_style.css"> +<script src="../_lib.js"></script> +<body> +<main> + <p>Goal: discover controls inside an <code>open</code> Shadow DOM, click the button, then type <code>shadow-ok</code> into the shadow input.</p> + <div id="host"></div> +</main> +<script> +Challenge.init({ id: "shadow-dom-controls", instructions: "click shadow button, type 'shadow-ok' in shadow input" }); +const host = document.getElementById("host"); +const root = host.attachShadow({ mode: "open" }); +root.innerHTML = ` + <style> + .card { border:1px solid #555; border-radius:8px; padding:16px; background:#222; display:inline-block; } + button,input { font:16px system-ui; padding:8px 10px; margin:6px; } + </style> + <div class="card" role="group" aria-label="Shadow test controls"> + <button id="shadowBtn">Arm shadow form</button> + <input id="shadowInput" aria-label="Shadow value" placeholder="shadow-ok" disabled> + <span id="status">waiting</span> + </div> +`; +const btn = root.getElementById("shadowBtn"); +const input = root.getElementById("shadowInput"); +const status = root.getElementById("status"); +let armed = false; +btn.addEventListener("click", (e) => { + armed = true; + input.disabled = false; + input.focus(); + status.textContent = `armed; click trusted=${e.isTrusted}`; + Challenge.log("shadow-click", { isTrusted: e.isTrusted }); +}); +input.addEventListener("input", (e) => { + Challenge.log("shadow-input", { value: input.value, isTrusted: e.isTrusted }); + if (input.value !== "shadow-ok") return; + if (!armed) return Challenge.fail("input reached target value before shadow button click armed the form"); + Challenge.pass("shadow button clicked and shadow input filled"); +}); +</script> +</body> diff --git a/test-suite/challenges/30-iframe-targeting.html b/test-suite/challenges/30-iframe-targeting.html new file mode 100644 index 0000000..b77eac2 --- /dev/null +++ b/test-suite/challenges/30-iframe-targeting.html @@ -0,0 +1,44 @@ +<!doctype html> +<meta charset="utf-8"> +<title>30 iframe targeting</title> +<link rel="stylesheet" href="../_style.css"> +<script src="../_lib.js"></script> +<body> +<main> + <p>Goal: interact with controls inside a same-origin iframe. Type <code>frame-ok</code> then click Submit inside the frame.</p> + <iframe id="ifr" title="same-origin challenge frame" style="width:520px;height:220px;border:1px solid #555;border-radius:8px;background:#fff"></iframe> +</main> +<script> +Challenge.init({ id: "iframe-targeting", instructions: "type 'frame-ok' and click Submit inside iframe" }); +const frame = document.getElementById("ifr"); +frame.srcdoc = `<!doctype html><meta charset="utf-8"> +<style>body{font:16px system-ui;margin:24px;background:#202020;color:#eee}input,button{font:16px system-ui;padding:8px 10px;margin:6px}</style> +<p>Inside frame: fill and submit.</p> +<input id="insideText" aria-label="Frame value" placeholder="frame-ok"> +<button id="insideButton">Submit frame</button> +<pre id="log"></pre> +<script> + const input = document.getElementById('insideText'); + const btn = document.getElementById('insideButton'); + const log = document.getElementById('log'); + let focused = false; + input.addEventListener('focus', () => { focused = true; parent.postMessage({ type:'frame-focus' }, '*'); }); + input.addEventListener('input', e => parent.postMessage({ type:'frame-input', value: input.value, trusted: e.isTrusted }, '*')); + btn.addEventListener('click', e => { + const payload = { type:'frame-submit', value: input.value, focused, trusted: e.isTrusted }; + log.textContent = JSON.stringify(payload, null, 2); + parent.postMessage(payload, '*'); + }); +<\/script>`; +window.addEventListener("message", (e) => { + if (!e.data || typeof e.data !== "object") return; + Challenge.log("frame-message", e.data); + if (e.data.type !== "frame-submit") return; + const bad = []; + if (e.data.value !== "frame-ok") bad.push(`iframe input value="${e.data.value}" (need frame-ok)`); + if (!e.data.focused) bad.push("iframe input was not focused before submit"); + if (bad.length) Challenge.fail(...bad); + else Challenge.pass(`iframe controls reached; submit trusted=${e.data.trusted}`); +}); +</script> +</body> diff --git a/test-suite/challenges/31-file-upload.html b/test-suite/challenges/31-file-upload.html new file mode 100644 index 0000000..0fddc60 --- /dev/null +++ b/test-suite/challenges/31-file-upload.html @@ -0,0 +1,30 @@ +<!doctype html> +<meta charset="utf-8"> +<title>31 file upload</title> +<link rel="stylesheet" href="../_style.css"> +<script src="../_lib.js"></script> +<body> +<main> + <p>Goal: attach <code>test-suite/fixtures/pi-chrome-upload.txt</code>. Page reads name and contents from the File object.</p> + <input id="file" type="file" aria-label="Upload fixture file" style="font-size:16px;padding:10px;border:1px solid #555;border-radius:6px"> + <pre id="rep" style="font:12px monospace;background:#111;color:#eee;padding:12px;border-radius:6px"></pre> +</main> +<script> +Challenge.init({ id: "file-upload", instructions: "upload fixtures/pi-chrome-upload.txt" }); +const input = document.getElementById("file"); +const rep = document.getElementById("rep"); +input.addEventListener("change", async (e) => { + const f = input.files && input.files[0]; + if (!f) return Challenge.fail("change fired but input.files[0] missing"); + const text = await f.text(); + const info = { name: f.name, size: f.size, type: f.type, trusted: e.isTrusted, text }; + rep.textContent = JSON.stringify(info, null, 2); + Challenge.log("file-change", info); + const bad = []; + if (f.name !== "pi-chrome-upload.txt") bad.push(`file.name=${f.name}`); + if (text !== "pi-chrome upload fixture\n") bad.push(`file content=${JSON.stringify(text)}`); + if (bad.length) Challenge.fail(...bad); + else Challenge.pass(`uploaded ${f.name}; change.isTrusted=${e.isTrusted}`); +}); +</script> +</body> diff --git a/test-suite/challenges/32-keyboard-tab-navigation.html b/test-suite/challenges/32-keyboard-tab-navigation.html new file mode 100644 index 0000000..16c7966 --- /dev/null +++ b/test-suite/challenges/32-keyboard-tab-navigation.html @@ -0,0 +1,61 @@ +<!doctype html> +<meta charset="utf-8"> +<title>32 keyboard tab navigation</title> +<link rel="stylesheet" href="../_style.css"> +<script src="../_lib.js"></script> +<body> +<main> + <p>Goal: use keyboard only. Click Start, then Tab through fields, type <code>one</code>, <code>two</code>, <code>three</code>, Tab to Submit, press Enter.</p> + <button id="start">Start here</button> + <form id="f" style="margin-top:16px;display:grid;gap:10px;max-width:360px"> + <label>First <input id="a" name="a" autocomplete="off"></label> + <label>Second <input id="b" name="b" autocomplete="off"></label> + <label>Third <input id="c" name="c" autocomplete="off"></label> + <button id="submit" type="submit">Submit</button> + </form> + <pre id="rep" style="font:12px monospace;background:#111;color:#eee;padding:12px;border-radius:6px"></pre> +</main> +<script> +Challenge.init({ id: "keyboard-tab-navigation", instructions: "keyboard-only tab flow; submit with Enter" }); +const ids = ["start", "a", "b", "c", "submit"]; +const focusLog = []; +const keyLog = []; +const rep = document.getElementById("rep"); +for (const id of ids) { + const el = document.getElementById(id); + el.addEventListener("focus", (e) => { + focusLog.push({ id, t: e.timeStamp }); + rep.textContent = JSON.stringify({ focusLog, keyLog }, null, 2); + }); + el.addEventListener("keydown", (e) => { + keyLog.push({ id, key: e.key, trusted: e.isTrusted, t: e.timeStamp }); + }); +} +let sawPointerAfterStart = false; +document.addEventListener("pointerdown", (e) => { + if (e.target && e.target.id !== "start") sawPointerAfterStart = true; +}); +document.getElementById("start").addEventListener("click", () => setTimeout(() => document.getElementById("start").focus(), 0)); +document.getElementById("f").addEventListener("submit", (e) => { + e.preventDefault(); + const vals = { + a: document.getElementById("a").value, + b: document.getElementById("b").value, + c: document.getElementById("c").value + }; + const bad = []; + if (vals.a !== "one" || vals.b !== "two" || vals.c !== "three") bad.push(`values=${JSON.stringify(vals)}`); + const seq = focusLog.map(x => x.id).join(">"); + for (const id of ids) if (!focusLog.some(x => x.id === id)) bad.push(`never focused ${id}`); + const enter = keyLog.find(x => x.id === "submit" && x.key === "Enter"); + if (!enter) bad.push("no Enter keydown on submit button"); + else if (!enter.trusted) bad.push("Enter on submit isTrusted=false"); + const tabCount = keyLog.filter(x => x.key === "Tab").length; + if (tabCount < 4) bad.push(`only ${tabCount} Tab keydowns (need ≥4)`); + if (sawPointerAfterStart) bad.push("pointerdown after Start; flow was not keyboard-only"); + Challenge.log("keyboard-submit", { vals, seq, keyLog, sawPointerAfterStart }); + if (bad.length) Challenge.fail(...bad); + else Challenge.pass(`keyboard-only focus sequence ${seq}`); +}); +</script> +</body> diff --git a/test-suite/challenges/33-network-console-capture.html b/test-suite/challenges/33-network-console-capture.html new file mode 100644 index 0000000..812345b --- /dev/null +++ b/test-suite/challenges/33-network-console-capture.html @@ -0,0 +1,33 @@ +<!doctype html> +<meta charset="utf-8"> +<title>33 network and console capture</title> +<link rel="stylesheet" href="../_style.css"> +<script src="../_lib.js"></script> +<body> +<main> + <p>Goal: click button. Page emits console messages and performs a fetch. Benchmark harness should verify <code>chrome_list_console_messages</code> and <code>chrome_list_network_requests</code> captured them.</p> + <button id="go" style="padding:16px 24px;font-size:18px">Emit console + fetch</button> + <pre id="rep" style="font:12px monospace;background:#111;color:#eee;padding:12px;border-radius:6px"></pre> +</main> +<script> +Challenge.init({ id: "network-console-capture", instructions: "clear capture, click, then inspect console/network tools" }); +document.getElementById("go").addEventListener("click", async (e) => { + console.log("pi-chrome-benchmark-console", { clicked: true, trusted: e.isTrusted }); + console.warn("pi-chrome-benchmark-warning"); + let data; + try { + const res = await fetch("data:application/json,%7B%22ok%22%3Atrue%2C%22source%22%3A%22pi-chrome-benchmark%22%7D"); + data = await res.json(); + } catch (err) { + return Challenge.fail("fetch failed: " + err.message); + } + document.getElementById("rep").textContent = JSON.stringify(data, null, 2); + Challenge.log("fetch-result", data); + if (data && data.ok && data.source === "pi-chrome-benchmark") { + Challenge.pass("page fetch completed; verify tool-level console/network capture separately"); + } else { + Challenge.fail("unexpected fetch payload: " + JSON.stringify(data)); + } +}); +</script> +</body> diff --git a/test-suite/fixtures/pi-chrome-upload.txt b/test-suite/fixtures/pi-chrome-upload.txt new file mode 100644 index 0000000..b1901d8 --- /dev/null +++ b/test-suite/fixtures/pi-chrome-upload.txt @@ -0,0 +1 @@ +pi-chrome upload fixture diff --git a/test-suite/index.html b/test-suite/index.html index e125f55..a4a3c0c 100644 --- a/test-suite/index.html +++ b/test-suite/index.html @@ -1,90 +1,157 @@ <!doctype html> <meta charset="utf-8"> -<title>pi-chrome anti-automation test suite</title> +<title>pi-chrome browser-control benchmark</title> <link rel="stylesheet" href="_style.css"> <body> -<main style="max-width:1100px"> -<h1>pi-chrome anti-automation test suite</h1> -<p>Each row links to a challenge page. Open it, drive the interaction with <code>chrome_*</code> tools, -then read <code>window.__verdict</code> via <code>chrome_evaluate</code>. Use the "Run all" button below -to poll every challenge's verdict into a single table.</p> +<main style="max-width:1280px"> +<h1>pi-chrome browser-control benchmark</h1> +<p>This benchmark measures how well Chrome-control tools let agents do real browser work: DOM discovery, trusted input, keyboard/focus, scroll, drag/drop, files, frames, clipboard, and observability.</p> -<table id="tbl" style="border-collapse:collapse;width:100%;font:13px monospace"> - <thead><tr><th align=left>id</th><th align=left>file</th><th align=left>verdict</th><th align=left>open</th></tr></thead> +<section class="panel"> + <div class="controls"> + <button id="refresh">Refresh verdicts</button> + <button id="clear">Clear local verdicts</button> + <button id="copy">Copy JSON report</button> + <label>Filter <input id="filter" placeholder="category, id, expected..." /></label> + <label>Mode + <select id="mode"> + <option value="trusted">trusted</option> + <option value="synthetic">synthetic</option> + <option value="manual">manual</option> + </select> + </label> + </div> + <p class="hint">Open a row, drive it with <code>chrome_*</code> tools, then return here. Verdicts are read from <code>localStorage</code>. Expected outcomes come from <code>manifest.json</code>.</p> + <textarea id="copyFallback" class="copy-fallback" readonly aria-label="JSON report fallback"></textarea> +</section> + +<div id="summary" class="summary"></div> +<table id="tbl"> + <thead> + <tr> + <th>id</th><th>category</th><th>verdict</th><th>expected</th><th>baseline</th><th>risk</th><th>goal / notes</th><th>open</th> + </tr> + </thead> <tbody></tbody> </table> -<p><button id="poll" style="margin-top:12px;padding:8px 14px">Refresh verdicts from any tab on this origin</button></p> - -<h2>How to drive from pi</h2> -<pre style="background:#111;padding:12px;border-radius:6px;overflow:auto"> -# 1. Start static server in test-suite/ -python3 -m http.server 8765 - -# 2. From pi: -chrome_navigate http://127.0.0.1:8765/challenges/01-is-trusted-click.html -chrome_click selector=#go -chrome_evaluate JSON.stringify({v:window.__verdict, r:window.__reason}) -</pre> - +<h2>Agent run loop</h2> +<pre class="code">python3 -m http.server 8765 +# For each manifest entry: +# 1. chrome_navigate http://127.0.0.1:8765/<file> +# 2. chrome_snapshot before acting; prefer uid selectors. +# 3. Execute recipe with selected mode, adapting descriptive frame/shadow selectors to tool uids. +# 4. chrome_evaluate JSON.stringify({v:window.__verdict,r:window.__reason,e:window.__events?.slice(-20)}) +# 5. Compare verdict to manifest.expected[mode]; CONDITIONAL means inspect prerequisites/notes.</pre> +</main> <script> -const challenges = [ - ["is-trusted-click", "challenges/01-is-trusted-click.html"], - ["is-trusted-keyboard", "challenges/02-is-trusted-keyboard.html"], - ["webdriver-flag", "challenges/03-webdriver-flag.html"], - ["mouse-entropy", "challenges/04-mouse-entropy.html"], - ["event-timing", "challenges/05-event-timing.html"], - ["click-coordinates", "challenges/06-click-coordinates.html"], - ["pointer-properties", "challenges/07-pointer-properties.html"], - ["keyboard-cadence", "challenges/08-keyboard-cadence.html"], - ["composition-input", "challenges/09-composition-input.html"], - ["user-activation", "challenges/10-user-activation.html"], - ["honeypot", "challenges/11-honeypot.html"], - ["fingerprint", "challenges/12-fingerprint.html"], - ["focus-order", "challenges/13-focus-order.html"], - ["wheel-scroll", "challenges/14-wheel-scroll.html"], - ["drag-drop-datatransfer", "challenges/15-drag-drop-datatransfer.html"], - ["contenteditable-selection", "challenges/16-contenteditable-selection.html"], - ["paste-clipboard", "challenges/17-paste-clipboard.html"], - ["native-select", "challenges/18-native-select.html"], - ["hover-dwell", "challenges/19-hover-dwell.html"], - ["react-value-tracker", "challenges/20-react-value-tracker.html"], - ["keyboard-modifiers", "challenges/21-keyboard-modifiers.html"], - ["touch-events", "challenges/22-touch-events.html"], - ["stack-trace-fingerprint", "challenges/23-stack-trace-fingerprint.html"], - ["viewport-edge-clicks", "challenges/24-viewport-edge-clicks.html"], - ["pointer-continuity", "challenges/25-pointer-continuity.html"], - ["mousemove-rate", "challenges/26-mousemove-rate.html"], - ["scroll-momentum", "challenges/27-scroll-momentum.html"], - ["intersection-visibility", "challenges/28-intersection-visibility.html"], -]; +let manifest = []; const tbody = document.querySelector("#tbl tbody"); -const rowFor = {}; -for (const [id, file] of challenges) { - const tr = document.createElement("tr"); - tr.innerHTML = `<td>${id}</td><td>${file}</td><td class=v>—</td><td><a target=_blank href="${file}">open</a></td>`; - tbody.appendChild(tr); - rowFor[id] = tr; -} +const modeEl = document.getElementById("mode"); +const filterEl = document.getElementById("filter"); -// localStorage rendezvous: each challenge page writes its verdict on change. +function verdictFor(id) { + const raw = localStorage.getItem("pi-chrome-suite:" + id); + if (!raw) return null; + try { return JSON.parse(raw); } catch { return null; } +} +function classFor(v) { + if (!v) return "pending"; + return String(v).toLowerCase(); +} +function expectedClass(actual, expected) { + if (!actual) return "pending"; + if (expected === "CONDITIONAL") return actual === "PASS" ? "ok" : "conditional"; + return actual === expected ? "ok" : "mismatch"; +} +function report() { + const mode = modeEl.value; + return manifest.map(m => { + const v = verdictFor(m.id); + return { + id: m.id, + category: m.category, + file: m.file, + verdict: v?.verdict ?? "PENDING", + reason: v?.reason ?? [], + expected: m.expected?.[mode] ?? "—", + timestamp: v?.ts ?? null, + goal: m.goal, + prerequisites: m.prerequisites ?? [], + notes: m.notes ?? [] + }; + }); +} function paint() { - for (const [id] of challenges) { - const raw = localStorage.getItem("pi-chrome-suite:" + id); - if (!raw) continue; - try { - const { verdict, reason } = JSON.parse(raw); - const td = rowFor[id].querySelector(".v"); - td.textContent = verdict + (reason?.length ? " — " + reason.join(" / ") : ""); - td.style.background = verdict === "PASS" ? "#1f7a1f" : verdict === "FAIL" ? "#a11" : "#444"; - td.style.color = "#fff"; - } catch {} + const q = filterEl.value.trim().toLowerCase(); + const mode = modeEl.value; + tbody.innerHTML = ""; + const rows = report(); + const counts = rows.reduce((acc, r) => (acc[r.verdict] = (acc[r.verdict] || 0) + 1, acc), {}); + const mismatches = rows.filter(r => r.verdict !== "PENDING" && r.expected !== "CONDITIONAL" && r.expected !== "—" && r.verdict !== r.expected).length; + document.getElementById("summary").innerHTML = ` + <span class="pill pass">PASS ${counts.PASS || 0}</span> + <span class="pill fail">FAIL ${counts.FAIL || 0}</span> + <span class="pill skip">SKIP ${counts.SKIP || 0}</span> + <span class="pill warn">WARN ${counts.WARN || 0}</span> + <span class="pill pending">PENDING ${counts.PENDING || 0}</span> + <span class="pill ${mismatches ? "fail" : "pass"}">unexpected ${mismatches}</span> + `; + for (const m of manifest) { + const v = verdictFor(m.id); + const verdict = v?.verdict ?? "PENDING"; + const expected = m.expected?.[mode] ?? "—"; + const hay = JSON.stringify({ id:m.id, category:m.category, verdict, expected, goal:m.goal, notes:m.notes }).toLowerCase(); + if (q && !hay.includes(q)) continue; + const tr = document.createElement("tr"); + tr.className = expectedClass(verdict, expected); + const reason = (v?.reason || []).join(" / "); + const notes = [...(m.prerequisites || []).map(x => "pre: " + x), ...(m.notes || [])].join("; "); + tr.innerHTML = ` + <td><code>${m.id}</code></td> + <td>${m.category || "—"}</td> + <td><span class="pill ${classFor(verdict)}">${verdict}</span><div class="reason">${escapeHtml(reason)}</div></td> + <td><span class="pill expected">${expected}</span></td> + <td><span class="pill ${m.manualBaseline === "verified" ? "pass" : "pending"}">${m.manualBaseline || "unverified"}</span></td> + <td>${m.flakeRisk ? `<span class="risk">${m.flakeRisk}</span>` : ""}</td> + <td><div>${escapeHtml(m.goal || "")}</div><div class="notes">${escapeHtml(notes)}</div><details><summary>recipe</summary><pre>${escapeHtml(JSON.stringify(m.recipe || [], null, 2))}</pre></details></td> + <td><a target="_blank" href="${m.file}">open</a></td> + `; + tbody.appendChild(tr); } } -document.getElementById("poll").addEventListener("click", paint); +function escapeHtml(s) { + return String(s).replace(/[&<>"']/g, c => ({"&":"&","<":"<",">":">","\"":""","'":"'"}[c])); +} +async function init() { + manifest = await fetch("manifest.json", { cache: "no-store" }).then(r => r.json()); + paint(); +} +document.getElementById("refresh").addEventListener("click", paint); +document.getElementById("clear").addEventListener("click", () => { + if (!confirm("Clear pi-chrome-suite verdicts for this origin?")) return; + for (const m of manifest) localStorage.removeItem("pi-chrome-suite:" + m.id); + paint(); +}); +document.getElementById("copy").addEventListener("click", async () => { + const text = JSON.stringify({ mode: modeEl.value, generatedAt: new Date().toISOString(), rows: report() }, null, 2); + const box = document.getElementById("copyFallback"); + box.value = text; + box.style.display = "block"; + box.focus(); + box.select(); + let copied = false; + try { await navigator.clipboard.writeText(text); copied = true; } catch {} + if (!copied) { + try { copied = document.execCommand("copy"); } catch {} + } + box.dataset.copied = copied ? "true" : "false"; + console.log(text); +}); +modeEl.addEventListener("change", paint); +filterEl.addEventListener("input", paint); window.addEventListener("storage", paint); setInterval(paint, 1000); -paint(); +init(); </script> -</main> </body> diff --git a/test-suite/manifest.json b/test-suite/manifest.json new file mode 100644 index 0000000..c09cf31 --- /dev/null +++ b/test-suite/manifest.json @@ -0,0 +1,952 @@ +[ + { + "id": "is-trusted-click", + "file": "challenges/01-is-trusted-click.html", + "category": "trusted-input", + "goal": "Click handler receives browser-trusted pointer/click events.", + "expected": { + "synthetic": "FAIL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#go", + "trusted": true + } + } + ], + "notes": [ + "Core debugger/CDP smoke test." + ], + "manualBaseline": "unverified" + }, + { + "id": "is-trusted-keyboard", + "file": "challenges/02-is-trusted-keyboard.html", + "category": "trusted-input", + "goal": "Typing produces trusted keydown/input sequence.", + "expected": { + "synthetic": "FAIL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_type", + "params": { + "selector": "#t", + "text": "hello", + "trusted": true + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "webdriver-flag", + "file": "challenges/03-webdriver-flag.html", + "category": "fingerprint", + "goal": "Real profile has no WebDriver/headless tells.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_evaluate", + "params": { + "expression": "JSON.stringify({v:window.__verdict,r:window.__reason})" + } + } + ], + "notes": [ + "Environment-sensitive; classify unexpected failures as profile warnings first." + ], + "manualBaseline": "unverified" + }, + { + "id": "mouse-entropy", + "file": "challenges/04-mouse-entropy.html", + "category": "pointer-humanization", + "goal": "Click is preceded by organic mousemove path.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "event-timing", + "file": "challenges/05-event-timing.html", + "category": "timing", + "goal": "Repeated clicks have plausible holds and gaps.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + } + ], + "flakeRisk": "medium", + "manualBaseline": "unverified" + }, + { + "id": "click-coordinates", + "file": "challenges/06-click-coordinates.html", + "category": "pointer-humanization", + "goal": "Clicks land inside target but not always exact center.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "pointer-properties", + "file": "challenges/07-pointer-properties.html", + "category": "pointer-humanization", + "goal": "Pointer events carry plausible pointerType, pressure, movement, id.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "keyboard-cadence", + "file": "challenges/08-keyboard-cadence.html", + "category": "keyboard", + "goal": "Typing emits per-character key events with non-uniform cadence.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_type", + "params": { + "selector": "#t", + "text": "secret" + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "composition-input", + "file": "challenges/09-composition-input.html", + "category": "keyboard", + "goal": "Plain ASCII typing has correct beforeinput/input order without fake composition.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_type", + "params": { + "selector": "#t", + "text": "abc" + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "user-activation", + "file": "challenges/10-user-activation.html", + "category": "activation-gates", + "goal": "User activation unlocks clipboard/fullscreen-style gated APIs.", + "expected": { + "synthetic": "FAIL", + "trusted": "CONDITIONAL", + "manual": "CONDITIONAL" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#go", + "trusted": true + } + } + ], + "verdictDelayMs": 800, + "prerequisites": [ + "secure origin", + "clipboard/fullscreen allowed by browser policy" + ], + "flakeRisk": "high", + "manualBaseline": "unverified" + }, + { + "id": "honeypot", + "file": "challenges/11-honeypot.html", + "category": "agent-safety", + "goal": "Agent fills only visible intended field, not hidden traps.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_fill", + "params": { + "selector": "#name", + "text": "alex" + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "button[type=submit]" + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "fingerprint", + "file": "challenges/12-fingerprint.html", + "category": "fingerprint", + "goal": "Browser fingerprint is plausible for normal Chrome profile.", + "expected": { + "synthetic": "CONDITIONAL", + "trusted": "CONDITIONAL", + "manual": "CONDITIONAL" + }, + "recipe": [ + { + "tool": "chrome_evaluate", + "params": { + "expression": "JSON.stringify({v:window.__verdict,r:window.__reason})" + } + } + ], + "prerequisites": [ + "non-headless normal Chrome profile" + ], + "flakeRisk": "medium", + "manualBaseline": "unverified" + }, + { + "id": "focus-order", + "file": "challenges/13-focus-order.html", + "category": "focus-keyboard", + "goal": "Pointer-driven focus precedes input and does not set focus-visible keyboard affordance.", + "expected": { + "synthetic": "CONDITIONAL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_type", + "params": { + "selector": "#t", + "text": "x", + "trusted": true + } + } + ], + "flakeRisk": "medium", + "manualBaseline": "unverified" + }, + { + "id": "wheel-scroll", + "file": "challenges/14-wheel-scroll.html", + "category": "scroll", + "goal": "Scrollable element moves via wheel events, not scrollTop teleport.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_scroll", + "params": { + "selector": "#box", + "deltaY": 1200 + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "drag-drop-datatransfer", + "file": "challenges/15-drag-drop-datatransfer.html", + "category": "drag-drop", + "goal": "HTML5 drag cycle delivers trusted events and populated DataTransfer.", + "expected": { + "synthetic": "FAIL", + "trusted": "CONDITIONAL", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_drag", + "params": { + "fromSelector": "#src", + "toSelector": "#dst", + "trusted": true + } + } + ], + "notes": [ + "May need CDP Input.dispatchDragEvent rather than mouse drag on some Chrome versions." + ], + "manualBaseline": "unverified" + }, + { + "id": "contenteditable-selection", + "file": "challenges/16-contenteditable-selection.html", + "category": "editing", + "goal": "Contenteditable typing advances selection/caret like user typing.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_type", + "params": { + "selector": "#ed", + "text": "hello" + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "paste-clipboard", + "file": "challenges/17-paste-clipboard.html", + "category": "clipboard", + "goal": "Paste uses OS/browser clipboard path with trusted paste event and clipboardData.", + "expected": { + "synthetic": "FAIL", + "trusted": "CONDITIONAL", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#t", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "v", + "modifiers": { + "metaKey": true + }, + "trusted": true + } + } + ], + "prerequisites": [ + "OS clipboard contains pi-chrome", + "use ctrlKey instead of metaKey on Windows/Linux" + ], + "manualBaseline": "unverified" + }, + { + "id": "native-select", + "file": "challenges/18-native-select.html", + "category": "native-controls", + "goal": "Native select changes via trusted picker/keyboard path, not .value assignment.", + "expected": { + "synthetic": "FAIL", + "trusted": "CONDITIONAL", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#s", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "ArrowDown", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "ArrowDown", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Enter", + "trusted": true + } + } + ], + "flakeRisk": "high", + "manualBaseline": "unverified" + }, + { + "id": "hover-dwell", + "file": "challenges/19-hover-dwell.html", + "category": "hover", + "goal": "Hover reveal includes dwell, pointer activity, and delayed trusted confirm click.", + "expected": { + "synthetic": "FAIL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_hover", + "params": { + "selector": "#trig", + "trusted": true + } + }, + { + "tool": "wait", + "params": { + "ms": 800 + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#go", + "trusted": true + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "react-value-tracker", + "file": "challenges/20-react-value-tracker.html", + "category": "frameworks", + "goal": "Programmatic fill uses native value setter so React-style tracker sees change.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_fill", + "params": { + "selector": "#t", + "text": "react-ok" + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "keyboard-modifiers", + "file": "challenges/21-keyboard-modifiers.html", + "category": "keyboard", + "goal": "Shift+a chord has correct ordering, code/key/modifier state, trusted events.", + "expected": { + "synthetic": "FAIL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_key", + "params": { + "key": "a", + "modifiers": { + "shiftKey": true + }, + "trusted": true + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "touch-events", + "file": "challenges/22-touch-events.html", + "category": "touch", + "goal": "Touch tap produces real TouchEvent/Touch surface with radius/force.", + "expected": { + "synthetic": "FAIL", + "trusted": "CONDITIONAL", + "manual": "CONDITIONAL" + }, + "recipe": [ + { + "tool": "chrome_tap", + "params": { + "selector": "#pad" + } + } + ], + "prerequisites": [ + "touch events enabled/supported" + ], + "manualBaseline": "unverified" + }, + { + "id": "stack-trace-fingerprint", + "file": "challenges/23-stack-trace-fingerprint.html", + "category": "fingerprint", + "goal": "Click handler stack does not expose extension/eval automation frames.", + "expected": { + "synthetic": "FAIL", + "trusted": "CONDITIONAL", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#go", + "trusted": true + } + } + ], + "notes": [ + "Adversarial probe; selector resolution itself can be detected by monkey-patched DOM APIs." + ], + "manualBaseline": "unverified" + }, + { + "id": "viewport-edge-clicks", + "file": "challenges/24-viewport-edge-clicks.html", + "category": "pointer-humanization", + "goal": "Click coordinates are valid viewport coordinates inside target.", + "expected": { + "synthetic": "FAIL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#go", + "trusted": true + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "pointer-continuity", + "file": "challenges/25-pointer-continuity.html", + "category": "pointer-humanization", + "goal": "Far-apart clicks are bridged by mid-span pointermove samples.", + "expected": { + "synthetic": "FAIL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#a", + "trusted": true + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#b", + "trusted": true + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "mousemove-rate", + "file": "challenges/26-mousemove-rate.html", + "category": "pointer-humanization", + "goal": "Mouse sweep produces plausible move frequency and jitter before trusted Done click.", + "expected": { + "synthetic": "FAIL", + "trusted": "CONDITIONAL", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_hover", + "params": { + "selector": "#pad", + "trusted": true + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#go", + "trusted": true + } + } + ], + "notes": [ + "Needs a first-class pointer path/sweep tool for deterministic automation." + ], + "flakeRisk": "high", + "manualBaseline": "unverified" + }, + { + "id": "scroll-momentum", + "file": "challenges/27-scroll-momentum.html", + "category": "scroll", + "goal": "Wheel gesture has trusted events with ramp/decay momentum tail.", + "expected": { + "synthetic": "FAIL", + "trusted": "PASS", + "manual": "CONDITIONAL" + }, + "recipe": [ + { + "tool": "chrome_scroll", + "params": { + "selector": "#box", + "deltaY": 1600, + "trusted": true + } + } + ], + "notes": [ + "Physical detent mouse wheels may fail decay requirement; trackpads usually pass." + ], + "manualBaseline": "unverified" + }, + { + "id": "intersection-visibility", + "file": "challenges/28-intersection-visibility.html", + "category": "scroll-visibility", + "goal": "Scrolling reveals target gradually with IO/rAF samples before trusted click.", + "expected": { + "synthetic": "FAIL", + "trusted": "CONDITIONAL", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_scroll", + "params": { + "selector": "#scroller", + "deltaY": 1200, + "trusted": true + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#target", + "trusted": true + } + } + ], + "flakeRisk": "medium", + "manualBaseline": "unverified" + }, + { + "id": "shadow-dom-controls", + "file": "challenges/29-shadow-dom-controls.html", + "category": "dom-complexity", + "goal": "Agent discovers and interacts with controls inside open Shadow DOM.", + "expected": { + "synthetic": "CONDITIONAL", + "trusted": "CONDITIONAL", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#host >>> button" + } + }, + { + "tool": "chrome_type", + "params": { + "selector": "#host >>> input", + "text": "shadow-ok" + } + } + ], + "notes": [ + "Selector syntax is descriptive; tools may need uid from snapshot rather than CSS.", + "Current recipe uses descriptive >>> shadow selector syntax; runners must adapt to supported uids/selectors or use evaluate until bridge supports shadow piercing." + ], + "manualBaseline": "unverified", + "prerequisites": [ + "pi-chrome shadow-piercing selectors or shadow-root-aware snapshot uids" + ] + }, + { + "id": "iframe-targeting", + "file": "challenges/30-iframe-targeting.html", + "category": "frames", + "goal": "Agent targets interactive controls inside a same-origin iframe and reports verdict to parent.", + "expected": { + "synthetic": "CONDITIONAL", + "trusted": "CONDITIONAL", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_type", + "params": { + "selector": "iframe#ifr >> #insideText", + "text": "frame-ok" + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "iframe#ifr >> #insideButton" + } + } + ], + "notes": [ + "Selector syntax is descriptive; benchmark checks whether tool exposes frame-aware uids/selectors." + ], + "manualBaseline": "unverified", + "prerequisites": [ + "pi-chrome currently targets frameId 0 for many actions; iframe drilling needs frame-targeting support" + ] + }, + { + "id": "file-upload", + "file": "challenges/31-file-upload.html", + "category": "files", + "goal": "Agent attaches a local file to an input and page reads expected name/content.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_upload_file", + "params": { + "selector": "#file", + "paths": [ + "$PWD/test-suite/fixtures/pi-chrome-upload.txt" + ] + } + } + ], + "manualBaseline": "unverified", + "notes": [ + "Recipe paths may contain $PWD; external runners should expand shell-style placeholders before calling chrome_upload_file." + ] + }, + { + "id": "keyboard-tab-navigation", + "file": "challenges/32-keyboard-tab-navigation.html", + "category": "focus-keyboard", + "goal": "Keyboard-only user flow tabs through fields and submits with Enter.", + "expected": { + "synthetic": "CONDITIONAL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#start", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Tab", + "trusted": true + } + }, + { + "tool": "chrome_type", + "params": { + "text": "one", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Tab", + "trusted": true + } + }, + { + "tool": "chrome_type", + "params": { + "text": "two", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Tab", + "trusted": true + } + }, + { + "tool": "chrome_type", + "params": { + "text": "three", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Tab", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Enter", + "trusted": true + } + } + ], + "manualBaseline": "unverified" + }, + { + "id": "network-console-capture", + "file": "challenges/33-network-console-capture.html", + "category": "observability", + "goal": "Interaction emits console logs and fetch traffic for tool-level capture APIs.", + "expected": { + "synthetic": "PASS", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_list_console_messages", + "params": { + "clear": true + } + }, + { + "tool": "chrome_list_network_requests", + "params": { + "clear": true + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#go" + } + }, + { + "tool": "chrome_list_console_messages", + "params": {} + }, + { + "tool": "chrome_list_network_requests", + "params": {} + } + ], + "notes": [ + "Page verdict only checks in-page fetch result; benchmark harness should also assert captured console/network entries.", + "First chrome_list_console_messages/chrome_list_network_requests call installs capture hooks; do not move clear/list calls after click." + ], + "manualBaseline": "unverified" + } +] diff --git a/test-suite/manifest.schema.json b/test-suite/manifest.schema.json new file mode 100644 index 0000000..79371e8 --- /dev/null +++ b/test-suite/manifest.schema.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "pi-chrome benchmark manifest", + "description": "Manifest for browser-control benchmark pages. Recipes are canonical intent, not guaranteed raw tool JSON: runners may need to adapt descriptive selectors (frame/shadow) and expand path placeholders such as $PWD before invoking tools.", + "type": "array", + "items": { + "type": "object", + "required": ["id", "file", "category", "goal", "expected", "recipe"], + "additionalProperties": true, + "properties": { + "id": { "type": "string" }, + "file": { "type": "string" }, + "category": { "$ref": "#/$defs/category" }, + "goal": { "type": "string" }, + "expected": { + "type": "object", + "description": "Expected verdict by execution mode. CONDITIONAL means prerequisites, browser policy, hardware, or missing tool primitives decide outcome.", + "required": ["synthetic", "trusted", "manual"], + "properties": { + "synthetic": { "$ref": "#/$defs/outcome" }, + "trusted": { "$ref": "#/$defs/outcome" }, + "manual": { "$ref": "#/$defs/outcome" } + } + }, + "manualBaseline": { + "description": "Whether human baseline has been explicitly verified in this repo/environment.", + "enum": ["unverified", "verified"] + }, + "manualBaselineVerifiedAt": { "type": "string", "format": "date-time" }, + "recipe": { + "type": "array", + "description": "Ordered tool intent. Params may include descriptive selectors (e.g. frame/shadow notation) or path placeholders for runners to adapt.", + "items": { + "type": "object", + "required": ["tool", "params"], + "properties": { + "tool": { "type": "string" }, + "params": { + "type": "object", + "description": "Tool params. For upload recipes, paths are strings after runner expansion; $PWD means repository working directory." + } + } + } + }, + "prerequisites": { "type": "array", "items": { "type": "string" } }, + "notes": { "type": "array", "items": { "type": "string" } }, + "verdictDelayMs": { "type": "number" }, + "flakeRisk": { "enum": ["low", "medium", "high"] } + } + }, + "$defs": { + "outcome": { "enum": ["PASS", "FAIL", "SKIP", "WARN", "CONDITIONAL"] }, + "category": { + "enum": [ + "activation-gates", + "agent-safety", + "clipboard", + "dom-complexity", + "drag-drop", + "editing", + "files", + "fingerprint", + "focus-keyboard", + "frames", + "frameworks", + "hover", + "keyboard", + "native-controls", + "observability", + "pointer-humanization", + "scroll", + "scroll-visibility", + "timing", + "touch", + "trusted-input" + ] + } + } +} diff --git a/test-suite/notes/bypass-ideas.md b/test-suite/notes/bypass-ideas.md index 7a019b6..7980aaa 100644 --- a/test-suite/notes/bypass-ideas.md +++ b/test-suite/notes/bypass-ideas.md @@ -1,9 +1,12 @@ -# What it would take to PASS each challenge +# Historical bypass notes -`pi-chrome` runs through a content-script bridge, not CDP. That limits the -solution space — most "real-event" tricks need the chrome.debugger API or -`chrome.input.synthesizeMouseEvent`. The notes below mark which are reachable -with the current manifest and which require new permissions. +This file predates the current `trusted:true` / `chrome.debugger` path and is +kept as design history. For current benchmark expectations, recipes, and +capability notes, use `../manifest.json`. + +Older context: `pi-chrome` originally ran mostly through a content-script bridge. +That limited the solution space — most "real-event" tricks needed the +`chrome.debugger` API or `chrome.input.synthesizeMouseEvent`. ## 01–02 isTrusted click / keyboard **Hard.** `Event.isTrusted` is true only for events the browser dispatched |