From 4c54b05dc15b81e449dc8ac53da8bae536fa2798 Mon Sep 17 00:00:00 2001 From: Tianren Dong Date: Wed, 13 May 2026 07:29:41 -0400 Subject: pi-chrome: snapshot pre-migration state --- .maintainer/GROWTH-IDEAS.md | 156 ++++++ CHANGELOG.md | 88 ++++ CONTRIBUTING.md | 50 ++ LICENSE | 21 + README.md | 301 +++++++---- SECURITY.md | 38 ++ docs/COMPARISON.md | 166 ++++++ docs/EXAMPLES.md | 166 ++++++ docs/FAQ.md | 79 +++ .../browser-extension/manifest.json | 2 +- package.json | 43 +- test-suite/README.md | 60 ++- test-suite/_lib.js | 35 +- test-suite/browsergym-action-space.json | 44 ++ test-suite/challenges/34-dialog-handling.html | 28 + test-suite/challenges/35-target-blank-popup.html | 35 ++ test-suite/challenges/36-modal-focus-trap.html | 49 ++ .../challenges/37-autocomplete-combobox.html | 42 ++ test-suite/challenges/38-spa-route-change.html | 40 ++ test-suite/fixtures/sites/mini-shop/cheats.js | 12 + test-suite/fixtures/sites/mini-shop/grader.js | 29 ++ test-suite/fixtures/sites/mini-shop/index.html | 55 ++ test-suite/fixtures/sites/mini-shop/tasks.json | 9 + test-suite/index.html | 28 + test-suite/manifest.json | 569 +++++++++++++++++++-- test-suite/manifest.schema.json | 53 +- test-suite/notes/browsergym-compat.md | 70 +++ test-suite/notes/profiles.md | 22 + test-suite/notes/runner-spec.md | 26 + test-suite/notes/scoring.md | 36 ++ test-suite/scenarios/choredesk/cheats.js | 49 ++ test-suite/scenarios/choredesk/index.html | 239 +++++++++ test-suite/task-manifest.json | 416 +++++++++++++++ test-suite/task-manifest.schema.json | 59 +++ 34 files changed, 2921 insertions(+), 194 deletions(-) create mode 100644 .maintainer/GROWTH-IDEAS.md create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 SECURITY.md create mode 100644 docs/COMPARISON.md create mode 100644 docs/EXAMPLES.md create mode 100644 docs/FAQ.md create mode 100644 test-suite/browsergym-action-space.json create mode 100644 test-suite/challenges/34-dialog-handling.html create mode 100644 test-suite/challenges/35-target-blank-popup.html create mode 100644 test-suite/challenges/36-modal-focus-trap.html create mode 100644 test-suite/challenges/37-autocomplete-combobox.html create mode 100644 test-suite/challenges/38-spa-route-change.html create mode 100644 test-suite/fixtures/sites/mini-shop/cheats.js create mode 100644 test-suite/fixtures/sites/mini-shop/grader.js create mode 100644 test-suite/fixtures/sites/mini-shop/index.html create mode 100644 test-suite/fixtures/sites/mini-shop/tasks.json create mode 100644 test-suite/notes/browsergym-compat.md create mode 100644 test-suite/notes/profiles.md create mode 100644 test-suite/notes/runner-spec.md create mode 100644 test-suite/notes/scoring.md create mode 100644 test-suite/scenarios/choredesk/cheats.js create mode 100644 test-suite/scenarios/choredesk/index.html create mode 100644 test-suite/task-manifest.json create mode 100644 test-suite/task-manifest.schema.json diff --git a/.maintainer/GROWTH-IDEAS.md b/.maintainer/GROWTH-IDEAS.md new file mode 100644 index 0000000..dc80fe2 --- /dev/null +++ b/.maintainer/GROWTH-IDEAS.md @@ -0,0 +1,156 @@ +# Growth ideas — for your review + +This file is **draft copy and ideas for the maintainer to review and choose from**, not anything that's been sent or published anywhere. Nothing here goes out without you doing it yourself. + +It also collects outside-the-package ideas (things I can't do without your accounts/approval) so we have one place to triage. + +--- + +## Inside-the-package — already shipped in this growth pass + +- README rewrite (hero, comparison table, complete tool list grouped, killer recipes, architecture diagram, honest-results section, benchmark plug) +- `package.json` keywords expanded for npm search SEO (`browser-automation`, `playwright-alternative`, `puppeteer-alternative`, `agent`, `llm-tools`, `browser-use`, `stagehand-alternative`, etc.) +- `package.json` description rewritten as a one-line elevator pitch +- `homepage`, `repository`, `bugs` fields added (npm shows them on the package page) +- `CHANGELOG.md` (signals active maintenance) +- `LICENSE` (MIT) +- `CONTRIBUTING.md` (signals openness; sets bar) +- `SECURITY.md` (signals maturity) +- `docs/COMPARISON.md` (deep comparison vs Playwright / Puppeteer / Stagehand / browser-use / Selenium) +- `docs/EXAMPLES.md` (ready-to-paste agent prompts) +- `docs/FAQ.md` (kills common adoption friction) +- Cross-links from `pi-bar` and `pi-qq` READMEs back to pi-chrome + +--- + +## Inside-the-package — candidates you should consider + +These are package-only changes you could approve quickly. Listed roughly by ROI. + +1. **Demo GIF or animated SVG in the README.** A 10–15s clip of `/chrome onboard` → `chrome_tab list` → `chrome_snapshot` → summary lands the value prop in one glance. (I can't record it from inside Pi; you'd need to record once, drop it under `assets/` and link from the README.) +2. **`assets/screenshot-*.png`** with one or two static screenshots of the agent operating Chrome. Even one frame is huge for the npm page; npm renders images inline from README. +3. **Open-source-friendly badges** in README: `pi-package` badge, "tested with test-suite", coverage-of-categories badge generated from `manifest.json`. +4. **`prompt-primer.md`** — short prompt block the agent reads on first `chrome_*` use, teaching it `snapshot → uid → act → verify envelope`. Currently in-tool descriptions, but a primer lets users `include` it in their own agent prompts. +5. **`recipes/` directory** with `.md` files per workflow (PR triage, Linear standup, etc.) so each one has a stable URL on npm. +6. **Auto-generated tool reference** (`docs/TOOLS.md`) derived from `index.ts` Type.Object schemas. Shows agents and humans the exact param shapes. +7. **Reload-on-update prompt.** After `pi install npm:pi-chrome` upgrades the package, `/chrome doctor` could auto-suggest the exact `chrome://extensions` reload step. Already half there — could be more aggressive. +8. **A "scorecard" badge** the README can render after running `test-suite/` — e.g. `pi-chrome • 27/30 challenges passing`. Auto-generated from the manifest + a local run. +9. **Quickstart video** — same as #1 but longer, hosted on YouTube / Vimeo. README can embed a thumbnail link. +10. **`pi-chrome --version` CLI helper** (if pi-package convention supports it) so users can sanity-check from a shell. + +--- + +## Outside-the-package — for you to do, when you want + +Nothing here is sent or published. These are drafts. + +### Short copy (one-liner) + +> **pi-chrome** — give your Pi coding agent your real, logged-in Chrome. No re-login, no throwaway profile, no CDP wrangling. `pi install npm:pi-chrome`. + +### Long copy (X/Twitter single tweet) + +> Built pi-chrome: 20 browser tools for Pi agents that drive your **real Chrome** — every tab you're already signed into. +> +> No re-login. No throwaway profile. Watch the agent work live, or run silent. +> +> `pi install npm:pi-chrome` +> +> https://www.npmjs.com/package/pi-chrome + +### X/Twitter thread (8 posts) — draft + +1/ Most browser agents make you log in again. They spawn a throwaway Chrome, replay cookies, and pray your SSO/MFA doesn't catch on. + +2/ pi-chrome flips it. A tiny MIT-licensed Chrome extension lives in your **normal profile**. Your agent drives the tabs you already have open — GitHub, Linear, Stripe, your admin panel, all of it. + +3/ 20 tools: click, type, navigate, screenshot, drag, scroll, touch (real CDP), file upload (no picker), network capture with response bodies, console capture, snapshot with stable element uids. + +4/ Honest result envelopes. When a click doesn't take effect, you get back: "occluded by div#overlay" or "pageMutated=false" — not a silent retry loop. + +5/ Multi-session safe. Planner / worker / audit Pi sessions share one bridge automatically. They all see the same Chrome state because it's your real one. + +6/ Quiet by default (synthetic DOM events, no Chrome banner). One toggle for trusted CDP input when sites need it. Or per-call. + +7/ Comes with a benchmark suite. 30+ static pages graded on trusted clicks, pointer humanization, keyboard fidelity, drag/drop, Shadow DOM, file inputs, network observability, fingerprint leaks. + +8/ `pi install npm:pi-chrome` → `/chrome onboard` → done in 60s. https://www.npmjs.com/package/pi-chrome + +### Hacker News — Show HN draft + +**Title:** Show HN: pi-chrome – give an LLM agent your real, logged-in Chrome (no re-login) + +**Body:** + +I've been frustrated that browser agents always make you log in again — they spin up Playwright/Puppeteer with an isolated profile and try to replay cookies, which falls over on real-world SSO/MFA/admin tools. + +pi-chrome is the opposite approach: a small MIT-licensed unpacked Chrome extension lives in the Chrome profile you already use. Your Pi agent drives the tabs you already have open. No re-login. Ever. + +20 tools: snapshot/click/type/fill/navigate/scroll/drag/upload/screenshot, plus network and console capture with response bodies, plus real CDP touch events for mobile PWAs. + +What makes it different from everything else: + +- **Synthetic input first**, trusted CDP only when needed. Quiet by default — no "Chrome is being debugged" banner unless a site forces it. +- **Honest result envelopes.** Every action returns pageMutated, defaultPrevented, elementVisible, occludedBy, valueMatches. Agents stop looping blindly on broken clicks. +- **Multi-session.** Multiple Pi sessions on the same machine share one bridge automatically. +- **A built-in benchmark suite** with 30+ pages grading any browser-control tool. PRs welcome with your competitor's scores. + +It is Chrome-only by design (it's a Chrome extension) and not a fit for headless scraping at scale — it's for **your editor session** acting on **your real apps**. + +Install: `pi install npm:pi-chrome` then `/chrome onboard`. + +Source: https://github.com/tianrendong/pi-packs/tree/main/packages/pi-chrome + +### Reddit — r/ChatGPTCoding / r/LocalLLaMA draft + +**Title:** pi-chrome: lets your LLM coding agent use the Chrome you're already logged into + +**Body:** + +If you've tried browser agents you've hit the wall where they want you to log in again. pi-chrome (MIT) is a Pi extension that does the obvious thing instead: a tiny Chrome extension in your normal profile, agent drives your real tabs. + +20 tools for the agent — click, type, navigate, screenshot, drag, scroll, real touch, file upload through React inputs (no native picker), network capture with response bodies, console capture, snapshot with stable element uids. + +Why pi-chrome over Playwright/Puppeteer/Selenium when the use case is "agent in your editor": + +- You stay signed in. SSO/MFA/admin pinning works because the agent IS you in those tabs. +- Watch it work live, or toggle silent mode for the whole session. +- Honest result envelopes (pageMutated/occludedBy/valueMatches) — the agent knows when a click failed. +- Multi-session safe — planner + worker Pi sessions share one bridge. +- Benchmark suite included; we test in public. + +`pi install npm:pi-chrome` → `/chrome onboard` → 60 seconds to first useful run. + +https://www.npmjs.com/package/pi-chrome + +--- + +## Outside-the-package — bigger growth bets + +Not draft copy — strategic moves you might run later. + +1. **Public benchmark leaderboard.** Run `test-suite/` against every major browser-control tool monthly, publish results in `docs/SCORES.md` with a link from README. We've got the moat; this surfaces it. Initial bar: pi-chrome (synthetic), pi-chrome (trusted), Playwright, Puppeteer, Stagehand, browser-use, Selenium. +2. **Awesome-pi list PR.** Submit pi-chrome to any "awesome agentic tools" / "awesome browser automation" list on GitHub. Each one is a backlink that costs nothing. +3. **"Built on pi-chrome" gallery.** A `docs/GALLERY.md` showcasing real workflows people built — invites users to submit their own and self-promotes the package. +4. **YouTube demo, 90 seconds.** "Watch a coding agent triage my GitHub notifications using my logged-in Chrome." Highest-leverage single asset for adoption. +5. **A `pi-chrome-recipes` companion package.** Curated recipe library installable via `pi install npm:pi-chrome-recipes` so users get inspirational `/recipe` slash commands. Doubles install count. +6. **Pi-coding-agent feature highlight.** Coordinate with the Pi team to feature pi-chrome in a "Tool of the month" or in onboarding docs. +7. **Comparison blog post on dev.to / hashnode.** Same content as `docs/COMPARISON.md` but as an indexable blog post with a discoverable title like "Playwright vs Puppeteer vs pi-chrome: when to use which for AI agents." +8. **Hook into agentic-AI Discords / Slacks.** browser-use, Stagehand, and AutoGPT communities are full of people whose first instinct is to ship a throwaway-profile bot. pi-chrome solves a real complaint they have weekly. +9. **VS Code / Cursor extension that surfaces `/chrome` commands from the editor command palette.** Low-effort, high-discoverability. +10. **Auto-generated case-study page** per recipe in `docs/EXAMPLES.md` with a screenshot, hosted at `https://pi-chrome.dev` (or a GitHub Pages site). Each one is an indexable landing page. +11. **Sponsorship / featured listing on awesome-llm-agents lists** if any take paid placement. +12. **`pi install npm:pi-chrome` mentioned in pi-coding-agent's own README** if there's a "popular extensions" section. + +--- + +## Demo / show-off prompt (paste in Pi after install) + +```text +Use chrome_tab list to find my GitHub notifications tab. +chrome_snapshot it. Write me a 5-bullet ranked PR triage: +which PRs need my review today, sorted by staleness. +Do not click anything. +``` + +If this lands cleanly in 30 seconds with their real GitHub session, you've sold them. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..171fd21 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,88 @@ +# Changelog + +All notable user-facing changes to `pi-chrome`. + +## 0.14.7 + +- Replace "30+ challenges" hand-wave in README + COMPARISON.md with the accurate framing from chrome-benchmark: **38 primitive challenges + 4 hermetic BrowserGym-style long-horizon tasks**, scored by **expected-outcome-by-mode** (not raw PASS count). Explains why a synthetic-events tool isn't supposed to satisfy a clipboard user-activation gate — matching that expectation is the pass. + +## 0.14.6 + +- Fix Browser Use license in `docs/COMPARISON.md`: MIT (not Apache-2.0). Confirmed against upstream LICENSE on GitHub. + +## 0.14.5 + +- `docs/COMPARISON.md` rewritten with a three-axis landscape (drivers / agent frameworks / cloud providers). Adds Browser Use, Stagehand, Skyvern, Magnitude, Alumnium, OpenAI Operator, Project Mariner, Surfer 2, Anthropic Computer Use, Browserbase, Steel.dev, Hyperbrowser, Anchor, Browserless. Adds Interop section, public-benchmark cheat sheet (WebArena, WorkArena++, BrowseComp, Mind2Web 2, WebChoreArena, MiniWoB++, BrowserGym). +- README gains a one-liner pointing at the new three-axis framing. +- Sourced from `benchmark-search` session research. No code changes. + +## 0.14.4 + +- Sync `manifest.json` version to match `package.json` (0.14.3 shipped with stale manifest, would trigger spurious `/chrome doctor` drift warnings). No code or behavior changes. + +## 0.14.3 + +- Documentation & discoverability overhaul. +- New README: hero, alternatives comparison table, 20-tool reference grouped by job, killer recipes, architecture diagram, honest-results explainer, benchmark suite plug. +- `docs/COMPARISON.md` — deep comparison vs Playwright / Puppeteer / Stagehand / browser-use / Selenium. +- `docs/EXAMPLES.md` — ready-to-paste agent prompts (PR triage, Linear standup, bug repro, network forensics, multi-tab admin cross-check, etc.). +- `docs/FAQ.md` — covers Brave/Arc, incognito, detection, banner, multi-session, CSP, file uploads, common envelope causes. +- `CHANGELOG.md`, `CONTRIBUTING.md`, `SECURITY.md`, `LICENSE` added. +- `package.json`: `homepage`, `repository`, `bugs`, expanded keywords for npm search. +- No code changes to the tools or extension. + +## 0.14.2 + +- Recover from foreign-extension input overlays so input tools don't get hijacked by other Chrome extensions. + +## 0.14.1 + +- Harder `attachDebugger` retry path so trusted clicks survive transient Chrome debugger contention. +- New `trusted.debug` diagnostic surface. + +## 0.14.0 + +- Bare `/chrome` opens a `SettingsList` dialog. `space` cycles values, `enter` saves. + +## 0.13.0 + +- Flattened `/chrome` tree. Cycle-on-pick for `clicks` / `quiet`. Status header at the top of the picker. + +## 0.12.1 + +- Fixed anti-automation regressions across 7 benchmark challenges (`07/13/21/27/28`). + +## 0.12.0 + +- Unified all slash commands under `/chrome` (`/chrome doctor`, `/chrome onboard`, `/chrome clicks`, `/chrome quiet`). + +## 0.11.x + +- Plain-English audit pass across `/chrome doctor`, `/chrome onboard`, `/chrome quiet`, README. +- `chrome_tap` (real CDP touch events). +- Smoother `trustedScroll`; CDP debugger auto-recovers from detach. +- Smart-auto trusted mode default. Extension renamed to **Pi Chrome Connector**. +- Per-event scroll delta cap so IntersectionObserver thresholds land naturally. + +## 0.10.x + +- Trusted-input mode via `chrome.debugger` (CDP) — opt-in, indistinguishable-from-human clicks/keys. +- `chrome_key` modifiers (Cmd+V, Ctrl+Shift+Tab, etc.) for trusted chords. +- Interactive `/chrome-trusted` picker (later folded into `/chrome`). + +## 0.9.x + +- Humanized synthetic input (pointer paths, key cadence variance). +- Anti-automation benchmark test suite landed in `test-suite/`. + +## 0.8.0 + +- `chrome_evaluate` no longer returns `null` for valid expressions — dedicated MAIN-world `evaluateInTab` pipeline with statement-mode fallback and tagged envelope for `undefined`/errors/symbols/bigints. +- Truthful action result envelopes: `isTrusted`, `defaultPrevented`, `elementVisible`, `occludedBy`, `valueMatches`, `pageMutated`. +- Extended `/chrome doctor`: bridge mode/URL, extension version drift, MAIN-world helper injection, `navigator.webdriver` fingerprint, CDP availability probe. +- Removed misleading `returnByValue` param. Implemented `chrome_screenshot.fullPage` via tile stitching. +- Autoplay-gate heuristic for `chrome_click`. + +## 0.7.0 + +- Initial public `pi-chrome` release. Companion Chrome extension + local bridge + first tool set. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c16b5fe --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,50 @@ +# Contributing to pi-chrome + +Thanks for considering a contribution. pi-chrome aims to be the **de-facto browser-control toolkit for Pi agents** — that means a few non-negotiables. + +## Non-negotiables + +1. **No re-login.** Every change must keep working against the user's already-signed-in Chrome profile. Anything that requires a fresh profile or extra auth steps is out of scope. +2. **Honest result envelopes.** Every action tool returns `pageMutated`, `defaultPrevented`, `elementVisible`, `occludedBy` (when relevant), `valueMatches` (for input). Agents need to know **why** something didn't take effect. +3. **Quiet by default, trusted by opt-in.** Synthetic DOM events first. CDP/`chrome.debugger` only when explicitly requested (`trusted: true`) or when the smart-auto heuristic detects an obvious user-activation gate. +4. **Benchmarks gate features.** Add a page in `test-suite/` that fails before your change and passes after. We accept PRs faster when there's a green/red verdict to point at. + +## Local dev + +```bash +# Link from a checkout +pi install ./pi-chrome + +# Run the benchmark dashboard +cd test-suite +python3 -m http.server 8765 +# open http://127.0.0.1:8765/ in the Chrome window pi-chrome controls +``` + +## Adding a new tool + +1. Register in `extensions/chrome-profile-bridge/index.ts` (the `register*Tool` calls near line 840+). +2. Implement the handler in `extensions/chrome-profile-bridge/browser-extension/service_worker.js`. +3. Return a `pageMutated` + relevant fields. +4. Add a benchmark page under `test-suite/challenges/` and a manifest entry. +5. Update `README.md` "What an agent gets" table. +6. Add a `CHANGELOG.md` entry. + +## Filing a bug + +Include: + +- `/chrome doctor` output +- `pi-chrome` version + extension version (the `doctor` output prints both) +- The exact tool call + the result envelope you got +- Page URL or a minimal repro page in `test-suite/` + +## Releasing + +- Bump `package.json` version. +- Move `CHANGELOG.md` notes from the working section to the new version header. +- `npm publish --access public`. + +## Code of conduct + +Be kind, be precise, ship things. PRs that break the "no re-login" promise will be closed with a note explaining which non-negotiable they hit. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a975012 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 pi-chrome contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 0387dce..b01487b 100644 --- a/README.md +++ b/README.md @@ -1,63 +1,82 @@ # pi-chrome [![npm version](https://img.shields.io/npm/v/pi-chrome.svg)](https://www.npmjs.com/package/pi-chrome) +[![npm downloads](https://img.shields.io/npm/dm/pi-chrome.svg)](https://www.npmjs.com/package/pi-chrome) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](./LICENSE) -Control the Chrome profile you already use from Pi. +> **The fastest way to give a [Pi](https://pi.dev) agent your real Chrome.** +> No CDP. No throwaway profile. No re-login. Watch it work — or run silent. -`pi-chrome` gives Pi agents browser tools for your **real Chrome windows, tabs, and authenticated sessions**. It uses a companion Chrome extension instead of the Chrome DevTools Protocol (CDP), so it does not launch a throwaway debug browser profile and does not require re-signing into the apps you already have open. +```text +You: "Find my open GitHub PR tab, summarize review state, and screenshot the failing CI." +Agent: chrome_tab(list) → chrome_snapshot(uid:…) → chrome_screenshot(...) + ✓ 3 reviewers, 1 change requested, CI red on iOS. Saved → .pi/chrome-screenshots/ci.png +You: [keeps coding — agent never asked you to log in] +``` -Multiple Pi sessions can use Chrome at the same time. The first Pi session starts the local bridge; later sessions automatically detect that bridge and submit their Chrome commands through it. +`pi-chrome` ships **20+ browser tools** for Pi agents, backed by a small MIT-licensed Chrome extension that runs inside the Chrome profile **you already use** — including every site you're already signed into. -## Why try it? +--- -- **Uses your existing Chrome profile** — works with the Chrome windows/tabs you are already using, including logged-in GitHub, admin dashboards, local apps, and internal tools. -- **Watch your authenticated Chrome work** — by default, `chrome_*` tool calls focus Chrome and activate the target tab so you can see the agent inspect, navigate, click, and type in real time. Switch to silent/background mode for the whole session with `/chrome quiet`, or pass `background: true` on a single tool call when you want quiet. -- **Full browser automation toolkit for Pi** — list/create/activate/close tabs, snapshot pages with usable CSS selectors, navigate, evaluate JavaScript, click, type, press keys, wait for page state, and capture screenshots. -- **Built-in setup and agent guidance** — `/chrome onboard` walks users through installing the companion extension, `/chrome doctor` checks connectivity and version drift, screenshots save to disk, and the prompt primer tells agents to inspect with `chrome_snapshot` before acting and avoid destructive actions unless explicitly requested. +## Why pi-chrome vs. everything else -## Install +> Short version: **pi-chrome is primitives — "Playwright for the Chrome you're already signed into."** Not an agent loop. Plug it under any agent framework (Browser Use, Stagehand, LangGraph) or call its tools directly from a Pi agent. See [docs/COMPARISON.md](./docs/COMPARISON.md) for the full three-axis landscape (drivers, agents, cloud providers). -```bash -pi install npm:pi-chrome -``` +| | **pi-chrome** | Playwright / Puppeteer | CDP-based agents | Selenium / WebDriver | +| ------------------------------ | --------------------------------- | ----------------------------- | ----------------------------- | ----------------------------- | +| Uses your real signed-in Chrome | ✅ yes (extension in your profile) | ❌ throwaway profile | ⚠️ requires `--remote-debug` | ❌ throwaway profile | +| Re-login required | **Never** | Every run | Sometimes | Every run | +| Watch agent work, live | ✅ default; toggle quiet | ❌ headless or new window | ⚠️ debugger banner always | ❌ new window | +| Works on strict-CSP pages | ✅ `new Function` MAIN-world | ✅ | ✅ | ✅ | +| Real browser-trusted clicks | ✅ opt-in (`chrome clicks on`) | ✅ | ✅ | ✅ | +| Multi-session safe | ✅ shared local bridge | ❌ port collisions | ❌ | ❌ | +| Network/console capture | ✅ built-in | ✅ | ✅ | ⚠️ via extensions | +| Honest result envelopes¹ | ✅ | ⚠️ | ❌ | ❌ | +| Built-in benchmark suite² | ✅ 38 primitives + 4 long-horizon | n/a | n/a | n/a | -For local development from a checkout: +¹ Every action returns `pageMutated`, `defaultPrevented`, `elementVisible`, `occludedBy`, and `valueMatches` so the agent knows when a click didn't take effect — instead of looping blindly. +² See [`test-suite/`](./test-suite) — 38 primitive challenges plus 4 hermetic BrowserGym-style tasks. Scoring is expected-outcome-by-mode (`synthetic` / `trusted` / `manual`), not raw PASS count. Pages grade any browser-control tool on trusted clicks, pointer humanization, keyboard fidelity, drag/drop, clipboard, Shadow DOM, iframes, file uploads, network capture, and fingerprint leaks. -```bash -pi install ./pi-chrome -``` +--- -### Why an unpacked Chrome extension? +## What an agent gets -`pi-chrome` cannot ship through the Chrome Web Store: a Web Store extension is not allowed to talk to a local bridge controlled by another tool. Instead it ships as a small, MIT-licensed unpacked extension in `extensions/chrome-profile-bridge/browser-extension/` — read the source before loading. `/chrome doctor` reports the loaded extension version and warns when it drifts from the installed `pi-chrome`. +**20 tools**, grouped by job. Every one runs against your already-open tabs. -## First-time setup +| Category | Tools | +| --------------- | ---------------------------------------------------------------------------------------------- | +| **Tabs** | `chrome_tab` (list/new/activate/close/version), `chrome_launch` | +| **Inspect** | `chrome_snapshot` (uids + selectors + text + viewport), `chrome_screenshot`, `chrome_evaluate` | +| **Navigate** | `chrome_navigate` (with optional `initScript` at `document_start`), `chrome_wait_for` | +| **Interact** | `chrome_click`, `chrome_type`, `chrome_fill`, `chrome_key`, `chrome_hover` | +| **Gesture** | `chrome_drag` (HTML5 DataTransfer), `chrome_scroll` (wheel + momentum), `chrome_tap` (touch) | +| **Files** | `chrome_upload_file` (no native picker; works with React/Vue/Angular file inputs) | +| **Observe** | `chrome_list_console_messages`, `chrome_list_network_requests`, `chrome_get_network_request` (with response body) | -In Pi, run: +Each tool is documented inline in Pi — agents see the parameters and the gotchas (synthetic vs. trusted, autoplay gates, file picker limits) without trial-and-error. -```text -/chrome onboard +--- + +## 60-second install + +```bash +pi install npm:pi-chrome ``` -Pi first shows setup instructions and waits for confirmation. Press Enter to continue. On macOS it will: +Then in Pi: -- open `chrome://extensions` -- reveal the bundled `browser-extension` folder in Finder -- copy the extension folder path to your clipboard +```text +/chrome onboard +``` -Then in Chrome: +On macOS this opens `chrome://extensions`, reveals the bundled `browser-extension/` folder in Finder, and copies its path to your clipboard. In Chrome: **Developer mode** → **Load unpacked** → paste the path. Done. -1. Enable **Developer mode**. -2. Click **Load unpacked**. -3. Select the revealed/copied `browser-extension` folder. -4. Return to Pi and run: +Verify: ```text /chrome doctor ``` -Expected output: - ```text Performing Chrome bridge health check pi-chrome v @@ -65,122 +84,202 @@ pi-chrome v ✓ Companion Chrome extension responding (ID: , ext v) ``` -## Click modes +--- -pi-chrome can drive Chrome two ways: +## Try this in 30 seconds after install -- **Quiet clicks** — fast and unobtrusive. They work on most sites, but some pages (sign-in flows, copy-to-clipboard buttons, file pickers, autoplay videos, fullscreen, paywalls) ignore them because they don't look like a real human action. -- **Real-looking clicks** — indistinguishable from a person clicking. They unlock the cases above, but Chrome shows a *"Pi Chrome Connector started debugging this browser"* banner at the top of every tab pi-chrome touches while it's working. +```text +Use chrome_tab list to find my GitHub notifications tab. +chrome_snapshot it, then write a 5-bullet triage: +which PRs need my review today, sorted by staleness. +Do not click anything yet. +``` -Pick a mode with `/chrome clicks`: +You'll watch the agent jump to your GitHub tab and read the page — using **your** session, **your** filters, **your** orgs. -```text -/chrome clicks auto # default; quiet by default, real-looking only when needed -/chrome clicks off # always quiet, no banner ever -/chrome clicks on # always real-looking, banner stays up the whole session -/chrome clicks status # show the current mode +--- + +## Killer recipes (copy-paste into Pi) + +Each one assumes tabs you already have open + accounts you're already signed into. + +**PR triage** +> Use chrome_tab list to find my GitHub notifications tab, snapshot it, summarize PRs needing my review. + +**Linear standup** +> Open my Linear current cycle in the active tab, snapshot it, write a 5-bullet standup. + +**Bug repro with evidence** +> Open the staging app I'm already signed into, reproduce \, save a screenshot of each step under `./repro/`. + +**Form auto-fill (no submit)** +> Open the vendor portal, fill the new-vendor form from this JSON, stop before submit. + +**Admin cross-check** +> Across my Stripe / Postmark / our admin tabs, find any user where state disagrees. + +**Local dev visual diff** +> Snapshot `localhost:3000` and the staging URL of the same page; tell me what's visually different. + +**Auth-only data pull** +> Open my analytics dashboard tab and `chrome_evaluate` to extract today's KPIs from page state. + +**Network forensics** +> Reproduce the checkout bug, then use `chrome_list_network_requests` to find the failing call and dump its response body. + +**File upload through React** +> Open the photo uploader, `chrome_upload_file` with `./fixtures/sample.png`, confirm preview rendered. + +--- + +## Architecture + +``` + ┌──────────────────────┐ ┌──────────────────────────┐ + │ Pi agent (terminal) │ ─── http://127.0.0.1:17318 ─→ │ Chrome extension │ + │ chrome_* tools │ │ (your real profile) │ + └──────────┬───────────┘ └─────────┬────────────────┘ + │ same machine │ + ▼ ▼ + Other Pi sessions Tabs you already have open + share the same bridge (signed in to GitHub, + automatically Linear, Stripe, etc.) ``` -For a one-off call, pass `trusted: true` (or `false`) on `chrome_click`, `chrome_type`, `chrome_fill`, `chrome_key`, `chrome_hover`, `chrome_drag`, or `chrome_scroll`. The per-call value wins over the global mode. +Multiple Pi sessions (planner / worker / audit) can all drive the same Chrome at once. The first session opens the local bridge; later sessions detect it and pipe their commands through. -First time you update pi-chrome to a version that supports real-looking clicks, Chrome will ask you to re-approve the extension. Open `chrome://extensions` and accept the new permission once. +--- -## Background mode +## Click & input modes -By default, `chrome_*` tools focus Chrome and activate the target tab so you can watch the agent work — great for demos, pair-driving, debugging, and first-time confidence that things are happening. +`pi-chrome` can drive Chrome two ways: -When you want quiet (planner / audit / worker sessions running alongside your editor), turn background mode on for the whole Pi session: +- **Quiet** — synthetic DOM events. Fast, no UI banners. Drives React/Vue/Angular state. Won't satisfy autoplay, clipboard, file picker, fullscreen, or user-activation gates. +- **Trusted** — `chrome.debugger` / CDP under the hood. Indistinguishable from a person clicking. Shows Chrome's *"Pi Chrome Connector started debugging this browser"* banner while active. ```text -/chrome quiet # toggle -/chrome quiet on # explicit -/chrome quiet off # explicit +/chrome clicks auto # default: quiet, upgrade to trusted only when needed +/chrome clicks off # always quiet, never banner +/chrome clicks on # always trusted, banner stays up +/chrome clicks status ``` -For a single tool call, the agent can pass `background: true` directly. The per-call value always wins over the session toggle. +Per-call `trusted: true / false` on any input tool wins over the global mode. -## Quick demo prompts +--- -After setup, try one of these in Pi: +## Background / watch modes -Silent inspection (no Chrome interruption): +By default, every `chrome_*` call focuses Chrome and activates the target tab so you can **watch the agent work** — invaluable for demos, debugging, and first-time confidence. ```text -Inspect my active GitHub tab with chrome_snapshot using background:true and summarize the PR state without focusing Chrome. +/chrome quiet # toggle for the whole session +/chrome quiet on # explicit +/chrome quiet off # explicit ``` -Existing authenticated tab: +Per-call `background: true` wins over the session toggle. + +--- + +## Honest results + +Most browser-automation libraries return `void` or a generic ack. `pi-chrome` returns a structured envelope on every interaction: ```text -Use chrome_tab list to find my existing GitHub tab, chrome_snapshot it, then summarize the visible PR state. Do not click anything yet. +chrome_click(occluded-button) → + "Clicked el-3 — pageMutated=false; occluded by " ``` -Local web app repro with screenshot: - ```text -Use chrome_tab list to find my localhost app, inspect it with chrome_snapshot, navigate through the bug repro flow, and save a screenshot when you reach the broken state. +chrome_type(react-input, "hello") → + "Typed into el-7 — valueMatches=true; pageMutated=true" ``` -## Recipes +This is why agents using pi-chrome don't get stuck in retry loops on broken sites. They get the **reason** the action didn't land and can fix course in one turn. -Copy-paste these into Pi after setup. Each one uses tabs you already have open and accounts you are already signed into. +--- -- **PR triage:** "Use chrome_tab list to find my GitHub notifications tab, snapshot it, and summarize PRs needing my review." -- **Linear standup:** "Open my Linear current cycle in the active tab, snapshot it, and write me a 5-bullet standup." -- **Bug repro with evidence:** "Open the staging app I'm already signed into, reproduce , and save a screenshot of each step under ./repro/." -- **Form auto-fill (no submit):** "Open portal, fill the new-vendor form from this JSON, but stop before submit." -- **Admin cross-check:** "Across my Stripe / Postmark / our admin tabs, find any user where state disagrees." -- **Local dev visual diff:** "Snapshot localhost:3000 and the staging URL of the same page; tell me what's visually different." -- **Auth-only data pull:** "Open my analytics dashboard tab and chrome_evaluate to extract today's KPIs from the page state." +## Diagnostics -Screenshots save under `.pi/chrome-screenshots/` by default, which composes nicely with PR demo workflows. +- `/chrome doctor` — single command: connectivity, extension version, bridge owner, version drift, MAIN-world helper injection, `chrome_evaluate("1+1") === 2`, fingerprint flags. +- `/chrome onboard` — guided first-time setup. +- `/chrome quiet status`, `/chrome clicks status` — current modes. -## Diagnostics +If the loaded Chrome extension is older than `pi-chrome` on disk, `/chrome doctor` tells you to reload it from `chrome://extensions`. -- `/chrome doctor` — single command that checks connectivity and reports the loaded Chrome extension ID + version, plus a one-line fix for common setup failures (extension not loaded, bridge owner stale after `pi update`, version mismatch between pi-chrome and the loaded Chrome extension). +--- -If the Chrome extension you have loaded is older than `pi-chrome` on disk, `/chrome doctor` will tell you to reload it from `chrome://extensions`. +## Composes with -## Compose with +- **[pi-qq](https://www.npmjs.com/package/pi-qq)** — `/qq summarize what the active GitHub tab shows` without polluting the main transcript. +- **[pi-bar](https://www.npmjs.com/package/pi-bar)** — when the agent scrapes large pages, watch the context-usage segment turn yellow → red as a signal to `/qq` for a recap. +- **PR demo skills** — screenshots write to `.pi/chrome-screenshots/` so you can attach them to PR descriptions or demo bundles. -- **pi-qq** — ask side questions about what the agent saw in Chrome without polluting the main transcript: `/qq summarize what the active GitHub tab shows`. -- **pi-bar** — watch context pressure as the agent scrapes large pages; the footer's red threshold is a clean signal to `/qq` for a recap before context overflows. -- **PR demo skills** (such as `ios-pr-agent` / `ios-demo-record` workflows) — `chrome_screenshot` writes to `.pi/chrome-screenshots/` so you can attach images to PR descriptions or demo bundles. +--- -## Tools +## Why an unpacked Chrome extension? -The package registers these Pi tools: +`pi-chrome` cannot ship through the Chrome Web Store — a Web Store extension cannot talk to a local bridge controlled by another tool on the same machine. So it ships as a small MIT-licensed unpacked extension in [`extensions/chrome-profile-bridge/browser-extension/`](./extensions/chrome-profile-bridge/browser-extension). **Read the source before loading.** `/chrome doctor` reports the extension version and warns when it drifts from your installed `pi-chrome`. -- `chrome_launch` — setup/help entry point; opens a URL if the bridge is connected -- `chrome_tab` — list, create, activate, close, or inspect tabs -- `chrome_snapshot` — inspect title, URL, visible text, viewport, and clickable/focusable selectors -- `chrome_navigate` — navigate an existing tab -- `chrome_evaluate` — evaluate JavaScript in a tab -- `chrome_click` — click by selector or coordinates -- `chrome_type` — type text, optionally focusing a selector first -- `chrome_key` — send keyboard keys -- `chrome_wait_for` — wait for a selector or expression -- `chrome_screenshot` — capture viewport screenshots to disk +--- -These tools are especially useful for authenticated web app debugging, repro flows, admin workflows, visual checks, and inspecting local development pages without rebuilding login state. +## Security model -
-How it works (technical details) +The companion extension runs in the Chrome profile where you install it and has broad tab/scripting permissions. Only install it from a package source you trust. -Pi starts a local bridge on `127.0.0.1:17318`. The companion Chrome extension, installed in your normal Chrome profile, polls that local bridge for commands and executes them using Chrome extension APIs. +The Pi side listens on `127.0.0.1:17318` by default. Override before starting Pi: -If another Pi session is already running the bridge, additional Pi sessions automatically act as clients of that shared bridge. This lets planner/audit/worker sessions all use the same authenticated Chrome profile concurrently without fighting over the port. +```bash +PI_CHROME_BRIDGE_PORT=17319 pi +``` -This is intentionally different from CDP-based tools: the browser extension lives inside the profile you already use, so Pi can interact with existing tabs and authenticated page state. +There is no network exposure; the bridge binds to loopback only. -
+--- -## Security model +## Built-in benchmark suite -The companion Chrome extension runs in the Chrome profile where you install it and has broad tab/scripting permissions. Only install it from a package source you trust. +[`test-suite/`](./test-suite) is a benchmark for **any** browser-control agent (not just pi-chrome). It includes **38 primitive challenges** plus **4 hermetic BrowserGym-style long-horizon tasks**. -The Pi side listens on `127.0.0.1:17318` by default. Override before starting Pi with: +Scoring is **expected-outcome-by-mode**, not raw PASS count: each challenge has an expected verdict per mode (`synthetic`, `trusted`, `manual`) and a tool grades itself by whether its actual outcome matches the expected one. This avoids false equivalence between modes — a synthetic-events tool isn't supposed to satisfy a clipboard user-activation gate; matching that expectation is the pass. + +Each challenge exposes `window.__verdict` / `window.__reason` / `window.__events` and a manifest entry with expected results per mode. ```bash -PI_CHROME_BRIDGE_PORT=17319 pi +cd test-suite && python3 -m http.server 8765 +# open http://127.0.0.1:8765/ in the Chrome window pi-chrome controls ``` + +Categories: `trusted-input`, `pointer-humanization`, `keyboard`, `activation-gates`, `scroll`, `drag-drop`, `clipboard`, `native-controls`, `frameworks`, `editing`, `dom-complexity`, `frames`, `files`, `observability`, `fingerprint`, `agent-safety`. + +If you build a competing tool, please open a PR with your scores. We benchmark in public. + +--- + +## Roadmap signals + +`pi-chrome` is actively shipped. Things on the near roadmap: + +- More observability tools (DOM mutation streams, performance traces) +- First-class iframe + Shadow-DOM uid stability across snapshots +- Web Push & service worker introspection +- Recorder mode that emits agent prompts from your own clicks + +If you want one of those next, open an issue. + +--- + +## Contributing + +PRs welcome. The bar: + +1. Add a benchmark page in `test-suite/` that fails before your change and passes after. +2. Keep `chrome_*` tool results honest — surface `pageMutated`, `valueMatches`, `defaultPrevented`, etc. +3. Don't break the "no re-login" guarantee. Anything that requires a fresh profile is out of scope. + +--- + +## License + +MIT. See [LICENSE](./LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..8627ca6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,38 @@ +# Security policy + +## Reporting a vulnerability + +Open a GitHub issue prefixed with `[security]` at https://github.com/tianrendong/pi-packs/issues, or contact the maintainer directly if the issue is sensitive. Please do **not** include exploit details in a public issue without coordinating first. + +## Threat model + +`pi-chrome` is a developer tool you install knowingly. It is **not** designed to defend against: + +- Hostile pages running in your Chrome trying to detect or escape automation. (Standard browser security boundaries still apply, but a hostile page that already runs in your tab can do anything that page can already do.) +- Other processes on your local machine. The bridge binds to `127.0.0.1:17318` (loopback only) but **does not authenticate** local callers. Any process running as your user can issue commands. If your threat model includes hostile local processes, run pi-chrome on a separate user account. + +`pi-chrome` **is** designed to: + +- Never exfiltrate page state to the network. All communication is loopback (`127.0.0.1`). +- Surface every action with an honest result envelope so the agent can't silently do the wrong thing. +- Require explicit opt-in for trusted-input mode (`/chrome clicks on` or `trusted: true`), which uses `chrome.debugger` and shows Chrome's banner. + +## The companion extension + +The Chrome extension under `extensions/chrome-profile-bridge/browser-extension/` runs with broad permissions: `tabs`, `scripting`, `debugger`, `webNavigation`, etc. **Only install it from a package source you trust.** Read the source before loading. Pin a known-good commit if you're security-sensitive. + +## Defaults + +- Loopback bridge only. No remote port. No telemetry. +- Synthetic events first; trusted CDP only when explicitly enabled. +- Quiet mode optional; tab/window focus is observable (the user can see Pi acting). + +## Override the port + +```bash +PI_CHROME_BRIDGE_PORT=17319 pi +``` + +## Supported versions + +The latest minor on npm is supported. Security patches will be released as soon as practical. diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md new file mode 100644 index 0000000..4f51291 --- /dev/null +++ b/docs/COMPARISON.md @@ -0,0 +1,166 @@ +# pi-chrome vs. the rest of the browser-automation landscape + +This is the honest "which tool when" page. The browser-automation space has three different layers and people often compare across them — let's not do that. + +We benchmark in public — see [`../test-suite/`](../test-suite). Where exact scores matter (WebVoyager, WorkArena++, BrowseComp, Mind2Web 2), check the live leaderboards; they shift monthly. + +--- + +## TL;DR + +| You are… | Use… | +| -------------------------------------------------------------- | ------------------------------- | +| A Pi agent operator who wants the agent to use **your real Chrome** (logged-in tabs, cookies, extensions) | **pi-chrome** | +| Building a Pi/LLM agent and want low-level browser primitives | **pi-chrome** | +| Writing deterministic end-to-end tests in CI | Playwright / Cypress | +| Building a hosted scraping/agent fleet on isolated profiles | Playwright / Puppeteer + Browserbase / Steel | +| Want a turnkey "natural-language agent" with built-in loop | Browser Use / Stagehand / Skyvern | +| Want a hosted, vendor-managed agent | OpenAI Operator / Project Mariner / Surfer | +| Debugging your own app from your editor without leaving your real session | **pi-chrome** | + +`pi-chrome` is **primitives**, not an opinionated agent loop. Think of it as **"Playwright for the Chrome you're already signed into"** — and pluggable under any agent framework above the line. + +--- + +## The three axes you should compare on + +| Axis | Examples | What it gives you | +| --------------------------------- | --------------------------------------------------------------------- | ----------------------------- | +| **1. Driver / transport** | Playwright, Puppeteer, Selenium, CDP raw, `puppeteer-stealth`, **pi-chrome** | low-level tools (click, type, navigate) | +| **2. Agent framework** | Browser Use, Stagehand, Skyvern, Magnitude, Alumnium, LangGraph-with-Playwright, Operator, Mariner, Surfer | LLM loop, planning, NL API | +| **3. Cloud browser provider** | Browserbase, Steel.dev, Hyperbrowser, Anchor, Browserless | managed Chromes, sessions, quotas | + +`pi-chrome` is **axis 1**. You can wrap it with any axis-2 agent framework, or run it directly from a Pi agent's `chrome_*` tool calls. It explicitly does NOT compete with axis 3 — it runs locally inside *your* Chrome. + +--- + +## Axis 1 — drivers (where pi-chrome lives) + +| Tool | Transport | Profile | Trusted events | Banner when controlling | Default detectable as bot | +| --------------------------------- | ------------------------------------------ | ---------------------------------- | -------------------- | ----------------------------------- | ------------------------- | +| Playwright | CDP (own driver) | throwaway by default | always | always ("controlled by test software") | yes (webdriver flag, automation flags) | +| Puppeteer | CDP | throwaway by default | always | always | yes | +| Selenium | WebDriver / BiDi | throwaway | partial (BiDi improves) | always | most detectable | +| puppeteer-stealth / playwright-extra | CDP + patches | throwaway | always | always | medium (patches flags) | +| Raw CDP | direct devtools protocol | either (needs `--remote-debugging-port`) | always | always | yes | +| **pi-chrome** | **Chrome extension bridge → local loopback** | **your real Chrome profile, signed-in cookies, extensions, history** | **opt-in** (`/chrome clicks on` or `trusted: true`) | **only when trusted mode is active** | **synthetic mode bypasses common detection signals**¹ | + +¹ pi-chrome synthetic mode dispatches DOM events with `isTrusted=false` — most sites don't check; some anti-bot defenses do. The [`test-suite/`](../test-suite) grades both modes against common detection signals. Trusted mode uses `chrome.debugger` and shows Chrome's banner like every other CDP-based tool. + +### What makes pi-chrome different on this axis + +1. **Profile attach, not driver launch.** Every other driver fights cookie persistence, login walls, MFA, and extension state. pi-chrome inherits all of it because it *is* your Chrome. +2. **Synthetic-first, trusted-on-demand.** Two-tier event model — agents pick the right tradeoff per call. Competitors are all-trusted (CDP) and always show the banner. pi-chrome avoids it by default; you opt in when a site needs it (autoplay gate, clipboard, file picker). +3. **Extension bridge transport.** No `--remote-debugging-port`, no throwaway Chromium. Survives Chrome auto-updates. Works alongside your normal Chrome usage. +4. **Honest result envelopes.** Every action returns `pageMutated`, `defaultPrevented`, `elementVisible`, `occludedBy`, `valueMatches`. Competitors return `void` or generic acks; agents loop blindly on broken clicks. +5. **Multi-session shared bridge.** Planner + worker + audit Pi sessions all drive the same Chrome concurrently. +6. **Stable element uids.** `chrome_snapshot` returns deterministic uids you can pass to subsequent actions — similar to BrowserGym's `bid`, but built into the snapshot tool itself. + +--- + +## Axis 2 — agent frameworks (built on top of axis 1) + +These wrap a driver with an LLM loop. They are **higher-level than pi-chrome** and **complementary**, not competitors. + +| Framework | Driver underneath | Approach | Open source | +| ------------------------ | ------------------------------ | --------------------------------------------------------------------------------------------- | --------------- | +| **Browser Use** | Playwright | DOM + a11y tree → LLM → action JSON. Open-source leader; widely cited on WebVoyager. | MIT (Python) | +| **Stagehand** (Browserbase) | Playwright | Natural-language `.act()` / `.observe()` / `.extract()`; deterministic + AI mix. | MIT (TypeScript)| +| **Skyvern** | Playwright + own DOM model | Vision-first + DOM; YAML workflows for form/workflow automation. | AGPL (Python) | +| **Magnitude** | Playwright | NL test authoring; QA-focused. | open | +| **Alumnium** | Selenium / Playwright | Test-author NL → agent. QA-focused. | open | +| **LangGraph / AutoGen + Playwright** | Playwright | Generic agent graph + browser tools. | open | +| **OpenAI Operator** | proprietary | OpenAI's own VLM + browser; ChatGPT-integrated. | closed, hosted | +| **Project Mariner** (Google) | proprietary Chrome integration | Google's own VLM Chrome experiment. | closed | +| **Surfer 2 / Surfer-H** (H Company) | proprietary | Hosted proprietary agent stack. | closed, hosted | +| **Anthropic Computer Use** | OS-level screenshots + mouse/keyboard | Broader than browser; trusted events at OS level. | closed (API) | + +**Why pi-chrome is not on this list:** it's intentionally **not an agent**. There's no LLM loop, no `.act("click the blue button")`. Pi handles the loop; pi-chrome provides the primitives. This means: + +- You can use pi-chrome under Browser Use, Stagehand, LangGraph, or any other agent framework (see [Interop](#interop)). +- You don't pay for an opinion you don't want. +- Agent improvements compound across all your workflows because the primitives stay stable. + +--- + +## Axis 3 — cloud browser providers (orthogonal) + +| Provider | What it sells | +| ----------------- | -------------------------------------------------------- | +| **Browserbase** | Managed browsers; pairs with Stagehand. | +| **Steel.dev** | Managed browsers + public agent leaderboards. | +| **Hyperbrowser** | Managed browsers, session APIs. | +| **Anchor Browser**| Managed browsers. | +| **Browserless** | Managed Chrome, scraping focus. | + +**pi-chrome doesn't compete here.** It runs locally in *your* Chrome. The right framing if someone asks: *"no cloud cost, no session-handoff, no rate limits — and the agent runs against your real logged-in state."* If you need fleets of isolated Chromes in CI, you want one of these. + +--- + +## Interop + +`pi-chrome` exposes tools that any Pi agent can call. If you want to use it from outside Pi: + +1. The local bridge speaks HTTP JSON-RPC at `127.0.0.1:17318` (default). The API is internal but stable across patch versions. +2. Tool surface mirrors Playwright closely (click/type/navigate/snapshot/screenshot/evaluate/wait_for) so adapter code is short. +3. Honest envelopes (`pageMutated`, `valueMatches`, `occludedBy`) let agent harnesses skip retry/heal logic. + +If you want a first-class pi-chrome adapter for Browser Use / Stagehand / LangGraph, file an issue with your use case. + +--- + +## "But Playwright has `storageState` / Puppeteer has user-data-dir" + +Yes — you can export cookies and replay them, or point Playwright at your existing profile directory. In practice for agent workflows that breaks down fast: + +1. **OAuth + SSO** providers (Okta, Google, GitHub) often pin sessions to TLS fingerprints, device IDs, and browser-extension state that doesn't survive replay or a parallel Chrome instance. +2. **MFA** tokens expire mid-run. +3. **Internal admin tools** hard-pin to your real device. +4. **Pointing Playwright at your real `user-data-dir`** requires closing your normal Chrome (Chrome won't share the profile lock). pi-chrome doesn't fight you for the profile because it lives *inside* it. +5. **Watching the agent work** in your real window is a different UX than a hidden parallel Chrome. Demos, pair-driving, and confidence-building all want axis-1-with-attach. + +--- + +## "Is this safer than CDP?" + +Different security boundary, not strictly safer. + +- **CDP-based tools** require `chrome --remote-debugging-port=...`. That port is unauthenticated and exposes the whole browser to any local process. Easy to misconfigure. +- **pi-chrome** runs through an extension you install yourself with broad permissions (tabs, scripting, debugger, webNavigation). The bridge listens on `127.0.0.1:17318` loopback only. **Only install the bundled extension if you trust the source you got the npm package from.** + +If your threat model excludes extensions with broad permissions, neither approach is a fit — you want a sandboxed CI runner. + +--- + +## Public benchmarks worth knowing (for axis 2 / axis 3 comparison) + +Pi-chrome itself ships a benchmark suite ([`../test-suite/`](../test-suite)) of **38 primitive challenges** plus **4 hermetic BrowserGym-style long-horizon tasks** covering trusted-input, pointer humanization, keyboard fidelity, drag/drop, Shadow DOM, file uploads, network observability, fingerprint leaks, and agent-safety honeypots. Scoring is **expected-outcome-by-mode** (not raw PASS count): each challenge has expected verdicts per mode (`synthetic` / `trusted` / `manual`) and a tool grades itself by whether its actual outcome matches expectations. That's **driver-level** grading. + +For **agent-level** comparison (axis 2), the public benchmarks worth citing: + +| Benchmark | What it measures | Notes | +| ---------------- | ------------------------------------------------- | ------------------------------------------------ | +| **WebArena** (CMU) | Hermetic, programmatic graders | Gold standard for reproducibility. | +| **WorkArena++** (ServiceNow) | Enterprise SaaS workflows | Hardest realistic benchmark; <5% frontier. | +| **BrowseComp** (OpenAI) | Hard info-retrieval | Not saturated. | +| **Mind2Web 2** (NeurIPS '25) | Long-horizon, rubric-tree judge | New, well-designed. | +| **WebChoreArena** | Tedious cross-page workflows | Reflects real ops work. | +| **WebVoyager** | Live web tasks | **Saturated** — 90%+ scores common; cite only as smoke test. | +| **VisualWebArena** | Multimodal | | +| **MiniWoB++** | Classic unit-task suite | | +| **BrowserGym + AgentLab** | Research harness covering the above | The de-facto research API; pi-chrome's snapshot uid is comparable to BrowserGym's `bid`. | + +Cite live leaderboards rather than hard-coded numbers; agent scores shift monthly. + +--- + +## Reproducing pi-chrome's driver-level claims + +Run [`../test-suite/`](../test-suite) against any browser-control tool. Each challenge exposes `window.__verdict` / `window.__reason` / `window.__events`, so any tool (Playwright, Puppeteer, Selenium, Stagehand, pi-chrome) can grade itself deterministically. + +```bash +cd test-suite && python3 -m http.server 8765 +# open http://127.0.0.1:8765/ in the Chrome window the tool controls +``` + +If you build a competing tool, please open a PR with your scores. diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md new file mode 100644 index 0000000..905d278 --- /dev/null +++ b/docs/EXAMPLES.md @@ -0,0 +1,166 @@ +# pi-chrome examples + +Real, useful agent prompts. Drop any of these into Pi after running `/chrome onboard`. Each one uses Chrome tabs and accounts you already have. + +## Daily workflow + +### PR triage + +```text +Use chrome_tab list to find my GitHub notifications tab. +chrome_snapshot it. Group PRs by: + - awaiting my review + - blocked on me (changes requested back) + - mergeable (approved + green CI) +Output a 5-bullet ranked triage. Do not click anything. +``` + +### Linear standup + +```text +Open my Linear current cycle in the active tab. +chrome_snapshot, then write yesterday/today/blockers +in the exact format my standup channel uses. +``` + +### Slack catch-up + +```text +For each unread channel in my Slack tab, chrome_snapshot, +extract the top 3 messages that mention me or my team, +and summarize what I missed in <100 words total. +``` + +## Debugging + +### Reproduce a customer bug with evidence + +```text +1. chrome_navigate to https://staging.acme.com/orders/ +2. chrome_snapshot +3. Click "Refund" with chrome_click +4. Use chrome_list_network_requests to capture the API call +5. chrome_get_network_request on the failing one — give me the response body +6. chrome_screenshot the error state to ./repro/refund-bug.png +``` + +### Visual diff local vs staging + +```text +chrome_screenshot http://localhost:3000/pricing → ./diff/local.png +chrome_screenshot https://staging.acme.com/pricing → ./diff/staging.png +Then read both, describe layout differences in plain English. +``` + +### Console + network forensics + +```text +Reproduce the checkout bug on the active tab. +After the failure: + - chrome_list_console_messages + - chrome_list_network_requests +Cross-reference the timestamps and tell me what broke first. +``` + +## Admin / ops + +### Multi-tab cross-check + +```text +I have Stripe, Postmark, and our internal admin open in 3 tabs. +For user , chrome_snapshot each tab in turn and find +any field where state disagrees. Output a 3-column table. +``` + +### Bulk gentle action (safe form-fill, no submit) + +```text +Open our vendor portal "Add Vendor" form. +For each row in ./vendors.csv: + - chrome_fill the form + - chrome_screenshot it + - STOP before submit + - chrome_evaluate "history.back()" to return to the list +I will review screenshots and submit manually. +``` + +### Auth-only data pull + +```text +My analytics dashboard is open and the cookie auth would die in headless mode. +chrome_evaluate to read window.__APP_STATE__.dashboardData +and dump today's KPIs as JSON. +``` + +## Demos / PRs + +### Capture screenshots for a PR description + +```text +On localhost:3000/feature-x: + - empty state → ./pr/01-empty.png + - filled state → ./pr/02-filled.png + - error state (delete the API key from devtools first) → ./pr/03-error.png +Save each with chrome_screenshot. Output a markdown block I can paste into the PR. +``` + +### Record a guided demo flow + +```text +On my staging app: +1. Walk the new-onboarding flow start to finish +2. After each chrome_click or chrome_navigate, chrome_screenshot +3. Save numbered PNGs under ./demo/ +4. Write narration captions for each step +``` + +## Forms with frameworks + +### React controlled inputs + +```text +chrome_fill (not chrome_type) for React inputs — it uses the +framework-aware native value setter so the form's state actually updates. +After each fill, the result envelope's valueMatches=true confirms the +component re-rendered with the new value. +``` + +### File upload without the native picker + +```text +chrome_upload_file paths=[./fixtures/avatar.png] selector="input[type=file]" +# No native file picker opens. Works with React/Vue/Angular controlled inputs. +``` + +### Drag-to-reorder lists + +```text +chrome_drag fromUid=row-3 toUid=row-1 +# Fires real HTML5 dragstart/dragover/drop with a shared DataTransfer. +``` + +## Multi-session patterns + +`pi-chrome` shares one bridge across all Pi sessions on the same machine. Useful patterns: + +### Planner + Worker + +- **Planner session** stays high level: "find the bug, decide the fix." +- **Worker session** runs the actual `chrome_*` tools. +- Both see the same Chrome state because they're both pointing at your real profile. + +### Watcher + +A third Pi session can run `chrome_snapshot` periodically in `background: true` mode and post summaries via `pi-qq` — handy for long-running flows. + +## When to prefer trusted clicks + +Pass `trusted: true` on `chrome_click` (or run `/chrome clicks on`) when: + +- the click should open a file picker +- the click should write to the clipboard or read it +- the click should start an audio/video play +- the click should request fullscreen / push permission +- the page is wrapped in a strict user-activation guard (some paywalls / login flows) + +Everything else is faster and quieter without it. diff --git a/docs/FAQ.md b/docs/FAQ.md new file mode 100644 index 0000000..2839f8f --- /dev/null +++ b/docs/FAQ.md @@ -0,0 +1,79 @@ +# pi-chrome FAQ + +## Does this work with Brave / Arc / Edge / Vivaldi? + +Yes. Any Chromium-based browser that supports unpacked extensions and the `chrome.debugger` API will work. The extension is named "Pi Chrome Connector" but the source is browser-agnostic. Firefox / WebKit are out of scope (different extension models). + +## Will it slow my browser down? + +The companion extension is idle when no Pi command is in flight. It uses Manifest V3 service worker activation, so it wakes for a request and goes back to sleep. No content script is injected globally. + +## Does it work in Chrome incognito? + +By default no — extensions need explicit "Allow in incognito" permission. Toggle it on `chrome://extensions` if you want pi-chrome to see incognito tabs. We don't recommend it for sensitive work. + +## Will sites detect that I'm automating? + +For **synthetic** (quiet) input: yes, technically. `event.isTrusted` is `false`. Most sites don't check; some anti-bot defenses do. + +For **trusted** (CDP) input: events are `isTrusted=true`, pointer paths are humanized, key cadence has variance. Most fingerprint-based detectors don't fire. Some specifically check for the `chrome.debugger` API attached and will show the "Chrome is being debugged" banner. That banner is the visible cost of trusted mode. + +The [`test-suite/`](../test-suite) grades both modes against common detection signals. + +## Why do I see a banner saying "Pi Chrome Connector started debugging this browser"? + +That's Chrome's built-in warning when an extension uses `chrome.debugger`. pi-chrome uses it only in trusted-input mode. If you don't want to see it, run `/chrome clicks off` and accept that some sign-in flows / file pickers / clipboard ops won't work. + +## Can a malicious page escape and access my other tabs? + +No — pages cannot directly talk to extensions. Commands flow agent → local bridge (`127.0.0.1:17318`) → extension → tab. The bridge binds to loopback only. The risk surface is **other local processes** that could connect to `127.0.0.1:17318` and impersonate Pi. If that's in your threat model, run pi-chrome on a separate user account. + +## Can multiple Pi sessions use it at once? + +Yes. The first session opens the local bridge; later sessions detect it and pipe their commands through the same bridge. Planner + worker + audit can all drive the same Chrome concurrently. + +## Why can't this be on the Chrome Web Store? + +Web Store extensions cannot communicate with a local process bridge controlled by another tool — Google's policy. pi-chrome must ship as an unpacked extension you load yourself. The upside: you can read the source. The downside: each Chrome update may prompt you to re-confirm. + +## What happens when I update pi-chrome? + +`/chrome doctor` will warn you if the loaded extension is older than the installed `pi-chrome`. Reload it from `chrome://extensions` to pick up the new version. Trusted-input mode in particular requires re-approving the `debugger` permission once. + +## What's the install footprint? + +- Pi side: one extension that registers ~20 tools and a few slash commands. +- Chrome side: one unpacked extension, ~5000 LOC of plain JavaScript, no dependencies. + +## Can I script it without Pi? + +The Pi-facing tools are thin wrappers around an HTTP bridge at `127.0.0.1:17318`. You could call it directly from any process, but the API is internal and may change. If you need a stable scripting interface, file an issue and we'll consider stabilizing. + +## Does `chrome_evaluate` work on strict-CSP pages? + +Yes. The handler compiles with `new Function(...)` in the MAIN world, which works under `script-src 'self'` without `'unsafe-eval'`. Multi-statement bodies are supported via a statement-mode fallback. Exceptions are surfaced to the agent. + +## Why does my click return `pageMutated=false`? + +Either: +- The element was occluded (look for `occludedBy: ` in the envelope). +- The click handler called `event.preventDefault()` and the page intentionally ignored it. +- The site rejects synthetic events. Try `trusted: true` or `/chrome clicks on`. + +The result envelope tells you which one. **Don't blind-retry.** + +## Why does `chrome_type` return `valueMatches=false`? + +The editor rejected the synthetic input. Common culprits: contenteditable rich-text editors, native date pickers, masked-input libraries. Try `chrome_fill` (uses framework-aware native setters) or `trusted: true`. + +## How do I attach a file to a React file input? + +`chrome_upload_file` — populates `input.files` via a real `DataTransfer` and fires `input` + `change` events. It does **not** open the native file picker (no synthetic event can; that's a user-activation gate). Works with React/Vue/Angular controlled inputs. + +## Can it record videos? + +Not yet. Screenshots only. Video recording is on the roadmap. + +## How do I file a good bug report? + +Include `/chrome doctor` output, the exact tool call, and the result envelope. If the page is public, link to it; if private, distill it into a benchmark page under `test-suite/challenges/`. See [CONTRIBUTING.md](../CONTRIBUTING.md). diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json index f0f8a6a..eccc6d5 100644 --- a/extensions/chrome-profile-bridge/browser-extension/manifest.json +++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "Pi Chrome Connector", - "version": "0.14.2", + "version": "0.14.7", "description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.", "permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation", "debugger"], "host_permissions": ["", "http://127.0.0.1:17318/*"], diff --git a/package.json b/package.json index dcaaa0e..316dcc9 100644 --- a/package.json +++ b/package.json @@ -1,22 +1,55 @@ { "name": "pi-chrome", - "version": "0.14.2", - "description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).", + "version": "0.14.7", + "description": "The de-facto browser automation toolkit for Pi agents. Drive your existing logged-in Chrome — no re-login, no throwaway profile, no CDP. 20+ tools (click, type, navigate, screenshot, network capture, file upload, drag, scroll, touch) + honest result envelopes + a built-in benchmark suite.", "keywords": [ + "pi", "pi-package", "pi-extension", + "pi-agent", "chrome", + "chrome-extension", "browser", - "automation", + "browser-automation", + "browser-control", + "web-automation", + "agent", + "ai-agent", + "llm-agent", + "llm-tools", + "agentic", + "playwright-alternative", + "puppeteer-alternative", + "selenium-alternative", + "cdp-alternative", "authenticated-session", "real-profile", - "web-debugging" + "web-debugging", + "web-scraping", + "screenshot", + "automation-tools", + "browser-use", + "stagehand-alternative" ], "license": "MIT", + "homepage": "https://github.com/tianrendong/pi-packs/tree/main/packages/pi-chrome#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/tianrendong/pi-packs.git", + "directory": "packages/pi-chrome" + }, + "bugs": { + "url": "https://github.com/tianrendong/pi-packs/issues" + }, "type": "commonjs", "files": [ "extensions", - "README.md" + "docs", + "README.md", + "CHANGELOG.md", + "CONTRIBUTING.md", + "SECURITY.md", + "LICENSE" ], "pi": { "extensions": [ diff --git a/test-suite/README.md b/test-suite/README.md index bb8becb..a96e0d0 100644 --- a/test-suite/README.md +++ b/test-suite/README.md @@ -1,13 +1,14 @@ # pi-chrome browser-control benchmark -Static benchmark pages for evaluating tools that let agents control Chrome. The -suite covers two related questions: +Static benchmark pages for evaluating tools that let agents control Chrome. The suite has two layers: -1. **Can the agent complete normal browser work?** Forms, scroll containers, - contenteditable, files, frames, Shadow DOM, network/console inspection. -2. **Does the interaction look like a human/browser action when sites care?** - `isTrusted`, user activation, pointer paths, key cadence, native controls, - drag/drop, touch, paste, and scroll momentum. +1. **Unit challenges** (`manifest.json`) — MiniWoB-style capability probes for + forms, scroll containers, contenteditable, files, frames, Shadow DOM, + network/console inspection, `isTrusted`, user activation, pointer paths, key + cadence, native controls, drag/drop, touch, paste, and scroll momentum. +2. **Long-horizon hermetic tasks** (`task-manifest.json`) — WebArena / + BrowserGym-inspired multi-step tasks with fresh run IDs and deterministic + programmatic graders. ## Run @@ -24,11 +25,17 @@ Each challenge page exposes: - `window.__reason` — array of reasons - `window.__events` — raw event log for forensics -`manifest.json` is the source of truth for benchmark metadata: category, goal, -expected result per mode, prerequisites, flake risk, manual baseline status, and -canonical tool recipe. `manifest.schema.json` documents the manifest shape. -Recipes express tool intent; runners may need to adapt descriptive selectors -(e.g. shadow/iframe notation) and expand path placeholders like `$PWD`. +`manifest.json` is the source of truth for unit-challenge metadata: category, +goal, expected result per mode, prerequisites, flake risk, manual baseline +status, and canonical tool recipe. `manifest.schema.json` documents the manifest +shape. Recipes express tool intent; runners may need to adapt descriptive +selectors (e.g. shadow/iframe notation) and expand path placeholders like `$PWD`. + +`task-manifest.json` is the source of truth for long-horizon tasks: BrowserGym-style +`taskId`, seed, viewport, goal object, difficulty tier, max steps, declared +action subsets, reset/setup URL, validate hook, optional cheat recipe, and +programmatic grader expression. `task-manifest.schema.json` documents this shape. +`browsergym-action-space.json` records BrowserGym-compatible action subsets. ## Modes / expected outcomes @@ -49,7 +56,7 @@ Expected values in `manifest.json`: Manual baselines are tracked separately with `manualBaseline`. `unverified` means the manual expectation is a target, not a recorded contract. -## Recommended agent flow +## Recommended unit-challenge agent flow 1. Navigate to dashboard: `http://127.0.0.1:8765/`. @@ -70,6 +77,29 @@ means the manual expectation is a target, not a recorded contract. 4. Return to dashboard and compare actual verdicts with expected values. 5. Copy JSON report from dashboard for PRs or regression notes. +## Recommended long-horizon task flow + +1. Load `task-manifest.json`. +2. Replace `$RUN_ID` in `startUrl` with a fresh value. +3. Navigate to the start URL and read the visible task instruction. +4. Solve using normal browser tools only; avoid direct state mutation unless the + benchmark mode explicitly allows evaluate-based actions. +5. Click **Grade now** or evaluate the task grader expression: + ```js + JSON.stringify({ v: window.__taskVerdict, r: window.__taskReason }) + ``` +6. Record action count, observations used, tools used, verdict, and reason. + +## Design principles copied from browser-agent benchmarks + +- Prefer hermetic sites and deterministic graders over live sites and LLM judges. +- Report action API and observation format; these strongly affect scores. +- Use difficulty tiers: L1 atomic, L2 compositional, L3 cross-page/context-rich. +- Include tedious cross-page memory and exact-value transfer tasks; short unit + probes hide these failures. +- Keep synthetic-event-gated tests because extension bridges face failures that + CDP/Playwright-style benchmarks usually do not measure. + ## Challenge categories - `trusted-input` — browser-trusted click/key events. @@ -136,3 +166,7 @@ The dashboard renders this from `manifest.json`. In brief: software WebGL in VMs, remote desktops, or policy-constrained environments. - `notes/bypass-ideas.md` is historical guidance for older synthetic-only versions. Prefer `manifest.json` for current expected outcomes. +- `notes/browsergym-compat.md` defines the reset/step/validate/observation/BID + contract for external BrowserGym-style agents. +- `notes/runner-spec.md`, `notes/scoring.md`, and `notes/profiles.md` define + runner output, scoring, retry policy, and environment metadata. diff --git a/test-suite/_lib.js b/test-suite/_lib.js index 95d6393..f3845e5 100644 --- a/test-suite/_lib.js +++ b/test-suite/_lib.js @@ -8,6 +8,8 @@ verdict: "PENDING", reason: [], events, + details: [], + thresholds: {}, }; function render() { @@ -31,8 +33,9 @@ } const Challenge = { - init({ id, instructions }) { + init({ id, instructions, thresholds = {} }) { state.id = id; + state.thresholds = parseThresholds(thresholds); document.title = `[${state.verdict}] ${id}`; const root = document.body; const bar = document.createElement("div"); @@ -82,6 +85,20 @@ document.title = `[WARN] ${state.id}`; persist(); render(); }, + partial({ name, pass, reason, data }) { + state.details.push({ name, pass: !!pass, reason: reason || "", data }); + log("partial", { name, pass: !!pass, reason, data }); + persist(); render(); + return !!pass; + }, + finishPartials() { + const failed = state.details.filter(d => !d.pass); + if (failed.length) Challenge.fail(...failed.map(d => `${d.name}: ${d.reason || "failed"}`)); + else Challenge.pass(...state.details.map(d => `${d.name}: ok`)); + }, + getThreshold(name, fallback) { + return Object.prototype.hasOwnProperty.call(state.thresholds, name) ? state.thresholds[name] : fallback; + }, log, state, }; @@ -90,10 +107,24 @@ try { localStorage.setItem( "pi-chrome-suite:" + state.id, - JSON.stringify({ id: state.id, verdict: state.verdict, reason: state.reason, events: state.events.slice(-50), ts: Date.now() }) + JSON.stringify({ id: state.id, verdict: state.verdict, reason: state.reason, details: state.details, thresholds: state.thresholds, events: state.events.slice(-50), ts: Date.now() }) ); } catch {} } + function parseThresholds(defaults) { + const out = { ...defaults }; + try { + const qs = new URLSearchParams(location.search); + for (const [k, v] of qs) { + if (!k.startsWith("threshold.")) continue; + const key = k.slice("threshold.".length); + const num = Number(v); + out[key] = Number.isFinite(num) ? num : v; + } + } catch {} + return out; + } + window.Challenge = Challenge; })(); diff --git a/test-suite/browsergym-action-space.json b/test-suite/browsergym-action-space.json new file mode 100644 index 0000000..5d7c873 --- /dev/null +++ b/test-suite/browsergym-action-space.json @@ -0,0 +1,44 @@ +{ + "subsets": { + "chat": ["send_msg_to_user"], + "infeas": ["report_infeasible"], + "bid": ["click", "dblclick", "hover", "fill", "clear", "press", "focus", "select_option", "scroll", "drag_and_drop", "upload_file"], + "coord": ["mouse_move", "mouse_up", "mouse_down", "mouse_click", "mouse_dblclick", "mouse_drag_and_drop", "mouse_upload_file", "scroll_at", "keyboard_down", "keyboard_up", "keyboard_press", "keyboard_type", "keyboard_insert_text"], + "nav": ["go_back", "go_forward", "goto"], + "tab": ["tab_close", "tab_focus", "new_tab"] + }, + "signatures": { + "fill": "fill(bid, value, enable_autocomplete_menu=false)", + "click": "click(bid, button='left', modifiers=[])", + "dblclick": "dblclick(bid, button='left', modifiers=[])", + "hover": "hover(bid)", + "press": "press(bid, key_comb)", + "focus": "focus(bid)", + "clear": "clear(bid)", + "select_option": "select_option(bid, options)", + "scroll": "scroll(delta_x, delta_y)", + "drag_and_drop": "drag_and_drop(from_bid, to_bid)", + "upload_file": "upload_file(bid, file)", + "mouse_move": "mouse_move(x, y)", + "mouse_click": "mouse_click(x, y, button='left')", + "mouse_dblclick": "mouse_dblclick(x, y, button='left')", + "mouse_up": "mouse_up(x, y, button='left')", + "mouse_down": "mouse_down(x, y, button='left')", + "mouse_drag_and_drop": "mouse_drag_and_drop(from_x, from_y, to_x, to_y)", + "scroll_at": "scroll_at(x, y, dx, dy)", + "keyboard_press": "keyboard_press(key)", + "keyboard_down": "keyboard_down(key)", + "keyboard_up": "keyboard_up(key)", + "keyboard_type": "keyboard_type(text)", + "keyboard_insert_text": "keyboard_insert_text(text)", + "goto": "goto(url)", + "go_back": "go_back()", + "go_forward": "go_forward()", + "tab_close": "tab_close()", + "tab_focus": "tab_focus(index)", + "new_tab": "new_tab()", + "send_msg_to_user": "send_msg_to_user(text)", + "report_infeasible": "report_infeasible(reason)", + "noop": "noop(wait_ms=1000)" + } +} diff --git a/test-suite/challenges/34-dialog-handling.html b/test-suite/challenges/34-dialog-handling.html new file mode 100644 index 0000000..25845a7 --- /dev/null +++ b/test-suite/challenges/34-dialog-handling.html @@ -0,0 +1,28 @@ + + +34 dialog handling + + + +
+

Goal: handle browser dialogs. Click Prompt, enter pi-chrome, accept it; then click Confirm and accept it. Browser automation must expose/dismiss native dialogs.

+ + + + 

+

+
+
diff --git a/test-suite/challenges/35-target-blank-popup.html b/test-suite/challenges/35-target-blank-popup.html
new file mode 100644
index 0000000..0813625
--- /dev/null
+++ b/test-suite/challenges/35-target-blank-popup.html
@@ -0,0 +1,35 @@
+
+
+35 target blank popup
+
+
+
+

+  
Goal: click target=_blank link, detect new tab, switch to it, and verify child page reports PASS.

+  Open child tab
+  

+

+
+
diff --git a/test-suite/challenges/36-modal-focus-trap.html b/test-suite/challenges/36-modal-focus-trap.html
new file mode 100644
index 0000000..811207c
--- /dev/null
+++ b/test-suite/challenges/36-modal-focus-trap.html
@@ -0,0 +1,49 @@
+
+
+36 modal focus trap
+
+
+
+

+  
Goal: open modal, use Tab to cycle focus inside modal only, then Escape closes it and focus returns to opener.

+  
+  
+

+
+
diff --git a/test-suite/challenges/37-autocomplete-combobox.html b/test-suite/challenges/37-autocomplete-combobox.html
new file mode 100644
index 0000000..91f35c2
--- /dev/null
+++ b/test-suite/challenges/37-autocomplete-combobox.html
@@ -0,0 +1,42 @@
+
+
+37 autocomplete combobox
+
+
+
+

+  
Goal: type as, use ArrowDown then Enter to choose Aster Lamp from ARIA combobox.

+  
+  
+

+
+
diff --git a/test-suite/challenges/38-spa-route-change.html b/test-suite/challenges/38-spa-route-change.html
new file mode 100644
index 0000000..533c38d
--- /dev/null
+++ b/test-suite/challenges/38-spa-route-change.html
@@ -0,0 +1,40 @@
+
+
+38 SPA route change
+
+
+
+

+  
Goal: navigate within SPA using links/buttons. URL must change via history.pushState without full reload. Then go Back to Settings, forward to Billing, and Save.

+  
+  

+

+
+
diff --git a/test-suite/fixtures/sites/mini-shop/cheats.js b/test-suite/fixtures/sites/mini-shop/cheats.js
new file mode 100644
index 0000000..7407ee4
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/cheats.js
@@ -0,0 +1,12 @@
+window.MiniShopCheats = {
+  "mini-shop-red-second-cheapest": [
+    { tool: "chrome_click", params: { selector: "button[aria-label=\"add Ruby Notebook\"]" } },
+    { tool: "chrome_click", params: { selector: "#checkout" } }
+  ],
+  run: {
+    "mini-shop-red-second-cheapest": async () => {
+      document.querySelector('button[aria-label="add Ruby Notebook"]').click();
+      document.querySelector('#checkout').click();
+    }
+  }
+};
diff --git a/test-suite/fixtures/sites/mini-shop/grader.js b/test-suite/fixtures/sites/mini-shop/grader.js
new file mode 100644
index 0000000..74605f4
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/grader.js
@@ -0,0 +1,29 @@
+window.MiniShopGrader = {
+  validate(task, state, products, run, seed = 0) {
+    let bad = [];
+    const rubric = {};
+    if (task === 'mini-shop-red-second-cheapest') {
+      const reds = products.filter(p => p.color === 'red').sort((a,b)=>a.price-b.price);
+      const target = reds[1];
+      rubric.addedTarget = state.cart.includes(target.id);
+      rubric.noWrongItems = state.cart.every(id => id === target.id);
+      rubric.checkedOut = /^MS-\d{4}$/.test(state.orderId || '');
+      if (!rubric.addedTarget) bad.push(`cart missing second-cheapest red item ${target.name}`);
+      const wrong = state.cart.filter(id => id !== target.id);
+      if (wrong.length) bad.push(`cart contains wrong product(s): ${wrong.join(',')}`);
+      if (!rubric.checkedOut) bad.push('missing checkout order id');
+    } else bad.push(`unknown task ${task}`);
+    const passed = bad.length === 0;
+    return { reward: passed ? 1 : 0, done: passed, message: passed ? 'all deterministic mini-shop checks passed' : bad.join('; '), info: { task, run, seed, rubric, bad, state } };
+  },
+  grade(task, state, products, run, seed = 0) {
+    const validation = this.validate(task, state, products, run, seed);
+    const verdict = validation.done ? 'PASS' : 'PENDING';
+    window.__taskId = task;
+    window.__taskVerdict = verdict;
+    window.__taskReason = validation.done ? [validation.message] : validation.info.bad;
+    window.__taskValidation = validation;
+    try { localStorage.setItem(`pi-chrome-task:${task}:${run}`, JSON.stringify({ id: task, run, seed, verdict, reason: window.__taskReason, validation, state, ts: Date.now() })); } catch {}
+    return validation;
+  }
+};
diff --git a/test-suite/fixtures/sites/mini-shop/index.html b/test-suite/fixtures/sites/mini-shop/index.html
new file mode 100644
index 0000000..0fd63ff
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/index.html
@@ -0,0 +1,55 @@
+
+
+MiniShop hermetic site
+
+
+
+

+
MiniShop

+
Cart 0 items   

+

+

+

+
+
+
+
diff --git a/test-suite/fixtures/sites/mini-shop/tasks.json b/test-suite/fixtures/sites/mini-shop/tasks.json
new file mode 100644
index 0000000..6812baf
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/tasks.json
@@ -0,0 +1,9 @@
+[
+  {
+    "id": "mini-shop-red-second-cheapest",
+    "difficulty": "L2",
+    "intent": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
+    "gradeSource": "harness",
+    "grader": "MiniShopGrader.grade(task, state, products, run)"
+  }
+]
diff --git a/test-suite/index.html b/test-suite/index.html
index a4a3c0c..00ef204 100644
--- a/test-suite/index.html
+++ b/test-suite/index.html
@@ -35,6 +35,13 @@
   
 
 
+
Long-horizon hermetic tasks

+
These WebArena/BrowserGym-inspired tasks use deterministic in-page graders and fresh $RUN_ID state. They test multi-step browser work beyond event fidelity.

+
+  
+  
+
iddifficultycategoryintentrequiresopen

+
 
Agent run loop

 
python3 -m http.server 8765
 # For each manifest entry:
@@ -46,7 +53,9 @@
 
 
+
+
diff --git a/test-suite/task-manifest.json b/test-suite/task-manifest.json
new file mode 100644
index 0000000..22e9590
--- /dev/null
+++ b/test-suite/task-manifest.json
@@ -0,0 +1,416 @@
+[
+  {
+    "id": "choredesk-l1-ticket-priority",
+    "site": "choredesk",
+    "startUrl": "scenarios/choredesk/index.html?task=choredesk-l1-ticket-priority&run=$RUN_ID&seed=$SEED",
+    "difficulty": "L1",
+    "category": "enterprise-crud",
+    "intent": "Set ticket T-104 priority to High and status to In Progress, then save it.",
+    "expectedActions": {
+      "min": 3,
+      "max": 6
+    },
+    "requires": [
+      "snapshot",
+      "click",
+      "select/change",
+      "form editing"
+    ],
+    "grader": {
+      "type": "inPageProgrammatic",
+      "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+      "passValue": "PASS"
+    },
+    "browserGymLike": {
+      "observation": [
+        "dom",
+        "accessibility_tree",
+        "screenshot_optional"
+      ],
+      "actionSpace": [
+        "click",
+        "type",
+        "select",
+        "evaluate_optional"
+      ],
+      "reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
+      "stepReturn": "obs,reward,terminated,truncated,info"
+    },
+    "viewport": {
+      "width": 1280,
+      "height": 720
+    },
+    "slowMoMs": 1000,
+    "timeoutMs": 5000,
+    "locale": null,
+    "timezoneId": null,
+    "humanBaseline": null,
+    "actionSubsets": [
+      "bid",
+      "nav",
+      "tab",
+      "infeas",
+      "chat"
+    ],
+    "rewardShape": "binary",
+    "taskId": "choredesk.L1.ticket-priority",
+    "seed": 101,
+    "maxSteps": 8,
+    "allowedActions": [
+      "click",
+      "fill",
+      "select_option",
+      "goto",
+      "report_infeasible",
+      "send_msg_to_user"
+    ],
+    "setupUrl": "scenarios/choredesk/index.html?task=choredesk-l1-ticket-priority&run=$RUN_ID&seed=$SEED",
+    "validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
+    "cheatScript": "scenarios/choredesk/cheats.js#choredesk-l1-ticket-priority",
+    "tags": [
+      "form-fill",
+      "ticket",
+      "select"
+    ],
+    "rubric": [
+      {
+        "id": "priorityHigh",
+        "weight": 0.33,
+        "description": "T-104 priority is High"
+      },
+      {
+        "id": "statusInProgress",
+        "weight": 0.33,
+        "description": "T-104 status is In Progress"
+      },
+      {
+        "id": "saved",
+        "weight": 0.34,
+        "description": "T-104 was saved"
+      }
+    ],
+    "goal": "Set ticket T-104 priority to High and status to In Progress, then save it.",
+    "goalObject": [
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": "Set ticket T-104 priority to High and status to In Progress, then save it."
+          }
+        ]
+      }
+    ],
+    "gradeSource": "page"
+  },
+  {
+    "id": "choredesk-l2-refund-message",
+    "site": "choredesk",
+    "startUrl": "scenarios/choredesk/index.html?task=choredesk-l2-refund-message&run=$RUN_ID&seed=$SEED",
+    "difficulty": "L2",
+    "category": "cross-page-memory",
+    "intent": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields.",
+    "expectedActions": {
+      "min": 8,
+      "max": 14
+    },
+    "requires": [
+      "multi-page navigation",
+      "table lookup",
+      "working memory",
+      "form composition"
+    ],
+    "grader": {
+      "type": "inPageProgrammatic",
+      "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+      "passValue": "PASS"
+    },
+    "browserGymLike": {
+      "observation": [
+        "dom",
+        "accessibility_tree",
+        "screenshot_optional"
+      ],
+      "actionSpace": [
+        "click",
+        "type",
+        "select",
+        "evaluate_optional"
+      ],
+      "reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
+      "stepReturn": "obs,reward,terminated,truncated,info"
+    },
+    "viewport": {
+      "width": 1280,
+      "height": 720
+    },
+    "slowMoMs": 1000,
+    "timeoutMs": 5000,
+    "locale": null,
+    "timezoneId": null,
+    "humanBaseline": null,
+    "actionSubsets": [
+      "bid",
+      "nav",
+      "tab",
+      "infeas",
+      "chat"
+    ],
+    "rewardShape": "binary",
+    "taskId": "choredesk.L2.refund-message",
+    "seed": 202,
+    "maxSteps": 16,
+    "allowedActions": [
+      "click",
+      "fill",
+      "select_option",
+      "goto",
+      "report_infeasible",
+      "send_msg_to_user"
+    ],
+    "setupUrl": "scenarios/choredesk/index.html?task=choredesk-l2-refund-message&run=$RUN_ID&seed=$SEED",
+    "validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
+    "cheatScript": "scenarios/choredesk/cheats.js#choredesk-l2-refund-message",
+    "tags": [
+      "lookup",
+      "cross-page-memory",
+      "message-compose"
+    ],
+    "rubric": [
+      {
+        "id": "messageSubject",
+        "weight": 0.34,
+        "description": "Billing message subject matches"
+      },
+      {
+        "id": "email",
+        "weight": 0.33,
+        "description": "Body includes Noah email"
+      },
+      {
+        "id": "total",
+        "weight": 0.33,
+        "description": "Body includes exact order total"
+      }
+    ],
+    "goal": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields.",
+    "goalObject": [
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields."
+          }
+        ]
+      }
+    ],
+    "gradeSource": "page"
+  },
+  {
+    "id": "choredesk-l3-restock-ticket",
+    "site": "choredesk",
+    "startUrl": "scenarios/choredesk/index.html?task=choredesk-l3-restock-ticket&run=$RUN_ID&seed=$SEED",
+    "difficulty": "L3",
+    "category": "cross-page-state-update",
+    "intent": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket.",
+    "expectedActions": {
+      "min": 12,
+      "max": 20
+    },
+    "requires": [
+      "multi-page navigation",
+      "entity resolution",
+      "state mutation",
+      "exact text note",
+      "audit-save behavior"
+    ],
+    "grader": {
+      "type": "inPageProgrammatic",
+      "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+      "passValue": "PASS"
+    },
+    "browserGymLike": {
+      "observation": [
+        "dom",
+        "accessibility_tree",
+        "screenshot_optional"
+      ],
+      "actionSpace": [
+        "click",
+        "type",
+        "select",
+        "evaluate_optional"
+      ],
+      "reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
+      "stepReturn": "obs,reward,terminated,truncated,info"
+    },
+    "viewport": {
+      "width": 1280,
+      "height": 720
+    },
+    "slowMoMs": 1000,
+    "timeoutMs": 5000,
+    "locale": null,
+    "timezoneId": null,
+    "humanBaseline": null,
+    "actionSubsets": [
+      "bid",
+      "nav",
+      "tab",
+      "infeas",
+      "chat"
+    ],
+    "rewardShape": "binary",
+    "taskId": "choredesk.L3.restock-ticket",
+    "seed": 303,
+    "maxSteps": 24,
+    "allowedActions": [
+      "click",
+      "fill",
+      "select_option",
+      "goto",
+      "report_infeasible",
+      "send_msg_to_user"
+    ],
+    "setupUrl": "scenarios/choredesk/index.html?task=choredesk-l3-restock-ticket&run=$RUN_ID&seed=$SEED",
+    "validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
+    "cheatScript": "scenarios/choredesk/cheats.js#choredesk-l3-restock-ticket",
+    "tags": [
+      "entity-resolution",
+      "cross-page-state-update",
+      "exact-note"
+    ],
+    "rubric": [
+      {
+        "id": "inventory",
+        "weight": 0.4,
+        "description": "AST-LAMP reorder quantity is 12 and saved"
+      },
+      {
+        "id": "ticketNote",
+        "weight": 0.4,
+        "description": "T-205 note contains exact phrase"
+      },
+      {
+        "id": "audit",
+        "weight": 0.2,
+        "description": "Both save audit entries exist"
+      }
+    ],
+    "goal": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket.",
+    "goalObject": [
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket."
+          }
+        ]
+      }
+    ],
+    "gradeSource": "page"
+  },
+  {
+    "id": "mini-shop-red-second-cheapest",
+    "site": "mini-shop",
+    "startUrl": "fixtures/sites/mini-shop/index.html?task=mini-shop-red-second-cheapest&run=$RUN_ID&seed=$SEED",
+    "difficulty": "L2",
+    "category": "task-completion",
+    "intent": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
+    "expectedActions": {
+      "min": 5,
+      "max": 9
+    },
+    "requires": [
+      "filter/sort reasoning",
+      "cart state",
+      "checkout flow"
+    ],
+    "grader": {
+      "type": "inPageProgrammatic",
+      "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+      "passValue": "PASS"
+    },
+    "browserGymLike": {
+      "observation": [
+        "dom",
+        "accessibility_tree",
+        "screenshot_optional"
+      ],
+      "actionSpace": [
+        "click",
+        "read_table",
+        "evaluate_optional"
+      ],
+      "reset": "navigate startUrl with fresh $RUN_ID",
+      "stepReturn": "obs,reward,terminated,truncated,info"
+    },
+    "viewport": {
+      "width": 1280,
+      "height": 720
+    },
+    "slowMoMs": 1000,
+    "timeoutMs": 5000,
+    "locale": null,
+    "timezoneId": null,
+    "humanBaseline": null,
+    "actionSubsets": [
+      "bid",
+      "nav",
+      "tab",
+      "infeas",
+      "chat"
+    ],
+    "rewardShape": "binary",
+    "taskId": "mini-shop.L2.red-second-cheapest",
+    "seed": 404,
+    "maxSteps": 12,
+    "allowedActions": [
+      "click",
+      "goto",
+      "report_infeasible",
+      "send_msg_to_user"
+    ],
+    "setupUrl": "fixtures/sites/mini-shop/index.html?task=mini-shop-red-second-cheapest&run=$RUN_ID&seed=$SEED",
+    "validateScript": "fixtures/sites/mini-shop/grader.js#MiniShopGrader.validate",
+    "cheatScript": "fixtures/sites/mini-shop/cheats.js#mini-shop-red-second-cheapest",
+    "tags": [
+      "shopping",
+      "sort",
+      "cart",
+      "checkout"
+    ],
+    "rubric": [
+      {
+        "id": "addedTarget",
+        "weight": 0.4,
+        "description": "Cart includes second-cheapest red product"
+      },
+      {
+        "id": "noWrongItems",
+        "weight": 0.3,
+        "description": "Cart has no wrong products"
+      },
+      {
+        "id": "checkedOut",
+        "weight": 0.3,
+        "description": "Checkout produced order id"
+      }
+    ],
+    "goal": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
+    "goalObject": [
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible."
+          }
+        ]
+      }
+    ],
+    "gradeSource": "both"
+  }
+]
diff --git a/test-suite/task-manifest.schema.json b/test-suite/task-manifest.schema.json
new file mode 100644
index 0000000..b8031fc
--- /dev/null
+++ b/test-suite/task-manifest.schema.json
@@ -0,0 +1,59 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "pi-chrome long-horizon task manifest",
+  "description": "BrowserGym-inspired task manifest: setup URL, deterministic seed, goal object, action subsets, reward contract, optional cheat recipe, and in-page validate hook.",
+  "type": "array",
+  "items": {
+    "type": "object",
+    "required": ["id", "taskId", "site", "startUrl", "setupUrl", "difficulty", "category", "goal", "goalObject", "grader", "actionSubsets", "maxSteps"],
+    "additionalProperties": true,
+    "properties": {
+      "id": { "type": "string" },
+      "taskId": { "type": "string" },
+      "site": { "type": "string" },
+      "seed": { "type": "integer" },
+      "startUrl": { "type": "string", "description": "Relative URL. $RUN_ID and $SEED should be replaced by runner for isolated deterministic state." },
+      "setupUrl": { "type": "string" },
+      "validateScript": { "type": "string" },
+      "cheatScript": { "type": "string" },
+      "viewport": { "type": "object", "properties": { "width": { "type": "integer" }, "height": { "type": "integer" } } },
+      "slowMoMs": { "type": "integer" },
+      "timeoutMs": { "type": "integer" },
+      "locale": { "type": ["string", "null"] },
+      "timezoneId": { "type": ["string", "null"] },
+      "difficulty": { "enum": ["L1", "L2", "L3"] },
+      "category": { "enum": ["enterprise-crud", "cross-page-memory", "cross-page-state-update", "task-completion"] },
+      "gradeSource": { "enum": ["page", "harness", "both"] },
+      "tags": { "type": "array", "items": { "type": "string" } },
+      "intent": { "type": "string" },
+      "goal": { "type": "string" },
+      "goalObject": { "type": "array", "items": { "type": "object" } },
+      "expectedActions": { "type": "object", "required": ["min", "max"], "properties": { "min": { "type": "integer" }, "max": { "type": "integer" } } },
+      "maxSteps": { "type": "integer" },
+      "humanBaseline": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
+      "actionSubsets": { "type": "array", "items": { "enum": ["chat", "infeas", "bid", "coord", "nav", "tab"] } },
+      "allowedActions": { "type": "array", "items": { "type": "string" } },
+      "requires": { "type": "array", "items": { "type": "string" } },
+      "rewardShape": { "enum": ["binary", "graded", "shaped"] },
+      "rubric": { "type": "array", "items": { "type": "object", "required": ["id", "weight", "description"], "properties": { "id": { "type": "string" }, "weight": { "type": "number" }, "description": { "type": "string" } } } },
+      "grader": {
+        "type": "object",
+        "required": ["type", "expression", "passValue"],
+        "properties": {
+          "type": { "enum": ["inPageProgrammatic"] },
+          "expression": { "type": "string", "description": "Should evaluate to BrowserGym-like validate object or legacy {v,r}." },
+          "passValue": { "type": "string" }
+        }
+      },
+      "browserGymLike": {
+        "type": "object",
+        "properties": {
+          "observation": { "type": "array", "items": { "type": "string" } },
+          "actionSpace": { "type": "array", "items": { "type": "string" } },
+          "reset": { "type": "string" },
+          "stepReturn": { "const": "obs,reward,terminated,truncated,info" }
+        }
+      }
+    }
+  }
+}
-- 
cgit v1.2.3