aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTianren Dong <tianrendong.dtr@gmail.com>2026-05-13 07:29:41 -0400
committerTianren Dong <tianrendong.dtr@gmail.com>2026-05-13 07:29:41 -0400
commit4c54b05dc15b81e449dc8ac53da8bae536fa2798 (patch)
treef81c5e627dcc4dd81d2fe2fd3b2b584e72a73eca
parent5951b3fb04aaf53b875afe265a57a944d6fce641 (diff)
pi-chrome: snapshot pre-migration state
-rw-r--r--.maintainer/GROWTH-IDEAS.md156
-rw-r--r--CHANGELOG.md88
-rw-r--r--CONTRIBUTING.md50
-rw-r--r--LICENSE21
-rw-r--r--README.md301
-rw-r--r--SECURITY.md38
-rw-r--r--docs/COMPARISON.md166
-rw-r--r--docs/EXAMPLES.md166
-rw-r--r--docs/FAQ.md79
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/manifest.json2
-rw-r--r--package.json43
-rw-r--r--test-suite/README.md60
-rw-r--r--test-suite/_lib.js35
-rw-r--r--test-suite/browsergym-action-space.json44
-rw-r--r--test-suite/challenges/34-dialog-handling.html28
-rw-r--r--test-suite/challenges/35-target-blank-popup.html35
-rw-r--r--test-suite/challenges/36-modal-focus-trap.html49
-rw-r--r--test-suite/challenges/37-autocomplete-combobox.html42
-rw-r--r--test-suite/challenges/38-spa-route-change.html40
-rw-r--r--test-suite/fixtures/sites/mini-shop/cheats.js12
-rw-r--r--test-suite/fixtures/sites/mini-shop/grader.js29
-rw-r--r--test-suite/fixtures/sites/mini-shop/index.html55
-rw-r--r--test-suite/fixtures/sites/mini-shop/tasks.json9
-rw-r--r--test-suite/index.html28
-rw-r--r--test-suite/manifest.json569
-rw-r--r--test-suite/manifest.schema.json53
-rw-r--r--test-suite/notes/browsergym-compat.md70
-rw-r--r--test-suite/notes/profiles.md22
-rw-r--r--test-suite/notes/runner-spec.md26
-rw-r--r--test-suite/notes/scoring.md36
-rw-r--r--test-suite/scenarios/choredesk/cheats.js49
-rw-r--r--test-suite/scenarios/choredesk/index.html239
-rw-r--r--test-suite/task-manifest.json416
-rw-r--r--test-suite/task-manifest.schema.json59
34 files changed, 2921 insertions, 194 deletions
diff --git a/.maintainer/GROWTH-IDEAS.md b/.maintainer/GROWTH-IDEAS.md
new file mode 100644
index 0000000..dc80fe2
--- /dev/null
+++ b/.maintainer/GROWTH-IDEAS.md
@@ -0,0 +1,156 @@
+# Growth ideas — for your review
+
+This file is **draft copy and ideas for the maintainer to review and choose from**, not anything that's been sent or published anywhere. Nothing here goes out without you doing it yourself.
+
+It also collects outside-the-package ideas (things I can't do without your accounts/approval) so we have one place to triage.
+
+---
+
+## Inside-the-package — already shipped in this growth pass
+
+- README rewrite (hero, comparison table, complete tool list grouped, killer recipes, architecture diagram, honest-results section, benchmark plug)
+- `package.json` keywords expanded for npm search SEO (`browser-automation`, `playwright-alternative`, `puppeteer-alternative`, `agent`, `llm-tools`, `browser-use`, `stagehand-alternative`, etc.)
+- `package.json` description rewritten as a one-line elevator pitch
+- `homepage`, `repository`, `bugs` fields added (npm shows them on the package page)
+- `CHANGELOG.md` (signals active maintenance)
+- `LICENSE` (MIT)
+- `CONTRIBUTING.md` (signals openness; sets bar)
+- `SECURITY.md` (signals maturity)
+- `docs/COMPARISON.md` (deep comparison vs Playwright / Puppeteer / Stagehand / browser-use / Selenium)
+- `docs/EXAMPLES.md` (ready-to-paste agent prompts)
+- `docs/FAQ.md` (kills common adoption friction)
+- Cross-links from `pi-bar` and `pi-qq` READMEs back to pi-chrome
+
+---
+
+## Inside-the-package — candidates you should consider
+
+These are package-only changes you could approve quickly. Listed roughly by ROI.
+
+1. **Demo GIF or animated SVG in the README.** A 10–15s clip of `/chrome onboard` → `chrome_tab list` → `chrome_snapshot` → summary lands the value prop in one glance. (I can't record it from inside Pi; you'd need to record once, drop it under `assets/` and link from the README.)
+2. **`assets/screenshot-*.png`** with one or two static screenshots of the agent operating Chrome. Even one frame is huge for the npm page; npm renders images inline from README.
+3. **Open-source-friendly badges** in README: `pi-package` badge, "tested with test-suite", coverage-of-categories badge generated from `manifest.json`.
+4. **`prompt-primer.md`** — short prompt block the agent reads on first `chrome_*` use, teaching it `snapshot → uid → act → verify envelope`. Currently in-tool descriptions, but a primer lets users `include` it in their own agent prompts.
+5. **`recipes/` directory** with `.md` files per workflow (PR triage, Linear standup, etc.) so each one has a stable URL on npm.
+6. **Auto-generated tool reference** (`docs/TOOLS.md`) derived from `index.ts` Type.Object schemas. Shows agents and humans the exact param shapes.
+7. **Reload-on-update prompt.** After `pi install npm:pi-chrome` upgrades the package, `/chrome doctor` could auto-suggest the exact `chrome://extensions` reload step. Already half there — could be more aggressive.
+8. **A "scorecard" badge** the README can render after running `test-suite/` — e.g. `pi-chrome • 27/30 challenges passing`. Auto-generated from the manifest + a local run.
+9. **Quickstart video** — same as #1 but longer, hosted on YouTube / Vimeo. README can embed a thumbnail link.
+10. **`pi-chrome --version` CLI helper** (if pi-package convention supports it) so users can sanity-check from a shell.
+
+---
+
+## Outside-the-package — for you to do, when you want
+
+Nothing here is sent or published. These are drafts.
+
+### Short copy (one-liner)
+
+> **pi-chrome** — give your Pi coding agent your real, logged-in Chrome. No re-login, no throwaway profile, no CDP wrangling. `pi install npm:pi-chrome`.
+
+### Long copy (X/Twitter single tweet)
+
+> Built pi-chrome: 20 browser tools for Pi agents that drive your **real Chrome** — every tab you're already signed into.
+>
+> No re-login. No throwaway profile. Watch the agent work live, or run silent.
+>
+> `pi install npm:pi-chrome`
+>
+> https://www.npmjs.com/package/pi-chrome
+
+### X/Twitter thread (8 posts) — draft
+
+1/ Most browser agents make you log in again. They spawn a throwaway Chrome, replay cookies, and pray your SSO/MFA doesn't catch on.
+
+2/ pi-chrome flips it. A tiny MIT-licensed Chrome extension lives in your **normal profile**. Your agent drives the tabs you already have open — GitHub, Linear, Stripe, your admin panel, all of it.
+
+3/ 20 tools: click, type, navigate, screenshot, drag, scroll, touch (real CDP), file upload (no picker), network capture with response bodies, console capture, snapshot with stable element uids.
+
+4/ Honest result envelopes. When a click doesn't take effect, you get back: "occluded by div#overlay" or "pageMutated=false" — not a silent retry loop.
+
+5/ Multi-session safe. Planner / worker / audit Pi sessions share one bridge automatically. They all see the same Chrome state because it's your real one.
+
+6/ Quiet by default (synthetic DOM events, no Chrome banner). One toggle for trusted CDP input when sites need it. Or per-call.
+
+7/ Comes with a benchmark suite. 30+ static pages graded on trusted clicks, pointer humanization, keyboard fidelity, drag/drop, Shadow DOM, file inputs, network observability, fingerprint leaks.
+
+8/ `pi install npm:pi-chrome` → `/chrome onboard` → done in 60s. https://www.npmjs.com/package/pi-chrome
+
+### Hacker News — Show HN draft
+
+**Title:** Show HN: pi-chrome – give an LLM agent your real, logged-in Chrome (no re-login)
+
+**Body:**
+
+I've been frustrated that browser agents always make you log in again — they spin up Playwright/Puppeteer with an isolated profile and try to replay cookies, which falls over on real-world SSO/MFA/admin tools.
+
+pi-chrome is the opposite approach: a small MIT-licensed unpacked Chrome extension lives in the Chrome profile you already use. Your Pi agent drives the tabs you already have open. No re-login. Ever.
+
+20 tools: snapshot/click/type/fill/navigate/scroll/drag/upload/screenshot, plus network and console capture with response bodies, plus real CDP touch events for mobile PWAs.
+
+What makes it different from everything else:
+
+- **Synthetic input first**, trusted CDP only when needed. Quiet by default — no "Chrome is being debugged" banner unless a site forces it.
+- **Honest result envelopes.** Every action returns pageMutated, defaultPrevented, elementVisible, occludedBy, valueMatches. Agents stop looping blindly on broken clicks.
+- **Multi-session.** Multiple Pi sessions on the same machine share one bridge automatically.
+- **A built-in benchmark suite** with 30+ pages grading any browser-control tool. PRs welcome with your competitor's scores.
+
+It is Chrome-only by design (it's a Chrome extension) and not a fit for headless scraping at scale — it's for **your editor session** acting on **your real apps**.
+
+Install: `pi install npm:pi-chrome` then `/chrome onboard`.
+
+Source: https://github.com/tianrendong/pi-packs/tree/main/packages/pi-chrome
+
+### Reddit — r/ChatGPTCoding / r/LocalLLaMA draft
+
+**Title:** pi-chrome: lets your LLM coding agent use the Chrome you're already logged into
+
+**Body:**
+
+If you've tried browser agents you've hit the wall where they want you to log in again. pi-chrome (MIT) is a Pi extension that does the obvious thing instead: a tiny Chrome extension in your normal profile, agent drives your real tabs.
+
+20 tools for the agent — click, type, navigate, screenshot, drag, scroll, real touch, file upload through React inputs (no native picker), network capture with response bodies, console capture, snapshot with stable element uids.
+
+Why pi-chrome over Playwright/Puppeteer/Selenium when the use case is "agent in your editor":
+
+- You stay signed in. SSO/MFA/admin pinning works because the agent IS you in those tabs.
+- Watch it work live, or toggle silent mode for the whole session.
+- Honest result envelopes (pageMutated/occludedBy/valueMatches) — the agent knows when a click failed.
+- Multi-session safe — planner + worker Pi sessions share one bridge.
+- Benchmark suite included; we test in public.
+
+`pi install npm:pi-chrome` → `/chrome onboard` → 60 seconds to first useful run.
+
+https://www.npmjs.com/package/pi-chrome
+
+---
+
+## Outside-the-package — bigger growth bets
+
+Not draft copy — strategic moves you might run later.
+
+1. **Public benchmark leaderboard.** Run `test-suite/` against every major browser-control tool monthly, publish results in `docs/SCORES.md` with a link from README. We've got the moat; this surfaces it. Initial bar: pi-chrome (synthetic), pi-chrome (trusted), Playwright, Puppeteer, Stagehand, browser-use, Selenium.
+2. **Awesome-pi list PR.** Submit pi-chrome to any "awesome agentic tools" / "awesome browser automation" list on GitHub. Each one is a backlink that costs nothing.
+3. **"Built on pi-chrome" gallery.** A `docs/GALLERY.md` showcasing real workflows people built — invites users to submit their own and self-promotes the package.
+4. **YouTube demo, 90 seconds.** "Watch a coding agent triage my GitHub notifications using my logged-in Chrome." Highest-leverage single asset for adoption.
+5. **A `pi-chrome-recipes` companion package.** Curated recipe library installable via `pi install npm:pi-chrome-recipes` so users get inspirational `/recipe` slash commands. Doubles install count.
+6. **Pi-coding-agent feature highlight.** Coordinate with the Pi team to feature pi-chrome in a "Tool of the month" or in onboarding docs.
+7. **Comparison blog post on dev.to / hashnode.** Same content as `docs/COMPARISON.md` but as an indexable blog post with a discoverable title like "Playwright vs Puppeteer vs pi-chrome: when to use which for AI agents."
+8. **Hook into agentic-AI Discords / Slacks.** browser-use, Stagehand, and AutoGPT communities are full of people whose first instinct is to ship a throwaway-profile bot. pi-chrome solves a real complaint they have weekly.
+9. **VS Code / Cursor extension that surfaces `/chrome` commands from the editor command palette.** Low-effort, high-discoverability.
+10. **Auto-generated case-study page** per recipe in `docs/EXAMPLES.md` with a screenshot, hosted at `https://pi-chrome.dev` (or a GitHub Pages site). Each one is an indexable landing page.
+11. **Sponsorship / featured listing on awesome-llm-agents lists** if any take paid placement.
+12. **`pi install npm:pi-chrome` mentioned in pi-coding-agent's own README** if there's a "popular extensions" section.
+
+---
+
+## Demo / show-off prompt (paste in Pi after install)
+
+```text
+Use chrome_tab list to find my GitHub notifications tab.
+chrome_snapshot it. Write me a 5-bullet ranked PR triage:
+which PRs need my review today, sorted by staleness.
+Do not click anything.
+```
+
+If this lands cleanly in 30 seconds with their real GitHub session, you've sold them.
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..171fd21
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,88 @@
+# Changelog
+
+All notable user-facing changes to `pi-chrome`.
+
+## 0.14.7
+
+- Replace "30+ challenges" hand-wave in README + COMPARISON.md with the accurate framing from chrome-benchmark: **38 primitive challenges + 4 hermetic BrowserGym-style long-horizon tasks**, scored by **expected-outcome-by-mode** (not raw PASS count). Explains why a synthetic-events tool isn't supposed to satisfy a clipboard user-activation gate — matching that expectation is the pass.
+
+## 0.14.6
+
+- Fix Browser Use license in `docs/COMPARISON.md`: MIT (not Apache-2.0). Confirmed against upstream LICENSE on GitHub.
+
+## 0.14.5
+
+- `docs/COMPARISON.md` rewritten with a three-axis landscape (drivers / agent frameworks / cloud providers). Adds Browser Use, Stagehand, Skyvern, Magnitude, Alumnium, OpenAI Operator, Project Mariner, Surfer 2, Anthropic Computer Use, Browserbase, Steel.dev, Hyperbrowser, Anchor, Browserless. Adds Interop section, public-benchmark cheat sheet (WebArena, WorkArena++, BrowseComp, Mind2Web 2, WebChoreArena, MiniWoB++, BrowserGym).
+- README gains a one-liner pointing at the new three-axis framing.
+- Sourced from `benchmark-search` session research. No code changes.
+
+## 0.14.4
+
+- Sync `manifest.json` version to match `package.json` (0.14.3 shipped with stale manifest, would trigger spurious `/chrome doctor` drift warnings). No code or behavior changes.
+
+## 0.14.3
+
+- Documentation & discoverability overhaul.
+- New README: hero, alternatives comparison table, 20-tool reference grouped by job, killer recipes, architecture diagram, honest-results explainer, benchmark suite plug.
+- `docs/COMPARISON.md` — deep comparison vs Playwright / Puppeteer / Stagehand / browser-use / Selenium.
+- `docs/EXAMPLES.md` — ready-to-paste agent prompts (PR triage, Linear standup, bug repro, network forensics, multi-tab admin cross-check, etc.).
+- `docs/FAQ.md` — covers Brave/Arc, incognito, detection, banner, multi-session, CSP, file uploads, common envelope causes.
+- `CHANGELOG.md`, `CONTRIBUTING.md`, `SECURITY.md`, `LICENSE` added.
+- `package.json`: `homepage`, `repository`, `bugs`, expanded keywords for npm search.
+- No code changes to the tools or extension.
+
+## 0.14.2
+
+- Recover from foreign-extension input overlays so input tools don't get hijacked by other Chrome extensions.
+
+## 0.14.1
+
+- Harder `attachDebugger` retry path so trusted clicks survive transient Chrome debugger contention.
+- New `trusted.debug` diagnostic surface.
+
+## 0.14.0
+
+- Bare `/chrome` opens a `SettingsList` dialog. `space` cycles values, `enter` saves.
+
+## 0.13.0
+
+- Flattened `/chrome` tree. Cycle-on-pick for `clicks` / `quiet`. Status header at the top of the picker.
+
+## 0.12.1
+
+- Fixed anti-automation regressions across 7 benchmark challenges (`07/13/21/27/28`).
+
+## 0.12.0
+
+- Unified all slash commands under `/chrome` (`/chrome doctor`, `/chrome onboard`, `/chrome clicks`, `/chrome quiet`).
+
+## 0.11.x
+
+- Plain-English audit pass across `/chrome doctor`, `/chrome onboard`, `/chrome quiet`, README.
+- `chrome_tap` (real CDP touch events).
+- Smoother `trustedScroll`; CDP debugger auto-recovers from detach.
+- Smart-auto trusted mode default. Extension renamed to **Pi Chrome Connector**.
+- Per-event scroll delta cap so IntersectionObserver thresholds land naturally.
+
+## 0.10.x
+
+- Trusted-input mode via `chrome.debugger` (CDP) — opt-in, indistinguishable-from-human clicks/keys.
+- `chrome_key` modifiers (Cmd+V, Ctrl+Shift+Tab, etc.) for trusted chords.
+- Interactive `/chrome-trusted` picker (later folded into `/chrome`).
+
+## 0.9.x
+
+- Humanized synthetic input (pointer paths, key cadence variance).
+- Anti-automation benchmark test suite landed in `test-suite/`.
+
+## 0.8.0
+
+- `chrome_evaluate` no longer returns `null` for valid expressions — dedicated MAIN-world `evaluateInTab` pipeline with statement-mode fallback and tagged envelope for `undefined`/errors/symbols/bigints.
+- Truthful action result envelopes: `isTrusted`, `defaultPrevented`, `elementVisible`, `occludedBy`, `valueMatches`, `pageMutated`.
+- Extended `/chrome doctor`: bridge mode/URL, extension version drift, MAIN-world helper injection, `navigator.webdriver` fingerprint, CDP availability probe.
+- Removed misleading `returnByValue` param. Implemented `chrome_screenshot.fullPage` via tile stitching.
+- Autoplay-gate heuristic for `chrome_click`.
+
+## 0.7.0
+
+- Initial public `pi-chrome` release. Companion Chrome extension + local bridge + first tool set.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..c16b5fe
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,50 @@
+# Contributing to pi-chrome
+
+Thanks for considering a contribution. pi-chrome aims to be the **de-facto browser-control toolkit for Pi agents** — that means a few non-negotiables.
+
+## Non-negotiables
+
+1. **No re-login.** Every change must keep working against the user's already-signed-in Chrome profile. Anything that requires a fresh profile or extra auth steps is out of scope.
+2. **Honest result envelopes.** Every action tool returns `pageMutated`, `defaultPrevented`, `elementVisible`, `occludedBy` (when relevant), `valueMatches` (for input). Agents need to know **why** something didn't take effect.
+3. **Quiet by default, trusted by opt-in.** Synthetic DOM events first. CDP/`chrome.debugger` only when explicitly requested (`trusted: true`) or when the smart-auto heuristic detects an obvious user-activation gate.
+4. **Benchmarks gate features.** Add a page in `test-suite/` that fails before your change and passes after. We accept PRs faster when there's a green/red verdict to point at.
+
+## Local dev
+
+```bash
+# Link from a checkout
+pi install ./pi-chrome
+
+# Run the benchmark dashboard
+cd test-suite
+python3 -m http.server 8765
+# open http://127.0.0.1:8765/ in the Chrome window pi-chrome controls
+```
+
+## Adding a new tool
+
+1. Register in `extensions/chrome-profile-bridge/index.ts` (the `register*Tool` calls near line 840+).
+2. Implement the handler in `extensions/chrome-profile-bridge/browser-extension/service_worker.js`.
+3. Return a `pageMutated` + relevant fields.
+4. Add a benchmark page under `test-suite/challenges/` and a manifest entry.
+5. Update `README.md` "What an agent gets" table.
+6. Add a `CHANGELOG.md` entry.
+
+## Filing a bug
+
+Include:
+
+- `/chrome doctor` output
+- `pi-chrome` version + extension version (the `doctor` output prints both)
+- The exact tool call + the result envelope you got
+- Page URL or a minimal repro page in `test-suite/`
+
+## Releasing
+
+- Bump `package.json` version.
+- Move `CHANGELOG.md` notes from the working section to the new version header.
+- `npm publish --access public`.
+
+## Code of conduct
+
+Be kind, be precise, ship things. PRs that break the "no re-login" promise will be closed with a note explaining which non-negotiable they hit.
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..a975012
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-2026 pi-chrome contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
index 0387dce..b01487b 100644
--- a/README.md
+++ b/README.md
@@ -1,63 +1,82 @@
# pi-chrome
[![npm version](https://img.shields.io/npm/v/pi-chrome.svg)](https://www.npmjs.com/package/pi-chrome)
+[![npm downloads](https://img.shields.io/npm/dm/pi-chrome.svg)](https://www.npmjs.com/package/pi-chrome)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](./LICENSE)
-Control the Chrome profile you already use from Pi.
+> **The fastest way to give a [Pi](https://pi.dev) agent your real Chrome.**
+> No CDP. No throwaway profile. No re-login. Watch it work — or run silent.
-`pi-chrome` gives Pi agents browser tools for your **real Chrome windows, tabs, and authenticated sessions**. It uses a companion Chrome extension instead of the Chrome DevTools Protocol (CDP), so it does not launch a throwaway debug browser profile and does not require re-signing into the apps you already have open.
+```text
+You: "Find my open GitHub PR tab, summarize review state, and screenshot the failing CI."
+Agent: chrome_tab(list) → chrome_snapshot(uid:…) → chrome_screenshot(...)
+ ✓ 3 reviewers, 1 change requested, CI red on iOS. Saved → .pi/chrome-screenshots/ci.png
+You: [keeps coding — agent never asked you to log in]
+```
-Multiple Pi sessions can use Chrome at the same time. The first Pi session starts the local bridge; later sessions automatically detect that bridge and submit their Chrome commands through it.
+`pi-chrome` ships **20+ browser tools** for Pi agents, backed by a small MIT-licensed Chrome extension that runs inside the Chrome profile **you already use** — including every site you're already signed into.
-## Why try it?
+---
-- **Uses your existing Chrome profile** — works with the Chrome windows/tabs you are already using, including logged-in GitHub, admin dashboards, local apps, and internal tools.
-- **Watch your authenticated Chrome work** — by default, `chrome_*` tool calls focus Chrome and activate the target tab so you can see the agent inspect, navigate, click, and type in real time. Switch to silent/background mode for the whole session with `/chrome quiet`, or pass `background: true` on a single tool call when you want quiet.
-- **Full browser automation toolkit for Pi** — list/create/activate/close tabs, snapshot pages with usable CSS selectors, navigate, evaluate JavaScript, click, type, press keys, wait for page state, and capture screenshots.
-- **Built-in setup and agent guidance** — `/chrome onboard` walks users through installing the companion extension, `/chrome doctor` checks connectivity and version drift, screenshots save to disk, and the prompt primer tells agents to inspect with `chrome_snapshot` before acting and avoid destructive actions unless explicitly requested.
+## Why pi-chrome vs. everything else
-## Install
+> Short version: **pi-chrome is primitives — "Playwright for the Chrome you're already signed into."** Not an agent loop. Plug it under any agent framework (Browser Use, Stagehand, LangGraph) or call its tools directly from a Pi agent. See [docs/COMPARISON.md](./docs/COMPARISON.md) for the full three-axis landscape (drivers, agents, cloud providers).
-```bash
-pi install npm:pi-chrome
-```
+| | **pi-chrome** | Playwright / Puppeteer | CDP-based agents | Selenium / WebDriver |
+| ------------------------------ | --------------------------------- | ----------------------------- | ----------------------------- | ----------------------------- |
+| Uses your real signed-in Chrome | ✅ yes (extension in your profile) | ❌ throwaway profile | ⚠️ requires `--remote-debug` | ❌ throwaway profile |
+| Re-login required | **Never** | Every run | Sometimes | Every run |
+| Watch agent work, live | ✅ default; toggle quiet | ❌ headless or new window | ⚠️ debugger banner always | ❌ new window |
+| Works on strict-CSP pages | ✅ `new Function` MAIN-world | ✅ | ✅ | ✅ |
+| Real browser-trusted clicks | ✅ opt-in (`chrome clicks on`) | ✅ | ✅ | ✅ |
+| Multi-session safe | ✅ shared local bridge | ❌ port collisions | ❌ | ❌ |
+| Network/console capture | ✅ built-in | ✅ | ✅ | ⚠️ via extensions |
+| Honest result envelopes¹ | ✅ | ⚠️ | ❌ | ❌ |
+| Built-in benchmark suite² | ✅ 38 primitives + 4 long-horizon | n/a | n/a | n/a |
-For local development from a checkout:
+¹ Every action returns `pageMutated`, `defaultPrevented`, `elementVisible`, `occludedBy`, and `valueMatches` so the agent knows when a click didn't take effect — instead of looping blindly.
+² See [`test-suite/`](./test-suite) — 38 primitive challenges plus 4 hermetic BrowserGym-style tasks. Scoring is expected-outcome-by-mode (`synthetic` / `trusted` / `manual`), not raw PASS count. Pages grade any browser-control tool on trusted clicks, pointer humanization, keyboard fidelity, drag/drop, clipboard, Shadow DOM, iframes, file uploads, network capture, and fingerprint leaks.
-```bash
-pi install ./pi-chrome
-```
+---
-### Why an unpacked Chrome extension?
+## What an agent gets
-`pi-chrome` cannot ship through the Chrome Web Store: a Web Store extension is not allowed to talk to a local bridge controlled by another tool. Instead it ships as a small, MIT-licensed unpacked extension in `extensions/chrome-profile-bridge/browser-extension/` — read the source before loading. `/chrome doctor` reports the loaded extension version and warns when it drifts from the installed `pi-chrome`.
+**20 tools**, grouped by job. Every one runs against your already-open tabs.
-## First-time setup
+| Category | Tools |
+| --------------- | ---------------------------------------------------------------------------------------------- |
+| **Tabs** | `chrome_tab` (list/new/activate/close/version), `chrome_launch` |
+| **Inspect** | `chrome_snapshot` (uids + selectors + text + viewport), `chrome_screenshot`, `chrome_evaluate` |
+| **Navigate** | `chrome_navigate` (with optional `initScript` at `document_start`), `chrome_wait_for` |
+| **Interact** | `chrome_click`, `chrome_type`, `chrome_fill`, `chrome_key`, `chrome_hover` |
+| **Gesture** | `chrome_drag` (HTML5 DataTransfer), `chrome_scroll` (wheel + momentum), `chrome_tap` (touch) |
+| **Files** | `chrome_upload_file` (no native picker; works with React/Vue/Angular file inputs) |
+| **Observe** | `chrome_list_console_messages`, `chrome_list_network_requests`, `chrome_get_network_request` (with response body) |
-In Pi, run:
+Each tool is documented inline in Pi — agents see the parameters and the gotchas (synthetic vs. trusted, autoplay gates, file picker limits) without trial-and-error.
-```text
-/chrome onboard
+---
+
+## 60-second install
+
+```bash
+pi install npm:pi-chrome
```
-Pi first shows setup instructions and waits for confirmation. Press Enter to continue. On macOS it will:
+Then in Pi:
-- open `chrome://extensions`
-- reveal the bundled `browser-extension` folder in Finder
-- copy the extension folder path to your clipboard
+```text
+/chrome onboard
+```
-Then in Chrome:
+On macOS this opens `chrome://extensions`, reveals the bundled `browser-extension/` folder in Finder, and copies its path to your clipboard. In Chrome: **Developer mode** → **Load unpacked** → paste the path. Done.
-1. Enable **Developer mode**.
-2. Click **Load unpacked**.
-3. Select the revealed/copied `browser-extension` folder.
-4. Return to Pi and run:
+Verify:
```text
/chrome doctor
```
-Expected output:
-
```text
Performing Chrome bridge health check
pi-chrome v<version>
@@ -65,122 +84,202 @@ pi-chrome v<version>
✓ Companion Chrome extension responding (ID: <chrome-extension-id>, ext v<version>)
```
-## Click modes
+---
-pi-chrome can drive Chrome two ways:
+## Try this in 30 seconds after install
-- **Quiet clicks** — fast and unobtrusive. They work on most sites, but some pages (sign-in flows, copy-to-clipboard buttons, file pickers, autoplay videos, fullscreen, paywalls) ignore them because they don't look like a real human action.
-- **Real-looking clicks** — indistinguishable from a person clicking. They unlock the cases above, but Chrome shows a *"Pi Chrome Connector started debugging this browser"* banner at the top of every tab pi-chrome touches while it's working.
+```text
+Use chrome_tab list to find my GitHub notifications tab.
+chrome_snapshot it, then write a 5-bullet triage:
+which PRs need my review today, sorted by staleness.
+Do not click anything yet.
+```
-Pick a mode with `/chrome clicks`:
+You'll watch the agent jump to your GitHub tab and read the page — using **your** session, **your** filters, **your** orgs.
-```text
-/chrome clicks auto # default; quiet by default, real-looking only when needed
-/chrome clicks off # always quiet, no banner ever
-/chrome clicks on # always real-looking, banner stays up the whole session
-/chrome clicks status # show the current mode
+---
+
+## Killer recipes (copy-paste into Pi)
+
+Each one assumes tabs you already have open + accounts you're already signed into.
+
+**PR triage**
+> Use chrome_tab list to find my GitHub notifications tab, snapshot it, summarize PRs needing my review.
+
+**Linear standup**
+> Open my Linear current cycle in the active tab, snapshot it, write a 5-bullet standup.
+
+**Bug repro with evidence**
+> Open the staging app I'm already signed into, reproduce \<bug>, save a screenshot of each step under `./repro/`.
+
+**Form auto-fill (no submit)**
+> Open the vendor portal, fill the new-vendor form from this JSON, stop before submit.
+
+**Admin cross-check**
+> Across my Stripe / Postmark / our admin tabs, find any user where state disagrees.
+
+**Local dev visual diff**
+> Snapshot `localhost:3000` and the staging URL of the same page; tell me what's visually different.
+
+**Auth-only data pull**
+> Open my analytics dashboard tab and `chrome_evaluate` to extract today's KPIs from page state.
+
+**Network forensics**
+> Reproduce the checkout bug, then use `chrome_list_network_requests` to find the failing call and dump its response body.
+
+**File upload through React**
+> Open the photo uploader, `chrome_upload_file` with `./fixtures/sample.png`, confirm preview rendered.
+
+---
+
+## Architecture
+
+```
+ ┌──────────────────────┐ ┌──────────────────────────┐
+ │ Pi agent (terminal) │ ─── http://127.0.0.1:17318 ─→ │ Chrome extension │
+ │ chrome_* tools │ │ (your real profile) │
+ └──────────┬───────────┘ └─────────┬────────────────┘
+ │ same machine │
+ ▼ ▼
+ Other Pi sessions Tabs you already have open
+ share the same bridge (signed in to GitHub,
+ automatically Linear, Stripe, etc.)
```
-For a one-off call, pass `trusted: true` (or `false`) on `chrome_click`, `chrome_type`, `chrome_fill`, `chrome_key`, `chrome_hover`, `chrome_drag`, or `chrome_scroll`. The per-call value wins over the global mode.
+Multiple Pi sessions (planner / worker / audit) can all drive the same Chrome at once. The first session opens the local bridge; later sessions detect it and pipe their commands through.
-First time you update pi-chrome to a version that supports real-looking clicks, Chrome will ask you to re-approve the extension. Open `chrome://extensions` and accept the new permission once.
+---
-## Background mode
+## Click & input modes
-By default, `chrome_*` tools focus Chrome and activate the target tab so you can watch the agent work — great for demos, pair-driving, debugging, and first-time confidence that things are happening.
+`pi-chrome` can drive Chrome two ways:
-When you want quiet (planner / audit / worker sessions running alongside your editor), turn background mode on for the whole Pi session:
+- **Quiet** — synthetic DOM events. Fast, no UI banners. Drives React/Vue/Angular state. Won't satisfy autoplay, clipboard, file picker, fullscreen, or user-activation gates.
+- **Trusted** — `chrome.debugger` / CDP under the hood. Indistinguishable from a person clicking. Shows Chrome's *"Pi Chrome Connector started debugging this browser"* banner while active.
```text
-/chrome quiet # toggle
-/chrome quiet on # explicit
-/chrome quiet off # explicit
+/chrome clicks auto # default: quiet, upgrade to trusted only when needed
+/chrome clicks off # always quiet, never banner
+/chrome clicks on # always trusted, banner stays up
+/chrome clicks status
```
-For a single tool call, the agent can pass `background: true` directly. The per-call value always wins over the session toggle.
+Per-call `trusted: true / false` on any input tool wins over the global mode.
-## Quick demo prompts
+---
-After setup, try one of these in Pi:
+## Background / watch modes
-Silent inspection (no Chrome interruption):
+By default, every `chrome_*` call focuses Chrome and activates the target tab so you can **watch the agent work** — invaluable for demos, debugging, and first-time confidence.
```text
-Inspect my active GitHub tab with chrome_snapshot using background:true and summarize the PR state without focusing Chrome.
+/chrome quiet # toggle for the whole session
+/chrome quiet on # explicit
+/chrome quiet off # explicit
```
-Existing authenticated tab:
+Per-call `background: true` wins over the session toggle.
+
+---
+
+## Honest results
+
+Most browser-automation libraries return `void` or a generic ack. `pi-chrome` returns a structured envelope on every interaction:
```text
-Use chrome_tab list to find my existing GitHub tab, chrome_snapshot it, then summarize the visible PR state. Do not click anything yet.
+chrome_click(occluded-button) →
+ "Clicked el-3 — pageMutated=false; occluded by <div#overlay>"
```
-Local web app repro with screenshot:
-
```text
-Use chrome_tab list to find my localhost app, inspect it with chrome_snapshot, navigate through the bug repro flow, and save a screenshot when you reach the broken state.
+chrome_type(react-input, "hello") →
+ "Typed into el-7 — valueMatches=true; pageMutated=true"
```
-## Recipes
+This is why agents using pi-chrome don't get stuck in retry loops on broken sites. They get the **reason** the action didn't land and can fix course in one turn.
-Copy-paste these into Pi after setup. Each one uses tabs you already have open and accounts you are already signed into.
+---
-- **PR triage:** "Use chrome_tab list to find my GitHub notifications tab, snapshot it, and summarize PRs needing my review."
-- **Linear standup:** "Open my Linear current cycle in the active tab, snapshot it, and write me a 5-bullet standup."
-- **Bug repro with evidence:** "Open the staging app I'm already signed into, reproduce <bug>, and save a screenshot of each step under ./repro/."
-- **Form auto-fill (no submit):** "Open <vendor> portal, fill the new-vendor form from this JSON, but stop before submit."
-- **Admin cross-check:** "Across my Stripe / Postmark / our admin tabs, find any user where state disagrees."
-- **Local dev visual diff:** "Snapshot localhost:3000 and the staging URL of the same page; tell me what's visually different."
-- **Auth-only data pull:** "Open my analytics dashboard tab and chrome_evaluate to extract today's KPIs from the page state."
+## Diagnostics
-Screenshots save under `.pi/chrome-screenshots/` by default, which composes nicely with PR demo workflows.
+- `/chrome doctor` — single command: connectivity, extension version, bridge owner, version drift, MAIN-world helper injection, `chrome_evaluate("1+1") === 2`, fingerprint flags.
+- `/chrome onboard` — guided first-time setup.
+- `/chrome quiet status`, `/chrome clicks status` — current modes.
-## Diagnostics
+If the loaded Chrome extension is older than `pi-chrome` on disk, `/chrome doctor` tells you to reload it from `chrome://extensions`.
-- `/chrome doctor` — single command that checks connectivity and reports the loaded Chrome extension ID + version, plus a one-line fix for common setup failures (extension not loaded, bridge owner stale after `pi update`, version mismatch between pi-chrome and the loaded Chrome extension).
+---
-If the Chrome extension you have loaded is older than `pi-chrome` on disk, `/chrome doctor` will tell you to reload it from `chrome://extensions`.
+## Composes with
-## Compose with
+- **[pi-qq](https://www.npmjs.com/package/pi-qq)** — `/qq summarize what the active GitHub tab shows` without polluting the main transcript.
+- **[pi-bar](https://www.npmjs.com/package/pi-bar)** — when the agent scrapes large pages, watch the context-usage segment turn yellow → red as a signal to `/qq` for a recap.
+- **PR demo skills** — screenshots write to `.pi/chrome-screenshots/` so you can attach them to PR descriptions or demo bundles.
-- **pi-qq** — ask side questions about what the agent saw in Chrome without polluting the main transcript: `/qq summarize what the active GitHub tab shows`.
-- **pi-bar** — watch context pressure as the agent scrapes large pages; the footer's red threshold is a clean signal to `/qq` for a recap before context overflows.
-- **PR demo skills** (such as `ios-pr-agent` / `ios-demo-record` workflows) — `chrome_screenshot` writes to `.pi/chrome-screenshots/` so you can attach images to PR descriptions or demo bundles.
+---
-## Tools
+## Why an unpacked Chrome extension?
-The package registers these Pi tools:
+`pi-chrome` cannot ship through the Chrome Web Store — a Web Store extension cannot talk to a local bridge controlled by another tool on the same machine. So it ships as a small MIT-licensed unpacked extension in [`extensions/chrome-profile-bridge/browser-extension/`](./extensions/chrome-profile-bridge/browser-extension). **Read the source before loading.** `/chrome doctor` reports the extension version and warns when it drifts from your installed `pi-chrome`.
-- `chrome_launch` — setup/help entry point; opens a URL if the bridge is connected
-- `chrome_tab` — list, create, activate, close, or inspect tabs
-- `chrome_snapshot` — inspect title, URL, visible text, viewport, and clickable/focusable selectors
-- `chrome_navigate` — navigate an existing tab
-- `chrome_evaluate` — evaluate JavaScript in a tab
-- `chrome_click` — click by selector or coordinates
-- `chrome_type` — type text, optionally focusing a selector first
-- `chrome_key` — send keyboard keys
-- `chrome_wait_for` — wait for a selector or expression
-- `chrome_screenshot` — capture viewport screenshots to disk
+---
-These tools are especially useful for authenticated web app debugging, repro flows, admin workflows, visual checks, and inspecting local development pages without rebuilding login state.
+## Security model
-<details>
-<summary><strong>How it works (technical details)</strong></summary>
+The companion extension runs in the Chrome profile where you install it and has broad tab/scripting permissions. Only install it from a package source you trust.
-Pi starts a local bridge on `127.0.0.1:17318`. The companion Chrome extension, installed in your normal Chrome profile, polls that local bridge for commands and executes them using Chrome extension APIs.
+The Pi side listens on `127.0.0.1:17318` by default. Override before starting Pi:
-If another Pi session is already running the bridge, additional Pi sessions automatically act as clients of that shared bridge. This lets planner/audit/worker sessions all use the same authenticated Chrome profile concurrently without fighting over the port.
+```bash
+PI_CHROME_BRIDGE_PORT=17319 pi
+```
-This is intentionally different from CDP-based tools: the browser extension lives inside the profile you already use, so Pi can interact with existing tabs and authenticated page state.
+There is no network exposure; the bridge binds to loopback only.
-</details>
+---
-## Security model
+## Built-in benchmark suite
-The companion Chrome extension runs in the Chrome profile where you install it and has broad tab/scripting permissions. Only install it from a package source you trust.
+[`test-suite/`](./test-suite) is a benchmark for **any** browser-control agent (not just pi-chrome). It includes **38 primitive challenges** plus **4 hermetic BrowserGym-style long-horizon tasks**.
-The Pi side listens on `127.0.0.1:17318` by default. Override before starting Pi with:
+Scoring is **expected-outcome-by-mode**, not raw PASS count: each challenge has an expected verdict per mode (`synthetic`, `trusted`, `manual`) and a tool grades itself by whether its actual outcome matches the expected one. This avoids false equivalence between modes — a synthetic-events tool isn't supposed to satisfy a clipboard user-activation gate; matching that expectation is the pass.
+
+Each challenge exposes `window.__verdict` / `window.__reason` / `window.__events` and a manifest entry with expected results per mode.
```bash
-PI_CHROME_BRIDGE_PORT=17319 pi
+cd test-suite && python3 -m http.server 8765
+# open http://127.0.0.1:8765/ in the Chrome window pi-chrome controls
```
+
+Categories: `trusted-input`, `pointer-humanization`, `keyboard`, `activation-gates`, `scroll`, `drag-drop`, `clipboard`, `native-controls`, `frameworks`, `editing`, `dom-complexity`, `frames`, `files`, `observability`, `fingerprint`, `agent-safety`.
+
+If you build a competing tool, please open a PR with your scores. We benchmark in public.
+
+---
+
+## Roadmap signals
+
+`pi-chrome` is actively shipped. Things on the near roadmap:
+
+- More observability tools (DOM mutation streams, performance traces)
+- First-class iframe + Shadow-DOM uid stability across snapshots
+- Web Push & service worker introspection
+- Recorder mode that emits agent prompts from your own clicks
+
+If you want one of those next, open an issue.
+
+---
+
+## Contributing
+
+PRs welcome. The bar:
+
+1. Add a benchmark page in `test-suite/` that fails before your change and passes after.
+2. Keep `chrome_*` tool results honest — surface `pageMutated`, `valueMatches`, `defaultPrevented`, etc.
+3. Don't break the "no re-login" guarantee. Anything that requires a fresh profile is out of scope.
+
+---
+
+## License
+
+MIT. See [LICENSE](./LICENSE).
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000..8627ca6
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,38 @@
+# Security policy
+
+## Reporting a vulnerability
+
+Open a GitHub issue prefixed with `[security]` at https://github.com/tianrendong/pi-packs/issues, or contact the maintainer directly if the issue is sensitive. Please do **not** include exploit details in a public issue without coordinating first.
+
+## Threat model
+
+`pi-chrome` is a developer tool you install knowingly. It is **not** designed to defend against:
+
+- Hostile pages running in your Chrome trying to detect or escape automation. (Standard browser security boundaries still apply, but a hostile page that already runs in your tab can do anything that page can already do.)
+- Other processes on your local machine. The bridge binds to `127.0.0.1:17318` (loopback only) but **does not authenticate** local callers. Any process running as your user can issue commands. If your threat model includes hostile local processes, run pi-chrome on a separate user account.
+
+`pi-chrome` **is** designed to:
+
+- Never exfiltrate page state to the network. All communication is loopback (`127.0.0.1`).
+- Surface every action with an honest result envelope so the agent can't silently do the wrong thing.
+- Require explicit opt-in for trusted-input mode (`/chrome clicks on` or `trusted: true`), which uses `chrome.debugger` and shows Chrome's banner.
+
+## The companion extension
+
+The Chrome extension under `extensions/chrome-profile-bridge/browser-extension/` runs with broad permissions: `tabs`, `scripting`, `debugger`, `webNavigation`, etc. **Only install it from a package source you trust.** Read the source before loading. Pin a known-good commit if you're security-sensitive.
+
+## Defaults
+
+- Loopback bridge only. No remote port. No telemetry.
+- Synthetic events first; trusted CDP only when explicitly enabled.
+- Quiet mode optional; tab/window focus is observable (the user can see Pi acting).
+
+## Override the port
+
+```bash
+PI_CHROME_BRIDGE_PORT=17319 pi
+```
+
+## Supported versions
+
+The latest minor on npm is supported. Security patches will be released as soon as practical.
diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md
new file mode 100644
index 0000000..4f51291
--- /dev/null
+++ b/docs/COMPARISON.md
@@ -0,0 +1,166 @@
+# pi-chrome vs. the rest of the browser-automation landscape
+
+This is the honest "which tool when" page. The browser-automation space has three different layers and people often compare across them — let's not do that.
+
+We benchmark in public — see [`../test-suite/`](../test-suite). Where exact scores matter (WebVoyager, WorkArena++, BrowseComp, Mind2Web 2), check the live leaderboards; they shift monthly.
+
+---
+
+## TL;DR
+
+| You are… | Use… |
+| -------------------------------------------------------------- | ------------------------------- |
+| A Pi agent operator who wants the agent to use **your real Chrome** (logged-in tabs, cookies, extensions) | **pi-chrome** |
+| Building a Pi/LLM agent and want low-level browser primitives | **pi-chrome** |
+| Writing deterministic end-to-end tests in CI | Playwright / Cypress |
+| Building a hosted scraping/agent fleet on isolated profiles | Playwright / Puppeteer + Browserbase / Steel |
+| Want a turnkey "natural-language agent" with built-in loop | Browser Use / Stagehand / Skyvern |
+| Want a hosted, vendor-managed agent | OpenAI Operator / Project Mariner / Surfer |
+| Debugging your own app from your editor without leaving your real session | **pi-chrome** |
+
+`pi-chrome` is **primitives**, not an opinionated agent loop. Think of it as **"Playwright for the Chrome you're already signed into"** — and pluggable under any agent framework above the line.
+
+---
+
+## The three axes you should compare on
+
+| Axis | Examples | What it gives you |
+| --------------------------------- | --------------------------------------------------------------------- | ----------------------------- |
+| **1. Driver / transport** | Playwright, Puppeteer, Selenium, CDP raw, `puppeteer-stealth`, **pi-chrome** | low-level tools (click, type, navigate) |
+| **2. Agent framework** | Browser Use, Stagehand, Skyvern, Magnitude, Alumnium, LangGraph-with-Playwright, Operator, Mariner, Surfer | LLM loop, planning, NL API |
+| **3. Cloud browser provider** | Browserbase, Steel.dev, Hyperbrowser, Anchor, Browserless | managed Chromes, sessions, quotas |
+
+`pi-chrome` is **axis 1**. You can wrap it with any axis-2 agent framework, or run it directly from a Pi agent's `chrome_*` tool calls. It explicitly does NOT compete with axis 3 — it runs locally inside *your* Chrome.
+
+---
+
+## Axis 1 — drivers (where pi-chrome lives)
+
+| Tool | Transport | Profile | Trusted events | Banner when controlling | Default detectable as bot |
+| --------------------------------- | ------------------------------------------ | ---------------------------------- | -------------------- | ----------------------------------- | ------------------------- |
+| Playwright | CDP (own driver) | throwaway by default | always | always ("controlled by test software") | yes (webdriver flag, automation flags) |
+| Puppeteer | CDP | throwaway by default | always | always | yes |
+| Selenium | WebDriver / BiDi | throwaway | partial (BiDi improves) | always | most detectable |
+| puppeteer-stealth / playwright-extra | CDP + patches | throwaway | always | always | medium (patches flags) |
+| Raw CDP | direct devtools protocol | either (needs `--remote-debugging-port`) | always | always | yes |
+| **pi-chrome** | **Chrome extension bridge → local loopback** | **your real Chrome profile, signed-in cookies, extensions, history** | **opt-in** (`/chrome clicks on` or `trusted: true`) | **only when trusted mode is active** | **synthetic mode bypasses common detection signals**¹ |
+
+¹ pi-chrome synthetic mode dispatches DOM events with `isTrusted=false` — most sites don't check; some anti-bot defenses do. The [`test-suite/`](../test-suite) grades both modes against common detection signals. Trusted mode uses `chrome.debugger` and shows Chrome's banner like every other CDP-based tool.
+
+### What makes pi-chrome different on this axis
+
+1. **Profile attach, not driver launch.** Every other driver fights cookie persistence, login walls, MFA, and extension state. pi-chrome inherits all of it because it *is* your Chrome.
+2. **Synthetic-first, trusted-on-demand.** Two-tier event model — agents pick the right tradeoff per call. Competitors are all-trusted (CDP) and always show the banner. pi-chrome avoids it by default; you opt in when a site needs it (autoplay gate, clipboard, file picker).
+3. **Extension bridge transport.** No `--remote-debugging-port`, no throwaway Chromium. Survives Chrome auto-updates. Works alongside your normal Chrome usage.
+4. **Honest result envelopes.** Every action returns `pageMutated`, `defaultPrevented`, `elementVisible`, `occludedBy`, `valueMatches`. Competitors return `void` or generic acks; agents loop blindly on broken clicks.
+5. **Multi-session shared bridge.** Planner + worker + audit Pi sessions all drive the same Chrome concurrently.
+6. **Stable element uids.** `chrome_snapshot` returns deterministic uids you can pass to subsequent actions — similar to BrowserGym's `bid`, but built into the snapshot tool itself.
+
+---
+
+## Axis 2 — agent frameworks (built on top of axis 1)
+
+These wrap a driver with an LLM loop. They are **higher-level than pi-chrome** and **complementary**, not competitors.
+
+| Framework | Driver underneath | Approach | Open source |
+| ------------------------ | ------------------------------ | --------------------------------------------------------------------------------------------- | --------------- |
+| **Browser Use** | Playwright | DOM + a11y tree → LLM → action JSON. Open-source leader; widely cited on WebVoyager. | MIT (Python) |
+| **Stagehand** (Browserbase) | Playwright | Natural-language `.act()` / `.observe()` / `.extract()`; deterministic + AI mix. | MIT (TypeScript)|
+| **Skyvern** | Playwright + own DOM model | Vision-first + DOM; YAML workflows for form/workflow automation. | AGPL (Python) |
+| **Magnitude** | Playwright | NL test authoring; QA-focused. | open |
+| **Alumnium** | Selenium / Playwright | Test-author NL → agent. QA-focused. | open |
+| **LangGraph / AutoGen + Playwright** | Playwright | Generic agent graph + browser tools. | open |
+| **OpenAI Operator** | proprietary | OpenAI's own VLM + browser; ChatGPT-integrated. | closed, hosted |
+| **Project Mariner** (Google) | proprietary Chrome integration | Google's own VLM Chrome experiment. | closed |
+| **Surfer 2 / Surfer-H** (H Company) | proprietary | Hosted proprietary agent stack. | closed, hosted |
+| **Anthropic Computer Use** | OS-level screenshots + mouse/keyboard | Broader than browser; trusted events at OS level. | closed (API) |
+
+**Why pi-chrome is not on this list:** it's intentionally **not an agent**. There's no LLM loop, no `.act("click the blue button")`. Pi handles the loop; pi-chrome provides the primitives. This means:
+
+- You can use pi-chrome under Browser Use, Stagehand, LangGraph, or any other agent framework (see [Interop](#interop)).
+- You don't pay for an opinion you don't want.
+- Agent improvements compound across all your workflows because the primitives stay stable.
+
+---
+
+## Axis 3 — cloud browser providers (orthogonal)
+
+| Provider | What it sells |
+| ----------------- | -------------------------------------------------------- |
+| **Browserbase** | Managed browsers; pairs with Stagehand. |
+| **Steel.dev** | Managed browsers + public agent leaderboards. |
+| **Hyperbrowser** | Managed browsers, session APIs. |
+| **Anchor Browser**| Managed browsers. |
+| **Browserless** | Managed Chrome, scraping focus. |
+
+**pi-chrome doesn't compete here.** It runs locally in *your* Chrome. The right framing if someone asks: *"no cloud cost, no session-handoff, no rate limits — and the agent runs against your real logged-in state."* If you need fleets of isolated Chromes in CI, you want one of these.
+
+---
+
+## Interop
+
+`pi-chrome` exposes tools that any Pi agent can call. If you want to use it from outside Pi:
+
+1. The local bridge speaks HTTP JSON-RPC at `127.0.0.1:17318` (default). The API is internal but stable across patch versions.
+2. Tool surface mirrors Playwright closely (click/type/navigate/snapshot/screenshot/evaluate/wait_for) so adapter code is short.
+3. Honest envelopes (`pageMutated`, `valueMatches`, `occludedBy`) let agent harnesses skip retry/heal logic.
+
+If you want a first-class pi-chrome adapter for Browser Use / Stagehand / LangGraph, file an issue with your use case.
+
+---
+
+## "But Playwright has `storageState` / Puppeteer has user-data-dir"
+
+Yes — you can export cookies and replay them, or point Playwright at your existing profile directory. In practice for agent workflows that breaks down fast:
+
+1. **OAuth + SSO** providers (Okta, Google, GitHub) often pin sessions to TLS fingerprints, device IDs, and browser-extension state that doesn't survive replay or a parallel Chrome instance.
+2. **MFA** tokens expire mid-run.
+3. **Internal admin tools** hard-pin to your real device.
+4. **Pointing Playwright at your real `user-data-dir`** requires closing your normal Chrome (Chrome won't share the profile lock). pi-chrome doesn't fight you for the profile because it lives *inside* it.
+5. **Watching the agent work** in your real window is a different UX than a hidden parallel Chrome. Demos, pair-driving, and confidence-building all want axis-1-with-attach.
+
+---
+
+## "Is this safer than CDP?"
+
+Different security boundary, not strictly safer.
+
+- **CDP-based tools** require `chrome --remote-debugging-port=...`. That port is unauthenticated and exposes the whole browser to any local process. Easy to misconfigure.
+- **pi-chrome** runs through an extension you install yourself with broad permissions (tabs, scripting, debugger, webNavigation). The bridge listens on `127.0.0.1:17318` loopback only. **Only install the bundled extension if you trust the source you got the npm package from.**
+
+If your threat model excludes extensions with broad permissions, neither approach is a fit — you want a sandboxed CI runner.
+
+---
+
+## Public benchmarks worth knowing (for axis 2 / axis 3 comparison)
+
+Pi-chrome itself ships a benchmark suite ([`../test-suite/`](../test-suite)) of **38 primitive challenges** plus **4 hermetic BrowserGym-style long-horizon tasks** covering trusted-input, pointer humanization, keyboard fidelity, drag/drop, Shadow DOM, file uploads, network observability, fingerprint leaks, and agent-safety honeypots. Scoring is **expected-outcome-by-mode** (not raw PASS count): each challenge has expected verdicts per mode (`synthetic` / `trusted` / `manual`) and a tool grades itself by whether its actual outcome matches expectations. That's **driver-level** grading.
+
+For **agent-level** comparison (axis 2), the public benchmarks worth citing:
+
+| Benchmark | What it measures | Notes |
+| ---------------- | ------------------------------------------------- | ------------------------------------------------ |
+| **WebArena** (CMU) | Hermetic, programmatic graders | Gold standard for reproducibility. |
+| **WorkArena++** (ServiceNow) | Enterprise SaaS workflows | Hardest realistic benchmark; <5% frontier. |
+| **BrowseComp** (OpenAI) | Hard info-retrieval | Not saturated. |
+| **Mind2Web 2** (NeurIPS '25) | Long-horizon, rubric-tree judge | New, well-designed. |
+| **WebChoreArena** | Tedious cross-page workflows | Reflects real ops work. |
+| **WebVoyager** | Live web tasks | **Saturated** — 90%+ scores common; cite only as smoke test. |
+| **VisualWebArena** | Multimodal | |
+| **MiniWoB++** | Classic unit-task suite | |
+| **BrowserGym + AgentLab** | Research harness covering the above | The de-facto research API; pi-chrome's snapshot uid is comparable to BrowserGym's `bid`. |
+
+Cite live leaderboards rather than hard-coded numbers; agent scores shift monthly.
+
+---
+
+## Reproducing pi-chrome's driver-level claims
+
+Run [`../test-suite/`](../test-suite) against any browser-control tool. Each challenge exposes `window.__verdict` / `window.__reason` / `window.__events`, so any tool (Playwright, Puppeteer, Selenium, Stagehand, pi-chrome) can grade itself deterministically.
+
+```bash
+cd test-suite && python3 -m http.server 8765
+# open http://127.0.0.1:8765/ in the Chrome window the tool controls
+```
+
+If you build a competing tool, please open a PR with your scores.
diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md
new file mode 100644
index 0000000..905d278
--- /dev/null
+++ b/docs/EXAMPLES.md
@@ -0,0 +1,166 @@
+# pi-chrome examples
+
+Real, useful agent prompts. Drop any of these into Pi after running `/chrome onboard`. Each one uses Chrome tabs and accounts you already have.
+
+## Daily workflow
+
+### PR triage
+
+```text
+Use chrome_tab list to find my GitHub notifications tab.
+chrome_snapshot it. Group PRs by:
+ - awaiting my review
+ - blocked on me (changes requested back)
+ - mergeable (approved + green CI)
+Output a 5-bullet ranked triage. Do not click anything.
+```
+
+### Linear standup
+
+```text
+Open my Linear current cycle in the active tab.
+chrome_snapshot, then write yesterday/today/blockers
+in the exact format my standup channel uses.
+```
+
+### Slack catch-up
+
+```text
+For each unread channel in my Slack tab, chrome_snapshot,
+extract the top 3 messages that mention me or my team,
+and summarize what I missed in <100 words total.
+```
+
+## Debugging
+
+### Reproduce a customer bug with evidence
+
+```text
+1. chrome_navigate to https://staging.acme.com/orders/<id>
+2. chrome_snapshot
+3. Click "Refund" with chrome_click
+4. Use chrome_list_network_requests to capture the API call
+5. chrome_get_network_request on the failing one — give me the response body
+6. chrome_screenshot the error state to ./repro/refund-bug.png
+```
+
+### Visual diff local vs staging
+
+```text
+chrome_screenshot http://localhost:3000/pricing → ./diff/local.png
+chrome_screenshot https://staging.acme.com/pricing → ./diff/staging.png
+Then read both, describe layout differences in plain English.
+```
+
+### Console + network forensics
+
+```text
+Reproduce the checkout bug on the active tab.
+After the failure:
+ - chrome_list_console_messages
+ - chrome_list_network_requests
+Cross-reference the timestamps and tell me what broke first.
+```
+
+## Admin / ops
+
+### Multi-tab cross-check
+
+```text
+I have Stripe, Postmark, and our internal admin open in 3 tabs.
+For user <id>, chrome_snapshot each tab in turn and find
+any field where state disagrees. Output a 3-column table.
+```
+
+### Bulk gentle action (safe form-fill, no submit)
+
+```text
+Open our vendor portal "Add Vendor" form.
+For each row in ./vendors.csv:
+ - chrome_fill the form
+ - chrome_screenshot it
+ - STOP before submit
+ - chrome_evaluate "history.back()" to return to the list
+I will review screenshots and submit manually.
+```
+
+### Auth-only data pull
+
+```text
+My analytics dashboard is open and the cookie auth would die in headless mode.
+chrome_evaluate to read window.__APP_STATE__.dashboardData
+and dump today's KPIs as JSON.
+```
+
+## Demos / PRs
+
+### Capture screenshots for a PR description
+
+```text
+On localhost:3000/feature-x:
+ - empty state → ./pr/01-empty.png
+ - filled state → ./pr/02-filled.png
+ - error state (delete the API key from devtools first) → ./pr/03-error.png
+Save each with chrome_screenshot. Output a markdown block I can paste into the PR.
+```
+
+### Record a guided demo flow
+
+```text
+On my staging app:
+1. Walk the new-onboarding flow start to finish
+2. After each chrome_click or chrome_navigate, chrome_screenshot
+3. Save numbered PNGs under ./demo/
+4. Write narration captions for each step
+```
+
+## Forms with frameworks
+
+### React controlled inputs
+
+```text
+chrome_fill (not chrome_type) for React inputs — it uses the
+framework-aware native value setter so the form's state actually updates.
+After each fill, the result envelope's valueMatches=true confirms the
+component re-rendered with the new value.
+```
+
+### File upload without the native picker
+
+```text
+chrome_upload_file paths=[./fixtures/avatar.png] selector="input[type=file]"
+# No native file picker opens. Works with React/Vue/Angular controlled inputs.
+```
+
+### Drag-to-reorder lists
+
+```text
+chrome_drag fromUid=row-3 toUid=row-1
+# Fires real HTML5 dragstart/dragover/drop with a shared DataTransfer.
+```
+
+## Multi-session patterns
+
+`pi-chrome` shares one bridge across all Pi sessions on the same machine. Useful patterns:
+
+### Planner + Worker
+
+- **Planner session** stays high level: "find the bug, decide the fix."
+- **Worker session** runs the actual `chrome_*` tools.
+- Both see the same Chrome state because they're both pointing at your real profile.
+
+### Watcher
+
+A third Pi session can run `chrome_snapshot` periodically in `background: true` mode and post summaries via `pi-qq` — handy for long-running flows.
+
+## When to prefer trusted clicks
+
+Pass `trusted: true` on `chrome_click` (or run `/chrome clicks on`) when:
+
+- the click should open a file picker
+- the click should write to the clipboard or read it
+- the click should start an audio/video play
+- the click should request fullscreen / push permission
+- the page is wrapped in a strict user-activation guard (some paywalls / login flows)
+
+Everything else is faster and quieter without it.
diff --git a/docs/FAQ.md b/docs/FAQ.md
new file mode 100644
index 0000000..2839f8f
--- /dev/null
+++ b/docs/FAQ.md
@@ -0,0 +1,79 @@
+# pi-chrome FAQ
+
+## Does this work with Brave / Arc / Edge / Vivaldi?
+
+Yes. Any Chromium-based browser that supports unpacked extensions and the `chrome.debugger` API will work. The extension is named "Pi Chrome Connector" but the source is browser-agnostic. Firefox / WebKit are out of scope (different extension models).
+
+## Will it slow my browser down?
+
+The companion extension is idle when no Pi command is in flight. It uses Manifest V3 service worker activation, so it wakes for a request and goes back to sleep. No content script is injected globally.
+
+## Does it work in Chrome incognito?
+
+By default no — extensions need explicit "Allow in incognito" permission. Toggle it on `chrome://extensions` if you want pi-chrome to see incognito tabs. We don't recommend it for sensitive work.
+
+## Will sites detect that I'm automating?
+
+For **synthetic** (quiet) input: yes, technically. `event.isTrusted` is `false`. Most sites don't check; some anti-bot defenses do.
+
+For **trusted** (CDP) input: events are `isTrusted=true`, pointer paths are humanized, key cadence has variance. Most fingerprint-based detectors don't fire. Some specifically check for the `chrome.debugger` API attached and will show the "Chrome is being debugged" banner. That banner is the visible cost of trusted mode.
+
+The [`test-suite/`](../test-suite) grades both modes against common detection signals.
+
+## Why do I see a banner saying "Pi Chrome Connector started debugging this browser"?
+
+That's Chrome's built-in warning when an extension uses `chrome.debugger`. pi-chrome uses it only in trusted-input mode. If you don't want to see it, run `/chrome clicks off` and accept that some sign-in flows / file pickers / clipboard ops won't work.
+
+## Can a malicious page escape and access my other tabs?
+
+No — pages cannot directly talk to extensions. Commands flow agent → local bridge (`127.0.0.1:17318`) → extension → tab. The bridge binds to loopback only. The risk surface is **other local processes** that could connect to `127.0.0.1:17318` and impersonate Pi. If that's in your threat model, run pi-chrome on a separate user account.
+
+## Can multiple Pi sessions use it at once?
+
+Yes. The first session opens the local bridge; later sessions detect it and pipe their commands through the same bridge. Planner + worker + audit can all drive the same Chrome concurrently.
+
+## Why can't this be on the Chrome Web Store?
+
+Web Store extensions cannot communicate with a local process bridge controlled by another tool — Google's policy. pi-chrome must ship as an unpacked extension you load yourself. The upside: you can read the source. The downside: each Chrome update may prompt you to re-confirm.
+
+## What happens when I update pi-chrome?
+
+`/chrome doctor` will warn you if the loaded extension is older than the installed `pi-chrome`. Reload it from `chrome://extensions` to pick up the new version. Trusted-input mode in particular requires re-approving the `debugger` permission once.
+
+## What's the install footprint?
+
+- Pi side: one extension that registers ~20 tools and a few slash commands.
+- Chrome side: one unpacked extension, ~5000 LOC of plain JavaScript, no dependencies.
+
+## Can I script it without Pi?
+
+The Pi-facing tools are thin wrappers around an HTTP bridge at `127.0.0.1:17318`. You could call it directly from any process, but the API is internal and may change. If you need a stable scripting interface, file an issue and we'll consider stabilizing.
+
+## Does `chrome_evaluate` work on strict-CSP pages?
+
+Yes. The handler compiles with `new Function(...)` in the MAIN world, which works under `script-src 'self'` without `'unsafe-eval'`. Multi-statement bodies are supported via a statement-mode fallback. Exceptions are surfaced to the agent.
+
+## Why does my click return `pageMutated=false`?
+
+Either:
+- The element was occluded (look for `occludedBy: <selector>` in the envelope).
+- The click handler called `event.preventDefault()` and the page intentionally ignored it.
+- The site rejects synthetic events. Try `trusted: true` or `/chrome clicks on`.
+
+The result envelope tells you which one. **Don't blind-retry.**
+
+## Why does `chrome_type` return `valueMatches=false`?
+
+The editor rejected the synthetic input. Common culprits: contenteditable rich-text editors, native date pickers, masked-input libraries. Try `chrome_fill` (uses framework-aware native setters) or `trusted: true`.
+
+## How do I attach a file to a React file input?
+
+`chrome_upload_file` — populates `input.files` via a real `DataTransfer` and fires `input` + `change` events. It does **not** open the native file picker (no synthetic event can; that's a user-activation gate). Works with React/Vue/Angular controlled inputs.
+
+## Can it record videos?
+
+Not yet. Screenshots only. Video recording is on the roadmap.
+
+## How do I file a good bug report?
+
+Include `/chrome doctor` output, the exact tool call, and the result envelope. If the page is public, link to it; if private, distill it into a benchmark page under `test-suite/challenges/`. See [CONTRIBUTING.md](../CONTRIBUTING.md).
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json
index f0f8a6a..eccc6d5 100644
--- a/extensions/chrome-profile-bridge/browser-extension/manifest.json
+++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json
@@ -1,7 +1,7 @@
{
"manifest_version": 3,
"name": "Pi Chrome Connector",
- "version": "0.14.2",
+ "version": "0.14.7",
"description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.",
"permissions": ["tabs", "scripting", "storage", "activeTab", "alarms", "webNavigation", "debugger"],
"host_permissions": ["<all_urls>", "http://127.0.0.1:17318/*"],
diff --git a/package.json b/package.json
index dcaaa0e..316dcc9 100644
--- a/package.json
+++ b/package.json
@@ -1,22 +1,55 @@
{
"name": "pi-chrome",
- "version": "0.14.2",
- "description": "Drive your existing logged-in Chrome from Pi — no re-login, no throwaway profile, watch the agent work in real time (or toggle quiet background mode).",
+ "version": "0.14.7",
+ "description": "The de-facto browser automation toolkit for Pi agents. Drive your existing logged-in Chrome — no re-login, no throwaway profile, no CDP. 20+ tools (click, type, navigate, screenshot, network capture, file upload, drag, scroll, touch) + honest result envelopes + a built-in benchmark suite.",
"keywords": [
+ "pi",
"pi-package",
"pi-extension",
+ "pi-agent",
"chrome",
+ "chrome-extension",
"browser",
- "automation",
+ "browser-automation",
+ "browser-control",
+ "web-automation",
+ "agent",
+ "ai-agent",
+ "llm-agent",
+ "llm-tools",
+ "agentic",
+ "playwright-alternative",
+ "puppeteer-alternative",
+ "selenium-alternative",
+ "cdp-alternative",
"authenticated-session",
"real-profile",
- "web-debugging"
+ "web-debugging",
+ "web-scraping",
+ "screenshot",
+ "automation-tools",
+ "browser-use",
+ "stagehand-alternative"
],
"license": "MIT",
+ "homepage": "https://github.com/tianrendong/pi-packs/tree/main/packages/pi-chrome#readme",
+ "repository": {
+ "type": "git",
+ "url": "git+https://github.com/tianrendong/pi-packs.git",
+ "directory": "packages/pi-chrome"
+ },
+ "bugs": {
+ "url": "https://github.com/tianrendong/pi-packs/issues"
+ },
"type": "commonjs",
"files": [
"extensions",
- "README.md"
+ "docs",
+ "README.md",
+ "CHANGELOG.md",
+ "CONTRIBUTING.md",
+ "SECURITY.md",
+ "LICENSE"
],
"pi": {
"extensions": [
diff --git a/test-suite/README.md b/test-suite/README.md
index bb8becb..a96e0d0 100644
--- a/test-suite/README.md
+++ b/test-suite/README.md
@@ -1,13 +1,14 @@
# pi-chrome browser-control benchmark
-Static benchmark pages for evaluating tools that let agents control Chrome. The
-suite covers two related questions:
+Static benchmark pages for evaluating tools that let agents control Chrome. The suite has two layers:
-1. **Can the agent complete normal browser work?** Forms, scroll containers,
- contenteditable, files, frames, Shadow DOM, network/console inspection.
-2. **Does the interaction look like a human/browser action when sites care?**
- `isTrusted`, user activation, pointer paths, key cadence, native controls,
- drag/drop, touch, paste, and scroll momentum.
+1. **Unit challenges** (`manifest.json`) — MiniWoB-style capability probes for
+ forms, scroll containers, contenteditable, files, frames, Shadow DOM,
+ network/console inspection, `isTrusted`, user activation, pointer paths, key
+ cadence, native controls, drag/drop, touch, paste, and scroll momentum.
+2. **Long-horizon hermetic tasks** (`task-manifest.json`) — WebArena /
+ BrowserGym-inspired multi-step tasks with fresh run IDs and deterministic
+ programmatic graders.
## Run
@@ -24,11 +25,17 @@ Each challenge page exposes:
- `window.__reason` — array of reasons
- `window.__events` — raw event log for forensics
-`manifest.json` is the source of truth for benchmark metadata: category, goal,
-expected result per mode, prerequisites, flake risk, manual baseline status, and
-canonical tool recipe. `manifest.schema.json` documents the manifest shape.
-Recipes express tool intent; runners may need to adapt descriptive selectors
-(e.g. shadow/iframe notation) and expand path placeholders like `$PWD`.
+`manifest.json` is the source of truth for unit-challenge metadata: category,
+goal, expected result per mode, prerequisites, flake risk, manual baseline
+status, and canonical tool recipe. `manifest.schema.json` documents the manifest
+shape. Recipes express tool intent; runners may need to adapt descriptive
+selectors (e.g. shadow/iframe notation) and expand path placeholders like `$PWD`.
+
+`task-manifest.json` is the source of truth for long-horizon tasks: BrowserGym-style
+`taskId`, seed, viewport, goal object, difficulty tier, max steps, declared
+action subsets, reset/setup URL, validate hook, optional cheat recipe, and
+programmatic grader expression. `task-manifest.schema.json` documents this shape.
+`browsergym-action-space.json` records BrowserGym-compatible action subsets.
## Modes / expected outcomes
@@ -49,7 +56,7 @@ Expected values in `manifest.json`:
Manual baselines are tracked separately with `manualBaseline`. `unverified`
means the manual expectation is a target, not a recorded contract.
-## Recommended agent flow
+## Recommended unit-challenge agent flow
1. Navigate to dashboard:
`http://127.0.0.1:8765/`.
@@ -70,6 +77,29 @@ means the manual expectation is a target, not a recorded contract.
4. Return to dashboard and compare actual verdicts with expected values.
5. Copy JSON report from dashboard for PRs or regression notes.
+## Recommended long-horizon task flow
+
+1. Load `task-manifest.json`.
+2. Replace `$RUN_ID` in `startUrl` with a fresh value.
+3. Navigate to the start URL and read the visible task instruction.
+4. Solve using normal browser tools only; avoid direct state mutation unless the
+ benchmark mode explicitly allows evaluate-based actions.
+5. Click **Grade now** or evaluate the task grader expression:
+ ```js
+ JSON.stringify({ v: window.__taskVerdict, r: window.__taskReason })
+ ```
+6. Record action count, observations used, tools used, verdict, and reason.
+
+## Design principles copied from browser-agent benchmarks
+
+- Prefer hermetic sites and deterministic graders over live sites and LLM judges.
+- Report action API and observation format; these strongly affect scores.
+- Use difficulty tiers: L1 atomic, L2 compositional, L3 cross-page/context-rich.
+- Include tedious cross-page memory and exact-value transfer tasks; short unit
+ probes hide these failures.
+- Keep synthetic-event-gated tests because extension bridges face failures that
+ CDP/Playwright-style benchmarks usually do not measure.
+
## Challenge categories
- `trusted-input` — browser-trusted click/key events.
@@ -136,3 +166,7 @@ The dashboard renders this from `manifest.json`. In brief:
software WebGL in VMs, remote desktops, or policy-constrained environments.
- `notes/bypass-ideas.md` is historical guidance for older synthetic-only
versions. Prefer `manifest.json` for current expected outcomes.
+- `notes/browsergym-compat.md` defines the reset/step/validate/observation/BID
+ contract for external BrowserGym-style agents.
+- `notes/runner-spec.md`, `notes/scoring.md`, and `notes/profiles.md` define
+ runner output, scoring, retry policy, and environment metadata.
diff --git a/test-suite/_lib.js b/test-suite/_lib.js
index 95d6393..f3845e5 100644
--- a/test-suite/_lib.js
+++ b/test-suite/_lib.js
@@ -8,6 +8,8 @@
verdict: "PENDING",
reason: [],
events,
+ details: [],
+ thresholds: {},
};
function render() {
@@ -31,8 +33,9 @@
}
const Challenge = {
- init({ id, instructions }) {
+ init({ id, instructions, thresholds = {} }) {
state.id = id;
+ state.thresholds = parseThresholds(thresholds);
document.title = `[${state.verdict}] ${id}`;
const root = document.body;
const bar = document.createElement("div");
@@ -82,6 +85,20 @@
document.title = `[WARN] ${state.id}`;
persist(); render();
},
+ partial({ name, pass, reason, data }) {
+ state.details.push({ name, pass: !!pass, reason: reason || "", data });
+ log("partial", { name, pass: !!pass, reason, data });
+ persist(); render();
+ return !!pass;
+ },
+ finishPartials() {
+ const failed = state.details.filter(d => !d.pass);
+ if (failed.length) Challenge.fail(...failed.map(d => `${d.name}: ${d.reason || "failed"}`));
+ else Challenge.pass(...state.details.map(d => `${d.name}: ok`));
+ },
+ getThreshold(name, fallback) {
+ return Object.prototype.hasOwnProperty.call(state.thresholds, name) ? state.thresholds[name] : fallback;
+ },
log,
state,
};
@@ -90,10 +107,24 @@
try {
localStorage.setItem(
"pi-chrome-suite:" + state.id,
- JSON.stringify({ id: state.id, verdict: state.verdict, reason: state.reason, events: state.events.slice(-50), ts: Date.now() })
+ JSON.stringify({ id: state.id, verdict: state.verdict, reason: state.reason, details: state.details, thresholds: state.thresholds, events: state.events.slice(-50), ts: Date.now() })
);
} catch {}
}
+ function parseThresholds(defaults) {
+ const out = { ...defaults };
+ try {
+ const qs = new URLSearchParams(location.search);
+ for (const [k, v] of qs) {
+ if (!k.startsWith("threshold.")) continue;
+ const key = k.slice("threshold.".length);
+ const num = Number(v);
+ out[key] = Number.isFinite(num) ? num : v;
+ }
+ } catch {}
+ return out;
+ }
+
window.Challenge = Challenge;
})();
diff --git a/test-suite/browsergym-action-space.json b/test-suite/browsergym-action-space.json
new file mode 100644
index 0000000..5d7c873
--- /dev/null
+++ b/test-suite/browsergym-action-space.json
@@ -0,0 +1,44 @@
+{
+ "subsets": {
+ "chat": ["send_msg_to_user"],
+ "infeas": ["report_infeasible"],
+ "bid": ["click", "dblclick", "hover", "fill", "clear", "press", "focus", "select_option", "scroll", "drag_and_drop", "upload_file"],
+ "coord": ["mouse_move", "mouse_up", "mouse_down", "mouse_click", "mouse_dblclick", "mouse_drag_and_drop", "mouse_upload_file", "scroll_at", "keyboard_down", "keyboard_up", "keyboard_press", "keyboard_type", "keyboard_insert_text"],
+ "nav": ["go_back", "go_forward", "goto"],
+ "tab": ["tab_close", "tab_focus", "new_tab"]
+ },
+ "signatures": {
+ "fill": "fill(bid, value, enable_autocomplete_menu=false)",
+ "click": "click(bid, button='left', modifiers=[])",
+ "dblclick": "dblclick(bid, button='left', modifiers=[])",
+ "hover": "hover(bid)",
+ "press": "press(bid, key_comb)",
+ "focus": "focus(bid)",
+ "clear": "clear(bid)",
+ "select_option": "select_option(bid, options)",
+ "scroll": "scroll(delta_x, delta_y)",
+ "drag_and_drop": "drag_and_drop(from_bid, to_bid)",
+ "upload_file": "upload_file(bid, file)",
+ "mouse_move": "mouse_move(x, y)",
+ "mouse_click": "mouse_click(x, y, button='left')",
+ "mouse_dblclick": "mouse_dblclick(x, y, button='left')",
+ "mouse_up": "mouse_up(x, y, button='left')",
+ "mouse_down": "mouse_down(x, y, button='left')",
+ "mouse_drag_and_drop": "mouse_drag_and_drop(from_x, from_y, to_x, to_y)",
+ "scroll_at": "scroll_at(x, y, dx, dy)",
+ "keyboard_press": "keyboard_press(key)",
+ "keyboard_down": "keyboard_down(key)",
+ "keyboard_up": "keyboard_up(key)",
+ "keyboard_type": "keyboard_type(text)",
+ "keyboard_insert_text": "keyboard_insert_text(text)",
+ "goto": "goto(url)",
+ "go_back": "go_back()",
+ "go_forward": "go_forward()",
+ "tab_close": "tab_close()",
+ "tab_focus": "tab_focus(index)",
+ "new_tab": "new_tab()",
+ "send_msg_to_user": "send_msg_to_user(text)",
+ "report_infeasible": "report_infeasible(reason)",
+ "noop": "noop(wait_ms=1000)"
+ }
+}
diff --git a/test-suite/challenges/34-dialog-handling.html b/test-suite/challenges/34-dialog-handling.html
new file mode 100644
index 0000000..25845a7
--- /dev/null
+++ b/test-suite/challenges/34-dialog-handling.html
@@ -0,0 +1,28 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>34 dialog handling</title>
+<link rel="stylesheet" href="../_style.css">
+<script src="../_lib.js"></script>
+<body>
+<main>
+ <p>Goal: handle browser dialogs. Click Prompt, enter <code>pi-chrome</code>, accept it; then click Confirm and accept it. Browser automation must expose/dismiss native dialogs.</p>
+ <button id="promptBtn">Prompt</button>
+ <button id="confirmBtn">Confirm</button>
+ <button id="alertBtn">Alert smoke</button>
+ <pre id="rep" class="code"></pre>
+</main>
+<script>
+Challenge.init({ id: "dialog-handling", instructions: "accept prompt('pi-chrome') and confirm()" });
+const state = { prompt: null, confirm: null, alert: false };
+function paint(){ rep.textContent = JSON.stringify(state, null, 2); }
+function check(){
+ paint();
+ if (state.prompt === "pi-chrome" && state.confirm === true) Challenge.pass("prompt and confirm handled with expected values");
+}
+promptBtn.addEventListener("click", () => { state.prompt = prompt("Enter token", ""); Challenge.log("prompt-result", { value: state.prompt }); check(); });
+confirmBtn.addEventListener("click", () => { state.confirm = confirm("Accept benchmark confirm?"); Challenge.log("confirm-result", { value: state.confirm }); check(); });
+alertBtn.addEventListener("click", () => { alert("pi-chrome alert smoke"); state.alert = true; Challenge.log("alert-dismissed", {}); paint(); });
+window.addEventListener("beforeunload", e => { e.preventDefault(); e.returnValue = "benchmark beforeunload"; });
+paint();
+</script>
+</body>
diff --git a/test-suite/challenges/35-target-blank-popup.html b/test-suite/challenges/35-target-blank-popup.html
new file mode 100644
index 0000000..0813625
--- /dev/null
+++ b/test-suite/challenges/35-target-blank-popup.html
@@ -0,0 +1,35 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>35 target blank popup</title>
+<link rel="stylesheet" href="../_style.css">
+<script src="../_lib.js"></script>
+<body>
+<main>
+ <p>Goal: click <code>target=_blank</code> link, detect new tab, switch to it, and verify child page reports PASS.</p>
+ <a id="open" target="_blank" rel="opener" href="35-target-blank-popup.html?child=1" style="font-size:18px">Open child tab</a>
+ <p id="msg"></p>
+</main>
+<script>
+Challenge.init({ id: "target-blank-popup", instructions: "open child tab and inspect it" });
+const qs = new URLSearchParams(location.search);
+if (qs.get("child") === "1") {
+ document.getElementById("msg").textContent = "Child tab opened.";
+ Challenge.pass("child tab loaded on same origin");
+ try { localStorage.setItem("pi-chrome-suite:target-blank-popup", JSON.stringify({ id:"target-blank-popup", verdict:"PASS", reason:["✓ child tab loaded"], ts: Date.now() })); } catch {}
+} else {
+ document.getElementById("msg").textContent = "Parent page. Link should open new tab.";
+ function maybePassFromStorage(raw) {
+ if (!raw) return;
+ try {
+ const v = JSON.parse(raw);
+ if (v.verdict === "PASS") Challenge.pass("child reported PASS via storage event/localStorage");
+ } catch {}
+ }
+ window.addEventListener("storage", e => {
+ if (e.key === "pi-chrome-suite:target-blank-popup") maybePassFromStorage(e.newValue);
+ });
+ maybePassFromStorage(localStorage.getItem("pi-chrome-suite:target-blank-popup"));
+ document.getElementById("open").addEventListener("click", e => Challenge.log("popup-click", { trusted: e.isTrusted }));
+}
+</script>
+</body>
diff --git a/test-suite/challenges/36-modal-focus-trap.html b/test-suite/challenges/36-modal-focus-trap.html
new file mode 100644
index 0000000..811207c
--- /dev/null
+++ b/test-suite/challenges/36-modal-focus-trap.html
@@ -0,0 +1,49 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>36 modal focus trap</title>
+<link rel="stylesheet" href="../_style.css">
+<script src="../_lib.js"></script>
+<body>
+<main>
+ <p>Goal: open modal, use Tab to cycle focus inside modal only, then Escape closes it and focus returns to opener.</p>
+ <button id="open">Open modal</button>
+ <div id="modal" role="dialog" aria-modal="true" aria-label="Focus trap modal" hidden style="position:fixed;inset:20%;background:#222;border:2px solid #6cf;border-radius:10px;padding:20px;z-index:5">
+ <button id="first">First</button>
+ <input id="middle" aria-label="Modal input">
+ <button id="last">Last</button>
+ </div>
+</main>
+<script>
+Challenge.init({ id: "modal-focus-trap", instructions: "open modal, Tab cycles inside, Escape closes" });
+const order = [];
+const modal = document.getElementById("modal");
+const nodes = [first, middle, last];
+let escaped = false, modalOpened = false, outsideFocusWhileOpen = false;
+for (const el of [open, ...nodes]) el.addEventListener("focus", () => { order.push(el.id); Challenge.log("focus", { id: el.id }); });
+document.addEventListener("focusin", e => {
+ if (modalOpened && !modal.hidden && !modal.contains(e.target)) outsideFocusWhileOpen = true;
+});
+open.addEventListener("click", () => { modal.hidden = false; modalOpened = true; first.focus(); });
+document.addEventListener("keydown", e => {
+ if (modal.hidden) return;
+ if (e.key === "Tab") {
+ const i = nodes.indexOf(document.activeElement);
+ if (i >= 0) { e.preventDefault(); nodes[(i + (e.shiftKey ? -1 : 1) + nodes.length) % nodes.length].focus(); }
+ }
+ if (e.key === "Escape") { modal.hidden = true; escaped = true; open.focus(); queueMicrotask(check); }
+});
+function hasAdjacent(seq, a, b) {
+ return seq.some((x, i) => x === a && seq[i + 1] === b);
+}
+function check(){
+ const afterOpen = order.slice(order.indexOf("first"));
+ const bad = [];
+ if (!escaped) bad.push("Escape did not close modal");
+ if (document.activeElement !== open) bad.push("focus did not return to opener");
+ if (!afterOpen.includes("middle") || !afterOpen.includes("last")) bad.push(`focus cycle incomplete: ${afterOpen.join(">")}`);
+ if (!hasAdjacent(afterOpen, "last", "first") && !hasAdjacent(afterOpen, "first", "last")) bad.push(`no wrap-around focus trap observed: ${afterOpen.join(">")}`);
+ if (outsideFocusWhileOpen) bad.push("focus escaped outside modal while open");
+ if (bad.length) Challenge.fail(...bad); else Challenge.pass(`modal focus trap ok: ${afterOpen.join(">")}`);
+}
+</script>
+</body>
diff --git a/test-suite/challenges/37-autocomplete-combobox.html b/test-suite/challenges/37-autocomplete-combobox.html
new file mode 100644
index 0000000..91f35c2
--- /dev/null
+++ b/test-suite/challenges/37-autocomplete-combobox.html
@@ -0,0 +1,42 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>37 autocomplete combobox</title>
+<link rel="stylesheet" href="../_style.css">
+<script src="../_lib.js"></script>
+<body>
+<main>
+ <p>Goal: type <code>as</code>, use ArrowDown then Enter to choose <code>Aster Lamp</code> from ARIA combobox.</p>
+ <label>Product <input id="combo" role="combobox" aria-autocomplete="list" aria-controls="list" aria-expanded="false" autocomplete="off"></label>
+ <ul id="list" role="listbox" style="border:1px solid #555;max-width:260px;padding:6px;display:none"></ul>
+</main>
+<script>
+Challenge.init({ id: "autocomplete-combobox", instructions: "type 'as', ArrowDown, Enter to choose Aster Lamp" });
+const items = ["Aster Lamp", "Aspen Desk", "Cable Kit", "Desk Mat", "Red Mug"];
+let active = -1, opened = false, keyPath = [];
+combo.addEventListener("input", renderList);
+combo.addEventListener("keydown", e => {
+ keyPath.push(e.key);
+ const opts = [...list.querySelectorAll('[role="option"]')];
+ if (e.key === "ArrowDown") { e.preventDefault(); active = Math.min(active + 1, opts.length - 1); paintActive(opts); }
+ if (e.key === "Enter" && active >= 0 && opts[active]) { e.preventDefault(); combo.value = opts[active].textContent; close(); check(e); }
+});
+function renderList(){
+ const q = combo.value.toLowerCase();
+ const matches = items.filter(x => x.toLowerCase().includes(q));
+ list.innerHTML = matches.map((x,i)=>`<li role="option" id="opt-${i}" style="padding:4px;cursor:pointer">${x}</li>`).join("");
+ list.style.display = matches.length ? "block" : "none";
+ combo.setAttribute("aria-expanded", matches.length ? "true" : "false");
+ active = -1; opened ||= matches.length > 0;
+}
+function paintActive(opts){ opts.forEach((o,i)=>o.style.background = i===active ? "#2b4b6b" : ""); if(opts[active]) combo.setAttribute("aria-activedescendant", opts[active].id); }
+function close(){ list.style.display="none"; combo.setAttribute("aria-expanded","false"); }
+function check(e){
+ const bad=[];
+ if (!opened) bad.push("listbox never opened");
+ if (combo.value !== "Aster Lamp") bad.push(`selected ${combo.value}`);
+ if (!keyPath.includes("ArrowDown") || !keyPath.includes("Enter")) bad.push(`missing keyboard selection path: ${keyPath.join(",")}`);
+ if (!e.isTrusted) bad.push("Enter key isTrusted=false");
+ if (bad.length) Challenge.fail(...bad); else Challenge.pass("ARIA combobox selected via keyboard");
+}
+</script>
+</body>
diff --git a/test-suite/challenges/38-spa-route-change.html b/test-suite/challenges/38-spa-route-change.html
new file mode 100644
index 0000000..533c38d
--- /dev/null
+++ b/test-suite/challenges/38-spa-route-change.html
@@ -0,0 +1,40 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>38 SPA route change</title>
+<link rel="stylesheet" href="../_style.css">
+<script src="../_lib.js"></script>
+<body>
+<main>
+ <p>Goal: navigate within SPA using links/buttons. URL must change via <code>history.pushState</code> without full reload. Then go Back to Settings, forward to Billing, and Save.</p>
+ <nav><a href="/settings" id="settings">Settings</a> <a href="/settings/billing" id="billing">Billing</a></nav>
+ <section id="view"></section>
+</main>
+<script>
+Challenge.init({ id: "spa-route-change", instructions: "Settings → Billing → Back → Billing → Save, no full reload" });
+let navs = 0, popstates = 0;
+window.__spaBootId = window.__spaBootId || Math.random().toString(36).slice(2);
+const bootId = window.__spaBootId;
+const navigationEntriesAtBoot = performance.getEntriesByType("navigation").length;
+function route(path){ history.pushState({}, "", path); navs++; render(); Challenge.log("route", { path, navs }); }
+settings.addEventListener("click", e => { e.preventDefault(); route("/settings"); });
+billing.addEventListener("click", e => { e.preventDefault(); route("/settings/billing"); });
+window.addEventListener("popstate", () => { popstates++; render(); Challenge.log("popstate", { path: location.pathname, popstates }); });
+function render(){
+ if (location.pathname.endsWith("/settings/billing")) view.innerHTML = `<h2>Billing</h2><button id="save">Save billing settings</button>`;
+ else if (location.pathname.endsWith("/settings")) view.innerHTML = `<h2>Settings</h2><p>Choose Billing.</p>`;
+ else view.innerHTML = `<h2>Home</h2>`;
+ const save = document.getElementById("save");
+ if (save) save.addEventListener("click", e => {
+ const bad=[];
+ if (!location.pathname.endsWith("/settings/billing")) bad.push(`wrong route ${location.pathname}`);
+ if (navs < 2) bad.push(`only ${navs} SPA route changes`);
+ if (popstates < 1) bad.push("browser Back/popstate was not observed");
+ if (window.__spaBootId !== bootId) bad.push("window sentinel changed; reload detected");
+ if (performance.getEntriesByType("navigation").length !== navigationEntriesAtBoot) bad.push("navigation entry count changed; reload suspected");
+ if (!e.isTrusted) bad.push("save click isTrusted=false");
+ if (bad.length) Challenge.fail(...bad); else Challenge.pass("SPA pushState flow completed without reload");
+ });
+}
+render();
+</script>
+</body>
diff --git a/test-suite/fixtures/sites/mini-shop/cheats.js b/test-suite/fixtures/sites/mini-shop/cheats.js
new file mode 100644
index 0000000..7407ee4
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/cheats.js
@@ -0,0 +1,12 @@
+window.MiniShopCheats = {
+ "mini-shop-red-second-cheapest": [
+ { tool: "chrome_click", params: { selector: "button[aria-label=\"add Ruby Notebook\"]" } },
+ { tool: "chrome_click", params: { selector: "#checkout" } }
+ ],
+ run: {
+ "mini-shop-red-second-cheapest": async () => {
+ document.querySelector('button[aria-label="add Ruby Notebook"]').click();
+ document.querySelector('#checkout').click();
+ }
+ }
+};
diff --git a/test-suite/fixtures/sites/mini-shop/grader.js b/test-suite/fixtures/sites/mini-shop/grader.js
new file mode 100644
index 0000000..74605f4
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/grader.js
@@ -0,0 +1,29 @@
+window.MiniShopGrader = {
+ validate(task, state, products, run, seed = 0) {
+ let bad = [];
+ const rubric = {};
+ if (task === 'mini-shop-red-second-cheapest') {
+ const reds = products.filter(p => p.color === 'red').sort((a,b)=>a.price-b.price);
+ const target = reds[1];
+ rubric.addedTarget = state.cart.includes(target.id);
+ rubric.noWrongItems = state.cart.every(id => id === target.id);
+ rubric.checkedOut = /^MS-\d{4}$/.test(state.orderId || '');
+ if (!rubric.addedTarget) bad.push(`cart missing second-cheapest red item ${target.name}`);
+ const wrong = state.cart.filter(id => id !== target.id);
+ if (wrong.length) bad.push(`cart contains wrong product(s): ${wrong.join(',')}`);
+ if (!rubric.checkedOut) bad.push('missing checkout order id');
+ } else bad.push(`unknown task ${task}`);
+ const passed = bad.length === 0;
+ return { reward: passed ? 1 : 0, done: passed, message: passed ? 'all deterministic mini-shop checks passed' : bad.join('; '), info: { task, run, seed, rubric, bad, state } };
+ },
+ grade(task, state, products, run, seed = 0) {
+ const validation = this.validate(task, state, products, run, seed);
+ const verdict = validation.done ? 'PASS' : 'PENDING';
+ window.__taskId = task;
+ window.__taskVerdict = verdict;
+ window.__taskReason = validation.done ? [validation.message] : validation.info.bad;
+ window.__taskValidation = validation;
+ try { localStorage.setItem(`pi-chrome-task:${task}:${run}`, JSON.stringify({ id: task, run, seed, verdict, reason: window.__taskReason, validation, state, ts: Date.now() })); } catch {}
+ return validation;
+ }
+};
diff --git a/test-suite/fixtures/sites/mini-shop/index.html b/test-suite/fixtures/sites/mini-shop/index.html
new file mode 100644
index 0000000..0fd63ff
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/index.html
@@ -0,0 +1,55 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>MiniShop hermetic site</title>
+<link rel="stylesheet" href="../../../_style.css">
+<style>
+main{max-width:1100px}.products{display:grid;grid-template-columns:repeat(auto-fit,minmax(210px,1fr));gap:12px}.card{border:1px solid #333;border-radius:10px;background:#202020;padding:12px}.cart{position:sticky;top:8px;background:#111;border:1px solid #555;border-radius:10px;padding:12px;margin-bottom:12px}.red{color:#ff8b8b}.blue{color:#8bbcff}.green{color:#9f9}.muted{color:#aaa;font-size:12px}</style>
+<body>
+<main>
+<h1>MiniShop</h1>
+<div class="cart"><b>Cart</b> <span id="cartCount">0</span> items <button id="checkout" data-task-role="checkout">Checkout</button> <span id="order"></span> <span id="reward" class="muted"></span><div id="cartItems"></div></div>
+<p id="taskText"></p>
+<section class="products" id="productsEl"></section>
+</main>
+<script src="grader.js?v=2"></script>
+<script src="cheats.js?v=2"></script>
+<script>
+(function(){
+const params=new URLSearchParams(location.search);
+if(!params.get('run')){params.set('run', crypto.randomUUID ? crypto.randomUUID() : String(Date.now())); history.replaceState(null,'',`?${params}`);}
+const task=params.get('task')||'mini-shop-red-second-cheapest'; const run=params.get('run'); const seed=Number(params.get('seed')||0);
+const key=`mini-shop:${task}:${run}:${seed}`;
+const products=[
+ {id:'p1',name:'Crimson Mug',color:'red',price:12.00},
+ {id:'p2',name:'Ruby Notebook',color:'red',price:7.50},
+ {id:'p3',name:'Scarlet Pen',color:'red',price:3.25},
+ {id:'p4',name:'Blue Cable',color:'blue',price:5.10},
+ {id:'p5',name:'Green Mat',color:'green',price:9.00},
+ {id:'p6',name:'Red Stapler',color:'red',price:18.00},
+ {id:'p7',name:'Blue Lamp',color:'blue',price:22.00},
+ {id:'p8',name:'Green Clip',color:'green',price:2.40}
+];
+let state=JSON.parse(localStorage.getItem(key)||'{"cart":[],"orderId":"","actionCount":0,"startedAt":0}');
+if(!state.startedAt) state.startedAt=Date.now();
+taskText.textContent='Task: add the second-cheapest red product to cart, checkout, and leave order id visible.';
+function save(){localStorage.setItem(key,JSON.stringify(state)); window.__miniShopState=state; const v=MiniShopGrader.grade(task,state,products,run,seed); window.__taskReward=v.reward; window.__taskTerminated=v.done; window.__taskInfo=v.info; reward.textContent=`reward=${v.reward} actions=${state.actionCount||0}`; renderCart();}
+function assignBids(){let n=0; document.querySelectorAll('button,a,input,select,textarea').forEach(el=>{const bid='b'+(++n).toString(36); el.setAttribute('bid',bid); el.setAttribute('data-bid',bid); const r=el.getBoundingClientRect(); el.setAttribute('browsergym_visibility_ratio',(r.width&&r.height?1:0).toString());});}
+function render(){productsEl.innerHTML=''; products.forEach(p=>{const d=document.createElement('div');d.className='card';d.innerHTML=`<h3 class="${p.color}">${p.name}</h3><p>Color: ${p.color}</p><p>Price: $${p.price.toFixed(2)}</p><button aria-label="add ${p.name}" data-id="${p.id}" data-task-role="add-to-cart">Add to cart</button>`;productsEl.appendChild(d)});document.querySelectorAll('[data-id]').forEach(b=>b.onclick=()=>{state.actionCount=(state.actionCount||0)+1;state.cart.push(b.dataset.id);save()});renderCart();assignBids();}
+function renderCart(){
+ cartCount.textContent=state.cart.length; order.textContent=state.orderId?`Order ${state.orderId}`:'';
+ cartItems.innerHTML=state.cart.map((id,i)=>{const p=products.find(x=>x.id===id);return `<div>${p?.name||id} <button data-remove-index="${i}" aria-label="remove ${p?.name||id}" data-task-role="remove-from-cart">Remove</button></div>`}).join('');
+ cartItems.querySelectorAll('[data-remove-index]').forEach(b=>b.onclick=()=>{state.actionCount=(state.actionCount||0)+1;state.cart.splice(Number(b.dataset.removeIndex),1);save();render();});
+ assignBids();
+}
+function orderId(){ return 'MS-' + String(1000 + ((seed * 2654435761 + state.cart.join('').length * 97) % 9000)).padStart(4, '0'); }
+checkout.onclick=()=>{state.actionCount=(state.actionCount||0)+1;state.orderId=orderId();save();};
+function observation(){return {goal:taskText.textContent,url:location.href,title:document.title,reward:window.__taskReward,actionCount:state.actionCount||0,focusedElementBid:document.activeElement?.getAttribute('bid')||null};}
+async function runCheat(){const fn=window.MiniShopCheats?.run?.[task]; if(fn) await fn(); else for(const step of (window.MiniShopCheats?.[task]||[])) console.log('cheat step',step); save();}
+window.__miniShopState=state; window.__miniShopProducts=products; window.__miniShopTask=task;
+window.__task={id:task,run,seed,goal:taskText.textContent,difficulty:'L2',step:()=>{const v=MiniShopGrader.grade(task,state,products,run,seed);return {observation:observation(),reward:v.reward,terminated:v.done,truncated:false,info:v.info}},reset:()=>{localStorage.removeItem(key);state={cart:[],orderId:'',actionCount:0,startedAt:Date.now()};render();save();return {observation:observation(),info:{task,run,seed}}},rubric:()=>MiniShopGrader.validate(task,state,products,run,seed).info.rubric,cheat:runCheat,validate:()=>MiniShopGrader.validate(task,state,products,run,seed),teardown:()=>undefined};
+window.__miniShop=window.__task;
+window.__miniShop.cheatHelpers={click:(sel)=>document.querySelector(sel).click()};
+render(); save();
+})();
+</script>
+</body>
diff --git a/test-suite/fixtures/sites/mini-shop/tasks.json b/test-suite/fixtures/sites/mini-shop/tasks.json
new file mode 100644
index 0000000..6812baf
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/tasks.json
@@ -0,0 +1,9 @@
+[
+ {
+ "id": "mini-shop-red-second-cheapest",
+ "difficulty": "L2",
+ "intent": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
+ "gradeSource": "harness",
+ "grader": "MiniShopGrader.grade(task, state, products, run)"
+ }
+]
diff --git a/test-suite/index.html b/test-suite/index.html
index a4a3c0c..00ef204 100644
--- a/test-suite/index.html
+++ b/test-suite/index.html
@@ -35,6 +35,13 @@
<tbody></tbody>
</table>
+<h2>Long-horizon hermetic tasks</h2>
+<p class="hint">These WebArena/BrowserGym-inspired tasks use deterministic in-page graders and fresh <code>$RUN_ID</code> state. They test multi-step browser work beyond event fidelity.</p>
+<table id="taskTbl">
+ <thead><tr><th>id</th><th>difficulty</th><th>category</th><th>intent</th><th>requires</th><th>open</th></tr></thead>
+ <tbody></tbody>
+</table>
+
<h2>Agent run loop</h2>
<pre class="code">python3 -m http.server 8765
# For each manifest entry:
@@ -46,7 +53,9 @@
</main>
<script>
let manifest = [];
+let taskManifest = [];
const tbody = document.querySelector("#tbl tbody");
+const taskTbody = document.querySelector("#taskTbl tbody");
const modeEl = document.getElementById("mode");
const filterEl = document.getElementById("filter");
@@ -123,9 +132,28 @@ function paint() {
function escapeHtml(s) {
return String(s).replace(/[&<>"']/g, c => ({"&":"&amp;","<":"&lt;",">":"&gt;","\"":"&quot;","'":"&#39;"}[c]));
}
+function paintTasks() {
+ taskTbody.innerHTML = "";
+ const run = "manual-" + new Date().toISOString().slice(0,10);
+ for (const t of taskManifest) {
+ const url = t.startUrl.replaceAll("$RUN_ID", run).replaceAll("$SEED", String(t.seed ?? 0));
+ const tr = document.createElement("tr");
+ tr.innerHTML = `
+ <td><code>${t.taskId || t.id}</code><div class="notes">seed=${t.seed ?? 0} maxSteps=${t.maxSteps ?? "—"}</div></td>
+ <td><span class="pill expected">${t.difficulty}</span></td>
+ <td>${t.category}</td>
+ <td>${escapeHtml(t.goal || t.intent)}</td>
+ <td><div class="notes">${escapeHtml((t.requires || []).join(", "))}</div><div class="notes">actions: ${escapeHtml((t.actionSubsets || []).join(", "))}</div></td>
+ <td><a target="_blank" href="${url}">open</a></td>
+ `;
+ taskTbody.appendChild(tr);
+ }
+}
async function init() {
manifest = await fetch("manifest.json", { cache: "no-store" }).then(r => r.json());
+ taskManifest = await fetch("task-manifest.json", { cache: "no-store" }).then(r => r.json()).catch(() => []);
paint();
+ paintTasks();
}
document.getElementById("refresh").addEventListener("click", paint);
document.getElementById("clear").addEventListener("click", () => {
diff --git a/test-suite/manifest.json b/test-suite/manifest.json
index c09cf31..900b5d3 100644
--- a/test-suite/manifest.json
+++ b/test-suite/manifest.json
@@ -21,7 +21,13 @@
"notes": [
"Core debugger/CDP smoke test."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "trusted-input"
+ ],
+ "requires": {}
},
{
"id": "is-trusted-keyboard",
@@ -43,7 +49,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "trusted-input"
+ ],
+ "requires": {}
},
{
"id": "webdriver-flag",
@@ -66,7 +78,13 @@
"notes": [
"Environment-sensitive; classify unexpected failures as profile warnings first."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "fingerprint"
+ ],
+ "requires": {}
},
{
"id": "mouse-entropy",
@@ -74,7 +92,7 @@
"category": "pointer-humanization",
"goal": "Click is preceded by organic mousemove path.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "FAIL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -86,7 +104,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "event-timing",
@@ -94,7 +118,7 @@
"category": "timing",
"goal": "Repeated clicks have plausible holds and gaps.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -119,7 +143,17 @@
}
],
"flakeRisk": "medium",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "timing"
+ ],
+ "requires": {},
+ "thresholds": {},
+ "notes": [
+ "Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices."
+ ]
},
{
"id": "click-coordinates",
@@ -127,7 +161,7 @@
"category": "pointer-humanization",
"goal": "Clicks land inside target but not always exact center.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -163,7 +197,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "pointer-properties",
@@ -171,7 +211,7 @@
"category": "pointer-humanization",
"goal": "Pointer events carry plausible pointerType, pressure, movement, id.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -183,7 +223,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "keyboard-cadence",
@@ -191,7 +237,7 @@
"category": "keyboard",
"goal": "Typing emits per-character key events with non-uniform cadence.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -204,7 +250,17 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "keyboard"
+ ],
+ "requires": {},
+ "thresholds": {},
+ "notes": [
+ "Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices."
+ ]
},
{
"id": "composition-input",
@@ -225,7 +281,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "keyboard"
+ ],
+ "requires": {}
},
{
"id": "user-activation",
@@ -252,7 +314,13 @@
"clipboard/fullscreen allowed by browser policy"
],
"flakeRisk": "high",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "activation-gates"
+ ],
+ "requires": {}
},
{
"id": "honeypot",
@@ -279,7 +347,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "agent-safety"
+ ],
+ "requires": {}
},
{
"id": "fingerprint",
@@ -303,7 +377,13 @@
"non-headless normal Chrome profile"
],
"flakeRisk": "medium",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "fingerprint"
+ ],
+ "requires": {}
},
{
"id": "focus-order",
@@ -326,7 +406,13 @@
}
],
"flakeRisk": "medium",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "focus-keyboard"
+ ],
+ "requires": {}
},
{
"id": "wheel-scroll",
@@ -347,7 +433,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "scroll"
+ ],
+ "requires": {}
},
{
"id": "drag-drop-datatransfer",
@@ -372,7 +464,13 @@
"notes": [
"May need CDP Input.dispatchDragEvent rather than mouse drag on some Chrome versions."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "drag-drop"
+ ],
+ "requires": {}
},
{
"id": "contenteditable-selection",
@@ -380,7 +478,7 @@
"category": "editing",
"goal": "Contenteditable typing advances selection/caret like user typing.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -393,7 +491,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "editing"
+ ],
+ "requires": {}
},
{
"id": "paste-clipboard",
@@ -428,7 +532,13 @@
"OS clipboard contains pi-chrome",
"use ctrlKey instead of metaKey on Windows/Linux"
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "clipboard"
+ ],
+ "requires": {}
},
{
"id": "native-select",
@@ -471,7 +581,13 @@
}
],
"flakeRisk": "high",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "native-controls"
+ ],
+ "requires": {}
},
{
"id": "hover-dwell",
@@ -505,7 +621,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "hover"
+ ],
+ "requires": {}
},
{
"id": "react-value-tracker",
@@ -526,7 +648,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "frameworks"
+ ],
+ "requires": {}
},
{
"id": "keyboard-modifiers",
@@ -550,7 +678,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "keyboard"
+ ],
+ "requires": {}
},
{
"id": "touch-events",
@@ -573,7 +707,13 @@
"prerequisites": [
"touch events enabled/supported"
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "touch"
+ ],
+ "requires": {}
},
{
"id": "stack-trace-fingerprint",
@@ -595,9 +735,16 @@
}
],
"notes": [
- "Adversarial probe; selector resolution itself can be detected by monkey-patched DOM APIs."
+ "Adversarial probe; selector resolution itself can be detected by monkey-patched DOM APIs.",
+ "Should fail only on concrete automation URLs/Runtime frames, not generic anonymous stack formatting."
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "fingerprint"
],
- "manualBaseline": "unverified"
+ "requires": {}
},
{
"id": "viewport-edge-clicks",
@@ -618,7 +765,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "pointer-continuity",
@@ -646,7 +799,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "mousemove-rate",
@@ -675,10 +834,18 @@
}
],
"notes": [
- "Needs a first-class pointer path/sweep tool for deterministic automation."
+ "Needs a first-class pointer path/sweep tool for deterministic automation.",
+ "Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices."
],
"flakeRisk": "high",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {},
+ "thresholds": {}
},
{
"id": "scroll-momentum",
@@ -703,7 +870,16 @@
"notes": [
"Physical detent mouse wheels may fail decay requirement; trackpads usually pass."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "scroll"
+ ],
+ "requires": {},
+ "prerequisites": [
+ "Trackpad-like momentum; physical detent mouse wheels may fail decay requirement even for manual users."
+ ]
},
{
"id": "intersection-visibility",
@@ -733,7 +909,13 @@
}
],
"flakeRisk": "medium",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "scroll-visibility"
+ ],
+ "requires": {}
},
{
"id": "shadow-dom-controls",
@@ -767,7 +949,13 @@
"manualBaseline": "unverified",
"prerequisites": [
"pi-chrome shadow-piercing selectors or shadow-root-aware snapshot uids"
- ]
+ ],
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "dom-complexity"
+ ],
+ "requires": {}
},
{
"id": "iframe-targeting",
@@ -800,7 +988,13 @@
"manualBaseline": "unverified",
"prerequisites": [
"pi-chrome currently targets frameId 0 for many actions; iframe drilling needs frame-targeting support"
- ]
+ ],
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "frames"
+ ],
+ "requires": {}
},
{
"id": "file-upload",
@@ -826,7 +1020,13 @@
"manualBaseline": "unverified",
"notes": [
"Recipe paths may contain $PWD; external runners should expand shell-style placeholders before calling chrome_upload_file."
- ]
+ ],
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "files"
+ ],
+ "requires": {}
},
{
"id": "keyboard-tab-navigation",
@@ -903,7 +1103,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "focus-keyboard"
+ ],
+ "requires": {}
},
{
"id": "network-console-capture",
@@ -911,7 +1117,7 @@
"category": "observability",
"goal": "Interaction emits console logs and fetch traffic for tool-level capture APIs.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -947,6 +1153,285 @@
"Page verdict only checks in-page fetch result; benchmark harness should also assert captured console/network entries.",
"First chrome_list_console_messages/chrome_list_network_requests call installs capture hooks; do not move clear/list calls after click."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "observability"
+ ],
+ "requires": {}
+ },
+ {
+ "id": "dialog-handling",
+ "file": "challenges/34-dialog-handling.html",
+ "category": "dialogs",
+ "difficulty": "L2",
+ "goal": "Agent detects and handles alert/confirm/prompt/beforeunload browser dialogs.",
+ "expected": {
+ "synthetic": "CONDITIONAL",
+ "trusted": "CONDITIONAL",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#promptBtn",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "dialog",
+ "params": {
+ "type": "prompt",
+ "text": "pi-chrome",
+ "accept": true
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#confirmBtn",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "dialog",
+ "params": {
+ "type": "confirm",
+ "accept": true
+ }
+ }
+ ],
+ "requires": {
+ "dialogs": true
+ },
+ "prerequisites": [
+ "browser-dialog handling API or manual user intervention"
+ ],
+ "tags": [
+ "dialogs",
+ "native-browser-ui"
+ ],
+ "notes": [
+ "If dialog handling is unsupported, page may block after click; this is expected signal."
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "target-blank-popup",
+ "file": "challenges/35-target-blank-popup.html",
+ "category": "popups",
+ "difficulty": "L1",
+ "goal": "Agent handles target=_blank new tab, discovers it via tab list, and inspects child tab verdict.",
+ "expected": {
+ "synthetic": "CONDITIONAL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#open",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_tab",
+ "params": {
+ "action": "list"
+ }
+ },
+ {
+ "tool": "chrome_snapshot",
+ "params": {
+ "titleIncludes": "target blank popup"
+ }
+ }
+ ],
+ "requires": {
+ "popups": true
+ },
+ "tags": [
+ "tabs",
+ "popup"
+ ],
+ "notes": [
+ "Child tab writes PASS to shared localStorage; harness should verify tab count/title too."
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "modal-focus-trap",
+ "file": "challenges/36-modal-focus-trap.html",
+ "category": "focus-keyboard",
+ "difficulty": "L2",
+ "goal": "Keyboard focus stays trapped inside modal and Escape closes with focus restoration.",
+ "expected": {
+ "synthetic": "CONDITIONAL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#open",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Tab",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Tab",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Tab",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Escape",
+ "trusted": true
+ }
+ }
+ ],
+ "requires": {
+ "cdp": true
+ },
+ "tags": [
+ "modal",
+ "focus",
+ "keyboard"
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "autocomplete-combobox",
+ "file": "challenges/37-autocomplete-combobox.html",
+ "category": "forms",
+ "difficulty": "L2",
+ "goal": "Agent uses ARIA combobox keyboard path: type filter, ArrowDown, Enter.",
+ "expected": {
+ "synthetic": "FAIL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_type",
+ "params": {
+ "selector": "#combo",
+ "text": "as",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "ArrowDown",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Enter",
+ "trusted": true
+ }
+ }
+ ],
+ "requires": {
+ "cdp": true
+ },
+ "tags": [
+ "aria",
+ "combobox",
+ "forms"
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "spa-route-change",
+ "file": "challenges/38-spa-route-change.html",
+ "category": "spa-routing",
+ "difficulty": "L1",
+ "goal": "Agent recognizes SPA pushState navigation and completes route-local action without full reload.",
+ "expected": {
+ "synthetic": "CONDITIONAL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#settings",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#billing",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "ArrowLeft",
+ "modifiers": {
+ "altKey": true
+ },
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#billing",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#save",
+ "trusted": true
+ }
+ }
+ ],
+ "requires": {
+ "cdp": true
+ },
+ "tags": [
+ "spa",
+ "history",
+ "navigation"
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "notes": [
+ "Requires a real browser Back action (e.g. Alt+Left) to exercise popstate before Save."
+ ]
}
]
diff --git a/test-suite/manifest.schema.json b/test-suite/manifest.schema.json
index 79371e8..cd6dcce 100644
--- a/test-suite/manifest.schema.json
+++ b/test-suite/manifest.schema.json
@@ -1,7 +1,7 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "pi-chrome benchmark manifest",
- "description": "Manifest for browser-control benchmark pages. Recipes are canonical intent, not guaranteed raw tool JSON: runners may need to adapt descriptive selectors (frame/shadow) and expand path placeholders such as $PWD before invoking tools.",
+ "description": "Manifest for browser-control benchmark pages. Recipes are canonical intent, not guaranteed raw tool JSON: runners may need to adapt descriptive selectors (frame/shadow) and expand path placeholders such as ${REPO_ROOT} before invoking tools.",
"type": "array",
"items": {
"type": "object",
@@ -11,7 +11,10 @@
"id": { "type": "string" },
"file": { "type": "string" },
"category": { "$ref": "#/$defs/category" },
+ "difficulty": { "enum": ["L1", "L2", "L3"] },
+ "tags": { "type": "array", "items": { "type": "string" } },
"goal": { "type": "string" },
+ "gradeSource": { "enum": ["page", "harness", "both"], "default": "page" },
"expected": {
"type": "object",
"description": "Expected verdict by execution mode. CONDITIONAL means prerequisites, browser policy, hardware, or missing tool primitives decide outcome.",
@@ -22,10 +25,7 @@
"manual": { "$ref": "#/$defs/outcome" }
}
},
- "manualBaseline": {
- "description": "Whether human baseline has been explicitly verified in this repo/environment.",
- "enum": ["unverified", "verified"]
- },
+ "manualBaseline": { "description": "Whether human baseline has been explicitly verified in this repo/environment.", "enum": ["unverified", "verified"] },
"manualBaselineVerifiedAt": { "type": "string", "format": "date-time" },
"recipe": {
"type": "array",
@@ -35,15 +35,28 @@
"required": ["tool", "params"],
"properties": {
"tool": { "type": "string" },
- "params": {
- "type": "object",
- "description": "Tool params. For upload recipes, paths are strings after runner expansion; $PWD means repository working directory."
- }
+ "params": { "type": "object", "description": "Tool params. For upload recipes, paths are strings after runner expansion; ${REPO_ROOT} means repository root." }
}
}
},
+ "requires": {
+ "type": "object",
+ "additionalProperties": true,
+ "properties": {
+ "cdp": { "type": "boolean" },
+ "clipboard": { "type": "boolean" },
+ "touch": { "type": "boolean" },
+ "secureOrigin": { "type": "boolean" },
+ "fileSystem": { "type": "boolean" },
+ "dialogs": { "type": "boolean" },
+ "downloads": { "type": "boolean" },
+ "popups": { "type": "boolean" }
+ }
+ },
"prerequisites": { "type": "array", "items": { "type": "string" } },
"notes": { "type": "array", "items": { "type": "string" } },
+ "thresholds": { "type": "object", "additionalProperties": true },
+ "timeoutMs": { "type": "number" },
"verdictDelayMs": { "type": "number" },
"flakeRisk": { "enum": ["low", "medium", "high"] }
}
@@ -52,27 +65,7 @@
"outcome": { "enum": ["PASS", "FAIL", "SKIP", "WARN", "CONDITIONAL"] },
"category": {
"enum": [
- "activation-gates",
- "agent-safety",
- "clipboard",
- "dom-complexity",
- "drag-drop",
- "editing",
- "files",
- "fingerprint",
- "focus-keyboard",
- "frames",
- "frameworks",
- "hover",
- "keyboard",
- "native-controls",
- "observability",
- "pointer-humanization",
- "scroll",
- "scroll-visibility",
- "timing",
- "touch",
- "trusted-input"
+ "activation-gates", "agent-safety", "aria-snapshot", "canvas-svg", "clipboard", "contextmenu", "dialogs", "dom-complexity", "downloads", "drag-drop", "editing", "files", "fingerprint", "focus-keyboard", "forms", "frames", "frameworks", "hover", "keyboard", "lazy-loading", "native-controls", "observability", "pointer-humanization", "popups", "scroll", "scroll-visibility", "spa-routing", "task-completion", "timing", "touch", "trusted-input"
]
}
}
diff --git a/test-suite/notes/browsergym-compat.md b/test-suite/notes/browsergym-compat.md
new file mode 100644
index 0000000..1eaea41
--- /dev/null
+++ b/test-suite/notes/browsergym-compat.md
@@ -0,0 +1,70 @@
+# BrowserGym compatibility contract
+
+Pi-chrome benchmark tasks mirror BrowserGym without importing Gymnasium.
+
+## Task fields
+
+Each `task-manifest.json` entry maps to `AbstractBrowserTask` concepts:
+
+- `taskId` / `id` — stable namespaced id
+- `seed` — deterministic state seed
+- `viewport`, `slowMoMs`, `timeoutMs`, `locale`, `timezoneId`
+- `goal` and `goalObject` — text and OpenAI-style message objects
+- `setupUrl` — page URL that performs setup from `task`, `run`, `seed`
+- `validateScript` — JS validate hook returning reward contract
+- `cheatScript` — gold recipe used to sanity-check grader
+- `maxSteps`, `humanBaseline`, `difficulty`, `tags`
+- `actionSubsets`, `allowedActions` — BrowserGym HighLevelActionSet metadata
+
+## Reset / step return
+
+Runner should expose:
+
+```text
+reset() -> (obs, info)
+step(action) -> (obs, reward, terminated, truncated, info)
+```
+
+`terminated` means task done. `truncated` means max step or timeout.
+
+## Validate return
+
+In-page validators return:
+
+```js
+{ reward, done, message, info }
+```
+
+`reward` is incremental. Current hermetic tasks use binary terminal reward. `info.rubric` carries per-check details.
+
+## Observation schema target
+
+Runner observations should include BrowserGym-like keys:
+
+- `chat_messages`
+- `goal`, `goal_object`
+- `open_pages_urls`, `open_pages_titles`, `active_page_index`
+- `url`, `screenshot`, `dom_object`, `axtree_object`
+- `extra_element_properties`
+- `focused_element_bid`
+- `last_action`, `last_action_error`
+- `elapsed_time`
+
+## BID plan
+
+BrowserGym uses `bid` attributes as stable element handles. Pi-chrome currently returns snapshot `uid`. Compatibility path:
+
+1. During snapshot, assign every interactive element a deterministic `bid`.
+2. Return both `uid` and `bid`.
+3. Add `visibility`, `bbox`, `clickable`, `set_of_marks` metadata.
+4. Maintain uid↔bid mapping so agents can use either addressing mode.
+
+Target attributes:
+
+- `bid`
+- `browsergym_visibility_ratio`
+- `browsergym_set_of_marks`
+
+## Action subsets
+
+See `../browsergym-action-space.json` for BrowserGym-compatible subsets and function signatures.
diff --git a/test-suite/notes/profiles.md b/test-suite/notes/profiles.md
new file mode 100644
index 0000000..8f55797
--- /dev/null
+++ b/test-suite/notes/profiles.md
@@ -0,0 +1,22 @@
+# Calibration profiles
+
+Benchmark results depend on Chrome profile, OS, input device, and extension mode.
+Record these with every run:
+
+- OS and version
+- Chrome version
+- pi-chrome package version
+- companion extension version
+- trusted mode: off / auto / on
+- input hardware: trackpad vs detent mouse, touch support
+- permissions: clipboard, fullscreen, downloads
+- viewport size and deviceScaleFactor
+- loaded privacy/security extensions
+
+Known environment-sensitive areas:
+
+- WebGL fingerprint can warn/fail in VMs, remote desktops, GPU-disabled Chrome.
+- Scroll momentum expects trackpad-like decay; detent mouse wheels may fail manually.
+- Touch tests require touch events enabled/supported.
+- Dialog/download/file-picker tests require native browser UI handling or manual intervention.
+- Stack traces vary across Chrome versions; tests should only fail concrete automation URLs/Runtime frames.
diff --git a/test-suite/notes/runner-spec.md b/test-suite/notes/runner-spec.md
new file mode 100644
index 0000000..b040cb4
--- /dev/null
+++ b/test-suite/notes/runner-spec.md
@@ -0,0 +1,26 @@
+# Recipe runner spec
+
+Future Node/Pi runner should:
+
+1. Read `manifest.json`.
+2. Probe environment capabilities (`trusted`, `clipboard`, `touch`, `dialogs`, `downloads`, `fileSystem`).
+3. For each entry:
+ - skip if `requires` is unsatisfied and expected is `CONDITIONAL`
+ - navigate to challenge URL
+ - execute `recipe` in order
+ - wait `verdictDelayMs || 500`
+ - evaluate `JSON.stringify({v:window.__verdict,r:window.__reason,d:window.Challenge?.state?.details,e:window.__events?.slice(-20)})`
+ - compare to `expected[mode]`
+4. Output:
+ - Markdown summary
+ - JSON details
+ - JUnit XML for CI
+
+Runner must adapt recipe intent:
+
+- expand `${REPO_ROOT}` path placeholders
+- adapt unsupported shadow/iframe selector notation to snapshot uids or evaluate fallback
+- preserve hook install ordering for console/network capture tests
+- record whether trusted/CDP path was used
+
+Long-horizon task runner should read `task-manifest.json`, replace `$RUN_ID`, solve task, then evaluate task grader expression.
diff --git a/test-suite/notes/scoring.md b/test-suite/notes/scoring.md
new file mode 100644
index 0000000..970a706
--- /dev/null
+++ b/test-suite/notes/scoring.md
@@ -0,0 +1,36 @@
+# Benchmark scoring
+
+## Unit challenges (`manifest.json`)
+
+Score actual verdict against `expected[mode]`, not raw PASS count.
+
+- `PASS` / `FAIL` expected: exact match required.
+- `CONDITIONAL`: exclude from headline score unless environment prerequisite is declared present.
+- `SKIP` / `WARN`: report separately.
+
+Recommended flake policy:
+
+- Run each non-destructive challenge up to 2 retries.
+- Take best verdict only for known-flaky timing/scroll tests.
+- Always keep all reasons/events in detailed report.
+
+Difficulty weighting:
+
+- L1 = 1
+- L2 = 2
+- L3 = 3
+
+Report both unweighted and weighted score. Also report per-category score.
+
+## Long-horizon tasks (`task-manifest.json`)
+
+Task score is deterministic grader PASS / not PASS. Record:
+
+- action count
+- wall time
+- tools used
+- observation mode (`snapshot`, screenshot, accessibility, evaluate)
+- whether direct state mutation/evaluate was allowed
+- final grader reason
+
+Do not use LLM judge for hermetic tasks unless grader cannot express the target.
diff --git a/test-suite/scenarios/choredesk/cheats.js b/test-suite/scenarios/choredesk/cheats.js
new file mode 100644
index 0000000..112679f
--- /dev/null
+++ b/test-suite/scenarios/choredesk/cheats.js
@@ -0,0 +1,49 @@
+window.ChoreDeskCheats = {
+ "choredesk-l1-ticket-priority": [
+ { tool: "chrome_fill", params: { selector: "select[aria-label=\"T-104 priority\"]", text: "High" } },
+ { tool: "chrome_fill", params: { selector: "select[aria-label=\"T-104 status\"]", text: "In Progress" } },
+ { tool: "chrome_click", params: { selector: "button[aria-label=\"save tickets T-104\"]" } }
+ ],
+ "choredesk-l2-refund-message": [
+ { tool: "chrome_click", params: { selector: "a[data-view=\"customers\"]" } },
+ { tool: "chrome_click", params: { selector: "a[data-view=\"orders\"]" } },
+ { tool: "chrome_click", params: { selector: "a[data-view=\"messages\"]" } },
+ { tool: "chrome_fill", params: { selector: "#msgSubject", text: "Refund review ORD-778" } },
+ { tool: "chrome_fill", params: { selector: "#msgBody", text: "Noah Kim noah.kim@example.com order ORD-778 total $84.20" } },
+ { tool: "chrome_click", params: { selector: "#sendMsg" } }
+ ],
+ "choredesk-l3-restock-ticket": [
+ { tool: "chrome_click", params: { selector: "a[data-view=\"catalog\"]" } },
+ { tool: "chrome_click", params: { selector: "a[data-view=\"inventory\"]" } },
+ { tool: "chrome_fill", params: { selector: "input[aria-label=\"AST-LAMP reorder quantity\"]", text: "12" } },
+ { tool: "chrome_click", params: { selector: "button[aria-label=\"save inventory AST-LAMP\"]" } },
+ { tool: "chrome_click", params: { selector: "a[data-view=\"tickets\"]" } },
+ { tool: "chrome_fill", params: { selector: "input[aria-label=\"T-205 internal note\"]", text: "reordered 12 AST-LAMP" } },
+ { tool: "chrome_click", params: { selector: "button[aria-label=\"save tickets T-205\"]" } }
+ ],
+ run: {
+ "choredesk-l1-ticket-priority": async () => {
+ await window.__choredesk.cheatHelpers.route("tickets");
+ window.__choredesk.cheatHelpers.set("select[aria-label=\"T-104 priority\"]", "High");
+ window.__choredesk.cheatHelpers.set("select[aria-label=\"T-104 status\"]", "In Progress");
+ document.querySelector("button[aria-label=\"save tickets T-104\"]").click();
+ },
+ "choredesk-l2-refund-message": async () => {
+ await window.__choredesk.cheatHelpers.route("customers");
+ await window.__choredesk.cheatHelpers.route("orders");
+ await window.__choredesk.cheatHelpers.route("messages");
+ window.__choredesk.cheatHelpers.set("#msgSubject", "Refund review ORD-778");
+ window.__choredesk.cheatHelpers.set("#msgBody", "Noah Kim noah.kim@example.com order ORD-778 total $84.20");
+ document.querySelector("#sendMsg").click();
+ },
+ "choredesk-l3-restock-ticket": async () => {
+ await window.__choredesk.cheatHelpers.route("catalog");
+ await window.__choredesk.cheatHelpers.route("inventory");
+ window.__choredesk.cheatHelpers.set("input[aria-label=\"AST-LAMP reorder quantity\"]", "12");
+ document.querySelector("button[aria-label=\"save inventory AST-LAMP\"]").click();
+ await window.__choredesk.cheatHelpers.route("tickets");
+ window.__choredesk.cheatHelpers.set("input[aria-label=\"T-205 internal note\"]", "reordered 12 AST-LAMP");
+ document.querySelector("button[aria-label=\"save tickets T-205\"]").click();
+ }
+ }
+};
diff --git a/test-suite/scenarios/choredesk/index.html b/test-suite/scenarios/choredesk/index.html
new file mode 100644
index 0000000..6481ac0
--- /dev/null
+++ b/test-suite/scenarios/choredesk/index.html
@@ -0,0 +1,239 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>ChoreDesk hermetic task site</title>
+<link rel="stylesheet" href="../../_style.css">
+<style>
+ main { max-width: 1180px; }
+ .top { display:flex; gap:12px; align-items:center; flex-wrap:wrap; }
+ .tabs { display:flex; gap:8px; margin:16px 0; flex-wrap:wrap; }
+ .tab { border:1px solid #444; border-radius:999px; padding:6px 10px; color:#eee; text-decoration:none; background:#222; }
+ .tab[aria-current="page"] { background:#2b4b6b; }
+ .card { border:1px solid #333; border-radius:10px; background:#202020; padding:14px; }
+ table { font-size:13px; }
+ td input, td select { width: 100%; box-sizing:border-box; }
+ textarea { width:100%; min-height:110px; background:#111; color:#eee; border:1px solid #555; border-radius:6px; padding:8px; }
+ .muted { color:#aaa; font-size:12px; }
+ .taskbar { border:1px solid #4b3b16; background:#221b0d; border-radius:10px; padding:12px; margin:12px 0; }
+ .verdict { display:inline-block; border-radius:999px; padding:2px 8px; font-weight:700; background:#444; }
+ .verdict.PASS { background:#1f7a1f; }
+ .verdict.FAIL { background:#a11; }
+ .verdict.PENDING { background:#444; }
+</style>
+<body>
+<main>
+ <div class="top">
+ <h1 style="margin-right:auto">ChoreDesk</h1>
+ <button id="reset" data-task-role="reset-button">Reset run</button>
+ <button id="grade" data-task-role="grade-button">Grade now</button>
+ <button id="cheat" data-task-role="cheat-button">Run cheat</button>
+ </div>
+ <div class="taskbar">
+ <div><b id="taskId"></b> <span id="difficulty"></span> <span id="verdict" class="verdict PENDING">PENDING</span> <span id="reward" class="muted"></span></div>
+ <p id="instruction"></p>
+ <pre id="reason" class="muted"></pre>
+ </div>
+ <nav class="tabs" aria-label="ChoreDesk sections"></nav>
+ <section id="view"></section>
+</main>
+<script src="cheats.js"></script>
+<script>
+(function () {
+const params = new URLSearchParams(location.search);
+if (!params.get("run")) {
+ params.set("run", crypto.randomUUID ? crypto.randomUUID() : String(Date.now()));
+ history.replaceState(null, "", `?${params}`);
+}
+const task = params.get("task") || "choredesk-l1-ticket-priority";
+const run = params.get("run");
+const seed = Number(params.get("seed") || 0);
+const nonce = params.get("nonce") || (crypto.randomUUID ? crypto.randomUUID() : Math.random().toString(36).slice(2));
+if (!params.get("nonce")) { params.set("nonce", nonce); history.replaceState(null, "", `?${params}`); }
+const storeKey = `pi-chrome-task-state:${task}:${run}:${seed}:${nonce}`;
+const verdictKey = `pi-chrome-task:${task}:${run}`;
+const views = ["tickets", "customers", "orders", "catalog", "inventory", "messages", "audit"];
+let view = params.get("view") || "tickets";
+let bidCounter = 0;
+
+const TASKS = {
+ "choredesk-l1-ticket-priority": {
+ difficulty: "L1 atomic",
+ instruction: "Set ticket T-104 priority to High and status to In Progress. Save the ticket.",
+ rubric(s) {
+ const t = s.tickets.find(x => x.id === "T-104");
+ return [
+ { id:"priorityHigh", weight:.33, pass:t?.priority === "High", description:"T-104 priority is High", reason:`T-104 priority=${t?.priority}` },
+ { id:"statusInProgress", weight:.33, pass:t?.status === "In Progress", description:"T-104 status is In Progress", reason:`T-104 status=${t?.status}` },
+ { id:"saved", weight:.34, pass:s.audit.some(a => a.kind === "ticket-save" && a.id === "T-104" && a.via === "ui-event"), description:"T-104 saved through UI", reason:"no UI save audit for T-104" }
+ ];
+ }
+ },
+ "choredesk-l2-refund-message": {
+ difficulty: "L2 compositional",
+ instruction: "A customer named Noah Kim reports a duplicate charge. Find Noah's email and order ORD-778 total, then send Billing a message with subject 'Refund review ORD-778'. Body must include Noah's email and the exact order total.",
+ rubric(s) {
+ const msg = s.messages.find(m => m.to === "Billing" && m.subject.trim() === "Refund review ORD-778");
+ return [
+ { id:"visitedCustomers", weight:.15, pass:!!s.viewsVisited.customers, description:"Opened Customers view", reason:"never opened Customers view" },
+ { id:"visitedOrders", weight:.15, pass:!!s.viewsVisited.orders, description:"Opened Orders view", reason:"never opened Orders view" },
+ { id:"subject", weight:.20, pass:!!msg, description:"Billing message with exact subject exists", reason:"missing Billing message with exact subject" },
+ { id:"email", weight:.25, pass:!!msg && /\bnoah\.kim@example\.com\b/i.test(msg.body), description:"Body includes Noah email", reason:"message body missing Noah email" },
+ { id:"total", weight:.25, pass:!!msg && /(?<![\d.])\$?84\.20\b/.test(msg.body), description:"Body includes exact ORD-778 total", reason:"message body missing exact ORD-778 total 84.20" }
+ ];
+ }
+ },
+ "choredesk-l3-restock-ticket": {
+ difficulty: "L3 cross-page tedious",
+ instruction: "Ticket T-205 says Aster Lamp is delayed. Find the Aster Lamp SKU in Catalog, set its Inventory reorder quantity to 12, then add an internal note to T-205 saying 'reordered 12 AST-LAMP'.",
+ rubric(s) {
+ const inv = s.inventory.find(x => x.sku === "AST-LAMP");
+ const t = s.tickets.find(x => x.id === "T-205");
+ return [
+ { id:"visitedCatalog", weight:.15, pass:!!s.viewsVisited.catalog, description:"Opened Catalog view", reason:"never opened Catalog view" },
+ { id:"visitedInventory", weight:.15, pass:!!s.viewsVisited.inventory, description:"Opened Inventory view", reason:"never opened Inventory view" },
+ { id:"inventory", weight:.25, pass:Number(inv?.reorderQty) === 12 && s.audit.some(a => a.kind === "inventory-save" && a.id === "AST-LAMP" && a.via === "ui-event"), description:"AST-LAMP reorderQty=12 saved through UI", reason:`AST-LAMP reorderQty=${inv?.reorderQty}; missing UI save audit` },
+ { id:"ticketNote", weight:.25, pass:!!t && t.note.includes("reordered 12 AST-LAMP"), description:"T-205 note contains exact phrase", reason:"T-205 internal note missing exact phrase" },
+ { id:"ticketSaved", weight:.20, pass:s.audit.some(a => a.kind === "ticket-save" && a.id === "T-205" && a.via === "ui-event"), description:"T-205 saved through UI", reason:"no UI save audit for T-205" }
+ ];
+ }
+ }
+};
+const SEED = {
+ tickets: [
+ { id:"T-101", customer:"Rosa Patel", subject:"Where is my desk mat?", priority:"Normal", status:"Open", note:"" },
+ { id:"T-104", customer:"Iris Zhao", subject:"Invoice address typo", priority:"Low", status:"Open", note:"" },
+ { id:"T-205", customer:"Mateo Silva", subject:"Aster Lamp delayed", priority:"Normal", status:"Open", note:"" },
+ { id:"T-309", customer:"Noah Kim", subject:"Duplicate charge", priority:"Normal", status:"Open", note:"" }
+ ],
+ customers: [
+ { name:"Noah Kim", email:"noah.kim@example.com", tier:"Gold" },
+ { name:"Rosa Patel", email:"rosa.patel@example.com", tier:"Silver" },
+ { name:"Iris Zhao", email:"iris.zhao@example.com", tier:"Bronze" },
+ { name:"Mateo Silva", email:"mateo.silva@example.com", tier:"Gold" }
+ ],
+ orders: [
+ { id:"ORD-778", customer:"Noah Kim", item:"Cable Kit", total:"$84.20", status:"Paid" },
+ { id:"ORD-812", customer:"Rosa Patel", item:"Desk Mat", total:"$28.00", status:"Shipped" },
+ { id:"ORD-919", customer:"Mateo Silva", item:"Aster Lamp", total:"$134.50", status:"Delayed" }
+ ],
+ catalog: [
+ { item:"Aster Lamp", sku:"AST-LAMP", vendor:"Northwind Lighting" },
+ { item:"Desk Mat", sku:"DSK-MAT", vendor:"Woven Works" },
+ { item:"Cable Kit", sku:"CBL-KIT", vendor:"Copper Lane" }
+ ],
+ inventory: [
+ { sku:"AST-LAMP", onHand:0, reorderQty:0 },
+ { sku:"DSK-MAT", onHand:42, reorderQty:0 },
+ { sku:"CBL-KIT", onHand:6, reorderQty:0 }
+ ],
+ messages: [],
+ audit: [],
+ viewsVisited: {},
+ actionCount: 0,
+ startedAt: Date.now()
+};
+let state = loadState();
+function clone(x){ return JSON.parse(JSON.stringify(x)); }
+function loadState(){ const raw = localStorage.getItem(storeKey); return raw ? JSON.parse(raw) : clone(SEED); }
+function saveState(){ localStorage.setItem(storeKey, JSON.stringify(state)); }
+function countAction(kind){ state.actionCount = (state.actionCount || 0) + 1; state.lastAction = kind; }
+function audit(entry){ state.audit.push({ ts:new Date().toISOString(), via:"ui-event", trusted:!!entry.trusted, ...entry }); saveState(); grade(false); }
+function rubric(){ return (TASKS[task] || TASKS["choredesk-l1-ticket-priority"]).rubric(state); }
+function validate(){
+ const items = rubric();
+ const total = items.reduce((a,x)=>a+x.weight,0) || 1;
+ const got = items.filter(x=>x.pass).reduce((a,x)=>a+x.weight,0);
+ const reward = Math.round((got / total) * 1000) / 1000;
+ const done = reward === 1;
+ const bad = items.filter(x=>!x.pass).map(x=>x.reason || `${x.id} failed`);
+ return { reward, done, message: done ? "all deterministic checks passed" : bad.join("; "), info: { task, seed, run, actionCount:state.actionCount||0, elapsedMs:Date.now()-(state.startedAt||Date.now()), viewsVisited:state.viewsVisited, rubric:items, state } };
+}
+function observation(){
+ return { goal:TASKS[task]?.instruction, url:location.href, activeView:view, title:document.title, reward:window.__taskReward, actionCount:state.actionCount||0, viewsVisited:state.viewsVisited, focusedElementBid:document.activeElement?.getAttribute("bid") || null };
+}
+function setVerdict(verdict, reason, validation = validate()){
+ window.__taskVerdict = verdict;
+ window.__taskReason = reason;
+ window.__taskValidation = validation;
+ window.__taskReward = validation.reward;
+ window.__taskTerminated = validation.done;
+ window.__taskTruncated = (state.actionCount || 0) >= Number(params.get("maxSteps") || 9999);
+ window.__taskInfo = validation.info;
+ localStorage.setItem(verdictKey, JSON.stringify({ id: task, run, seed, verdict, reason, validation, state, ts: Date.now() }));
+ document.getElementById("verdict").textContent = verdict;
+ document.getElementById("verdict").className = "verdict " + verdict;
+ document.getElementById("reward").textContent = `reward=${validation.reward} actions=${state.actionCount || 0}`;
+ document.getElementById("reason").textContent = reason.join("\n");
+}
+function grade(showPass=true){ const v = validate(); if (!v.done) setVerdict(showPass ? "FAIL" : "PENDING", v.info.rubric.filter(x=>!x.pass).map(x=>`${x.id}: ${x.reason}`), v); else setVerdict("PASS", [v.message], v); return v; }
+function route(v){ view = v; state.viewsVisited[v] = (state.viewsVisited[v] || 0) + 1; countAction("route:" + v); saveState(); history.replaceState(null, "", `?task=${encodeURIComponent(task)}&run=${encodeURIComponent(run)}&seed=${seed}&nonce=${nonce}&view=${encodeURIComponent(view)}`); render(); }
+function renderShell(){
+ const spec = TASKS[task] || TASKS["choredesk-l1-ticket-priority"];
+ document.getElementById("taskId").textContent = task;
+ document.getElementById("difficulty").textContent = spec.difficulty;
+ document.getElementById("instruction").textContent = spec.instruction;
+ const nav = document.querySelector(".tabs"); nav.innerHTML = "";
+ for (const v of views) { const a = document.createElement("a"); a.href="#"; a.className="tab"; a.textContent=v[0].toUpperCase()+v.slice(1); a.dataset.view=v; a.setAttribute("data-task-role", "nav-tab"); if(v===view)a.setAttribute("aria-current","page"); a.addEventListener("click", e=>{e.preventDefault(); route(v);}); nav.appendChild(a); }
+}
+function render(){
+ renderShell(); bidCounter = 0;
+ const root = document.getElementById("view"); root.innerHTML = "";
+ if (view === "tickets") root.appendChild(tableEditor("tickets", state.tickets, ["id","customer","subject","priority","status","note"], saveTicket));
+ if (view === "customers") root.appendChild(readTable(state.customers, ["name","email","tier"]));
+ if (view === "orders") root.appendChild(readTable(state.orders, ["id","customer","item","total","status"]));
+ if (view === "catalog") root.appendChild(readTable(state.catalog, ["item","sku","vendor"]));
+ if (view === "inventory") root.appendChild(tableEditor("inventory", state.inventory, ["sku","onHand","reorderQty"], saveInventory));
+ if (view === "messages") root.appendChild(messageComposer());
+ if (view === "audit") root.appendChild(readTable(state.audit.map((entry,i)=>({ n:i+1, entry: JSON.stringify(entry) })), ["n","entry"]));
+ assignBids(); grade(false);
+}
+function readTable(rows, cols){
+ const wrap = document.createElement("div"); wrap.className="card";
+ wrap.innerHTML = `<table><thead><tr>${cols.map(c=>`<th>${c}</th>`).join("")}</tr></thead><tbody></tbody></table>`;
+ const tb = wrap.querySelector("tbody");
+ for (const r of rows) { const tr = document.createElement("tr"); tr.innerHTML = cols.map(c => `<td>${escapeHtml(r[c] ?? "")}</td>`).join(""); tb.appendChild(tr); }
+ return wrap;
+}
+function tableEditor(kind, rows, cols, onSave){
+ const wrap = document.createElement("div"); wrap.className="card";
+ wrap.innerHTML = `<table><thead><tr>${cols.map(c=>`<th>${c}</th>`).join("")}<th>save</th></tr></thead><tbody></tbody></table>`;
+ const tb = wrap.querySelector("tbody");
+ rows.forEach((r) => {
+ const key = r.id || r.sku; const tr = document.createElement("tr"); tr.dataset.itemId = key; if (r.id) tr.dataset.ticketId = r.id; if (r.sku) tr.dataset.sku = r.sku;
+ for (const c of cols) {
+ const td = document.createElement("td");
+ const stableAttrs = `${r.id ? `data-ticket-id="${r.id}"` : ""} ${r.sku ? `data-sku="${r.sku}"` : ""}`;
+ if (c === "priority") td.innerHTML = `<select data-item-id="${key}" ${stableAttrs} data-col="${c}" aria-label="${key} priority"><option>Low</option><option>Normal</option><option>High</option></select>`;
+ else if (c === "status") td.innerHTML = `<select data-item-id="${key}" ${stableAttrs} data-col="${c}" aria-label="${key} status"><option>Open</option><option>In Progress</option><option>Waiting</option><option>Closed</option></select>`;
+ else if (c === "note") td.innerHTML = `<input data-item-id="${key}" ${stableAttrs} data-col="${c}" aria-label="${key} internal note">`;
+ else if (c === "reorderQty") td.innerHTML = `<input type="number" data-item-id="${key}" ${stableAttrs} data-col="${c}" aria-label="${key} reorder quantity">`;
+ else td.textContent = r[c];
+ tr.appendChild(td);
+ }
+ const save = document.createElement("td"); save.innerHTML = `<button data-save-id="${key}" data-task-role="save-button" aria-label="save ${kind} ${key}">Save</button>`; tr.appendChild(save); tb.appendChild(tr);
+ });
+ wrap.querySelectorAll("[data-item-id]").forEach(el => { const row = rows.find(r => (r.id || r.sku) === el.dataset.itemId); el.value = row[el.dataset.col]; el.addEventListener("input", () => { row[el.dataset.col] = el.value; }); });
+ wrap.querySelectorAll("[data-save-id]").forEach(btn => btn.addEventListener("click", e => onSave(rows.find(r => (r.id || r.sku) === btn.dataset.saveId), e)));
+ return wrap;
+}
+function saveTicket(t, e){ countAction("save-ticket"); saveState(); audit({ kind:"ticket-save", id:t.id, trusted:e?.isTrusted }); render(); }
+function saveInventory(i, e){ countAction("save-inventory"); i.reorderQty = Number(i.reorderQty); saveState(); audit({ kind:"inventory-save", id:i.sku, trusted:e?.isTrusted }); render(); }
+function messageComposer(){
+ const wrap = document.createElement("div"); wrap.className="card";
+ wrap.innerHTML = `<label>From <input id="msgFrom" aria-label="message from" value="Agent"></label><br><br><label>To <select id="msgTo"><option>Billing</option><option>Support</option><option>Warehouse</option></select></label><br><br><label>Subject <input id="msgSubject" aria-label="message subject"></label><br><br><label>Body <textarea id="msgBody" aria-label="message body"></textarea></label><br><button id="sendMsg" data-task-role="send-message">Send message</button><h3>Sent messages</h3><div id="sent"></div>`;
+ wrap.querySelector("#sendMsg").addEventListener("click", e => { countAction("send-message"); state.messages.push({ from:msgFrom.value, to:msgTo.value, subject:msgSubject.value, body:msgBody.value, trusted:e.isTrusted }); audit({ kind:"message-send", id:msgSubject.value, trusted:e.isTrusted }); render(); });
+ wrap.querySelector("#sent").appendChild(readTable(state.messages, ["from","to","subject","body"]));
+ return wrap;
+}
+function assignBids(){ document.querySelectorAll("button,input,select,textarea,a[href]").forEach(el => { const id = `b${(++bidCounter).toString(36)}`; el.setAttribute("bid", id); el.setAttribute("data-bid", id); const r = el.getBoundingClientRect(); el.setAttribute("browsergym_visibility_ratio", (r.width > 0 && r.height > 0 ? 1 : 0).toString()); }); }
+function escapeHtml(s){ return String(s).replace(/[&<>"']/g, c => ({"&":"&amp;","<":"&lt;",">":"&gt;","\"":"&quot;","'":"&#39;"}[c])); }
+async function runCheat(){ const fn = window.ChoreDeskCheats?.run?.[task]; if (fn) await fn(); else for (const step of (window.ChoreDeskCheats?.[task] || [])) console.log("cheat step", step); grade(true); }
+document.getElementById("reset").addEventListener("click", () => { localStorage.removeItem(storeKey); localStorage.removeItem(verdictKey); state = loadState(); render(); });
+document.getElementById("grade").addEventListener("click", () => grade(true));
+document.getElementById("cheat").addEventListener("click", () => runCheat());
+window.__task = { id:task, run, seed, goal:TASKS[task]?.instruction, difficulty:TASKS[task]?.difficulty, step:() => { const v=grade(false); return { observation:observation(), reward:v.reward, terminated:v.done, truncated:window.__taskTruncated, info:v.info }; }, reset:() => { localStorage.removeItem(storeKey); state=loadState(); render(); return { observation:observation(), info:{task,run,seed} }; }, rubric:() => rubric(), cheat:runCheat, validate, teardown:()=>undefined };
+window.__choredesk = window.__task;
+window.__choredesk.cheatHelpers = { route: async (v) => { route(v); await new Promise(r => setTimeout(r, 0)); }, set: (sel, value) => { const el = document.querySelector(sel); if (!el) throw new Error('cheat selector missing: ' + sel); el.value = value; el.dispatchEvent(new Event('input', { bubbles:true })); el.dispatchEvent(new Event('change', { bubbles:true })); } };
+state.viewsVisited[view] = (state.viewsVisited[view] || 0) + 1; saveState(); render();
+})();
+</script>
+</body>
diff --git a/test-suite/task-manifest.json b/test-suite/task-manifest.json
new file mode 100644
index 0000000..22e9590
--- /dev/null
+++ b/test-suite/task-manifest.json
@@ -0,0 +1,416 @@
+[
+ {
+ "id": "choredesk-l1-ticket-priority",
+ "site": "choredesk",
+ "startUrl": "scenarios/choredesk/index.html?task=choredesk-l1-ticket-priority&run=$RUN_ID&seed=$SEED",
+ "difficulty": "L1",
+ "category": "enterprise-crud",
+ "intent": "Set ticket T-104 priority to High and status to In Progress, then save it.",
+ "expectedActions": {
+ "min": 3,
+ "max": 6
+ },
+ "requires": [
+ "snapshot",
+ "click",
+ "select/change",
+ "form editing"
+ ],
+ "grader": {
+ "type": "inPageProgrammatic",
+ "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+ "passValue": "PASS"
+ },
+ "browserGymLike": {
+ "observation": [
+ "dom",
+ "accessibility_tree",
+ "screenshot_optional"
+ ],
+ "actionSpace": [
+ "click",
+ "type",
+ "select",
+ "evaluate_optional"
+ ],
+ "reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
+ "stepReturn": "obs,reward,terminated,truncated,info"
+ },
+ "viewport": {
+ "width": 1280,
+ "height": 720
+ },
+ "slowMoMs": 1000,
+ "timeoutMs": 5000,
+ "locale": null,
+ "timezoneId": null,
+ "humanBaseline": null,
+ "actionSubsets": [
+ "bid",
+ "nav",
+ "tab",
+ "infeas",
+ "chat"
+ ],
+ "rewardShape": "binary",
+ "taskId": "choredesk.L1.ticket-priority",
+ "seed": 101,
+ "maxSteps": 8,
+ "allowedActions": [
+ "click",
+ "fill",
+ "select_option",
+ "goto",
+ "report_infeasible",
+ "send_msg_to_user"
+ ],
+ "setupUrl": "scenarios/choredesk/index.html?task=choredesk-l1-ticket-priority&run=$RUN_ID&seed=$SEED",
+ "validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
+ "cheatScript": "scenarios/choredesk/cheats.js#choredesk-l1-ticket-priority",
+ "tags": [
+ "form-fill",
+ "ticket",
+ "select"
+ ],
+ "rubric": [
+ {
+ "id": "priorityHigh",
+ "weight": 0.33,
+ "description": "T-104 priority is High"
+ },
+ {
+ "id": "statusInProgress",
+ "weight": 0.33,
+ "description": "T-104 status is In Progress"
+ },
+ {
+ "id": "saved",
+ "weight": 0.34,
+ "description": "T-104 was saved"
+ }
+ ],
+ "goal": "Set ticket T-104 priority to High and status to In Progress, then save it.",
+ "goalObject": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Set ticket T-104 priority to High and status to In Progress, then save it."
+ }
+ ]
+ }
+ ],
+ "gradeSource": "page"
+ },
+ {
+ "id": "choredesk-l2-refund-message",
+ "site": "choredesk",
+ "startUrl": "scenarios/choredesk/index.html?task=choredesk-l2-refund-message&run=$RUN_ID&seed=$SEED",
+ "difficulty": "L2",
+ "category": "cross-page-memory",
+ "intent": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields.",
+ "expectedActions": {
+ "min": 8,
+ "max": 14
+ },
+ "requires": [
+ "multi-page navigation",
+ "table lookup",
+ "working memory",
+ "form composition"
+ ],
+ "grader": {
+ "type": "inPageProgrammatic",
+ "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+ "passValue": "PASS"
+ },
+ "browserGymLike": {
+ "observation": [
+ "dom",
+ "accessibility_tree",
+ "screenshot_optional"
+ ],
+ "actionSpace": [
+ "click",
+ "type",
+ "select",
+ "evaluate_optional"
+ ],
+ "reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
+ "stepReturn": "obs,reward,terminated,truncated,info"
+ },
+ "viewport": {
+ "width": 1280,
+ "height": 720
+ },
+ "slowMoMs": 1000,
+ "timeoutMs": 5000,
+ "locale": null,
+ "timezoneId": null,
+ "humanBaseline": null,
+ "actionSubsets": [
+ "bid",
+ "nav",
+ "tab",
+ "infeas",
+ "chat"
+ ],
+ "rewardShape": "binary",
+ "taskId": "choredesk.L2.refund-message",
+ "seed": 202,
+ "maxSteps": 16,
+ "allowedActions": [
+ "click",
+ "fill",
+ "select_option",
+ "goto",
+ "report_infeasible",
+ "send_msg_to_user"
+ ],
+ "setupUrl": "scenarios/choredesk/index.html?task=choredesk-l2-refund-message&run=$RUN_ID&seed=$SEED",
+ "validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
+ "cheatScript": "scenarios/choredesk/cheats.js#choredesk-l2-refund-message",
+ "tags": [
+ "lookup",
+ "cross-page-memory",
+ "message-compose"
+ ],
+ "rubric": [
+ {
+ "id": "messageSubject",
+ "weight": 0.34,
+ "description": "Billing message subject matches"
+ },
+ {
+ "id": "email",
+ "weight": 0.33,
+ "description": "Body includes Noah email"
+ },
+ {
+ "id": "total",
+ "weight": 0.33,
+ "description": "Body includes exact order total"
+ }
+ ],
+ "goal": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields.",
+ "goalObject": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields."
+ }
+ ]
+ }
+ ],
+ "gradeSource": "page"
+ },
+ {
+ "id": "choredesk-l3-restock-ticket",
+ "site": "choredesk",
+ "startUrl": "scenarios/choredesk/index.html?task=choredesk-l3-restock-ticket&run=$RUN_ID&seed=$SEED",
+ "difficulty": "L3",
+ "category": "cross-page-state-update",
+ "intent": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket.",
+ "expectedActions": {
+ "min": 12,
+ "max": 20
+ },
+ "requires": [
+ "multi-page navigation",
+ "entity resolution",
+ "state mutation",
+ "exact text note",
+ "audit-save behavior"
+ ],
+ "grader": {
+ "type": "inPageProgrammatic",
+ "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+ "passValue": "PASS"
+ },
+ "browserGymLike": {
+ "observation": [
+ "dom",
+ "accessibility_tree",
+ "screenshot_optional"
+ ],
+ "actionSpace": [
+ "click",
+ "type",
+ "select",
+ "evaluate_optional"
+ ],
+ "reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
+ "stepReturn": "obs,reward,terminated,truncated,info"
+ },
+ "viewport": {
+ "width": 1280,
+ "height": 720
+ },
+ "slowMoMs": 1000,
+ "timeoutMs": 5000,
+ "locale": null,
+ "timezoneId": null,
+ "humanBaseline": null,
+ "actionSubsets": [
+ "bid",
+ "nav",
+ "tab",
+ "infeas",
+ "chat"
+ ],
+ "rewardShape": "binary",
+ "taskId": "choredesk.L3.restock-ticket",
+ "seed": 303,
+ "maxSteps": 24,
+ "allowedActions": [
+ "click",
+ "fill",
+ "select_option",
+ "goto",
+ "report_infeasible",
+ "send_msg_to_user"
+ ],
+ "setupUrl": "scenarios/choredesk/index.html?task=choredesk-l3-restock-ticket&run=$RUN_ID&seed=$SEED",
+ "validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
+ "cheatScript": "scenarios/choredesk/cheats.js#choredesk-l3-restock-ticket",
+ "tags": [
+ "entity-resolution",
+ "cross-page-state-update",
+ "exact-note"
+ ],
+ "rubric": [
+ {
+ "id": "inventory",
+ "weight": 0.4,
+ "description": "AST-LAMP reorder quantity is 12 and saved"
+ },
+ {
+ "id": "ticketNote",
+ "weight": 0.4,
+ "description": "T-205 note contains exact phrase"
+ },
+ {
+ "id": "audit",
+ "weight": 0.2,
+ "description": "Both save audit entries exist"
+ }
+ ],
+ "goal": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket.",
+ "goalObject": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket."
+ }
+ ]
+ }
+ ],
+ "gradeSource": "page"
+ },
+ {
+ "id": "mini-shop-red-second-cheapest",
+ "site": "mini-shop",
+ "startUrl": "fixtures/sites/mini-shop/index.html?task=mini-shop-red-second-cheapest&run=$RUN_ID&seed=$SEED",
+ "difficulty": "L2",
+ "category": "task-completion",
+ "intent": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
+ "expectedActions": {
+ "min": 5,
+ "max": 9
+ },
+ "requires": [
+ "filter/sort reasoning",
+ "cart state",
+ "checkout flow"
+ ],
+ "grader": {
+ "type": "inPageProgrammatic",
+ "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+ "passValue": "PASS"
+ },
+ "browserGymLike": {
+ "observation": [
+ "dom",
+ "accessibility_tree",
+ "screenshot_optional"
+ ],
+ "actionSpace": [
+ "click",
+ "read_table",
+ "evaluate_optional"
+ ],
+ "reset": "navigate startUrl with fresh $RUN_ID",
+ "stepReturn": "obs,reward,terminated,truncated,info"
+ },
+ "viewport": {
+ "width": 1280,
+ "height": 720
+ },
+ "slowMoMs": 1000,
+ "timeoutMs": 5000,
+ "locale": null,
+ "timezoneId": null,
+ "humanBaseline": null,
+ "actionSubsets": [
+ "bid",
+ "nav",
+ "tab",
+ "infeas",
+ "chat"
+ ],
+ "rewardShape": "binary",
+ "taskId": "mini-shop.L2.red-second-cheapest",
+ "seed": 404,
+ "maxSteps": 12,
+ "allowedActions": [
+ "click",
+ "goto",
+ "report_infeasible",
+ "send_msg_to_user"
+ ],
+ "setupUrl": "fixtures/sites/mini-shop/index.html?task=mini-shop-red-second-cheapest&run=$RUN_ID&seed=$SEED",
+ "validateScript": "fixtures/sites/mini-shop/grader.js#MiniShopGrader.validate",
+ "cheatScript": "fixtures/sites/mini-shop/cheats.js#mini-shop-red-second-cheapest",
+ "tags": [
+ "shopping",
+ "sort",
+ "cart",
+ "checkout"
+ ],
+ "rubric": [
+ {
+ "id": "addedTarget",
+ "weight": 0.4,
+ "description": "Cart includes second-cheapest red product"
+ },
+ {
+ "id": "noWrongItems",
+ "weight": 0.3,
+ "description": "Cart has no wrong products"
+ },
+ {
+ "id": "checkedOut",
+ "weight": 0.3,
+ "description": "Checkout produced order id"
+ }
+ ],
+ "goal": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
+ "goalObject": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible."
+ }
+ ]
+ }
+ ],
+ "gradeSource": "both"
+ }
+]
diff --git a/test-suite/task-manifest.schema.json b/test-suite/task-manifest.schema.json
new file mode 100644
index 0000000..b8031fc
--- /dev/null
+++ b/test-suite/task-manifest.schema.json
@@ -0,0 +1,59 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "title": "pi-chrome long-horizon task manifest",
+ "description": "BrowserGym-inspired task manifest: setup URL, deterministic seed, goal object, action subsets, reward contract, optional cheat recipe, and in-page validate hook.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": ["id", "taskId", "site", "startUrl", "setupUrl", "difficulty", "category", "goal", "goalObject", "grader", "actionSubsets", "maxSteps"],
+ "additionalProperties": true,
+ "properties": {
+ "id": { "type": "string" },
+ "taskId": { "type": "string" },
+ "site": { "type": "string" },
+ "seed": { "type": "integer" },
+ "startUrl": { "type": "string", "description": "Relative URL. $RUN_ID and $SEED should be replaced by runner for isolated deterministic state." },
+ "setupUrl": { "type": "string" },
+ "validateScript": { "type": "string" },
+ "cheatScript": { "type": "string" },
+ "viewport": { "type": "object", "properties": { "width": { "type": "integer" }, "height": { "type": "integer" } } },
+ "slowMoMs": { "type": "integer" },
+ "timeoutMs": { "type": "integer" },
+ "locale": { "type": ["string", "null"] },
+ "timezoneId": { "type": ["string", "null"] },
+ "difficulty": { "enum": ["L1", "L2", "L3"] },
+ "category": { "enum": ["enterprise-crud", "cross-page-memory", "cross-page-state-update", "task-completion"] },
+ "gradeSource": { "enum": ["page", "harness", "both"] },
+ "tags": { "type": "array", "items": { "type": "string" } },
+ "intent": { "type": "string" },
+ "goal": { "type": "string" },
+ "goalObject": { "type": "array", "items": { "type": "object" } },
+ "expectedActions": { "type": "object", "required": ["min", "max"], "properties": { "min": { "type": "integer" }, "max": { "type": "integer" } } },
+ "maxSteps": { "type": "integer" },
+ "humanBaseline": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
+ "actionSubsets": { "type": "array", "items": { "enum": ["chat", "infeas", "bid", "coord", "nav", "tab"] } },
+ "allowedActions": { "type": "array", "items": { "type": "string" } },
+ "requires": { "type": "array", "items": { "type": "string" } },
+ "rewardShape": { "enum": ["binary", "graded", "shaped"] },
+ "rubric": { "type": "array", "items": { "type": "object", "required": ["id", "weight", "description"], "properties": { "id": { "type": "string" }, "weight": { "type": "number" }, "description": { "type": "string" } } } },
+ "grader": {
+ "type": "object",
+ "required": ["type", "expression", "passValue"],
+ "properties": {
+ "type": { "enum": ["inPageProgrammatic"] },
+ "expression": { "type": "string", "description": "Should evaluate to BrowserGym-like validate object or legacy {v,r}." },
+ "passValue": { "type": "string" }
+ }
+ },
+ "browserGymLike": {
+ "type": "object",
+ "properties": {
+ "observation": { "type": "array", "items": { "type": "string" } },
+ "actionSpace": { "type": "array", "items": { "type": "string" } },
+ "reset": { "type": "string" },
+ "stepReturn": { "const": "obs,reward,terminated,truncated,info" }
+ }
+ }
+ }
+ }
+}