From 4c54b05dc15b81e449dc8ac53da8bae536fa2798 Mon Sep 17 00:00:00 2001 From: Tianren Dong Date: Wed, 13 May 2026 07:29:41 -0400 Subject: pi-chrome: snapshot pre-migration state --- test-suite/README.md | 60 ++- test-suite/_lib.js | 35 +- test-suite/browsergym-action-space.json | 44 ++ test-suite/challenges/34-dialog-handling.html | 28 + test-suite/challenges/35-target-blank-popup.html | 35 ++ test-suite/challenges/36-modal-focus-trap.html | 49 ++ .../challenges/37-autocomplete-combobox.html | 42 ++ test-suite/challenges/38-spa-route-change.html | 40 ++ test-suite/fixtures/sites/mini-shop/cheats.js | 12 + test-suite/fixtures/sites/mini-shop/grader.js | 29 ++ test-suite/fixtures/sites/mini-shop/index.html | 55 ++ test-suite/fixtures/sites/mini-shop/tasks.json | 9 + test-suite/index.html | 28 + test-suite/manifest.json | 569 +++++++++++++++++++-- test-suite/manifest.schema.json | 53 +- test-suite/notes/browsergym-compat.md | 70 +++ test-suite/notes/profiles.md | 22 + test-suite/notes/runner-spec.md | 26 + test-suite/notes/scoring.md | 36 ++ test-suite/scenarios/choredesk/cheats.js | 49 ++ test-suite/scenarios/choredesk/index.html | 239 +++++++++ test-suite/task-manifest.json | 416 +++++++++++++++ test-suite/task-manifest.schema.json | 59 +++ 23 files changed, 1918 insertions(+), 87 deletions(-) create mode 100644 test-suite/browsergym-action-space.json create mode 100644 test-suite/challenges/34-dialog-handling.html create mode 100644 test-suite/challenges/35-target-blank-popup.html create mode 100644 test-suite/challenges/36-modal-focus-trap.html create mode 100644 test-suite/challenges/37-autocomplete-combobox.html create mode 100644 test-suite/challenges/38-spa-route-change.html create mode 100644 test-suite/fixtures/sites/mini-shop/cheats.js create mode 100644 test-suite/fixtures/sites/mini-shop/grader.js create mode 100644 test-suite/fixtures/sites/mini-shop/index.html create mode 100644 test-suite/fixtures/sites/mini-shop/tasks.json create mode 100644 test-suite/notes/browsergym-compat.md create mode 100644 test-suite/notes/profiles.md create mode 100644 test-suite/notes/runner-spec.md create mode 100644 test-suite/notes/scoring.md create mode 100644 test-suite/scenarios/choredesk/cheats.js create mode 100644 test-suite/scenarios/choredesk/index.html create mode 100644 test-suite/task-manifest.json create mode 100644 test-suite/task-manifest.schema.json (limited to 'test-suite') diff --git a/test-suite/README.md b/test-suite/README.md index bb8becb..a96e0d0 100644 --- a/test-suite/README.md +++ b/test-suite/README.md @@ -1,13 +1,14 @@ # pi-chrome browser-control benchmark -Static benchmark pages for evaluating tools that let agents control Chrome. The -suite covers two related questions: +Static benchmark pages for evaluating tools that let agents control Chrome. The suite has two layers: -1. **Can the agent complete normal browser work?** Forms, scroll containers, - contenteditable, files, frames, Shadow DOM, network/console inspection. -2. **Does the interaction look like a human/browser action when sites care?** - `isTrusted`, user activation, pointer paths, key cadence, native controls, - drag/drop, touch, paste, and scroll momentum. +1. **Unit challenges** (`manifest.json`) — MiniWoB-style capability probes for + forms, scroll containers, contenteditable, files, frames, Shadow DOM, + network/console inspection, `isTrusted`, user activation, pointer paths, key + cadence, native controls, drag/drop, touch, paste, and scroll momentum. +2. **Long-horizon hermetic tasks** (`task-manifest.json`) — WebArena / + BrowserGym-inspired multi-step tasks with fresh run IDs and deterministic + programmatic graders. ## Run @@ -24,11 +25,17 @@ Each challenge page exposes: - `window.__reason` — array of reasons - `window.__events` — raw event log for forensics -`manifest.json` is the source of truth for benchmark metadata: category, goal, -expected result per mode, prerequisites, flake risk, manual baseline status, and -canonical tool recipe. `manifest.schema.json` documents the manifest shape. -Recipes express tool intent; runners may need to adapt descriptive selectors -(e.g. shadow/iframe notation) and expand path placeholders like `$PWD`. +`manifest.json` is the source of truth for unit-challenge metadata: category, +goal, expected result per mode, prerequisites, flake risk, manual baseline +status, and canonical tool recipe. `manifest.schema.json` documents the manifest +shape. Recipes express tool intent; runners may need to adapt descriptive +selectors (e.g. shadow/iframe notation) and expand path placeholders like `$PWD`. + +`task-manifest.json` is the source of truth for long-horizon tasks: BrowserGym-style +`taskId`, seed, viewport, goal object, difficulty tier, max steps, declared +action subsets, reset/setup URL, validate hook, optional cheat recipe, and +programmatic grader expression. `task-manifest.schema.json` documents this shape. +`browsergym-action-space.json` records BrowserGym-compatible action subsets. ## Modes / expected outcomes @@ -49,7 +56,7 @@ Expected values in `manifest.json`: Manual baselines are tracked separately with `manualBaseline`. `unverified` means the manual expectation is a target, not a recorded contract. -## Recommended agent flow +## Recommended unit-challenge agent flow 1. Navigate to dashboard: `http://127.0.0.1:8765/`. @@ -70,6 +77,29 @@ means the manual expectation is a target, not a recorded contract. 4. Return to dashboard and compare actual verdicts with expected values. 5. Copy JSON report from dashboard for PRs or regression notes. +## Recommended long-horizon task flow + +1. Load `task-manifest.json`. +2. Replace `$RUN_ID` in `startUrl` with a fresh value. +3. Navigate to the start URL and read the visible task instruction. +4. Solve using normal browser tools only; avoid direct state mutation unless the + benchmark mode explicitly allows evaluate-based actions. +5. Click **Grade now** or evaluate the task grader expression: + ```js + JSON.stringify({ v: window.__taskVerdict, r: window.__taskReason }) + ``` +6. Record action count, observations used, tools used, verdict, and reason. + +## Design principles copied from browser-agent benchmarks + +- Prefer hermetic sites and deterministic graders over live sites and LLM judges. +- Report action API and observation format; these strongly affect scores. +- Use difficulty tiers: L1 atomic, L2 compositional, L3 cross-page/context-rich. +- Include tedious cross-page memory and exact-value transfer tasks; short unit + probes hide these failures. +- Keep synthetic-event-gated tests because extension bridges face failures that + CDP/Playwright-style benchmarks usually do not measure. + ## Challenge categories - `trusted-input` — browser-trusted click/key events. @@ -136,3 +166,7 @@ The dashboard renders this from `manifest.json`. In brief: software WebGL in VMs, remote desktops, or policy-constrained environments. - `notes/bypass-ideas.md` is historical guidance for older synthetic-only versions. Prefer `manifest.json` for current expected outcomes. +- `notes/browsergym-compat.md` defines the reset/step/validate/observation/BID + contract for external BrowserGym-style agents. +- `notes/runner-spec.md`, `notes/scoring.md`, and `notes/profiles.md` define + runner output, scoring, retry policy, and environment metadata. diff --git a/test-suite/_lib.js b/test-suite/_lib.js index 95d6393..f3845e5 100644 --- a/test-suite/_lib.js +++ b/test-suite/_lib.js @@ -8,6 +8,8 @@ verdict: "PENDING", reason: [], events, + details: [], + thresholds: {}, }; function render() { @@ -31,8 +33,9 @@ } const Challenge = { - init({ id, instructions }) { + init({ id, instructions, thresholds = {} }) { state.id = id; + state.thresholds = parseThresholds(thresholds); document.title = `[${state.verdict}] ${id}`; const root = document.body; const bar = document.createElement("div"); @@ -82,6 +85,20 @@ document.title = `[WARN] ${state.id}`; persist(); render(); }, + partial({ name, pass, reason, data }) { + state.details.push({ name, pass: !!pass, reason: reason || "", data }); + log("partial", { name, pass: !!pass, reason, data }); + persist(); render(); + return !!pass; + }, + finishPartials() { + const failed = state.details.filter(d => !d.pass); + if (failed.length) Challenge.fail(...failed.map(d => `${d.name}: ${d.reason || "failed"}`)); + else Challenge.pass(...state.details.map(d => `${d.name}: ok`)); + }, + getThreshold(name, fallback) { + return Object.prototype.hasOwnProperty.call(state.thresholds, name) ? state.thresholds[name] : fallback; + }, log, state, }; @@ -90,10 +107,24 @@ try { localStorage.setItem( "pi-chrome-suite:" + state.id, - JSON.stringify({ id: state.id, verdict: state.verdict, reason: state.reason, events: state.events.slice(-50), ts: Date.now() }) + JSON.stringify({ id: state.id, verdict: state.verdict, reason: state.reason, details: state.details, thresholds: state.thresholds, events: state.events.slice(-50), ts: Date.now() }) ); } catch {} } + function parseThresholds(defaults) { + const out = { ...defaults }; + try { + const qs = new URLSearchParams(location.search); + for (const [k, v] of qs) { + if (!k.startsWith("threshold.")) continue; + const key = k.slice("threshold.".length); + const num = Number(v); + out[key] = Number.isFinite(num) ? num : v; + } + } catch {} + return out; + } + window.Challenge = Challenge; })(); diff --git a/test-suite/browsergym-action-space.json b/test-suite/browsergym-action-space.json new file mode 100644 index 0000000..5d7c873 --- /dev/null +++ b/test-suite/browsergym-action-space.json @@ -0,0 +1,44 @@ +{ + "subsets": { + "chat": ["send_msg_to_user"], + "infeas": ["report_infeasible"], + "bid": ["click", "dblclick", "hover", "fill", "clear", "press", "focus", "select_option", "scroll", "drag_and_drop", "upload_file"], + "coord": ["mouse_move", "mouse_up", "mouse_down", "mouse_click", "mouse_dblclick", "mouse_drag_and_drop", "mouse_upload_file", "scroll_at", "keyboard_down", "keyboard_up", "keyboard_press", "keyboard_type", "keyboard_insert_text"], + "nav": ["go_back", "go_forward", "goto"], + "tab": ["tab_close", "tab_focus", "new_tab"] + }, + "signatures": { + "fill": "fill(bid, value, enable_autocomplete_menu=false)", + "click": "click(bid, button='left', modifiers=[])", + "dblclick": "dblclick(bid, button='left', modifiers=[])", + "hover": "hover(bid)", + "press": "press(bid, key_comb)", + "focus": "focus(bid)", + "clear": "clear(bid)", + "select_option": "select_option(bid, options)", + "scroll": "scroll(delta_x, delta_y)", + "drag_and_drop": "drag_and_drop(from_bid, to_bid)", + "upload_file": "upload_file(bid, file)", + "mouse_move": "mouse_move(x, y)", + "mouse_click": "mouse_click(x, y, button='left')", + "mouse_dblclick": "mouse_dblclick(x, y, button='left')", + "mouse_up": "mouse_up(x, y, button='left')", + "mouse_down": "mouse_down(x, y, button='left')", + "mouse_drag_and_drop": "mouse_drag_and_drop(from_x, from_y, to_x, to_y)", + "scroll_at": "scroll_at(x, y, dx, dy)", + "keyboard_press": "keyboard_press(key)", + "keyboard_down": "keyboard_down(key)", + "keyboard_up": "keyboard_up(key)", + "keyboard_type": "keyboard_type(text)", + "keyboard_insert_text": "keyboard_insert_text(text)", + "goto": "goto(url)", + "go_back": "go_back()", + "go_forward": "go_forward()", + "tab_close": "tab_close()", + "tab_focus": "tab_focus(index)", + "new_tab": "new_tab()", + "send_msg_to_user": "send_msg_to_user(text)", + "report_infeasible": "report_infeasible(reason)", + "noop": "noop(wait_ms=1000)" + } +} diff --git a/test-suite/challenges/34-dialog-handling.html b/test-suite/challenges/34-dialog-handling.html new file mode 100644 index 0000000..25845a7 --- /dev/null +++ b/test-suite/challenges/34-dialog-handling.html @@ -0,0 +1,28 @@ + + +34 dialog handling + + + +
+

Goal: handle browser dialogs. Click Prompt, enter pi-chrome, accept it; then click Confirm and accept it. Browser automation must expose/dismiss native dialogs.

+ + + + 

+

+
+
diff --git a/test-suite/challenges/35-target-blank-popup.html b/test-suite/challenges/35-target-blank-popup.html
new file mode 100644
index 0000000..0813625
--- /dev/null
+++ b/test-suite/challenges/35-target-blank-popup.html
@@ -0,0 +1,35 @@
+
+
+35 target blank popup
+
+
+
+

+  
Goal: click target=_blank link, detect new tab, switch to it, and verify child page reports PASS.

+  Open child tab
+  

+

+
+
diff --git a/test-suite/challenges/36-modal-focus-trap.html b/test-suite/challenges/36-modal-focus-trap.html
new file mode 100644
index 0000000..811207c
--- /dev/null
+++ b/test-suite/challenges/36-modal-focus-trap.html
@@ -0,0 +1,49 @@
+
+
+36 modal focus trap
+
+
+
+

+  
Goal: open modal, use Tab to cycle focus inside modal only, then Escape closes it and focus returns to opener.

+  
+  
+

+
+
diff --git a/test-suite/challenges/37-autocomplete-combobox.html b/test-suite/challenges/37-autocomplete-combobox.html
new file mode 100644
index 0000000..91f35c2
--- /dev/null
+++ b/test-suite/challenges/37-autocomplete-combobox.html
@@ -0,0 +1,42 @@
+
+
+37 autocomplete combobox
+
+
+
+

+  
Goal: type as, use ArrowDown then Enter to choose Aster Lamp from ARIA combobox.

+  
+  
+

+
+
diff --git a/test-suite/challenges/38-spa-route-change.html b/test-suite/challenges/38-spa-route-change.html
new file mode 100644
index 0000000..533c38d
--- /dev/null
+++ b/test-suite/challenges/38-spa-route-change.html
@@ -0,0 +1,40 @@
+
+
+38 SPA route change
+
+
+
+

+  
Goal: navigate within SPA using links/buttons. URL must change via history.pushState without full reload. Then go Back to Settings, forward to Billing, and Save.

+  
+  

+

+
+
diff --git a/test-suite/fixtures/sites/mini-shop/cheats.js b/test-suite/fixtures/sites/mini-shop/cheats.js
new file mode 100644
index 0000000..7407ee4
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/cheats.js
@@ -0,0 +1,12 @@
+window.MiniShopCheats = {
+  "mini-shop-red-second-cheapest": [
+    { tool: "chrome_click", params: { selector: "button[aria-label=\"add Ruby Notebook\"]" } },
+    { tool: "chrome_click", params: { selector: "#checkout" } }
+  ],
+  run: {
+    "mini-shop-red-second-cheapest": async () => {
+      document.querySelector('button[aria-label="add Ruby Notebook"]').click();
+      document.querySelector('#checkout').click();
+    }
+  }
+};
diff --git a/test-suite/fixtures/sites/mini-shop/grader.js b/test-suite/fixtures/sites/mini-shop/grader.js
new file mode 100644
index 0000000..74605f4
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/grader.js
@@ -0,0 +1,29 @@
+window.MiniShopGrader = {
+  validate(task, state, products, run, seed = 0) {
+    let bad = [];
+    const rubric = {};
+    if (task === 'mini-shop-red-second-cheapest') {
+      const reds = products.filter(p => p.color === 'red').sort((a,b)=>a.price-b.price);
+      const target = reds[1];
+      rubric.addedTarget = state.cart.includes(target.id);
+      rubric.noWrongItems = state.cart.every(id => id === target.id);
+      rubric.checkedOut = /^MS-\d{4}$/.test(state.orderId || '');
+      if (!rubric.addedTarget) bad.push(`cart missing second-cheapest red item ${target.name}`);
+      const wrong = state.cart.filter(id => id !== target.id);
+      if (wrong.length) bad.push(`cart contains wrong product(s): ${wrong.join(',')}`);
+      if (!rubric.checkedOut) bad.push('missing checkout order id');
+    } else bad.push(`unknown task ${task}`);
+    const passed = bad.length === 0;
+    return { reward: passed ? 1 : 0, done: passed, message: passed ? 'all deterministic mini-shop checks passed' : bad.join('; '), info: { task, run, seed, rubric, bad, state } };
+  },
+  grade(task, state, products, run, seed = 0) {
+    const validation = this.validate(task, state, products, run, seed);
+    const verdict = validation.done ? 'PASS' : 'PENDING';
+    window.__taskId = task;
+    window.__taskVerdict = verdict;
+    window.__taskReason = validation.done ? [validation.message] : validation.info.bad;
+    window.__taskValidation = validation;
+    try { localStorage.setItem(`pi-chrome-task:${task}:${run}`, JSON.stringify({ id: task, run, seed, verdict, reason: window.__taskReason, validation, state, ts: Date.now() })); } catch {}
+    return validation;
+  }
+};
diff --git a/test-suite/fixtures/sites/mini-shop/index.html b/test-suite/fixtures/sites/mini-shop/index.html
new file mode 100644
index 0000000..0fd63ff
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/index.html
@@ -0,0 +1,55 @@
+
+
+MiniShop hermetic site
+
+
+
+

+
MiniShop

+
Cart 0 items   

+

+

+

+
+
+
+
diff --git a/test-suite/fixtures/sites/mini-shop/tasks.json b/test-suite/fixtures/sites/mini-shop/tasks.json
new file mode 100644
index 0000000..6812baf
--- /dev/null
+++ b/test-suite/fixtures/sites/mini-shop/tasks.json
@@ -0,0 +1,9 @@
+[
+  {
+    "id": "mini-shop-red-second-cheapest",
+    "difficulty": "L2",
+    "intent": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
+    "gradeSource": "harness",
+    "grader": "MiniShopGrader.grade(task, state, products, run)"
+  }
+]
diff --git a/test-suite/index.html b/test-suite/index.html
index a4a3c0c..00ef204 100644
--- a/test-suite/index.html
+++ b/test-suite/index.html
@@ -35,6 +35,13 @@
   
 
 
+
Long-horizon hermetic tasks

+
These WebArena/BrowserGym-inspired tasks use deterministic in-page graders and fresh $RUN_ID state. They test multi-step browser work beyond event fidelity.

+
+  
+  
+
iddifficultycategoryintentrequiresopen

+
 
Agent run loop

 
python3 -m http.server 8765
 # For each manifest entry:
@@ -46,7 +53,9 @@
 
 
+
+
diff --git a/test-suite/task-manifest.json b/test-suite/task-manifest.json
new file mode 100644
index 0000000..22e9590
--- /dev/null
+++ b/test-suite/task-manifest.json
@@ -0,0 +1,416 @@
+[
+  {
+    "id": "choredesk-l1-ticket-priority",
+    "site": "choredesk",
+    "startUrl": "scenarios/choredesk/index.html?task=choredesk-l1-ticket-priority&run=$RUN_ID&seed=$SEED",
+    "difficulty": "L1",
+    "category": "enterprise-crud",
+    "intent": "Set ticket T-104 priority to High and status to In Progress, then save it.",
+    "expectedActions": {
+      "min": 3,
+      "max": 6
+    },
+    "requires": [
+      "snapshot",
+      "click",
+      "select/change",
+      "form editing"
+    ],
+    "grader": {
+      "type": "inPageProgrammatic",
+      "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+      "passValue": "PASS"
+    },
+    "browserGymLike": {
+      "observation": [
+        "dom",
+        "accessibility_tree",
+        "screenshot_optional"
+      ],
+      "actionSpace": [
+        "click",
+        "type",
+        "select",
+        "evaluate_optional"
+      ],
+      "reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
+      "stepReturn": "obs,reward,terminated,truncated,info"
+    },
+    "viewport": {
+      "width": 1280,
+      "height": 720
+    },
+    "slowMoMs": 1000,
+    "timeoutMs": 5000,
+    "locale": null,
+    "timezoneId": null,
+    "humanBaseline": null,
+    "actionSubsets": [
+      "bid",
+      "nav",
+      "tab",
+      "infeas",
+      "chat"
+    ],
+    "rewardShape": "binary",
+    "taskId": "choredesk.L1.ticket-priority",
+    "seed": 101,
+    "maxSteps": 8,
+    "allowedActions": [
+      "click",
+      "fill",
+      "select_option",
+      "goto",
+      "report_infeasible",
+      "send_msg_to_user"
+    ],
+    "setupUrl": "scenarios/choredesk/index.html?task=choredesk-l1-ticket-priority&run=$RUN_ID&seed=$SEED",
+    "validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
+    "cheatScript": "scenarios/choredesk/cheats.js#choredesk-l1-ticket-priority",
+    "tags": [
+      "form-fill",
+      "ticket",
+      "select"
+    ],
+    "rubric": [
+      {
+        "id": "priorityHigh",
+        "weight": 0.33,
+        "description": "T-104 priority is High"
+      },
+      {
+        "id": "statusInProgress",
+        "weight": 0.33,
+        "description": "T-104 status is In Progress"
+      },
+      {
+        "id": "saved",
+        "weight": 0.34,
+        "description": "T-104 was saved"
+      }
+    ],
+    "goal": "Set ticket T-104 priority to High and status to In Progress, then save it.",
+    "goalObject": [
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": "Set ticket T-104 priority to High and status to In Progress, then save it."
+          }
+        ]
+      }
+    ],
+    "gradeSource": "page"
+  },
+  {
+    "id": "choredesk-l2-refund-message",
+    "site": "choredesk",
+    "startUrl": "scenarios/choredesk/index.html?task=choredesk-l2-refund-message&run=$RUN_ID&seed=$SEED",
+    "difficulty": "L2",
+    "category": "cross-page-memory",
+    "intent": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields.",
+    "expectedActions": {
+      "min": 8,
+      "max": 14
+    },
+    "requires": [
+      "multi-page navigation",
+      "table lookup",
+      "working memory",
+      "form composition"
+    ],
+    "grader": {
+      "type": "inPageProgrammatic",
+      "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+      "passValue": "PASS"
+    },
+    "browserGymLike": {
+      "observation": [
+        "dom",
+        "accessibility_tree",
+        "screenshot_optional"
+      ],
+      "actionSpace": [
+        "click",
+        "type",
+        "select",
+        "evaluate_optional"
+      ],
+      "reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
+      "stepReturn": "obs,reward,terminated,truncated,info"
+    },
+    "viewport": {
+      "width": 1280,
+      "height": 720
+    },
+    "slowMoMs": 1000,
+    "timeoutMs": 5000,
+    "locale": null,
+    "timezoneId": null,
+    "humanBaseline": null,
+    "actionSubsets": [
+      "bid",
+      "nav",
+      "tab",
+      "infeas",
+      "chat"
+    ],
+    "rewardShape": "binary",
+    "taskId": "choredesk.L2.refund-message",
+    "seed": 202,
+    "maxSteps": 16,
+    "allowedActions": [
+      "click",
+      "fill",
+      "select_option",
+      "goto",
+      "report_infeasible",
+      "send_msg_to_user"
+    ],
+    "setupUrl": "scenarios/choredesk/index.html?task=choredesk-l2-refund-message&run=$RUN_ID&seed=$SEED",
+    "validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
+    "cheatScript": "scenarios/choredesk/cheats.js#choredesk-l2-refund-message",
+    "tags": [
+      "lookup",
+      "cross-page-memory",
+      "message-compose"
+    ],
+    "rubric": [
+      {
+        "id": "messageSubject",
+        "weight": 0.34,
+        "description": "Billing message subject matches"
+      },
+      {
+        "id": "email",
+        "weight": 0.33,
+        "description": "Body includes Noah email"
+      },
+      {
+        "id": "total",
+        "weight": 0.33,
+        "description": "Body includes exact order total"
+      }
+    ],
+    "goal": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields.",
+    "goalObject": [
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": "Resolve a customer billing task by looking up customer email and order total on separate pages, then composing a message with exact required fields."
+          }
+        ]
+      }
+    ],
+    "gradeSource": "page"
+  },
+  {
+    "id": "choredesk-l3-restock-ticket",
+    "site": "choredesk",
+    "startUrl": "scenarios/choredesk/index.html?task=choredesk-l3-restock-ticket&run=$RUN_ID&seed=$SEED",
+    "difficulty": "L3",
+    "category": "cross-page-state-update",
+    "intent": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket.",
+    "expectedActions": {
+      "min": 12,
+      "max": 20
+    },
+    "requires": [
+      "multi-page navigation",
+      "entity resolution",
+      "state mutation",
+      "exact text note",
+      "audit-save behavior"
+    ],
+    "grader": {
+      "type": "inPageProgrammatic",
+      "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+      "passValue": "PASS"
+    },
+    "browserGymLike": {
+      "observation": [
+        "dom",
+        "accessibility_tree",
+        "screenshot_optional"
+      ],
+      "actionSpace": [
+        "click",
+        "type",
+        "select",
+        "evaluate_optional"
+      ],
+      "reset": "navigate startUrl with fresh $RUN_ID or click Reset run",
+      "stepReturn": "obs,reward,terminated,truncated,info"
+    },
+    "viewport": {
+      "width": 1280,
+      "height": 720
+    },
+    "slowMoMs": 1000,
+    "timeoutMs": 5000,
+    "locale": null,
+    "timezoneId": null,
+    "humanBaseline": null,
+    "actionSubsets": [
+      "bid",
+      "nav",
+      "tab",
+      "infeas",
+      "chat"
+    ],
+    "rewardShape": "binary",
+    "taskId": "choredesk.L3.restock-ticket",
+    "seed": 303,
+    "maxSteps": 24,
+    "allowedActions": [
+      "click",
+      "fill",
+      "select_option",
+      "goto",
+      "report_infeasible",
+      "send_msg_to_user"
+    ],
+    "setupUrl": "scenarios/choredesk/index.html?task=choredesk-l3-restock-ticket&run=$RUN_ID&seed=$SEED",
+    "validateScript": "scenarios/choredesk/index.html#window.__choredesk.validate()",
+    "cheatScript": "scenarios/choredesk/cheats.js#choredesk-l3-restock-ticket",
+    "tags": [
+      "entity-resolution",
+      "cross-page-state-update",
+      "exact-note"
+    ],
+    "rubric": [
+      {
+        "id": "inventory",
+        "weight": 0.4,
+        "description": "AST-LAMP reorder quantity is 12 and saved"
+      },
+      {
+        "id": "ticketNote",
+        "weight": 0.4,
+        "description": "T-205 note contains exact phrase"
+      },
+      {
+        "id": "audit",
+        "weight": 0.2,
+        "description": "Both save audit entries exist"
+      }
+    ],
+    "goal": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket.",
+    "goalObject": [
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": "Use ticket text to identify an item, map item to SKU in Catalog, update Inventory reorder quantity, and add exact internal note back on the ticket."
+          }
+        ]
+      }
+    ],
+    "gradeSource": "page"
+  },
+  {
+    "id": "mini-shop-red-second-cheapest",
+    "site": "mini-shop",
+    "startUrl": "fixtures/sites/mini-shop/index.html?task=mini-shop-red-second-cheapest&run=$RUN_ID&seed=$SEED",
+    "difficulty": "L2",
+    "category": "task-completion",
+    "intent": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
+    "expectedActions": {
+      "min": 5,
+      "max": 9
+    },
+    "requires": [
+      "filter/sort reasoning",
+      "cart state",
+      "checkout flow"
+    ],
+    "grader": {
+      "type": "inPageProgrammatic",
+      "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+      "passValue": "PASS"
+    },
+    "browserGymLike": {
+      "observation": [
+        "dom",
+        "accessibility_tree",
+        "screenshot_optional"
+      ],
+      "actionSpace": [
+        "click",
+        "read_table",
+        "evaluate_optional"
+      ],
+      "reset": "navigate startUrl with fresh $RUN_ID",
+      "stepReturn": "obs,reward,terminated,truncated,info"
+    },
+    "viewport": {
+      "width": 1280,
+      "height": 720
+    },
+    "slowMoMs": 1000,
+    "timeoutMs": 5000,
+    "locale": null,
+    "timezoneId": null,
+    "humanBaseline": null,
+    "actionSubsets": [
+      "bid",
+      "nav",
+      "tab",
+      "infeas",
+      "chat"
+    ],
+    "rewardShape": "binary",
+    "taskId": "mini-shop.L2.red-second-cheapest",
+    "seed": 404,
+    "maxSteps": 12,
+    "allowedActions": [
+      "click",
+      "goto",
+      "report_infeasible",
+      "send_msg_to_user"
+    ],
+    "setupUrl": "fixtures/sites/mini-shop/index.html?task=mini-shop-red-second-cheapest&run=$RUN_ID&seed=$SEED",
+    "validateScript": "fixtures/sites/mini-shop/grader.js#MiniShopGrader.validate",
+    "cheatScript": "fixtures/sites/mini-shop/cheats.js#mini-shop-red-second-cheapest",
+    "tags": [
+      "shopping",
+      "sort",
+      "cart",
+      "checkout"
+    ],
+    "rubric": [
+      {
+        "id": "addedTarget",
+        "weight": 0.4,
+        "description": "Cart includes second-cheapest red product"
+      },
+      {
+        "id": "noWrongItems",
+        "weight": 0.3,
+        "description": "Cart has no wrong products"
+      },
+      {
+        "id": "checkedOut",
+        "weight": 0.3,
+        "description": "Checkout produced order id"
+      }
+    ],
+    "goal": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible.",
+    "goalObject": [
+      {
+        "role": "user",
+        "content": [
+          {
+            "type": "text",
+            "text": "Add the second-cheapest red product to cart, checkout, and leave the generated order id visible."
+          }
+        ]
+      }
+    ],
+    "gradeSource": "both"
+  }
+]
diff --git a/test-suite/task-manifest.schema.json b/test-suite/task-manifest.schema.json
new file mode 100644
index 0000000..b8031fc
--- /dev/null
+++ b/test-suite/task-manifest.schema.json
@@ -0,0 +1,59 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "pi-chrome long-horizon task manifest",
+  "description": "BrowserGym-inspired task manifest: setup URL, deterministic seed, goal object, action subsets, reward contract, optional cheat recipe, and in-page validate hook.",
+  "type": "array",
+  "items": {
+    "type": "object",
+    "required": ["id", "taskId", "site", "startUrl", "setupUrl", "difficulty", "category", "goal", "goalObject", "grader", "actionSubsets", "maxSteps"],
+    "additionalProperties": true,
+    "properties": {
+      "id": { "type": "string" },
+      "taskId": { "type": "string" },
+      "site": { "type": "string" },
+      "seed": { "type": "integer" },
+      "startUrl": { "type": "string", "description": "Relative URL. $RUN_ID and $SEED should be replaced by runner for isolated deterministic state." },
+      "setupUrl": { "type": "string" },
+      "validateScript": { "type": "string" },
+      "cheatScript": { "type": "string" },
+      "viewport": { "type": "object", "properties": { "width": { "type": "integer" }, "height": { "type": "integer" } } },
+      "slowMoMs": { "type": "integer" },
+      "timeoutMs": { "type": "integer" },
+      "locale": { "type": ["string", "null"] },
+      "timezoneId": { "type": ["string", "null"] },
+      "difficulty": { "enum": ["L1", "L2", "L3"] },
+      "category": { "enum": ["enterprise-crud", "cross-page-memory", "cross-page-state-update", "task-completion"] },
+      "gradeSource": { "enum": ["page", "harness", "both"] },
+      "tags": { "type": "array", "items": { "type": "string" } },
+      "intent": { "type": "string" },
+      "goal": { "type": "string" },
+      "goalObject": { "type": "array", "items": { "type": "object" } },
+      "expectedActions": { "type": "object", "required": ["min", "max"], "properties": { "min": { "type": "integer" }, "max": { "type": "integer" } } },
+      "maxSteps": { "type": "integer" },
+      "humanBaseline": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
+      "actionSubsets": { "type": "array", "items": { "enum": ["chat", "infeas", "bid", "coord", "nav", "tab"] } },
+      "allowedActions": { "type": "array", "items": { "type": "string" } },
+      "requires": { "type": "array", "items": { "type": "string" } },
+      "rewardShape": { "enum": ["binary", "graded", "shaped"] },
+      "rubric": { "type": "array", "items": { "type": "object", "required": ["id", "weight", "description"], "properties": { "id": { "type": "string" }, "weight": { "type": "number" }, "description": { "type": "string" } } } },
+      "grader": {
+        "type": "object",
+        "required": ["type", "expression", "passValue"],
+        "properties": {
+          "type": { "enum": ["inPageProgrammatic"] },
+          "expression": { "type": "string", "description": "Should evaluate to BrowserGym-like validate object or legacy {v,r}." },
+          "passValue": { "type": "string" }
+        }
+      },
+      "browserGymLike": {
+        "type": "object",
+        "properties": {
+          "observation": { "type": "array", "items": { "type": "string" } },
+          "actionSpace": { "type": "array", "items": { "type": "string" } },
+          "reset": { "type": "string" },
+          "stepReturn": { "const": "obs,reward,terminated,truncated,info" }
+        }
+      }
+    }
+  }
+}
-- 
cgit v1.2.3