aboutsummaryrefslogtreecommitdiff
path: root/test-suite/manifest.json
diff options
context:
space:
mode:
authorTianren Dong <tianrendong.dtr@gmail.com>2026-05-13 07:29:41 -0400
committerTianren Dong <tianrendong.dtr@gmail.com>2026-05-13 07:29:41 -0400
commit4c54b05dc15b81e449dc8ac53da8bae536fa2798 (patch)
treef81c5e627dcc4dd81d2fe2fd3b2b584e72a73eca /test-suite/manifest.json
parent5951b3fb04aaf53b875afe265a57a944d6fce641 (diff)
pi-chrome: snapshot pre-migration state
Diffstat (limited to 'test-suite/manifest.json')
-rw-r--r--test-suite/manifest.json569
1 files changed, 527 insertions, 42 deletions
diff --git a/test-suite/manifest.json b/test-suite/manifest.json
index c09cf31..900b5d3 100644
--- a/test-suite/manifest.json
+++ b/test-suite/manifest.json
@@ -21,7 +21,13 @@
"notes": [
"Core debugger/CDP smoke test."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "trusted-input"
+ ],
+ "requires": {}
},
{
"id": "is-trusted-keyboard",
@@ -43,7 +49,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "trusted-input"
+ ],
+ "requires": {}
},
{
"id": "webdriver-flag",
@@ -66,7 +78,13 @@
"notes": [
"Environment-sensitive; classify unexpected failures as profile warnings first."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "fingerprint"
+ ],
+ "requires": {}
},
{
"id": "mouse-entropy",
@@ -74,7 +92,7 @@
"category": "pointer-humanization",
"goal": "Click is preceded by organic mousemove path.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "FAIL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -86,7 +104,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "event-timing",
@@ -94,7 +118,7 @@
"category": "timing",
"goal": "Repeated clicks have plausible holds and gaps.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -119,7 +143,17 @@
}
],
"flakeRisk": "medium",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "timing"
+ ],
+ "requires": {},
+ "thresholds": {},
+ "notes": [
+ "Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices."
+ ]
},
{
"id": "click-coordinates",
@@ -127,7 +161,7 @@
"category": "pointer-humanization",
"goal": "Clicks land inside target but not always exact center.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -163,7 +197,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "pointer-properties",
@@ -171,7 +211,7 @@
"category": "pointer-humanization",
"goal": "Pointer events carry plausible pointerType, pressure, movement, id.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -183,7 +223,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "keyboard-cadence",
@@ -191,7 +237,7 @@
"category": "keyboard",
"goal": "Typing emits per-character key events with non-uniform cadence.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -204,7 +250,17 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "keyboard"
+ ],
+ "requires": {},
+ "thresholds": {},
+ "notes": [
+ "Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices."
+ ]
},
{
"id": "composition-input",
@@ -225,7 +281,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "keyboard"
+ ],
+ "requires": {}
},
{
"id": "user-activation",
@@ -252,7 +314,13 @@
"clipboard/fullscreen allowed by browser policy"
],
"flakeRisk": "high",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "activation-gates"
+ ],
+ "requires": {}
},
{
"id": "honeypot",
@@ -279,7 +347,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "agent-safety"
+ ],
+ "requires": {}
},
{
"id": "fingerprint",
@@ -303,7 +377,13 @@
"non-headless normal Chrome profile"
],
"flakeRisk": "medium",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "fingerprint"
+ ],
+ "requires": {}
},
{
"id": "focus-order",
@@ -326,7 +406,13 @@
}
],
"flakeRisk": "medium",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "focus-keyboard"
+ ],
+ "requires": {}
},
{
"id": "wheel-scroll",
@@ -347,7 +433,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "scroll"
+ ],
+ "requires": {}
},
{
"id": "drag-drop-datatransfer",
@@ -372,7 +464,13 @@
"notes": [
"May need CDP Input.dispatchDragEvent rather than mouse drag on some Chrome versions."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "drag-drop"
+ ],
+ "requires": {}
},
{
"id": "contenteditable-selection",
@@ -380,7 +478,7 @@
"category": "editing",
"goal": "Contenteditable typing advances selection/caret like user typing.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -393,7 +491,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "editing"
+ ],
+ "requires": {}
},
{
"id": "paste-clipboard",
@@ -428,7 +532,13 @@
"OS clipboard contains pi-chrome",
"use ctrlKey instead of metaKey on Windows/Linux"
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "clipboard"
+ ],
+ "requires": {}
},
{
"id": "native-select",
@@ -471,7 +581,13 @@
}
],
"flakeRisk": "high",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "native-controls"
+ ],
+ "requires": {}
},
{
"id": "hover-dwell",
@@ -505,7 +621,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "hover"
+ ],
+ "requires": {}
},
{
"id": "react-value-tracker",
@@ -526,7 +648,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "frameworks"
+ ],
+ "requires": {}
},
{
"id": "keyboard-modifiers",
@@ -550,7 +678,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "keyboard"
+ ],
+ "requires": {}
},
{
"id": "touch-events",
@@ -573,7 +707,13 @@
"prerequisites": [
"touch events enabled/supported"
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "touch"
+ ],
+ "requires": {}
},
{
"id": "stack-trace-fingerprint",
@@ -595,9 +735,16 @@
}
],
"notes": [
- "Adversarial probe; selector resolution itself can be detected by monkey-patched DOM APIs."
+ "Adversarial probe; selector resolution itself can be detected by monkey-patched DOM APIs.",
+ "Should fail only on concrete automation URLs/Runtime frames, not generic anonymous stack formatting."
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "fingerprint"
],
- "manualBaseline": "unverified"
+ "requires": {}
},
{
"id": "viewport-edge-clicks",
@@ -618,7 +765,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "pointer-continuity",
@@ -646,7 +799,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {}
},
{
"id": "mousemove-rate",
@@ -675,10 +834,18 @@
}
],
"notes": [
- "Needs a first-class pointer path/sweep tool for deterministic automation."
+ "Needs a first-class pointer path/sweep tool for deterministic automation.",
+ "Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices."
],
"flakeRisk": "high",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "pointer-humanization"
+ ],
+ "requires": {},
+ "thresholds": {}
},
{
"id": "scroll-momentum",
@@ -703,7 +870,16 @@
"notes": [
"Physical detent mouse wheels may fail decay requirement; trackpads usually pass."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "scroll"
+ ],
+ "requires": {},
+ "prerequisites": [
+ "Trackpad-like momentum; physical detent mouse wheels may fail decay requirement even for manual users."
+ ]
},
{
"id": "intersection-visibility",
@@ -733,7 +909,13 @@
}
],
"flakeRisk": "medium",
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "scroll-visibility"
+ ],
+ "requires": {}
},
{
"id": "shadow-dom-controls",
@@ -767,7 +949,13 @@
"manualBaseline": "unverified",
"prerequisites": [
"pi-chrome shadow-piercing selectors or shadow-root-aware snapshot uids"
- ]
+ ],
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "dom-complexity"
+ ],
+ "requires": {}
},
{
"id": "iframe-targeting",
@@ -800,7 +988,13 @@
"manualBaseline": "unverified",
"prerequisites": [
"pi-chrome currently targets frameId 0 for many actions; iframe drilling needs frame-targeting support"
- ]
+ ],
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "frames"
+ ],
+ "requires": {}
},
{
"id": "file-upload",
@@ -826,7 +1020,13 @@
"manualBaseline": "unverified",
"notes": [
"Recipe paths may contain $PWD; external runners should expand shell-style placeholders before calling chrome_upload_file."
- ]
+ ],
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "files"
+ ],
+ "requires": {}
},
{
"id": "keyboard-tab-navigation",
@@ -903,7 +1103,13 @@
}
}
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "focus-keyboard"
+ ],
+ "requires": {}
},
{
"id": "network-console-capture",
@@ -911,7 +1117,7 @@
"category": "observability",
"goal": "Interaction emits console logs and fetch traffic for tool-level capture APIs.",
"expected": {
- "synthetic": "PASS",
+ "synthetic": "CONDITIONAL",
"trusted": "PASS",
"manual": "PASS"
},
@@ -947,6 +1153,285 @@
"Page verdict only checks in-page fetch result; benchmark harness should also assert captured console/network entries.",
"First chrome_list_console_messages/chrome_list_network_requests call installs capture hooks; do not move clear/list calls after click."
],
- "manualBaseline": "unverified"
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "difficulty": "L1",
+ "tags": [
+ "observability"
+ ],
+ "requires": {}
+ },
+ {
+ "id": "dialog-handling",
+ "file": "challenges/34-dialog-handling.html",
+ "category": "dialogs",
+ "difficulty": "L2",
+ "goal": "Agent detects and handles alert/confirm/prompt/beforeunload browser dialogs.",
+ "expected": {
+ "synthetic": "CONDITIONAL",
+ "trusted": "CONDITIONAL",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#promptBtn",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "dialog",
+ "params": {
+ "type": "prompt",
+ "text": "pi-chrome",
+ "accept": true
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#confirmBtn",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "dialog",
+ "params": {
+ "type": "confirm",
+ "accept": true
+ }
+ }
+ ],
+ "requires": {
+ "dialogs": true
+ },
+ "prerequisites": [
+ "browser-dialog handling API or manual user intervention"
+ ],
+ "tags": [
+ "dialogs",
+ "native-browser-ui"
+ ],
+ "notes": [
+ "If dialog handling is unsupported, page may block after click; this is expected signal."
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "target-blank-popup",
+ "file": "challenges/35-target-blank-popup.html",
+ "category": "popups",
+ "difficulty": "L1",
+ "goal": "Agent handles target=_blank new tab, discovers it via tab list, and inspects child tab verdict.",
+ "expected": {
+ "synthetic": "CONDITIONAL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#open",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_tab",
+ "params": {
+ "action": "list"
+ }
+ },
+ {
+ "tool": "chrome_snapshot",
+ "params": {
+ "titleIncludes": "target blank popup"
+ }
+ }
+ ],
+ "requires": {
+ "popups": true
+ },
+ "tags": [
+ "tabs",
+ "popup"
+ ],
+ "notes": [
+ "Child tab writes PASS to shared localStorage; harness should verify tab count/title too."
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "modal-focus-trap",
+ "file": "challenges/36-modal-focus-trap.html",
+ "category": "focus-keyboard",
+ "difficulty": "L2",
+ "goal": "Keyboard focus stays trapped inside modal and Escape closes with focus restoration.",
+ "expected": {
+ "synthetic": "CONDITIONAL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#open",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Tab",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Tab",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Tab",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Escape",
+ "trusted": true
+ }
+ }
+ ],
+ "requires": {
+ "cdp": true
+ },
+ "tags": [
+ "modal",
+ "focus",
+ "keyboard"
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "autocomplete-combobox",
+ "file": "challenges/37-autocomplete-combobox.html",
+ "category": "forms",
+ "difficulty": "L2",
+ "goal": "Agent uses ARIA combobox keyboard path: type filter, ArrowDown, Enter.",
+ "expected": {
+ "synthetic": "FAIL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_type",
+ "params": {
+ "selector": "#combo",
+ "text": "as",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "ArrowDown",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "Enter",
+ "trusted": true
+ }
+ }
+ ],
+ "requires": {
+ "cdp": true
+ },
+ "tags": [
+ "aria",
+ "combobox",
+ "forms"
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "spa-route-change",
+ "file": "challenges/38-spa-route-change.html",
+ "category": "spa-routing",
+ "difficulty": "L1",
+ "goal": "Agent recognizes SPA pushState navigation and completes route-local action without full reload.",
+ "expected": {
+ "synthetic": "CONDITIONAL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#settings",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#billing",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_key",
+ "params": {
+ "key": "ArrowLeft",
+ "modifiers": {
+ "altKey": true
+ },
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#billing",
+ "trusted": true
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#save",
+ "trusted": true
+ }
+ }
+ ],
+ "requires": {
+ "cdp": true
+ },
+ "tags": [
+ "spa",
+ "history",
+ "navigation"
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page",
+ "notes": [
+ "Requires a real browser Back action (e.g. Alt+Left) to exercise popstate before Save."
+ ]
}
]