diff options
| author | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-13 07:29:41 -0400 |
|---|---|---|
| committer | Tianren Dong <tianrendong.dtr@gmail.com> | 2026-05-13 07:29:41 -0400 |
| commit | 4c54b05dc15b81e449dc8ac53da8bae536fa2798 (patch) | |
| tree | f81c5e627dcc4dd81d2fe2fd3b2b584e72a73eca /test-suite/manifest.json | |
| parent | 5951b3fb04aaf53b875afe265a57a944d6fce641 (diff) | |
pi-chrome: snapshot pre-migration state
Diffstat (limited to 'test-suite/manifest.json')
| -rw-r--r-- | test-suite/manifest.json | 569 |
1 files changed, 527 insertions, 42 deletions
diff --git a/test-suite/manifest.json b/test-suite/manifest.json index c09cf31..900b5d3 100644 --- a/test-suite/manifest.json +++ b/test-suite/manifest.json @@ -21,7 +21,13 @@ "notes": [ "Core debugger/CDP smoke test." ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "trusted-input" + ], + "requires": {} }, { "id": "is-trusted-keyboard", @@ -43,7 +49,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "trusted-input" + ], + "requires": {} }, { "id": "webdriver-flag", @@ -66,7 +78,13 @@ "notes": [ "Environment-sensitive; classify unexpected failures as profile warnings first." ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "fingerprint" + ], + "requires": {} }, { "id": "mouse-entropy", @@ -74,7 +92,7 @@ "category": "pointer-humanization", "goal": "Click is preceded by organic mousemove path.", "expected": { - "synthetic": "PASS", + "synthetic": "FAIL", "trusted": "PASS", "manual": "PASS" }, @@ -86,7 +104,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "pointer-humanization" + ], + "requires": {} }, { "id": "event-timing", @@ -94,7 +118,7 @@ "category": "timing", "goal": "Repeated clicks have plausible holds and gaps.", "expected": { - "synthetic": "PASS", + "synthetic": "CONDITIONAL", "trusted": "PASS", "manual": "PASS" }, @@ -119,7 +143,17 @@ } ], "flakeRisk": "medium", - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "timing" + ], + "requires": {}, + "thresholds": {}, + "notes": [ + "Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices." + ] }, { "id": "click-coordinates", @@ -127,7 +161,7 @@ "category": "pointer-humanization", "goal": "Clicks land inside target but not always exact center.", "expected": { - "synthetic": "PASS", + "synthetic": "CONDITIONAL", "trusted": "PASS", "manual": "PASS" }, @@ -163,7 +197,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "pointer-humanization" + ], + "requires": {} }, { "id": "pointer-properties", @@ -171,7 +211,7 @@ "category": "pointer-humanization", "goal": "Pointer events carry plausible pointerType, pressure, movement, id.", "expected": { - "synthetic": "PASS", + "synthetic": "CONDITIONAL", "trusted": "PASS", "manual": "PASS" }, @@ -183,7 +223,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "pointer-humanization" + ], + "requires": {} }, { "id": "keyboard-cadence", @@ -191,7 +237,7 @@ "category": "keyboard", "goal": "Typing emits per-character key events with non-uniform cadence.", "expected": { - "synthetic": "PASS", + "synthetic": "CONDITIONAL", "trusted": "PASS", "manual": "PASS" }, @@ -204,7 +250,17 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "keyboard" + ], + "requires": {}, + "thresholds": {}, + "notes": [ + "Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices." + ] }, { "id": "composition-input", @@ -225,7 +281,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "keyboard" + ], + "requires": {} }, { "id": "user-activation", @@ -252,7 +314,13 @@ "clipboard/fullscreen allowed by browser policy" ], "flakeRisk": "high", - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "activation-gates" + ], + "requires": {} }, { "id": "honeypot", @@ -279,7 +347,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "agent-safety" + ], + "requires": {} }, { "id": "fingerprint", @@ -303,7 +377,13 @@ "non-headless normal Chrome profile" ], "flakeRisk": "medium", - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "fingerprint" + ], + "requires": {} }, { "id": "focus-order", @@ -326,7 +406,13 @@ } ], "flakeRisk": "medium", - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "focus-keyboard" + ], + "requires": {} }, { "id": "wheel-scroll", @@ -347,7 +433,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "scroll" + ], + "requires": {} }, { "id": "drag-drop-datatransfer", @@ -372,7 +464,13 @@ "notes": [ "May need CDP Input.dispatchDragEvent rather than mouse drag on some Chrome versions." ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "drag-drop" + ], + "requires": {} }, { "id": "contenteditable-selection", @@ -380,7 +478,7 @@ "category": "editing", "goal": "Contenteditable typing advances selection/caret like user typing.", "expected": { - "synthetic": "PASS", + "synthetic": "CONDITIONAL", "trusted": "PASS", "manual": "PASS" }, @@ -393,7 +491,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "editing" + ], + "requires": {} }, { "id": "paste-clipboard", @@ -428,7 +532,13 @@ "OS clipboard contains pi-chrome", "use ctrlKey instead of metaKey on Windows/Linux" ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "clipboard" + ], + "requires": {} }, { "id": "native-select", @@ -471,7 +581,13 @@ } ], "flakeRisk": "high", - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "native-controls" + ], + "requires": {} }, { "id": "hover-dwell", @@ -505,7 +621,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "hover" + ], + "requires": {} }, { "id": "react-value-tracker", @@ -526,7 +648,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "frameworks" + ], + "requires": {} }, { "id": "keyboard-modifiers", @@ -550,7 +678,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "keyboard" + ], + "requires": {} }, { "id": "touch-events", @@ -573,7 +707,13 @@ "prerequisites": [ "touch events enabled/supported" ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "touch" + ], + "requires": {} }, { "id": "stack-trace-fingerprint", @@ -595,9 +735,16 @@ } ], "notes": [ - "Adversarial probe; selector resolution itself can be detected by monkey-patched DOM APIs." + "Adversarial probe; selector resolution itself can be detected by monkey-patched DOM APIs.", + "Should fail only on concrete automation URLs/Runtime frames, not generic anonymous stack formatting." + ], + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "fingerprint" ], - "manualBaseline": "unverified" + "requires": {} }, { "id": "viewport-edge-clicks", @@ -618,7 +765,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "pointer-humanization" + ], + "requires": {} }, { "id": "pointer-continuity", @@ -646,7 +799,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "pointer-humanization" + ], + "requires": {} }, { "id": "mousemove-rate", @@ -675,10 +834,18 @@ } ], "notes": [ - "Needs a first-class pointer path/sweep tool for deterministic automation." + "Needs a first-class pointer path/sweep tool for deterministic automation.", + "Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices." ], "flakeRisk": "high", - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "pointer-humanization" + ], + "requires": {}, + "thresholds": {} }, { "id": "scroll-momentum", @@ -703,7 +870,16 @@ "notes": [ "Physical detent mouse wheels may fail decay requirement; trackpads usually pass." ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "scroll" + ], + "requires": {}, + "prerequisites": [ + "Trackpad-like momentum; physical detent mouse wheels may fail decay requirement even for manual users." + ] }, { "id": "intersection-visibility", @@ -733,7 +909,13 @@ } ], "flakeRisk": "medium", - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "scroll-visibility" + ], + "requires": {} }, { "id": "shadow-dom-controls", @@ -767,7 +949,13 @@ "manualBaseline": "unverified", "prerequisites": [ "pi-chrome shadow-piercing selectors or shadow-root-aware snapshot uids" - ] + ], + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "dom-complexity" + ], + "requires": {} }, { "id": "iframe-targeting", @@ -800,7 +988,13 @@ "manualBaseline": "unverified", "prerequisites": [ "pi-chrome currently targets frameId 0 for many actions; iframe drilling needs frame-targeting support" - ] + ], + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "frames" + ], + "requires": {} }, { "id": "file-upload", @@ -826,7 +1020,13 @@ "manualBaseline": "unverified", "notes": [ "Recipe paths may contain $PWD; external runners should expand shell-style placeholders before calling chrome_upload_file." - ] + ], + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "files" + ], + "requires": {} }, { "id": "keyboard-tab-navigation", @@ -903,7 +1103,13 @@ } } ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "focus-keyboard" + ], + "requires": {} }, { "id": "network-console-capture", @@ -911,7 +1117,7 @@ "category": "observability", "goal": "Interaction emits console logs and fetch traffic for tool-level capture APIs.", "expected": { - "synthetic": "PASS", + "synthetic": "CONDITIONAL", "trusted": "PASS", "manual": "PASS" }, @@ -947,6 +1153,285 @@ "Page verdict only checks in-page fetch result; benchmark harness should also assert captured console/network entries.", "First chrome_list_console_messages/chrome_list_network_requests call installs capture hooks; do not move clear/list calls after click." ], - "manualBaseline": "unverified" + "manualBaseline": "unverified", + "gradeSource": "page", + "difficulty": "L1", + "tags": [ + "observability" + ], + "requires": {} + }, + { + "id": "dialog-handling", + "file": "challenges/34-dialog-handling.html", + "category": "dialogs", + "difficulty": "L2", + "goal": "Agent detects and handles alert/confirm/prompt/beforeunload browser dialogs.", + "expected": { + "synthetic": "CONDITIONAL", + "trusted": "CONDITIONAL", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#promptBtn", + "trusted": true + } + }, + { + "tool": "dialog", + "params": { + "type": "prompt", + "text": "pi-chrome", + "accept": true + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#confirmBtn", + "trusted": true + } + }, + { + "tool": "dialog", + "params": { + "type": "confirm", + "accept": true + } + } + ], + "requires": { + "dialogs": true + }, + "prerequisites": [ + "browser-dialog handling API or manual user intervention" + ], + "tags": [ + "dialogs", + "native-browser-ui" + ], + "notes": [ + "If dialog handling is unsupported, page may block after click; this is expected signal." + ], + "manualBaseline": "unverified", + "gradeSource": "page" + }, + { + "id": "target-blank-popup", + "file": "challenges/35-target-blank-popup.html", + "category": "popups", + "difficulty": "L1", + "goal": "Agent handles target=_blank new tab, discovers it via tab list, and inspects child tab verdict.", + "expected": { + "synthetic": "CONDITIONAL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#open", + "trusted": true + } + }, + { + "tool": "chrome_tab", + "params": { + "action": "list" + } + }, + { + "tool": "chrome_snapshot", + "params": { + "titleIncludes": "target blank popup" + } + } + ], + "requires": { + "popups": true + }, + "tags": [ + "tabs", + "popup" + ], + "notes": [ + "Child tab writes PASS to shared localStorage; harness should verify tab count/title too." + ], + "manualBaseline": "unverified", + "gradeSource": "page" + }, + { + "id": "modal-focus-trap", + "file": "challenges/36-modal-focus-trap.html", + "category": "focus-keyboard", + "difficulty": "L2", + "goal": "Keyboard focus stays trapped inside modal and Escape closes with focus restoration.", + "expected": { + "synthetic": "CONDITIONAL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#open", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Tab", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Tab", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Tab", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Escape", + "trusted": true + } + } + ], + "requires": { + "cdp": true + }, + "tags": [ + "modal", + "focus", + "keyboard" + ], + "manualBaseline": "unverified", + "gradeSource": "page" + }, + { + "id": "autocomplete-combobox", + "file": "challenges/37-autocomplete-combobox.html", + "category": "forms", + "difficulty": "L2", + "goal": "Agent uses ARIA combobox keyboard path: type filter, ArrowDown, Enter.", + "expected": { + "synthetic": "FAIL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_type", + "params": { + "selector": "#combo", + "text": "as", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "ArrowDown", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "Enter", + "trusted": true + } + } + ], + "requires": { + "cdp": true + }, + "tags": [ + "aria", + "combobox", + "forms" + ], + "manualBaseline": "unverified", + "gradeSource": "page" + }, + { + "id": "spa-route-change", + "file": "challenges/38-spa-route-change.html", + "category": "spa-routing", + "difficulty": "L1", + "goal": "Agent recognizes SPA pushState navigation and completes route-local action without full reload.", + "expected": { + "synthetic": "CONDITIONAL", + "trusted": "PASS", + "manual": "PASS" + }, + "recipe": [ + { + "tool": "chrome_click", + "params": { + "selector": "#settings", + "trusted": true + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#billing", + "trusted": true + } + }, + { + "tool": "chrome_key", + "params": { + "key": "ArrowLeft", + "modifiers": { + "altKey": true + }, + "trusted": true + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#billing", + "trusted": true + } + }, + { + "tool": "chrome_click", + "params": { + "selector": "#save", + "trusted": true + } + } + ], + "requires": { + "cdp": true + }, + "tags": [ + "spa", + "history", + "navigation" + ], + "manualBaseline": "unverified", + "gradeSource": "page", + "notes": [ + "Requires a real browser Back action (e.g. Alt+Left) to exercise popstate before Save." + ] } ] |
