aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTianren Dong <tianrendong.dtr@gmail.com>2026-05-16 14:01:00 -0400
committerTianren Dong <tianrendong.dtr@gmail.com>2026-05-16 14:01:00 -0400
commitb792f22f2c2ef4dfb7bd300e7381a186e34e0c18 (patch)
treea31a4e6d1213478f5ac96e2208d1a1c16fc825b9
parentf133011c8ecda06ae63068649a9714c88a7b901e (diff)
0.15.25: fix reload after stale singleton
-rw-r--r--CHANGELOG.md5
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/manifest.json2
-rw-r--r--extensions/chrome-profile-bridge/index.ts12
-rw-r--r--package-lock.json4
-rw-r--r--package.json2
-rw-r--r--test-suite/README.md29
-rw-r--r--test-suite/challenges/39-strict-csp-fallback.html14
-rw-r--r--test-suite/challenges/39-strict-csp-fallback.js27
-rw-r--r--test-suite/challenges/40-dynamic-wait-readiness.html41
-rw-r--r--test-suite/challenges/41-tab-lifecycle.html44
-rw-r--r--test-suite/index.html14
-rw-r--r--test-suite/manifest.json269
-rw-r--r--test-suite/manifest.schema.json3
-rw-r--r--test-suite/notes/runner-spec.md7
-rw-r--r--test-suite/notes/scoring.md12
-rw-r--r--test-suite/task-manifest.json8
16 files changed, 431 insertions, 62 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 673bb26..ecf9272 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
All notable user-facing changes to `pi-chrome`.
+## 0.15.25 — 2026-05-16
+
+- **Reload after older installs.** `/reload` now recovers from stale singleton state left by pi-chrome 0.15.19 and earlier instead of skipping the freshly loaded extension.
+- **Test suite coverage.** Added gate buckets plus strict-CSP fallback, dynamic wait readiness, and explicit tab lifecycle challenges.
+
## 0.15.24 — 2026-05-16
- **Unload Chrome tools on lock.** `chrome_*` tools now deactivate when `/chrome revoke` runs or timed authorization expires, keeping the prompt/tool list small after Chrome control locks again.
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json
index 36da9f4..84e3986 100644
--- a/extensions/chrome-profile-bridge/browser-extension/manifest.json
+++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json
@@ -1,7 +1,7 @@
{
"manifest_version": 3,
"name": "Pi Chrome Connector",
- "version": "0.15.24",
+ "version": "0.15.25",
"description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.",
"permissions": [
"tabs",
diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts
index c8c921a..77ae767 100644
--- a/extensions/chrome-profile-bridge/index.ts
+++ b/extensions/chrome-profile-bridge/index.ts
@@ -470,17 +470,21 @@ function StringEnum<T extends readonly [string, ...string[]]>(values: T) {
export default function (pi: ExtensionAPI): void {
const instanceToken = Symbol("pi-chrome-instance");
+ const currentRoot = extensionRoot();
const globalState = globalThis as typeof globalThis & {
- [PI_CHROME_GLOBAL_KEY]?: { version: string; root: string; token: symbol };
+ [PI_CHROME_GLOBAL_KEY]?: { version: string; root: string; token?: symbol };
};
const alreadyLoaded = globalState[PI_CHROME_GLOBAL_KEY];
- if (alreadyLoaded) {
+ if (alreadyLoaded?.token || (alreadyLoaded && alreadyLoaded.root !== currentRoot)) {
console.warn(
- `pi-chrome already loaded from ${alreadyLoaded.root} (v${alreadyLoaded.version}); skipping duplicate from ${extensionRoot()}.`,
+ `pi-chrome already loaded from ${alreadyLoaded.root} (v${alreadyLoaded.version}); skipping duplicate from ${currentRoot}.`,
);
return;
}
- globalState[PI_CHROME_GLOBAL_KEY] = { version: PI_CHROME_VERSION, root: extensionRoot(), token: instanceToken };
+ // pi-chrome <=0.15.19 set the singleton flag but did not clear it on reload.
+ // If the stale flag points at this same extension root, replace it instead of
+ // skipping the freshly reloaded extension.
+ globalState[PI_CHROME_GLOBAL_KEY] = { version: PI_CHROME_VERSION, root: currentRoot, token: instanceToken };
const bridge = new ChromeProfileBridge(DEFAULT_HOST, DEFAULT_PORT);
let backgroundDefault = false;
diff --git a/package-lock.json b/package-lock.json
index 38bf360..f211b82 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "pi-chrome",
- "version": "0.15.24",
+ "version": "0.15.25",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "pi-chrome",
- "version": "0.15.24",
+ "version": "0.15.25",
"license": "MIT",
"peerDependencies": {
"@earendil-works/pi-coding-agent": "*",
diff --git a/package.json b/package.json
index be8e113..b24fdea 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "pi-chrome",
- "version": "0.15.24",
+ "version": "0.15.25",
"scripts": {
"version": "node scripts/sync-manifest-version.js",
"prepublishOnly": "node scripts/sync-manifest-version.js"
diff --git a/test-suite/README.md b/test-suite/README.md
index a96e0d0..5e64efe 100644
--- a/test-suite/README.md
+++ b/test-suite/README.md
@@ -26,10 +26,11 @@ Each challenge page exposes:
- `window.__events` — raw event log for forensics
`manifest.json` is the source of truth for unit-challenge metadata: category,
-goal, expected result per mode, prerequisites, flake risk, manual baseline
-status, and canonical tool recipe. `manifest.schema.json` documents the manifest
-shape. Recipes express tool intent; runners may need to adapt descriptive
-selectors (e.g. shadow/iframe notation) and expand path placeholders like `$PWD`.
+gate bucket, goal, expected result per mode, prerequisites, flake risk, manual
+baseline status, and canonical tool recipe. `manifest.schema.json` documents the
+manifest shape. Recipes express tool intent; runners may need to adapt
+descriptive selectors (e.g. shadow/iframe notation), dynamic tab ids, and expand
+path placeholders like `$PWD`.
`task-manifest.json` is the source of truth for long-horizon tasks: BrowserGym-style
`taskId`, seed, viewport, goal object, difficulty tier, max steps, declared
@@ -56,6 +57,16 @@ Expected values in `manifest.json`:
Manual baselines are tracked separately with `manualBaseline`. `unverified`
means the manual expectation is a target, not a recorded contract.
+## Gate buckets
+
+Each unit challenge has a `gate` field:
+
+- `core` — required release blocker for normal trusted-mode pi-chrome shipping.
+- `conditional` — blocks only when declared prerequisites/capabilities are present
+ (clipboard, touch, dialogs, native UI, etc.).
+- `quality` — adversarial humanization/fingerprint signal. Track regressions, but
+ do not block general ship without an explicit product decision.
+
## Recommended unit-challenge agent flow
1. Navigate to dashboard:
@@ -114,6 +125,8 @@ means the manual expectation is a target, not a recorded contract.
- `dom-complexity` / `frames` — Shadow DOM and iframe targeting.
- `files` — file attachment to `<input type=file>`.
- `observability` — console/network capture tools.
+- `csp` — strict Content Security Policy fallback where eval/snapshot may fail.
+- `lazy-loading` — dynamic DOM readiness and wait behavior.
- `fingerprint` — environment and stack fingerprint probes.
- `agent-safety` — hidden honeypots and safe target selection.
@@ -154,6 +167,14 @@ The dashboard renders this from `manifest.json`. In brief:
31. file upload
32. keyboard Tab navigation
33. network/console capture
+34. dialog handling
+35. target blank popup
+36. modal focus trap
+37. autocomplete combobox
+38. SPA route change
+39. strict CSP screenshot/coordinate fallback
+40. dynamic wait/readiness
+41. explicit tab lifecycle
## Design notes
diff --git a/test-suite/challenges/39-strict-csp-fallback.html b/test-suite/challenges/39-strict-csp-fallback.html
new file mode 100644
index 0000000..93bd9cf
--- /dev/null
+++ b/test-suite/challenges/39-strict-csp-fallback.html
@@ -0,0 +1,14 @@
+<!doctype html>
+<meta charset="utf-8">
+<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; object-src 'none'; base-uri 'none'">
+<title>39 strict CSP fallback</title>
+<link rel="stylesheet" href="../_style.css">
+<script src="../_lib.js"></script>
+<body>
+<main>
+ <p>Goal: page blocks <code>unsafe-eval</code>. Agent should fall back to screenshot/coordinate observation and click the target without relying on snapshot/evaluate.</p>
+ <p id="hint">The green target is fixed at known viewport coordinates. Use a screenshot, then click inside it.</p>
+ <button id="cspTarget" aria-label="strict CSP target">CSP TARGET</button>
+</main>
+<script src="39-strict-csp-fallback.js"></script>
+</body>
diff --git a/test-suite/challenges/39-strict-csp-fallback.js b/test-suite/challenges/39-strict-csp-fallback.js
new file mode 100644
index 0000000..4f2d91e
--- /dev/null
+++ b/test-suite/challenges/39-strict-csp-fallback.js
@@ -0,0 +1,27 @@
+Challenge.init({ id: "strict-csp-fallback", instructions: "use screenshot/coordinates; click CSP TARGET" });
+
+const btn = document.getElementById("cspTarget");
+btn.style.cssText = [
+ "position:fixed",
+ "left:220px",
+ "top:220px",
+ "width:180px",
+ "height:72px",
+ "font:700 16px system-ui",
+ "background:#1f7a1f",
+ "color:white",
+ "border:0",
+ "border-radius:10px",
+ "box-shadow:0 0 0 4px rgba(31,122,31,.25)"
+].join(";");
+
+document.getElementById("cspTarget").addEventListener("click", (e) => {
+ const r = btn.getBoundingClientRect();
+ const bad = [];
+ if (!e.isTrusted) bad.push("click isTrusted=false");
+ if (e.clientX < r.left || e.clientX > r.right || e.clientY < r.top || e.clientY > r.bottom) {
+ bad.push(`click coordinates ${e.clientX},${e.clientY} outside target rect ${Math.round(r.left)},${Math.round(r.top)},${Math.round(r.right)},${Math.round(r.bottom)}`);
+ }
+ if (bad.length) Challenge.fail(...bad);
+ else Challenge.pass("strict CSP page completed via trusted viewport click");
+});
diff --git a/test-suite/challenges/40-dynamic-wait-readiness.html b/test-suite/challenges/40-dynamic-wait-readiness.html
new file mode 100644
index 0000000..adf006d
--- /dev/null
+++ b/test-suite/challenges/40-dynamic-wait-readiness.html
@@ -0,0 +1,41 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>40 dynamic wait readiness</title>
+<link rel="stylesheet" href="../_style.css">
+<script src="../_lib.js"></script>
+<body>
+<main>
+ <p>Goal: wait until an async control exists and is enabled, then click it. Direct early clicks or fixed sleeps should not be required.</p>
+ <div id="status" role="status">Loading async action…</div>
+ <div id="mount" style="margin-top:20px"></div>
+</main>
+<script>
+Challenge.init({ id: "dynamic-wait-readiness", instructions: "wait for async button, then click" });
+const start = performance.now();
+setTimeout(() => {
+ const btn = document.createElement("button");
+ btn.id = "readyAction";
+ btn.textContent = "Run ready action";
+ btn.disabled = true;
+ btn.style.cssText = "padding:14px 22px;font-size:16px;background:#1f7a1f;color:#fff;border:0;border-radius:6px";
+ document.getElementById("mount").appendChild(btn);
+ document.getElementById("status").textContent = "Button mounted; enabling soon…";
+ Challenge.log("mounted", { at: performance.now() - start });
+ setTimeout(() => {
+ btn.disabled = false;
+ btn.dataset.ready = "true";
+ document.getElementById("status").textContent = "Ready.";
+ Challenge.log("enabled", { at: performance.now() - start });
+ }, 450);
+ btn.addEventListener("click", (e) => {
+ const elapsed = performance.now() - start;
+ const bad = [];
+ if (!e.isTrusted) bad.push("click isTrusted=false");
+ if (btn.disabled || btn.dataset.ready !== "true") bad.push("clicked before enabled/ready");
+ if (elapsed < 650) bad.push(`clicked too early at ${elapsed.toFixed(0)}ms`);
+ if (bad.length) Challenge.fail(...bad);
+ else Challenge.pass(`waited for async ready state (${elapsed.toFixed(0)}ms)`);
+ });
+}, 350);
+</script>
+</body>
diff --git a/test-suite/challenges/41-tab-lifecycle.html b/test-suite/challenges/41-tab-lifecycle.html
new file mode 100644
index 0000000..2ba002d
--- /dev/null
+++ b/test-suite/challenges/41-tab-lifecycle.html
@@ -0,0 +1,44 @@
+<!doctype html>
+<meta charset="utf-8">
+<title>41 tab lifecycle</title>
+<link rel="stylesheet" href="../_style.css">
+<script src="../_lib.js"></script>
+<body>
+<main>
+ <p>Goal: create a new tab, inspect it, close it, return to parent, then verify lifecycle state.</p>
+ <p id="msg"></p>
+ <button id="verify">Verify tab lifecycle</button>
+</main>
+<script>
+Challenge.init({ id: "tab-lifecycle", instructions: "new tab → inspect child → close child → return parent → Verify" });
+const qs = new URLSearchParams(location.search);
+const key = "pi-chrome-suite:tab-lifecycle:state";
+function readState(){ try { return JSON.parse(localStorage.getItem(key) || "{}"); } catch { return {}; } }
+function replaceState(next){ localStorage.setItem(key, JSON.stringify({ ...next, ts: Date.now() })); }
+function writeState(patch){ localStorage.setItem(key, JSON.stringify({ ...readState(), ...patch, ts: Date.now() })); }
+
+if (qs.get("child") === "1") {
+ document.title = "[CHILD] 41 tab lifecycle";
+ document.getElementById("msg").textContent = "Child tab opened. Close this tab, then return to parent.";
+ writeState({ opened: true, childUrl: location.href, childTitle: document.title });
+ window.addEventListener("pagehide", () => writeState({ closed: true, closedAt: Date.now() }));
+ window.addEventListener("beforeunload", () => writeState({ closed: true, closedAt: Date.now() }));
+ // Do not call Challenge.pass here: parent verification must be the only suite PASS.
+ Challenge.log("child-loaded", { url: location.href });
+} else {
+ document.getElementById("msg").textContent = "Parent tab. Runner should open this same file with ?child=1 in a new tab, inspect it, close it, then return here.";
+ replaceState({ parentLoaded: true, closed: false });
+ document.getElementById("verify").addEventListener("click", (e) => {
+ const s = readState();
+ const bad = [];
+ if (!e.isTrusted) bad.push("verify click isTrusted=false");
+ if (!s.opened) bad.push("child tab did not record opened=true");
+ if (!s.childUrl || !s.childUrl.includes("child=1")) bad.push("child URL not recorded");
+ if (!s.closed) bad.push("child tab close/pagehide was not recorded");
+ Challenge.log("tab-state", s);
+ if (bad.length) Challenge.fail(...bad);
+ else Challenge.pass("new/inspect/close/return tab lifecycle completed");
+ });
+}
+</script>
+</body>
diff --git a/test-suite/index.html b/test-suite/index.html
index 00ef204..ac95c9d 100644
--- a/test-suite/index.html
+++ b/test-suite/index.html
@@ -29,7 +29,7 @@
<table id="tbl">
<thead>
<tr>
- <th>id</th><th>category</th><th>verdict</th><th>expected</th><th>baseline</th><th>risk</th><th>goal / notes</th><th>open</th>
+ <th>id</th><th>gate</th><th>category</th><th>verdict</th><th>expected</th><th>baseline</th><th>risk</th><th>goal / notes</th><th>open</th>
</tr>
</thead>
<tbody></tbody>
@@ -80,6 +80,7 @@ function report() {
return {
id: m.id,
category: m.category,
+ gate: m.gate ?? "core",
file: m.file,
verdict: v?.verdict ?? "PENDING",
reason: v?.reason ?? [],
@@ -98,19 +99,25 @@ function paint() {
const rows = report();
const counts = rows.reduce((acc, r) => (acc[r.verdict] = (acc[r.verdict] || 0) + 1, acc), {});
const mismatches = rows.filter(r => r.verdict !== "PENDING" && r.expected !== "CONDITIONAL" && r.expected !== "—" && r.verdict !== r.expected).length;
+ const coreMismatches = rows.filter(r => (r.gate ?? "core") === "core" && r.verdict !== "PENDING" && r.expected !== "CONDITIONAL" && r.expected !== "—" && r.verdict !== r.expected).length;
+ const gates = rows.reduce((acc, r) => (acc[r.gate ?? "core"] = (acc[r.gate ?? "core"] || 0) + 1, acc), {});
document.getElementById("summary").innerHTML = `
<span class="pill pass">PASS ${counts.PASS || 0}</span>
<span class="pill fail">FAIL ${counts.FAIL || 0}</span>
<span class="pill skip">SKIP ${counts.SKIP || 0}</span>
<span class="pill warn">WARN ${counts.WARN || 0}</span>
<span class="pill pending">PENDING ${counts.PENDING || 0}</span>
- <span class="pill ${mismatches ? "fail" : "pass"}">unexpected ${mismatches}</span>
+ <span class="pill ${coreMismatches ? "fail" : "pass"}">core unexpected ${coreMismatches}</span>
+ <span class="pill ${mismatches ? "fail" : "pass"}">all unexpected ${mismatches}</span>
+ <span class="pill expected">core ${gates.core || 0}</span>
+ <span class="pill expected">conditional ${gates.conditional || 0}</span>
+ <span class="pill expected">quality ${gates.quality || 0}</span>
`;
for (const m of manifest) {
const v = verdictFor(m.id);
const verdict = v?.verdict ?? "PENDING";
const expected = m.expected?.[mode] ?? "—";
- const hay = JSON.stringify({ id:m.id, category:m.category, verdict, expected, goal:m.goal, notes:m.notes }).toLowerCase();
+ const hay = JSON.stringify({ id:m.id, gate:m.gate, category:m.category, verdict, expected, goal:m.goal, notes:m.notes }).toLowerCase();
if (q && !hay.includes(q)) continue;
const tr = document.createElement("tr");
tr.className = expectedClass(verdict, expected);
@@ -118,6 +125,7 @@ function paint() {
const notes = [...(m.prerequisites || []).map(x => "pre: " + x), ...(m.notes || [])].join("; ");
tr.innerHTML = `
<td><code>${m.id}</code></td>
+ <td><span class="pill expected">${m.gate || "core"}</span></td>
<td>${m.category || "—"}</td>
<td><span class="pill ${classFor(verdict)}">${verdict}</span><div class="reason">${escapeHtml(reason)}</div></td>
<td><span class="pill expected">${expected}</span></td>
diff --git a/test-suite/manifest.json b/test-suite/manifest.json
index 900b5d3..752c455 100644
--- a/test-suite/manifest.json
+++ b/test-suite/manifest.json
@@ -27,7 +27,8 @@
"tags": [
"trusted-input"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "is-trusted-keyboard",
@@ -55,7 +56,8 @@
"tags": [
"trusted-input"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "webdriver-flag",
@@ -84,7 +86,8 @@
"tags": [
"fingerprint"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "mouse-entropy",
@@ -110,7 +113,8 @@
"tags": [
"pointer-humanization"
],
- "requires": {}
+ "requires": {},
+ "gate": "quality"
},
{
"id": "event-timing",
@@ -153,7 +157,8 @@
"thresholds": {},
"notes": [
"Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices."
- ]
+ ],
+ "gate": "core"
},
{
"id": "click-coordinates",
@@ -203,7 +208,8 @@
"tags": [
"pointer-humanization"
],
- "requires": {}
+ "requires": {},
+ "gate": "quality"
},
{
"id": "pointer-properties",
@@ -229,7 +235,8 @@
"tags": [
"pointer-humanization"
],
- "requires": {}
+ "requires": {},
+ "gate": "quality"
},
{
"id": "keyboard-cadence",
@@ -260,7 +267,8 @@
"thresholds": {},
"notes": [
"Timing thresholds can be tuned via ?threshold.<name>=value for slower CI or high-polling devices."
- ]
+ ],
+ "gate": "quality"
},
{
"id": "composition-input",
@@ -287,7 +295,8 @@
"tags": [
"keyboard"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "user-activation",
@@ -320,7 +329,8 @@
"tags": [
"activation-gates"
],
- "requires": {}
+ "requires": {},
+ "gate": "conditional"
},
{
"id": "honeypot",
@@ -353,7 +363,8 @@
"tags": [
"agent-safety"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "fingerprint",
@@ -383,7 +394,8 @@
"tags": [
"fingerprint"
],
- "requires": {}
+ "requires": {},
+ "gate": "quality"
},
{
"id": "focus-order",
@@ -412,7 +424,8 @@
"tags": [
"focus-keyboard"
],
- "requires": {}
+ "requires": {},
+ "gate": "quality"
},
{
"id": "wheel-scroll",
@@ -439,7 +452,8 @@
"tags": [
"scroll"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "drag-drop-datatransfer",
@@ -470,7 +484,8 @@
"tags": [
"drag-drop"
],
- "requires": {}
+ "requires": {},
+ "gate": "conditional"
},
{
"id": "contenteditable-selection",
@@ -497,7 +512,8 @@
"tags": [
"editing"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "paste-clipboard",
@@ -538,7 +554,8 @@
"tags": [
"clipboard"
],
- "requires": {}
+ "requires": {},
+ "gate": "conditional"
},
{
"id": "native-select",
@@ -587,7 +604,8 @@
"tags": [
"native-controls"
],
- "requires": {}
+ "requires": {},
+ "gate": "conditional"
},
{
"id": "hover-dwell",
@@ -627,7 +645,8 @@
"tags": [
"hover"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "react-value-tracker",
@@ -654,7 +673,8 @@
"tags": [
"frameworks"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "keyboard-modifiers",
@@ -684,7 +704,8 @@
"tags": [
"keyboard"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "touch-events",
@@ -713,7 +734,8 @@
"tags": [
"touch"
],
- "requires": {}
+ "requires": {},
+ "gate": "conditional"
},
{
"id": "stack-trace-fingerprint",
@@ -744,7 +766,8 @@
"tags": [
"fingerprint"
],
- "requires": {}
+ "requires": {},
+ "gate": "quality"
},
{
"id": "viewport-edge-clicks",
@@ -771,7 +794,8 @@
"tags": [
"pointer-humanization"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "pointer-continuity",
@@ -805,7 +829,8 @@
"tags": [
"pointer-humanization"
],
- "requires": {}
+ "requires": {},
+ "gate": "quality"
},
{
"id": "mousemove-rate",
@@ -845,7 +870,8 @@
"pointer-humanization"
],
"requires": {},
- "thresholds": {}
+ "thresholds": {},
+ "gate": "quality"
},
{
"id": "scroll-momentum",
@@ -879,7 +905,8 @@
"requires": {},
"prerequisites": [
"Trackpad-like momentum; physical detent mouse wheels may fail decay requirement even for manual users."
- ]
+ ],
+ "gate": "quality"
},
{
"id": "intersection-visibility",
@@ -915,7 +942,8 @@
"tags": [
"scroll-visibility"
],
- "requires": {}
+ "requires": {},
+ "gate": "quality"
},
{
"id": "shadow-dom-controls",
@@ -955,7 +983,8 @@
"tags": [
"dom-complexity"
],
- "requires": {}
+ "requires": {},
+ "gate": "conditional"
},
{
"id": "iframe-targeting",
@@ -994,7 +1023,8 @@
"tags": [
"frames"
],
- "requires": {}
+ "requires": {},
+ "gate": "conditional"
},
{
"id": "file-upload",
@@ -1026,7 +1056,8 @@
"tags": [
"files"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "keyboard-tab-navigation",
@@ -1109,7 +1140,8 @@
"tags": [
"focus-keyboard"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "network-console-capture",
@@ -1159,7 +1191,8 @@
"tags": [
"observability"
],
- "requires": {}
+ "requires": {},
+ "gate": "core"
},
{
"id": "dialog-handling",
@@ -1217,7 +1250,8 @@
"If dialog handling is unsupported, page may block after click; this is expected signal."
],
"manualBaseline": "unverified",
- "gradeSource": "page"
+ "gradeSource": "page",
+ "gate": "conditional"
},
{
"id": "target-blank-popup",
@@ -1262,7 +1296,8 @@
"Child tab writes PASS to shared localStorage; harness should verify tab count/title too."
],
"manualBaseline": "unverified",
- "gradeSource": "page"
+ "gradeSource": "page",
+ "gate": "core"
},
{
"id": "modal-focus-trap",
@@ -1321,7 +1356,8 @@
"keyboard"
],
"manualBaseline": "unverified",
- "gradeSource": "page"
+ "gradeSource": "page",
+ "gate": "core"
},
{
"id": "autocomplete-combobox",
@@ -1367,7 +1403,8 @@
"forms"
],
"manualBaseline": "unverified",
- "gradeSource": "page"
+ "gradeSource": "page",
+ "gate": "core"
},
{
"id": "spa-route-change",
@@ -1432,6 +1469,162 @@
"gradeSource": "page",
"notes": [
"Requires a real browser Back action (e.g. Alt+Left) to exercise popstate before Save."
- ]
+ ],
+ "gate": "core"
+ },
+ {
+ "id": "strict-csp-fallback",
+ "file": "challenges/39-strict-csp-fallback.html",
+ "category": "csp",
+ "difficulty": "L2",
+ "gate": "core",
+ "goal": "Operate a page with strict CSP that blocks unsafe-eval by falling back to screenshot/viewport-coordinate input.",
+ "expected": {
+ "synthetic": "FAIL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_screenshot",
+ "params": {
+ "path": ".pi/chrome-screenshots/strict-csp-fallback.png"
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "x": 310,
+ "y": 256,
+ "trusted": true
+ }
+ }
+ ],
+ "requires": {
+ "cdp": true
+ },
+ "tags": [
+ "csp",
+ "screenshot",
+ "coordinate-click",
+ "strict-csp"
+ ],
+ "notes": [
+ "chrome_snapshot/chrome_evaluate may fail on this page because script-src omits unsafe-eval; this is intentional. Use screenshot plus viewport coordinates, then read verdict from dashboard/localStorage after leaving CSP page."
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "dynamic-wait-readiness",
+ "file": "challenges/40-dynamic-wait-readiness.html",
+ "category": "lazy-loading",
+ "difficulty": "L1",
+ "gate": "core",
+ "goal": "Wait for an asynchronously mounted and enabled control before clicking it.",
+ "expected": {
+ "synthetic": "FAIL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_wait_for",
+ "params": {
+ "kind": "selector",
+ "value": "#readyAction[data-ready=\"true\"]:not([disabled])",
+ "timeoutMs": 3000
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#readyAction",
+ "trusted": true
+ }
+ }
+ ],
+ "requires": {
+ "cdp": true
+ },
+ "tags": [
+ "wait",
+ "async-dom",
+ "readiness"
+ ],
+ "notes": [
+ "Covers explicit wait primitive and avoids fixed sleep as readiness strategy."
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
+ },
+ {
+ "id": "tab-lifecycle",
+ "file": "challenges/41-tab-lifecycle.html",
+ "category": "popups",
+ "difficulty": "L2",
+ "gate": "core",
+ "goal": "Create a new tab, inspect it, close it, return to the parent tab, and verify tab lifecycle state.",
+ "expected": {
+ "synthetic": "CONDITIONAL",
+ "trusted": "PASS",
+ "manual": "PASS"
+ },
+ "recipe": [
+ {
+ "tool": "chrome_tab",
+ "params": {
+ "action": "new",
+ "url": "http://127.0.0.1:8765/challenges/41-tab-lifecycle.html?child=1"
+ }
+ },
+ {
+ "tool": "chrome_tab",
+ "params": {
+ "action": "list"
+ }
+ },
+ {
+ "tool": "chrome_snapshot",
+ "params": {
+ "titleIncludes": "tab lifecycle"
+ }
+ },
+ {
+ "tool": "chrome_tab",
+ "params": {
+ "action": "close",
+ "targetId": "<child tab id from list>"
+ }
+ },
+ {
+ "tool": "chrome_tab",
+ "params": {
+ "action": "activate",
+ "targetId": "<parent tab id>"
+ }
+ },
+ {
+ "tool": "chrome_click",
+ "params": {
+ "selector": "#verify",
+ "trusted": true
+ }
+ }
+ ],
+ "requires": {
+ "popups": true
+ },
+ "tags": [
+ "tabs",
+ "new-tab",
+ "close-tab",
+ "activate-tab"
+ ],
+ "notes": [
+ "Recipe contains dynamic tab IDs; runner must substitute ids from chrome_tab list. Complements target=_blank by testing explicit new/close/activate tools."
+ ],
+ "manualBaseline": "unverified",
+ "gradeSource": "page"
}
]
diff --git a/test-suite/manifest.schema.json b/test-suite/manifest.schema.json
index cd6dcce..fa63c58 100644
--- a/test-suite/manifest.schema.json
+++ b/test-suite/manifest.schema.json
@@ -12,6 +12,7 @@
"file": { "type": "string" },
"category": { "$ref": "#/$defs/category" },
"difficulty": { "enum": ["L1", "L2", "L3"] },
+ "gate": { "description": "Ship-gate bucket. core blocks releases, conditional requires declared capability/environment, quality is adversarial/flaky signal and should not block general ship.", "enum": ["core", "conditional", "quality"] },
"tags": { "type": "array", "items": { "type": "string" } },
"goal": { "type": "string" },
"gradeSource": { "enum": ["page", "harness", "both"], "default": "page" },
@@ -65,7 +66,7 @@
"outcome": { "enum": ["PASS", "FAIL", "SKIP", "WARN", "CONDITIONAL"] },
"category": {
"enum": [
- "activation-gates", "agent-safety", "aria-snapshot", "canvas-svg", "clipboard", "contextmenu", "dialogs", "dom-complexity", "downloads", "drag-drop", "editing", "files", "fingerprint", "focus-keyboard", "forms", "frames", "frameworks", "hover", "keyboard", "lazy-loading", "native-controls", "observability", "pointer-humanization", "popups", "scroll", "scroll-visibility", "spa-routing", "task-completion", "timing", "touch", "trusted-input"
+ "activation-gates", "agent-safety", "aria-snapshot", "canvas-svg", "clipboard", "contextmenu", "csp", "dialogs", "dom-complexity", "downloads", "drag-drop", "editing", "files", "fingerprint", "focus-keyboard", "forms", "frames", "frameworks", "hover", "keyboard", "lazy-loading", "native-controls", "observability", "pointer-humanization", "popups", "scroll", "scroll-visibility", "spa-routing", "task-completion", "timing", "touch", "trusted-input"
]
}
}
diff --git a/test-suite/notes/runner-spec.md b/test-suite/notes/runner-spec.md
index b040cb4..24c97ab 100644
--- a/test-suite/notes/runner-spec.md
+++ b/test-suite/notes/runner-spec.md
@@ -3,13 +3,14 @@
Future Node/Pi runner should:
1. Read `manifest.json`.
-2. Probe environment capabilities (`trusted`, `clipboard`, `touch`, `dialogs`, `downloads`, `fileSystem`).
+2. Probe environment capabilities (`trusted`, `clipboard`, `touch`, `dialogs`, `downloads`, `fileSystem`, `strictCspFallback`).
3. For each entry:
+ - read `gate` (`core`, `conditional`, `quality`) and score in that bucket
- skip if `requires` is unsatisfied and expected is `CONDITIONAL`
- navigate to challenge URL
- execute `recipe` in order
- wait `verdictDelayMs || 500`
- - evaluate `JSON.stringify({v:window.__verdict,r:window.__reason,d:window.Challenge?.state?.details,e:window.__events?.slice(-20)})`
+ - evaluate `JSON.stringify({v:window.__verdict,r:window.__reason,d:window.Challenge?.state?.details,e:window.__events?.slice(-20)})`, except strict-CSP pages where eval is expected to fail; read persisted verdict from dashboard/localStorage after leaving the CSP page
- compare to `expected[mode]`
4. Output:
- Markdown summary
@@ -21,6 +22,8 @@ Runner must adapt recipe intent:
- expand `${REPO_ROOT}` path placeholders
- adapt unsupported shadow/iframe selector notation to snapshot uids or evaluate fallback
- preserve hook install ordering for console/network capture tests
+- substitute dynamic tab ids from `chrome_tab list` recipes
+- support screenshot/coordinate fallback for strict CSP pages where snapshot/evaluate are blocked
- record whether trusted/CDP path was used
Long-horizon task runner should read `task-manifest.json`, replace `$RUN_ID`, solve task, then evaluate task grader expression.
diff --git a/test-suite/notes/scoring.md b/test-suite/notes/scoring.md
index 970a706..5edb7f5 100644
--- a/test-suite/notes/scoring.md
+++ b/test-suite/notes/scoring.md
@@ -4,8 +4,16 @@
Score actual verdict against `expected[mode]`, not raw PASS count.
-- `PASS` / `FAIL` expected: exact match required.
-- `CONDITIONAL`: exclude from headline score unless environment prerequisite is declared present.
+Gate buckets:
+
+- `core`: headline release gate. Exact expected verdict required.
+- `conditional`: gate only when declared capability/prerequisite is present; otherwise report skipped/conditional.
+- `quality`: adversarial humanization/fingerprint signal. Report trend and regressions, but do not block general release unless explicitly promoted.
+
+Expected values:
+
+- `PASS` / `FAIL` expected: exact match required inside active gate bucket.
+- `CONDITIONAL`: exclude from core headline score unless environment prerequisite is declared present.
- `SKIP` / `WARN`: report separately.
Recommended flake policy:
diff --git a/test-suite/task-manifest.json b/test-suite/task-manifest.json
index 22e9590..e780510 100644
--- a/test-suite/task-manifest.json
+++ b/test-suite/task-manifest.json
@@ -18,7 +18,7 @@
],
"grader": {
"type": "inPageProgrammatic",
- "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+ "expression": "window.__taskVerdict || (window.__taskValidation?.done ? \"PASS\" : \"FAIL\")",
"passValue": "PASS"
},
"browserGymLike": {
@@ -122,7 +122,7 @@
],
"grader": {
"type": "inPageProgrammatic",
- "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+ "expression": "window.__taskVerdict || (window.__taskValidation?.done ? \"PASS\" : \"FAIL\")",
"passValue": "PASS"
},
"browserGymLike": {
@@ -227,7 +227,7 @@
],
"grader": {
"type": "inPageProgrammatic",
- "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+ "expression": "window.__taskVerdict || (window.__taskValidation?.done ? \"PASS\" : \"FAIL\")",
"passValue": "PASS"
},
"browserGymLike": {
@@ -330,7 +330,7 @@
],
"grader": {
"type": "inPageProgrammatic",
- "expression": "JSON.stringify(window.__taskValidation || {v:window.__taskVerdict,r:window.__taskReason})",
+ "expression": "window.__taskVerdict || (window.__taskValidation?.done ? \"PASS\" : \"FAIL\")",
"passValue": "PASS"
},
"browserGymLike": {