aboutsummaryrefslogtreecommitdiff
path: root/extensions
diff options
context:
space:
mode:
authorTianren Dong <tianrendong.dtr@gmail.com>2026-09-26 19:07:56 -0400
committerTianren Dong <tianrendong.dtr@gmail.com>2026-09-26 19:07:56 -0400
commit238f347a03b2cefb21198bb78a90699e931d8504 (patch)
tree4c7221441428a086e93f2767a9fb332ee97c9995 /extensions
parentdf1472b8c777d40daa70da23115f4c8f5afb6623 (diff)
Report hidden-tab input, append targeted typing, smarter uploads, prune screenshots; release 0.15.54
- Check page visibility before trusted input: Chrome drops input to hidden pages (inactive tabs, covered/minimized windows, macOS window tabs behind full screen). Type/key/hover/scroll/tap/drag fail fast with the reason; click/fill use their DOM fallback immediately (or reject with domFallback:false). - Use an inactive tab instead of a new window for the automation target while the user's window is full screen. - chrome_type with uid/selector moves the caret to the end (moveToEndOfDocument) after its focus click. - chrome_upload_file: labels/wrappers resolve to their file input, same-origin iframe search, and intercepted file chooser for upload buttons (trusted click when visible, user-gesture click when hidden); reject multiple paths for single-file inputs; no duplicate change event. - Prune default-folder chrome_screenshot captures (7 days, newest 20 kept, retentionDays override). - input.debug without a target no longer calls tabs.get(-1). - New hidden-input unit suite.
Diffstat (limited to 'extensions')
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/manifest.json2
-rw-r--r--extensions/chrome-profile-bridge/browser-extension/service_worker.js274
-rw-r--r--extensions/chrome-profile-bridge/index.ts83
3 files changed, 320 insertions, 39 deletions
diff --git a/extensions/chrome-profile-bridge/browser-extension/manifest.json b/extensions/chrome-profile-bridge/browser-extension/manifest.json
index 952c9cc..aed24ef 100644
--- a/extensions/chrome-profile-bridge/browser-extension/manifest.json
+++ b/extensions/chrome-profile-bridge/browser-extension/manifest.json
@@ -1,7 +1,7 @@
{
"manifest_version": 3,
"name": "Pi Chrome Connector",
- "version": "0.15.53",
+ "version": "0.15.54",
"description": "Lets Pi control tabs in Chrome via a local connector at 127.0.0.1.",
"permissions": [
"tabs",
diff --git a/extensions/chrome-profile-bridge/browser-extension/service_worker.js b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
index 4bfce34..ead8ae0 100644
--- a/extensions/chrome-profile-bridge/browser-extension/service_worker.js
+++ b/extensions/chrome-profile-bridge/browser-extension/service_worker.js
@@ -174,6 +174,17 @@ function isPiChromeOwnedTarget(tabId, sessionKey) {
return false;
}
+// A new window created while the user's Chrome window is full screen does not become a visible
+// standalone window: macOS either merges it into the full-screen window as a hidden native window
+// tab ("Prefer tabs: in full screen") or places it on another Space. Its page is hidden, so trusted
+// input cannot reach it. Use a grouped background tab in the user's window instead, which is at least
+// visible in the tab strip and does not add a macOS tab bar to the user's full-screen window.
+async function userWindowIsFullscreen() {
+ if (typeof chrome.windows?.getLastFocused !== "function") return false;
+ const win = await chrome.windows.getLastFocused({ windowTypes: ["normal"] }).catch(() => null);
+ return win?.state === "fullscreen";
+}
+
// Create a fresh automation target for `sessionKey`. If this session already has a tab group,
// create the tab inside that group's window so one Pi session keeps one Chrome tab group (Chrome
// groups cannot span windows). If no group exists yet, prefer an isolated window; fall back to a
@@ -187,7 +198,7 @@ async function createAutomationTarget(sessionKey, groupTitle) {
await persistAutomationTargets();
return tab;
}
- if (chrome.windows && typeof chrome.windows.create === "function") {
+ if (chrome.windows && typeof chrome.windows.create === "function" && !(await userWindowIsFullscreen())) {
try {
const win = await chrome.windows.create({ url: "about:blank", focused: false });
const created = win && Array.isArray(win.tabs) ? win.tabs[0] : undefined;
@@ -391,7 +402,8 @@ async function attachDebugger(tabId) {
}
async function describeInputTarget(tabId) {
- const tab = await chrome.tabs.get(Number(tabId)).catch(() => null);
+ const id = Number(tabId);
+ const tab = Number.isInteger(id) && id >= 0 ? await chrome.tabs.get(id).catch(() => null) : null;
const active = (await chrome.tabs.query({ active: true, lastFocusedWindow: true }).catch(() => []))[0] || null;
let targets = [];
try { targets = await new Promise((resolve) => chrome.debugger.getTargets((t) => resolve(t || []))); } catch {}
@@ -408,7 +420,13 @@ function targetMetaSuffix(meta) {
}
async function inputDebug(params) {
- const requested = params?.targetId ? await describeInputTarget(Number(params.targetId)) : await describeInputTarget(-1);
+ // No explicit target: describe this session's automation tab if it exists, without creating one.
+ let tabId = params?.targetId !== undefined ? Number(params.targetId) : null;
+ if (tabId === null) {
+ const owned = await resolveOwnedAutomationTarget(sessionKeyOf(params || {})).catch(() => null);
+ tabId = owned?.id ?? null;
+ }
+ const requested = await describeInputTarget(tabId);
return {
extensionVersion: chrome.runtime.getManifest().version,
extensionId: chrome.runtime.id,
@@ -417,6 +435,57 @@ async function inputDebug(params) {
};
}
+// Chrome drops trusted CDP input to hidden pages: mouse presses and keys are silently ignored and
+// mouse moves can hang waiting for a frame. A page is hidden when its tab is inactive, its window is
+// minimized, or (macOS) its window became a native window-tab behind a full-screen window. Check
+// before sending input so the caller gets an actionable error instead of a false success.
+async function pageVisibilityState(tabId) {
+ try {
+ const evaluated = await cdp(tabId, "Runtime.evaluate", { expression: "document.visibilityState", returnByValue: true });
+ const value = evaluated?.result?.value;
+ return typeof value === "string" ? value : null;
+ } catch {
+ return null; // Unknown: do not block input on a failed probe.
+ }
+}
+
+function hiddenTabReason(tab, win) {
+ if (win?.state === "minimized") return "its window is minimized";
+ if (tab && tab.active === false) return "it is an inactive tab in its window";
+ if (win?.state === "fullscreen") return "its window is behind a full-screen window (on macOS, new windows can become hidden window tabs)";
+ return "its window is covered by another window or is on another Space (Chrome treats fully covered windows as hidden)";
+}
+
+async function assertTabVisibleForInput(tab, params, label) {
+ let state = await pageVisibilityState(tab.id);
+ if (state !== "hidden") return;
+ if (foregroundRequested(params)) {
+ // bringToFront just ran; give Chrome a moment to report the page visible.
+ for (let i = 0; i < 6 && state === "hidden"; i++) {
+ await sleep(100);
+ state = await pageVisibilityState(tab.id);
+ }
+ if (state !== "hidden") return;
+ }
+ const fresh = await chrome.tabs.get(tab.id).catch(() => tab);
+ const win = typeof chrome.windows?.get === "function" ? await chrome.windows.get(fresh?.windowId ?? tab.windowId).catch(() => null) : null;
+ const error = new Error(
+ `${label}: tab ${tab.id} is hidden because ${hiddenTabReason(fresh, win)}. Chrome ignores trusted input to hidden pages, so the events would be dropped. ` +
+ `Pass background:false for this call (or run /chrome background off) to bring the tab forward. chrome_click and chrome_fill can instead use their DOM fallback (untrusted page events).`,
+ );
+ error.hiddenTab = true;
+ error.windowState = win?.state;
+ throw error;
+}
+
+// Put the caret at the end of the focused field with Chrome's editing command (platform-neutral,
+// works for inputs, textareas, and contenteditables). Used after chrome_type clicks a field to focus
+// it, so the random click point never splices text into the middle of existing content.
+async function moveCaretToEnd(tabId) {
+ await cdp(tabId, "Input.dispatchKeyEvent", { type: "rawKeyDown", key: "End", code: "End", windowsVirtualKeyCode: 35, commands: ["moveToEndOfDocument"] });
+ await cdp(tabId, "Input.dispatchKeyEvent", { type: "keyUp", key: "End", code: "End", windowsVirtualKeyCode: 35 });
+}
+
async function detachDebugger(tabId) {
const entry = attachedTabs.get(tabId);
if (!entry) return;
@@ -825,6 +894,7 @@ async function chromeInputClick(params) {
await bringToFront(tab, params);
try {
await attachDebugger(tab.id);
+ await assertTabVisibleForInput(tab, params, "chrome.click");
const resolved = await resolveTargetInTab(tab.id, params);
const point = resolved.rect ? pickInsideRect(resolved.rect) : { x: resolved.x, y: resolved.y };
await cdpMoveTo(tab.id, point.x, point.y);
@@ -861,6 +931,7 @@ async function chromeInputHover(params) {
const tab = await getTabByParams(params);
await bringToFront(tab, params);
await attachDebugger(tab.id);
+ await assertTabVisibleForInput(tab, params, "chrome.hover");
const resolved = await resolveTargetInTab(tab.id, params);
const point = resolved.rect ? pickInsideRect(resolved.rect) : { x: resolved.x, y: resolved.y };
await cdpMoveTo(tab.id, point.x, point.y);
@@ -872,6 +943,7 @@ async function chromeInputKey(params) {
const tab = await getTabByParams(params);
await bringToFront(tab, params);
await attachDebugger(tab.id);
+ await assertTabVisibleForInput(tab, params, "chrome.key");
const key = String(params.key || "");
if (!key) throw new Error("chrome.key: missing key");
const mods = params.modifiers || {};
@@ -954,6 +1026,7 @@ async function chromeInputType(params) {
const tab = await getTabByParams(params);
await bringToFront(tab, params);
await attachDebugger(tab.id);
+ await assertTabVisibleForInput(tab, params, "chrome.type");
if (params.selector || params.uid) {
// Focus target by clicking it first.
const resolved = await resolveTargetInTab(tab.id, params);
@@ -963,6 +1036,9 @@ async function chromeInputType(params) {
await sleep(rng(45, 110));
await cdp(tab.id, "Input.dispatchMouseEvent", { type: "mouseReleased", x: point.x, y: point.y, button: "left", buttons: 0, clickCount: 1, pointerType: "mouse" });
await sleep(rng(50, 120));
+ // The click lands at a random point inside the field, which leaves the caret mid-text. Move it to
+ // the end so a targeted chrome_type appends; replace=true selects everything below anyway.
+ if (params.replace !== true) await moveCaretToEnd(tab.id);
}
const text = String(params.text || "");
// chrome_type inserts at the caret. Read the field before and after so a splice into existing
@@ -1144,6 +1220,7 @@ async function chromeInputFill(params) {
await bringToFront(tab, params);
try {
await attachDebugger(tab.id);
+ await assertTabVisibleForInput(tab, params, "chrome.fill");
if (!(params.selector || params.uid)) throw new Error("chrome.fill: selector or uid required");
const resolved = await resolveTargetInTab(tab.id, params);
const point = resolved.rect ? pickInsideRect(resolved.rect) : { x: resolved.x, y: resolved.y };
@@ -1174,6 +1251,7 @@ async function chromeInputScroll(params) {
const tab = await getTabByParams(params);
await bringToFront(tab, params);
await attachDebugger(tab.id);
+ await assertTabVisibleForInput(tab, params, "chrome.scroll");
const resolved = (params.selector || params.uid) ? await resolveTargetInTab(tab.id, params) : { x: 100, y: 100, rect: null };
const x = resolved.rect ? resolved.rect.left + Math.min(resolved.rect.width, 800) / 2 : resolved.x;
const y = resolved.rect ? resolved.rect.top + Math.min(resolved.rect.height, 600) / 2 : resolved.y;
@@ -1224,6 +1302,7 @@ async function chromeInputTap(params) {
const tab = await getTabByParams(params);
await bringToFront(tab, params);
await attachDebugger(tab.id);
+ await assertTabVisibleForInput(tab, params, "chrome.tap");
const resolved = (params.selector || params.uid || (typeof params.x === "number" && typeof params.y === "number"))
? await resolveTargetInTab(tab.id, params)
: null;
@@ -1240,6 +1319,7 @@ async function chromeInputDrag(params) {
const tab = await getTabByParams(params);
await bringToFront(tab, params);
await attachDebugger(tab.id);
+ await assertTabVisibleForInput(tab, params, "chrome.drag");
const from = await resolveTargetInTab(tab.id, { selector: params.fromSelector ?? null, uid: params.fromUid ?? null, x: params.fromX ?? null, y: params.fromY ?? null });
const to = await resolveTargetInTab(tab.id, { selector: params.toSelector ?? null, uid: params.toUid ?? null, x: params.toX ?? null, y: params.toY ?? null });
const fp = from.rect ? pickInsideRect(from.rect) : { x: from.x, y: from.y };
@@ -1261,6 +1341,100 @@ async function chromeInputDrag(params) {
return { input: "chrome", from: fp, to: tp, steps };
}
+// Pending native file-chooser intercepts, keyed by tabId. chrome_upload_file uses these when the
+// target is an upload trigger (button/dropzone) rather than a persistent <input type=file>:
+// Page.setInterceptFileChooserDialog -> click the trigger -> Page.fileChooserOpened ->
+// DOM.setFileInputFiles(backendNodeId). No OS dialog is shown.
+const pendingFileChoosers = new Map();
+const FILE_CHOOSER_TIMEOUT_MS = 8_000;
+
+function fileChooserTabIdForSource(source) {
+ if (typeof source?.tabId === "number") return source.tabId;
+ if (source?.targetId) {
+ for (const [tabId, entry] of attachedTabs) if (entry?.debuggee?.targetId === source.targetId) return tabId;
+ }
+ return undefined;
+}
+
+function handleDebuggerEvent(source, method, params) {
+ if (method !== "Page.fileChooserOpened") return;
+ const tabId = fileChooserTabIdForSource(source);
+ const entry = tabId === undefined ? undefined : pendingFileChoosers.get(tabId);
+ if (!entry) return;
+ pendingFileChoosers.delete(tabId);
+ entry.resolve({ backendNodeId: params?.backendNodeId, mode: params?.mode });
+}
+
+if (chrome.debugger && chrome.debugger.onEvent) {
+ chrome.debugger.onEvent.addListener(handleDebuggerEvent);
+}
+
+function waitForFileChooser(tabId, timeoutMs) {
+ let entry;
+ const promise = new Promise((resolve, reject) => {
+ entry = { resolve, reject };
+ entry.timer = setTimeout(() => {
+ if (pendingFileChoosers.get(tabId) === entry) pendingFileChoosers.delete(tabId);
+ reject(new Error("chrome.upload: no file chooser opened after clicking the target. Pass the <input type=file> itself, or an element that opens the file picker when clicked."));
+ }, timeoutMs);
+ });
+ pendingFileChoosers.set(tabId, entry);
+ return { promise: promise.finally(() => clearTimeout(entry.timer)), cancel: () => { if (pendingFileChoosers.get(tabId) === entry) pendingFileChoosers.delete(tabId); clearTimeout(entry.timer); } };
+}
+
+// Resolve the upload target in the page. Selectors also search same-origin iframes (editors often
+// host their file input in one). A label for a file input, or a wrapper/dropzone containing exactly
+// one file input, resolves to that input so no click is needed.
+function uploadTargetExpression(selector, uid) {
+ return `(() => {
+ const selector = ${JSON.stringify(selector ?? null)};
+ const uid = ${JSON.stringify(uid ?? null)};
+ const state = window.__PI_CHROME_STATE__;
+ const findIn = (doc, depth) => {
+ let found = null;
+ try { found = doc.querySelector(selector); } catch (error) { throw new Error("Invalid selector: " + selector); }
+ if (found || depth >= 3) return found;
+ for (const frame of doc.querySelectorAll("iframe,frame")) {
+ let inner = null;
+ try { inner = frame.contentDocument; } catch {}
+ if (!inner) continue;
+ found = findIn(inner, depth + 1);
+ if (found) return found;
+ }
+ return null;
+ };
+ let el = uid ? state?.elements?.[uid] : (selector ? findIn(document, 0) : null);
+ if (uid && (!el || !el.isConnected)) throw new Error("snapshot uid " + uid + " is stale; refresh chrome_snapshot");
+ if (!el) throw new Error("chrome.upload: target not found: " + (uid || selector));
+ const isFile = (node) => node && node.tagName === "INPUT" && String(node.type).toLowerCase() === "file";
+ if (!isFile(el)) {
+ const control = el.tagName === "LABEL" && isFile(el.control) ? el.control : null;
+ const nested = control ? null : Array.from(el.querySelectorAll ? el.querySelectorAll("input[type=file]") : []);
+ if (control) el = control;
+ else if (nested && nested.length === 1) el = nested[0];
+ }
+ return el;
+ })()`;
+}
+
+const UPLOAD_TARGET_INFO_FN = `function() {
+ const isFile = this.tagName === "INPUT" && String(this.type).toLowerCase() === "file";
+ if (!isFile) this.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
+ const r = this.getBoundingClientRect();
+ let x = r.left + r.width / 2, y = r.top + r.height / 2;
+ let win = this.ownerDocument.defaultView;
+ // Convert iframe-relative coordinates to top-level viewport coordinates (same-origin frames only).
+ while (win && win.frameElement) {
+ const frame = win.frameElement;
+ const fr = frame.getBoundingClientRect();
+ const cs = frame.ownerDocument.defaultView.getComputedStyle(frame);
+ x += fr.left + (parseFloat(cs.borderLeftWidth) || 0) + (parseFloat(cs.paddingLeft) || 0);
+ y += fr.top + (parseFloat(cs.borderTopWidth) || 0) + (parseFloat(cs.paddingTop) || 0);
+ win = win.parent === win ? null : win.parent;
+ }
+ return { tag: this.tagName, isFile, multiple: isFile ? this.multiple === true : undefined, inFrame: this.ownerDocument !== document, x, y, width: r.width, height: r.height };
+}`;
+
async function chromeInputUpload(params) {
const tab = await getTabByParams(params);
await bringToFront(tab, params);
@@ -1268,36 +1442,80 @@ async function chromeInputUpload(params) {
if (!(params.selector || params.uid)) throw new Error("chrome.upload: selector or uid required");
const paths = Array.isArray(params.paths) ? params.paths.map(String) : [];
if (!paths.length) throw new Error("chrome.upload: no file paths provided");
- const expression = `(() => {
- const selector = ${JSON.stringify(params.selector ?? null)};
- const uid = ${JSON.stringify(params.uid ?? null)};
- const state = window.__PI_CHROME_STATE__;
- const el = uid ? state?.elements?.[uid] : (selector ? document.querySelector(selector) : null);
- if (uid && (!el || !el.isConnected)) throw new Error("snapshot uid " + uid + " is stale; refresh chrome_snapshot");
- if (!el || el.tagName !== "INPUT" || el.type !== "file") throw new Error("Target must be <input type=file>");
- el.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
- return el;
- })()`;
- const evaluated = await cdp(tab.id, "Runtime.evaluate", { expression, objectGroup: "pi-chrome-upload", includeCommandLineAPI: false, returnByValue: false });
- if (evaluated.exceptionDetails) throw new Error(cdpExceptionText(evaluated.exceptionDetails) || "Could not resolve file input");
+ const evaluated = await cdp(tab.id, "Runtime.evaluate", { expression: uploadTargetExpression(params.selector, params.uid), objectGroup: "pi-chrome-upload", includeCommandLineAPI: false, returnByValue: false });
+ if (evaluated.exceptionDetails) throw new Error(cdpExceptionText(evaluated.exceptionDetails) || "Could not resolve upload target");
const objectId = evaluated.result?.objectId;
- if (!objectId) throw new Error("Could not resolve file input object");
+ if (!objectId) throw new Error("Could not resolve upload target object");
try {
- await cdp(tab.id, "DOM.enable", {}).catch(() => undefined);
- // Some DOM agents return nodeId:0 (or reject conversion) for a valid remote
- // element. CDP accepts that same objectId directly, before any file mutation.
- const requested = await cdp(tab.id, "DOM.requestNode", { objectId }).catch(() => null);
- const target = requested?.nodeId ? { nodeId: requested.nodeId } : { objectId };
- await cdp(tab.id, "DOM.setFileInputFiles", { ...target, files: paths });
- await cdp(tab.id, "Runtime.callFunctionOn", {
- objectId,
- functionDeclaration: `function() { this.dispatchEvent(new Event("input", { bubbles: true })); this.dispatchEvent(new Event("change", { bubbles: true })); return this.files ? this.files.length : 0; }`,
- returnByValue: true,
- }).catch(() => undefined);
+ const infoResult = await cdp(tab.id, "Runtime.callFunctionOn", { objectId, functionDeclaration: UPLOAD_TARGET_INFO_FN, returnByValue: true });
+ const info = infoResult?.result?.value || {};
+ if (info.isFile) {
+ if (paths.length > 1 && info.multiple === false) throw new Error(`chrome.upload: the file input accepts one file but ${paths.length} paths were given`);
+ await cdp(tab.id, "DOM.enable", {}).catch(() => undefined);
+ // Some DOM agents return nodeId:0 (or reject conversion) for a valid remote
+ // element. CDP accepts that same objectId directly, before any file mutation.
+ const requested = await cdp(tab.id, "DOM.requestNode", { objectId }).catch(() => null);
+ const target = requested?.nodeId ? { nodeId: requested.nodeId } : { objectId };
+ // Chrome's setFileInputFiles normally fires input/change itself. Watch for that and only
+ // dispatch our own events when it did not, so apps never see a duplicate change (double upload).
+ await cdp(tab.id, "Runtime.callFunctionOn", {
+ objectId,
+ functionDeclaration: `function() { const el = this; el.__piChromeChangeSeen = false; el.addEventListener("change", () => { el.__piChromeChangeSeen = true; }, { once: true, capture: true }); }`,
+ }).catch(() => undefined);
+ await cdp(tab.id, "DOM.setFileInputFiles", { ...target, files: paths });
+ const notified = await cdp(tab.id, "Runtime.callFunctionOn", {
+ objectId,
+ functionDeclaration: `function() { const seen = this.__piChromeChangeSeen === true; delete this.__piChromeChangeSeen; if (seen) return "native"; this.dispatchEvent(new Event("input", { bubbles: true })); this.dispatchEvent(new Event("change", { bubbles: true })); return "dispatched"; }`,
+ returnByValue: true,
+ }).catch(() => null);
+ const events = notified?.result?.value;
+ return { input: "chrome", mode: "file-input", events: typeof events === "string" ? events : undefined, inFrame: info.inFrame || undefined, uploaded: paths.map((path) => ({ path })) };
+ }
+ return await uploadViaFileChooser(tab, params, paths, objectId, info);
} finally {
await cdp(tab.id, "Runtime.releaseObject", { objectId }).catch(() => undefined);
}
- return { input: "chrome", uploaded: paths.map((path) => ({ path })) };
+}
+
+// The target opens the native picker when clicked. Intercept the chooser so no OS dialog appears,
+// then attach the files to the input Chrome reports. The click is trusted Chrome input when the page
+// is visible; a hidden page drops trusted input, so there we call element.click() with a CDP user
+// gesture instead (an untrusted event, like chrome_click's DOM fallback) unless domFallback:false.
+async function uploadViaFileChooser(tab, params, paths, objectId, info) {
+ const hidden = (await pageVisibilityState(tab.id)) === "hidden";
+ if (hidden && params.domFallback === false) await assertTabVisibleForInput(tab, params, "chrome.upload");
+ await cdp(tab.id, "Page.enable", {});
+ await cdp(tab.id, "Page.setInterceptFileChooserDialog", { enabled: true });
+ const chooser = waitForFileChooser(tab.id, FILE_CHOOSER_TIMEOUT_MS);
+ let trigger;
+ try {
+ if (!hidden && typeof info.x === "number" && typeof info.y === "number" && info.width > 0 && info.height > 0) {
+ trigger = "chrome";
+ await cdpMoveTo(tab.id, info.x, info.y);
+ await cdp(tab.id, "Input.dispatchMouseEvent", { type: "mousePressed", x: info.x, y: info.y, button: "left", buttons: 1, clickCount: 1, pointerType: "mouse" });
+ await sleep(rng(40, 110));
+ await cdp(tab.id, "Input.dispatchMouseEvent", { type: "mouseReleased", x: info.x, y: info.y, button: "left", buttons: 0, clickCount: 1, pointerType: "mouse" });
+ } else {
+ trigger = "activation-click";
+ await cdp(tab.id, "Runtime.callFunctionOn", { objectId, functionDeclaration: "function() { this.click(); }", userGesture: true, awaitPromise: false });
+ }
+ const opened = await chooser.promise;
+ if (typeof opened.backendNodeId !== "number") throw new Error("chrome.upload: file chooser opened without an input node");
+ if (paths.length > 1 && opened.mode === "selectSingle") throw new Error(`chrome.upload: the file chooser accepts one file but ${paths.length} paths were given`);
+ await cdp(tab.id, "DOM.enable", {}).catch(() => undefined);
+ await cdp(tab.id, "DOM.setFileInputFiles", { backendNodeId: opened.backendNodeId, files: paths });
+ return {
+ input: "chrome",
+ mode: "file-chooser",
+ trigger,
+ reason: trigger === "activation-click" ? "target tab is hidden; Chrome drops trusted input to hidden pages" : undefined,
+ inFrame: info.inFrame || undefined,
+ uploaded: paths.map((path) => ({ path })),
+ };
+ } finally {
+ chooser.cancel();
+ await cdp(tab.id, "Page.setInterceptFileChooserDialog", { enabled: false }).catch(() => undefined);
+ }
}
// ===============================================================
diff --git a/extensions/chrome-profile-bridge/index.ts b/extensions/chrome-profile-bridge/index.ts
index 46be3d9..e9e21e9 100644
--- a/extensions/chrome-profile-bridge/index.ts
+++ b/extensions/chrome-profile-bridge/index.ts
@@ -1,7 +1,7 @@
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
import { Type } from "typebox";
import { existsSync, readFileSync, statSync } from "node:fs";
-import { mkdir, writeFile } from "node:fs/promises";
+import { mkdir, readdir, unlink, writeFile } from "node:fs/promises";
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
import { dirname, join, resolve } from "node:path";
@@ -236,6 +236,58 @@ function formatCdpResult(method: string, value: unknown): ToolTextResult {
return { content: [{ type: "text", text: truncateText(text) }], details: { value } };
}
+// chrome_screenshot writes a timestamped file per capture (plus -tileN files and a .json manifest for
+// full-page captures) into .pi/chrome-screenshots. Prune that folder at capture time so it stays
+// bounded. Only files matching the tool's own naming are eligible; the newest `keep` captures are always
+// kept, and nothing inside the retention window is removed. Best-effort: errors never fail a capture.
+const SCREENSHOT_NAME_RE = /^(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z)(?:-tile\d+)?\.(?:png|jpeg)(?:\.json)?$/;
+const SCREENSHOT_KEEP_NEWEST = 20;
+const SCREENSHOT_RETENTION_DAYS = 7;
+
+function screenshotCaptureTime(stamp: string): number {
+ const iso = stamp.replace(/^(\d{4}-\d{2}-\d{2}T\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, "$1:$2:$3.$4Z");
+ return Date.parse(iso);
+}
+
+function screenshotFilesToPrune(names: string[], now: number, retentionDays = SCREENSHOT_RETENTION_DAYS, keep = SCREENSHOT_KEEP_NEWEST): string[] {
+ if (!(retentionDays > 0)) return [];
+ const captures = new Map<string, string[]>();
+ for (const name of names) {
+ const match = SCREENSHOT_NAME_RE.exec(name);
+ if (!match) continue;
+ const files = captures.get(match[1]) ?? [];
+ files.push(name);
+ captures.set(match[1], files);
+ }
+ const cutoff = now - retentionDays * 24 * 60 * 60 * 1000;
+ const newestFirst = [...captures.keys()].sort().reverse();
+ const prune: string[] = [];
+ for (const stamp of newestFirst.slice(keep)) {
+ const takenAt = screenshotCaptureTime(stamp);
+ if (Number.isFinite(takenAt) && takenAt < cutoff) prune.push(...(captures.get(stamp) ?? []));
+ }
+ return prune;
+}
+
+async function pruneScreenshotDir(dir: string, retentionDays?: number): Promise<number> {
+ try {
+ const names = await readdir(dir);
+ let removed = 0;
+ for (const name of screenshotFilesToPrune(names, Date.now(), retentionDays)) {
+ try {
+ await unlink(join(dir, name));
+ removed++;
+ } catch {
+ // Locked or already gone: keep going.
+ }
+ }
+ return removed;
+ } catch {
+ return 0;
+ }
+}
+// End screenshot pruning.
+
function formatChromeInspect(inspect: any): string {
if (!inspect || typeof inspect !== "object") return safeJson(inspect);
const t = inspect.target || {};
@@ -1066,7 +1118,8 @@ Capability model (important):
- Input bypasses page CSP because it is injected at browser input layer, not page JavaScript. Chrome may show the “Pi Chrome Connector started debugging this browser” banner while attached.
- \`chrome_evaluate\` and \`chrome_snapshot\` run in MAIN world via **CDP \`Runtime.evaluate\`**, which is not subject to the page's Content-Security-Policy. They work even on strict-CSP pages (e.g. github.com, many bank/SaaS apps) that block \`'unsafe-eval'\`. \`chrome_navigate initScript\` likewise injects at document_start via CDP and bypasses CSP. \`chrome_screenshot\`, \`chrome_tab\`, and Chrome input also work under any CSP.
- Input tools return structured details and support \`includeSnapshot=true\` on click/type/fill/key. Use the fresh snapshot to verify state instead of repeating blindly. If the action started a navigation, the snapshot waits (up to 5s) for the new page.
-- \`chrome_type\` inserts at the caret and never replaces; check its before/after report. Use \`chrome_fill\` to replace a field.
+- \`chrome_type\` never replaces: with a uid/selector it appends at the end of the field; without one it types at the current caret. Check its before/after report. Use \`chrome_fill\` to replace a field.
+- Chrome ignores real input to hidden pages (inactive tabs, minimized windows, windows behind a macOS full-screen window). Input tools report this instead of silently doing nothing; retry with background:false only when bringing the tab forward is acceptable.
- \`chrome_cdp\` runs any raw CDP method on a tab (emulation, cookies, PDF, accessibility tree, etc.) when no dedicated chrome_* tool fits; \`chrome_cdp_targets\` diagnoses debugger/overlay conflicts.
Usage rules:
@@ -1630,7 +1683,7 @@ Usage rules:
selector: Type.Optional(Type.String({ description: "CSS selector to click. Prefer uid from chrome_snapshot when available." })),
x: Type.Optional(Type.Number({ description: "Viewport x coordinate if uid/selector is omitted." })),
y: Type.Optional(Type.Number({ description: "Viewport y coordinate if uid/selector is omitted." })),
- domFallback: Type.Optional(Type.Boolean({ description: "If true (default), fall back to DOM-dispatched click if Chrome's CDP input path is blocked by another extension overlay or debugger failure." })),
+ domFallback: Type.Optional(Type.Boolean({ description: "If true (default), fall back to DOM-dispatched click if Chrome's CDP input path is blocked (extension overlay, debugger failure, or a hidden tab that ignores trusted input)." })),
includeSnapshot: Type.Optional(Type.Boolean({ description: "If true, include a fresh chrome_snapshot result after the click." })),
maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS, description: "Max elements in the included snapshot." })),
targetId: Type.Optional(Type.String()),
@@ -1654,8 +1707,8 @@ Usage rules:
name: "chrome_type",
label: "Chrome Type",
description:
- "Focus an optional snapshot uid or CSS selector, then type at the caret using Chrome's real input. It does NOT replace existing text: in a non-empty field the text is inserted wherever the caret lands after the focus click. Use chrome_fill (or replace=true) to replace a field's value. Contenteditables use one native text insertion; other fields use key events. Set perCharacter=true for editors needing individual keydown events. The result reports the field value before/after and insertedAt (empty|caret-end|caret-middle|replaced-selection|replaced-all), with a warning when text was spliced into existing content. Pass includeSnapshot=true to verify after typing; if the action starts a navigation, the snapshot waits (up to 5s) for it to load.",
- promptSnippet: "Type text at the caret in Chrome (does not replace existing text; use chrome_fill or replace=true).",
+ "Type using Chrome's real input. With a snapshot uid or CSS selector, chrome_type clicks the field and moves the caret to the end, so text is appended to existing content; without one it types at the current caret of the focused element. It does NOT replace existing text. Use chrome_fill (or replace=true) to replace a field's value. Contenteditables use one native text insertion; other fields use key events. Set perCharacter=true for editors needing individual keydown events. The result reports the field value before/after and insertedAt (empty|caret-end|caret-middle|replaced-selection|replaced-all), with a warning when text was spliced into existing content. Pass includeSnapshot=true to verify after typing; if the action starts a navigation, the snapshot waits (up to 5s) for it to load.",
+ promptSnippet: "Type text in Chrome; a uid/selector target appends at the end (does not replace existing text; use chrome_fill or replace=true).",
parameters: Type.Object({
text: Type.String(),
uid: Type.Optional(Type.String({ description: "Stable element uid from chrome_snapshot." })),
@@ -1695,7 +1748,7 @@ Usage rules:
selector: Type.Optional(Type.String({ description: "CSS selector to fill if uid is omitted." })),
perCharacter: Type.Optional(Type.Boolean({ default: false, description: "Send individual key events even in contenteditables. Default: one native text insertion for contenteditables; key events for other fields." })),
submit: Type.Optional(Type.Boolean({ description: "If true, press Enter after filling." })),
- domFallback: Type.Optional(Type.Boolean({ description: "If true (default), fall back to DOM value-setting if Chrome's CDP input path is blocked by another extension overlay or debugger failure." })),
+ domFallback: Type.Optional(Type.Boolean({ description: "If true (default), fall back to DOM value-setting if Chrome's CDP input path is blocked (extension overlay, debugger failure, or a hidden tab that ignores trusted input)." })),
includeSnapshot: Type.Optional(Type.Boolean({ description: "If true, include a fresh chrome_snapshot result after filling." })),
maxElements: Type.Optional(Type.Number({ default: MAX_ELEMENTS, description: "Max elements in the included snapshot." })),
targetId: Type.Optional(Type.String()),
@@ -1845,6 +1898,7 @@ Usage rules:
format: Type.Optional(StringEnum(imageFormatValues)),
quality: Type.Optional(Type.Number({ minimum: 0, maximum: 100, description: "JPEG quality 0-100." })),
fullPage: Type.Optional(Type.Boolean({ description: "Capture full-page tiles plus a JSON manifest. Temporarily scrolls the target page; does not activate background tabs." })),
+ retentionDays: Type.Optional(Type.Number({ minimum: 0, description: "Default-folder captures older than this many days are pruned (the newest 20 are always kept). Default 7; 0 disables pruning. Explicit paths are never pruned." })),
targetId: Type.Optional(Type.String()),
urlIncludes: Type.Optional(Type.String()),
titleIncludes: Type.Optional(Type.String()),
@@ -1855,8 +1909,10 @@ Usage rules:
async execute(_id, params, signal, _onUpdate, ctx: ExtensionContext): Promise<ToolTextResult> {
const format = params.format ?? "png";
const cwd = workspaceCwd(ctx);
- const defaultPath = join(cwd, ".pi", "chrome-screenshots", `${new Date().toISOString().replace(/[:.]/g, "-")}.${format}`);
+ const screenshotDir = join(cwd, ".pi", "chrome-screenshots");
+ const defaultPath = join(screenshotDir, `${new Date().toISOString().replace(/[:.]/g, "-")}.${format}`);
const outputPath = params.path ? resolve(cwd, params.path) : defaultPath;
+ const prune = () => (params.path ? Promise.resolve(0) : pruneScreenshotDir(screenshotDir, params.retentionDays));
const result = (await authorizedBridgeSend("page.screenshot", params, params.fullPage ? 120_000 : DEFAULT_TIMEOUT_MS, signal)) as {
dataUrl?: string;
method?: string;
@@ -1880,6 +1936,7 @@ Usage rules:
manifest.push({ path: tilePath, y: tile.y });
}
await writeFile(outputPath + ".json", JSON.stringify({ width, height, viewportHeight, dpr, tiles: manifest }, null, 2));
+ await prune();
return {
content: [{ type: "text", text: `Saved ${result.tiles.length} full-page tile(s) for ${width}×${height}px page. Manifest: ${outputPath}.json` }],
details: { manifest: outputPath + ".json", tiles: manifest, dimensions: result.dimensions, tab: result.tab, method: result.method } as unknown as Record<string, unknown>,
@@ -1888,6 +1945,7 @@ Usage rules:
if (!result.dataUrl) throw new Error("Screenshot returned no dataUrl");
const base64 = result.dataUrl.replace(/^data:image\/(?:png|jpeg);base64,/, "");
await writeFile(outputPath, Buffer.from(base64, "base64"));
+ await prune();
return { content: [{ type: "text", text: `Saved Chrome screenshot to ${outputPath}` }], details: { path: outputPath, format, tab: result.tab, method: result.method } };
},
});
@@ -1987,12 +2045,13 @@ Usage rules:
pi.registerTool({
name: "chrome_upload_file",
label: "Chrome Upload File",
- description: "Attach local files to an <input type=file> element using Chrome DevTools file-input control. Does NOT open the native file picker; works with React/Vue/Angular controlled inputs.",
+ description: "Attach local files using Chrome DevTools file-input control without opening a native file picker. Target the <input type=file>, a label/wrapper containing one, or an upload button that opens the picker (the chooser is intercepted). Selectors also search same-origin iframes. Works with React/Vue/Angular controlled inputs.",
promptSnippet: "Attach local files to a Chrome <input type=file> without opening the native file picker.",
parameters: Type.Object({
uid: Type.Optional(Type.String()),
selector: Type.Optional(Type.String()),
paths: Type.Array(Type.String(), { description: "Local absolute file paths to upload." }),
+ domFallback: Type.Optional(Type.Boolean({ description: "Upload buttons need a click to open the (intercepted) file chooser. If true (default) and the tab is hidden, use a page-level element.click() with a user gesture instead of trusted input. Set false to require trusted input." })),
targetId: Type.Optional(Type.String()),
urlIncludes: Type.Optional(Type.String()),
titleIncludes: Type.Optional(Type.String()),
@@ -2001,8 +2060,12 @@ Usage rules:
async execute(_id, params, signal, _onUpdate, ctx): Promise<ToolTextResult> {
const cwd = workspaceCwd(ctx);
const paths = params.paths.map((p) => resolve(cwd, p));
- const result = await authorizedBridgeSend("page.upload", { ...params, paths }, DEFAULT_TIMEOUT_MS, signal);
- return { content: [{ type: "text", text: `Uploaded ${paths.length} file(s) to ${params.uid ?? params.selector}` }], details: { result: result as Json } };
+ const result = (await authorizedBridgeSend("page.upload", { ...params, paths }, DEFAULT_TIMEOUT_MS, signal)) as { mode?: string; trigger?: string; inFrame?: boolean } | undefined;
+ const via = result?.mode === "file-chooser"
+ ? ` via intercepted file chooser (${result.trigger === "activation-click" ? "page-level click because the tab is hidden" : "trusted click"})`
+ : "";
+ const frame = result?.inFrame ? " inside an iframe" : "";
+ return { content: [{ type: "text", text: `Uploaded ${paths.length} file(s) to ${params.uid ?? params.selector}${frame}${via}` }], details: { result: result as Json } };
},
});