From 495de0d5283dd3e4a6ef715b596c4a2892e95915 Mon Sep 17 00:00:00 2001 From: Igor Soarez Date: Mon, 3 Aug 2026 21:18:02 +0100 Subject: Web search for pi through an existing Chrome over CDP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Attaches to a browser that is already running — never launches one — using the same endpoint configuration as pi-browser-harness, so a single /browser-target choice governs both packages. One tool, castle_cdp_search, deliberately not named web_search so it coexists with pi-web-access rather than shadowing it. Notes from validating against castle's Chrome: - tbs=qdr:*, the parameter Google's own Tools menu writes, renders an empty page on this profile; the older as_qdr=* works. Any date filter combined with udm=14 is also empty, so recency drops udm. - Target.createTarget must not be raced against the abort signal: raceAbort abandons the promise but cannot cancel the command, and the command's side effect is a tab nothing is left holding. - A search cancelled while queued has to be removed from the semaphore queue, or the slot handed to it later is never counted back. - A decaying rate-limit block stops serving /sorry/ and returns an empty results page instead, indistinguishable from a genuine zero-hit search. --- src/cdp.ts | 342 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/endpoint.ts | 108 +++++++++++++++++ src/errors.ts | 56 +++++++++ src/extract.ts | 163 +++++++++++++++++++++++++ src/format.ts | 46 +++++++ src/index.ts | 198 ++++++++++++++++++++++++++++++ src/paths.ts | 41 +++++++ src/search.ts | 279 ++++++++++++++++++++++++++++++++++++++++++ src/target-store.ts | 42 +++++++ 9 files changed, 1275 insertions(+) create mode 100644 src/cdp.ts create mode 100644 src/endpoint.ts create mode 100644 src/errors.ts create mode 100644 src/extract.ts create mode 100644 src/format.ts create mode 100644 src/index.ts create mode 100644 src/paths.ts create mode 100644 src/search.ts create mode 100644 src/target-store.ts (limited to 'src') diff --git a/src/cdp.ts b/src/cdp.ts new file mode 100644 index 0000000..aa01b56 --- /dev/null +++ b/src/cdp.ts @@ -0,0 +1,342 @@ +/** + * A minimal CDP client — just enough to open a page, navigate it, read the DOM, + * and close it again. + * + * Deliberately dependency-free: Node 22 ships a global `WebSocket`, so this + * needs no `ws`. That matters because an extension installed by `pi install` + * gets its own `node_modules`, and fewer moving parts there is fewer things to + * go wrong on a machine that is not the one it was written on. + * + * The browser is *not* ours. It is a shared Chrome someone may be looking at, + * recycled hourly (`me.soarez.chrome-cdp-recycle`). Two consequences run + * through this file: + * + * 1. The websocket URL is re-resolved from `/json/version` on every connect, + * never cached. Chrome re-mints its browser UUID on each launch, so a + * pinned ws:// URL breaks the first time the browser is recycled. + * 2. Every page this opens is created in the background and closed again, + * including on error paths, so a failed search does not leave tabs behind + * in someone else's window. + */ + +import { BrowserUnavailableError, SearchAbortedError, messageOf } from "./errors.ts"; +import type { CdpTarget } from "./endpoint.ts"; + +/** How long a single CDP command may take before the socket is presumed dead. */ +const COMMAND_TIMEOUT_MS = 15_000; +/** How long to wait for /json/version and for the websocket handshake. */ +const CONNECT_TIMEOUT_MS = 5_000; + +type CdpMessage = { + id?: number; + method?: string; + params?: Record; + sessionId?: string; + result?: Record; + error?: { code?: number; message?: string }; +}; + +type Pending = { + resolve: (value: Record) => void; + reject: (reason: Error) => void; + timer: ReturnType; +}; + +/** + * Reject if `signal` fires before `promise` settles, cleaning up the listener + * either way. Node's own abort-aware APIs are not available here — the + * websocket predates them — so aborts are raced in rather than plumbed through. + */ +export const raceAbort = (promise: Promise, signal: AbortSignal | undefined): Promise => { + if (!signal) return promise; + if (signal.aborted) return Promise.reject(abortError(signal)); + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(abortError(signal)); + signal.addEventListener("abort", onAbort, { once: true }); + promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort)); + }); +}; + +const abortError = (signal: AbortSignal): Error => { + const reason: unknown = signal.reason; + if (reason instanceof Error && reason.name === "TimeoutError") { + return new SearchAbortedError("search timed out"); + } + return new SearchAbortedError("search was cancelled"); +}; + +/** Ask the browser for its current websocket endpoint. Never cached — see above. */ +const queryWebSocketUrl = async (target: CdpTarget, signal?: AbortSignal): Promise => { + const url = `http://${target.host}:${target.port}/json/version`; + let payload: unknown; + try { + const res = await fetch(url, { + signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(CONNECT_TIMEOUT_MS)]) : AbortSignal.timeout(CONNECT_TIMEOUT_MS), + }); + if (!res.ok) { + throw new BrowserUnavailableError(`${url} answered HTTP ${res.status}`); + } + payload = await res.json(); + } catch (e) { + if (e instanceof BrowserUnavailableError) throw e; + if (signal?.aborted) throw abortError(signal); + throw new BrowserUnavailableError( + `No CDP endpoint answered at ${url} (${messageOf(e)}). Is the browser running and reachable on the overlay?`, + ); + } + const ws = (payload as Record | null)?.["webSocketDebuggerUrl"]; + if (typeof ws !== "string" || ws === "") { + throw new BrowserUnavailableError(`${url} answered without a webSocketDebuggerUrl`); + } + return ws; +}; + +/** A live browser-level CDP connection. */ +export class BrowserConnection { + #ws: WebSocket; + #nextId = 1; + #pending = new Map(); + #listeners = new Set<(msg: CdpMessage) => void>(); + #closedReason: string | null = null; + + readonly target: CdpTarget; + + private constructor(ws: WebSocket, target: CdpTarget) { + this.#ws = ws; + this.target = target; + + ws.addEventListener("message", (ev: MessageEvent) => { + let msg: CdpMessage; + try { + msg = JSON.parse(String(ev.data)) as CdpMessage; + } catch { + return; + } + if (msg.id !== undefined) { + const pending = this.#pending.get(msg.id); + if (!pending) return; + this.#pending.delete(msg.id); + clearTimeout(pending.timer); + if (msg.error) { + pending.reject(new BrowserUnavailableError(`CDP error: ${msg.error.message ?? "unknown"}`)); + } else { + pending.resolve(msg.result ?? {}); + } + return; + } + for (const listener of [...this.#listeners]) listener(msg); + }); + + const fail = (reason: string): void => { + this.#closedReason ??= reason; + for (const [, pending] of this.#pending) { + clearTimeout(pending.timer); + pending.reject(new BrowserUnavailableError(reason)); + } + this.#pending.clear(); + }; + ws.addEventListener("close", () => fail("CDP connection closed (the browser may have been recycled)")); + ws.addEventListener("error", () => fail("CDP connection failed")); + } + + /** Resolve the endpoint afresh and dial it. */ + static async connect(target: CdpTarget, signal?: AbortSignal): Promise { + const wsUrl = await queryWebSocketUrl(target, signal); + const ws = new WebSocket(wsUrl); + try { + await raceAbort( + new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new BrowserUnavailableError(`timed out opening ${wsUrl}`)), + CONNECT_TIMEOUT_MS, + ); + ws.addEventListener( + "open", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + ws.addEventListener( + "error", + () => { + clearTimeout(timer); + reject(new BrowserUnavailableError(`could not open ${wsUrl}`)); + }, + { once: true }, + ); + }), + signal, + ); + } catch (e) { + try { + ws.close(); + } catch { + // Already failed; nothing useful to do with a second failure. + } + throw e; + } + return new BrowserConnection(ws, target); + } + + /** False once the socket has closed or failed — the reconnect trigger. */ + get isOpen(): boolean { + return this.#closedReason === null && this.#ws.readyState === 1 /* OPEN */; + } + + send(method: string, params: Record = {}, sessionId?: string): Promise> { + if (!this.isOpen) { + return Promise.reject(new BrowserUnavailableError(this.#closedReason ?? "CDP connection is not open")); + } + const id = this.#nextId++; + const payload: CdpMessage = sessionId ? { id, method, params, sessionId } : { id, method, params }; + return new Promise>((resolve, reject) => { + const timer = setTimeout(() => { + this.#pending.delete(id); + reject(new BrowserUnavailableError(`CDP command ${method} timed out after ${COMMAND_TIMEOUT_MS}ms`)); + }, COMMAND_TIMEOUT_MS); + this.#pending.set(id, { resolve, reject, timer }); + try { + this.#ws.send(JSON.stringify(payload)); + } catch (e) { + this.#pending.delete(id); + clearTimeout(timer); + reject(new BrowserUnavailableError(`could not send ${method}: ${messageOf(e)}`)); + } + }); + } + + /** Subscribe to unsolicited protocol events. Returns an unsubscribe function. */ + onEvent(listener: (msg: CdpMessage) => void): () => void { + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + } + + close(): void { + this.#closedReason ??= "CDP connection closed by this extension"; + this.#listeners.clear(); + try { + this.#ws.close(); + } catch { + // Closing an already-dead socket is not a failure worth reporting. + } + } +} + +/** + * One throwaway page. + * + * Created in the background so it does not steal focus from whoever is using + * the browser, and closed by {@link close} on every path. + */ +export class PageSession { + readonly #conn: BrowserConnection; + readonly #targetId: string; + readonly #sessionId: string; + #closed = false; + + private constructor(conn: BrowserConnection, targetId: string, sessionId: string) { + this.#conn = conn; + this.#targetId = targetId; + this.#sessionId = sessionId; + } + + static async open(conn: BrowserConnection, signal?: AbortSignal): Promise { + if (signal?.aborted) throw abortError(signal); + + // Deliberately NOT raced against the abort signal. raceAbort abandons the + // promise, it cannot cancel the command — and this command's side effect is + // a tab. Abandoning it would leave a tab open in a browser someone else is + // using, with nothing left holding its targetId to close it. It is a fast + // command already bounded by COMMAND_TIMEOUT_MS, so waiting it out costs + // little and guarantees we own whatever it created. + const created = await conn.send("Target.createTarget", { url: "about:blank", background: true }); + const targetId = created["targetId"]; + if (typeof targetId !== "string") { + throw new BrowserUnavailableError("Target.createTarget did not return a targetId"); + } + + // From here on the page exists, so every failure — including a late abort — + // must still close it. + try { + if (signal?.aborted) throw abortError(signal); + const attached = await raceAbort( + conn.send("Target.attachToTarget", { targetId, flatten: true }), + signal, + ); + const sessionId = attached["sessionId"]; + if (typeof sessionId !== "string") { + throw new BrowserUnavailableError("Target.attachToTarget did not return a sessionId"); + } + const page = new PageSession(conn, targetId, sessionId); + await raceAbort(conn.send("Page.enable", {}, sessionId), signal); + return page; + } catch (e) { + await conn.send("Target.closeTarget", { targetId }).catch(() => {}); + throw e; + } + } + + /** Navigate and wait for the load event, or for `signal` to fire. */ + async navigate(url: string, signal?: AbortSignal): Promise { + // Subscribed before navigating, so a page that loads faster than the + // Page.navigate reply cannot slip its load event past us. + let off: () => void = () => {}; + const loaded = new Promise((resolve) => { + off = this.#conn.onEvent((msg) => { + if (msg.sessionId === this.#sessionId && msg.method === "Page.loadEventFired") resolve(); + }); + }); + + try { + const nav = await raceAbort(this.#conn.send("Page.navigate", { url }, this.#sessionId), signal); + const errorText = nav["errorText"]; + if (typeof errorText === "string" && errorText !== "") { + throw new BrowserUnavailableError(`navigation to ${url} failed: ${errorText}`); + } + + try { + await raceAbort(loaded, signal); + } catch (e) { + // Stop the in-flight load so the page is not still fetching while we + // tear it down; the tab is closed either way, this just makes it quicker. + await this.#conn.send("Page.stopLoading", {}, this.#sessionId).catch(() => {}); + throw e; + } + } finally { + // Unsubscribing here rather than inside the listener: on the failure paths + // the listener never fires, and a subscription left on a connection that + // outlives many searches accumulates. + off(); + } + } + + /** Evaluate an expression in the page and return its value by value. */ + async evaluate(expression: string, signal?: AbortSignal): Promise { + const res = await raceAbort( + this.#conn.send( + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise: true }, + this.#sessionId, + ), + signal, + ); + const exception = res["exceptionDetails"] as { text?: string; exception?: { description?: string } } | undefined; + if (exception) { + const text = exception.exception?.description ?? exception.text ?? "unknown error"; + throw new BrowserUnavailableError(`page script failed: ${text}`); + } + return (res["result"] as { value?: T } | undefined)?.value as T; + } + + /** + * Close the tab. Idempotent and never throws: it runs on error paths, where a + * second failure would mask the one worth reporting. + */ + async close(): Promise { + if (this.#closed) return; + this.#closed = true; + await this.#conn.send("Target.closeTarget", { targetId: this.#targetId }).catch(() => {}); + } +} diff --git a/src/endpoint.ts b/src/endpoint.ts new file mode 100644 index 0000000..d3f4b0d --- /dev/null +++ b/src/endpoint.ts @@ -0,0 +1,108 @@ +/** + * Which browser this extension searches through. + * + * Deliberately the *same* configuration as pi-browser-harness, so one decision + * governs both packages and they never drive different browsers in the same + * session: `BU_CDP_HTTP`, then the `/browser-target` file that harness writes, + * then the built-in castle default. + * + * Resolution is pure — no I/O — so it can be unit-tested and reported without + * touching the network. + */ + +/** Default CDP endpoint: castle's Chrome, exposed on the wg8 overlay. */ +export const DEFAULT_REMOTE_CDP = "10.88.0.25:9223"; + +/** Values of BU_CDP_HTTP that mean "ignore the default, use this machine". */ +const LOCAL_ALIASES = new Set(["local", "localhost", "off", "none", "0", "no"]); + +/** Port a local Chrome is expected to serve CDP on when the target is "local". */ +export const LOCAL_CDP_PORT = 9222; + +/** + * Where a target came from. "env" is BU_CDP_HTTP, "stored" is the harness's + * `/browser-target`, "default" is the built-in. + */ +export type CdpTargetSource = "env" | "stored" | "default"; + +export type CdpTarget = { + readonly host: string; + readonly port: number; + readonly source: CdpTargetSource; +}; + +export type CdpTargetResolution = + | { readonly kind: "ok"; readonly target: CdpTarget } + | { readonly kind: "invalid"; readonly raw: string; readonly reason: string; readonly source: CdpTargetSource }; + +/** + * Precedence: BU_CDP_HTTP, then the stored `/browser-target` choice, then the + * built-in default. The environment wins so a one-off `BU_CDP_HTTP=… pi` is + * never silently overridden by a choice made days ago. + * + * Unlike pi-browser-harness this never falls back from the default to a local + * browser on its own. The harness can fall back because it drives a browser the + * operator is watching; here a silent fallback would mean *searching from the + * operator's own signed-in Chrome* without saying so. When the configured + * endpoint is unreachable the caller reports that, and the operator can say + * `local` explicitly. + * + * `stored` is passed in rather than read here so this stays pure and synchronous. + */ +export const resolveCdpTarget = ( + env: NodeJS.ProcessEnv = process.env, + stored: string | null = null, +): CdpTargetResolution => { + const fromEnv = env["BU_CDP_HTTP"]?.trim(); + const fromStore = stored?.trim(); + + const hasEnv = fromEnv !== undefined && fromEnv !== ""; + const raw = hasEnv ? fromEnv : fromStore; + const source: CdpTargetSource = hasEnv ? "env" : "stored"; + + if (raw === undefined || raw === "") { + const parsed = parseHostPort(DEFAULT_REMOTE_CDP); + // Unreachable in practice: the constant above is covered by its own test. + if (!parsed) { + return { kind: "invalid", raw: DEFAULT_REMOTE_CDP, source: "default", reason: "built-in default is malformed" }; + } + return { kind: "ok", target: { ...parsed, source: "default" } }; + } + + if (LOCAL_ALIASES.has(raw.toLowerCase())) { + return { kind: "ok", target: { host: "127.0.0.1", port: LOCAL_CDP_PORT, source } }; + } + + const parsed = parseHostPort(raw); + if (!parsed) { + return { + kind: "invalid", + raw, + source, + reason: `expected "host:port" or one of ${[...LOCAL_ALIASES].join(", ")}`, + }; + } + return { kind: "ok", target: { ...parsed, source } }; +}; + +const parseHostPort = (value: string): { host: string; port: number } | null => { + const sep = value.lastIndexOf(":"); + if (sep === -1) return null; + const host = value.slice(0, sep).trim(); + const port = Number(value.slice(sep + 1).trim()); + if (!host) return null; + if (!Number.isInteger(port) || port <= 0 || port >= 65536) return null; + return { host, port }; +}; + +const sourceLabel = (source: CdpTargetSource): string => + source === "env" ? "BU_CDP_HTTP" : source === "stored" ? "/browser-target" : "built-in default"; + +/** One-line description for progress messages and errors. */ +export const describeCdpTarget = (target: CdpTarget): string => + `${target.host}:${target.port} (${sourceLabel(target.source)})`; + +/** One-line description of a resolution failure. */ +export const describeResolutionFailure = ( + failure: Extract, +): string => `invalid CDP target "${failure.raw}" from ${sourceLabel(failure.source)} — ${failure.reason}`; diff --git a/src/errors.ts b/src/errors.ts new file mode 100644 index 0000000..d003e4f --- /dev/null +++ b/src/errors.ts @@ -0,0 +1,56 @@ +/** + * Failure vocabulary. + * + * Every one of these is *thrown*, never returned. pi only sets `isError: true` + * on a tool result when `execute()` throws — a returned object with an `error` + * field looks to the model exactly like a successful search that found nothing. + * + * `SearchChallengeError` is the one the agent is expected to act on rather than + * just report: it means a human has to touch the browser before search can work + * again, so its message says so in words the agent can pass to the operator. + */ + +/** The search engine served a captcha, consent wall, or other human check. */ +export class SearchChallengeError extends Error { + override readonly name = "SearchChallengeError"; + /** Machine-readable flavour of the challenge, for callers that branch on it. */ + readonly challenge: string; + /** The page the human needs to visit to clear it. */ + readonly pageUrl: string; + /** Which browser is blocked, e.g. "10.88.0.25:9223 (built-in default)". */ + readonly endpoint: string; + + constructor(options: { challenge: string; pageUrl: string; endpoint: string; detail?: string | undefined }) { + super( + [ + `Search is blocked by a human verification challenge (${options.challenge}).`, + options.detail ? `Page said: ${options.detail}` : null, + `This cannot be solved by the agent — a person has to clear it in the browser at ${options.endpoint}.`, + `Ask the user to open ${options.pageUrl} in that Chrome, complete the challenge or accept the consent dialog,`, + `then retry the search. The cookie it sets persists in that browser profile, so one pass unblocks later searches.`, + ] + .filter((line) => line !== null) + .join(" "), + ); + this.challenge = options.challenge; + this.pageUrl = options.pageUrl; + this.endpoint = options.endpoint; + } +} + +/** The browser could not be reached or spoke CDP badly. */ +export class BrowserUnavailableError extends Error { + override readonly name = "BrowserUnavailableError"; +} + +/** The search itself ran but produced nothing usable. */ +export class NoResultsError extends Error { + override readonly name = "NoResultsError"; +} + +/** The per-search deadline expired, or the user pressed Esc. */ +export class SearchAbortedError extends Error { + override readonly name = "SearchAbortedError"; +} + +export const messageOf = (e: unknown): string => (e instanceof Error ? e.message : String(e)); diff --git a/src/extract.ts b/src/extract.ts new file mode 100644 index 0000000..c4d263a --- /dev/null +++ b/src/extract.ts @@ -0,0 +1,163 @@ +/** + * The script that runs inside the search results page. + * + * It is a string rather than a function because it is shipped to Chrome via + * `Runtime.evaluate` — nothing in here is typechecked, so it is kept small, + * defensive, and free of anything that could throw on an unexpected DOM. + * + * ## Why it does not select on class names + * + * Google's result classes (`MjjYud`, `kb0PBd`, `yuRUbf`, …) are generated and + * change without notice; the `data-` hooks (`data-snhf` for the title/source + * header, `data-sncf` for the description) are more stable but not promised + * either. So the extractor uses them when present and otherwise falls back to a + * structural walk that only assumes "an

inside a link, with the + * description somewhere in a shared ancestor": + * + * - anchor = nearest enclosing + * - header = nearest [data-snhf], else the anchor itself + * - container = climb from the header until the text grows past the header's, + * abandoning the climb if a second

comes into scope + * (that would mean we had swallowed the next result) + * - snippet = [data-sncf] if present, else the container's lines minus the + * header's lines, minus URL/breadcrumb noise + * + * Verified against both the classic SERP and the `udm=14` "Web" layout. + */ + +/** One search hit, as the page script reports it. */ +export type SearchResult = { + title: string; + url: string; + snippet: string; +}; + +/** What the page script returns in a single round trip. */ +export type PageProbe = { + /** Where the page actually ended up — redirects and consent walls move it. */ + url: string; + title: string; + readyState: string; + /** Non-null when a human check is in the way. */ + challenge: { kind: string; detail: string } | null; + /** + * Which results container the page rendered. "none" means Google served a + * shell with no results area at all — a different failure from an empty one, + * and worth telling apart when diagnosing a zero-hit search. + */ + container: "rso" | "search" | "none"; + results: SearchResult[]; +}; + +/** + * Detection order matters: a `/sorry/` interstitial also contains a recaptcha + * iframe, and reporting the specific page beats reporting the widget on it. + */ +export const PROBE_SCRIPT = String.raw`(() => { + const norm = (s) => (s || "").replace(/\s+/g, " ").trim(); + const linesOf = (el) => + el && el.innerText ? el.innerText.split("\n").map((l) => l.trim()).filter(Boolean) : []; + + const bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : ""; + + const detectChallenge = () => { + const href = location.href; + if (/\/sorry\//.test(href)) { + return { kind: "google-block-page", detail: bodyText.slice(0, 300) }; + } + if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname)) { + return { kind: "consent-wall", detail: bodyText.slice(0, 300) }; + } + if (document.querySelector("#captcha-form, form#captcha-form")) { + return { kind: "captcha-form", detail: bodyText.slice(0, 300) }; + } + if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[title*="challenge"]')) { + return { kind: "captcha-widget", detail: bodyText.slice(0, 300) }; + } + if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running")) { + return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) }; + } + if (/unusual traffic|are you a robot|verify (that )?you('| a)?re human|not a robot|automated queries/i.test(bodyText)) { + return { kind: "bot-check", detail: bodyText.slice(0, 300) }; + } + return null; + }; + + const extract = () => { + const root = document.querySelector("#rso") || document.querySelector("#search"); + if (!root) return []; + const out = []; + const seen = new Set(); + + const headings = root.querySelectorAll("h3"); + for (let i = 0; i < headings.length; i++) { + const h3 = headings[i]; + const anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]")); + if (!anchor) continue; + + const url = anchor.href; + if (!/^https?:/.test(url)) continue; + // Drop Google's own links (image search, cached copies, "more results"). + if (/^https?:\/\/(www\.)?google\.[a-z.]+\//.test(url)) continue; + if (seen.has(url)) continue; + + const title = norm(h3.innerText); + if (!title) continue; + + const header = h3.closest("[data-snhf]") || anchor; + const headerLines = new Set(linesOf(header).map(norm)); + const baseline = norm(header.innerText || "").length; + + let container = header.parentElement; + for (let depth = 0; depth < 6 && container && container !== root; depth++) { + if (container.querySelectorAll("h3").length > 1) { + container = null; + break; + } + if (norm(container.innerText || "").length > baseline + 40) break; + container = container.parentElement; + } + + let snippet = ""; + if (container && container !== root) { + const explicit = container.querySelector("[data-sncf]"); + const body = explicit ? linesOf(explicit) : linesOf(container).filter((l) => !headerLines.has(norm(l))); + snippet = norm( + body + .filter((l) => !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results") + .join(" "), + ) + .replace(/Read more$/, "") + .trim(); + } + + seen.add(url); + out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); + } + return out; + }; + + let challenge = null; + let results = []; + let container = "none"; + try { + challenge = detectChallenge(); + } catch (e) { + challenge = null; + } + try { + container = document.querySelector("#rso") ? "rso" : document.querySelector("#search") ? "search" : "none"; + results = challenge ? [] : extract(); + } catch (e) { + results = []; + } + + return { + url: location.href, + title: document.title || "", + readyState: document.readyState, + challenge: challenge, + container: container, + results: results, + }; +})()`; diff --git a/src/format.ts b/src/format.ts new file mode 100644 index 0000000..d3872a1 --- /dev/null +++ b/src/format.ts @@ -0,0 +1,46 @@ +/** + * Turning results into the text the model reads. + * + * Twenty results with 600-character snippets is roughly 15KB, comfortably under + * pi's 50KB / 2000-line budget — but the truncation is applied anyway rather + * than argued about, because a pathological page could blow past it and an + * overflowing tool result costs the whole session, not just this call. + */ + +import type { SearchResult } from "./extract.ts"; + +export type FormatInput = { + query: string; + searchUrl: string; + finalUrl: string; + endpoint: string; + results: SearchResult[]; +}; + +export const formatResults = (input: FormatInput): string => { + const lines: string[] = [ + `Search results for "${input.query}" (${input.results.length} hits, via Chrome at ${input.endpoint})`, + `Query URL: ${input.searchUrl}`, + ]; + if (input.finalUrl !== input.searchUrl) lines.push(`Landed on: ${input.finalUrl}`); + lines.push(""); + + input.results.forEach((result, index) => { + lines.push(`${index + 1}. ${result.title}`); + lines.push(` ${result.url}`); + if (result.snippet) lines.push(` ${result.snippet}`); + lines.push(""); + }); + + return lines.join("\n").trimEnd(); +}; + +/** + * The note appended when pi's truncation utilities cut the output. Says what was + * lost and what to do about it — a bare "[truncated]" tells the model nothing it + * can act on, and there is no temp file to point at because the full text only + * ever existed in memory. + */ +export const truncationNote = (shown: number, total: number, bytesShown: string, bytesTotal: string): string => + `\n\n[Output truncated: ${shown} of ${total} lines (${bytesShown} of ${bytesTotal}). ` + + `Re-run castle_cdp_search with a smaller numResults to see the rest.]`; diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..ad0ba6f --- /dev/null +++ b/src/index.ts @@ -0,0 +1,198 @@ +/** + * castle-cdp-search — web search for pi through a Chrome that is already + * running somewhere on the wire network, driven over CDP. + * + * It does not launch a browser. It attaches to one that a person or a launchd + * job started, using the same endpoint configuration as pi-browser-harness + * (`BU_CDP_HTTP` > `/browser-target` > castle), so a single choice governs both + * packages and they can never end up driving different browsers. + * + * The tool is called `castle_cdp_search`, not `web_search`, on purpose: it is + * meant to sit alongside pi-web-access rather than shadow it. The model picks + * between them, so both stay available in one session. + */ + +import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, truncateHead } from "@earendil-works/pi-coding-agent"; +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { StringEnum } from "@earendil-works/pi-ai"; +import { Type } from "typebox"; + +import { messageOf } from "./errors.ts"; +import type { SearchResult } from "./extract.ts"; +import { formatResults, truncationNote } from "./format.ts"; +import { + MAX_CONCURRENT_SEARCHES, + SEARCH_TIMEOUT_MS, + type Recency, + type SearchPhase, + describeConfiguredEndpoint, + disconnect, + search, +} from "./search.ts"; + +const DEFAULT_NUM_RESULTS = 10; +const MIN_NUM_RESULTS = 1; +const MAX_NUM_RESULTS = 20; + +const parameters = Type.Object({ + query: Type.String({ + description: "The search query, phrased as you would type it into a search engine.", + minLength: 1, + }), + numResults: Type.Optional( + Type.Integer({ + description: `How many results to return (${MIN_NUM_RESULTS}-${MAX_NUM_RESULTS}, default ${DEFAULT_NUM_RESULTS}).`, + minimum: MIN_NUM_RESULTS, + maximum: MAX_NUM_RESULTS, + default: DEFAULT_NUM_RESULTS, + }), + ), + // StringEnum, not Type.Union of Type.Literal: Google's API rejects the + // anyOf/const shape typebox emits for unions. + recency: Type.Optional( + StringEnum(["day", "week", "month", "year"] as const, { + description: "Restrict results to pages published within this window. Omit for no time limit.", + }), + ), +}); + +/** Exported so `isToolCallEventType<"castle_cdp_search", CastleCdpSearchInput>` can type it. */ +export type CastleCdpSearchInput = { + query: string; + numResults?: number; + recency?: Recency; +}; + +export type CastleCdpSearchDetails = { + query: string; + numResults: number; + recency: Recency | null; + phase: SearchPhase | "done"; + note: string; + endpoint: string | null; + searchUrl: string | null; + results: SearchResult[]; + truncated: boolean; +}; + +const initialDetails = (query: string, numResults: number, recency: Recency | null): CastleCdpSearchDetails => ({ + query, + numResults, + recency, + phase: "queued", + note: "starting", + endpoint: null, + searchUrl: null, + results: [], + truncated: false, +}); + +export default function (pi: ExtensionAPI): void { + // Nothing is dialled here. pi runs extension factories in invocations that + // never start a session, so opening a socket from a factory would leave one + // behind on, say, `pi --list-models`. The connection is made on first use and + // torn down below. + + pi.on("session_shutdown", async () => { + disconnect(); + }); + + pi.registerTool({ + name: "castle_cdp_search", + label: "Castle Search", + description: + "Search the web using a real Chrome browser running on the private network, driven over the Chrome " + + "DevTools Protocol. Returns ranked results with titles, URLs and snippets. Because it uses a real " + + "logged-in browser rather than a search API, it reaches pages that block datacentre traffic. It " + + "returns search results only — it does not fetch or read the linked pages, so follow up with a " + + "fetch/read tool for full page content. Output is truncated at " + + `${formatSize(DEFAULT_MAX_BYTES)} or ${DEFAULT_MAX_LINES} lines.`, + promptSnippet: + "Search the web through a real Chrome on the private network (castle_cdp_search); returns titles, URLs and snippets", + promptGuidelines: [ + "Use castle_cdp_search to find pages on the open web when you need current information, documentation, " + + "or sources you do not already have — it drives a real signed-in Chrome, so it works on sites that " + + "reject scripted clients.", + "castle_cdp_search returns search results only. To read a result, follow it up with a tool that fetches " + + "page content; do not treat the snippet as the full page.", + "Pass recency to castle_cdp_search when the answer depends on how recent a page is, and keep numResults " + + "small (5-10) unless you genuinely need a wide sweep.", + "If castle_cdp_search fails with a SearchChallengeError, the browser is sitting behind a captcha or " + + "consent wall that only a person can clear: stop searching, tell the user the URL from the error, and " + + "ask them to complete the challenge in that browser before you retry.", + ], + parameters, + + async execute(_toolCallId, params, signal, onUpdate, _ctx) { + const query = params.query.trim(); + if (query === "") throw new Error("query must not be empty"); + + const numResults = Math.min( + MAX_NUM_RESULTS, + Math.max(MIN_NUM_RESULTS, params.numResults ?? DEFAULT_NUM_RESULTS), + ); + const recency = (params.recency ?? null) as Recency | null; + + const details = initialDetails(query, numResults, recency); + + // One deadline covering connect + navigate + extract, plus the user's own + // abort so Esc drops an in-flight navigation rather than waiting it out. + const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS); + const combined = signal ? AbortSignal.any([signal, timeout]) : timeout; + + const report = (phase: SearchPhase, note: string): void => { + details.phase = phase; + details.note = note; + // Keep the TUI moving during the seconds Chrome spends navigating. + onUpdate?.({ content: [{ type: "text", text: `${note}…` }], details: { ...details } }); + }; + + try { + const outcome = await search({ query, numResults, recency: recency ?? undefined }, combined, report); + + const body = formatResults({ + query, + searchUrl: outcome.searchUrl, + finalUrl: outcome.finalUrl, + endpoint: outcome.endpoint, + results: outcome.results, + }); + + const truncation = truncateHead(body, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES }); + const text = truncation.truncated + ? truncation.content + + truncationNote( + truncation.outputLines, + truncation.totalLines, + formatSize(truncation.outputBytes), + formatSize(truncation.totalBytes), + ) + : truncation.content; + + details.phase = "done"; + details.note = `${outcome.results.length} results`; + details.endpoint = outcome.endpoint; + details.searchUrl = outcome.searchUrl; + details.results = outcome.results; + details.truncated = truncation.truncated; + + return { content: [{ type: "text", text }], details }; + } catch (e) { + // Rethrow, always. pi only marks a tool result as an error when execute() + // throws — returning `{ error }` would read to the model as a search that + // simply found nothing, which is the one wrong conclusion to invite here. + // The endpoint is worth naming: "which browser failed" is most of the + // diagnosis when the same config can point at three different machines. + const endpoint = await describeConfiguredEndpoint().catch(() => "unknown endpoint"); + const e2 = e instanceof Error ? e : new Error(messageOf(e)); + // Naming the phase turns "it timed out" into something actionable: + // "queued" means the cap was the bottleneck, "navigating" means the + // browser or the network was. + e2.message = + `${e2.message} [castle_cdp_search: failed during "${details.phase}" — browser ${endpoint}, ` + + `up to ${MAX_CONCURRENT_SEARCHES} concurrent searches, ${SEARCH_TIMEOUT_MS / 1000}s deadline covering the whole call]`; + throw e2; + } + }, + }); +} diff --git a/src/paths.ts b/src/paths.ts new file mode 100644 index 0000000..b008f4f --- /dev/null +++ b/src/paths.ts @@ -0,0 +1,41 @@ +/** + * pi's agent config directory, and the file pi-browser-harness stores its + * `/browser-target` choice in. + * + * This extension reads that file so a single `/browser-target` decision governs + * both packages. It never writes it — the harness owns that setting, and two + * writers of one file is a bug waiting to happen. + * + * pi exports `getAgentDir()`, but resolving it here would be a *runtime* import + * of the host package from an installed extension's own tree. Every other pi + * import in this codebase is `import type` and erased at build time, so the + * three lines are replicated instead (pi's dist/config.js: `PI_CODING_AGENT_DIR`, + * CONFIG_DIR_NAME `.pi`). Same reasoning as pi-browser-harness/src/profile/paths.ts. + */ + +import { homedir } from "node:os"; +import { join } from "node:path"; + +/** pi's env override for the agent dir (APP_NAME.toUpperCase() + "_CODING_AGENT_DIR"). */ +const ENV_AGENT_DIR = "PI_CODING_AGENT_DIR"; + +/** + * Expand a leading `~` exactly as pi's expandTildePath does — `~` and `~/…` + * only, never `~\…`, on every platform. Being more lenient here would look in a + * different directory than pi resolves from the same variable. + */ +const expandTildePiCompatible = (path: string): string => { + if (path === "~") return homedir(); + if (path.startsWith("~/")) return homedir() + path.slice(1); + return path; +}; + +/** pi's agent config directory — `$PI_CODING_AGENT_DIR` or `~/.pi/agent`. */ +export const agentDir = (): string => { + const fromEnv = process.env[ENV_AGENT_DIR]; + if (fromEnv) return expandTildePiCompatible(fromEnv); + return join(homedir(), ".pi", "agent"); +}; + +/** Where pi-browser-harness persists the `/browser-target` choice. */ +export const targetFilePath = (): string => join(agentDir(), "browser-target.json"); diff --git a/src/search.ts b/src/search.ts new file mode 100644 index 0000000..fa7ce35 --- /dev/null +++ b/src/search.ts @@ -0,0 +1,279 @@ +/** + * Search orchestration: resolve a browser, borrow a page, navigate, extract, + * hand the page back. + * + * The connection is process-wide and lazy. It is *not* opened by the extension + * factory — pi runs factories in invocations that never start a session, and + * the docs are explicit that sockets must not start there. It is opened on the + * first search and closed by an idempotent `session_shutdown`. + */ + +import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts"; +import { + type CdpTarget, + describeCdpTarget, + describeResolutionFailure, + resolveCdpTarget, +} from "./endpoint.ts"; +import { BrowserUnavailableError, NoResultsError, SearchChallengeError } from "./errors.ts"; +import { PROBE_SCRIPT, type PageProbe, type SearchResult } from "./extract.ts"; +import { readStoredTarget } from "./target-store.ts"; + +/** Per-search wall clock, covering connect, navigate and extract together. */ +export const SEARCH_TIMEOUT_MS = 20_000; +/** How many searches may drive the shared browser at once. */ +export const MAX_CONCURRENT_SEARCHES = 2; +/** Results sometimes land a beat after the load event; re-read a couple of times. */ +const EXTRACT_RETRIES = 2; +const EXTRACT_RETRY_DELAY_MS = 700; + +export type Recency = "day" | "week" | "month" | "year"; + +export type SearchRequest = { + query: string; + numResults: number; + recency?: Recency | undefined; +}; + +export type SearchOutcome = { + results: SearchResult[]; + searchUrl: string; + /** The URL the page settled on, which differs from searchUrl after a redirect. */ + finalUrl: string; + endpoint: string; +}; + +/** Google's date-restrict vocabulary. */ +const RECENCY_CODE: Record = { day: "d", week: "w", month: "m", year: "y" }; + +/** + * `udm=14` asks for the plain "Web" tab: no AI overview, no carousels, just + * ranked links. That is both cheaper to parse and closer to what the agent + * asked for. `hl=en` pins the result language so snippets do not change shape + * with whatever locale the browser profile happens to carry. + * + * Two findings from testing against the real browser, both counter-intuitive + * enough to be worth writing down: + * + * - `tbs=qdr:*` — the parameter the Tools menu puts in the URL — renders an + * empty page in this profile, with `#search` present but no `#rso` and no + * results at all. `as_qdr=*`, the older advanced-search parameter, works and + * genuinely filters (verified: d/w/m/y all return dated results). + * - Any date restriction combined with `udm=14` also renders that empty page. + * + * So a time-limited search drops `udm` and uses `as_qdr`. The classic layout it + * falls back to parses fine — the extractor was checked against both. + */ +export const buildSearchUrl = (request: SearchRequest): string => { + const params = new URLSearchParams({ + q: request.query, + num: String(request.numResults), + hl: "en", + }); + if (request.recency) params.set("as_qdr", RECENCY_CODE[request.recency]); + else params.set("udm", "14"); + return `https://www.google.com/search?${params.toString()}`; +}; + +// ── connection lifecycle ─────────────────────────────────────────────────── + +let connection: BrowserConnection | null = null; +/** In-flight connect, so two concurrent searches share one dial rather than racing. */ +let connecting: Promise | null = null; + +const resolveTarget = async (): Promise => { + const resolution = resolveCdpTarget(process.env, await readStoredTarget()); + if (resolution.kind === "invalid") { + throw new BrowserUnavailableError(describeResolutionFailure(resolution)); + } + return resolution.target; +}; + +/** + * Return a live connection, dialling one if there isn't one or the last one + * died. Reconnecting transparently matters here: castle recycles Chrome hourly, + * so a session that searched an hour ago is holding a dead socket. + */ +const ensureConnection = async (signal?: AbortSignal): Promise => { + if (connection && connection.isOpen) return connection; + if (connection) { + connection.close(); + connection = null; + } + + if (!connecting) { + // Assigned synchronously, before the first await inside: two searches + // starting in the same tick must share one dial, not open two sockets and + // leak whichever loses the assignment race. + connecting = (async () => { + const target = await resolveTarget(); + const conn = await BrowserConnection.connect(target); + connection = conn; + return conn; + })().finally(() => { + connecting = null; + }); + } + + // The caller's signal is raced here rather than passed into connect(). A + // shared dial must not be cancelled by whichever caller happened to start it + // — one search pressing Esc would otherwise fail the other search waiting on + // the same connection. connect() is bounded by its own timeouts regardless. + return raceAbort(connecting, signal); +}; + +/** Idempotent: safe to call from `session_shutdown` however many times it fires. */ +export const disconnect = (): void => { + connection?.close(); + connection = null; +}; + +/** For status reporting — never dials. */ +export const describeConfiguredEndpoint = async (): Promise => { + const resolution = resolveCdpTarget(process.env, await readStoredTarget()); + return resolution.kind === "invalid" + ? describeResolutionFailure(resolution) + : describeCdpTarget(resolution.target); +}; + +// ── concurrency cap ──────────────────────────────────────────────────────── + +let active = 0; +const waiting: Array<() => void> = []; + +/** Give a held slot to the next waiter, or return it to the pool. */ +const handOff = (): void => { + const next = waiting.shift(); + // Hand the slot straight over rather than decrementing and letting the waiter + // re-check, so a queued search cannot be overtaken by a newly arriving one. + if (next) next(); + else active--; +}; + +const acquire = async (signal?: AbortSignal): Promise<() => void> => { + if (active < MAX_CONCURRENT_SEARCHES) { + active++; + } else { + let resolver!: () => void; + const queued = new Promise((resolve) => { + resolver = resolve; + }); + waiting.push(resolver); + try { + await raceAbort(queued, signal); + // The slot was handed over by handOff(), which already counted it. + } catch (e) { + // An abort while queued must not strand the slot. Either we are still in + // the queue — drop out of it — or the slot was handed to us in the same + // tick the abort landed, in which case give it back. Without this, two + // cancelled-while-queued searches would wedge the semaphore permanently. + const index = waiting.indexOf(resolver); + if (index !== -1) waiting.splice(index, 1); + else handOff(); + throw e; + } + } + let released = false; + return () => { + if (released) return; + released = true; + handOff(); + }; +}; + +// ── the search itself ────────────────────────────────────────────────────── + +export type ProgressReporter = (phase: SearchPhase, note: string) => void; +export type SearchPhase = "queued" | "connecting" | "opening" | "navigating" | "extracting"; + +const sleep = (ms: number, signal?: AbortSignal): Promise => + raceAbort(new Promise((resolve) => setTimeout(resolve, ms)), signal); + +/** + * Run one search. Throws on every failure — see errors.ts for why a returned + * error object would be worse than useless here. + */ +export const search = async ( + request: SearchRequest, + signal: AbortSignal, + report: ProgressReporter, +): Promise => { + // Reported before acquiring, so a search that dies waiting for a slot is + // distinguishable from one that died reaching the browser. The deadline is + // the caller's and covers the queue wait too — with a cap of two and searches + // that take a couple of seconds, waiting is the rare case, and a tool call + // that silently takes twice its stated deadline would be worse. + report("queued", "waiting for a search slot"); + const release = await acquire(signal); + try { + report("connecting", "resolving browser"); + const conn = await ensureConnection(signal); + const endpoint = describeCdpTarget(conn.target); + + report("opening", `opening a page on ${endpoint}`); + const page = await PageSession.open(conn, signal); + const searchUrl = buildSearchUrl(request); + + try { + report("navigating", `searching for "${request.query}"`); + await page.navigate(searchUrl, signal); + + report("extracting", "reading results"); + let probe = await page.evaluate(PROBE_SCRIPT, signal); + for (let attempt = 0; attempt < EXTRACT_RETRIES; attempt++) { + if (probe?.challenge || (probe?.results?.length ?? 0) > 0) break; + await sleep(EXTRACT_RETRY_DELAY_MS, signal); + probe = await page.evaluate(PROBE_SCRIPT, signal); + } + + if (!probe) { + throw new BrowserUnavailableError("the page returned nothing — the extraction script did not run"); + } + + if (probe.challenge) { + throw new SearchChallengeError({ + challenge: probe.challenge.kind, + // The page the human must clear is where the browser *ended up*, which + // after a consent redirect is not the URL we asked for. + pageUrl: probe.url || searchUrl, + endpoint, + detail: probe.challenge.detail || undefined, + }); + } + + if (probe.results.length === 0) { + // "#rso missing entirely" and "#rso present but empty" are different + // faults, and saying which saves the next person a browser session. + const diagnosis = + probe.container === "rso" + ? "The results container was rendered but nothing could be parsed out of it, which suggests Google's " + + "result markup changed and castle_cdp_search's extractor needs updating." + : "Google served a page with no results area at all. Most likely this is a soft rate-limit: as a " + + "block decays Google stops serving the /sorry/ challenge page and returns an empty result page " + + "instead, which is indistinguishable from a genuine zero-hit search (observed lasting ~90 minutes " + + "after heavy querying). It can also mean the query truly has no hits, or that this combination of " + + "search parameters is not honoured for this browser profile."; + throw new NoResultsError( + `No results could be read from ${probe.url || searchUrl} (page title: "${probe.title}"). ${diagnosis} ` + + `Before concluding the extractor is broken, try a broader query and the same query without recency; ` + + `if those are also empty, treat it as rate limiting and wait rather than retrying in a loop.`, + ); + } + + return { + results: probe.results.slice(0, request.numResults), + searchUrl, + finalUrl: probe.url || searchUrl, + endpoint, + }; + } finally { + // Always, including the challenge path. Leaving the tab open would let the + // operator solve the captcha in place, but it would also litter a browser + // someone else is using with abandoned tabs on every failure; the error + // carries the URL instead, and the cookie it sets is profile-wide. + await page.close(); + } + } finally { + release(); + } +}; diff --git a/src/target-store.ts b/src/target-store.ts new file mode 100644 index 0000000..6ff0db4 --- /dev/null +++ b/src/target-store.ts @@ -0,0 +1,42 @@ +/** + * Read-only view of the target pi-browser-harness persists for `/browser-target`. + * + * The stored value is the raw string a user would have put in BU_CDP_HTTP + * ("local", "10.88.0.25:9223", …) — kept in that vocabulary deliberately, so + * there is one resolution path in endpoint.ts and a stored value can never mean + * something the variable could not. + * + * Every read failure degrades to "nothing stored" rather than throwing: a + * corrupt or absent file must fall back to the built-in default, not break search. + */ + +import { readFile } from "node:fs/promises"; +import { targetFilePath } from "./paths.ts"; + +const CURRENT_VERSION = 1; + +/** + * The persisted target, or null when nothing is stored, the file is missing, or + * it cannot be parsed. A file written by a newer version is treated as "nothing + * stored" rather than guessed at. + */ +export const readStoredTarget = async (): Promise => { + let raw: string; + try { + raw = await readFile(targetFilePath(), "utf8"); + } catch { + return null; + } + try { + const parsed: unknown = JSON.parse(raw); + if (typeof parsed !== "object" || parsed === null) return null; + const file = parsed as Record; + if (file["version"] !== CURRENT_VERSION) return null; + const target = file["target"]; + if (typeof target !== "string") return null; + const trimmed = target.trim(); + return trimmed.length > 0 ? trimmed : null; + } catch { + return null; + } +}; -- cgit v1.3.1