aboutsummaryrefslogtreecommitdiff
path: root/castle-search/src
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-08-04 15:08:47 +0100
committerIgor Soarez <igor@soarez.org>2026-08-04 15:08:47 +0100
commitbd03d3b8fff9ed18d3cdb104c55908ba5be6c424 (patch)
tree659afa1dd7e80c7ac97fb9c142026bf43c1ef8a0 /castle-search/src
parent08c0c3a8b4fe755f7fe65ea8d258d77971503882 (diff)
Write up what AI answer blocks would cost before building one
Google, Bing and Brave all put a generated answer above their results now, and none of it is being read. Investigated against castle's Chrome; nothing in src/ changed, because the cost turned out to be worth deciding on rather than absorbing. Two things make it expensive. Google's AI Overview exists only on the classic layout — udm=14 is the "Web" tab, and having no overview is part of what that tab is — so enabling it means giving back the clean page the extractor was built around. And all three stream in after the results are already parseable: Google settles ~2.3s in, Brave was still growing at 5s, against an extractor that returns at ~0.5s. The wait is often wasted, since the blocks are query-dependent — no Google overview for "how does postgres MVCC vacuum work" after 15s, no Bing answer for "who founded anthropic". Selectors, the traps behind them (#m-x-content briefly holds ~29k characters of unrelated page; Google's "Show more" hides only the disclaimer), the measured timings, three possible designs and their tradeoffs are all in the doc, so this does not have to be re-measured if the answer turns out to be yes.
Diffstat (limited to 'castle-search/src')
-rw-r--r--castle-search/src/cdp.ts342
-rw-r--r--castle-search/src/endpoint.ts108
-rw-r--r--castle-search/src/engine-store.ts81
-rw-r--r--castle-search/src/engine-target.ts79
-rw-r--r--castle-search/src/engines.ts229
-rw-r--r--castle-search/src/errors.ts83
-rw-r--r--castle-search/src/extract.ts294
-rw-r--r--castle-search/src/format.ts54
-rw-r--r--castle-search/src/index.ts315
-rw-r--r--castle-search/src/paths.ts41
-rw-r--r--castle-search/src/search.ts339
-rw-r--r--castle-search/src/target-store.ts42
12 files changed, 2007 insertions, 0 deletions
diff --git a/castle-search/src/cdp.ts b/castle-search/src/cdp.ts
new file mode 100644
index 0000000..aa01b56
--- /dev/null
+++ b/castle-search/src/cdp.ts
@@ -0,0 +1,342 @@
+/**
+ * A minimal CDP client — just enough to open a page, navigate it, read the DOM,
+ * and close it again.
+ *
+ * Deliberately dependency-free: Node 22 ships a global `WebSocket`, so this
+ * needs no `ws`. That matters because an extension installed by `pi install`
+ * gets its own `node_modules`, and fewer moving parts there is fewer things to
+ * go wrong on a machine that is not the one it was written on.
+ *
+ * The browser is *not* ours. It is a shared Chrome someone may be looking at,
+ * recycled hourly (`me.soarez.chrome-cdp-recycle`). Two consequences run
+ * through this file:
+ *
+ * 1. The websocket URL is re-resolved from `/json/version` on every connect,
+ * never cached. Chrome re-mints its browser UUID on each launch, so a
+ * pinned ws:// URL breaks the first time the browser is recycled.
+ * 2. Every page this opens is created in the background and closed again,
+ * including on error paths, so a failed search does not leave tabs behind
+ * in someone else's window.
+ */
+
+import { BrowserUnavailableError, SearchAbortedError, messageOf } from "./errors.ts";
+import type { CdpTarget } from "./endpoint.ts";
+
+/** How long a single CDP command may take before the socket is presumed dead. */
+const COMMAND_TIMEOUT_MS = 15_000;
+/** How long to wait for /json/version and for the websocket handshake. */
+const CONNECT_TIMEOUT_MS = 5_000;
+
+type CdpMessage = {
+ id?: number;
+ method?: string;
+ params?: Record<string, unknown>;
+ sessionId?: string;
+ result?: Record<string, unknown>;
+ error?: { code?: number; message?: string };
+};
+
+type Pending = {
+ resolve: (value: Record<string, unknown>) => void;
+ reject: (reason: Error) => void;
+ timer: ReturnType<typeof setTimeout>;
+};
+
+/**
+ * Reject if `signal` fires before `promise` settles, cleaning up the listener
+ * either way. Node's own abort-aware APIs are not available here — the
+ * websocket predates them — so aborts are raced in rather than plumbed through.
+ */
+export const raceAbort = <T>(promise: Promise<T>, signal: AbortSignal | undefined): Promise<T> => {
+ if (!signal) return promise;
+ if (signal.aborted) return Promise.reject(abortError(signal));
+ return new Promise<T>((resolve, reject) => {
+ const onAbort = (): void => reject(abortError(signal));
+ signal.addEventListener("abort", onAbort, { once: true });
+ promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort));
+ });
+};
+
+const abortError = (signal: AbortSignal): Error => {
+ const reason: unknown = signal.reason;
+ if (reason instanceof Error && reason.name === "TimeoutError") {
+ return new SearchAbortedError("search timed out");
+ }
+ return new SearchAbortedError("search was cancelled");
+};
+
+/** Ask the browser for its current websocket endpoint. Never cached — see above. */
+const queryWebSocketUrl = async (target: CdpTarget, signal?: AbortSignal): Promise<string> => {
+ const url = `http://${target.host}:${target.port}/json/version`;
+ let payload: unknown;
+ try {
+ const res = await fetch(url, {
+ signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(CONNECT_TIMEOUT_MS)]) : AbortSignal.timeout(CONNECT_TIMEOUT_MS),
+ });
+ if (!res.ok) {
+ throw new BrowserUnavailableError(`${url} answered HTTP ${res.status}`);
+ }
+ payload = await res.json();
+ } catch (e) {
+ if (e instanceof BrowserUnavailableError) throw e;
+ if (signal?.aborted) throw abortError(signal);
+ throw new BrowserUnavailableError(
+ `No CDP endpoint answered at ${url} (${messageOf(e)}). Is the browser running and reachable on the overlay?`,
+ );
+ }
+ const ws = (payload as Record<string, unknown> | null)?.["webSocketDebuggerUrl"];
+ if (typeof ws !== "string" || ws === "") {
+ throw new BrowserUnavailableError(`${url} answered without a webSocketDebuggerUrl`);
+ }
+ return ws;
+};
+
+/** A live browser-level CDP connection. */
+export class BrowserConnection {
+ #ws: WebSocket;
+ #nextId = 1;
+ #pending = new Map<number, Pending>();
+ #listeners = new Set<(msg: CdpMessage) => void>();
+ #closedReason: string | null = null;
+
+ readonly target: CdpTarget;
+
+ private constructor(ws: WebSocket, target: CdpTarget) {
+ this.#ws = ws;
+ this.target = target;
+
+ ws.addEventListener("message", (ev: MessageEvent) => {
+ let msg: CdpMessage;
+ try {
+ msg = JSON.parse(String(ev.data)) as CdpMessage;
+ } catch {
+ return;
+ }
+ if (msg.id !== undefined) {
+ const pending = this.#pending.get(msg.id);
+ if (!pending) return;
+ this.#pending.delete(msg.id);
+ clearTimeout(pending.timer);
+ if (msg.error) {
+ pending.reject(new BrowserUnavailableError(`CDP error: ${msg.error.message ?? "unknown"}`));
+ } else {
+ pending.resolve(msg.result ?? {});
+ }
+ return;
+ }
+ for (const listener of [...this.#listeners]) listener(msg);
+ });
+
+ const fail = (reason: string): void => {
+ this.#closedReason ??= reason;
+ for (const [, pending] of this.#pending) {
+ clearTimeout(pending.timer);
+ pending.reject(new BrowserUnavailableError(reason));
+ }
+ this.#pending.clear();
+ };
+ ws.addEventListener("close", () => fail("CDP connection closed (the browser may have been recycled)"));
+ ws.addEventListener("error", () => fail("CDP connection failed"));
+ }
+
+ /** Resolve the endpoint afresh and dial it. */
+ static async connect(target: CdpTarget, signal?: AbortSignal): Promise<BrowserConnection> {
+ const wsUrl = await queryWebSocketUrl(target, signal);
+ const ws = new WebSocket(wsUrl);
+ try {
+ await raceAbort(
+ new Promise<void>((resolve, reject) => {
+ const timer = setTimeout(
+ () => reject(new BrowserUnavailableError(`timed out opening ${wsUrl}`)),
+ CONNECT_TIMEOUT_MS,
+ );
+ ws.addEventListener(
+ "open",
+ () => {
+ clearTimeout(timer);
+ resolve();
+ },
+ { once: true },
+ );
+ ws.addEventListener(
+ "error",
+ () => {
+ clearTimeout(timer);
+ reject(new BrowserUnavailableError(`could not open ${wsUrl}`));
+ },
+ { once: true },
+ );
+ }),
+ signal,
+ );
+ } catch (e) {
+ try {
+ ws.close();
+ } catch {
+ // Already failed; nothing useful to do with a second failure.
+ }
+ throw e;
+ }
+ return new BrowserConnection(ws, target);
+ }
+
+ /** False once the socket has closed or failed — the reconnect trigger. */
+ get isOpen(): boolean {
+ return this.#closedReason === null && this.#ws.readyState === 1 /* OPEN */;
+ }
+
+ send(method: string, params: Record<string, unknown> = {}, sessionId?: string): Promise<Record<string, unknown>> {
+ if (!this.isOpen) {
+ return Promise.reject(new BrowserUnavailableError(this.#closedReason ?? "CDP connection is not open"));
+ }
+ const id = this.#nextId++;
+ const payload: CdpMessage = sessionId ? { id, method, params, sessionId } : { id, method, params };
+ return new Promise<Record<string, unknown>>((resolve, reject) => {
+ const timer = setTimeout(() => {
+ this.#pending.delete(id);
+ reject(new BrowserUnavailableError(`CDP command ${method} timed out after ${COMMAND_TIMEOUT_MS}ms`));
+ }, COMMAND_TIMEOUT_MS);
+ this.#pending.set(id, { resolve, reject, timer });
+ try {
+ this.#ws.send(JSON.stringify(payload));
+ } catch (e) {
+ this.#pending.delete(id);
+ clearTimeout(timer);
+ reject(new BrowserUnavailableError(`could not send ${method}: ${messageOf(e)}`));
+ }
+ });
+ }
+
+ /** Subscribe to unsolicited protocol events. Returns an unsubscribe function. */
+ onEvent(listener: (msg: CdpMessage) => void): () => void {
+ this.#listeners.add(listener);
+ return () => this.#listeners.delete(listener);
+ }
+
+ close(): void {
+ this.#closedReason ??= "CDP connection closed by this extension";
+ this.#listeners.clear();
+ try {
+ this.#ws.close();
+ } catch {
+ // Closing an already-dead socket is not a failure worth reporting.
+ }
+ }
+}
+
+/**
+ * One throwaway page.
+ *
+ * Created in the background so it does not steal focus from whoever is using
+ * the browser, and closed by {@link close} on every path.
+ */
+export class PageSession {
+ readonly #conn: BrowserConnection;
+ readonly #targetId: string;
+ readonly #sessionId: string;
+ #closed = false;
+
+ private constructor(conn: BrowserConnection, targetId: string, sessionId: string) {
+ this.#conn = conn;
+ this.#targetId = targetId;
+ this.#sessionId = sessionId;
+ }
+
+ static async open(conn: BrowserConnection, signal?: AbortSignal): Promise<PageSession> {
+ if (signal?.aborted) throw abortError(signal);
+
+ // Deliberately NOT raced against the abort signal. raceAbort abandons the
+ // promise, it cannot cancel the command — and this command's side effect is
+ // a tab. Abandoning it would leave a tab open in a browser someone else is
+ // using, with nothing left holding its targetId to close it. It is a fast
+ // command already bounded by COMMAND_TIMEOUT_MS, so waiting it out costs
+ // little and guarantees we own whatever it created.
+ const created = await conn.send("Target.createTarget", { url: "about:blank", background: true });
+ const targetId = created["targetId"];
+ if (typeof targetId !== "string") {
+ throw new BrowserUnavailableError("Target.createTarget did not return a targetId");
+ }
+
+ // From here on the page exists, so every failure — including a late abort —
+ // must still close it.
+ try {
+ if (signal?.aborted) throw abortError(signal);
+ const attached = await raceAbort(
+ conn.send("Target.attachToTarget", { targetId, flatten: true }),
+ signal,
+ );
+ const sessionId = attached["sessionId"];
+ if (typeof sessionId !== "string") {
+ throw new BrowserUnavailableError("Target.attachToTarget did not return a sessionId");
+ }
+ const page = new PageSession(conn, targetId, sessionId);
+ await raceAbort(conn.send("Page.enable", {}, sessionId), signal);
+ return page;
+ } catch (e) {
+ await conn.send("Target.closeTarget", { targetId }).catch(() => {});
+ throw e;
+ }
+ }
+
+ /** Navigate and wait for the load event, or for `signal` to fire. */
+ async navigate(url: string, signal?: AbortSignal): Promise<void> {
+ // Subscribed before navigating, so a page that loads faster than the
+ // Page.navigate reply cannot slip its load event past us.
+ let off: () => void = () => {};
+ const loaded = new Promise<void>((resolve) => {
+ off = this.#conn.onEvent((msg) => {
+ if (msg.sessionId === this.#sessionId && msg.method === "Page.loadEventFired") resolve();
+ });
+ });
+
+ try {
+ const nav = await raceAbort(this.#conn.send("Page.navigate", { url }, this.#sessionId), signal);
+ const errorText = nav["errorText"];
+ if (typeof errorText === "string" && errorText !== "") {
+ throw new BrowserUnavailableError(`navigation to ${url} failed: ${errorText}`);
+ }
+
+ try {
+ await raceAbort(loaded, signal);
+ } catch (e) {
+ // Stop the in-flight load so the page is not still fetching while we
+ // tear it down; the tab is closed either way, this just makes it quicker.
+ await this.#conn.send("Page.stopLoading", {}, this.#sessionId).catch(() => {});
+ throw e;
+ }
+ } finally {
+ // Unsubscribing here rather than inside the listener: on the failure paths
+ // the listener never fires, and a subscription left on a connection that
+ // outlives many searches accumulates.
+ off();
+ }
+ }
+
+ /** Evaluate an expression in the page and return its value by value. */
+ async evaluate<T>(expression: string, signal?: AbortSignal): Promise<T> {
+ const res = await raceAbort(
+ this.#conn.send(
+ "Runtime.evaluate",
+ { expression, returnByValue: true, awaitPromise: true },
+ this.#sessionId,
+ ),
+ signal,
+ );
+ const exception = res["exceptionDetails"] as { text?: string; exception?: { description?: string } } | undefined;
+ if (exception) {
+ const text = exception.exception?.description ?? exception.text ?? "unknown error";
+ throw new BrowserUnavailableError(`page script failed: ${text}`);
+ }
+ return (res["result"] as { value?: T } | undefined)?.value as T;
+ }
+
+ /**
+ * Close the tab. Idempotent and never throws: it runs on error paths, where a
+ * second failure would mask the one worth reporting.
+ */
+ async close(): Promise<void> {
+ if (this.#closed) return;
+ this.#closed = true;
+ await this.#conn.send("Target.closeTarget", { targetId: this.#targetId }).catch(() => {});
+ }
+}
diff --git a/castle-search/src/endpoint.ts b/castle-search/src/endpoint.ts
new file mode 100644
index 0000000..d3f4b0d
--- /dev/null
+++ b/castle-search/src/endpoint.ts
@@ -0,0 +1,108 @@
+/**
+ * Which browser this extension searches through.
+ *
+ * Deliberately the *same* configuration as pi-browser-harness, so one decision
+ * governs both packages and they never drive different browsers in the same
+ * session: `BU_CDP_HTTP`, then the `/browser-target` file that harness writes,
+ * then the built-in castle default.
+ *
+ * Resolution is pure — no I/O — so it can be unit-tested and reported without
+ * touching the network.
+ */
+
+/** Default CDP endpoint: castle's Chrome, exposed on the wg8 overlay. */
+export const DEFAULT_REMOTE_CDP = "10.88.0.25:9223";
+
+/** Values of BU_CDP_HTTP that mean "ignore the default, use this machine". */
+const LOCAL_ALIASES = new Set(["local", "localhost", "off", "none", "0", "no"]);
+
+/** Port a local Chrome is expected to serve CDP on when the target is "local". */
+export const LOCAL_CDP_PORT = 9222;
+
+/**
+ * Where a target came from. "env" is BU_CDP_HTTP, "stored" is the harness's
+ * `/browser-target`, "default" is the built-in.
+ */
+export type CdpTargetSource = "env" | "stored" | "default";
+
+export type CdpTarget = {
+ readonly host: string;
+ readonly port: number;
+ readonly source: CdpTargetSource;
+};
+
+export type CdpTargetResolution =
+ | { readonly kind: "ok"; readonly target: CdpTarget }
+ | { readonly kind: "invalid"; readonly raw: string; readonly reason: string; readonly source: CdpTargetSource };
+
+/**
+ * Precedence: BU_CDP_HTTP, then the stored `/browser-target` choice, then the
+ * built-in default. The environment wins so a one-off `BU_CDP_HTTP=… pi` is
+ * never silently overridden by a choice made days ago.
+ *
+ * Unlike pi-browser-harness this never falls back from the default to a local
+ * browser on its own. The harness can fall back because it drives a browser the
+ * operator is watching; here a silent fallback would mean *searching from the
+ * operator's own signed-in Chrome* without saying so. When the configured
+ * endpoint is unreachable the caller reports that, and the operator can say
+ * `local` explicitly.
+ *
+ * `stored` is passed in rather than read here so this stays pure and synchronous.
+ */
+export const resolveCdpTarget = (
+ env: NodeJS.ProcessEnv = process.env,
+ stored: string | null = null,
+): CdpTargetResolution => {
+ const fromEnv = env["BU_CDP_HTTP"]?.trim();
+ const fromStore = stored?.trim();
+
+ const hasEnv = fromEnv !== undefined && fromEnv !== "";
+ const raw = hasEnv ? fromEnv : fromStore;
+ const source: CdpTargetSource = hasEnv ? "env" : "stored";
+
+ if (raw === undefined || raw === "") {
+ const parsed = parseHostPort(DEFAULT_REMOTE_CDP);
+ // Unreachable in practice: the constant above is covered by its own test.
+ if (!parsed) {
+ return { kind: "invalid", raw: DEFAULT_REMOTE_CDP, source: "default", reason: "built-in default is malformed" };
+ }
+ return { kind: "ok", target: { ...parsed, source: "default" } };
+ }
+
+ if (LOCAL_ALIASES.has(raw.toLowerCase())) {
+ return { kind: "ok", target: { host: "127.0.0.1", port: LOCAL_CDP_PORT, source } };
+ }
+
+ const parsed = parseHostPort(raw);
+ if (!parsed) {
+ return {
+ kind: "invalid",
+ raw,
+ source,
+ reason: `expected "host:port" or one of ${[...LOCAL_ALIASES].join(", ")}`,
+ };
+ }
+ return { kind: "ok", target: { ...parsed, source } };
+};
+
+const parseHostPort = (value: string): { host: string; port: number } | null => {
+ const sep = value.lastIndexOf(":");
+ if (sep === -1) return null;
+ const host = value.slice(0, sep).trim();
+ const port = Number(value.slice(sep + 1).trim());
+ if (!host) return null;
+ if (!Number.isInteger(port) || port <= 0 || port >= 65536) return null;
+ return { host, port };
+};
+
+const sourceLabel = (source: CdpTargetSource): string =>
+ source === "env" ? "BU_CDP_HTTP" : source === "stored" ? "/browser-target" : "built-in default";
+
+/** One-line description for progress messages and errors. */
+export const describeCdpTarget = (target: CdpTarget): string =>
+ `${target.host}:${target.port} (${sourceLabel(target.source)})`;
+
+/** One-line description of a resolution failure. */
+export const describeResolutionFailure = (
+ failure: Extract<CdpTargetResolution, { kind: "invalid" }>,
+): string => `invalid CDP target "${failure.raw}" from ${sourceLabel(failure.source)} — ${failure.reason}`;
diff --git a/castle-search/src/engine-store.ts b/castle-search/src/engine-store.ts
new file mode 100644
index 0000000..d218d2b
--- /dev/null
+++ b/castle-search/src/engine-store.ts
@@ -0,0 +1,81 @@
+/**
+ * Persistence for the `/search-engine` choice.
+ *
+ * Deliberately the same shape, and the same atomic write, as the harness's
+ * browser-target store: a sibling temp file plus rename, so a crash mid-write
+ * cannot leave a half-written engine name behind. Its own file rather than a
+ * shared settings blob, so a corrupt engine choice cannot cost anything else.
+ *
+ * Like the CDP target, the stored value is the raw string a user would have put
+ * in the environment variable ("ddg", "bing", …), not a pre-parsed id. One
+ * vocabulary, one resolution path in engine-target.ts, and a stored value can
+ * never mean something the variable could not.
+ *
+ * Every read failure degrades to "nothing stored" rather than throwing.
+ */
+
+import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { dirname, join } from "node:path";
+
+import { agentDir } from "./paths.ts";
+
+const CURRENT_VERSION = 1;
+
+const engineFilePath = (): string => join(agentDir(), "castle-cdp-search-engine.json");
+
+type EngineFile = {
+ readonly version: 1;
+ /** Raw CCS_SEARCH_ENGINE-style value, or null when cleared. */
+ readonly engine: string | null;
+ readonly savedAt: string;
+};
+
+/**
+ * The persisted engine, or null when nothing is stored, the file is missing, or
+ * it cannot be parsed. A file written by a newer version is treated as "nothing
+ * stored" rather than guessed at.
+ */
+export const readStoredEngine = async (): Promise<string | null> => {
+ let raw: string;
+ try {
+ raw = await readFile(engineFilePath(), "utf8");
+ } catch {
+ return null;
+ }
+ try {
+ const parsed: unknown = JSON.parse(raw);
+ if (typeof parsed !== "object" || parsed === null) return null;
+ const file = parsed as Record<string, unknown>;
+ if (file["version"] !== CURRENT_VERSION) return null;
+ const engine = file["engine"];
+ if (typeof engine !== "string") return null;
+ const trimmed = engine.trim();
+ return trimmed.length > 0 ? trimmed : null;
+ } catch {
+ return null;
+ }
+};
+
+const write = async (engine: string | null): Promise<void> => {
+ const path = engineFilePath();
+ const tmp = `${path}.${randomUUID()}.tmp`;
+ const payload: EngineFile = { version: CURRENT_VERSION, engine, savedAt: new Date().toISOString() };
+ try {
+ await mkdir(dirname(path), { recursive: true });
+ await writeFile(tmp, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
+ await rename(tmp, path);
+ } catch (e) {
+ await unlink(tmp).catch(() => {});
+ throw new Error(`could not save search engine to ${path}: ${e instanceof Error ? e.message : String(e)}`);
+ }
+};
+
+/** Persist the chosen engine. */
+export const writeStoredEngine = (engine: string): Promise<void> => write(engine);
+
+/** Clear the choice, restoring the built-in default. */
+export const clearStoredEngine = (): Promise<void> => write(null);
+
+/** Exposed so the command can tell the user where the choice lives. */
+export const storedEngineLocation = (): string => engineFilePath();
diff --git a/castle-search/src/engine-target.ts b/castle-search/src/engine-target.ts
new file mode 100644
index 0000000..db4d81f
--- /dev/null
+++ b/castle-search/src/engine-target.ts
@@ -0,0 +1,79 @@
+/**
+ * Which search engine to use, resolved the same way the CDP host is:
+ * environment variable, then a persisted choice, then a built-in default.
+ *
+ * Pure — no I/O — so the `/search-engine` command can report the configured
+ * intent without touching disk twice, and so it is testable.
+ */
+
+import { type EngineId, parseEngineId, engineAliases } from "./engines.ts";
+
+/** Used when nothing else says otherwise. */
+export const DEFAULT_ENGINE: EngineId = "google";
+
+/** The environment variable that pins the engine for one process. */
+export const ENGINE_ENV_VAR = "CCS_SEARCH_ENGINE";
+
+export type EngineSource = "parameter" | "env" | "stored" | "default";
+
+export type EngineResolution =
+ | { readonly kind: "ok"; readonly id: EngineId; readonly source: EngineSource }
+ | { readonly kind: "invalid"; readonly raw: string; readonly source: EngineSource };
+
+/**
+ * Precedence: the per-call tool parameter, then `CCS_SEARCH_ENGINE`, then the
+ * `/search-engine` choice, then the built-in default.
+ *
+ * Note the tool parameter sits *above* the environment variable, which is the
+ * opposite of how the CDP target treats an explicit endpoint. That difference
+ * is deliberate. For the browser, an environment variable has to win, because
+ * quietly driving a different machine means automating the operator's own
+ * signed-in Chrome — a safety property. Choosing a different search engine has
+ * no equivalent hazard, and the ability to fall back to another engine when the
+ * first one is serving a captcha is the single most useful thing the agent can
+ * do with this tool. So the model is allowed to override the default; the
+ * operator sets what it starts from.
+ */
+export const resolveEngine = (
+ parameter: string | null | undefined,
+ env: NodeJS.ProcessEnv = process.env,
+ stored: string | null = null,
+): EngineResolution => {
+ const candidates: Array<{ raw: string; source: EngineSource }> = [];
+
+ const fromParam = parameter?.trim();
+ if (fromParam) candidates.push({ raw: fromParam, source: "parameter" });
+
+ const fromEnv = env[ENGINE_ENV_VAR]?.trim();
+ if (fromEnv) candidates.push({ raw: fromEnv, source: "env" });
+
+ const fromStore = stored?.trim();
+ if (fromStore) candidates.push({ raw: fromStore, source: "stored" });
+
+ for (const candidate of candidates) {
+ const id = parseEngineId(candidate.raw);
+ // An unparseable value is reported rather than skipped over. Falling
+ // through to the next source would mean a typo in CCS_SEARCH_ENGINE
+ // silently searches Google instead of saying so.
+ if (!id) return { kind: "invalid", raw: candidate.raw, source: candidate.source };
+ return { kind: "ok", id, source: candidate.source };
+ }
+
+ return { kind: "ok", id: DEFAULT_ENGINE, source: "default" };
+};
+
+const sourceLabel = (source: EngineSource): string =>
+ source === "parameter"
+ ? "engine parameter"
+ : source === "env"
+ ? ENGINE_ENV_VAR
+ : source === "stored"
+ ? "/search-engine"
+ : "built-in default";
+
+export const describeEngineSource = (source: EngineSource): string => sourceLabel(source);
+
+/** One-line description of a bad engine name, listing what would have worked. */
+export const describeEngineFailure = (failure: Extract<EngineResolution, { kind: "invalid" }>): string =>
+ `unknown search engine "${failure.raw}" from ${sourceLabel(failure.source)} — ` +
+ `expected one of ${engineAliases().join(", ")}`;
diff --git a/castle-search/src/engines.ts b/castle-search/src/engines.ts
new file mode 100644
index 0000000..ef8baef
--- /dev/null
+++ b/castle-search/src/engines.ts
@@ -0,0 +1,229 @@
+/**
+ * The search engines this extension knows how to drive.
+ *
+ * Every selector and every URL parameter below was read off the real SERP in
+ * castle's Chrome, not recalled or inferred. That is not pedantry: Google's own
+ * Tools menu writes `tbs=qdr:*`, which renders an *empty* page on that profile,
+ * while the undocumented-looking `as_qdr=*` works. Anything in here that was not
+ * observed is a bug waiting to be reported as "no results".
+ *
+ * Two extraction modes, because the engines genuinely differ:
+ *
+ * - "items" — the SERP has a clean per-result container (`li.b_algo`,
+ * `.result`, `.snippet[data-type=web]`). Straightforward.
+ * - "headings" — Google has no stable result container, so results are found
+ * from each <h3> outward. Kept as its own mode rather than
+ * forced into the item shape, because it is the one that has
+ * been through the most verification.
+ */
+
+export type EngineId = "google" | "duckduckgo" | "bing" | "brave";
+
+export type Recency = "day" | "week" | "month" | "year";
+
+export const RECENCY_VALUES = ["day", "week", "month", "year"] as const;
+
+export const ENGINE_IDS: readonly EngineId[] = ["google", "duckduckgo", "bing", "brave"];
+
+/** Aliases accepted from humans and from the model. */
+const ENGINE_ALIASES: Record<string, EngineId> = {
+ google: "google",
+ g: "google",
+ duckduckgo: "duckduckgo",
+ ddg: "duckduckgo",
+ duck: "duckduckgo",
+ bing: "bing",
+ b: "bing",
+ brave: "brave",
+};
+
+/**
+ * How the in-page script should read one engine's results. Passed into the
+ * browser as JSON, so everything here must be plain data.
+ */
+export type ExtractionConfig = {
+ mode: "items" | "headings";
+ /** Results container candidates, first match wins. */
+ roots: string[];
+ /** "items" mode: one result. */
+ item?: string;
+ /** Anchor carrying the outbound link, relative to the item. */
+ link?: string;
+ /** Title element, relative to the item. Falls back to the anchor's text. */
+ title?: string;
+ /** Description element, relative to the item. */
+ snippet?: string;
+ /**
+ * "items" mode with no snippet selector: text from these is subtracted from
+ * the item's text to leave the description behind.
+ */
+ subtract?: string[];
+ /** Items matching any of these are ads or non-web cards, and are skipped. */
+ exclude?: string[];
+ /** How outbound links are wrapped, if they are. */
+ unwrap: "none" | "ddg" | "bing";
+ /** Hosts belonging to the engine itself; links to them are not results. */
+ selfHostPattern: string;
+};
+
+export type EngineDefinition = {
+ readonly id: EngineId;
+ readonly label: string;
+ /** Human-facing note about what makes this engine worth choosing. */
+ readonly note: string;
+ /**
+ * Recency windows this engine can actually express, mapped to the parameter
+ * value. A window that is absent is one the engine does not support — never
+ * one that is silently dropped.
+ */
+ readonly recency: Partial<Record<Recency, string>>;
+ readonly buildUrl: (query: string, numResults: number, recency: Recency | undefined) => string;
+ readonly extraction: ExtractionConfig;
+};
+
+const GOOGLE: EngineDefinition = {
+ id: "google",
+ label: "Google",
+ note: "best result quality; blocks aggressively under repeated automated queries",
+ recency: { day: "d", week: "w", month: "m", year: "y" },
+ buildUrl: (query, numResults, recency) => {
+ const params = new URLSearchParams({ q: query, num: String(numResults), hl: "en" });
+ // `udm=14` is the plain "Web" tab: no AI overview, no carousels. But any
+ // date restriction combined with it renders an empty page, and `tbs=qdr:*`
+ // renders an empty page on its own — both observed. So a time-limited
+ // search drops udm and uses the older as_qdr instead.
+ if (recency) params.set("as_qdr", GOOGLE.recency[recency] as string);
+ else params.set("udm", "14");
+ return `https://www.google.com/search?${params.toString()}`;
+ },
+ extraction: {
+ mode: "headings",
+ roots: ["#rso", "#search"],
+ unwrap: "none",
+ selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/`,
+ },
+};
+
+const DUCKDUCKGO: EngineDefinition = {
+ id: "duckduckgo",
+ label: "DuckDuckGo",
+ note: "no-JS endpoint, the most stable markup here and the least likely to challenge",
+ // Read off DuckDuckGo's own <select name="df">: "", d, w, m, y.
+ recency: { day: "d", week: "w", month: "m", year: "y" },
+ buildUrl: (query, _numResults, recency) => {
+ // The html endpoint renders server-side with no JavaScript, which makes it
+ // both faster and far less fragile than the app at duckduckgo.com. It has
+ // no result-count parameter — it returns a full page and the caller slices.
+ const params = new URLSearchParams({ q: query });
+ if (recency) params.set("df", DUCKDUCKGO.recency[recency] as string);
+ return `https://html.duckduckgo.com/html/?${params.toString()}`;
+ },
+ extraction: {
+ mode: "items",
+ roots: [".results", "#links"],
+ item: ".result",
+ link: "a.result__a[href]",
+ title: "a.result__a",
+ snippet: ".result__snippet",
+ exclude: [".result--ad", ".badge--ad"],
+ unwrap: "ddg",
+ selfHostPattern: String.raw`^https?://(html\.|www\.)?duckduckgo\.com/`,
+ },
+};
+
+const BING: EngineDefinition = {
+ id: "bing",
+ label: "Bing",
+ note: "good coverage; supports day/week/month only — it has no year filter",
+ // ez1/ez2/ez3 verified to filter (results carried "4 hours ago", "1 day ago").
+ // There is deliberately no year: Bing's UI offers no such window, and the
+ // ez5 custom-range form returned nothing when tried.
+ recency: { day: "ez1", week: "ez2", month: "ez3" },
+ buildUrl: (query, numResults, recency) => {
+ const params = new URLSearchParams({ q: query });
+ if (recency) {
+ // `count` silently cancels `filters` — with ez1 alone every result is
+ // hours old, and adding count in either order brings back months-old
+ // ones. Observed directly, and it fails silently: the results look
+ // perfectly plausible, just unfiltered. Exactly the same trap as Google's
+ // udm+as_qdr, so the same answer — drop the count parameter and let the
+ // caller slice the list it gets.
+ params.set("filters", `ex1:"${BING.recency[recency] as string}"`);
+ } else {
+ params.set("count", String(numResults));
+ }
+ return `https://www.bing.com/search?${params.toString()}`;
+ },
+ extraction: {
+ mode: "items",
+ // Organic results only. Bing's answer cards also contain <h2>s, which is
+ // why this targets li.b_algo rather than walking headings.
+ roots: ["#b_results"],
+ item: "li.b_algo",
+ link: "h2 a[href]",
+ title: "h2",
+ snippet: ".b_caption p, .b_algoSlug, p",
+ exclude: [".b_ad", ".b_adBottom"],
+ unwrap: "bing",
+ selfHostPattern: String.raw`^https?://(www\.)?bing\.com/`,
+ },
+};
+
+const BRAVE: EngineDefinition = {
+ id: "brave",
+ label: "Brave Search",
+ note: "independent index, direct links; challenges after 1-2 queries, clears in ~15-20 min",
+ // Left empty deliberately — see the note in README. Brave's filter UI is
+ // client-rendered and the `tf=` values could not be confirmed against a page
+ // that was not simultaneously serving a bot check, so no window is claimed
+ // rather than one being guessed at.
+ recency: {},
+ buildUrl: (query, _numResults, _recency) => {
+ const params = new URLSearchParams({ q: query });
+ return `https://search.brave.com/search?${params.toString()}`;
+ },
+ extraction: {
+ mode: "items",
+ roots: ["#results"],
+ // data-type="web" excludes the AI summariser, video and news cards, which
+ // share the .snippet class but are not ranked web results.
+ item: '.snippet[data-type="web"]',
+ link: "a[href]",
+ title: ".title",
+ // No dedicated description element. The item's text is
+ // "source | breadcrumb | title | description", so the extractor keeps only
+ // the lines after the title. These selectors are the fallback for when the
+ // title is not on a line of its own.
+ subtract: [".title", "cite"],
+ unwrap: "none",
+ selfHostPattern: String.raw`^https?://(search\.)?brave\.com/`,
+ },
+};
+
+const BY_ID: Record<EngineId, EngineDefinition> = {
+ google: GOOGLE,
+ duckduckgo: DUCKDUCKGO,
+ bing: BING,
+ brave: BRAVE,
+};
+
+export const getEngine = (id: EngineId): EngineDefinition => BY_ID[id];
+
+export const allEngines = (): readonly EngineDefinition[] => ENGINE_IDS.map((id) => BY_ID[id]);
+
+/** Parse a user- or model-supplied engine name. Null when unrecognised. */
+export const parseEngineId = (raw: string): EngineId | null =>
+ ENGINE_ALIASES[raw.trim().toLowerCase()] ?? null;
+
+/** The names accepted for an engine, for help text and error messages. */
+export const engineAliases = (): readonly string[] => Object.keys(ENGINE_ALIASES);
+
+/** Which engines can express a given recency window. */
+export const enginesSupportingRecency = (recency: Recency): readonly EngineId[] =>
+ ENGINE_IDS.filter((id) => BY_ID[id].recency[recency] !== undefined);
+
+/** Human-readable list of the windows an engine supports, or "none". */
+export const describeRecencySupport = (engine: EngineDefinition): string => {
+ const windows = RECENCY_VALUES.filter((r) => engine.recency[r] !== undefined);
+ return windows.length > 0 ? windows.join(", ") : "none";
+};
diff --git a/castle-search/src/errors.ts b/castle-search/src/errors.ts
new file mode 100644
index 0000000..2318b27
--- /dev/null
+++ b/castle-search/src/errors.ts
@@ -0,0 +1,83 @@
+/**
+ * Failure vocabulary.
+ *
+ * Every one of these is *thrown*, never returned. pi only sets `isError: true`
+ * on a tool result when `execute()` throws — a returned object with an `error`
+ * field looks to the model exactly like a successful search that found nothing.
+ *
+ * `SearchChallengeError` is the one the agent is expected to act on rather than
+ * just report: it means a human has to touch the browser before search can work
+ * again, so its message says so in words the agent can pass to the operator.
+ */
+
+/** The search engine served a captcha, consent wall, or other human check. */
+export class SearchChallengeError extends Error {
+ override readonly name = "SearchChallengeError";
+ /** Machine-readable flavour of the challenge, for callers that branch on it. */
+ readonly challenge: string;
+ /** The page the human needs to visit to clear it. */
+ readonly pageUrl: string;
+ /** Which browser is blocked, e.g. "10.88.0.25:9223 (built-in default)". */
+ readonly endpoint: string;
+ /** Which engine is blocked. */
+ readonly engine: string;
+ /** Engines that are not blocked and could be tried instead. */
+ readonly alternatives: readonly string[];
+
+ constructor(options: {
+ challenge: string;
+ pageUrl: string;
+ endpoint: string;
+ engine: string;
+ alternatives: readonly string[];
+ detail?: string | undefined;
+ }) {
+ super(
+ [
+ `${options.engine} is blocked by a human verification challenge (${options.challenge}).`,
+ options.detail ? `Page said: ${options.detail}` : null,
+ // Retrying elsewhere comes first because it is the action the agent can
+ // take on its own; a challenge on one engine says nothing about the
+ // others, and asking the operator for help should be the fallback, not
+ // the first move.
+ options.alternatives.length > 0
+ ? `A challenge on one engine does not affect the others: retry the same query with engine set to ` +
+ `${options.alternatives.join(", ")} before asking anyone for help.`
+ : null,
+ `If every engine is blocked, this needs a person — the challenge cannot be solved by the agent.`,
+ `Ask the user to open ${options.pageUrl} in the Chrome at ${options.endpoint}, complete the challenge or`,
+ `accept the consent dialog, then retry. The cookie it sets persists in that browser profile, so one pass`,
+ `unblocks later searches.`,
+ ]
+ .filter((line) => line !== null)
+ .join(" "),
+ );
+ this.challenge = options.challenge;
+ this.pageUrl = options.pageUrl;
+ this.endpoint = options.endpoint;
+ this.engine = options.engine;
+ this.alternatives = options.alternatives;
+ }
+}
+
+/** The chosen engine cannot express the requested recency window. */
+export class RecencyUnsupportedError extends Error {
+ override readonly name = "RecencyUnsupportedError";
+}
+
+/** The browser could not be reached or spoke CDP badly. */
+export class BrowserUnavailableError extends Error {
+ override readonly name = "BrowserUnavailableError";
+}
+
+/** The search itself ran but produced nothing usable. */
+export class NoResultsError extends Error {
+ override readonly name = "NoResultsError";
+}
+
+/** The per-search deadline expired, or the user pressed Esc. */
+export class SearchAbortedError extends Error {
+ override readonly name = "SearchAbortedError";
+}
+
+export const messageOf = (e: unknown): string => (e instanceof Error ? e.message : String(e));
diff --git a/castle-search/src/extract.ts b/castle-search/src/extract.ts
new file mode 100644
index 0000000..8b5c57f
--- /dev/null
+++ b/castle-search/src/extract.ts
@@ -0,0 +1,294 @@
+/**
+ * The script that runs inside the search results page.
+ *
+ * It is built as a string rather than shipped as a function because it goes to
+ * Chrome via `Runtime.evaluate` — nothing in here is typechecked, so it is kept
+ * defensive and free of anything that could throw on an unexpected DOM. The
+ * per-engine configuration is injected as JSON by {@link buildProbeScript}.
+ *
+ * ## Two modes, because the engines genuinely differ
+ *
+ * **"items"** — DuckDuckGo, Bing and Brave each have a clean per-result
+ * container, so results are read directly out of it.
+ *
+ * **"headings"** — Google has no stable result container; its class names
+ * (`MjjYud`, `kb0PBd`, `yuRUbf`) are generated and change without notice. So
+ * results are found from each `<h3>` outward:
+ *
+ * - anchor = nearest enclosing `<a href>`
+ * - header = nearest `[data-snhf]`, else the anchor
+ * - container = climb from the header until the text grows past the header's,
+ * abandoning the climb if a second `<h3>` comes into scope
+ * (that would mean the next result had been swallowed)
+ * - snippet = `[data-sncf]` if present, else the container's lines minus the
+ * header's lines, minus URL and breadcrumb noise
+ *
+ * Verified against Google's classic SERP and its `udm=14` layout, DuckDuckGo's
+ * no-JS endpoint, Bing's `li.b_algo`, and Brave's `.snippet[data-type=web]`.
+ *
+ * ## Link unwrapping
+ *
+ * DuckDuckGo and Bing both route outbound links through a redirector, so the
+ * raw href is useless to the agent. Both are unwrapped in the page, where the
+ * URL and base64 primitives already exist:
+ *
+ * - DuckDuckGo: `//duckduckgo.com/l/?uddg=<percent-encoded target>`
+ * - Bing: `//bing.com/ck/a?…&u=a1<base64url of target>`
+ *
+ * Google and Brave link straight out and need no unwrapping.
+ */
+
+import type { ExtractionConfig } from "./engines.ts";
+
+/** One search hit, as the page script reports it. */
+export type SearchResult = {
+ title: string;
+ url: string;
+ snippet: string;
+};
+
+/** What the page script returns in a single round trip. */
+export type PageProbe = {
+ /** Where the page actually ended up — redirects and consent walls move it. */
+ url: string;
+ title: string;
+ readyState: string;
+ /** Non-null when a human check is in the way. */
+ challenge: { kind: string; detail: string } | null;
+ /**
+ * Whether the engine's results container was present at all. "none" means a
+ * page shell with no results area — a different failure from an empty one,
+ * and worth telling apart when diagnosing a zero-hit search.
+ */
+ container: "found" | "none";
+ results: SearchResult[];
+};
+
+/**
+ * Challenge detection, shared across engines.
+ *
+ * Every phrase here has been seen on a real page during development: Google's
+ * `/sorry/` interstitial, and Brave's "Verifying you're not a bot / Quick check
+ * before you continue searching" — the latter is why the wording list is broad
+ * rather than just matching Google's "unusual traffic". Ordering matters: a
+ * `/sorry/` page also contains a recaptcha iframe, and naming the page beats
+ * naming the widget sitting on it.
+ */
+const CHALLENGE_DETECTION = String.raw`
+ const detectChallenge = () => {
+ if (/\/sorry\//.test(location.href)) return { kind: "google-block-page", detail: bodyText.slice(0, 300) };
+ if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname))
+ return { kind: "consent-wall", detail: bodyText.slice(0, 300) };
+ if (document.querySelector("#captcha-form, form#captcha-form"))
+ return { kind: "captcha-form", detail: bodyText.slice(0, 300) };
+ if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running, [id^=cf-chl]"))
+ return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) };
+ if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[src*="turnstile"], iframe[title*="challenge"]'))
+ return { kind: "captcha-widget", detail: bodyText.slice(0, 300) };
+ if (/verifying (that )?you('| a)?re( not)? a? ?(human|bot|robot)|quick check before you continue|verify you are human|are you a robot|not a robot/i.test(bodyText))
+ return { kind: "bot-check", detail: bodyText.slice(0, 300) };
+ if (/unusual traffic|automated queries|suspicious activity from your/i.test(bodyText))
+ return { kind: "rate-limit-block", detail: bodyText.slice(0, 300) };
+ return null;
+ };
+`;
+
+/**
+ * Build the probe for one engine. The configuration is embedded as a JSON
+ * literal so the script stays a single self-contained expression.
+ */
+export const buildProbeScript = (config: ExtractionConfig): string => String.raw`(() => {
+ var CFG = ${JSON.stringify(config)};
+
+ var norm = function (s) { return (s || "").replace(/\s+/g, " ").trim(); };
+ var linesOf = function (el) {
+ return el && el.innerText ? el.innerText.split("\n").map(function (l) { return l.trim(); }).filter(Boolean) : [];
+ };
+ var bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : "";
+ var selfHost = new RegExp(CFG.selfHostPattern);
+
+ ${CHALLENGE_DETECTION}
+
+ // DuckDuckGo and Bing both hide the real destination behind a redirector.
+ var unwrap = function (href) {
+ try {
+ var u = new URL(href, location.href);
+ if (CFG.unwrap === "ddg") {
+ if (!/(^|\.)duckduckgo\.com$/.test(u.hostname)) return href;
+ return u.searchParams.get("uddg") || href;
+ }
+ if (CFG.unwrap === "bing") {
+ if (!/(^|\.)bing\.com$/.test(u.hostname)) return href;
+ var p = u.searchParams.get("u");
+ if (!p) return href;
+ var b64 = p.replace(/^a1/, "").replace(/-/g, "+").replace(/_/g, "/");
+ b64 += "=".repeat((4 - (b64.length % 4)) % 4);
+ var bin = atob(b64);
+ var bytes = new Uint8Array(bin.length);
+ for (var i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+ return new TextDecoder().decode(bytes);
+ }
+ return href;
+ } catch (e) {
+ // An unwrap that fails leaves the redirector URL in place rather than
+ // dropping the result: a working link the agent has to follow twice beats
+ // no link at all.
+ return href;
+ }
+ };
+
+ var findRoot = function () {
+ for (var i = 0; i < CFG.roots.length; i++) {
+ var el = document.querySelector(CFG.roots[i]);
+ if (el) return el;
+ }
+ return null;
+ };
+
+ var acceptable = function (url) {
+ return /^https?:/.test(url) && !selfHost.test(url);
+ };
+
+ // ── items mode ───────────────────────────────────────────────────────────
+ var extractItems = function (root) {
+ var out = [];
+ var seen = {};
+ var items = root.querySelectorAll(CFG.item);
+ for (var i = 0; i < items.length; i++) {
+ var item = items[i];
+
+ var skip = false;
+ for (var x = 0; CFG.exclude && x < CFG.exclude.length; x++) {
+ if (item.matches(CFG.exclude[x]) || item.querySelector(CFG.exclude[x])) { skip = true; break; }
+ }
+ if (skip) continue;
+
+ var anchor = item.querySelector(CFG.link);
+ if (!anchor) continue;
+ var url = unwrap(anchor.href);
+ if (!acceptable(url) || seen[url]) continue;
+
+ var titleEl = CFG.title ? item.querySelector(CFG.title) : null;
+ var title = norm(titleEl ? titleEl.innerText : anchor.innerText);
+ if (!title) continue;
+
+ var snippet = "";
+ var snippetEl = CFG.snippet ? item.querySelector(CFG.snippet) : null;
+ if (snippetEl) {
+ snippet = norm(snippetEl.innerText);
+ } else {
+ // No dedicated description element. Rather than guess at class names
+ // for the source and breadcrumb, use their position: an engine that
+ // renders "source / breadcrumb / title / description" puts every piece
+ // of metadata *before* the title, so everything after the title line is
+ // the description. Class names churn; that ordering does not.
+ var itemLines = linesOf(item);
+ var titleIndex = -1;
+ for (var t = 0; t < itemLines.length; t++) {
+ if (norm(itemLines[t]) === title) { titleIndex = t; break; }
+ }
+
+ var candidate;
+ if (titleIndex >= 0) {
+ candidate = itemLines.slice(titleIndex + 1);
+ } else {
+ // Title is not its own line (it may be inline with other text).
+ // Fall back to subtracting the parts we can name.
+ var drop = {};
+ for (var s = 0; CFG.subtract && s < CFG.subtract.length; s++) {
+ var parts = item.querySelectorAll(CFG.subtract[s]);
+ for (var p = 0; p < parts.length; p++) {
+ var pl = linesOf(parts[p]);
+ for (var q = 0; q < pl.length; q++) drop[norm(pl[q])] = true;
+ }
+ }
+ candidate = itemLines.filter(function (l) { return !drop[norm(l)]; });
+ }
+
+ snippet = norm(
+ candidate.filter(function (l) {
+ var n = norm(l);
+ return n !== title && !/^https?:\/\//.test(n) && n.indexOf("›") === -1;
+ }).join(" "),
+ );
+ }
+
+ seen[url] = true;
+ out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
+ }
+ return out;
+ };
+
+ // ── headings mode (Google) ───────────────────────────────────────────────
+ var extractHeadings = function (root) {
+ var out = [];
+ var seen = {};
+ var headings = root.querySelectorAll("h3");
+ for (var i = 0; i < headings.length; i++) {
+ var h3 = headings[i];
+ var anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]"));
+ if (!anchor) continue;
+
+ var url = unwrap(anchor.href);
+ if (!acceptable(url) || seen[url]) continue;
+
+ var title = norm(h3.innerText);
+ if (!title) continue;
+
+ var header = h3.closest("[data-snhf]") || anchor;
+ var headerLines = {};
+ var hl = linesOf(header);
+ for (var k = 0; k < hl.length; k++) headerLines[norm(hl[k])] = true;
+ var baseline = norm(header.innerText || "").length;
+
+ var container = header.parentElement;
+ for (var depth = 0; depth < 6 && container && container !== root; depth++) {
+ if (container.querySelectorAll("h3").length > 1) { container = null; break; }
+ if (norm(container.innerText || "").length > baseline + 40) break;
+ container = container.parentElement;
+ }
+
+ var snippet = "";
+ if (container && container !== root) {
+ var explicit = container.querySelector("[data-sncf]");
+ var body = explicit
+ ? linesOf(explicit)
+ : linesOf(container).filter(function (l) { return !headerLines[norm(l)]; });
+ snippet = norm(
+ body.filter(function (l) {
+ return !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results";
+ }).join(" "),
+ ).replace(/Read more$/, "").trim();
+ }
+
+ seen[url] = true;
+ out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
+ }
+ return out;
+ };
+
+ var challenge = null;
+ var results = [];
+ var container = "none";
+ try {
+ challenge = detectChallenge();
+ } catch (e) {
+ challenge = null;
+ }
+ try {
+ var root = findRoot();
+ container = root ? "found" : "none";
+ if (root && !challenge) results = CFG.mode === "items" ? extractItems(root) : extractHeadings(root);
+ } catch (e) {
+ results = [];
+ }
+
+ return {
+ url: location.href,
+ title: document.title || "",
+ readyState: document.readyState,
+ challenge: challenge,
+ container: container,
+ results: results,
+ };
+})()`;
diff --git a/castle-search/src/format.ts b/castle-search/src/format.ts
new file mode 100644
index 0000000..aef6c3b
--- /dev/null
+++ b/castle-search/src/format.ts
@@ -0,0 +1,54 @@
+/**
+ * Turning results into the text the model reads.
+ *
+ * Twenty results with 600-character snippets is roughly 15KB, comfortably under
+ * pi's 50KB / 2000-line budget — but the truncation is applied anyway rather
+ * than argued about, because a pathological page could blow past it and an
+ * overflowing tool result costs the whole session, not just this call.
+ */
+
+import type { SearchResult } from "./extract.ts";
+
+export type FormatInput = {
+ query: string;
+ searchUrl: string;
+ finalUrl: string;
+ endpoint: string;
+ /** Engine display name, e.g. "DuckDuckGo". */
+ engine: string;
+ /** Where the engine choice came from, e.g. "/search-engine". */
+ engineSource: string;
+ results: SearchResult[];
+};
+
+export const formatResults = (input: FormatInput): string => {
+ const lines: string[] = [
+ // Naming the engine on every result set matters more than it looks: the
+ // agent may have switched engines mid-conversation to get around a block,
+ // and results from different indexes are not interchangeable evidence.
+ `Search results for "${input.query}" — ${input.results.length} hits from ${input.engine} ` +
+ `(${input.engineSource}), via Chrome at ${input.endpoint}`,
+ `Query URL: ${input.searchUrl}`,
+ ];
+ if (input.finalUrl !== input.searchUrl) lines.push(`Landed on: ${input.finalUrl}`);
+ lines.push("");
+
+ input.results.forEach((result, index) => {
+ lines.push(`${index + 1}. ${result.title}`);
+ lines.push(` ${result.url}`);
+ if (result.snippet) lines.push(` ${result.snippet}`);
+ lines.push("");
+ });
+
+ return lines.join("\n").trimEnd();
+};
+
+/**
+ * The note appended when pi's truncation utilities cut the output. Says what was
+ * lost and what to do about it — a bare "[truncated]" tells the model nothing it
+ * can act on, and there is no temp file to point at because the full text only
+ * ever existed in memory.
+ */
+export const truncationNote = (shown: number, total: number, bytesShown: string, bytesTotal: string): string =>
+ `\n\n[Output truncated: ${shown} of ${total} lines (${bytesShown} of ${bytesTotal}). ` +
+ `Re-run castle_cdp_search with a smaller numResults to see the rest.]`;
diff --git a/castle-search/src/index.ts b/castle-search/src/index.ts
new file mode 100644
index 0000000..29cdaaf
--- /dev/null
+++ b/castle-search/src/index.ts
@@ -0,0 +1,315 @@
+/**
+ * castle-cdp-search — web search for pi through a Chrome that is already
+ * running somewhere on the wire network, driven over CDP.
+ *
+ * It does not launch a browser. It attaches to one that a person or a launchd
+ * job started, using the same endpoint configuration as pi-browser-harness
+ * (`BU_CDP_HTTP` > `/browser-target` > castle), so a single choice governs both
+ * packages and they can never end up driving different browsers.
+ *
+ * The engine is switchable the same way: `CCS_SEARCH_ENGINE` > `/search-engine`
+ * > Google, plus a per-call `engine` parameter so the agent can fall back when
+ * one engine starts serving captchas.
+ *
+ * The tool is called `castle_cdp_search`, not `web_search`, on purpose: it is
+ * meant to sit alongside pi-web-access rather than shadow it. The model picks
+ * between them, so both stay available in one session.
+ */
+
+import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, truncateHead } from "@earendil-works/pi-coding-agent";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { StringEnum } from "@earendil-works/pi-ai";
+import { Type } from "typebox";
+
+import { clearStoredEngine, storedEngineLocation, writeStoredEngine } from "./engine-store.ts";
+import { DEFAULT_ENGINE, ENGINE_ENV_VAR } from "./engine-target.ts";
+import {
+ ENGINE_IDS,
+ RECENCY_VALUES,
+ type EngineId,
+ type Recency,
+ allEngines,
+ describeRecencySupport,
+ engineAliases,
+ parseEngineId,
+} from "./engines.ts";
+import { messageOf } from "./errors.ts";
+import type { SearchResult } from "./extract.ts";
+import { formatResults, truncationNote } from "./format.ts";
+import {
+ MAX_CONCURRENT_SEARCHES,
+ SEARCH_TIMEOUT_MS,
+ type SearchPhase,
+ describeConfiguredEndpoint,
+ describeConfiguredEngine,
+ disconnect,
+ search,
+} from "./search.ts";
+
+const DEFAULT_NUM_RESULTS = 10;
+const MIN_NUM_RESULTS = 1;
+const MAX_NUM_RESULTS = 20;
+
+const parameters = Type.Object({
+ query: Type.String({
+ description: "The search query, phrased as you would type it into a search engine.",
+ minLength: 1,
+ }),
+ numResults: Type.Optional(
+ Type.Integer({
+ description: `How many results to return (${MIN_NUM_RESULTS}-${MAX_NUM_RESULTS}, default ${DEFAULT_NUM_RESULTS}).`,
+ minimum: MIN_NUM_RESULTS,
+ maximum: MAX_NUM_RESULTS,
+ default: DEFAULT_NUM_RESULTS,
+ }),
+ ),
+ // StringEnum, not Type.Union of Type.Literal: Google's API rejects the
+ // anyOf/const shape typebox emits for unions.
+ recency: Type.Optional(
+ StringEnum(RECENCY_VALUES, {
+ description:
+ "Restrict results to pages published within this window. Omit for no time limit. " +
+ "Not every engine supports every window — bing has no year, brave has no time filter at all.",
+ }),
+ ),
+ engine: Type.Optional(
+ StringEnum(ENGINE_IDS, {
+ description:
+ "Which search engine to use for this call. Omit to use the configured default. " +
+ "Switch engines when one returns a challenge or no results — they have independent indexes and blocks.",
+ }),
+ ),
+});
+
+/** Exported so `isToolCallEventType<"castle_cdp_search", CastleCdpSearchInput>` can type it. */
+export type CastleCdpSearchInput = {
+ query: string;
+ numResults?: number;
+ recency?: Recency;
+ engine?: EngineId;
+};
+
+export type CastleCdpSearchDetails = {
+ query: string;
+ numResults: number;
+ recency: Recency | null;
+ engine: EngineId | null;
+ phase: SearchPhase | "done";
+ note: string;
+ endpoint: string | null;
+ searchUrl: string | null;
+ results: SearchResult[];
+ truncated: boolean;
+};
+
+const initialDetails = (
+ query: string,
+ numResults: number,
+ recency: Recency | null,
+ engine: EngineId | null,
+): CastleCdpSearchDetails => ({
+ query,
+ numResults,
+ recency,
+ engine,
+ phase: "queued",
+ note: "starting",
+ endpoint: null,
+ searchUrl: null,
+ results: [],
+ truncated: false,
+});
+
+/** The engine table, rendered for `/search-engine` and for the tool description. */
+const engineTable = (): string =>
+ allEngines()
+ .map((e) => ` ${e.id.padEnd(11)} ${e.label.padEnd(14)} recency: ${describeRecencySupport(e).padEnd(24)} ${e.note}`)
+ .join("\n");
+
+/**
+ * Say something from a command, in whichever mode pi is running.
+ *
+ * `ctx.ui.notify` is a no-op when `hasUI` is false, which means `pi -p
+ * "/search-engine"` would change the stored engine and print absolutely nothing
+ * — the command would look broken. Print mode falls back to stdout. JSON mode
+ * deliberately does not: stdout there is a structured event stream, and a stray
+ * line would corrupt it.
+ */
+const say = (ctx: { hasUI: boolean; mode: string; ui: { notify: (m: string, t?: "info" | "warning" | "error") => void } },
+ message: string,
+ level: "info" | "warning" | "error" = "info",
+): void => {
+ if (ctx.hasUI) {
+ ctx.ui.notify(message, level);
+ return;
+ }
+ if (ctx.mode === "print") console.log(message);
+};
+
+export default function (pi: ExtensionAPI): void {
+ // Nothing is dialled here. pi runs extension factories in invocations that
+ // never start a session, so opening a socket from a factory would leave one
+ // behind on, say, `pi --list-models`. The connection is made on first use and
+ // torn down below.
+
+ pi.on("session_shutdown", async () => {
+ disconnect();
+ });
+
+ // The engine analogue of pi-browser-harness's /browser-target. Persisted
+ // machine-wide for the same reason: the choice should outlive the session
+ // that made it.
+ pi.registerCommand("search-engine", {
+ description: "Show or set the search engine castle_cdp_search uses",
+ handler: async (args, ctx) => {
+ const raw = args.trim();
+ const current = await describeConfiguredEngine();
+
+ if (raw === "") {
+ say(
+ ctx,
+ `castle_cdp_search engine: ${current.id} (${current.source})\n\n${engineTable()}\n\n` +
+ `Set with: /search-engine <name> Clear with: /search-engine default\n` +
+ `Stored in ${storedEngineLocation()}; ${ENGINE_ENV_VAR} overrides it for one process.`,
+ );
+ return;
+ }
+
+ if (raw.toLowerCase() === "default" || raw.toLowerCase() === "clear") {
+ await clearStoredEngine();
+ say(ctx, `castle_cdp_search engine reset to the built-in default (${DEFAULT_ENGINE}).`);
+ return;
+ }
+
+ const id = parseEngineId(raw);
+ if (!id) {
+ say(ctx, `Unknown search engine "${raw}". Expected one of: ${engineAliases().join(", ")}`, "error");
+ return;
+ }
+
+ await writeStoredEngine(id);
+ // Saying so explicitly, because the variable silently wins otherwise and
+ // the user would reasonably assume the command had taken effect.
+ const pinned = process.env[ENGINE_ENV_VAR]?.trim();
+ say(
+ ctx,
+ pinned
+ ? `Saved ${id}, but ${ENGINE_ENV_VAR}=${pinned} is set and overrides it for this process.`
+ : `castle_cdp_search will use ${id}.`,
+ pinned ? "warning" : "info",
+ );
+ },
+ });
+
+ pi.registerTool({
+ name: "castle_cdp_search",
+ label: "Castle Search",
+ description:
+ "Search the web using a real Chrome browser running on the private network, driven over the Chrome " +
+ "DevTools Protocol. Returns ranked results with titles, URLs and snippets. Because it uses a real " +
+ "logged-in browser rather than a search API, it reaches pages that block datacentre traffic. It " +
+ "returns search results only — it does not fetch or read the linked pages, so follow up with a " +
+ "fetch/read tool for full page content. Supports four engines with independent indexes and " +
+ `independent rate limits (${ENGINE_IDS.join(", ")}); pass engine to switch when one is blocked. ` +
+ `Output is truncated at ${formatSize(DEFAULT_MAX_BYTES)} or ${DEFAULT_MAX_LINES} lines.`,
+ promptSnippet:
+ "Search the web through a real Chrome on the private network (castle_cdp_search); google/duckduckgo/bing/brave, returns titles, URLs and snippets",
+ promptGuidelines: [
+ "Use castle_cdp_search to find pages on the open web when you need current information, documentation, " +
+ "or sources you do not already have — it drives a real signed-in Chrome, so it works on sites that " +
+ "reject scripted clients.",
+ "castle_cdp_search returns search results only. To read a result, follow it up with a tool that fetches " +
+ "page content; do not treat the snippet as the full page.",
+ "If castle_cdp_search fails with a SearchChallengeError or returns no results, retry the same query with " +
+ "its engine parameter set to a different engine (google, duckduckgo, bing, brave) before giving up — " +
+ "they have independent indexes and independent blocks, and a challenge on one says nothing about the others.",
+ "Only ask the user for help with a castle_cdp_search captcha once more than one engine has failed; " +
+ "when you do, give them the URL from the error so they can clear it in that browser.",
+ "Pass recency to castle_cdp_search when the answer depends on how recent a page is, and keep numResults " +
+ "small (5-10) unless you genuinely need a wide sweep. Not every engine supports every window: bing has " +
+ "no year, and brave has no time filter, so castle_cdp_search will tell you to switch engines rather than " +
+ "silently ignoring the request.",
+ ],
+ parameters,
+
+ async execute(_toolCallId, params, signal, onUpdate, _ctx) {
+ const query = params.query.trim();
+ if (query === "") throw new Error("query must not be empty");
+
+ const numResults = Math.min(
+ MAX_NUM_RESULTS,
+ Math.max(MIN_NUM_RESULTS, params.numResults ?? DEFAULT_NUM_RESULTS),
+ );
+ const recency = (params.recency ?? null) as Recency | null;
+ const engine = (params.engine ?? null) as EngineId | null;
+
+ const details = initialDetails(query, numResults, recency, engine);
+
+ // One deadline covering connect + navigate + extract, plus the user's own
+ // abort so Esc drops an in-flight navigation rather than waiting it out.
+ const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+ const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
+
+ const report = (phase: SearchPhase, note: string): void => {
+ details.phase = phase;
+ details.note = note;
+ // Keep the TUI moving during the seconds Chrome spends navigating.
+ onUpdate?.({ content: [{ type: "text", text: `${note}…` }], details: { ...details } });
+ };
+
+ try {
+ const outcome = await search(
+ { query, numResults, recency: recency ?? undefined, engine },
+ combined,
+ report,
+ );
+
+ const body = formatResults({
+ query,
+ searchUrl: outcome.searchUrl,
+ finalUrl: outcome.finalUrl,
+ endpoint: outcome.endpoint,
+ engine: outcome.engine.label,
+ engineSource: outcome.engineSource,
+ results: outcome.results,
+ });
+
+ const truncation = truncateHead(body, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
+ const text = truncation.truncated
+ ? truncation.content +
+ truncationNote(
+ truncation.outputLines,
+ truncation.totalLines,
+ formatSize(truncation.outputBytes),
+ formatSize(truncation.totalBytes),
+ )
+ : truncation.content;
+
+ details.phase = "done";
+ details.note = `${outcome.results.length} results from ${outcome.engine.id}`;
+ details.engine = outcome.engine.id;
+ details.endpoint = outcome.endpoint;
+ details.searchUrl = outcome.searchUrl;
+ details.results = outcome.results;
+ details.truncated = truncation.truncated;
+
+ return { content: [{ type: "text", text }], details };
+ } catch (e) {
+ // Rethrow, always. pi only marks a tool result as an error when execute()
+ // throws — returning `{ error }` would read to the model as a search that
+ // simply found nothing, which is the one wrong conclusion to invite here.
+ const endpoint = await describeConfiguredEndpoint().catch(() => "unknown endpoint");
+ const configured = await describeConfiguredEngine().catch(() => null);
+ const e2 = e instanceof Error ? e : new Error(messageOf(e));
+ // Naming the phase turns "it timed out" into something actionable:
+ // "queued" means the cap was the bottleneck, "navigating" means the
+ // browser or the network was.
+ e2.message =
+ `${e2.message} [castle_cdp_search: failed during "${details.phase}" — engine ` +
+ `${engine ?? configured?.id ?? "unknown"}, browser ${endpoint}, up to ${MAX_CONCURRENT_SEARCHES} ` +
+ `concurrent searches, ${SEARCH_TIMEOUT_MS / 1000}s deadline covering the whole call]`;
+ throw e2;
+ }
+ },
+ });
+}
diff --git a/castle-search/src/paths.ts b/castle-search/src/paths.ts
new file mode 100644
index 0000000..b008f4f
--- /dev/null
+++ b/castle-search/src/paths.ts
@@ -0,0 +1,41 @@
+/**
+ * pi's agent config directory, and the file pi-browser-harness stores its
+ * `/browser-target` choice in.
+ *
+ * This extension reads that file so a single `/browser-target` decision governs
+ * both packages. It never writes it — the harness owns that setting, and two
+ * writers of one file is a bug waiting to happen.
+ *
+ * pi exports `getAgentDir()`, but resolving it here would be a *runtime* import
+ * of the host package from an installed extension's own tree. Every other pi
+ * import in this codebase is `import type` and erased at build time, so the
+ * three lines are replicated instead (pi's dist/config.js: `PI_CODING_AGENT_DIR`,
+ * CONFIG_DIR_NAME `.pi`). Same reasoning as pi-browser-harness/src/profile/paths.ts.
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+/** pi's env override for the agent dir (APP_NAME.toUpperCase() + "_CODING_AGENT_DIR"). */
+const ENV_AGENT_DIR = "PI_CODING_AGENT_DIR";
+
+/**
+ * Expand a leading `~` exactly as pi's expandTildePath does — `~` and `~/…`
+ * only, never `~\…`, on every platform. Being more lenient here would look in a
+ * different directory than pi resolves from the same variable.
+ */
+const expandTildePiCompatible = (path: string): string => {
+ if (path === "~") return homedir();
+ if (path.startsWith("~/")) return homedir() + path.slice(1);
+ return path;
+};
+
+/** pi's agent config directory — `$PI_CODING_AGENT_DIR` or `~/.pi/agent`. */
+export const agentDir = (): string => {
+ const fromEnv = process.env[ENV_AGENT_DIR];
+ if (fromEnv) return expandTildePiCompatible(fromEnv);
+ return join(homedir(), ".pi", "agent");
+};
+
+/** Where pi-browser-harness persists the `/browser-target` choice. */
+export const targetFilePath = (): string => join(agentDir(), "browser-target.json");
diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts
new file mode 100644
index 0000000..1306443
--- /dev/null
+++ b/castle-search/src/search.ts
@@ -0,0 +1,339 @@
+/**
+ * Search orchestration: resolve a browser, borrow a page, navigate, extract,
+ * hand the page back.
+ *
+ * The connection is process-wide and lazy. It is *not* opened by the extension
+ * factory — pi runs factories in invocations that never start a session, and
+ * the docs are explicit that sockets must not start there. It is opened on the
+ * first search and closed by an idempotent `session_shutdown`.
+ */
+
+import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts";
+import {
+ type CdpTarget,
+ describeCdpTarget,
+ describeResolutionFailure,
+ resolveCdpTarget,
+} from "./endpoint.ts";
+import { describeEngineFailure, describeEngineSource, resolveEngine } from "./engine-target.ts";
+import { readStoredEngine } from "./engine-store.ts";
+import {
+ ENGINE_IDS as allEngineIdList,
+ type EngineDefinition,
+ type EngineId,
+ type Recency,
+ describeRecencySupport,
+ enginesSupportingRecency,
+ getEngine,
+} from "./engines.ts";
+
+const allEngineIds = (): readonly EngineId[] => allEngineIdList;
+import {
+ BrowserUnavailableError,
+ NoResultsError,
+ RecencyUnsupportedError,
+ SearchChallengeError,
+} from "./errors.ts";
+import { buildProbeScript, type PageProbe, type SearchResult } from "./extract.ts";
+import { readStoredTarget } from "./target-store.ts";
+
+/** Per-search wall clock, covering connect, navigate and extract together. */
+export const SEARCH_TIMEOUT_MS = 20_000;
+/** How many searches may drive the shared browser at once. */
+export const MAX_CONCURRENT_SEARCHES = 2;
+/** Results sometimes land a beat after the load event; re-read a couple of times. */
+const EXTRACT_RETRIES = 2;
+const EXTRACT_RETRY_DELAY_MS = 700;
+
+export type { Recency } from "./engines.ts";
+
+export type SearchRequest = {
+ query: string;
+ numResults: number;
+ recency?: Recency | undefined;
+ /** Per-call engine override; falls back to the configured default. */
+ engine?: string | null | undefined;
+};
+
+export type SearchOutcome = {
+ results: SearchResult[];
+ searchUrl: string;
+ /** The URL the page settled on, which differs from searchUrl after a redirect. */
+ finalUrl: string;
+ endpoint: string;
+ engine: EngineDefinition;
+ /** Where the engine choice came from, for the "via" line in the output. */
+ engineSource: string;
+};
+
+/** Build the SERP URL for a request. Exported for tests. */
+export const buildSearchUrl = (engine: EngineDefinition, request: SearchRequest): string =>
+ engine.buildUrl(request.query, request.numResults, request.recency);
+
+/**
+ * Resolve the engine, refusing rather than silently ignoring a recency window
+ * the chosen engine cannot express.
+ *
+ * Dropping an unsupported filter would be the same failure mode as Google's
+ * `tbs=qdr:` — the caller asks for recent results, gets whatever the engine
+ * ranked, and has no way to tell. Naming the engines that *do* support the
+ * window makes the error actionable, since the model can retry with one.
+ */
+export const selectEngine = async (
+ request: SearchRequest,
+): Promise<{ engine: EngineDefinition; source: string }> => {
+ const resolution = resolveEngine(request.engine, process.env, await readStoredEngine());
+ if (resolution.kind === "invalid") {
+ throw new BrowserUnavailableError(describeEngineFailure(resolution));
+ }
+ const engine = getEngine(resolution.id);
+
+ if (request.recency && engine.recency[request.recency] === undefined) {
+ const alternatives = enginesSupportingRecency(request.recency);
+ throw new RecencyUnsupportedError(
+ `${engine.label} cannot restrict results to the past ${request.recency}. ` +
+ `It supports: ${describeRecencySupport(engine)}. ` +
+ (alternatives.length > 0
+ ? `Engines that support "${request.recency}": ${alternatives.join(", ")}. ` +
+ `Retry with engine set to one of those, or drop the recency parameter.`
+ : `No configured engine supports that window; drop the recency parameter.`),
+ );
+ }
+
+ return { engine, source: describeEngineSource(resolution.source) };
+};
+
+/** For status reporting — resolves the configured engine without searching. */
+export const describeConfiguredEngine = async (): Promise<{ id: EngineId; source: string; label: string }> => {
+ const resolution = resolveEngine(null, process.env, await readStoredEngine());
+ if (resolution.kind === "invalid") {
+ return { id: "google", source: describeEngineFailure(resolution), label: "unresolved" };
+ }
+ return {
+ id: resolution.id,
+ source: describeEngineSource(resolution.source),
+ label: getEngine(resolution.id).label,
+ };
+};
+
+// ── connection lifecycle ───────────────────────────────────────────────────
+
+let connection: BrowserConnection | null = null;
+/** In-flight connect, so two concurrent searches share one dial rather than racing. */
+let connecting: Promise<BrowserConnection> | null = null;
+
+const resolveTarget = async (): Promise<CdpTarget> => {
+ const resolution = resolveCdpTarget(process.env, await readStoredTarget());
+ if (resolution.kind === "invalid") {
+ throw new BrowserUnavailableError(describeResolutionFailure(resolution));
+ }
+ return resolution.target;
+};
+
+/**
+ * Return a live connection, dialling one if there isn't one or the last one
+ * died. Reconnecting transparently matters here: castle recycles Chrome hourly,
+ * so a session that searched an hour ago is holding a dead socket.
+ */
+const ensureConnection = async (signal?: AbortSignal): Promise<BrowserConnection> => {
+ if (connection && connection.isOpen) return connection;
+ if (connection) {
+ connection.close();
+ connection = null;
+ }
+
+ if (!connecting) {
+ // Assigned synchronously, before the first await inside: two searches
+ // starting in the same tick must share one dial, not open two sockets and
+ // leak whichever loses the assignment race.
+ connecting = (async () => {
+ const target = await resolveTarget();
+ const conn = await BrowserConnection.connect(target);
+ connection = conn;
+ return conn;
+ })().finally(() => {
+ connecting = null;
+ });
+ }
+
+ // The caller's signal is raced here rather than passed into connect(). A
+ // shared dial must not be cancelled by whichever caller happened to start it
+ // — one search pressing Esc would otherwise fail the other search waiting on
+ // the same connection. connect() is bounded by its own timeouts regardless.
+ return raceAbort(connecting, signal);
+};
+
+/** Idempotent: safe to call from `session_shutdown` however many times it fires. */
+export const disconnect = (): void => {
+ connection?.close();
+ connection = null;
+};
+
+/** For status reporting — never dials. */
+export const describeConfiguredEndpoint = async (): Promise<string> => {
+ const resolution = resolveCdpTarget(process.env, await readStoredTarget());
+ return resolution.kind === "invalid"
+ ? describeResolutionFailure(resolution)
+ : describeCdpTarget(resolution.target);
+};
+
+// ── concurrency cap ────────────────────────────────────────────────────────
+
+let active = 0;
+const waiting: Array<() => void> = [];
+
+/** Give a held slot to the next waiter, or return it to the pool. */
+const handOff = (): void => {
+ const next = waiting.shift();
+ // Hand the slot straight over rather than decrementing and letting the waiter
+ // re-check, so a queued search cannot be overtaken by a newly arriving one.
+ if (next) next();
+ else active--;
+};
+
+const acquire = async (signal?: AbortSignal): Promise<() => void> => {
+ if (active < MAX_CONCURRENT_SEARCHES) {
+ active++;
+ } else {
+ let resolver!: () => void;
+ const queued = new Promise<void>((resolve) => {
+ resolver = resolve;
+ });
+ waiting.push(resolver);
+ try {
+ await raceAbort(queued, signal);
+ // The slot was handed over by handOff(), which already counted it.
+ } catch (e) {
+ // An abort while queued must not strand the slot. Either we are still in
+ // the queue — drop out of it — or the slot was handed to us in the same
+ // tick the abort landed, in which case give it back. Without this, two
+ // cancelled-while-queued searches would wedge the semaphore permanently.
+ const index = waiting.indexOf(resolver);
+ if (index !== -1) waiting.splice(index, 1);
+ else handOff();
+ throw e;
+ }
+ }
+ let released = false;
+ return () => {
+ if (released) return;
+ released = true;
+ handOff();
+ };
+};
+
+// ── the search itself ──────────────────────────────────────────────────────
+
+export type ProgressReporter = (phase: SearchPhase, note: string) => void;
+export type SearchPhase = "queued" | "connecting" | "opening" | "navigating" | "extracting";
+
+const sleep = (ms: number, signal?: AbortSignal): Promise<void> =>
+ raceAbort(new Promise<void>((resolve) => setTimeout(resolve, ms)), signal);
+
+/**
+ * The other engines, named in failure messages. A blocked engine is the case
+ * where the agent most needs to know it has somewhere else to go, and telling
+ * it inside the error beats hoping it remembers the guidelines.
+ */
+const otherEngines = (current: EngineId): readonly string[] =>
+ allEngineIds().filter((id) => id !== current);
+
+/**
+ * Run one search. Throws on every failure — see errors.ts for why a returned
+ * error object would be worse than useless here.
+ */
+export const search = async (
+ request: SearchRequest,
+ signal: AbortSignal,
+ report: ProgressReporter,
+): Promise<SearchOutcome> => {
+ // Reported before acquiring, so a search that dies waiting for a slot is
+ // distinguishable from one that died reaching the browser. The deadline is
+ // the caller's and covers the queue wait too — with a cap of two and searches
+ // that take a couple of seconds, waiting is the rare case, and a tool call
+ // that silently takes twice its stated deadline would be worse.
+ // Resolved before acquiring a slot: an unknown engine name or an unsupported
+ // recency window is a caller error, and should not wait behind other searches
+ // or touch the browser at all.
+ const { engine, source: engineSource } = await selectEngine(request);
+ const probeScript = buildProbeScript(engine.extraction);
+
+ report("queued", "waiting for a search slot");
+ const release = await acquire(signal);
+ try {
+ report("connecting", "resolving browser");
+ const conn = await ensureConnection(signal);
+ const endpoint = describeCdpTarget(conn.target);
+
+ report("opening", `opening a page on ${endpoint}`);
+ const page = await PageSession.open(conn, signal);
+ const searchUrl = buildSearchUrl(engine, request);
+
+ try {
+ report("navigating", `searching ${engine.label} for "${request.query}"`);
+ await page.navigate(searchUrl, signal);
+
+ report("extracting", "reading results");
+ let probe = await page.evaluate<PageProbe>(probeScript, signal);
+ for (let attempt = 0; attempt < EXTRACT_RETRIES; attempt++) {
+ if (probe?.challenge || (probe?.results?.length ?? 0) > 0) break;
+ await sleep(EXTRACT_RETRY_DELAY_MS, signal);
+ probe = await page.evaluate<PageProbe>(probeScript, signal);
+ }
+
+ if (!probe) {
+ throw new BrowserUnavailableError("the page returned nothing — the extraction script did not run");
+ }
+
+ if (probe.challenge) {
+ throw new SearchChallengeError({
+ challenge: probe.challenge.kind,
+ // The page the human must clear is where the browser *ended up*, which
+ // after a consent redirect is not the URL we asked for.
+ pageUrl: probe.url || searchUrl,
+ endpoint,
+ engine: engine.label,
+ alternatives: otherEngines(engine.id),
+ detail: probe.challenge.detail || undefined,
+ });
+ }
+
+ if (probe.results.length === 0) {
+ // "container missing entirely" and "container present but empty" are
+ // different faults, and saying which saves the next person a browser
+ // session.
+ const diagnosis =
+ probe.container === "found"
+ ? `${engine.label}'s results container was rendered but nothing could be parsed out of it, which ` +
+ `suggests its markup changed and castle_cdp_search's extractor for ${engine.id} needs updating.`
+ : `${engine.label} served a page with no results area at all. This is often a soft rate-limit: as a ` +
+ `block decays an engine stops serving its challenge page and returns an empty result page instead, ` +
+ `which is indistinguishable from a genuine zero-hit search (observed on Google lasting ~90 minutes ` +
+ `after heavy querying). It can also mean the query truly has no hits.`;
+ throw new NoResultsError(
+ `No results could be read from ${probe.url || searchUrl} (page title: "${probe.title}"). ${diagnosis} ` +
+ `Before concluding the extractor is broken, retry with a different engine (${otherEngines(engine.id).join(", ")}); ` +
+ `if those also come back empty the query is the problem, and if only this one does, it is rate limiting — ` +
+ `wait rather than retrying in a loop.`,
+ );
+ }
+
+ return {
+ results: probe.results.slice(0, request.numResults),
+ searchUrl,
+ finalUrl: probe.url || searchUrl,
+ endpoint,
+ engine,
+ engineSource,
+ };
+ } finally {
+ // Always, including the challenge path. Leaving the tab open would let the
+ // operator solve the captcha in place, but it would also litter a browser
+ // someone else is using with abandoned tabs on every failure; the error
+ // carries the URL instead, and the cookie it sets is profile-wide.
+ await page.close();
+ }
+ } finally {
+ release();
+ }
+};
diff --git a/castle-search/src/target-store.ts b/castle-search/src/target-store.ts
new file mode 100644
index 0000000..6ff0db4
--- /dev/null
+++ b/castle-search/src/target-store.ts
@@ -0,0 +1,42 @@
+/**
+ * Read-only view of the target pi-browser-harness persists for `/browser-target`.
+ *
+ * The stored value is the raw string a user would have put in BU_CDP_HTTP
+ * ("local", "10.88.0.25:9223", …) — kept in that vocabulary deliberately, so
+ * there is one resolution path in endpoint.ts and a stored value can never mean
+ * something the variable could not.
+ *
+ * Every read failure degrades to "nothing stored" rather than throwing: a
+ * corrupt or absent file must fall back to the built-in default, not break search.
+ */
+
+import { readFile } from "node:fs/promises";
+import { targetFilePath } from "./paths.ts";
+
+const CURRENT_VERSION = 1;
+
+/**
+ * The persisted target, or null when nothing is stored, the file is missing, or
+ * it cannot be parsed. A file written by a newer version is treated as "nothing
+ * stored" rather than guessed at.
+ */
+export const readStoredTarget = async (): Promise<string | null> => {
+ let raw: string;
+ try {
+ raw = await readFile(targetFilePath(), "utf8");
+ } catch {
+ return null;
+ }
+ try {
+ const parsed: unknown = JSON.parse(raw);
+ if (typeof parsed !== "object" || parsed === null) return null;
+ const file = parsed as Record<string, unknown>;
+ if (file["version"] !== CURRENT_VERSION) return null;
+ const target = file["target"];
+ if (typeof target !== "string") return null;
+ const trimmed = target.trim();
+ return trimmed.length > 0 ? trimmed : null;
+ } catch {
+ return null;
+ }
+};