From bd03d3b8fff9ed18d3cdb104c55908ba5be6c424 Mon Sep 17 00:00:00 2001 From: Igor Soarez Date: Tue, 4 Aug 2026 15:08:47 +0100 Subject: Write up what AI answer blocks would cost before building one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Google, Bing and Brave all put a generated answer above their results now, and none of it is being read. Investigated against castle's Chrome; nothing in src/ changed, because the cost turned out to be worth deciding on rather than absorbing. Two things make it expensive. Google's AI Overview exists only on the classic layout — udm=14 is the "Web" tab, and having no overview is part of what that tab is — so enabling it means giving back the clean page the extractor was built around. And all three stream in after the results are already parseable: Google settles ~2.3s in, Brave was still growing at 5s, against an extractor that returns at ~0.5s. The wait is often wasted, since the blocks are query-dependent — no Google overview for "how does postgres MVCC vacuum work" after 15s, no Bing answer for "who founded anthropic". Selectors, the traps behind them (#m-x-content briefly holds ~29k characters of unrelated page; Google's "Show more" hides only the disclaimer), the measured timings, three possible designs and their tradeoffs are all in the doc, so this does not have to be re-measured if the answer turns out to be yes. --- castle-search/src/cdp.ts | 342 +++++++++++++++++++++++++++++++++++++ castle-search/src/endpoint.ts | 108 ++++++++++++ castle-search/src/engine-store.ts | 81 +++++++++ castle-search/src/engine-target.ts | 79 +++++++++ castle-search/src/engines.ts | 229 +++++++++++++++++++++++++ castle-search/src/errors.ts | 83 +++++++++ castle-search/src/extract.ts | 294 +++++++++++++++++++++++++++++++ castle-search/src/format.ts | 54 ++++++ castle-search/src/index.ts | 315 ++++++++++++++++++++++++++++++++++ castle-search/src/paths.ts | 41 +++++ castle-search/src/search.ts | 339 ++++++++++++++++++++++++++++++++++++ castle-search/src/target-store.ts | 42 +++++ 12 files changed, 2007 insertions(+) create mode 100644 castle-search/src/cdp.ts create mode 100644 castle-search/src/endpoint.ts create mode 100644 castle-search/src/engine-store.ts create mode 100644 castle-search/src/engine-target.ts create mode 100644 castle-search/src/engines.ts create mode 100644 castle-search/src/errors.ts create mode 100644 castle-search/src/extract.ts create mode 100644 castle-search/src/format.ts create mode 100644 castle-search/src/index.ts create mode 100644 castle-search/src/paths.ts create mode 100644 castle-search/src/search.ts create mode 100644 castle-search/src/target-store.ts (limited to 'castle-search/src') diff --git a/castle-search/src/cdp.ts b/castle-search/src/cdp.ts new file mode 100644 index 0000000..aa01b56 --- /dev/null +++ b/castle-search/src/cdp.ts @@ -0,0 +1,342 @@ +/** + * A minimal CDP client — just enough to open a page, navigate it, read the DOM, + * and close it again. + * + * Deliberately dependency-free: Node 22 ships a global `WebSocket`, so this + * needs no `ws`. That matters because an extension installed by `pi install` + * gets its own `node_modules`, and fewer moving parts there is fewer things to + * go wrong on a machine that is not the one it was written on. + * + * The browser is *not* ours. It is a shared Chrome someone may be looking at, + * recycled hourly (`me.soarez.chrome-cdp-recycle`). Two consequences run + * through this file: + * + * 1. The websocket URL is re-resolved from `/json/version` on every connect, + * never cached. Chrome re-mints its browser UUID on each launch, so a + * pinned ws:// URL breaks the first time the browser is recycled. + * 2. Every page this opens is created in the background and closed again, + * including on error paths, so a failed search does not leave tabs behind + * in someone else's window. + */ + +import { BrowserUnavailableError, SearchAbortedError, messageOf } from "./errors.ts"; +import type { CdpTarget } from "./endpoint.ts"; + +/** How long a single CDP command may take before the socket is presumed dead. */ +const COMMAND_TIMEOUT_MS = 15_000; +/** How long to wait for /json/version and for the websocket handshake. */ +const CONNECT_TIMEOUT_MS = 5_000; + +type CdpMessage = { + id?: number; + method?: string; + params?: Record; + sessionId?: string; + result?: Record; + error?: { code?: number; message?: string }; +}; + +type Pending = { + resolve: (value: Record) => void; + reject: (reason: Error) => void; + timer: ReturnType; +}; + +/** + * Reject if `signal` fires before `promise` settles, cleaning up the listener + * either way. Node's own abort-aware APIs are not available here — the + * websocket predates them — so aborts are raced in rather than plumbed through. + */ +export const raceAbort = (promise: Promise, signal: AbortSignal | undefined): Promise => { + if (!signal) return promise; + if (signal.aborted) return Promise.reject(abortError(signal)); + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(abortError(signal)); + signal.addEventListener("abort", onAbort, { once: true }); + promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort)); + }); +}; + +const abortError = (signal: AbortSignal): Error => { + const reason: unknown = signal.reason; + if (reason instanceof Error && reason.name === "TimeoutError") { + return new SearchAbortedError("search timed out"); + } + return new SearchAbortedError("search was cancelled"); +}; + +/** Ask the browser for its current websocket endpoint. Never cached — see above. */ +const queryWebSocketUrl = async (target: CdpTarget, signal?: AbortSignal): Promise => { + const url = `http://${target.host}:${target.port}/json/version`; + let payload: unknown; + try { + const res = await fetch(url, { + signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(CONNECT_TIMEOUT_MS)]) : AbortSignal.timeout(CONNECT_TIMEOUT_MS), + }); + if (!res.ok) { + throw new BrowserUnavailableError(`${url} answered HTTP ${res.status}`); + } + payload = await res.json(); + } catch (e) { + if (e instanceof BrowserUnavailableError) throw e; + if (signal?.aborted) throw abortError(signal); + throw new BrowserUnavailableError( + `No CDP endpoint answered at ${url} (${messageOf(e)}). Is the browser running and reachable on the overlay?`, + ); + } + const ws = (payload as Record | null)?.["webSocketDebuggerUrl"]; + if (typeof ws !== "string" || ws === "") { + throw new BrowserUnavailableError(`${url} answered without a webSocketDebuggerUrl`); + } + return ws; +}; + +/** A live browser-level CDP connection. */ +export class BrowserConnection { + #ws: WebSocket; + #nextId = 1; + #pending = new Map(); + #listeners = new Set<(msg: CdpMessage) => void>(); + #closedReason: string | null = null; + + readonly target: CdpTarget; + + private constructor(ws: WebSocket, target: CdpTarget) { + this.#ws = ws; + this.target = target; + + ws.addEventListener("message", (ev: MessageEvent) => { + let msg: CdpMessage; + try { + msg = JSON.parse(String(ev.data)) as CdpMessage; + } catch { + return; + } + if (msg.id !== undefined) { + const pending = this.#pending.get(msg.id); + if (!pending) return; + this.#pending.delete(msg.id); + clearTimeout(pending.timer); + if (msg.error) { + pending.reject(new BrowserUnavailableError(`CDP error: ${msg.error.message ?? "unknown"}`)); + } else { + pending.resolve(msg.result ?? {}); + } + return; + } + for (const listener of [...this.#listeners]) listener(msg); + }); + + const fail = (reason: string): void => { + this.#closedReason ??= reason; + for (const [, pending] of this.#pending) { + clearTimeout(pending.timer); + pending.reject(new BrowserUnavailableError(reason)); + } + this.#pending.clear(); + }; + ws.addEventListener("close", () => fail("CDP connection closed (the browser may have been recycled)")); + ws.addEventListener("error", () => fail("CDP connection failed")); + } + + /** Resolve the endpoint afresh and dial it. */ + static async connect(target: CdpTarget, signal?: AbortSignal): Promise { + const wsUrl = await queryWebSocketUrl(target, signal); + const ws = new WebSocket(wsUrl); + try { + await raceAbort( + new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new BrowserUnavailableError(`timed out opening ${wsUrl}`)), + CONNECT_TIMEOUT_MS, + ); + ws.addEventListener( + "open", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + ws.addEventListener( + "error", + () => { + clearTimeout(timer); + reject(new BrowserUnavailableError(`could not open ${wsUrl}`)); + }, + { once: true }, + ); + }), + signal, + ); + } catch (e) { + try { + ws.close(); + } catch { + // Already failed; nothing useful to do with a second failure. + } + throw e; + } + return new BrowserConnection(ws, target); + } + + /** False once the socket has closed or failed — the reconnect trigger. */ + get isOpen(): boolean { + return this.#closedReason === null && this.#ws.readyState === 1 /* OPEN */; + } + + send(method: string, params: Record = {}, sessionId?: string): Promise> { + if (!this.isOpen) { + return Promise.reject(new BrowserUnavailableError(this.#closedReason ?? "CDP connection is not open")); + } + const id = this.#nextId++; + const payload: CdpMessage = sessionId ? { id, method, params, sessionId } : { id, method, params }; + return new Promise>((resolve, reject) => { + const timer = setTimeout(() => { + this.#pending.delete(id); + reject(new BrowserUnavailableError(`CDP command ${method} timed out after ${COMMAND_TIMEOUT_MS}ms`)); + }, COMMAND_TIMEOUT_MS); + this.#pending.set(id, { resolve, reject, timer }); + try { + this.#ws.send(JSON.stringify(payload)); + } catch (e) { + this.#pending.delete(id); + clearTimeout(timer); + reject(new BrowserUnavailableError(`could not send ${method}: ${messageOf(e)}`)); + } + }); + } + + /** Subscribe to unsolicited protocol events. Returns an unsubscribe function. */ + onEvent(listener: (msg: CdpMessage) => void): () => void { + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + } + + close(): void { + this.#closedReason ??= "CDP connection closed by this extension"; + this.#listeners.clear(); + try { + this.#ws.close(); + } catch { + // Closing an already-dead socket is not a failure worth reporting. + } + } +} + +/** + * One throwaway page. + * + * Created in the background so it does not steal focus from whoever is using + * the browser, and closed by {@link close} on every path. + */ +export class PageSession { + readonly #conn: BrowserConnection; + readonly #targetId: string; + readonly #sessionId: string; + #closed = false; + + private constructor(conn: BrowserConnection, targetId: string, sessionId: string) { + this.#conn = conn; + this.#targetId = targetId; + this.#sessionId = sessionId; + } + + static async open(conn: BrowserConnection, signal?: AbortSignal): Promise { + if (signal?.aborted) throw abortError(signal); + + // Deliberately NOT raced against the abort signal. raceAbort abandons the + // promise, it cannot cancel the command — and this command's side effect is + // a tab. Abandoning it would leave a tab open in a browser someone else is + // using, with nothing left holding its targetId to close it. It is a fast + // command already bounded by COMMAND_TIMEOUT_MS, so waiting it out costs + // little and guarantees we own whatever it created. + const created = await conn.send("Target.createTarget", { url: "about:blank", background: true }); + const targetId = created["targetId"]; + if (typeof targetId !== "string") { + throw new BrowserUnavailableError("Target.createTarget did not return a targetId"); + } + + // From here on the page exists, so every failure — including a late abort — + // must still close it. + try { + if (signal?.aborted) throw abortError(signal); + const attached = await raceAbort( + conn.send("Target.attachToTarget", { targetId, flatten: true }), + signal, + ); + const sessionId = attached["sessionId"]; + if (typeof sessionId !== "string") { + throw new BrowserUnavailableError("Target.attachToTarget did not return a sessionId"); + } + const page = new PageSession(conn, targetId, sessionId); + await raceAbort(conn.send("Page.enable", {}, sessionId), signal); + return page; + } catch (e) { + await conn.send("Target.closeTarget", { targetId }).catch(() => {}); + throw e; + } + } + + /** Navigate and wait for the load event, or for `signal` to fire. */ + async navigate(url: string, signal?: AbortSignal): Promise { + // Subscribed before navigating, so a page that loads faster than the + // Page.navigate reply cannot slip its load event past us. + let off: () => void = () => {}; + const loaded = new Promise((resolve) => { + off = this.#conn.onEvent((msg) => { + if (msg.sessionId === this.#sessionId && msg.method === "Page.loadEventFired") resolve(); + }); + }); + + try { + const nav = await raceAbort(this.#conn.send("Page.navigate", { url }, this.#sessionId), signal); + const errorText = nav["errorText"]; + if (typeof errorText === "string" && errorText !== "") { + throw new BrowserUnavailableError(`navigation to ${url} failed: ${errorText}`); + } + + try { + await raceAbort(loaded, signal); + } catch (e) { + // Stop the in-flight load so the page is not still fetching while we + // tear it down; the tab is closed either way, this just makes it quicker. + await this.#conn.send("Page.stopLoading", {}, this.#sessionId).catch(() => {}); + throw e; + } + } finally { + // Unsubscribing here rather than inside the listener: on the failure paths + // the listener never fires, and a subscription left on a connection that + // outlives many searches accumulates. + off(); + } + } + + /** Evaluate an expression in the page and return its value by value. */ + async evaluate(expression: string, signal?: AbortSignal): Promise { + const res = await raceAbort( + this.#conn.send( + "Runtime.evaluate", + { expression, returnByValue: true, awaitPromise: true }, + this.#sessionId, + ), + signal, + ); + const exception = res["exceptionDetails"] as { text?: string; exception?: { description?: string } } | undefined; + if (exception) { + const text = exception.exception?.description ?? exception.text ?? "unknown error"; + throw new BrowserUnavailableError(`page script failed: ${text}`); + } + return (res["result"] as { value?: T } | undefined)?.value as T; + } + + /** + * Close the tab. Idempotent and never throws: it runs on error paths, where a + * second failure would mask the one worth reporting. + */ + async close(): Promise { + if (this.#closed) return; + this.#closed = true; + await this.#conn.send("Target.closeTarget", { targetId: this.#targetId }).catch(() => {}); + } +} diff --git a/castle-search/src/endpoint.ts b/castle-search/src/endpoint.ts new file mode 100644 index 0000000..d3f4b0d --- /dev/null +++ b/castle-search/src/endpoint.ts @@ -0,0 +1,108 @@ +/** + * Which browser this extension searches through. + * + * Deliberately the *same* configuration as pi-browser-harness, so one decision + * governs both packages and they never drive different browsers in the same + * session: `BU_CDP_HTTP`, then the `/browser-target` file that harness writes, + * then the built-in castle default. + * + * Resolution is pure — no I/O — so it can be unit-tested and reported without + * touching the network. + */ + +/** Default CDP endpoint: castle's Chrome, exposed on the wg8 overlay. */ +export const DEFAULT_REMOTE_CDP = "10.88.0.25:9223"; + +/** Values of BU_CDP_HTTP that mean "ignore the default, use this machine". */ +const LOCAL_ALIASES = new Set(["local", "localhost", "off", "none", "0", "no"]); + +/** Port a local Chrome is expected to serve CDP on when the target is "local". */ +export const LOCAL_CDP_PORT = 9222; + +/** + * Where a target came from. "env" is BU_CDP_HTTP, "stored" is the harness's + * `/browser-target`, "default" is the built-in. + */ +export type CdpTargetSource = "env" | "stored" | "default"; + +export type CdpTarget = { + readonly host: string; + readonly port: number; + readonly source: CdpTargetSource; +}; + +export type CdpTargetResolution = + | { readonly kind: "ok"; readonly target: CdpTarget } + | { readonly kind: "invalid"; readonly raw: string; readonly reason: string; readonly source: CdpTargetSource }; + +/** + * Precedence: BU_CDP_HTTP, then the stored `/browser-target` choice, then the + * built-in default. The environment wins so a one-off `BU_CDP_HTTP=… pi` is + * never silently overridden by a choice made days ago. + * + * Unlike pi-browser-harness this never falls back from the default to a local + * browser on its own. The harness can fall back because it drives a browser the + * operator is watching; here a silent fallback would mean *searching from the + * operator's own signed-in Chrome* without saying so. When the configured + * endpoint is unreachable the caller reports that, and the operator can say + * `local` explicitly. + * + * `stored` is passed in rather than read here so this stays pure and synchronous. + */ +export const resolveCdpTarget = ( + env: NodeJS.ProcessEnv = process.env, + stored: string | null = null, +): CdpTargetResolution => { + const fromEnv = env["BU_CDP_HTTP"]?.trim(); + const fromStore = stored?.trim(); + + const hasEnv = fromEnv !== undefined && fromEnv !== ""; + const raw = hasEnv ? fromEnv : fromStore; + const source: CdpTargetSource = hasEnv ? "env" : "stored"; + + if (raw === undefined || raw === "") { + const parsed = parseHostPort(DEFAULT_REMOTE_CDP); + // Unreachable in practice: the constant above is covered by its own test. + if (!parsed) { + return { kind: "invalid", raw: DEFAULT_REMOTE_CDP, source: "default", reason: "built-in default is malformed" }; + } + return { kind: "ok", target: { ...parsed, source: "default" } }; + } + + if (LOCAL_ALIASES.has(raw.toLowerCase())) { + return { kind: "ok", target: { host: "127.0.0.1", port: LOCAL_CDP_PORT, source } }; + } + + const parsed = parseHostPort(raw); + if (!parsed) { + return { + kind: "invalid", + raw, + source, + reason: `expected "host:port" or one of ${[...LOCAL_ALIASES].join(", ")}`, + }; + } + return { kind: "ok", target: { ...parsed, source } }; +}; + +const parseHostPort = (value: string): { host: string; port: number } | null => { + const sep = value.lastIndexOf(":"); + if (sep === -1) return null; + const host = value.slice(0, sep).trim(); + const port = Number(value.slice(sep + 1).trim()); + if (!host) return null; + if (!Number.isInteger(port) || port <= 0 || port >= 65536) return null; + return { host, port }; +}; + +const sourceLabel = (source: CdpTargetSource): string => + source === "env" ? "BU_CDP_HTTP" : source === "stored" ? "/browser-target" : "built-in default"; + +/** One-line description for progress messages and errors. */ +export const describeCdpTarget = (target: CdpTarget): string => + `${target.host}:${target.port} (${sourceLabel(target.source)})`; + +/** One-line description of a resolution failure. */ +export const describeResolutionFailure = ( + failure: Extract, +): string => `invalid CDP target "${failure.raw}" from ${sourceLabel(failure.source)} — ${failure.reason}`; diff --git a/castle-search/src/engine-store.ts b/castle-search/src/engine-store.ts new file mode 100644 index 0000000..d218d2b --- /dev/null +++ b/castle-search/src/engine-store.ts @@ -0,0 +1,81 @@ +/** + * Persistence for the `/search-engine` choice. + * + * Deliberately the same shape, and the same atomic write, as the harness's + * browser-target store: a sibling temp file plus rename, so a crash mid-write + * cannot leave a half-written engine name behind. Its own file rather than a + * shared settings blob, so a corrupt engine choice cannot cost anything else. + * + * Like the CDP target, the stored value is the raw string a user would have put + * in the environment variable ("ddg", "bing", …), not a pre-parsed id. One + * vocabulary, one resolution path in engine-target.ts, and a stored value can + * never mean something the variable could not. + * + * Every read failure degrades to "nothing stored" rather than throwing. + */ + +import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { dirname, join } from "node:path"; + +import { agentDir } from "./paths.ts"; + +const CURRENT_VERSION = 1; + +const engineFilePath = (): string => join(agentDir(), "castle-cdp-search-engine.json"); + +type EngineFile = { + readonly version: 1; + /** Raw CCS_SEARCH_ENGINE-style value, or null when cleared. */ + readonly engine: string | null; + readonly savedAt: string; +}; + +/** + * The persisted engine, or null when nothing is stored, the file is missing, or + * it cannot be parsed. A file written by a newer version is treated as "nothing + * stored" rather than guessed at. + */ +export const readStoredEngine = async (): Promise => { + let raw: string; + try { + raw = await readFile(engineFilePath(), "utf8"); + } catch { + return null; + } + try { + const parsed: unknown = JSON.parse(raw); + if (typeof parsed !== "object" || parsed === null) return null; + const file = parsed as Record; + if (file["version"] !== CURRENT_VERSION) return null; + const engine = file["engine"]; + if (typeof engine !== "string") return null; + const trimmed = engine.trim(); + return trimmed.length > 0 ? trimmed : null; + } catch { + return null; + } +}; + +const write = async (engine: string | null): Promise => { + const path = engineFilePath(); + const tmp = `${path}.${randomUUID()}.tmp`; + const payload: EngineFile = { version: CURRENT_VERSION, engine, savedAt: new Date().toISOString() }; + try { + await mkdir(dirname(path), { recursive: true }); + await writeFile(tmp, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); + await rename(tmp, path); + } catch (e) { + await unlink(tmp).catch(() => {}); + throw new Error(`could not save search engine to ${path}: ${e instanceof Error ? e.message : String(e)}`); + } +}; + +/** Persist the chosen engine. */ +export const writeStoredEngine = (engine: string): Promise => write(engine); + +/** Clear the choice, restoring the built-in default. */ +export const clearStoredEngine = (): Promise => write(null); + +/** Exposed so the command can tell the user where the choice lives. */ +export const storedEngineLocation = (): string => engineFilePath(); diff --git a/castle-search/src/engine-target.ts b/castle-search/src/engine-target.ts new file mode 100644 index 0000000..db4d81f --- /dev/null +++ b/castle-search/src/engine-target.ts @@ -0,0 +1,79 @@ +/** + * Which search engine to use, resolved the same way the CDP host is: + * environment variable, then a persisted choice, then a built-in default. + * + * Pure — no I/O — so the `/search-engine` command can report the configured + * intent without touching disk twice, and so it is testable. + */ + +import { type EngineId, parseEngineId, engineAliases } from "./engines.ts"; + +/** Used when nothing else says otherwise. */ +export const DEFAULT_ENGINE: EngineId = "google"; + +/** The environment variable that pins the engine for one process. */ +export const ENGINE_ENV_VAR = "CCS_SEARCH_ENGINE"; + +export type EngineSource = "parameter" | "env" | "stored" | "default"; + +export type EngineResolution = + | { readonly kind: "ok"; readonly id: EngineId; readonly source: EngineSource } + | { readonly kind: "invalid"; readonly raw: string; readonly source: EngineSource }; + +/** + * Precedence: the per-call tool parameter, then `CCS_SEARCH_ENGINE`, then the + * `/search-engine` choice, then the built-in default. + * + * Note the tool parameter sits *above* the environment variable, which is the + * opposite of how the CDP target treats an explicit endpoint. That difference + * is deliberate. For the browser, an environment variable has to win, because + * quietly driving a different machine means automating the operator's own + * signed-in Chrome — a safety property. Choosing a different search engine has + * no equivalent hazard, and the ability to fall back to another engine when the + * first one is serving a captcha is the single most useful thing the agent can + * do with this tool. So the model is allowed to override the default; the + * operator sets what it starts from. + */ +export const resolveEngine = ( + parameter: string | null | undefined, + env: NodeJS.ProcessEnv = process.env, + stored: string | null = null, +): EngineResolution => { + const candidates: Array<{ raw: string; source: EngineSource }> = []; + + const fromParam = parameter?.trim(); + if (fromParam) candidates.push({ raw: fromParam, source: "parameter" }); + + const fromEnv = env[ENGINE_ENV_VAR]?.trim(); + if (fromEnv) candidates.push({ raw: fromEnv, source: "env" }); + + const fromStore = stored?.trim(); + if (fromStore) candidates.push({ raw: fromStore, source: "stored" }); + + for (const candidate of candidates) { + const id = parseEngineId(candidate.raw); + // An unparseable value is reported rather than skipped over. Falling + // through to the next source would mean a typo in CCS_SEARCH_ENGINE + // silently searches Google instead of saying so. + if (!id) return { kind: "invalid", raw: candidate.raw, source: candidate.source }; + return { kind: "ok", id, source: candidate.source }; + } + + return { kind: "ok", id: DEFAULT_ENGINE, source: "default" }; +}; + +const sourceLabel = (source: EngineSource): string => + source === "parameter" + ? "engine parameter" + : source === "env" + ? ENGINE_ENV_VAR + : source === "stored" + ? "/search-engine" + : "built-in default"; + +export const describeEngineSource = (source: EngineSource): string => sourceLabel(source); + +/** One-line description of a bad engine name, listing what would have worked. */ +export const describeEngineFailure = (failure: Extract): string => + `unknown search engine "${failure.raw}" from ${sourceLabel(failure.source)} — ` + + `expected one of ${engineAliases().join(", ")}`; diff --git a/castle-search/src/engines.ts b/castle-search/src/engines.ts new file mode 100644 index 0000000..ef8baef --- /dev/null +++ b/castle-search/src/engines.ts @@ -0,0 +1,229 @@ +/** + * The search engines this extension knows how to drive. + * + * Every selector and every URL parameter below was read off the real SERP in + * castle's Chrome, not recalled or inferred. That is not pedantry: Google's own + * Tools menu writes `tbs=qdr:*`, which renders an *empty* page on that profile, + * while the undocumented-looking `as_qdr=*` works. Anything in here that was not + * observed is a bug waiting to be reported as "no results". + * + * Two extraction modes, because the engines genuinely differ: + * + * - "items" — the SERP has a clean per-result container (`li.b_algo`, + * `.result`, `.snippet[data-type=web]`). Straightforward. + * - "headings" — Google has no stable result container, so results are found + * from each

outward. Kept as its own mode rather than + * forced into the item shape, because it is the one that has + * been through the most verification. + */ + +export type EngineId = "google" | "duckduckgo" | "bing" | "brave"; + +export type Recency = "day" | "week" | "month" | "year"; + +export const RECENCY_VALUES = ["day", "week", "month", "year"] as const; + +export const ENGINE_IDS: readonly EngineId[] = ["google", "duckduckgo", "bing", "brave"]; + +/** Aliases accepted from humans and from the model. */ +const ENGINE_ALIASES: Record = { + google: "google", + g: "google", + duckduckgo: "duckduckgo", + ddg: "duckduckgo", + duck: "duckduckgo", + bing: "bing", + b: "bing", + brave: "brave", +}; + +/** + * How the in-page script should read one engine's results. Passed into the + * browser as JSON, so everything here must be plain data. + */ +export type ExtractionConfig = { + mode: "items" | "headings"; + /** Results container candidates, first match wins. */ + roots: string[]; + /** "items" mode: one result. */ + item?: string; + /** Anchor carrying the outbound link, relative to the item. */ + link?: string; + /** Title element, relative to the item. Falls back to the anchor's text. */ + title?: string; + /** Description element, relative to the item. */ + snippet?: string; + /** + * "items" mode with no snippet selector: text from these is subtracted from + * the item's text to leave the description behind. + */ + subtract?: string[]; + /** Items matching any of these are ads or non-web cards, and are skipped. */ + exclude?: string[]; + /** How outbound links are wrapped, if they are. */ + unwrap: "none" | "ddg" | "bing"; + /** Hosts belonging to the engine itself; links to them are not results. */ + selfHostPattern: string; +}; + +export type EngineDefinition = { + readonly id: EngineId; + readonly label: string; + /** Human-facing note about what makes this engine worth choosing. */ + readonly note: string; + /** + * Recency windows this engine can actually express, mapped to the parameter + * value. A window that is absent is one the engine does not support — never + * one that is silently dropped. + */ + readonly recency: Partial>; + readonly buildUrl: (query: string, numResults: number, recency: Recency | undefined) => string; + readonly extraction: ExtractionConfig; +}; + +const GOOGLE: EngineDefinition = { + id: "google", + label: "Google", + note: "best result quality; blocks aggressively under repeated automated queries", + recency: { day: "d", week: "w", month: "m", year: "y" }, + buildUrl: (query, numResults, recency) => { + const params = new URLSearchParams({ q: query, num: String(numResults), hl: "en" }); + // `udm=14` is the plain "Web" tab: no AI overview, no carousels. But any + // date restriction combined with it renders an empty page, and `tbs=qdr:*` + // renders an empty page on its own — both observed. So a time-limited + // search drops udm and uses the older as_qdr instead. + if (recency) params.set("as_qdr", GOOGLE.recency[recency] as string); + else params.set("udm", "14"); + return `https://www.google.com/search?${params.toString()}`; + }, + extraction: { + mode: "headings", + roots: ["#rso", "#search"], + unwrap: "none", + selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/`, + }, +}; + +const DUCKDUCKGO: EngineDefinition = { + id: "duckduckgo", + label: "DuckDuckGo", + note: "no-JS endpoint, the most stable markup here and the least likely to challenge", + // Read off DuckDuckGo's own