From db69207c06e8d5233bff4996e3ef5b43332d533f Mon Sep 17 00:00:00 2001 From: Igor Soarez Date: Mon, 3 Aug 2026 21:43:56 +0100 Subject: Support DuckDuckGo, Bing and Brave, switchable like the CDP host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine resolution mirrors the browser target: CCS_SEARCH_ENGINE, then a /search-engine choice persisted machine-wide, then Google. A per-call `engine` parameter sits above both so the agent can fall back when one engine starts serving captchas — the one case where the model, not the operator, has to make the call. Unlike the CDP target there is no safety argument for the environment winning: driving the wrong browser means automating someone's signed-in Chrome, choosing a different index does not. An unsupported recency window is refused, naming the engines that support it, rather than dropped. Silently returning unfiltered results is indistinguishable from success, which is the failure this whole design is trying to avoid. Extraction grows a second mode. DuckDuckGo, Bing and Brave have clean per-result containers; Google does not, so its heading-walk stays as its own path rather than being bent into the item shape. DuckDuckGo and Bing route links through redirectors, unwrapped in the page. Third silent-failure trap found, alongside Google's two: on Bing, `count` cancels `filters`. With ex1:"ez1" alone every result is hours old; add count in either order and months-old results return, looking perfectly ordinary. Bing now drops count whenever a date filter is present. Challenge detection widened to Brave's "Verifying you're not a bot" and "Quick check before you continue searching", which the previous Google- shaped matcher missed entirely — found by tripping it. --- README.md | 144 +++++++++++++++-------- src/engine-store.ts | 81 +++++++++++++ src/engine-target.ts | 79 +++++++++++++ src/engines.ts | 227 +++++++++++++++++++++++++++++++++++++ src/errors.ts | 37 +++++- src/extract.ts | 276 +++++++++++++++++++++++++++++++-------------- src/format.ts | 10 +- src/index.ts | 132 +++++++++++++++++++--- src/search.ts | 150 ++++++++++++++++-------- test/engine-target.test.ts | 89 +++++++++++++++ test/format.test.ts | 20 +++- test/search-url.test.ts | 99 ++++++++++++---- 12 files changed, 1122 insertions(+), 222 deletions(-) create mode 100644 src/engine-store.ts create mode 100644 src/engine-target.ts create mode 100644 src/engines.ts create mode 100644 test/engine-target.test.ts diff --git a/README.md b/README.md index a97d105..684bd96 100644 --- a/README.md +++ b/README.md @@ -21,10 +21,56 @@ the model picks. | `query` | string, required | As you would type it into a search box | | `numResults` | integer, optional | 1–20, default 10 | | `recency` | `day` \| `week` \| `month` \| `year`, optional | Omit for no time limit | +| `engine` | `google` \| `duckduckgo` \| `bing` \| `brave`, optional | Overrides the configured default for one call | It returns titles, URLs and snippets — **search results only**. It does not fetch the linked pages; follow up with a read/fetch tool for full content. +## Engines + +Four, with independent indexes and independent rate limits. That last part is +the point: when one starts serving captchas, the others generally still work, +and the tool tells the agent so inside the error. + +| Engine | Recency windows | Notes | +|---|---|---| +| `google` | day, week, month, year | Best results; blocks aggressively under automation | +| `duckduckgo` | day, week, month, year | No-JS endpoint — the most stable markup here, least likely to challenge | +| `bing` | day, week, month | No year window exists | +| `brave` | none | Independent index, unwrapped links; challenges quickly | + +Aliases are accepted: `ddg`, `duck`, `g`, `b`. + +**An unsupported recency window is refused, not ignored.** Asking Bing for the +past year raises `RecencyUnsupportedError` naming the engines that can do it. +Silently returning unfiltered results would be indistinguishable from success. + +### Switching engines + +Exactly like switching the CDP host, plus a per-call override: + +``` +/search-engine show the current engine and where it came from +/search-engine ddg persist a choice (machine-wide, survives restarts) +/search-engine default back to Google +CCS_SEARCH_ENGINE=bing pi … pin for one process +``` + +Precedence: **`engine` parameter > `CCS_SEARCH_ENGINE` > `/search-engine` > +Google.** The stored choice lives in +`~/.pi/agent/castle-cdp-search-engine.json`, written atomically. + +Note the tool parameter sits *above* the environment variable, which is the +opposite of how the CDP target treats an explicit endpoint. That is deliberate: +for the browser, an env var must win because quietly driving a different machine +means automating the operator's own signed-in Chrome — a safety property. +Choosing a different search engine carries no such hazard, and letting the agent +fall back when one engine is blocked is the single most useful thing it can do +with this tool. + +An unrecognised engine name is reported rather than skipped, so a typo in +`CCS_SEARCH_ENGINE` never silently searches Google instead. + ## Which browser it drives Exactly the same configuration as [`pi-browser-harness`](../pi-browser-harness), @@ -61,8 +107,8 @@ transparently on the next search. and aborts. `background: true` so it does not steal focus from whoever is looking at that screen. - **Two concurrent searches**, queued beyond that. -- **20-second deadline** per search, covering connect, navigate and extract - together. Esc aborts an in-flight navigation. +- **20-second deadline** per call, covering the queue wait, connect, navigate and + extract together. Esc aborts an in-flight navigation. - **Every failure throws.** pi only sets `isError: true` when `execute()` throws; a returned `{ error }` object would read to the model as a search that simply found nothing. @@ -72,70 +118,73 @@ transparently on the next search. The browser is shared and belongs to a person. Worth being aware of: - Queries the agent runs land in that browser profile's history and cookies, and - in the Google account's search history if that profile is signed in. + in the search engine's account history if that profile is signed in. - Tabs open and close on someone's screen. `background: true` keeps them from stealing focus, but they are visible. -- Searching hard trips Google's rate limiter for the whole host — a handful of - queries in a few seconds is enough to earn a `/sorry/` page that affects the - human using that browser too. The cap of two concurrent searches limits this; - it does not eliminate it. Measured once: roughly 30 queries in a few minutes - cost about 90 minutes of blocking. -- **A decaying block does not look like a block.** Once the `/sorry/` page stops - being served, Google returns an *empty results page* for a while instead — - which is indistinguishable from a query that genuinely has no hits. It - surfaces as `NoResultsError`, not `SearchChallengeError`. If several unrelated - queries all come back empty, that is rate limiting; wait rather than retrying. +- Searching hard trips rate limiters for the whole host — a handful of queries in + a few seconds is enough to earn a challenge that affects the human using that + browser too. Measured on Google: roughly 30 queries in a few minutes cost about + 90 minutes of blocking. Brave challenges considerably sooner than that. +- **A decaying block does not look like a block.** Once the challenge page stops + being served, engines return an *empty results page* for a while instead — + indistinguishable from a query with no hits. It surfaces as `NoResultsError`, + not `SearchChallengeError`. If one engine comes back empty and another answers + the same query, that is rate limiting. ## When a captcha appears -Google will eventually serve a `/sorry/` interstitial, a consent wall, or a -recaptcha — especially if searches come in fast. The extension detects this and -throws a `SearchChallengeError` naming the challenge, the browser, and the URL a -human has to visit. +Every engine eventually serves an interstitial: Google's `/sorry/` page, Brave's +"Verifying you're not a bot", a consent wall, a Cloudflare challenge. The +extension detects these and throws `SearchChallengeError` naming the challenge, +the engine, the browser, and the URL a human would have to visit. -The agent cannot solve it. The guidelines tell it to stop searching and hand off -to the user, who opens that URL in the browser at the endpoint, clears the -challenge, and lets the agent retry. The cookie is profile-wide, so one pass -unblocks later searches. +The error tells the agent to **try another engine first**, because a challenge on +one engine says nothing about the others and that is a fix it can apply itself. +Only when engines run out should it ask the user to clear the challenge in the +browser. The cookie is profile-wide, so one pass unblocks later searches. The tab is closed rather than left open on the challenge page: leaving it would let the operator solve it in place, but would also litter a shared browser with abandoned tabs on every failure. The URL in the error is enough. -## Search parameters, and two surprises +## Search parameters, and three traps -Plain searches use `udm=14` — Google's "Web" tab: no AI overview, no carousels, -just ranked links, which is both cheaper to parse and closer to what was asked -for. Verified against the real browser: +Everything below was observed against castle's real Chrome, not inferred. All +three fail *silently* — the results look entirely plausible, just wrong. -- `tbs=qdr:*`, the parameter Google's own Tools menu writes, renders an **empty - page** for this profile — `#search` present, no `#rso`, no results. The older - `as_qdr=*` works and genuinely filters. -- Any date restriction combined with `udm=14` also renders that empty page. +1. **Google: `tbs=qdr:*` renders an empty page.** That is the parameter Google's + own Tools menu writes. The older `as_qdr=*` works. +2. **Google: any date filter combined with `udm=14` renders that same empty + page.** So a time-limited search drops `udm` and uses the classic layout. +3. **Bing: `count=` silently cancels `filters=`.** With `ex1:"ez1"` alone every + result is hours old; add `count` in either order and months-old results come + back, unfiltered and unremarkable-looking. So Bing drops `count` whenever a + date filter is present and the caller slices the list instead. -So a time-limited search drops `udm` and uses `as_qdr`. The extractor handles -both layouts. +Each has a regression test asserting the parameter combination, since none of +them would announce itself if it regressed. ## Result extraction -`src/extract.ts` avoids Google's generated class names (`MjjYud`, `kb0PBd`, …) -entirely. It prefers the `data-snhf` / `data-sncf` hooks and otherwise falls back -to a structural walk: from each `

`, take the enclosing link, climb until the -text grows past the header's, and abandon the climb if a second `

` comes into -scope. Checked against both the classic SERP and the `udm=14` layout. +Two modes, because the engines genuinely differ: -Google's markup will change anyway. When it does, a search returns -`NoResultsError` whose message distinguishes "results container rendered but -unparseable" (extractor needs updating) from "no results area at all" (query or -parameters). +- **items** — DuckDuckGo (`.result`), Bing (`li.b_algo`) and Brave + (`.snippet[data-type=web]`) each have a clean per-result container. +- **headings** — Google has no stable container; its class names (`MjjYud`, + `kb0PBd`, `yuRUbf`) are generated. Results are found from each `

` outward, + climbing to the enclosing block but abandoning the climb if a second `

` + comes into scope. -## Install +DuckDuckGo and Bing both route outbound links through redirectors +(`duckduckgo.com/l/?uddg=…`, `bing.com/ck/a?…&u=a1`); both are +unwrapped in the page so the agent gets real URLs. Google and Brave link +straight out. -``` -pi install /Volumes/Sense/src/soarez/pi-castle-cdp-search -``` +Markup will change. When it does, `NoResultsError` distinguishes "container +rendered but unparseable" (that engine's extractor needs updating) from "no +container at all" (query, or rate limiting). -Or, once it is on castle alongside the harness: +## Install ``` pi install ssh://sz@10.88.0.25/Users/sz/repos/pi-castle-cdp-search.git @@ -159,8 +208,9 @@ npm run typecheck npm test # unit tests, no browser needed ``` -The tests cover endpoint resolution, URL building, formatting, the challenge -error, and the concurrency semaphore. The semaphore tests point at an +The tests cover engine selection and precedence, URL building for all four +engines (including the three traps above), recency refusal, formatting, the +challenge error, and the concurrency semaphore. The semaphore tests point at an unreachable endpoint on purpose — the questions there are about slot bookkeeping, and hitting a real browser to test a counter would be slow, flaky, and rude to whoever is using it. diff --git a/src/engine-store.ts b/src/engine-store.ts new file mode 100644 index 0000000..d218d2b --- /dev/null +++ b/src/engine-store.ts @@ -0,0 +1,81 @@ +/** + * Persistence for the `/search-engine` choice. + * + * Deliberately the same shape, and the same atomic write, as the harness's + * browser-target store: a sibling temp file plus rename, so a crash mid-write + * cannot leave a half-written engine name behind. Its own file rather than a + * shared settings blob, so a corrupt engine choice cannot cost anything else. + * + * Like the CDP target, the stored value is the raw string a user would have put + * in the environment variable ("ddg", "bing", …), not a pre-parsed id. One + * vocabulary, one resolution path in engine-target.ts, and a stored value can + * never mean something the variable could not. + * + * Every read failure degrades to "nothing stored" rather than throwing. + */ + +import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { dirname, join } from "node:path"; + +import { agentDir } from "./paths.ts"; + +const CURRENT_VERSION = 1; + +const engineFilePath = (): string => join(agentDir(), "castle-cdp-search-engine.json"); + +type EngineFile = { + readonly version: 1; + /** Raw CCS_SEARCH_ENGINE-style value, or null when cleared. */ + readonly engine: string | null; + readonly savedAt: string; +}; + +/** + * The persisted engine, or null when nothing is stored, the file is missing, or + * it cannot be parsed. A file written by a newer version is treated as "nothing + * stored" rather than guessed at. + */ +export const readStoredEngine = async (): Promise => { + let raw: string; + try { + raw = await readFile(engineFilePath(), "utf8"); + } catch { + return null; + } + try { + const parsed: unknown = JSON.parse(raw); + if (typeof parsed !== "object" || parsed === null) return null; + const file = parsed as Record; + if (file["version"] !== CURRENT_VERSION) return null; + const engine = file["engine"]; + if (typeof engine !== "string") return null; + const trimmed = engine.trim(); + return trimmed.length > 0 ? trimmed : null; + } catch { + return null; + } +}; + +const write = async (engine: string | null): Promise => { + const path = engineFilePath(); + const tmp = `${path}.${randomUUID()}.tmp`; + const payload: EngineFile = { version: CURRENT_VERSION, engine, savedAt: new Date().toISOString() }; + try { + await mkdir(dirname(path), { recursive: true }); + await writeFile(tmp, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); + await rename(tmp, path); + } catch (e) { + await unlink(tmp).catch(() => {}); + throw new Error(`could not save search engine to ${path}: ${e instanceof Error ? e.message : String(e)}`); + } +}; + +/** Persist the chosen engine. */ +export const writeStoredEngine = (engine: string): Promise => write(engine); + +/** Clear the choice, restoring the built-in default. */ +export const clearStoredEngine = (): Promise => write(null); + +/** Exposed so the command can tell the user where the choice lives. */ +export const storedEngineLocation = (): string => engineFilePath(); diff --git a/src/engine-target.ts b/src/engine-target.ts new file mode 100644 index 0000000..db4d81f --- /dev/null +++ b/src/engine-target.ts @@ -0,0 +1,79 @@ +/** + * Which search engine to use, resolved the same way the CDP host is: + * environment variable, then a persisted choice, then a built-in default. + * + * Pure — no I/O — so the `/search-engine` command can report the configured + * intent without touching disk twice, and so it is testable. + */ + +import { type EngineId, parseEngineId, engineAliases } from "./engines.ts"; + +/** Used when nothing else says otherwise. */ +export const DEFAULT_ENGINE: EngineId = "google"; + +/** The environment variable that pins the engine for one process. */ +export const ENGINE_ENV_VAR = "CCS_SEARCH_ENGINE"; + +export type EngineSource = "parameter" | "env" | "stored" | "default"; + +export type EngineResolution = + | { readonly kind: "ok"; readonly id: EngineId; readonly source: EngineSource } + | { readonly kind: "invalid"; readonly raw: string; readonly source: EngineSource }; + +/** + * Precedence: the per-call tool parameter, then `CCS_SEARCH_ENGINE`, then the + * `/search-engine` choice, then the built-in default. + * + * Note the tool parameter sits *above* the environment variable, which is the + * opposite of how the CDP target treats an explicit endpoint. That difference + * is deliberate. For the browser, an environment variable has to win, because + * quietly driving a different machine means automating the operator's own + * signed-in Chrome — a safety property. Choosing a different search engine has + * no equivalent hazard, and the ability to fall back to another engine when the + * first one is serving a captcha is the single most useful thing the agent can + * do with this tool. So the model is allowed to override the default; the + * operator sets what it starts from. + */ +export const resolveEngine = ( + parameter: string | null | undefined, + env: NodeJS.ProcessEnv = process.env, + stored: string | null = null, +): EngineResolution => { + const candidates: Array<{ raw: string; source: EngineSource }> = []; + + const fromParam = parameter?.trim(); + if (fromParam) candidates.push({ raw: fromParam, source: "parameter" }); + + const fromEnv = env[ENGINE_ENV_VAR]?.trim(); + if (fromEnv) candidates.push({ raw: fromEnv, source: "env" }); + + const fromStore = stored?.trim(); + if (fromStore) candidates.push({ raw: fromStore, source: "stored" }); + + for (const candidate of candidates) { + const id = parseEngineId(candidate.raw); + // An unparseable value is reported rather than skipped over. Falling + // through to the next source would mean a typo in CCS_SEARCH_ENGINE + // silently searches Google instead of saying so. + if (!id) return { kind: "invalid", raw: candidate.raw, source: candidate.source }; + return { kind: "ok", id, source: candidate.source }; + } + + return { kind: "ok", id: DEFAULT_ENGINE, source: "default" }; +}; + +const sourceLabel = (source: EngineSource): string => + source === "parameter" + ? "engine parameter" + : source === "env" + ? ENGINE_ENV_VAR + : source === "stored" + ? "/search-engine" + : "built-in default"; + +export const describeEngineSource = (source: EngineSource): string => sourceLabel(source); + +/** One-line description of a bad engine name, listing what would have worked. */ +export const describeEngineFailure = (failure: Extract): string => + `unknown search engine "${failure.raw}" from ${sourceLabel(failure.source)} — ` + + `expected one of ${engineAliases().join(", ")}`; diff --git a/src/engines.ts b/src/engines.ts new file mode 100644 index 0000000..79462fc --- /dev/null +++ b/src/engines.ts @@ -0,0 +1,227 @@ +/** + * The search engines this extension knows how to drive. + * + * Every selector and every URL parameter below was read off the real SERP in + * castle's Chrome, not recalled or inferred. That is not pedantry: Google's own + * Tools menu writes `tbs=qdr:*`, which renders an *empty* page on that profile, + * while the undocumented-looking `as_qdr=*` works. Anything in here that was not + * observed is a bug waiting to be reported as "no results". + * + * Two extraction modes, because the engines genuinely differ: + * + * - "items" — the SERP has a clean per-result container (`li.b_algo`, + * `.result`, `.snippet[data-type=web]`). Straightforward. + * - "headings" — Google has no stable result container, so results are found + * from each

outward. Kept as its own mode rather than + * forced into the item shape, because it is the one that has + * been through the most verification. + */ + +export type EngineId = "google" | "duckduckgo" | "bing" | "brave"; + +export type Recency = "day" | "week" | "month" | "year"; + +export const RECENCY_VALUES = ["day", "week", "month", "year"] as const; + +export const ENGINE_IDS: readonly EngineId[] = ["google", "duckduckgo", "bing", "brave"]; + +/** Aliases accepted from humans and from the model. */ +const ENGINE_ALIASES: Record = { + google: "google", + g: "google", + duckduckgo: "duckduckgo", + ddg: "duckduckgo", + duck: "duckduckgo", + bing: "bing", + b: "bing", + brave: "brave", +}; + +/** + * How the in-page script should read one engine's results. Passed into the + * browser as JSON, so everything here must be plain data. + */ +export type ExtractionConfig = { + mode: "items" | "headings"; + /** Results container candidates, first match wins. */ + roots: string[]; + /** "items" mode: one result. */ + item?: string; + /** Anchor carrying the outbound link, relative to the item. */ + link?: string; + /** Title element, relative to the item. Falls back to the anchor's text. */ + title?: string; + /** Description element, relative to the item. */ + snippet?: string; + /** + * "items" mode with no snippet selector: text from these is subtracted from + * the item's text to leave the description behind. + */ + subtract?: string[]; + /** Items matching any of these are ads or non-web cards, and are skipped. */ + exclude?: string[]; + /** How outbound links are wrapped, if they are. */ + unwrap: "none" | "ddg" | "bing"; + /** Hosts belonging to the engine itself; links to them are not results. */ + selfHostPattern: string; +}; + +export type EngineDefinition = { + readonly id: EngineId; + readonly label: string; + /** Human-facing note about what makes this engine worth choosing. */ + readonly note: string; + /** + * Recency windows this engine can actually express, mapped to the parameter + * value. A window that is absent is one the engine does not support — never + * one that is silently dropped. + */ + readonly recency: Partial>; + readonly buildUrl: (query: string, numResults: number, recency: Recency | undefined) => string; + readonly extraction: ExtractionConfig; +}; + +const GOOGLE: EngineDefinition = { + id: "google", + label: "Google", + note: "best result quality; blocks aggressively under repeated automated queries", + recency: { day: "d", week: "w", month: "m", year: "y" }, + buildUrl: (query, numResults, recency) => { + const params = new URLSearchParams({ q: query, num: String(numResults), hl: "en" }); + // `udm=14` is the plain "Web" tab: no AI overview, no carousels. But any + // date restriction combined with it renders an empty page, and `tbs=qdr:*` + // renders an empty page on its own — both observed. So a time-limited + // search drops udm and uses the older as_qdr instead. + if (recency) params.set("as_qdr", GOOGLE.recency[recency] as string); + else params.set("udm", "14"); + return `https://www.google.com/search?${params.toString()}`; + }, + extraction: { + mode: "headings", + roots: ["#rso", "#search"], + unwrap: "none", + selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/`, + }, +}; + +const DUCKDUCKGO: EngineDefinition = { + id: "duckduckgo", + label: "DuckDuckGo", + note: "no-JS endpoint, the most stable markup here and the least likely to challenge", + // Read off DuckDuckGo's own