aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--castle-search/.gitignore4
-rw-r--r--castle-search/README.md224
-rw-r--r--castle-search/docs/2026-08-04-ai-answers.md235
-rw-r--r--castle-search/package.json38
-rw-r--r--castle-search/src/cdp.ts342
-rw-r--r--castle-search/src/endpoint.ts108
-rw-r--r--castle-search/src/engine-store.ts81
-rw-r--r--castle-search/src/engine-target.ts79
-rw-r--r--castle-search/src/engines.ts229
-rw-r--r--castle-search/src/errors.ts83
-rw-r--r--castle-search/src/extract.ts294
-rw-r--r--castle-search/src/format.ts54
-rw-r--r--castle-search/src/index.ts315
-rw-r--r--castle-search/src/paths.ts41
-rw-r--r--castle-search/src/search.ts339
-rw-r--r--castle-search/src/target-store.ts42
-rw-r--r--castle-search/test/concurrency.test.ts86
-rw-r--r--castle-search/test/endpoint.test.ts64
-rw-r--r--castle-search/test/engine-target.test.ts89
-rw-r--r--castle-search/test/format.test.ts70
-rw-r--r--castle-search/test/search-url.test.ts88
-rw-r--r--castle-search/tsconfig.json18
22 files changed, 2923 insertions, 0 deletions
diff --git a/castle-search/.gitignore b/castle-search/.gitignore
new file mode 100644
index 0000000..2d153b7
--- /dev/null
+++ b/castle-search/.gitignore
@@ -0,0 +1,4 @@
+node_modules/
+package-lock.json
+*.tsbuildinfo
+.DS_Store
diff --git a/castle-search/README.md b/castle-search/README.md
new file mode 100644
index 0000000..6dac8a7
--- /dev/null
+++ b/castle-search/README.md
@@ -0,0 +1,224 @@
+# pi-castle-cdp-search
+
+Web search for [pi](https://pi.dev) through a Chrome that is **already running**
+on the wire network, driven over the Chrome DevTools Protocol.
+
+It never launches a browser. It attaches to one a person or a launchd job
+started — by default castle's shared Chrome on `10.88.0.25:9223` — opens a
+background tab, reads the results page, and closes the tab again.
+
+Because the search runs in a real browser with a real profile, it reaches pages
+that reject datacentre traffic, and it costs nothing per query.
+
+## The tool
+
+One tool, `castle_cdp_search`. Deliberately **not** named `web_search`, so it
+coexists with `pi-web-access` rather than shadowing it — both stay registered and
+the model picks.
+
+| Parameter | Type | Notes |
+|---|---|---|
+| `query` | string, required | As you would type it into a search box |
+| `numResults` | integer, optional | 1–20, default 10 |
+| `recency` | `day` \| `week` \| `month` \| `year`, optional | Omit for no time limit |
+| `engine` | `google` \| `duckduckgo` \| `bing` \| `brave`, optional | Overrides the configured default for one call |
+
+It returns titles, URLs and snippets — **search results only**. It does not fetch
+the linked pages; follow up with a read/fetch tool for full content.
+
+## Engines
+
+Four, with independent indexes and independent rate limits. That last part is
+the point: when one starts serving captchas, the others generally still work,
+and the tool tells the agent so inside the error.
+
+| Engine | Recency windows | Notes |
+|---|---|---|
+| `google` | day, week, month, year | Best results; blocks aggressively under automation |
+| `duckduckgo` | day, week, month, year | No-JS endpoint — the most stable markup here, least likely to challenge |
+| `bing` | day, week, month | No year window exists |
+| `brave` | none | Independent index, unwrapped links; challenges quickly |
+
+Aliases are accepted: `ddg`, `duck`, `g`, `b`.
+
+**An unsupported recency window is refused, not ignored.** Asking Bing for the
+past year raises `RecencyUnsupportedError` naming the engines that can do it.
+Silently returning unfiltered results would be indistinguishable from success.
+
+### Switching engines
+
+Exactly like switching the CDP host, plus a per-call override:
+
+```
+/search-engine show the current engine and where it came from
+/search-engine ddg persist a choice (machine-wide, survives restarts)
+/search-engine default back to Google
+CCS_SEARCH_ENGINE=bing pi … pin for one process
+```
+
+Precedence: **`engine` parameter > `CCS_SEARCH_ENGINE` > `/search-engine` >
+Google.** The stored choice lives in
+`~/.pi/agent/castle-cdp-search-engine.json`, written atomically.
+
+Note the tool parameter sits *above* the environment variable, which is the
+opposite of how the CDP target treats an explicit endpoint. That is deliberate:
+for the browser, an env var must win because quietly driving a different machine
+means automating the operator's own signed-in Chrome — a safety property.
+Choosing a different search engine carries no such hazard, and letting the agent
+fall back when one engine is blocked is the single most useful thing it can do
+with this tool.
+
+An unrecognised engine name is reported rather than skipped, so a typo in
+`CCS_SEARCH_ENGINE` never silently searches Google instead.
+
+## Which browser it drives
+
+Exactly the same configuration as [`pi-browser-harness`](../pi-browser-harness),
+on purpose: one decision governs both packages, so they can never end up driving
+different browsers in the same session.
+
+| `BU_CDP_HTTP` | Behaviour |
+|---|---|
+| unset | the `/browser-target` choice stored by pi-browser-harness, else castle (`10.88.0.25:9223`) |
+| `host:port` | that endpoint |
+| `local` (also `localhost`, `off`, `none`, `0`, `no`) | this machine's own browser on `127.0.0.1:9222` |
+
+Precedence is `BU_CDP_HTTP` > `/browser-target` > built-in default, matching the
+harness. The `/browser-target` file (`~/.pi/agent/browser-target.json`) is read
+only, never written — the harness owns that setting.
+
+**One deliberate difference from the harness.** When the configured endpoint is
+unreachable, this extension reports that and stops. The harness falls back from
+its built-in default to a local browser; here that fallback would mean silently
+running searches through the operator's own signed-in Chrome without saying so.
+Say `local` explicitly if that is what you want.
+
+The endpoint is re-resolved from `/json/version` on every connect, never cached:
+castle recycles Chrome hourly and Chrome re-mints its browser UUID each launch,
+so a pinned `ws://` URL breaks on the first recycle. A dropped socket reconnects
+transparently on the next search.
+
+## Behaviour worth knowing
+
+- **Nothing is dialled at load.** pi runs extension factories in invocations that
+ never start a session, so the connection is made on first search and closed by
+ an idempotent `session_shutdown` handler.
+- **One fresh background tab per search**, closed on every path including errors
+ and aborts. `background: true` so it does not steal focus from whoever is
+ looking at that screen.
+- **Two concurrent searches**, queued beyond that.
+- **20-second deadline** per call, covering the queue wait, connect, navigate and
+ extract together. Esc aborts an in-flight navigation.
+- **Every failure throws.** pi only sets `isError: true` when `execute()` throws;
+ a returned `{ error }` object would read to the model as a search that simply
+ found nothing.
+
+## What it touches
+
+The browser is shared and belongs to a person. Worth being aware of:
+
+- Queries the agent runs land in that browser profile's history and cookies, and
+ in the search engine's account history if that profile is signed in.
+- Tabs open and close on someone's screen. `background: true` keeps them from
+ stealing focus, but they are visible.
+- Searching hard trips rate limiters for the whole host, and the challenge
+ affects the human using that browser too. Measured during development:
+ **Google** tolerated a lot, then roughly 30 queries in a few minutes cost about
+ 90 minutes of blocking. **Brave** is far stricter — it challenges after as few
+ as one or two queries and clears again in roughly 15–20 minutes. **DuckDuckGo**
+ and **Bing** never challenged once across the whole of development, which is
+ why DuckDuckGo is the one to reach for when you need to run several searches
+ in a row.
+- **A decaying block does not look like a block.** Once the challenge page stops
+ being served, engines return an *empty results page* for a while instead —
+ indistinguishable from a query with no hits. It surfaces as `NoResultsError`,
+ not `SearchChallengeError`. If one engine comes back empty and another answers
+ the same query, that is rate limiting.
+
+## When a captcha appears
+
+Every engine eventually serves an interstitial: Google's `/sorry/` page, Brave's
+"Verifying you're not a bot", a consent wall, a Cloudflare challenge. The
+extension detects these and throws `SearchChallengeError` naming the challenge,
+the engine, the browser, and the URL a human would have to visit.
+
+The error tells the agent to **try another engine first**, because a challenge on
+one engine says nothing about the others and that is a fix it can apply itself.
+Only when engines run out should it ask the user to clear the challenge in the
+browser. The cookie is profile-wide, so one pass unblocks later searches.
+
+The tab is closed rather than left open on the challenge page: leaving it would
+let the operator solve it in place, but would also litter a shared browser with
+abandoned tabs on every failure. The URL in the error is enough.
+
+## Search parameters, and three traps
+
+Everything below was observed against castle's real Chrome, not inferred. All
+three fail *silently* — the results look entirely plausible, just wrong.
+
+1. **Google: `tbs=qdr:*` renders an empty page.** That is the parameter Google's
+ own Tools menu writes. The older `as_qdr=*` works.
+2. **Google: any date filter combined with `udm=14` renders that same empty
+ page.** So a time-limited search drops `udm` and uses the classic layout.
+3. **Bing: `count=` silently cancels `filters=`.** With `ex1:"ez1"` alone every
+ result is hours old; add `count` in either order and months-old results come
+ back, unfiltered and unremarkable-looking. So Bing drops `count` whenever a
+ date filter is present and the caller slices the list instead.
+
+Each has a regression test asserting the parameter combination, since none of
+them would announce itself if it regressed.
+
+## Result extraction
+
+Two modes, because the engines genuinely differ:
+
+- **items** — DuckDuckGo (`.result`), Bing (`li.b_algo`) and Brave
+ (`.snippet[data-type=web]`) each have a clean per-result container.
+- **headings** — Google has no stable container; its class names (`MjjYud`,
+ `kb0PBd`, `yuRUbf`) are generated. Results are found from each `<h3>` outward,
+ climbing to the enclosing block but abandoning the climb if a second `<h3>`
+ comes into scope.
+
+DuckDuckGo and Bing both route outbound links through redirectors
+(`duckduckgo.com/l/?uddg=…`, `bing.com/ck/a?…&u=a1<base64url>`); both are
+unwrapped in the page so the agent gets real URLs. Google and Brave link
+straight out.
+
+Markup will change. When it does, `NoResultsError` distinguishes "container
+rendered but unparseable" (that engine's extractor needs updating) from "no
+container at all" (query, or rate limiting).
+
+## Install
+
+```
+pi install ssh://sz@10.88.0.25/Users/sz/repos/pi-castle-cdp-search.git
+```
+
+The ssh form is what pi-browser-harness uses, and it is the portable one: pi
+records local-path installs by resolved absolute path, and `settings.json` is
+shared across hosts through the dotfiles symlink.
+
+To try it without installing:
+
+```
+pi -e ./src/index.ts
+```
+
+## Development
+
+```
+npm install
+npm run typecheck
+npm test # unit tests, no browser needed
+```
+
+The tests cover engine selection and precedence, URL building for all four
+engines (including the three traps above), recency refusal, formatting, the
+challenge error, and the concurrency semaphore. The semaphore tests point at an
+unreachable endpoint on purpose — the questions there are about slot
+bookkeeping, and hitting a real browser to test a counter would be slow, flaky,
+and rude to whoever is using it.
+
+Requires Node 22+, for the global `WebSocket`. There are no runtime
+dependencies; pi supplies `@earendil-works/pi-coding-agent`,
+`@earendil-works/pi-ai` and `typebox` to extensions through its own loader.
diff --git a/castle-search/docs/2026-08-04-ai-answers.md b/castle-search/docs/2026-08-04-ai-answers.md
new file mode 100644
index 0000000..29b5a6b
--- /dev/null
+++ b/castle-search/docs/2026-08-04-ai-answers.md
@@ -0,0 +1,235 @@
+# AI answers on the SERP — findings, 2026-08-04
+
+Status: **investigated, not implemented, not decided.** Nothing in `src/` has
+changed. This is the field notes for whoever picks it up.
+
+The question was whether the AI answer block that Google, Bing and Brave now put
+above their results ("AI Overview", the Bing generative answer, Brave's AI
+summary) is worth pulling into `castle_cdp_search`'s output alongside the ranked
+results. It is not obviously worth it — the cost is real and is set out below —
+so the decision was deferred rather than taken.
+
+Everything here was read off the live SERPs in castle's Chrome on 2026-08-04, in
+the signed-in `~/.chrome-cdp-profile`, `hl=en`. Same rule as `engines.ts`: what
+is written down is what was observed. Where something was *not* verified it says
+so.
+
+## What is there, per engine
+
+| Engine | Block | Container observed | Sources exposed |
+| --- | --- | --- | --- |
+| Google | "AI Overview" | `#m-x-content` | yes, with titles |
+| Bing | generative answer | `li.b_ans.b_top.b_qnacdxcont` → `.gs_caphead_main` | yes, cards in `.gs_cit` |
+| Brave | AI summary | `#chatllm-main-answer-content` | no |
+| DuckDuckGo | none | — | — |
+
+DuckDuckGo is a genuine "none", not an unfinished check: the engine points at the
+no-JS `html.duckduckgo.com` endpoint, which cannot render one. There is no
+`.zci`/zero-click block on it either — that was checked directly.
+
+### Google
+
+The overview lives **outside `#search` and `#rso`**, above them. That was
+verified (`rso.contains(node) === false`), and it means the current extractor
+neither sees it nor is disturbed by it — no existing behaviour is at risk here.
+
+Structure of `#m-x-content`, on a query that had one:
+
+```
+#m-x-content
+ [jsname=txosbe] "AI Overview" ← the header
+ [jsname=HKDuG] the answer body ← also contains the citations
+ … answer prose
+ [role=dialog] "6 sites" + cards + "Show all"
+```
+
+Hooks, ranked by how much they can be trusted:
+
+- `#m-x-content` — an id, present on both loads. The most durable thing found.
+ **But see the trap below: the id alone is not sufficient.**
+- `[role=heading]` with the exact text `AI Overview`, and
+ `[aria-label="Show more AI Overview"]` on the collapse control. Semantic, and
+ the natural anchor if the id ever moves.
+- `[role=dialog]` around the citation carousel (`data-type=hovc`).
+- Class names — **useless**, as everywhere else on Google. The answer body was
+ `.LT6XE` on one load and `.jloFI GkDqAd rJweXb` on the next, same query,
+ minutes apart. This is the same churn `engines.ts` already warns about.
+
+**Citations are the good part.** Anchors inside `#m-x-content` carry the source
+title in `aria-label`, suffixed `". Opens in a new tab."`:
+
+```
+aria-label="Open source agentic startup LangChain hits $1.25B valuation | TechCrunch. Opens in a new tab."
+href="https://techcrunch.com/2025/10/21/open-source-agentic-startup-langchain-hits-1-25b-valuation/"
+```
+
+Their `innerText` is empty — the visible label is an icon — so `aria-label` is
+the only way to get a title. Links out directly, no redirector, so the existing
+`unwrap: "none"` holds. Google's own hosts appear among them (a privacy-policy
+link) and are already excluded by `selfHostPattern`.
+
+Two traps:
+
+1. **`#m-x-content` exists before the overview does, holding something else.**
+ Polling a fresh load at 250ms intervals caught it at 1502ms containing 28,864
+ characters — approximately the whole SERP — and at 1752ms containing 142, then
+ growing to its real 903. So the id is reused by, or briefly wraps, a larger
+ shell. An extractor that reads it on sight will occasionally return the entire
+ page as the "AI answer". **Require the text to begin with `AI Overview`, or
+ anchor on the heading and climb, before trusting the node.**
+2. **"Show more" is not hiding any of the answer.** Clicking it took the text
+ from 1124 to 1262 characters, and the 138 added were the disclaimer footer
+ ("Double-check important information… AI responses may include mistakes").
+ The answer body is complete in the DOM while still visually truncated. So
+ there is no need to click anything — which is good, because clicking in a
+ browser a human is using is exactly what this package avoids.
+
+To render usefully, the extractor has to drop the `AI Overview` header line, the
+`[role=dialog]` citation carousel, the "Show all" / "Dismiss" / "Learn how"
+buttons and the disclaimer. Cleanest approach found: `cloneNode(true)`, remove
+those by selector on the clone, take `innerText`. The page itself is never
+mutated, which matters on a shared browser.
+
+### Bing
+
+```
+li.b_ans.b_top.b_qnacdxcont
+ .gs_caphead_main the answer prose ← clean, nothing else in it
+ .gs_cit_cont
+ .gs_cit source card: name, title, snippet
+ .gs_cit …
+ .cit_exp_cont "View all"
+```
+
+The `gs_` prefix ("generative search") is descriptive rather than hashed, and
+`.gs_caphead_main` held the answer and *only* the answer — no header, no cards,
+no disclaimer. The nicest of the three to extract.
+
+`li.b_ans` on its own is **not** a safe selector: "People also ask" is also a
+`li.b_ans`. The discriminator is `b_qnacdxcont` on the `li`, or the presence of
+`.gs_caphead_main` inside it.
+
+Citation anchors go through Bing's `/ck/a?…&u=a1…` redirector, so the existing
+`unwrap: "bing"` already handles them — no new unwrapping needed.
+
+No URL change is needed: the answer appears on the plain
+`bing.com/search?q=…` the engine already builds.
+
+### Brave
+
+`#chatllm-main-answer-content` contained the answer and nothing else — no
+header, no disclaimer (that sits outside it, in `cite.chatllm-disclaimer`), no
+citations. The wrapper is `#llm-snippet`, a `.snippet.standalone` sitting first
+in `#results`.
+
+Worth noting: `engines.ts` already restricts Brave's results to
+`.snippet[data-type="web"]` specifically to exclude this block. That filter
+stays correct — the AI answer would be read separately, not as a result.
+
+No sources are exposed. The only anchors inside are `search.brave.com/ask?…`
+follow-up prompts ("Elaborate", "Who are LangChain's main competitors?"), which
+`selfHostPattern` already discards. There is a "+4 More" control that may reveal
+sources; **not investigated** — it would need a click, and clicking was avoided.
+
+No URL change needed.
+
+## The two reasons this is not free
+
+### 1. Google's AI Overview requires giving up `udm=14`
+
+`udm` is Google's undocumented vertical selector — the parameter its own tab
+strip writes. `udm=2` is Images, `udm=7` Videos, `udm=12` News, `udm=28`
+Shopping, and `udm=14` is the "Web" tab added in May 2024: plain ranked links,
+no AI Overview, no knowledge panel, no carousels. `engines.ts` sets it as
+Google's default precisely for that cleanliness.
+
+So "no AI Overview" is not incidental to `udm=14`, it is part of what the Web tab
+*is*. Every page where `#m-x-content` was found was a no-`udm` classic SERP.
+Enabling overviews on Google means dropping `udm`.
+
+That is less alarming than it sounds: the engine **already** drops `udm` for any
+time-limited search, because `udm=14` plus a date filter renders an empty page
+(README, trap 2). Heading-mode extraction is therefore already exercised against
+the classic layout. The cost is that the classic layout puts back all the
+furniture — knowledge panels, "People also ask", carousels — that the extractor
+then has to keep ignoring.
+
+### 2. They stream in after the results are ready, and are often absent
+
+Measured from navigation start, polling at 250ms via an `initScript`:
+
+| Engine | results parseable | AI answer appears | AI answer settles |
+| --- | --- | --- | --- |
+| Google | ~500ms | ~1500ms | ~2300ms |
+| Brave | ~290ms | ~1260ms | **still growing at 5000ms** |
+| Bing | ~520ms | — (absent on that query) | — |
+
+`search.ts` reads results and returns as soon as anything parses, retrying at
+most twice at 700ms (`EXTRACT_RETRIES`, `EXTRACT_RETRY_DELAY_MS`) and breaking
+the moment `results.length > 0`. In other words it returns roughly 2s before
+Google's overview lands and 5s+ before Brave's finishes. **Capturing an AI answer
+means deliberately waiting past the point where the useful part of the job is
+already done.**
+
+And the wait is frequently wasted, because the blocks are query-dependent:
+
+- `how does postgres MVCC vacuum work` → **no** Google AI Overview at all, still
+ absent after 15s, in a foreground tab. (Checked in a foreground tab
+ specifically to rule out background-tab throttling as the cause — an earlier
+ `window.open` probe was discarded for that reason.)
+- `who founded anthropic and when` → **no** Bing generative answer after 13.5s;
+ Bing served an ordinary big-Wikipedia card instead.
+- `LangChain valuation Series C funding 2026` → all three produced one.
+
+There is no cheap way to know in advance which case you are in, so a fixed wait
+pays the full cost on every query that was never going to have an answer.
+
+## The decision that was not taken
+
+Three shapes were on the table:
+
+1. **Opt-in `aiAnswer` parameter (the one that was going to be recommended).**
+ Default off: today's latency and today's `udm=14` layout, byte-identical
+ behaviour. When true: Google drops `udm`, and the probe polls until the text
+ stops growing or a budget (~6s) expires. Costs nothing unless asked for, and
+ the model can ask for it when the query is a question rather than a lookup.
+ Downside: another parameter on a tool that already has four, and the model has
+ to decide when to set it.
+2. **Always on.** Simplest surface, no new parameter. Every search pays ~2s
+ (Google) to ~5s (Brave), much of it waiting for answers that never arrive,
+ against a 20s whole-call deadline (`SEARCH_TIMEOUT_MS`) that also has to cover
+ connect and navigate.
+3. **Opportunistic, never wait.** Include an AI answer only if it is already
+ rendered when results are read. Zero added latency, but Google would
+ essentially never contribute one (`udm=14` stays), and Brave would usually be
+ caught mid-sentence — see the growth curve above. Risks putting a truncated,
+ confident-sounding half-answer in front of the model, which is worse than
+ putting none there.
+
+Unresolved either way: whether an AI answer belongs in the model's context at
+all. It is a summary generated by another model from pages the agent could read
+itself, it carries no provenance beyond its citations, and `promptGuidelines`
+currently tells the model in plain terms not to treat a snippet as the page.
+An AI answer invites exactly that confusion at greater length. If it goes in, it
+should be labelled unmistakably as the engine's generated text — engine name
+included, e.g. "Google AI Overview" — and never merged into the numbered results.
+
+## If you implement it
+
+- `ExtractionConfig` in `src/engines.ts` is the right place for the per-engine
+ selectors; it is already plain JSON crossing into the page.
+- `PageProbe` in `src/extract.ts` gains an optional field; `SearchResult` should
+ **not** — an AI answer is not a hit and must not be numbered among them.
+- The settle-poll belongs in the page script, not in round trips from
+ `search.ts`. One `Runtime.evaluate` that polls locally until the text stops
+ growing is one round trip; polling from the host is one per sample.
+- `formatResults` in `src/format.ts` puts it above the numbered list. Mind the
+ budget note at the top of that file — an overview plus citations is another
+ 1-2KB on top of the 15KB worst case.
+- Tests: `test/search-url.test.ts` is where a "Google drops `udm` when an AI
+ answer is wanted" assertion goes, next to the existing parameter-trap tests.
+ The extraction itself cannot be tested without a browser; the existing suite
+ does not try, and this should not either.
+- Re-verify every selector before trusting it. Google's classes churned between
+ two loads minutes apart during this very investigation, and these blocks are
+ changing faster than the rest of the SERP.
diff --git a/castle-search/package.json b/castle-search/package.json
new file mode 100644
index 0000000..3e31124
--- /dev/null
+++ b/castle-search/package.json
@@ -0,0 +1,38 @@
+{
+ "name": "pi-castle-cdp-search",
+ "version": "0.1.0",
+ "private": true,
+ "description": "Web search for pi through an already-running Chrome on the wire network, driven over CDP",
+ "keywords": [
+ "pi-package",
+ "pi-extension",
+ "search",
+ "cdp",
+ "chrome"
+ ],
+ "license": "MIT",
+ "type": "module",
+ "engines": {
+ "node": ">=22.0.0"
+ },
+ "files": [
+ "src/",
+ "tsconfig.json",
+ "README.md"
+ ],
+ "pi": {
+ "extensions": [
+ "./src/index.ts"
+ ]
+ },
+ "scripts": {
+ "typecheck": "tsc --noEmit",
+ "test": "node --experimental-strip-types --test test/*.test.ts"
+ },
+ "devDependencies": {
+ "@earendil-works/pi-ai": "*",
+ "@earendil-works/pi-coding-agent": "*",
+ "typebox": "*",
+ "typescript": "^5.0.0"
+ }
+}
diff --git a/castle-search/src/cdp.ts b/castle-search/src/cdp.ts
new file mode 100644
index 0000000..aa01b56
--- /dev/null
+++ b/castle-search/src/cdp.ts
@@ -0,0 +1,342 @@
+/**
+ * A minimal CDP client — just enough to open a page, navigate it, read the DOM,
+ * and close it again.
+ *
+ * Deliberately dependency-free: Node 22 ships a global `WebSocket`, so this
+ * needs no `ws`. That matters because an extension installed by `pi install`
+ * gets its own `node_modules`, and fewer moving parts there is fewer things to
+ * go wrong on a machine that is not the one it was written on.
+ *
+ * The browser is *not* ours. It is a shared Chrome someone may be looking at,
+ * recycled hourly (`me.soarez.chrome-cdp-recycle`). Two consequences run
+ * through this file:
+ *
+ * 1. The websocket URL is re-resolved from `/json/version` on every connect,
+ * never cached. Chrome re-mints its browser UUID on each launch, so a
+ * pinned ws:// URL breaks the first time the browser is recycled.
+ * 2. Every page this opens is created in the background and closed again,
+ * including on error paths, so a failed search does not leave tabs behind
+ * in someone else's window.
+ */
+
+import { BrowserUnavailableError, SearchAbortedError, messageOf } from "./errors.ts";
+import type { CdpTarget } from "./endpoint.ts";
+
+/** How long a single CDP command may take before the socket is presumed dead. */
+const COMMAND_TIMEOUT_MS = 15_000;
+/** How long to wait for /json/version and for the websocket handshake. */
+const CONNECT_TIMEOUT_MS = 5_000;
+
+type CdpMessage = {
+ id?: number;
+ method?: string;
+ params?: Record<string, unknown>;
+ sessionId?: string;
+ result?: Record<string, unknown>;
+ error?: { code?: number; message?: string };
+};
+
+type Pending = {
+ resolve: (value: Record<string, unknown>) => void;
+ reject: (reason: Error) => void;
+ timer: ReturnType<typeof setTimeout>;
+};
+
+/**
+ * Reject if `signal` fires before `promise` settles, cleaning up the listener
+ * either way. Node's own abort-aware APIs are not available here — the
+ * websocket predates them — so aborts are raced in rather than plumbed through.
+ */
+export const raceAbort = <T>(promise: Promise<T>, signal: AbortSignal | undefined): Promise<T> => {
+ if (!signal) return promise;
+ if (signal.aborted) return Promise.reject(abortError(signal));
+ return new Promise<T>((resolve, reject) => {
+ const onAbort = (): void => reject(abortError(signal));
+ signal.addEventListener("abort", onAbort, { once: true });
+ promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", onAbort));
+ });
+};
+
+const abortError = (signal: AbortSignal): Error => {
+ const reason: unknown = signal.reason;
+ if (reason instanceof Error && reason.name === "TimeoutError") {
+ return new SearchAbortedError("search timed out");
+ }
+ return new SearchAbortedError("search was cancelled");
+};
+
+/** Ask the browser for its current websocket endpoint. Never cached — see above. */
+const queryWebSocketUrl = async (target: CdpTarget, signal?: AbortSignal): Promise<string> => {
+ const url = `http://${target.host}:${target.port}/json/version`;
+ let payload: unknown;
+ try {
+ const res = await fetch(url, {
+ signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(CONNECT_TIMEOUT_MS)]) : AbortSignal.timeout(CONNECT_TIMEOUT_MS),
+ });
+ if (!res.ok) {
+ throw new BrowserUnavailableError(`${url} answered HTTP ${res.status}`);
+ }
+ payload = await res.json();
+ } catch (e) {
+ if (e instanceof BrowserUnavailableError) throw e;
+ if (signal?.aborted) throw abortError(signal);
+ throw new BrowserUnavailableError(
+ `No CDP endpoint answered at ${url} (${messageOf(e)}). Is the browser running and reachable on the overlay?`,
+ );
+ }
+ const ws = (payload as Record<string, unknown> | null)?.["webSocketDebuggerUrl"];
+ if (typeof ws !== "string" || ws === "") {
+ throw new BrowserUnavailableError(`${url} answered without a webSocketDebuggerUrl`);
+ }
+ return ws;
+};
+
+/** A live browser-level CDP connection. */
+export class BrowserConnection {
+ #ws: WebSocket;
+ #nextId = 1;
+ #pending = new Map<number, Pending>();
+ #listeners = new Set<(msg: CdpMessage) => void>();
+ #closedReason: string | null = null;
+
+ readonly target: CdpTarget;
+
+ private constructor(ws: WebSocket, target: CdpTarget) {
+ this.#ws = ws;
+ this.target = target;
+
+ ws.addEventListener("message", (ev: MessageEvent) => {
+ let msg: CdpMessage;
+ try {
+ msg = JSON.parse(String(ev.data)) as CdpMessage;
+ } catch {
+ return;
+ }
+ if (msg.id !== undefined) {
+ const pending = this.#pending.get(msg.id);
+ if (!pending) return;
+ this.#pending.delete(msg.id);
+ clearTimeout(pending.timer);
+ if (msg.error) {
+ pending.reject(new BrowserUnavailableError(`CDP error: ${msg.error.message ?? "unknown"}`));
+ } else {
+ pending.resolve(msg.result ?? {});
+ }
+ return;
+ }
+ for (const listener of [...this.#listeners]) listener(msg);
+ });
+
+ const fail = (reason: string): void => {
+ this.#closedReason ??= reason;
+ for (const [, pending] of this.#pending) {
+ clearTimeout(pending.timer);
+ pending.reject(new BrowserUnavailableError(reason));
+ }
+ this.#pending.clear();
+ };
+ ws.addEventListener("close", () => fail("CDP connection closed (the browser may have been recycled)"));
+ ws.addEventListener("error", () => fail("CDP connection failed"));
+ }
+
+ /** Resolve the endpoint afresh and dial it. */
+ static async connect(target: CdpTarget, signal?: AbortSignal): Promise<BrowserConnection> {
+ const wsUrl = await queryWebSocketUrl(target, signal);
+ const ws = new WebSocket(wsUrl);
+ try {
+ await raceAbort(
+ new Promise<void>((resolve, reject) => {
+ const timer = setTimeout(
+ () => reject(new BrowserUnavailableError(`timed out opening ${wsUrl}`)),
+ CONNECT_TIMEOUT_MS,
+ );
+ ws.addEventListener(
+ "open",
+ () => {
+ clearTimeout(timer);
+ resolve();
+ },
+ { once: true },
+ );
+ ws.addEventListener(
+ "error",
+ () => {
+ clearTimeout(timer);
+ reject(new BrowserUnavailableError(`could not open ${wsUrl}`));
+ },
+ { once: true },
+ );
+ }),
+ signal,
+ );
+ } catch (e) {
+ try {
+ ws.close();
+ } catch {
+ // Already failed; nothing useful to do with a second failure.
+ }
+ throw e;
+ }
+ return new BrowserConnection(ws, target);
+ }
+
+ /** False once the socket has closed or failed — the reconnect trigger. */
+ get isOpen(): boolean {
+ return this.#closedReason === null && this.#ws.readyState === 1 /* OPEN */;
+ }
+
+ send(method: string, params: Record<string, unknown> = {}, sessionId?: string): Promise<Record<string, unknown>> {
+ if (!this.isOpen) {
+ return Promise.reject(new BrowserUnavailableError(this.#closedReason ?? "CDP connection is not open"));
+ }
+ const id = this.#nextId++;
+ const payload: CdpMessage = sessionId ? { id, method, params, sessionId } : { id, method, params };
+ return new Promise<Record<string, unknown>>((resolve, reject) => {
+ const timer = setTimeout(() => {
+ this.#pending.delete(id);
+ reject(new BrowserUnavailableError(`CDP command ${method} timed out after ${COMMAND_TIMEOUT_MS}ms`));
+ }, COMMAND_TIMEOUT_MS);
+ this.#pending.set(id, { resolve, reject, timer });
+ try {
+ this.#ws.send(JSON.stringify(payload));
+ } catch (e) {
+ this.#pending.delete(id);
+ clearTimeout(timer);
+ reject(new BrowserUnavailableError(`could not send ${method}: ${messageOf(e)}`));
+ }
+ });
+ }
+
+ /** Subscribe to unsolicited protocol events. Returns an unsubscribe function. */
+ onEvent(listener: (msg: CdpMessage) => void): () => void {
+ this.#listeners.add(listener);
+ return () => this.#listeners.delete(listener);
+ }
+
+ close(): void {
+ this.#closedReason ??= "CDP connection closed by this extension";
+ this.#listeners.clear();
+ try {
+ this.#ws.close();
+ } catch {
+ // Closing an already-dead socket is not a failure worth reporting.
+ }
+ }
+}
+
+/**
+ * One throwaway page.
+ *
+ * Created in the background so it does not steal focus from whoever is using
+ * the browser, and closed by {@link close} on every path.
+ */
+export class PageSession {
+ readonly #conn: BrowserConnection;
+ readonly #targetId: string;
+ readonly #sessionId: string;
+ #closed = false;
+
+ private constructor(conn: BrowserConnection, targetId: string, sessionId: string) {
+ this.#conn = conn;
+ this.#targetId = targetId;
+ this.#sessionId = sessionId;
+ }
+
+ static async open(conn: BrowserConnection, signal?: AbortSignal): Promise<PageSession> {
+ if (signal?.aborted) throw abortError(signal);
+
+ // Deliberately NOT raced against the abort signal. raceAbort abandons the
+ // promise, it cannot cancel the command — and this command's side effect is
+ // a tab. Abandoning it would leave a tab open in a browser someone else is
+ // using, with nothing left holding its targetId to close it. It is a fast
+ // command already bounded by COMMAND_TIMEOUT_MS, so waiting it out costs
+ // little and guarantees we own whatever it created.
+ const created = await conn.send("Target.createTarget", { url: "about:blank", background: true });
+ const targetId = created["targetId"];
+ if (typeof targetId !== "string") {
+ throw new BrowserUnavailableError("Target.createTarget did not return a targetId");
+ }
+
+ // From here on the page exists, so every failure — including a late abort —
+ // must still close it.
+ try {
+ if (signal?.aborted) throw abortError(signal);
+ const attached = await raceAbort(
+ conn.send("Target.attachToTarget", { targetId, flatten: true }),
+ signal,
+ );
+ const sessionId = attached["sessionId"];
+ if (typeof sessionId !== "string") {
+ throw new BrowserUnavailableError("Target.attachToTarget did not return a sessionId");
+ }
+ const page = new PageSession(conn, targetId, sessionId);
+ await raceAbort(conn.send("Page.enable", {}, sessionId), signal);
+ return page;
+ } catch (e) {
+ await conn.send("Target.closeTarget", { targetId }).catch(() => {});
+ throw e;
+ }
+ }
+
+ /** Navigate and wait for the load event, or for `signal` to fire. */
+ async navigate(url: string, signal?: AbortSignal): Promise<void> {
+ // Subscribed before navigating, so a page that loads faster than the
+ // Page.navigate reply cannot slip its load event past us.
+ let off: () => void = () => {};
+ const loaded = new Promise<void>((resolve) => {
+ off = this.#conn.onEvent((msg) => {
+ if (msg.sessionId === this.#sessionId && msg.method === "Page.loadEventFired") resolve();
+ });
+ });
+
+ try {
+ const nav = await raceAbort(this.#conn.send("Page.navigate", { url }, this.#sessionId), signal);
+ const errorText = nav["errorText"];
+ if (typeof errorText === "string" && errorText !== "") {
+ throw new BrowserUnavailableError(`navigation to ${url} failed: ${errorText}`);
+ }
+
+ try {
+ await raceAbort(loaded, signal);
+ } catch (e) {
+ // Stop the in-flight load so the page is not still fetching while we
+ // tear it down; the tab is closed either way, this just makes it quicker.
+ await this.#conn.send("Page.stopLoading", {}, this.#sessionId).catch(() => {});
+ throw e;
+ }
+ } finally {
+ // Unsubscribing here rather than inside the listener: on the failure paths
+ // the listener never fires, and a subscription left on a connection that
+ // outlives many searches accumulates.
+ off();
+ }
+ }
+
+ /** Evaluate an expression in the page and return its value by value. */
+ async evaluate<T>(expression: string, signal?: AbortSignal): Promise<T> {
+ const res = await raceAbort(
+ this.#conn.send(
+ "Runtime.evaluate",
+ { expression, returnByValue: true, awaitPromise: true },
+ this.#sessionId,
+ ),
+ signal,
+ );
+ const exception = res["exceptionDetails"] as { text?: string; exception?: { description?: string } } | undefined;
+ if (exception) {
+ const text = exception.exception?.description ?? exception.text ?? "unknown error";
+ throw new BrowserUnavailableError(`page script failed: ${text}`);
+ }
+ return (res["result"] as { value?: T } | undefined)?.value as T;
+ }
+
+ /**
+ * Close the tab. Idempotent and never throws: it runs on error paths, where a
+ * second failure would mask the one worth reporting.
+ */
+ async close(): Promise<void> {
+ if (this.#closed) return;
+ this.#closed = true;
+ await this.#conn.send("Target.closeTarget", { targetId: this.#targetId }).catch(() => {});
+ }
+}
diff --git a/castle-search/src/endpoint.ts b/castle-search/src/endpoint.ts
new file mode 100644
index 0000000..d3f4b0d
--- /dev/null
+++ b/castle-search/src/endpoint.ts
@@ -0,0 +1,108 @@
+/**
+ * Which browser this extension searches through.
+ *
+ * Deliberately the *same* configuration as pi-browser-harness, so one decision
+ * governs both packages and they never drive different browsers in the same
+ * session: `BU_CDP_HTTP`, then the `/browser-target` file that harness writes,
+ * then the built-in castle default.
+ *
+ * Resolution is pure — no I/O — so it can be unit-tested and reported without
+ * touching the network.
+ */
+
+/** Default CDP endpoint: castle's Chrome, exposed on the wg8 overlay. */
+export const DEFAULT_REMOTE_CDP = "10.88.0.25:9223";
+
+/** Values of BU_CDP_HTTP that mean "ignore the default, use this machine". */
+const LOCAL_ALIASES = new Set(["local", "localhost", "off", "none", "0", "no"]);
+
+/** Port a local Chrome is expected to serve CDP on when the target is "local". */
+export const LOCAL_CDP_PORT = 9222;
+
+/**
+ * Where a target came from. "env" is BU_CDP_HTTP, "stored" is the harness's
+ * `/browser-target`, "default" is the built-in.
+ */
+export type CdpTargetSource = "env" | "stored" | "default";
+
+export type CdpTarget = {
+ readonly host: string;
+ readonly port: number;
+ readonly source: CdpTargetSource;
+};
+
+export type CdpTargetResolution =
+ | { readonly kind: "ok"; readonly target: CdpTarget }
+ | { readonly kind: "invalid"; readonly raw: string; readonly reason: string; readonly source: CdpTargetSource };
+
+/**
+ * Precedence: BU_CDP_HTTP, then the stored `/browser-target` choice, then the
+ * built-in default. The environment wins so a one-off `BU_CDP_HTTP=… pi` is
+ * never silently overridden by a choice made days ago.
+ *
+ * Unlike pi-browser-harness this never falls back from the default to a local
+ * browser on its own. The harness can fall back because it drives a browser the
+ * operator is watching; here a silent fallback would mean *searching from the
+ * operator's own signed-in Chrome* without saying so. When the configured
+ * endpoint is unreachable the caller reports that, and the operator can say
+ * `local` explicitly.
+ *
+ * `stored` is passed in rather than read here so this stays pure and synchronous.
+ */
+export const resolveCdpTarget = (
+ env: NodeJS.ProcessEnv = process.env,
+ stored: string | null = null,
+): CdpTargetResolution => {
+ const fromEnv = env["BU_CDP_HTTP"]?.trim();
+ const fromStore = stored?.trim();
+
+ const hasEnv = fromEnv !== undefined && fromEnv !== "";
+ const raw = hasEnv ? fromEnv : fromStore;
+ const source: CdpTargetSource = hasEnv ? "env" : "stored";
+
+ if (raw === undefined || raw === "") {
+ const parsed = parseHostPort(DEFAULT_REMOTE_CDP);
+ // Unreachable in practice: the constant above is covered by its own test.
+ if (!parsed) {
+ return { kind: "invalid", raw: DEFAULT_REMOTE_CDP, source: "default", reason: "built-in default is malformed" };
+ }
+ return { kind: "ok", target: { ...parsed, source: "default" } };
+ }
+
+ if (LOCAL_ALIASES.has(raw.toLowerCase())) {
+ return { kind: "ok", target: { host: "127.0.0.1", port: LOCAL_CDP_PORT, source } };
+ }
+
+ const parsed = parseHostPort(raw);
+ if (!parsed) {
+ return {
+ kind: "invalid",
+ raw,
+ source,
+ reason: `expected "host:port" or one of ${[...LOCAL_ALIASES].join(", ")}`,
+ };
+ }
+ return { kind: "ok", target: { ...parsed, source } };
+};
+
+const parseHostPort = (value: string): { host: string; port: number } | null => {
+ const sep = value.lastIndexOf(":");
+ if (sep === -1) return null;
+ const host = value.slice(0, sep).trim();
+ const port = Number(value.slice(sep + 1).trim());
+ if (!host) return null;
+ if (!Number.isInteger(port) || port <= 0 || port >= 65536) return null;
+ return { host, port };
+};
+
+const sourceLabel = (source: CdpTargetSource): string =>
+ source === "env" ? "BU_CDP_HTTP" : source === "stored" ? "/browser-target" : "built-in default";
+
+/** One-line description for progress messages and errors. */
+export const describeCdpTarget = (target: CdpTarget): string =>
+ `${target.host}:${target.port} (${sourceLabel(target.source)})`;
+
+/** One-line description of a resolution failure. */
+export const describeResolutionFailure = (
+ failure: Extract<CdpTargetResolution, { kind: "invalid" }>,
+): string => `invalid CDP target "${failure.raw}" from ${sourceLabel(failure.source)} — ${failure.reason}`;
diff --git a/castle-search/src/engine-store.ts b/castle-search/src/engine-store.ts
new file mode 100644
index 0000000..d218d2b
--- /dev/null
+++ b/castle-search/src/engine-store.ts
@@ -0,0 +1,81 @@
+/**
+ * Persistence for the `/search-engine` choice.
+ *
+ * Deliberately the same shape, and the same atomic write, as the harness's
+ * browser-target store: a sibling temp file plus rename, so a crash mid-write
+ * cannot leave a half-written engine name behind. Its own file rather than a
+ * shared settings blob, so a corrupt engine choice cannot cost anything else.
+ *
+ * Like the CDP target, the stored value is the raw string a user would have put
+ * in the environment variable ("ddg", "bing", …), not a pre-parsed id. One
+ * vocabulary, one resolution path in engine-target.ts, and a stored value can
+ * never mean something the variable could not.
+ *
+ * Every read failure degrades to "nothing stored" rather than throwing.
+ */
+
+import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { dirname, join } from "node:path";
+
+import { agentDir } from "./paths.ts";
+
+const CURRENT_VERSION = 1;
+
+const engineFilePath = (): string => join(agentDir(), "castle-cdp-search-engine.json");
+
+type EngineFile = {
+ readonly version: 1;
+ /** Raw CCS_SEARCH_ENGINE-style value, or null when cleared. */
+ readonly engine: string | null;
+ readonly savedAt: string;
+};
+
+/**
+ * The persisted engine, or null when nothing is stored, the file is missing, or
+ * it cannot be parsed. A file written by a newer version is treated as "nothing
+ * stored" rather than guessed at.
+ */
+export const readStoredEngine = async (): Promise<string | null> => {
+ let raw: string;
+ try {
+ raw = await readFile(engineFilePath(), "utf8");
+ } catch {
+ return null;
+ }
+ try {
+ const parsed: unknown = JSON.parse(raw);
+ if (typeof parsed !== "object" || parsed === null) return null;
+ const file = parsed as Record<string, unknown>;
+ if (file["version"] !== CURRENT_VERSION) return null;
+ const engine = file["engine"];
+ if (typeof engine !== "string") return null;
+ const trimmed = engine.trim();
+ return trimmed.length > 0 ? trimmed : null;
+ } catch {
+ return null;
+ }
+};
+
+const write = async (engine: string | null): Promise<void> => {
+ const path = engineFilePath();
+ const tmp = `${path}.${randomUUID()}.tmp`;
+ const payload: EngineFile = { version: CURRENT_VERSION, engine, savedAt: new Date().toISOString() };
+ try {
+ await mkdir(dirname(path), { recursive: true });
+ await writeFile(tmp, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
+ await rename(tmp, path);
+ } catch (e) {
+ await unlink(tmp).catch(() => {});
+ throw new Error(`could not save search engine to ${path}: ${e instanceof Error ? e.message : String(e)}`);
+ }
+};
+
+/** Persist the chosen engine. */
+export const writeStoredEngine = (engine: string): Promise<void> => write(engine);
+
+/** Clear the choice, restoring the built-in default. */
+export const clearStoredEngine = (): Promise<void> => write(null);
+
+/** Exposed so the command can tell the user where the choice lives. */
+export const storedEngineLocation = (): string => engineFilePath();
diff --git a/castle-search/src/engine-target.ts b/castle-search/src/engine-target.ts
new file mode 100644
index 0000000..db4d81f
--- /dev/null
+++ b/castle-search/src/engine-target.ts
@@ -0,0 +1,79 @@
+/**
+ * Which search engine to use, resolved the same way the CDP host is:
+ * environment variable, then a persisted choice, then a built-in default.
+ *
+ * Pure — no I/O — so the `/search-engine` command can report the configured
+ * intent without touching disk twice, and so it is testable.
+ */
+
+import { type EngineId, parseEngineId, engineAliases } from "./engines.ts";
+
+/** Used when nothing else says otherwise. */
+export const DEFAULT_ENGINE: EngineId = "google";
+
+/** The environment variable that pins the engine for one process. */
+export const ENGINE_ENV_VAR = "CCS_SEARCH_ENGINE";
+
+export type EngineSource = "parameter" | "env" | "stored" | "default";
+
+export type EngineResolution =
+ | { readonly kind: "ok"; readonly id: EngineId; readonly source: EngineSource }
+ | { readonly kind: "invalid"; readonly raw: string; readonly source: EngineSource };
+
+/**
+ * Precedence: the per-call tool parameter, then `CCS_SEARCH_ENGINE`, then the
+ * `/search-engine` choice, then the built-in default.
+ *
+ * Note the tool parameter sits *above* the environment variable, which is the
+ * opposite of how the CDP target treats an explicit endpoint. That difference
+ * is deliberate. For the browser, an environment variable has to win, because
+ * quietly driving a different machine means automating the operator's own
+ * signed-in Chrome — a safety property. Choosing a different search engine has
+ * no equivalent hazard, and the ability to fall back to another engine when the
+ * first one is serving a captcha is the single most useful thing the agent can
+ * do with this tool. So the model is allowed to override the default; the
+ * operator sets what it starts from.
+ */
+export const resolveEngine = (
+ parameter: string | null | undefined,
+ env: NodeJS.ProcessEnv = process.env,
+ stored: string | null = null,
+): EngineResolution => {
+ const candidates: Array<{ raw: string; source: EngineSource }> = [];
+
+ const fromParam = parameter?.trim();
+ if (fromParam) candidates.push({ raw: fromParam, source: "parameter" });
+
+ const fromEnv = env[ENGINE_ENV_VAR]?.trim();
+ if (fromEnv) candidates.push({ raw: fromEnv, source: "env" });
+
+ const fromStore = stored?.trim();
+ if (fromStore) candidates.push({ raw: fromStore, source: "stored" });
+
+ for (const candidate of candidates) {
+ const id = parseEngineId(candidate.raw);
+ // An unparseable value is reported rather than skipped over. Falling
+ // through to the next source would mean a typo in CCS_SEARCH_ENGINE
+ // silently searches Google instead of saying so.
+ if (!id) return { kind: "invalid", raw: candidate.raw, source: candidate.source };
+ return { kind: "ok", id, source: candidate.source };
+ }
+
+ return { kind: "ok", id: DEFAULT_ENGINE, source: "default" };
+};
+
+const sourceLabel = (source: EngineSource): string =>
+ source === "parameter"
+ ? "engine parameter"
+ : source === "env"
+ ? ENGINE_ENV_VAR
+ : source === "stored"
+ ? "/search-engine"
+ : "built-in default";
+
+export const describeEngineSource = (source: EngineSource): string => sourceLabel(source);
+
+/** One-line description of a bad engine name, listing what would have worked. */
+export const describeEngineFailure = (failure: Extract<EngineResolution, { kind: "invalid" }>): string =>
+ `unknown search engine "${failure.raw}" from ${sourceLabel(failure.source)} — ` +
+ `expected one of ${engineAliases().join(", ")}`;
diff --git a/castle-search/src/engines.ts b/castle-search/src/engines.ts
new file mode 100644
index 0000000..ef8baef
--- /dev/null
+++ b/castle-search/src/engines.ts
@@ -0,0 +1,229 @@
+/**
+ * The search engines this extension knows how to drive.
+ *
+ * Every selector and every URL parameter below was read off the real SERP in
+ * castle's Chrome, not recalled or inferred. That is not pedantry: Google's own
+ * Tools menu writes `tbs=qdr:*`, which renders an *empty* page on that profile,
+ * while the undocumented-looking `as_qdr=*` works. Anything in here that was not
+ * observed is a bug waiting to be reported as "no results".
+ *
+ * Two extraction modes, because the engines genuinely differ:
+ *
+ * - "items" — the SERP has a clean per-result container (`li.b_algo`,
+ * `.result`, `.snippet[data-type=web]`). Straightforward.
+ * - "headings" — Google has no stable result container, so results are found
+ * from each <h3> outward. Kept as its own mode rather than
+ * forced into the item shape, because it is the one that has
+ * been through the most verification.
+ */
+
+export type EngineId = "google" | "duckduckgo" | "bing" | "brave";
+
+export type Recency = "day" | "week" | "month" | "year";
+
+export const RECENCY_VALUES = ["day", "week", "month", "year"] as const;
+
+export const ENGINE_IDS: readonly EngineId[] = ["google", "duckduckgo", "bing", "brave"];
+
+/** Aliases accepted from humans and from the model. */
+const ENGINE_ALIASES: Record<string, EngineId> = {
+ google: "google",
+ g: "google",
+ duckduckgo: "duckduckgo",
+ ddg: "duckduckgo",
+ duck: "duckduckgo",
+ bing: "bing",
+ b: "bing",
+ brave: "brave",
+};
+
+/**
+ * How the in-page script should read one engine's results. Passed into the
+ * browser as JSON, so everything here must be plain data.
+ */
+export type ExtractionConfig = {
+ mode: "items" | "headings";
+ /** Results container candidates, first match wins. */
+ roots: string[];
+ /** "items" mode: one result. */
+ item?: string;
+ /** Anchor carrying the outbound link, relative to the item. */
+ link?: string;
+ /** Title element, relative to the item. Falls back to the anchor's text. */
+ title?: string;
+ /** Description element, relative to the item. */
+ snippet?: string;
+ /**
+ * "items" mode with no snippet selector: text from these is subtracted from
+ * the item's text to leave the description behind.
+ */
+ subtract?: string[];
+ /** Items matching any of these are ads or non-web cards, and are skipped. */
+ exclude?: string[];
+ /** How outbound links are wrapped, if they are. */
+ unwrap: "none" | "ddg" | "bing";
+ /** Hosts belonging to the engine itself; links to them are not results. */
+ selfHostPattern: string;
+};
+
+export type EngineDefinition = {
+ readonly id: EngineId;
+ readonly label: string;
+ /** Human-facing note about what makes this engine worth choosing. */
+ readonly note: string;
+ /**
+ * Recency windows this engine can actually express, mapped to the parameter
+ * value. A window that is absent is one the engine does not support — never
+ * one that is silently dropped.
+ */
+ readonly recency: Partial<Record<Recency, string>>;
+ readonly buildUrl: (query: string, numResults: number, recency: Recency | undefined) => string;
+ readonly extraction: ExtractionConfig;
+};
+
+const GOOGLE: EngineDefinition = {
+ id: "google",
+ label: "Google",
+ note: "best result quality; blocks aggressively under repeated automated queries",
+ recency: { day: "d", week: "w", month: "m", year: "y" },
+ buildUrl: (query, numResults, recency) => {
+ const params = new URLSearchParams({ q: query, num: String(numResults), hl: "en" });
+ // `udm=14` is the plain "Web" tab: no AI overview, no carousels. But any
+ // date restriction combined with it renders an empty page, and `tbs=qdr:*`
+ // renders an empty page on its own — both observed. So a time-limited
+ // search drops udm and uses the older as_qdr instead.
+ if (recency) params.set("as_qdr", GOOGLE.recency[recency] as string);
+ else params.set("udm", "14");
+ return `https://www.google.com/search?${params.toString()}`;
+ },
+ extraction: {
+ mode: "headings",
+ roots: ["#rso", "#search"],
+ unwrap: "none",
+ selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/`,
+ },
+};
+
+const DUCKDUCKGO: EngineDefinition = {
+ id: "duckduckgo",
+ label: "DuckDuckGo",
+ note: "no-JS endpoint, the most stable markup here and the least likely to challenge",
+ // Read off DuckDuckGo's own <select name="df">: "", d, w, m, y.
+ recency: { day: "d", week: "w", month: "m", year: "y" },
+ buildUrl: (query, _numResults, recency) => {
+ // The html endpoint renders server-side with no JavaScript, which makes it
+ // both faster and far less fragile than the app at duckduckgo.com. It has
+ // no result-count parameter — it returns a full page and the caller slices.
+ const params = new URLSearchParams({ q: query });
+ if (recency) params.set("df", DUCKDUCKGO.recency[recency] as string);
+ return `https://html.duckduckgo.com/html/?${params.toString()}`;
+ },
+ extraction: {
+ mode: "items",
+ roots: [".results", "#links"],
+ item: ".result",
+ link: "a.result__a[href]",
+ title: "a.result__a",
+ snippet: ".result__snippet",
+ exclude: [".result--ad", ".badge--ad"],
+ unwrap: "ddg",
+ selfHostPattern: String.raw`^https?://(html\.|www\.)?duckduckgo\.com/`,
+ },
+};
+
+const BING: EngineDefinition = {
+ id: "bing",
+ label: "Bing",
+ note: "good coverage; supports day/week/month only — it has no year filter",
+ // ez1/ez2/ez3 verified to filter (results carried "4 hours ago", "1 day ago").
+ // There is deliberately no year: Bing's UI offers no such window, and the
+ // ez5 custom-range form returned nothing when tried.
+ recency: { day: "ez1", week: "ez2", month: "ez3" },
+ buildUrl: (query, numResults, recency) => {
+ const params = new URLSearchParams({ q: query });
+ if (recency) {
+ // `count` silently cancels `filters` — with ez1 alone every result is
+ // hours old, and adding count in either order brings back months-old
+ // ones. Observed directly, and it fails silently: the results look
+ // perfectly plausible, just unfiltered. Exactly the same trap as Google's
+ // udm+as_qdr, so the same answer — drop the count parameter and let the
+ // caller slice the list it gets.
+ params.set("filters", `ex1:"${BING.recency[recency] as string}"`);
+ } else {
+ params.set("count", String(numResults));
+ }
+ return `https://www.bing.com/search?${params.toString()}`;
+ },
+ extraction: {
+ mode: "items",
+ // Organic results only. Bing's answer cards also contain <h2>s, which is
+ // why this targets li.b_algo rather than walking headings.
+ roots: ["#b_results"],
+ item: "li.b_algo",
+ link: "h2 a[href]",
+ title: "h2",
+ snippet: ".b_caption p, .b_algoSlug, p",
+ exclude: [".b_ad", ".b_adBottom"],
+ unwrap: "bing",
+ selfHostPattern: String.raw`^https?://(www\.)?bing\.com/`,
+ },
+};
+
+const BRAVE: EngineDefinition = {
+ id: "brave",
+ label: "Brave Search",
+ note: "independent index, direct links; challenges after 1-2 queries, clears in ~15-20 min",
+ // Left empty deliberately — see the note in README. Brave's filter UI is
+ // client-rendered and the `tf=` values could not be confirmed against a page
+ // that was not simultaneously serving a bot check, so no window is claimed
+ // rather than one being guessed at.
+ recency: {},
+ buildUrl: (query, _numResults, _recency) => {
+ const params = new URLSearchParams({ q: query });
+ return `https://search.brave.com/search?${params.toString()}`;
+ },
+ extraction: {
+ mode: "items",
+ roots: ["#results"],
+ // data-type="web" excludes the AI summariser, video and news cards, which
+ // share the .snippet class but are not ranked web results.
+ item: '.snippet[data-type="web"]',
+ link: "a[href]",
+ title: ".title",
+ // No dedicated description element. The item's text is
+ // "source | breadcrumb | title | description", so the extractor keeps only
+ // the lines after the title. These selectors are the fallback for when the
+ // title is not on a line of its own.
+ subtract: [".title", "cite"],
+ unwrap: "none",
+ selfHostPattern: String.raw`^https?://(search\.)?brave\.com/`,
+ },
+};
+
+const BY_ID: Record<EngineId, EngineDefinition> = {
+ google: GOOGLE,
+ duckduckgo: DUCKDUCKGO,
+ bing: BING,
+ brave: BRAVE,
+};
+
+export const getEngine = (id: EngineId): EngineDefinition => BY_ID[id];
+
+export const allEngines = (): readonly EngineDefinition[] => ENGINE_IDS.map((id) => BY_ID[id]);
+
+/** Parse a user- or model-supplied engine name. Null when unrecognised. */
+export const parseEngineId = (raw: string): EngineId | null =>
+ ENGINE_ALIASES[raw.trim().toLowerCase()] ?? null;
+
+/** The names accepted for an engine, for help text and error messages. */
+export const engineAliases = (): readonly string[] => Object.keys(ENGINE_ALIASES);
+
+/** Which engines can express a given recency window. */
+export const enginesSupportingRecency = (recency: Recency): readonly EngineId[] =>
+ ENGINE_IDS.filter((id) => BY_ID[id].recency[recency] !== undefined);
+
+/** Human-readable list of the windows an engine supports, or "none". */
+export const describeRecencySupport = (engine: EngineDefinition): string => {
+ const windows = RECENCY_VALUES.filter((r) => engine.recency[r] !== undefined);
+ return windows.length > 0 ? windows.join(", ") : "none";
+};
diff --git a/castle-search/src/errors.ts b/castle-search/src/errors.ts
new file mode 100644
index 0000000..2318b27
--- /dev/null
+++ b/castle-search/src/errors.ts
@@ -0,0 +1,83 @@
+/**
+ * Failure vocabulary.
+ *
+ * Every one of these is *thrown*, never returned. pi only sets `isError: true`
+ * on a tool result when `execute()` throws — a returned object with an `error`
+ * field looks to the model exactly like a successful search that found nothing.
+ *
+ * `SearchChallengeError` is the one the agent is expected to act on rather than
+ * just report: it means a human has to touch the browser before search can work
+ * again, so its message says so in words the agent can pass to the operator.
+ */
+
+/** The search engine served a captcha, consent wall, or other human check. */
+export class SearchChallengeError extends Error {
+ override readonly name = "SearchChallengeError";
+ /** Machine-readable flavour of the challenge, for callers that branch on it. */
+ readonly challenge: string;
+ /** The page the human needs to visit to clear it. */
+ readonly pageUrl: string;
+ /** Which browser is blocked, e.g. "10.88.0.25:9223 (built-in default)". */
+ readonly endpoint: string;
+ /** Which engine is blocked. */
+ readonly engine: string;
+ /** Engines that are not blocked and could be tried instead. */
+ readonly alternatives: readonly string[];
+
+ constructor(options: {
+ challenge: string;
+ pageUrl: string;
+ endpoint: string;
+ engine: string;
+ alternatives: readonly string[];
+ detail?: string | undefined;
+ }) {
+ super(
+ [
+ `${options.engine} is blocked by a human verification challenge (${options.challenge}).`,
+ options.detail ? `Page said: ${options.detail}` : null,
+ // Retrying elsewhere comes first because it is the action the agent can
+ // take on its own; a challenge on one engine says nothing about the
+ // others, and asking the operator for help should be the fallback, not
+ // the first move.
+ options.alternatives.length > 0
+ ? `A challenge on one engine does not affect the others: retry the same query with engine set to ` +
+ `${options.alternatives.join(", ")} before asking anyone for help.`
+ : null,
+ `If every engine is blocked, this needs a person — the challenge cannot be solved by the agent.`,
+ `Ask the user to open ${options.pageUrl} in the Chrome at ${options.endpoint}, complete the challenge or`,
+ `accept the consent dialog, then retry. The cookie it sets persists in that browser profile, so one pass`,
+ `unblocks later searches.`,
+ ]
+ .filter((line) => line !== null)
+ .join(" "),
+ );
+ this.challenge = options.challenge;
+ this.pageUrl = options.pageUrl;
+ this.endpoint = options.endpoint;
+ this.engine = options.engine;
+ this.alternatives = options.alternatives;
+ }
+}
+
+/** The chosen engine cannot express the requested recency window. */
+export class RecencyUnsupportedError extends Error {
+ override readonly name = "RecencyUnsupportedError";
+}
+
+/** The browser could not be reached or spoke CDP badly. */
+export class BrowserUnavailableError extends Error {
+ override readonly name = "BrowserUnavailableError";
+}
+
+/** The search itself ran but produced nothing usable. */
+export class NoResultsError extends Error {
+ override readonly name = "NoResultsError";
+}
+
+/** The per-search deadline expired, or the user pressed Esc. */
+export class SearchAbortedError extends Error {
+ override readonly name = "SearchAbortedError";
+}
+
+export const messageOf = (e: unknown): string => (e instanceof Error ? e.message : String(e));
diff --git a/castle-search/src/extract.ts b/castle-search/src/extract.ts
new file mode 100644
index 0000000..8b5c57f
--- /dev/null
+++ b/castle-search/src/extract.ts
@@ -0,0 +1,294 @@
+/**
+ * The script that runs inside the search results page.
+ *
+ * It is built as a string rather than shipped as a function because it goes to
+ * Chrome via `Runtime.evaluate` — nothing in here is typechecked, so it is kept
+ * defensive and free of anything that could throw on an unexpected DOM. The
+ * per-engine configuration is injected as JSON by {@link buildProbeScript}.
+ *
+ * ## Two modes, because the engines genuinely differ
+ *
+ * **"items"** — DuckDuckGo, Bing and Brave each have a clean per-result
+ * container, so results are read directly out of it.
+ *
+ * **"headings"** — Google has no stable result container; its class names
+ * (`MjjYud`, `kb0PBd`, `yuRUbf`) are generated and change without notice. So
+ * results are found from each `<h3>` outward:
+ *
+ * - anchor = nearest enclosing `<a href>`
+ * - header = nearest `[data-snhf]`, else the anchor
+ * - container = climb from the header until the text grows past the header's,
+ * abandoning the climb if a second `<h3>` comes into scope
+ * (that would mean the next result had been swallowed)
+ * - snippet = `[data-sncf]` if present, else the container's lines minus the
+ * header's lines, minus URL and breadcrumb noise
+ *
+ * Verified against Google's classic SERP and its `udm=14` layout, DuckDuckGo's
+ * no-JS endpoint, Bing's `li.b_algo`, and Brave's `.snippet[data-type=web]`.
+ *
+ * ## Link unwrapping
+ *
+ * DuckDuckGo and Bing both route outbound links through a redirector, so the
+ * raw href is useless to the agent. Both are unwrapped in the page, where the
+ * URL and base64 primitives already exist:
+ *
+ * - DuckDuckGo: `//duckduckgo.com/l/?uddg=<percent-encoded target>`
+ * - Bing: `//bing.com/ck/a?…&u=a1<base64url of target>`
+ *
+ * Google and Brave link straight out and need no unwrapping.
+ */
+
+import type { ExtractionConfig } from "./engines.ts";
+
+/** One search hit, as the page script reports it. */
+export type SearchResult = {
+ title: string;
+ url: string;
+ snippet: string;
+};
+
+/** What the page script returns in a single round trip. */
+export type PageProbe = {
+ /** Where the page actually ended up — redirects and consent walls move it. */
+ url: string;
+ title: string;
+ readyState: string;
+ /** Non-null when a human check is in the way. */
+ challenge: { kind: string; detail: string } | null;
+ /**
+ * Whether the engine's results container was present at all. "none" means a
+ * page shell with no results area — a different failure from an empty one,
+ * and worth telling apart when diagnosing a zero-hit search.
+ */
+ container: "found" | "none";
+ results: SearchResult[];
+};
+
+/**
+ * Challenge detection, shared across engines.
+ *
+ * Every phrase here has been seen on a real page during development: Google's
+ * `/sorry/` interstitial, and Brave's "Verifying you're not a bot / Quick check
+ * before you continue searching" — the latter is why the wording list is broad
+ * rather than just matching Google's "unusual traffic". Ordering matters: a
+ * `/sorry/` page also contains a recaptcha iframe, and naming the page beats
+ * naming the widget sitting on it.
+ */
+const CHALLENGE_DETECTION = String.raw`
+ const detectChallenge = () => {
+ if (/\/sorry\//.test(location.href)) return { kind: "google-block-page", detail: bodyText.slice(0, 300) };
+ if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname))
+ return { kind: "consent-wall", detail: bodyText.slice(0, 300) };
+ if (document.querySelector("#captcha-form, form#captcha-form"))
+ return { kind: "captcha-form", detail: bodyText.slice(0, 300) };
+ if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running, [id^=cf-chl]"))
+ return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) };
+ if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[src*="turnstile"], iframe[title*="challenge"]'))
+ return { kind: "captcha-widget", detail: bodyText.slice(0, 300) };
+ if (/verifying (that )?you('| a)?re( not)? a? ?(human|bot|robot)|quick check before you continue|verify you are human|are you a robot|not a robot/i.test(bodyText))
+ return { kind: "bot-check", detail: bodyText.slice(0, 300) };
+ if (/unusual traffic|automated queries|suspicious activity from your/i.test(bodyText))
+ return { kind: "rate-limit-block", detail: bodyText.slice(0, 300) };
+ return null;
+ };
+`;
+
+/**
+ * Build the probe for one engine. The configuration is embedded as a JSON
+ * literal so the script stays a single self-contained expression.
+ */
+export const buildProbeScript = (config: ExtractionConfig): string => String.raw`(() => {
+ var CFG = ${JSON.stringify(config)};
+
+ var norm = function (s) { return (s || "").replace(/\s+/g, " ").trim(); };
+ var linesOf = function (el) {
+ return el && el.innerText ? el.innerText.split("\n").map(function (l) { return l.trim(); }).filter(Boolean) : [];
+ };
+ var bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : "";
+ var selfHost = new RegExp(CFG.selfHostPattern);
+
+ ${CHALLENGE_DETECTION}
+
+ // DuckDuckGo and Bing both hide the real destination behind a redirector.
+ var unwrap = function (href) {
+ try {
+ var u = new URL(href, location.href);
+ if (CFG.unwrap === "ddg") {
+ if (!/(^|\.)duckduckgo\.com$/.test(u.hostname)) return href;
+ return u.searchParams.get("uddg") || href;
+ }
+ if (CFG.unwrap === "bing") {
+ if (!/(^|\.)bing\.com$/.test(u.hostname)) return href;
+ var p = u.searchParams.get("u");
+ if (!p) return href;
+ var b64 = p.replace(/^a1/, "").replace(/-/g, "+").replace(/_/g, "/");
+ b64 += "=".repeat((4 - (b64.length % 4)) % 4);
+ var bin = atob(b64);
+ var bytes = new Uint8Array(bin.length);
+ for (var i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+ return new TextDecoder().decode(bytes);
+ }
+ return href;
+ } catch (e) {
+ // An unwrap that fails leaves the redirector URL in place rather than
+ // dropping the result: a working link the agent has to follow twice beats
+ // no link at all.
+ return href;
+ }
+ };
+
+ var findRoot = function () {
+ for (var i = 0; i < CFG.roots.length; i++) {
+ var el = document.querySelector(CFG.roots[i]);
+ if (el) return el;
+ }
+ return null;
+ };
+
+ var acceptable = function (url) {
+ return /^https?:/.test(url) && !selfHost.test(url);
+ };
+
+ // ── items mode ───────────────────────────────────────────────────────────
+ var extractItems = function (root) {
+ var out = [];
+ var seen = {};
+ var items = root.querySelectorAll(CFG.item);
+ for (var i = 0; i < items.length; i++) {
+ var item = items[i];
+
+ var skip = false;
+ for (var x = 0; CFG.exclude && x < CFG.exclude.length; x++) {
+ if (item.matches(CFG.exclude[x]) || item.querySelector(CFG.exclude[x])) { skip = true; break; }
+ }
+ if (skip) continue;
+
+ var anchor = item.querySelector(CFG.link);
+ if (!anchor) continue;
+ var url = unwrap(anchor.href);
+ if (!acceptable(url) || seen[url]) continue;
+
+ var titleEl = CFG.title ? item.querySelector(CFG.title) : null;
+ var title = norm(titleEl ? titleEl.innerText : anchor.innerText);
+ if (!title) continue;
+
+ var snippet = "";
+ var snippetEl = CFG.snippet ? item.querySelector(CFG.snippet) : null;
+ if (snippetEl) {
+ snippet = norm(snippetEl.innerText);
+ } else {
+ // No dedicated description element. Rather than guess at class names
+ // for the source and breadcrumb, use their position: an engine that
+ // renders "source / breadcrumb / title / description" puts every piece
+ // of metadata *before* the title, so everything after the title line is
+ // the description. Class names churn; that ordering does not.
+ var itemLines = linesOf(item);
+ var titleIndex = -1;
+ for (var t = 0; t < itemLines.length; t++) {
+ if (norm(itemLines[t]) === title) { titleIndex = t; break; }
+ }
+
+ var candidate;
+ if (titleIndex >= 0) {
+ candidate = itemLines.slice(titleIndex + 1);
+ } else {
+ // Title is not its own line (it may be inline with other text).
+ // Fall back to subtracting the parts we can name.
+ var drop = {};
+ for (var s = 0; CFG.subtract && s < CFG.subtract.length; s++) {
+ var parts = item.querySelectorAll(CFG.subtract[s]);
+ for (var p = 0; p < parts.length; p++) {
+ var pl = linesOf(parts[p]);
+ for (var q = 0; q < pl.length; q++) drop[norm(pl[q])] = true;
+ }
+ }
+ candidate = itemLines.filter(function (l) { return !drop[norm(l)]; });
+ }
+
+ snippet = norm(
+ candidate.filter(function (l) {
+ var n = norm(l);
+ return n !== title && !/^https?:\/\//.test(n) && n.indexOf("›") === -1;
+ }).join(" "),
+ );
+ }
+
+ seen[url] = true;
+ out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
+ }
+ return out;
+ };
+
+ // ── headings mode (Google) ───────────────────────────────────────────────
+ var extractHeadings = function (root) {
+ var out = [];
+ var seen = {};
+ var headings = root.querySelectorAll("h3");
+ for (var i = 0; i < headings.length; i++) {
+ var h3 = headings[i];
+ var anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]"));
+ if (!anchor) continue;
+
+ var url = unwrap(anchor.href);
+ if (!acceptable(url) || seen[url]) continue;
+
+ var title = norm(h3.innerText);
+ if (!title) continue;
+
+ var header = h3.closest("[data-snhf]") || anchor;
+ var headerLines = {};
+ var hl = linesOf(header);
+ for (var k = 0; k < hl.length; k++) headerLines[norm(hl[k])] = true;
+ var baseline = norm(header.innerText || "").length;
+
+ var container = header.parentElement;
+ for (var depth = 0; depth < 6 && container && container !== root; depth++) {
+ if (container.querySelectorAll("h3").length > 1) { container = null; break; }
+ if (norm(container.innerText || "").length > baseline + 40) break;
+ container = container.parentElement;
+ }
+
+ var snippet = "";
+ if (container && container !== root) {
+ var explicit = container.querySelector("[data-sncf]");
+ var body = explicit
+ ? linesOf(explicit)
+ : linesOf(container).filter(function (l) { return !headerLines[norm(l)]; });
+ snippet = norm(
+ body.filter(function (l) {
+ return !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results";
+ }).join(" "),
+ ).replace(/Read more$/, "").trim();
+ }
+
+ seen[url] = true;
+ out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
+ }
+ return out;
+ };
+
+ var challenge = null;
+ var results = [];
+ var container = "none";
+ try {
+ challenge = detectChallenge();
+ } catch (e) {
+ challenge = null;
+ }
+ try {
+ var root = findRoot();
+ container = root ? "found" : "none";
+ if (root && !challenge) results = CFG.mode === "items" ? extractItems(root) : extractHeadings(root);
+ } catch (e) {
+ results = [];
+ }
+
+ return {
+ url: location.href,
+ title: document.title || "",
+ readyState: document.readyState,
+ challenge: challenge,
+ container: container,
+ results: results,
+ };
+})()`;
diff --git a/castle-search/src/format.ts b/castle-search/src/format.ts
new file mode 100644
index 0000000..aef6c3b
--- /dev/null
+++ b/castle-search/src/format.ts
@@ -0,0 +1,54 @@
+/**
+ * Turning results into the text the model reads.
+ *
+ * Twenty results with 600-character snippets is roughly 15KB, comfortably under
+ * pi's 50KB / 2000-line budget — but the truncation is applied anyway rather
+ * than argued about, because a pathological page could blow past it and an
+ * overflowing tool result costs the whole session, not just this call.
+ */
+
+import type { SearchResult } from "./extract.ts";
+
+export type FormatInput = {
+ query: string;
+ searchUrl: string;
+ finalUrl: string;
+ endpoint: string;
+ /** Engine display name, e.g. "DuckDuckGo". */
+ engine: string;
+ /** Where the engine choice came from, e.g. "/search-engine". */
+ engineSource: string;
+ results: SearchResult[];
+};
+
+export const formatResults = (input: FormatInput): string => {
+ const lines: string[] = [
+ // Naming the engine on every result set matters more than it looks: the
+ // agent may have switched engines mid-conversation to get around a block,
+ // and results from different indexes are not interchangeable evidence.
+ `Search results for "${input.query}" — ${input.results.length} hits from ${input.engine} ` +
+ `(${input.engineSource}), via Chrome at ${input.endpoint}`,
+ `Query URL: ${input.searchUrl}`,
+ ];
+ if (input.finalUrl !== input.searchUrl) lines.push(`Landed on: ${input.finalUrl}`);
+ lines.push("");
+
+ input.results.forEach((result, index) => {
+ lines.push(`${index + 1}. ${result.title}`);
+ lines.push(` ${result.url}`);
+ if (result.snippet) lines.push(` ${result.snippet}`);
+ lines.push("");
+ });
+
+ return lines.join("\n").trimEnd();
+};
+
+/**
+ * The note appended when pi's truncation utilities cut the output. Says what was
+ * lost and what to do about it — a bare "[truncated]" tells the model nothing it
+ * can act on, and there is no temp file to point at because the full text only
+ * ever existed in memory.
+ */
+export const truncationNote = (shown: number, total: number, bytesShown: string, bytesTotal: string): string =>
+ `\n\n[Output truncated: ${shown} of ${total} lines (${bytesShown} of ${bytesTotal}). ` +
+ `Re-run castle_cdp_search with a smaller numResults to see the rest.]`;
diff --git a/castle-search/src/index.ts b/castle-search/src/index.ts
new file mode 100644
index 0000000..29cdaaf
--- /dev/null
+++ b/castle-search/src/index.ts
@@ -0,0 +1,315 @@
+/**
+ * castle-cdp-search — web search for pi through a Chrome that is already
+ * running somewhere on the wire network, driven over CDP.
+ *
+ * It does not launch a browser. It attaches to one that a person or a launchd
+ * job started, using the same endpoint configuration as pi-browser-harness
+ * (`BU_CDP_HTTP` > `/browser-target` > castle), so a single choice governs both
+ * packages and they can never end up driving different browsers.
+ *
+ * The engine is switchable the same way: `CCS_SEARCH_ENGINE` > `/search-engine`
+ * > Google, plus a per-call `engine` parameter so the agent can fall back when
+ * one engine starts serving captchas.
+ *
+ * The tool is called `castle_cdp_search`, not `web_search`, on purpose: it is
+ * meant to sit alongside pi-web-access rather than shadow it. The model picks
+ * between them, so both stay available in one session.
+ */
+
+import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, truncateHead } from "@earendil-works/pi-coding-agent";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { StringEnum } from "@earendil-works/pi-ai";
+import { Type } from "typebox";
+
+import { clearStoredEngine, storedEngineLocation, writeStoredEngine } from "./engine-store.ts";
+import { DEFAULT_ENGINE, ENGINE_ENV_VAR } from "./engine-target.ts";
+import {
+ ENGINE_IDS,
+ RECENCY_VALUES,
+ type EngineId,
+ type Recency,
+ allEngines,
+ describeRecencySupport,
+ engineAliases,
+ parseEngineId,
+} from "./engines.ts";
+import { messageOf } from "./errors.ts";
+import type { SearchResult } from "./extract.ts";
+import { formatResults, truncationNote } from "./format.ts";
+import {
+ MAX_CONCURRENT_SEARCHES,
+ SEARCH_TIMEOUT_MS,
+ type SearchPhase,
+ describeConfiguredEndpoint,
+ describeConfiguredEngine,
+ disconnect,
+ search,
+} from "./search.ts";
+
+const DEFAULT_NUM_RESULTS = 10;
+const MIN_NUM_RESULTS = 1;
+const MAX_NUM_RESULTS = 20;
+
+const parameters = Type.Object({
+ query: Type.String({
+ description: "The search query, phrased as you would type it into a search engine.",
+ minLength: 1,
+ }),
+ numResults: Type.Optional(
+ Type.Integer({
+ description: `How many results to return (${MIN_NUM_RESULTS}-${MAX_NUM_RESULTS}, default ${DEFAULT_NUM_RESULTS}).`,
+ minimum: MIN_NUM_RESULTS,
+ maximum: MAX_NUM_RESULTS,
+ default: DEFAULT_NUM_RESULTS,
+ }),
+ ),
+ // StringEnum, not Type.Union of Type.Literal: Google's API rejects the
+ // anyOf/const shape typebox emits for unions.
+ recency: Type.Optional(
+ StringEnum(RECENCY_VALUES, {
+ description:
+ "Restrict results to pages published within this window. Omit for no time limit. " +
+ "Not every engine supports every window — bing has no year, brave has no time filter at all.",
+ }),
+ ),
+ engine: Type.Optional(
+ StringEnum(ENGINE_IDS, {
+ description:
+ "Which search engine to use for this call. Omit to use the configured default. " +
+ "Switch engines when one returns a challenge or no results — they have independent indexes and blocks.",
+ }),
+ ),
+});
+
+/** Exported so `isToolCallEventType<"castle_cdp_search", CastleCdpSearchInput>` can type it. */
+export type CastleCdpSearchInput = {
+ query: string;
+ numResults?: number;
+ recency?: Recency;
+ engine?: EngineId;
+};
+
+export type CastleCdpSearchDetails = {
+ query: string;
+ numResults: number;
+ recency: Recency | null;
+ engine: EngineId | null;
+ phase: SearchPhase | "done";
+ note: string;
+ endpoint: string | null;
+ searchUrl: string | null;
+ results: SearchResult[];
+ truncated: boolean;
+};
+
+const initialDetails = (
+ query: string,
+ numResults: number,
+ recency: Recency | null,
+ engine: EngineId | null,
+): CastleCdpSearchDetails => ({
+ query,
+ numResults,
+ recency,
+ engine,
+ phase: "queued",
+ note: "starting",
+ endpoint: null,
+ searchUrl: null,
+ results: [],
+ truncated: false,
+});
+
+/** The engine table, rendered for `/search-engine` and for the tool description. */
+const engineTable = (): string =>
+ allEngines()
+ .map((e) => ` ${e.id.padEnd(11)} ${e.label.padEnd(14)} recency: ${describeRecencySupport(e).padEnd(24)} ${e.note}`)
+ .join("\n");
+
+/**
+ * Say something from a command, in whichever mode pi is running.
+ *
+ * `ctx.ui.notify` is a no-op when `hasUI` is false, which means `pi -p
+ * "/search-engine"` would change the stored engine and print absolutely nothing
+ * — the command would look broken. Print mode falls back to stdout. JSON mode
+ * deliberately does not: stdout there is a structured event stream, and a stray
+ * line would corrupt it.
+ */
+const say = (ctx: { hasUI: boolean; mode: string; ui: { notify: (m: string, t?: "info" | "warning" | "error") => void } },
+ message: string,
+ level: "info" | "warning" | "error" = "info",
+): void => {
+ if (ctx.hasUI) {
+ ctx.ui.notify(message, level);
+ return;
+ }
+ if (ctx.mode === "print") console.log(message);
+};
+
+export default function (pi: ExtensionAPI): void {
+ // Nothing is dialled here. pi runs extension factories in invocations that
+ // never start a session, so opening a socket from a factory would leave one
+ // behind on, say, `pi --list-models`. The connection is made on first use and
+ // torn down below.
+
+ pi.on("session_shutdown", async () => {
+ disconnect();
+ });
+
+ // The engine analogue of pi-browser-harness's /browser-target. Persisted
+ // machine-wide for the same reason: the choice should outlive the session
+ // that made it.
+ pi.registerCommand("search-engine", {
+ description: "Show or set the search engine castle_cdp_search uses",
+ handler: async (args, ctx) => {
+ const raw = args.trim();
+ const current = await describeConfiguredEngine();
+
+ if (raw === "") {
+ say(
+ ctx,
+ `castle_cdp_search engine: ${current.id} (${current.source})\n\n${engineTable()}\n\n` +
+ `Set with: /search-engine <name> Clear with: /search-engine default\n` +
+ `Stored in ${storedEngineLocation()}; ${ENGINE_ENV_VAR} overrides it for one process.`,
+ );
+ return;
+ }
+
+ if (raw.toLowerCase() === "default" || raw.toLowerCase() === "clear") {
+ await clearStoredEngine();
+ say(ctx, `castle_cdp_search engine reset to the built-in default (${DEFAULT_ENGINE}).`);
+ return;
+ }
+
+ const id = parseEngineId(raw);
+ if (!id) {
+ say(ctx, `Unknown search engine "${raw}". Expected one of: ${engineAliases().join(", ")}`, "error");
+ return;
+ }
+
+ await writeStoredEngine(id);
+ // Saying so explicitly, because the variable silently wins otherwise and
+ // the user would reasonably assume the command had taken effect.
+ const pinned = process.env[ENGINE_ENV_VAR]?.trim();
+ say(
+ ctx,
+ pinned
+ ? `Saved ${id}, but ${ENGINE_ENV_VAR}=${pinned} is set and overrides it for this process.`
+ : `castle_cdp_search will use ${id}.`,
+ pinned ? "warning" : "info",
+ );
+ },
+ });
+
+ pi.registerTool({
+ name: "castle_cdp_search",
+ label: "Castle Search",
+ description:
+ "Search the web using a real Chrome browser running on the private network, driven over the Chrome " +
+ "DevTools Protocol. Returns ranked results with titles, URLs and snippets. Because it uses a real " +
+ "logged-in browser rather than a search API, it reaches pages that block datacentre traffic. It " +
+ "returns search results only — it does not fetch or read the linked pages, so follow up with a " +
+ "fetch/read tool for full page content. Supports four engines with independent indexes and " +
+ `independent rate limits (${ENGINE_IDS.join(", ")}); pass engine to switch when one is blocked. ` +
+ `Output is truncated at ${formatSize(DEFAULT_MAX_BYTES)} or ${DEFAULT_MAX_LINES} lines.`,
+ promptSnippet:
+ "Search the web through a real Chrome on the private network (castle_cdp_search); google/duckduckgo/bing/brave, returns titles, URLs and snippets",
+ promptGuidelines: [
+ "Use castle_cdp_search to find pages on the open web when you need current information, documentation, " +
+ "or sources you do not already have — it drives a real signed-in Chrome, so it works on sites that " +
+ "reject scripted clients.",
+ "castle_cdp_search returns search results only. To read a result, follow it up with a tool that fetches " +
+ "page content; do not treat the snippet as the full page.",
+ "If castle_cdp_search fails with a SearchChallengeError or returns no results, retry the same query with " +
+ "its engine parameter set to a different engine (google, duckduckgo, bing, brave) before giving up — " +
+ "they have independent indexes and independent blocks, and a challenge on one says nothing about the others.",
+ "Only ask the user for help with a castle_cdp_search captcha once more than one engine has failed; " +
+ "when you do, give them the URL from the error so they can clear it in that browser.",
+ "Pass recency to castle_cdp_search when the answer depends on how recent a page is, and keep numResults " +
+ "small (5-10) unless you genuinely need a wide sweep. Not every engine supports every window: bing has " +
+ "no year, and brave has no time filter, so castle_cdp_search will tell you to switch engines rather than " +
+ "silently ignoring the request.",
+ ],
+ parameters,
+
+ async execute(_toolCallId, params, signal, onUpdate, _ctx) {
+ const query = params.query.trim();
+ if (query === "") throw new Error("query must not be empty");
+
+ const numResults = Math.min(
+ MAX_NUM_RESULTS,
+ Math.max(MIN_NUM_RESULTS, params.numResults ?? DEFAULT_NUM_RESULTS),
+ );
+ const recency = (params.recency ?? null) as Recency | null;
+ const engine = (params.engine ?? null) as EngineId | null;
+
+ const details = initialDetails(query, numResults, recency, engine);
+
+ // One deadline covering connect + navigate + extract, plus the user's own
+ // abort so Esc drops an in-flight navigation rather than waiting it out.
+ const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+ const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
+
+ const report = (phase: SearchPhase, note: string): void => {
+ details.phase = phase;
+ details.note = note;
+ // Keep the TUI moving during the seconds Chrome spends navigating.
+ onUpdate?.({ content: [{ type: "text", text: `${note}…` }], details: { ...details } });
+ };
+
+ try {
+ const outcome = await search(
+ { query, numResults, recency: recency ?? undefined, engine },
+ combined,
+ report,
+ );
+
+ const body = formatResults({
+ query,
+ searchUrl: outcome.searchUrl,
+ finalUrl: outcome.finalUrl,
+ endpoint: outcome.endpoint,
+ engine: outcome.engine.label,
+ engineSource: outcome.engineSource,
+ results: outcome.results,
+ });
+
+ const truncation = truncateHead(body, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES });
+ const text = truncation.truncated
+ ? truncation.content +
+ truncationNote(
+ truncation.outputLines,
+ truncation.totalLines,
+ formatSize(truncation.outputBytes),
+ formatSize(truncation.totalBytes),
+ )
+ : truncation.content;
+
+ details.phase = "done";
+ details.note = `${outcome.results.length} results from ${outcome.engine.id}`;
+ details.engine = outcome.engine.id;
+ details.endpoint = outcome.endpoint;
+ details.searchUrl = outcome.searchUrl;
+ details.results = outcome.results;
+ details.truncated = truncation.truncated;
+
+ return { content: [{ type: "text", text }], details };
+ } catch (e) {
+ // Rethrow, always. pi only marks a tool result as an error when execute()
+ // throws — returning `{ error }` would read to the model as a search that
+ // simply found nothing, which is the one wrong conclusion to invite here.
+ const endpoint = await describeConfiguredEndpoint().catch(() => "unknown endpoint");
+ const configured = await describeConfiguredEngine().catch(() => null);
+ const e2 = e instanceof Error ? e : new Error(messageOf(e));
+ // Naming the phase turns "it timed out" into something actionable:
+ // "queued" means the cap was the bottleneck, "navigating" means the
+ // browser or the network was.
+ e2.message =
+ `${e2.message} [castle_cdp_search: failed during "${details.phase}" — engine ` +
+ `${engine ?? configured?.id ?? "unknown"}, browser ${endpoint}, up to ${MAX_CONCURRENT_SEARCHES} ` +
+ `concurrent searches, ${SEARCH_TIMEOUT_MS / 1000}s deadline covering the whole call]`;
+ throw e2;
+ }
+ },
+ });
+}
diff --git a/castle-search/src/paths.ts b/castle-search/src/paths.ts
new file mode 100644
index 0000000..b008f4f
--- /dev/null
+++ b/castle-search/src/paths.ts
@@ -0,0 +1,41 @@
+/**
+ * pi's agent config directory, and the file pi-browser-harness stores its
+ * `/browser-target` choice in.
+ *
+ * This extension reads that file so a single `/browser-target` decision governs
+ * both packages. It never writes it — the harness owns that setting, and two
+ * writers of one file is a bug waiting to happen.
+ *
+ * pi exports `getAgentDir()`, but resolving it here would be a *runtime* import
+ * of the host package from an installed extension's own tree. Every other pi
+ * import in this codebase is `import type` and erased at build time, so the
+ * three lines are replicated instead (pi's dist/config.js: `PI_CODING_AGENT_DIR`,
+ * CONFIG_DIR_NAME `.pi`). Same reasoning as pi-browser-harness/src/profile/paths.ts.
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+/** pi's env override for the agent dir (APP_NAME.toUpperCase() + "_CODING_AGENT_DIR"). */
+const ENV_AGENT_DIR = "PI_CODING_AGENT_DIR";
+
+/**
+ * Expand a leading `~` exactly as pi's expandTildePath does — `~` and `~/…`
+ * only, never `~\…`, on every platform. Being more lenient here would look in a
+ * different directory than pi resolves from the same variable.
+ */
+const expandTildePiCompatible = (path: string): string => {
+ if (path === "~") return homedir();
+ if (path.startsWith("~/")) return homedir() + path.slice(1);
+ return path;
+};
+
+/** pi's agent config directory — `$PI_CODING_AGENT_DIR` or `~/.pi/agent`. */
+export const agentDir = (): string => {
+ const fromEnv = process.env[ENV_AGENT_DIR];
+ if (fromEnv) return expandTildePiCompatible(fromEnv);
+ return join(homedir(), ".pi", "agent");
+};
+
+/** Where pi-browser-harness persists the `/browser-target` choice. */
+export const targetFilePath = (): string => join(agentDir(), "browser-target.json");
diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts
new file mode 100644
index 0000000..1306443
--- /dev/null
+++ b/castle-search/src/search.ts
@@ -0,0 +1,339 @@
+/**
+ * Search orchestration: resolve a browser, borrow a page, navigate, extract,
+ * hand the page back.
+ *
+ * The connection is process-wide and lazy. It is *not* opened by the extension
+ * factory — pi runs factories in invocations that never start a session, and
+ * the docs are explicit that sockets must not start there. It is opened on the
+ * first search and closed by an idempotent `session_shutdown`.
+ */
+
+import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts";
+import {
+ type CdpTarget,
+ describeCdpTarget,
+ describeResolutionFailure,
+ resolveCdpTarget,
+} from "./endpoint.ts";
+import { describeEngineFailure, describeEngineSource, resolveEngine } from "./engine-target.ts";
+import { readStoredEngine } from "./engine-store.ts";
+import {
+ ENGINE_IDS as allEngineIdList,
+ type EngineDefinition,
+ type EngineId,
+ type Recency,
+ describeRecencySupport,
+ enginesSupportingRecency,
+ getEngine,
+} from "./engines.ts";
+
+const allEngineIds = (): readonly EngineId[] => allEngineIdList;
+import {
+ BrowserUnavailableError,
+ NoResultsError,
+ RecencyUnsupportedError,
+ SearchChallengeError,
+} from "./errors.ts";
+import { buildProbeScript, type PageProbe, type SearchResult } from "./extract.ts";
+import { readStoredTarget } from "./target-store.ts";
+
+/** Per-search wall clock, covering connect, navigate and extract together. */
+export const SEARCH_TIMEOUT_MS = 20_000;
+/** How many searches may drive the shared browser at once. */
+export const MAX_CONCURRENT_SEARCHES = 2;
+/** Results sometimes land a beat after the load event; re-read a couple of times. */
+const EXTRACT_RETRIES = 2;
+const EXTRACT_RETRY_DELAY_MS = 700;
+
+export type { Recency } from "./engines.ts";
+
+export type SearchRequest = {
+ query: string;
+ numResults: number;
+ recency?: Recency | undefined;
+ /** Per-call engine override; falls back to the configured default. */
+ engine?: string | null | undefined;
+};
+
+export type SearchOutcome = {
+ results: SearchResult[];
+ searchUrl: string;
+ /** The URL the page settled on, which differs from searchUrl after a redirect. */
+ finalUrl: string;
+ endpoint: string;
+ engine: EngineDefinition;
+ /** Where the engine choice came from, for the "via" line in the output. */
+ engineSource: string;
+};
+
+/** Build the SERP URL for a request. Exported for tests. */
+export const buildSearchUrl = (engine: EngineDefinition, request: SearchRequest): string =>
+ engine.buildUrl(request.query, request.numResults, request.recency);
+
+/**
+ * Resolve the engine, refusing rather than silently ignoring a recency window
+ * the chosen engine cannot express.
+ *
+ * Dropping an unsupported filter would be the same failure mode as Google's
+ * `tbs=qdr:` — the caller asks for recent results, gets whatever the engine
+ * ranked, and has no way to tell. Naming the engines that *do* support the
+ * window makes the error actionable, since the model can retry with one.
+ */
+export const selectEngine = async (
+ request: SearchRequest,
+): Promise<{ engine: EngineDefinition; source: string }> => {
+ const resolution = resolveEngine(request.engine, process.env, await readStoredEngine());
+ if (resolution.kind === "invalid") {
+ throw new BrowserUnavailableError(describeEngineFailure(resolution));
+ }
+ const engine = getEngine(resolution.id);
+
+ if (request.recency && engine.recency[request.recency] === undefined) {
+ const alternatives = enginesSupportingRecency(request.recency);
+ throw new RecencyUnsupportedError(
+ `${engine.label} cannot restrict results to the past ${request.recency}. ` +
+ `It supports: ${describeRecencySupport(engine)}. ` +
+ (alternatives.length > 0
+ ? `Engines that support "${request.recency}": ${alternatives.join(", ")}. ` +
+ `Retry with engine set to one of those, or drop the recency parameter.`
+ : `No configured engine supports that window; drop the recency parameter.`),
+ );
+ }
+
+ return { engine, source: describeEngineSource(resolution.source) };
+};
+
+/** For status reporting — resolves the configured engine without searching. */
+export const describeConfiguredEngine = async (): Promise<{ id: EngineId; source: string; label: string }> => {
+ const resolution = resolveEngine(null, process.env, await readStoredEngine());
+ if (resolution.kind === "invalid") {
+ return { id: "google", source: describeEngineFailure(resolution), label: "unresolved" };
+ }
+ return {
+ id: resolution.id,
+ source: describeEngineSource(resolution.source),
+ label: getEngine(resolution.id).label,
+ };
+};
+
+// ── connection lifecycle ───────────────────────────────────────────────────
+
+let connection: BrowserConnection | null = null;
+/** In-flight connect, so two concurrent searches share one dial rather than racing. */
+let connecting: Promise<BrowserConnection> | null = null;
+
+const resolveTarget = async (): Promise<CdpTarget> => {
+ const resolution = resolveCdpTarget(process.env, await readStoredTarget());
+ if (resolution.kind === "invalid") {
+ throw new BrowserUnavailableError(describeResolutionFailure(resolution));
+ }
+ return resolution.target;
+};
+
+/**
+ * Return a live connection, dialling one if there isn't one or the last one
+ * died. Reconnecting transparently matters here: castle recycles Chrome hourly,
+ * so a session that searched an hour ago is holding a dead socket.
+ */
+const ensureConnection = async (signal?: AbortSignal): Promise<BrowserConnection> => {
+ if (connection && connection.isOpen) return connection;
+ if (connection) {
+ connection.close();
+ connection = null;
+ }
+
+ if (!connecting) {
+ // Assigned synchronously, before the first await inside: two searches
+ // starting in the same tick must share one dial, not open two sockets and
+ // leak whichever loses the assignment race.
+ connecting = (async () => {
+ const target = await resolveTarget();
+ const conn = await BrowserConnection.connect(target);
+ connection = conn;
+ return conn;
+ })().finally(() => {
+ connecting = null;
+ });
+ }
+
+ // The caller's signal is raced here rather than passed into connect(). A
+ // shared dial must not be cancelled by whichever caller happened to start it
+ // — one search pressing Esc would otherwise fail the other search waiting on
+ // the same connection. connect() is bounded by its own timeouts regardless.
+ return raceAbort(connecting, signal);
+};
+
+/** Idempotent: safe to call from `session_shutdown` however many times it fires. */
+export const disconnect = (): void => {
+ connection?.close();
+ connection = null;
+};
+
+/** For status reporting — never dials. */
+export const describeConfiguredEndpoint = async (): Promise<string> => {
+ const resolution = resolveCdpTarget(process.env, await readStoredTarget());
+ return resolution.kind === "invalid"
+ ? describeResolutionFailure(resolution)
+ : describeCdpTarget(resolution.target);
+};
+
+// ── concurrency cap ────────────────────────────────────────────────────────
+
+let active = 0;
+const waiting: Array<() => void> = [];
+
+/** Give a held slot to the next waiter, or return it to the pool. */
+const handOff = (): void => {
+ const next = waiting.shift();
+ // Hand the slot straight over rather than decrementing and letting the waiter
+ // re-check, so a queued search cannot be overtaken by a newly arriving one.
+ if (next) next();
+ else active--;
+};
+
+const acquire = async (signal?: AbortSignal): Promise<() => void> => {
+ if (active < MAX_CONCURRENT_SEARCHES) {
+ active++;
+ } else {
+ let resolver!: () => void;
+ const queued = new Promise<void>((resolve) => {
+ resolver = resolve;
+ });
+ waiting.push(resolver);
+ try {
+ await raceAbort(queued, signal);
+ // The slot was handed over by handOff(), which already counted it.
+ } catch (e) {
+ // An abort while queued must not strand the slot. Either we are still in
+ // the queue — drop out of it — or the slot was handed to us in the same
+ // tick the abort landed, in which case give it back. Without this, two
+ // cancelled-while-queued searches would wedge the semaphore permanently.
+ const index = waiting.indexOf(resolver);
+ if (index !== -1) waiting.splice(index, 1);
+ else handOff();
+ throw e;
+ }
+ }
+ let released = false;
+ return () => {
+ if (released) return;
+ released = true;
+ handOff();
+ };
+};
+
+// ── the search itself ──────────────────────────────────────────────────────
+
+export type ProgressReporter = (phase: SearchPhase, note: string) => void;
+export type SearchPhase = "queued" | "connecting" | "opening" | "navigating" | "extracting";
+
+const sleep = (ms: number, signal?: AbortSignal): Promise<void> =>
+ raceAbort(new Promise<void>((resolve) => setTimeout(resolve, ms)), signal);
+
+/**
+ * The other engines, named in failure messages. A blocked engine is the case
+ * where the agent most needs to know it has somewhere else to go, and telling
+ * it inside the error beats hoping it remembers the guidelines.
+ */
+const otherEngines = (current: EngineId): readonly string[] =>
+ allEngineIds().filter((id) => id !== current);
+
+/**
+ * Run one search. Throws on every failure — see errors.ts for why a returned
+ * error object would be worse than useless here.
+ */
+export const search = async (
+ request: SearchRequest,
+ signal: AbortSignal,
+ report: ProgressReporter,
+): Promise<SearchOutcome> => {
+ // Reported before acquiring, so a search that dies waiting for a slot is
+ // distinguishable from one that died reaching the browser. The deadline is
+ // the caller's and covers the queue wait too — with a cap of two and searches
+ // that take a couple of seconds, waiting is the rare case, and a tool call
+ // that silently takes twice its stated deadline would be worse.
+ // Resolved before acquiring a slot: an unknown engine name or an unsupported
+ // recency window is a caller error, and should not wait behind other searches
+ // or touch the browser at all.
+ const { engine, source: engineSource } = await selectEngine(request);
+ const probeScript = buildProbeScript(engine.extraction);
+
+ report("queued", "waiting for a search slot");
+ const release = await acquire(signal);
+ try {
+ report("connecting", "resolving browser");
+ const conn = await ensureConnection(signal);
+ const endpoint = describeCdpTarget(conn.target);
+
+ report("opening", `opening a page on ${endpoint}`);
+ const page = await PageSession.open(conn, signal);
+ const searchUrl = buildSearchUrl(engine, request);
+
+ try {
+ report("navigating", `searching ${engine.label} for "${request.query}"`);
+ await page.navigate(searchUrl, signal);
+
+ report("extracting", "reading results");
+ let probe = await page.evaluate<PageProbe>(probeScript, signal);
+ for (let attempt = 0; attempt < EXTRACT_RETRIES; attempt++) {
+ if (probe?.challenge || (probe?.results?.length ?? 0) > 0) break;
+ await sleep(EXTRACT_RETRY_DELAY_MS, signal);
+ probe = await page.evaluate<PageProbe>(probeScript, signal);
+ }
+
+ if (!probe) {
+ throw new BrowserUnavailableError("the page returned nothing — the extraction script did not run");
+ }
+
+ if (probe.challenge) {
+ throw new SearchChallengeError({
+ challenge: probe.challenge.kind,
+ // The page the human must clear is where the browser *ended up*, which
+ // after a consent redirect is not the URL we asked for.
+ pageUrl: probe.url || searchUrl,
+ endpoint,
+ engine: engine.label,
+ alternatives: otherEngines(engine.id),
+ detail: probe.challenge.detail || undefined,
+ });
+ }
+
+ if (probe.results.length === 0) {
+ // "container missing entirely" and "container present but empty" are
+ // different faults, and saying which saves the next person a browser
+ // session.
+ const diagnosis =
+ probe.container === "found"
+ ? `${engine.label}'s results container was rendered but nothing could be parsed out of it, which ` +
+ `suggests its markup changed and castle_cdp_search's extractor for ${engine.id} needs updating.`
+ : `${engine.label} served a page with no results area at all. This is often a soft rate-limit: as a ` +
+ `block decays an engine stops serving its challenge page and returns an empty result page instead, ` +
+ `which is indistinguishable from a genuine zero-hit search (observed on Google lasting ~90 minutes ` +
+ `after heavy querying). It can also mean the query truly has no hits.`;
+ throw new NoResultsError(
+ `No results could be read from ${probe.url || searchUrl} (page title: "${probe.title}"). ${diagnosis} ` +
+ `Before concluding the extractor is broken, retry with a different engine (${otherEngines(engine.id).join(", ")}); ` +
+ `if those also come back empty the query is the problem, and if only this one does, it is rate limiting — ` +
+ `wait rather than retrying in a loop.`,
+ );
+ }
+
+ return {
+ results: probe.results.slice(0, request.numResults),
+ searchUrl,
+ finalUrl: probe.url || searchUrl,
+ endpoint,
+ engine,
+ engineSource,
+ };
+ } finally {
+ // Always, including the challenge path. Leaving the tab open would let the
+ // operator solve the captcha in place, but it would also litter a browser
+ // someone else is using with abandoned tabs on every failure; the error
+ // carries the URL instead, and the cookie it sets is profile-wide.
+ await page.close();
+ }
+ } finally {
+ release();
+ }
+};
diff --git a/castle-search/src/target-store.ts b/castle-search/src/target-store.ts
new file mode 100644
index 0000000..6ff0db4
--- /dev/null
+++ b/castle-search/src/target-store.ts
@@ -0,0 +1,42 @@
+/**
+ * Read-only view of the target pi-browser-harness persists for `/browser-target`.
+ *
+ * The stored value is the raw string a user would have put in BU_CDP_HTTP
+ * ("local", "10.88.0.25:9223", …) — kept in that vocabulary deliberately, so
+ * there is one resolution path in endpoint.ts and a stored value can never mean
+ * something the variable could not.
+ *
+ * Every read failure degrades to "nothing stored" rather than throwing: a
+ * corrupt or absent file must fall back to the built-in default, not break search.
+ */
+
+import { readFile } from "node:fs/promises";
+import { targetFilePath } from "./paths.ts";
+
+const CURRENT_VERSION = 1;
+
+/**
+ * The persisted target, or null when nothing is stored, the file is missing, or
+ * it cannot be parsed. A file written by a newer version is treated as "nothing
+ * stored" rather than guessed at.
+ */
+export const readStoredTarget = async (): Promise<string | null> => {
+ let raw: string;
+ try {
+ raw = await readFile(targetFilePath(), "utf8");
+ } catch {
+ return null;
+ }
+ try {
+ const parsed: unknown = JSON.parse(raw);
+ if (typeof parsed !== "object" || parsed === null) return null;
+ const file = parsed as Record<string, unknown>;
+ if (file["version"] !== CURRENT_VERSION) return null;
+ const target = file["target"];
+ if (typeof target !== "string") return null;
+ const trimmed = target.trim();
+ return trimmed.length > 0 ? trimmed : null;
+ } catch {
+ return null;
+ }
+};
diff --git a/castle-search/test/concurrency.test.ts b/castle-search/test/concurrency.test.ts
new file mode 100644
index 0000000..500fee0
--- /dev/null
+++ b/castle-search/test/concurrency.test.ts
@@ -0,0 +1,86 @@
+/**
+ * The semaphore, exercised through the only door it has: search().
+ *
+ * These run against a deliberately unreachable CDP endpoint. Every search fails
+ * fast at the connect step, which is exactly what is wanted — the questions here
+ * are about slot bookkeeping, not about Google. Hitting the real browser to test
+ * a counter would be slow, flaky, and rude to whoever is using it.
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { MAX_CONCURRENT_SEARCHES, disconnect, search } from "../src/search.ts";
+
+const UNREACHABLE = "127.0.0.1:1"; // Nothing listens on port 1.
+const quiet = () => {};
+
+const withUnreachableBrowser = async (body: () => Promise<void>): Promise<void> => {
+ const saved = process.env["BU_CDP_HTTP"];
+ process.env["BU_CDP_HTTP"] = UNREACHABLE;
+ disconnect();
+ try {
+ await body();
+ } finally {
+ if (saved === undefined) delete process.env["BU_CDP_HTTP"];
+ else process.env["BU_CDP_HTTP"] = saved;
+ disconnect();
+ }
+};
+
+const attempt = (query: string, signal: AbortSignal) =>
+ search({ query, numResults: 1 }, signal, quiet).then(
+ () => "fulfilled" as const,
+ (e: Error) => e.name,
+ );
+
+test("a failed search returns its slot, so the next one still runs", async () => {
+ await withUnreachableBrowser(async () => {
+ // Three times the cap, run sequentially. If a slot leaked on the failure
+ // path, the run past the cap would hang until the test timed out.
+ for (let i = 0; i < MAX_CONCURRENT_SEARCHES * 3; i++) {
+ const name = await attempt(`q${i}`, AbortSignal.timeout(10_000));
+ assert.equal(name, "BrowserUnavailableError", `attempt ${i}`);
+ }
+ });
+});
+
+test("aborting while queued does not wedge the semaphore", async () => {
+ await withUnreachableBrowser(async () => {
+ // Saturate the cap and queue two more, then cancel the queued ones. The
+ // bug this guards against: a cancelled waiter that is later handed a slot
+ // resolves into nothing, and the slot is never counted back.
+ const cancel = new AbortController();
+ const running = Array.from({ length: MAX_CONCURRENT_SEARCHES }, (_, i) =>
+ attempt(`running${i}`, AbortSignal.timeout(10_000)),
+ );
+ const queued = Array.from({ length: 2 }, (_, i) => attempt(`queued${i}`, cancel.signal));
+ cancel.abort();
+
+ await Promise.all(running);
+ for (const outcome of await Promise.all(queued)) {
+ assert.ok(
+ outcome === "SearchAbortedError" || outcome === "BrowserUnavailableError",
+ `unexpected outcome ${outcome}`,
+ );
+ }
+
+ // The real assertion: the pool still works afterwards. A wedged semaphore
+ // would leave this hanging rather than failing.
+ for (let i = 0; i < MAX_CONCURRENT_SEARCHES + 1; i++) {
+ assert.equal(await attempt(`after${i}`, AbortSignal.timeout(10_000)), "BrowserUnavailableError");
+ }
+ });
+});
+
+test("an unreachable endpoint is reported, never silently swapped for a local browser", async () => {
+ await withUnreachableBrowser(async () => {
+ const error = await search({ query: "q", numResults: 1 }, AbortSignal.timeout(10_000), quiet).then(
+ () => null,
+ (e: Error) => e,
+ );
+ assert.ok(error, "expected a throw");
+ assert.equal(error.name, "BrowserUnavailableError");
+ assert.match(error.message, new RegExp(UNREACHABLE.replace(".", "\\.")));
+ });
+});
diff --git a/castle-search/test/endpoint.test.ts b/castle-search/test/endpoint.test.ts
new file mode 100644
index 0000000..f5c8236
--- /dev/null
+++ b/castle-search/test/endpoint.test.ts
@@ -0,0 +1,64 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+ DEFAULT_REMOTE_CDP,
+ LOCAL_CDP_PORT,
+ describeCdpTarget,
+ resolveCdpTarget,
+} from "../src/endpoint.ts";
+
+const ok = (resolution: ReturnType<typeof resolveCdpTarget>) => {
+ assert.equal(resolution.kind, "ok");
+ if (resolution.kind !== "ok") throw new Error("unreachable");
+ return resolution.target;
+};
+
+test("the built-in default parses — the fallback in resolveCdpTarget is unreachable", () => {
+ const target = ok(resolveCdpTarget({}, null));
+ assert.deepEqual(target, { host: "10.88.0.25", port: 9223, source: "default" });
+ assert.equal(`${target.host}:${target.port}`, DEFAULT_REMOTE_CDP);
+});
+
+test("BU_CDP_HTTP wins over a stored target", () => {
+ const target = ok(resolveCdpTarget({ BU_CDP_HTTP: "10.88.0.9:9333" }, "10.88.0.25:9223"));
+ assert.deepEqual(target, { host: "10.88.0.9", port: 9333, source: "env" });
+});
+
+test("a stored target is used when the environment is silent", () => {
+ const target = ok(resolveCdpTarget({}, "192.168.1.4:9222"));
+ assert.deepEqual(target, { host: "192.168.1.4", port: 9222, source: "stored" });
+});
+
+test("an empty BU_CDP_HTTP does not shadow the stored target", () => {
+ const target = ok(resolveCdpTarget({ BU_CDP_HTTP: " " }, "192.168.1.4:9222"));
+ assert.equal(target.source, "stored");
+});
+
+test("the local aliases all resolve to this machine", () => {
+ for (const alias of ["local", "localhost", "off", "none", "0", "no", "LOCAL"]) {
+ const target = ok(resolveCdpTarget({ BU_CDP_HTTP: alias }, null));
+ assert.deepEqual(target, { host: "127.0.0.1", port: LOCAL_CDP_PORT, source: "env" }, alias);
+ }
+});
+
+test("IPv6-ish and malformed values are rejected rather than half-parsed", () => {
+ for (const raw of ["nonsense", "host:", ":9223", "host:0", "host:65536", "host:notaport"]) {
+ const resolution = resolveCdpTarget({ BU_CDP_HTTP: raw }, null);
+ assert.equal(resolution.kind, "invalid", raw);
+ }
+});
+
+test("a host with a port takes the last colon, so hostnames survive", () => {
+ const target = ok(resolveCdpTarget({ BU_CDP_HTTP: "castle.local:9223" }, null));
+ assert.deepEqual(target, { host: "castle.local", port: 9223, source: "env" });
+});
+
+test("describeCdpTarget names where the choice came from", () => {
+ assert.equal(
+ describeCdpTarget({ host: "10.88.0.25", port: 9223, source: "default" }),
+ "10.88.0.25:9223 (built-in default)",
+ );
+ assert.equal(describeCdpTarget({ host: "h", port: 1, source: "env" }), "h:1 (BU_CDP_HTTP)");
+ assert.equal(describeCdpTarget({ host: "h", port: 1, source: "stored" }), "h:1 (/browser-target)");
+});
diff --git a/castle-search/test/engine-target.test.ts b/castle-search/test/engine-target.test.ts
new file mode 100644
index 0000000..e5ced6d
--- /dev/null
+++ b/castle-search/test/engine-target.test.ts
@@ -0,0 +1,89 @@
+/**
+ * Engine selection, and the promise that an unsupported recency window is
+ * refused rather than dropped.
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { ENGINE_ENV_VAR, DEFAULT_ENGINE, resolveEngine } from "../src/engine-target.ts";
+import { enginesSupportingRecency, parseEngineId } from "../src/engines.ts";
+import { selectEngine } from "../src/search.ts";
+
+const ok = (r: ReturnType<typeof resolveEngine>) => {
+ assert.equal(r.kind, "ok");
+ if (r.kind !== "ok") throw new Error("unreachable");
+ return r;
+};
+
+test("nothing configured means the built-in default", () => {
+ const r = ok(resolveEngine(null, {}, null));
+ assert.equal(r.id, DEFAULT_ENGINE);
+ assert.equal(r.source, "default");
+});
+
+test("precedence is parameter, then env, then stored", () => {
+ assert.equal(ok(resolveEngine(null, {}, "bing")).id, "bing");
+ assert.equal(ok(resolveEngine(null, { [ENGINE_ENV_VAR]: "brave" }, "bing")).id, "brave");
+ // The parameter beating the env var is the deliberate difference from the CDP
+ // target: it is what lets the agent fall back when an engine is blocked.
+ const r = ok(resolveEngine("duckduckgo", { [ENGINE_ENV_VAR]: "brave" }, "bing"));
+ assert.equal(r.id, "duckduckgo");
+ assert.equal(r.source, "parameter");
+});
+
+test("aliases resolve, including the short ones a human would type", () => {
+ for (const [raw, expected] of [
+ ["ddg", "duckduckgo"],
+ ["duck", "duckduckgo"],
+ ["DuckDuckGo", "duckduckgo"],
+ [" bing ", "bing"],
+ ["g", "google"],
+ ["BRAVE", "brave"],
+ ] as const) {
+ assert.equal(parseEngineId(raw), expected, raw);
+ }
+});
+
+test("an unknown name is reported, never quietly skipped for the next source", () => {
+ // A typo in the env var must not silently search Google instead.
+ const r = resolveEngine(null, { [ENGINE_ENV_VAR]: "gooogle" }, "bing");
+ assert.equal(r.kind, "invalid");
+ if (r.kind !== "invalid") throw new Error("unreachable");
+ assert.equal(r.source, "env");
+ assert.equal(r.raw, "gooogle");
+});
+
+test("blank values fall through rather than counting as a choice", () => {
+ assert.equal(ok(resolveEngine(" ", { [ENGINE_ENV_VAR]: " " }, "bing")).id, "bing");
+});
+
+test("an unsupported recency window is refused, and names engines that support it", async () => {
+ // Bing has no year window and Brave has no time filter; both must say so
+ // rather than returning unfiltered results that look filtered.
+ for (const engine of ["bing", "brave"] as const) {
+ const error = await selectEngine({ query: "q", numResults: 5, recency: "year", engine }).then(
+ () => null,
+ (e: Error) => e,
+ );
+ assert.ok(error, `${engine} should have refused`);
+ assert.equal(error.name, "RecencyUnsupportedError", engine);
+ for (const alt of enginesSupportingRecency("year")) assert.match(error.message, new RegExp(alt), engine);
+ }
+});
+
+test("windows an engine does support are accepted", async () => {
+ for (const [engine, recency] of [
+ ["google", "year"],
+ ["duckduckgo", "year"],
+ ["bing", "month"],
+ ] as const) {
+ const chosen = await selectEngine({ query: "q", numResults: 5, recency, engine });
+ assert.equal(chosen.engine.id, engine);
+ }
+});
+
+test("brave with no recency is fine — the refusal is about the window, not the engine", async () => {
+ const chosen = await selectEngine({ query: "q", numResults: 5, engine: "brave" });
+ assert.equal(chosen.engine.id, "brave");
+});
diff --git a/castle-search/test/format.test.ts b/castle-search/test/format.test.ts
new file mode 100644
index 0000000..74e37e1
--- /dev/null
+++ b/castle-search/test/format.test.ts
@@ -0,0 +1,70 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { formatResults } from "../src/format.ts";
+import { SearchChallengeError } from "../src/errors.ts";
+
+const results = [
+ { title: "First", url: "https://example.com/1", snippet: "A snippet." },
+ { title: "Second", url: "https://example.com/2", snippet: "" },
+];
+
+test("results are numbered, with the URL on its own line", () => {
+ const text = formatResults({
+ query: "q",
+ searchUrl: "https://www.google.com/search?q=q",
+ finalUrl: "https://www.google.com/search?q=q",
+ endpoint: "10.88.0.25:9223 (built-in default)",
+ engine: "Google",
+ engineSource: "built-in default",
+ results,
+ });
+ // The engine has to be named on every result set: the agent may have switched
+ // engines to get around a block, and two indexes are not interchangeable.
+ assert.match(
+ text,
+ /^Search results for "q" — 2 hits from Google \(built-in default\), via Chrome at 10\.88\.0\.25:9223 \(built-in default\)$/m,
+ );
+ assert.match(text, /^1\. First$/m);
+ assert.match(text, /^ {3}https:\/\/example\.com\/1$/m);
+ assert.match(text, /^2\. Second$/m);
+ // A missing snippet must not leave a stray blank indented line.
+ assert.ok(!/\n {3}\n/.test(text));
+});
+
+test("a redirect is reported, because the query URL is then not where we looked", () => {
+ const text = formatResults({
+ query: "q",
+ searchUrl: "https://www.google.com/search?q=q",
+ finalUrl: "https://www.google.com/search?q=q&sei=abc",
+ endpoint: "e",
+ engine: "Bing",
+ engineSource: "/search-engine",
+ results,
+ });
+ assert.match(text, /^Landed on: https:\/\/www\.google\.com\/search\?q=q&sei=abc$/m);
+});
+
+test("the challenge error tells the agent to hand off to a human, with the URL", () => {
+ const err = new SearchChallengeError({
+ challenge: "captcha-widget",
+ pageUrl: "https://www.google.com/sorry/index?continue=x",
+ endpoint: "10.88.0.25:9223 (built-in default)",
+ engine: "Google",
+ alternatives: ["duckduckgo", "bing", "brave"],
+ detail: "Our systems have detected unusual traffic",
+ });
+ assert.equal(err.name, "SearchChallengeError");
+ assert.ok(err instanceof Error, "must be throwable as an Error");
+ assert.match(err.message, /captcha-widget/);
+ assert.match(err.message, /https:\/\/www\.google\.com\/sorry\/index\?continue=x/);
+ assert.match(err.message, /Ask the user/);
+ assert.match(err.message, /10\.88\.0\.25:9223/);
+ // Switching engines is the agent's own move and must be offered before the
+ // human handoff, since a block on one engine says nothing about the others.
+ for (const alt of ["duckduckgo", "bing", "brave"]) assert.match(err.message, new RegExp(alt));
+ assert.ok(
+ err.message.indexOf("retry the same query with engine") < err.message.indexOf("Ask the user"),
+ "the self-service fallback must come before the human handoff",
+ );
+});
diff --git a/castle-search/test/search-url.test.ts b/castle-search/test/search-url.test.ts
new file mode 100644
index 0000000..2425f76
--- /dev/null
+++ b/castle-search/test/search-url.test.ts
@@ -0,0 +1,88 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { RECENCY_VALUES, allEngines, getEngine, type Recency } from "../src/engines.ts";
+import { buildSearchUrl } from "../src/search.ts";
+
+const urlFor = (id: Parameters<typeof getEngine>[0], recency?: Recency) =>
+ new URL(buildSearchUrl(getEngine(id), { query: "sqlite wal mode", numResults: 7, recency }));
+
+test("google uses the Web tab, and swaps it for as_qdr when time-limited", () => {
+ const plain = urlFor("google");
+ assert.equal(plain.origin + plain.pathname, "https://www.google.com/search");
+ assert.equal(plain.searchParams.get("udm"), "14");
+ assert.equal(plain.searchParams.get("num"), "7");
+ assert.equal(plain.searchParams.get("as_qdr"), null);
+
+ const dated = urlFor("google", "month");
+ // Both findings in one assertion pair: tbs is never used, and udm must be
+ // dropped when a date filter is present or the page renders empty.
+ assert.equal(dated.searchParams.get("as_qdr"), "m");
+ assert.equal(dated.searchParams.get("udm"), null);
+ assert.equal(dated.searchParams.get("tbs"), null);
+});
+
+test("duckduckgo uses the no-JS html endpoint and df=", () => {
+ const plain = urlFor("duckduckgo");
+ assert.equal(plain.origin + plain.pathname, "https://html.duckduckgo.com/html/");
+ assert.equal(plain.searchParams.get("q"), "sqlite wal mode");
+ assert.equal(plain.searchParams.get("df"), null);
+
+ for (const [recency, code] of [
+ ["day", "d"],
+ ["week", "w"],
+ ["month", "m"],
+ ["year", "y"],
+ ] as const) {
+ assert.equal(urlFor("duckduckgo", recency).searchParams.get("df"), code, recency);
+ }
+});
+
+test("bing uses count=, and drops it when filtering because count cancels filters", () => {
+ const plain = urlFor("bing");
+ assert.equal(plain.origin + plain.pathname, "https://www.bing.com/search");
+ assert.equal(plain.searchParams.get("count"), "7");
+ assert.equal(plain.searchParams.get("filters"), null);
+
+ for (const [recency, code] of [
+ ["day", "ez1"],
+ ["week", "ez2"],
+ ["month", "ez3"],
+ ] as const) {
+ const dated = urlFor("bing", recency);
+ assert.equal(dated.searchParams.get("filters"), `ex1:"${code}"`, recency);
+ // Observed: sending count alongside filters silently returns unfiltered
+ // results that look entirely plausible. This assertion is the regression.
+ assert.equal(dated.searchParams.get("count"), null, `${recency}: count must be dropped`);
+ }
+});
+
+test("brave is a plain query — it advertises no time filter at all", () => {
+ const plain = urlFor("brave");
+ assert.equal(plain.origin + plain.pathname, "https://search.brave.com/search");
+ assert.equal(plain.searchParams.get("q"), "sqlite wal mode");
+ assert.deepEqual(getEngine("brave").recency, {});
+});
+
+test("every advertised recency window maps to a non-empty parameter value", () => {
+ // Guards the failure this whole design exists to prevent: an engine claiming
+ // support for a window it then silently drops from the URL.
+ for (const engine of allEngines()) {
+ for (const recency of RECENCY_VALUES) {
+ const code = engine.recency[recency];
+ if (code === undefined) continue;
+ assert.ok(code.length > 0, `${engine.id}/${recency} maps to an empty value`);
+ const url = buildSearchUrl(engine, { query: "q", numResults: 5, recency });
+ assert.ok(url.includes(encodeURIComponent(code)) || url.includes(code), `${engine.id}/${recency} lost its value`);
+ }
+ }
+});
+
+test("queries with characters that would break a URL are encoded, on every engine", () => {
+ const query = 'site:example.com "exact phrase" a&b?c=d #frag +plus/slash';
+ for (const engine of allEngines()) {
+ const url = buildSearchUrl(engine, { query, numResults: 3 });
+ assert.equal(new URL(url).searchParams.get("q"), query, engine.id);
+ assert.ok(!url.includes(" "), `${engine.id}: no raw spaces in the URL`);
+ }
+});
diff --git a/castle-search/tsconfig.json b/castle-search/tsconfig.json
new file mode 100644
index 0000000..0cdaa06
--- /dev/null
+++ b/castle-search/tsconfig.json
@@ -0,0 +1,18 @@
+{
+ "compilerOptions": {
+ "target": "ES2023",
+ "lib": ["ES2023"],
+ "module": "ESNext",
+ "moduleResolution": "bundler",
+ "allowImportingTsExtensions": true,
+ "noEmit": true,
+ "strict": true,
+ "noUncheckedIndexedAccess": true,
+ "noImplicitOverride": true,
+ "exactOptionalPropertyTypes": true,
+ "esModuleInterop": true,
+ "skipLibCheck": true,
+ "types": ["node"]
+ },
+ "include": ["src/**/*.ts", "test/**/*.ts"]
+}
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-06 21:52:14 +0000