diff options
| author | Igor Soarez <igor@soarez.org> | 2026-08-04 15:08:47 +0100 |
|---|---|---|
| committer | Igor Soarez <igor@soarez.org> | 2026-08-04 15:08:47 +0100 |
| commit | bd03d3b8fff9ed18d3cdb104c55908ba5be6c424 (patch) | |
| tree | 659afa1dd7e80c7ac97fb9c142026bf43c1ef8a0 /castle-search/test | |
| parent | 08c0c3a8b4fe755f7fe65ea8d258d77971503882 (diff) | |
Write up what AI answer blocks would cost before building one
Google, Bing and Brave all put a generated answer above their results now,
and none of it is being read. Investigated against castle's Chrome; nothing
in src/ changed, because the cost turned out to be worth deciding on rather
than absorbing.
Two things make it expensive. Google's AI Overview exists only on the
classic layout — udm=14 is the "Web" tab, and having no overview is part of
what that tab is — so enabling it means giving back the clean page the
extractor was built around. And all three stream in after the results are
already parseable: Google settles ~2.3s in, Brave was still growing at 5s,
against an extractor that returns at ~0.5s. The wait is often wasted, since
the blocks are query-dependent — no Google overview for "how does postgres
MVCC vacuum work" after 15s, no Bing answer for "who founded anthropic".
Selectors, the traps behind them (#m-x-content briefly holds ~29k
characters of unrelated page; Google's "Show more" hides only the
disclaimer), the measured timings, three possible designs and their
tradeoffs are all in the doc, so this does not have to be re-measured if
the answer turns out to be yes.
Diffstat (limited to 'castle-search/test')
| -rw-r--r-- | castle-search/test/concurrency.test.ts | 86 | ||||
| -rw-r--r-- | castle-search/test/endpoint.test.ts | 64 | ||||
| -rw-r--r-- | castle-search/test/engine-target.test.ts | 89 | ||||
| -rw-r--r-- | castle-search/test/format.test.ts | 70 | ||||
| -rw-r--r-- | castle-search/test/search-url.test.ts | 88 |
5 files changed, 397 insertions, 0 deletions
diff --git a/castle-search/test/concurrency.test.ts b/castle-search/test/concurrency.test.ts new file mode 100644 index 0000000..500fee0 --- /dev/null +++ b/castle-search/test/concurrency.test.ts @@ -0,0 +1,86 @@ +/** + * The semaphore, exercised through the only door it has: search(). + * + * These run against a deliberately unreachable CDP endpoint. Every search fails + * fast at the connect step, which is exactly what is wanted — the questions here + * are about slot bookkeeping, not about Google. Hitting the real browser to test + * a counter would be slow, flaky, and rude to whoever is using it. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { MAX_CONCURRENT_SEARCHES, disconnect, search } from "../src/search.ts"; + +const UNREACHABLE = "127.0.0.1:1"; // Nothing listens on port 1. +const quiet = () => {}; + +const withUnreachableBrowser = async (body: () => Promise<void>): Promise<void> => { + const saved = process.env["BU_CDP_HTTP"]; + process.env["BU_CDP_HTTP"] = UNREACHABLE; + disconnect(); + try { + await body(); + } finally { + if (saved === undefined) delete process.env["BU_CDP_HTTP"]; + else process.env["BU_CDP_HTTP"] = saved; + disconnect(); + } +}; + +const attempt = (query: string, signal: AbortSignal) => + search({ query, numResults: 1 }, signal, quiet).then( + () => "fulfilled" as const, + (e: Error) => e.name, + ); + +test("a failed search returns its slot, so the next one still runs", async () => { + await withUnreachableBrowser(async () => { + // Three times the cap, run sequentially. If a slot leaked on the failure + // path, the run past the cap would hang until the test timed out. + for (let i = 0; i < MAX_CONCURRENT_SEARCHES * 3; i++) { + const name = await attempt(`q${i}`, AbortSignal.timeout(10_000)); + assert.equal(name, "BrowserUnavailableError", `attempt ${i}`); + } + }); +}); + +test("aborting while queued does not wedge the semaphore", async () => { + await withUnreachableBrowser(async () => { + // Saturate the cap and queue two more, then cancel the queued ones. The + // bug this guards against: a cancelled waiter that is later handed a slot + // resolves into nothing, and the slot is never counted back. + const cancel = new AbortController(); + const running = Array.from({ length: MAX_CONCURRENT_SEARCHES }, (_, i) => + attempt(`running${i}`, AbortSignal.timeout(10_000)), + ); + const queued = Array.from({ length: 2 }, (_, i) => attempt(`queued${i}`, cancel.signal)); + cancel.abort(); + + await Promise.all(running); + for (const outcome of await Promise.all(queued)) { + assert.ok( + outcome === "SearchAbortedError" || outcome === "BrowserUnavailableError", + `unexpected outcome ${outcome}`, + ); + } + + // The real assertion: the pool still works afterwards. A wedged semaphore + // would leave this hanging rather than failing. + for (let i = 0; i < MAX_CONCURRENT_SEARCHES + 1; i++) { + assert.equal(await attempt(`after${i}`, AbortSignal.timeout(10_000)), "BrowserUnavailableError"); + } + }); +}); + +test("an unreachable endpoint is reported, never silently swapped for a local browser", async () => { + await withUnreachableBrowser(async () => { + const error = await search({ query: "q", numResults: 1 }, AbortSignal.timeout(10_000), quiet).then( + () => null, + (e: Error) => e, + ); + assert.ok(error, "expected a throw"); + assert.equal(error.name, "BrowserUnavailableError"); + assert.match(error.message, new RegExp(UNREACHABLE.replace(".", "\\."))); + }); +}); diff --git a/castle-search/test/endpoint.test.ts b/castle-search/test/endpoint.test.ts new file mode 100644 index 0000000..f5c8236 --- /dev/null +++ b/castle-search/test/endpoint.test.ts @@ -0,0 +1,64 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { + DEFAULT_REMOTE_CDP, + LOCAL_CDP_PORT, + describeCdpTarget, + resolveCdpTarget, +} from "../src/endpoint.ts"; + +const ok = (resolution: ReturnType<typeof resolveCdpTarget>) => { + assert.equal(resolution.kind, "ok"); + if (resolution.kind !== "ok") throw new Error("unreachable"); + return resolution.target; +}; + +test("the built-in default parses — the fallback in resolveCdpTarget is unreachable", () => { + const target = ok(resolveCdpTarget({}, null)); + assert.deepEqual(target, { host: "10.88.0.25", port: 9223, source: "default" }); + assert.equal(`${target.host}:${target.port}`, DEFAULT_REMOTE_CDP); +}); + +test("BU_CDP_HTTP wins over a stored target", () => { + const target = ok(resolveCdpTarget({ BU_CDP_HTTP: "10.88.0.9:9333" }, "10.88.0.25:9223")); + assert.deepEqual(target, { host: "10.88.0.9", port: 9333, source: "env" }); +}); + +test("a stored target is used when the environment is silent", () => { + const target = ok(resolveCdpTarget({}, "192.168.1.4:9222")); + assert.deepEqual(target, { host: "192.168.1.4", port: 9222, source: "stored" }); +}); + +test("an empty BU_CDP_HTTP does not shadow the stored target", () => { + const target = ok(resolveCdpTarget({ BU_CDP_HTTP: " " }, "192.168.1.4:9222")); + assert.equal(target.source, "stored"); +}); + +test("the local aliases all resolve to this machine", () => { + for (const alias of ["local", "localhost", "off", "none", "0", "no", "LOCAL"]) { + const target = ok(resolveCdpTarget({ BU_CDP_HTTP: alias }, null)); + assert.deepEqual(target, { host: "127.0.0.1", port: LOCAL_CDP_PORT, source: "env" }, alias); + } +}); + +test("IPv6-ish and malformed values are rejected rather than half-parsed", () => { + for (const raw of ["nonsense", "host:", ":9223", "host:0", "host:65536", "host:notaport"]) { + const resolution = resolveCdpTarget({ BU_CDP_HTTP: raw }, null); + assert.equal(resolution.kind, "invalid", raw); + } +}); + +test("a host with a port takes the last colon, so hostnames survive", () => { + const target = ok(resolveCdpTarget({ BU_CDP_HTTP: "castle.local:9223" }, null)); + assert.deepEqual(target, { host: "castle.local", port: 9223, source: "env" }); +}); + +test("describeCdpTarget names where the choice came from", () => { + assert.equal( + describeCdpTarget({ host: "10.88.0.25", port: 9223, source: "default" }), + "10.88.0.25:9223 (built-in default)", + ); + assert.equal(describeCdpTarget({ host: "h", port: 1, source: "env" }), "h:1 (BU_CDP_HTTP)"); + assert.equal(describeCdpTarget({ host: "h", port: 1, source: "stored" }), "h:1 (/browser-target)"); +}); diff --git a/castle-search/test/engine-target.test.ts b/castle-search/test/engine-target.test.ts new file mode 100644 index 0000000..e5ced6d --- /dev/null +++ b/castle-search/test/engine-target.test.ts @@ -0,0 +1,89 @@ +/** + * Engine selection, and the promise that an unsupported recency window is + * refused rather than dropped. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { ENGINE_ENV_VAR, DEFAULT_ENGINE, resolveEngine } from "../src/engine-target.ts"; +import { enginesSupportingRecency, parseEngineId } from "../src/engines.ts"; +import { selectEngine } from "../src/search.ts"; + +const ok = (r: ReturnType<typeof resolveEngine>) => { + assert.equal(r.kind, "ok"); + if (r.kind !== "ok") throw new Error("unreachable"); + return r; +}; + +test("nothing configured means the built-in default", () => { + const r = ok(resolveEngine(null, {}, null)); + assert.equal(r.id, DEFAULT_ENGINE); + assert.equal(r.source, "default"); +}); + +test("precedence is parameter, then env, then stored", () => { + assert.equal(ok(resolveEngine(null, {}, "bing")).id, "bing"); + assert.equal(ok(resolveEngine(null, { [ENGINE_ENV_VAR]: "brave" }, "bing")).id, "brave"); + // The parameter beating the env var is the deliberate difference from the CDP + // target: it is what lets the agent fall back when an engine is blocked. + const r = ok(resolveEngine("duckduckgo", { [ENGINE_ENV_VAR]: "brave" }, "bing")); + assert.equal(r.id, "duckduckgo"); + assert.equal(r.source, "parameter"); +}); + +test("aliases resolve, including the short ones a human would type", () => { + for (const [raw, expected] of [ + ["ddg", "duckduckgo"], + ["duck", "duckduckgo"], + ["DuckDuckGo", "duckduckgo"], + [" bing ", "bing"], + ["g", "google"], + ["BRAVE", "brave"], + ] as const) { + assert.equal(parseEngineId(raw), expected, raw); + } +}); + +test("an unknown name is reported, never quietly skipped for the next source", () => { + // A typo in the env var must not silently search Google instead. + const r = resolveEngine(null, { [ENGINE_ENV_VAR]: "gooogle" }, "bing"); + assert.equal(r.kind, "invalid"); + if (r.kind !== "invalid") throw new Error("unreachable"); + assert.equal(r.source, "env"); + assert.equal(r.raw, "gooogle"); +}); + +test("blank values fall through rather than counting as a choice", () => { + assert.equal(ok(resolveEngine(" ", { [ENGINE_ENV_VAR]: " " }, "bing")).id, "bing"); +}); + +test("an unsupported recency window is refused, and names engines that support it", async () => { + // Bing has no year window and Brave has no time filter; both must say so + // rather than returning unfiltered results that look filtered. + for (const engine of ["bing", "brave"] as const) { + const error = await selectEngine({ query: "q", numResults: 5, recency: "year", engine }).then( + () => null, + (e: Error) => e, + ); + assert.ok(error, `${engine} should have refused`); + assert.equal(error.name, "RecencyUnsupportedError", engine); + for (const alt of enginesSupportingRecency("year")) assert.match(error.message, new RegExp(alt), engine); + } +}); + +test("windows an engine does support are accepted", async () => { + for (const [engine, recency] of [ + ["google", "year"], + ["duckduckgo", "year"], + ["bing", "month"], + ] as const) { + const chosen = await selectEngine({ query: "q", numResults: 5, recency, engine }); + assert.equal(chosen.engine.id, engine); + } +}); + +test("brave with no recency is fine — the refusal is about the window, not the engine", async () => { + const chosen = await selectEngine({ query: "q", numResults: 5, engine: "brave" }); + assert.equal(chosen.engine.id, "brave"); +}); diff --git a/castle-search/test/format.test.ts b/castle-search/test/format.test.ts new file mode 100644 index 0000000..74e37e1 --- /dev/null +++ b/castle-search/test/format.test.ts @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { formatResults } from "../src/format.ts"; +import { SearchChallengeError } from "../src/errors.ts"; + +const results = [ + { title: "First", url: "https://example.com/1", snippet: "A snippet." }, + { title: "Second", url: "https://example.com/2", snippet: "" }, +]; + +test("results are numbered, with the URL on its own line", () => { + const text = formatResults({ + query: "q", + searchUrl: "https://www.google.com/search?q=q", + finalUrl: "https://www.google.com/search?q=q", + endpoint: "10.88.0.25:9223 (built-in default)", + engine: "Google", + engineSource: "built-in default", + results, + }); + // The engine has to be named on every result set: the agent may have switched + // engines to get around a block, and two indexes are not interchangeable. + assert.match( + text, + /^Search results for "q" — 2 hits from Google \(built-in default\), via Chrome at 10\.88\.0\.25:9223 \(built-in default\)$/m, + ); + assert.match(text, /^1\. First$/m); + assert.match(text, /^ {3}https:\/\/example\.com\/1$/m); + assert.match(text, /^2\. Second$/m); + // A missing snippet must not leave a stray blank indented line. + assert.ok(!/\n {3}\n/.test(text)); +}); + +test("a redirect is reported, because the query URL is then not where we looked", () => { + const text = formatResults({ + query: "q", + searchUrl: "https://www.google.com/search?q=q", + finalUrl: "https://www.google.com/search?q=q&sei=abc", + endpoint: "e", + engine: "Bing", + engineSource: "/search-engine", + results, + }); + assert.match(text, /^Landed on: https:\/\/www\.google\.com\/search\?q=q&sei=abc$/m); +}); + +test("the challenge error tells the agent to hand off to a human, with the URL", () => { + const err = new SearchChallengeError({ + challenge: "captcha-widget", + pageUrl: "https://www.google.com/sorry/index?continue=x", + endpoint: "10.88.0.25:9223 (built-in default)", + engine: "Google", + alternatives: ["duckduckgo", "bing", "brave"], + detail: "Our systems have detected unusual traffic", + }); + assert.equal(err.name, "SearchChallengeError"); + assert.ok(err instanceof Error, "must be throwable as an Error"); + assert.match(err.message, /captcha-widget/); + assert.match(err.message, /https:\/\/www\.google\.com\/sorry\/index\?continue=x/); + assert.match(err.message, /Ask the user/); + assert.match(err.message, /10\.88\.0\.25:9223/); + // Switching engines is the agent's own move and must be offered before the + // human handoff, since a block on one engine says nothing about the others. + for (const alt of ["duckduckgo", "bing", "brave"]) assert.match(err.message, new RegExp(alt)); + assert.ok( + err.message.indexOf("retry the same query with engine") < err.message.indexOf("Ask the user"), + "the self-service fallback must come before the human handoff", + ); +}); diff --git a/castle-search/test/search-url.test.ts b/castle-search/test/search-url.test.ts new file mode 100644 index 0000000..2425f76 --- /dev/null +++ b/castle-search/test/search-url.test.ts @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { RECENCY_VALUES, allEngines, getEngine, type Recency } from "../src/engines.ts"; +import { buildSearchUrl } from "../src/search.ts"; + +const urlFor = (id: Parameters<typeof getEngine>[0], recency?: Recency) => + new URL(buildSearchUrl(getEngine(id), { query: "sqlite wal mode", numResults: 7, recency })); + +test("google uses the Web tab, and swaps it for as_qdr when time-limited", () => { + const plain = urlFor("google"); + assert.equal(plain.origin + plain.pathname, "https://www.google.com/search"); + assert.equal(plain.searchParams.get("udm"), "14"); + assert.equal(plain.searchParams.get("num"), "7"); + assert.equal(plain.searchParams.get("as_qdr"), null); + + const dated = urlFor("google", "month"); + // Both findings in one assertion pair: tbs is never used, and udm must be + // dropped when a date filter is present or the page renders empty. + assert.equal(dated.searchParams.get("as_qdr"), "m"); + assert.equal(dated.searchParams.get("udm"), null); + assert.equal(dated.searchParams.get("tbs"), null); +}); + +test("duckduckgo uses the no-JS html endpoint and df=", () => { + const plain = urlFor("duckduckgo"); + assert.equal(plain.origin + plain.pathname, "https://html.duckduckgo.com/html/"); + assert.equal(plain.searchParams.get("q"), "sqlite wal mode"); + assert.equal(plain.searchParams.get("df"), null); + + for (const [recency, code] of [ + ["day", "d"], + ["week", "w"], + ["month", "m"], + ["year", "y"], + ] as const) { + assert.equal(urlFor("duckduckgo", recency).searchParams.get("df"), code, recency); + } +}); + +test("bing uses count=, and drops it when filtering because count cancels filters", () => { + const plain = urlFor("bing"); + assert.equal(plain.origin + plain.pathname, "https://www.bing.com/search"); + assert.equal(plain.searchParams.get("count"), "7"); + assert.equal(plain.searchParams.get("filters"), null); + + for (const [recency, code] of [ + ["day", "ez1"], + ["week", "ez2"], + ["month", "ez3"], + ] as const) { + const dated = urlFor("bing", recency); + assert.equal(dated.searchParams.get("filters"), `ex1:"${code}"`, recency); + // Observed: sending count alongside filters silently returns unfiltered + // results that look entirely plausible. This assertion is the regression. + assert.equal(dated.searchParams.get("count"), null, `${recency}: count must be dropped`); + } +}); + +test("brave is a plain query — it advertises no time filter at all", () => { + const plain = urlFor("brave"); + assert.equal(plain.origin + plain.pathname, "https://search.brave.com/search"); + assert.equal(plain.searchParams.get("q"), "sqlite wal mode"); + assert.deepEqual(getEngine("brave").recency, {}); +}); + +test("every advertised recency window maps to a non-empty parameter value", () => { + // Guards the failure this whole design exists to prevent: an engine claiming + // support for a window it then silently drops from the URL. + for (const engine of allEngines()) { + for (const recency of RECENCY_VALUES) { + const code = engine.recency[recency]; + if (code === undefined) continue; + assert.ok(code.length > 0, `${engine.id}/${recency} maps to an empty value`); + const url = buildSearchUrl(engine, { query: "q", numResults: 5, recency }); + assert.ok(url.includes(encodeURIComponent(code)) || url.includes(code), `${engine.id}/${recency} lost its value`); + } + } +}); + +test("queries with characters that would break a URL are encoded, on every engine", () => { + const query = 'site:example.com "exact phrase" a&b?c=d #frag +plus/slash'; + for (const engine of allEngines()) { + const url = buildSearchUrl(engine, { query, numResults: 3 }); + assert.equal(new URL(url).searchParams.get("q"), query, engine.id); + assert.ok(!url.includes(" "), `${engine.id}: no raw spaces in the URL`); + } +}); |
