aboutsummaryrefslogtreecommitdiff
path: root/castle-search/test
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-08-04 15:08:47 +0100
committerIgor Soarez <igor@soarez.org>2026-08-04 15:08:47 +0100
commitbd03d3b8fff9ed18d3cdb104c55908ba5be6c424 (patch)
tree659afa1dd7e80c7ac97fb9c142026bf43c1ef8a0 /castle-search/test
parent08c0c3a8b4fe755f7fe65ea8d258d77971503882 (diff)
Write up what AI answer blocks would cost before building one
Google, Bing and Brave all put a generated answer above their results now, and none of it is being read. Investigated against castle's Chrome; nothing in src/ changed, because the cost turned out to be worth deciding on rather than absorbing. Two things make it expensive. Google's AI Overview exists only on the classic layout — udm=14 is the "Web" tab, and having no overview is part of what that tab is — so enabling it means giving back the clean page the extractor was built around. And all three stream in after the results are already parseable: Google settles ~2.3s in, Brave was still growing at 5s, against an extractor that returns at ~0.5s. The wait is often wasted, since the blocks are query-dependent — no Google overview for "how does postgres MVCC vacuum work" after 15s, no Bing answer for "who founded anthropic". Selectors, the traps behind them (#m-x-content briefly holds ~29k characters of unrelated page; Google's "Show more" hides only the disclaimer), the measured timings, three possible designs and their tradeoffs are all in the doc, so this does not have to be re-measured if the answer turns out to be yes.
Diffstat (limited to 'castle-search/test')
-rw-r--r--castle-search/test/concurrency.test.ts86
-rw-r--r--castle-search/test/endpoint.test.ts64
-rw-r--r--castle-search/test/engine-target.test.ts89
-rw-r--r--castle-search/test/format.test.ts70
-rw-r--r--castle-search/test/search-url.test.ts88
5 files changed, 397 insertions, 0 deletions
diff --git a/castle-search/test/concurrency.test.ts b/castle-search/test/concurrency.test.ts
new file mode 100644
index 0000000..500fee0
--- /dev/null
+++ b/castle-search/test/concurrency.test.ts
@@ -0,0 +1,86 @@
+/**
+ * The semaphore, exercised through the only door it has: search().
+ *
+ * These run against a deliberately unreachable CDP endpoint. Every search fails
+ * fast at the connect step, which is exactly what is wanted — the questions here
+ * are about slot bookkeeping, not about Google. Hitting the real browser to test
+ * a counter would be slow, flaky, and rude to whoever is using it.
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { MAX_CONCURRENT_SEARCHES, disconnect, search } from "../src/search.ts";
+
+const UNREACHABLE = "127.0.0.1:1"; // Nothing listens on port 1.
+const quiet = () => {};
+
+const withUnreachableBrowser = async (body: () => Promise<void>): Promise<void> => {
+ const saved = process.env["BU_CDP_HTTP"];
+ process.env["BU_CDP_HTTP"] = UNREACHABLE;
+ disconnect();
+ try {
+ await body();
+ } finally {
+ if (saved === undefined) delete process.env["BU_CDP_HTTP"];
+ else process.env["BU_CDP_HTTP"] = saved;
+ disconnect();
+ }
+};
+
+const attempt = (query: string, signal: AbortSignal) =>
+ search({ query, numResults: 1 }, signal, quiet).then(
+ () => "fulfilled" as const,
+ (e: Error) => e.name,
+ );
+
+test("a failed search returns its slot, so the next one still runs", async () => {
+ await withUnreachableBrowser(async () => {
+ // Three times the cap, run sequentially. If a slot leaked on the failure
+ // path, the run past the cap would hang until the test timed out.
+ for (let i = 0; i < MAX_CONCURRENT_SEARCHES * 3; i++) {
+ const name = await attempt(`q${i}`, AbortSignal.timeout(10_000));
+ assert.equal(name, "BrowserUnavailableError", `attempt ${i}`);
+ }
+ });
+});
+
+test("aborting while queued does not wedge the semaphore", async () => {
+ await withUnreachableBrowser(async () => {
+ // Saturate the cap and queue two more, then cancel the queued ones. The
+ // bug this guards against: a cancelled waiter that is later handed a slot
+ // resolves into nothing, and the slot is never counted back.
+ const cancel = new AbortController();
+ const running = Array.from({ length: MAX_CONCURRENT_SEARCHES }, (_, i) =>
+ attempt(`running${i}`, AbortSignal.timeout(10_000)),
+ );
+ const queued = Array.from({ length: 2 }, (_, i) => attempt(`queued${i}`, cancel.signal));
+ cancel.abort();
+
+ await Promise.all(running);
+ for (const outcome of await Promise.all(queued)) {
+ assert.ok(
+ outcome === "SearchAbortedError" || outcome === "BrowserUnavailableError",
+ `unexpected outcome ${outcome}`,
+ );
+ }
+
+ // The real assertion: the pool still works afterwards. A wedged semaphore
+ // would leave this hanging rather than failing.
+ for (let i = 0; i < MAX_CONCURRENT_SEARCHES + 1; i++) {
+ assert.equal(await attempt(`after${i}`, AbortSignal.timeout(10_000)), "BrowserUnavailableError");
+ }
+ });
+});
+
+test("an unreachable endpoint is reported, never silently swapped for a local browser", async () => {
+ await withUnreachableBrowser(async () => {
+ const error = await search({ query: "q", numResults: 1 }, AbortSignal.timeout(10_000), quiet).then(
+ () => null,
+ (e: Error) => e,
+ );
+ assert.ok(error, "expected a throw");
+ assert.equal(error.name, "BrowserUnavailableError");
+ assert.match(error.message, new RegExp(UNREACHABLE.replace(".", "\\.")));
+ });
+});
diff --git a/castle-search/test/endpoint.test.ts b/castle-search/test/endpoint.test.ts
new file mode 100644
index 0000000..f5c8236
--- /dev/null
+++ b/castle-search/test/endpoint.test.ts
@@ -0,0 +1,64 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import {
+ DEFAULT_REMOTE_CDP,
+ LOCAL_CDP_PORT,
+ describeCdpTarget,
+ resolveCdpTarget,
+} from "../src/endpoint.ts";
+
+const ok = (resolution: ReturnType<typeof resolveCdpTarget>) => {
+ assert.equal(resolution.kind, "ok");
+ if (resolution.kind !== "ok") throw new Error("unreachable");
+ return resolution.target;
+};
+
+test("the built-in default parses — the fallback in resolveCdpTarget is unreachable", () => {
+ const target = ok(resolveCdpTarget({}, null));
+ assert.deepEqual(target, { host: "10.88.0.25", port: 9223, source: "default" });
+ assert.equal(`${target.host}:${target.port}`, DEFAULT_REMOTE_CDP);
+});
+
+test("BU_CDP_HTTP wins over a stored target", () => {
+ const target = ok(resolveCdpTarget({ BU_CDP_HTTP: "10.88.0.9:9333" }, "10.88.0.25:9223"));
+ assert.deepEqual(target, { host: "10.88.0.9", port: 9333, source: "env" });
+});
+
+test("a stored target is used when the environment is silent", () => {
+ const target = ok(resolveCdpTarget({}, "192.168.1.4:9222"));
+ assert.deepEqual(target, { host: "192.168.1.4", port: 9222, source: "stored" });
+});
+
+test("an empty BU_CDP_HTTP does not shadow the stored target", () => {
+ const target = ok(resolveCdpTarget({ BU_CDP_HTTP: " " }, "192.168.1.4:9222"));
+ assert.equal(target.source, "stored");
+});
+
+test("the local aliases all resolve to this machine", () => {
+ for (const alias of ["local", "localhost", "off", "none", "0", "no", "LOCAL"]) {
+ const target = ok(resolveCdpTarget({ BU_CDP_HTTP: alias }, null));
+ assert.deepEqual(target, { host: "127.0.0.1", port: LOCAL_CDP_PORT, source: "env" }, alias);
+ }
+});
+
+test("IPv6-ish and malformed values are rejected rather than half-parsed", () => {
+ for (const raw of ["nonsense", "host:", ":9223", "host:0", "host:65536", "host:notaport"]) {
+ const resolution = resolveCdpTarget({ BU_CDP_HTTP: raw }, null);
+ assert.equal(resolution.kind, "invalid", raw);
+ }
+});
+
+test("a host with a port takes the last colon, so hostnames survive", () => {
+ const target = ok(resolveCdpTarget({ BU_CDP_HTTP: "castle.local:9223" }, null));
+ assert.deepEqual(target, { host: "castle.local", port: 9223, source: "env" });
+});
+
+test("describeCdpTarget names where the choice came from", () => {
+ assert.equal(
+ describeCdpTarget({ host: "10.88.0.25", port: 9223, source: "default" }),
+ "10.88.0.25:9223 (built-in default)",
+ );
+ assert.equal(describeCdpTarget({ host: "h", port: 1, source: "env" }), "h:1 (BU_CDP_HTTP)");
+ assert.equal(describeCdpTarget({ host: "h", port: 1, source: "stored" }), "h:1 (/browser-target)");
+});
diff --git a/castle-search/test/engine-target.test.ts b/castle-search/test/engine-target.test.ts
new file mode 100644
index 0000000..e5ced6d
--- /dev/null
+++ b/castle-search/test/engine-target.test.ts
@@ -0,0 +1,89 @@
+/**
+ * Engine selection, and the promise that an unsupported recency window is
+ * refused rather than dropped.
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { ENGINE_ENV_VAR, DEFAULT_ENGINE, resolveEngine } from "../src/engine-target.ts";
+import { enginesSupportingRecency, parseEngineId } from "../src/engines.ts";
+import { selectEngine } from "../src/search.ts";
+
+const ok = (r: ReturnType<typeof resolveEngine>) => {
+ assert.equal(r.kind, "ok");
+ if (r.kind !== "ok") throw new Error("unreachable");
+ return r;
+};
+
+test("nothing configured means the built-in default", () => {
+ const r = ok(resolveEngine(null, {}, null));
+ assert.equal(r.id, DEFAULT_ENGINE);
+ assert.equal(r.source, "default");
+});
+
+test("precedence is parameter, then env, then stored", () => {
+ assert.equal(ok(resolveEngine(null, {}, "bing")).id, "bing");
+ assert.equal(ok(resolveEngine(null, { [ENGINE_ENV_VAR]: "brave" }, "bing")).id, "brave");
+ // The parameter beating the env var is the deliberate difference from the CDP
+ // target: it is what lets the agent fall back when an engine is blocked.
+ const r = ok(resolveEngine("duckduckgo", { [ENGINE_ENV_VAR]: "brave" }, "bing"));
+ assert.equal(r.id, "duckduckgo");
+ assert.equal(r.source, "parameter");
+});
+
+test("aliases resolve, including the short ones a human would type", () => {
+ for (const [raw, expected] of [
+ ["ddg", "duckduckgo"],
+ ["duck", "duckduckgo"],
+ ["DuckDuckGo", "duckduckgo"],
+ [" bing ", "bing"],
+ ["g", "google"],
+ ["BRAVE", "brave"],
+ ] as const) {
+ assert.equal(parseEngineId(raw), expected, raw);
+ }
+});
+
+test("an unknown name is reported, never quietly skipped for the next source", () => {
+ // A typo in the env var must not silently search Google instead.
+ const r = resolveEngine(null, { [ENGINE_ENV_VAR]: "gooogle" }, "bing");
+ assert.equal(r.kind, "invalid");
+ if (r.kind !== "invalid") throw new Error("unreachable");
+ assert.equal(r.source, "env");
+ assert.equal(r.raw, "gooogle");
+});
+
+test("blank values fall through rather than counting as a choice", () => {
+ assert.equal(ok(resolveEngine(" ", { [ENGINE_ENV_VAR]: " " }, "bing")).id, "bing");
+});
+
+test("an unsupported recency window is refused, and names engines that support it", async () => {
+ // Bing has no year window and Brave has no time filter; both must say so
+ // rather than returning unfiltered results that look filtered.
+ for (const engine of ["bing", "brave"] as const) {
+ const error = await selectEngine({ query: "q", numResults: 5, recency: "year", engine }).then(
+ () => null,
+ (e: Error) => e,
+ );
+ assert.ok(error, `${engine} should have refused`);
+ assert.equal(error.name, "RecencyUnsupportedError", engine);
+ for (const alt of enginesSupportingRecency("year")) assert.match(error.message, new RegExp(alt), engine);
+ }
+});
+
+test("windows an engine does support are accepted", async () => {
+ for (const [engine, recency] of [
+ ["google", "year"],
+ ["duckduckgo", "year"],
+ ["bing", "month"],
+ ] as const) {
+ const chosen = await selectEngine({ query: "q", numResults: 5, recency, engine });
+ assert.equal(chosen.engine.id, engine);
+ }
+});
+
+test("brave with no recency is fine — the refusal is about the window, not the engine", async () => {
+ const chosen = await selectEngine({ query: "q", numResults: 5, engine: "brave" });
+ assert.equal(chosen.engine.id, "brave");
+});
diff --git a/castle-search/test/format.test.ts b/castle-search/test/format.test.ts
new file mode 100644
index 0000000..74e37e1
--- /dev/null
+++ b/castle-search/test/format.test.ts
@@ -0,0 +1,70 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { formatResults } from "../src/format.ts";
+import { SearchChallengeError } from "../src/errors.ts";
+
+const results = [
+ { title: "First", url: "https://example.com/1", snippet: "A snippet." },
+ { title: "Second", url: "https://example.com/2", snippet: "" },
+];
+
+test("results are numbered, with the URL on its own line", () => {
+ const text = formatResults({
+ query: "q",
+ searchUrl: "https://www.google.com/search?q=q",
+ finalUrl: "https://www.google.com/search?q=q",
+ endpoint: "10.88.0.25:9223 (built-in default)",
+ engine: "Google",
+ engineSource: "built-in default",
+ results,
+ });
+ // The engine has to be named on every result set: the agent may have switched
+ // engines to get around a block, and two indexes are not interchangeable.
+ assert.match(
+ text,
+ /^Search results for "q" — 2 hits from Google \(built-in default\), via Chrome at 10\.88\.0\.25:9223 \(built-in default\)$/m,
+ );
+ assert.match(text, /^1\. First$/m);
+ assert.match(text, /^ {3}https:\/\/example\.com\/1$/m);
+ assert.match(text, /^2\. Second$/m);
+ // A missing snippet must not leave a stray blank indented line.
+ assert.ok(!/\n {3}\n/.test(text));
+});
+
+test("a redirect is reported, because the query URL is then not where we looked", () => {
+ const text = formatResults({
+ query: "q",
+ searchUrl: "https://www.google.com/search?q=q",
+ finalUrl: "https://www.google.com/search?q=q&sei=abc",
+ endpoint: "e",
+ engine: "Bing",
+ engineSource: "/search-engine",
+ results,
+ });
+ assert.match(text, /^Landed on: https:\/\/www\.google\.com\/search\?q=q&sei=abc$/m);
+});
+
+test("the challenge error tells the agent to hand off to a human, with the URL", () => {
+ const err = new SearchChallengeError({
+ challenge: "captcha-widget",
+ pageUrl: "https://www.google.com/sorry/index?continue=x",
+ endpoint: "10.88.0.25:9223 (built-in default)",
+ engine: "Google",
+ alternatives: ["duckduckgo", "bing", "brave"],
+ detail: "Our systems have detected unusual traffic",
+ });
+ assert.equal(err.name, "SearchChallengeError");
+ assert.ok(err instanceof Error, "must be throwable as an Error");
+ assert.match(err.message, /captcha-widget/);
+ assert.match(err.message, /https:\/\/www\.google\.com\/sorry\/index\?continue=x/);
+ assert.match(err.message, /Ask the user/);
+ assert.match(err.message, /10\.88\.0\.25:9223/);
+ // Switching engines is the agent's own move and must be offered before the
+ // human handoff, since a block on one engine says nothing about the others.
+ for (const alt of ["duckduckgo", "bing", "brave"]) assert.match(err.message, new RegExp(alt));
+ assert.ok(
+ err.message.indexOf("retry the same query with engine") < err.message.indexOf("Ask the user"),
+ "the self-service fallback must come before the human handoff",
+ );
+});
diff --git a/castle-search/test/search-url.test.ts b/castle-search/test/search-url.test.ts
new file mode 100644
index 0000000..2425f76
--- /dev/null
+++ b/castle-search/test/search-url.test.ts
@@ -0,0 +1,88 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { RECENCY_VALUES, allEngines, getEngine, type Recency } from "../src/engines.ts";
+import { buildSearchUrl } from "../src/search.ts";
+
+const urlFor = (id: Parameters<typeof getEngine>[0], recency?: Recency) =>
+ new URL(buildSearchUrl(getEngine(id), { query: "sqlite wal mode", numResults: 7, recency }));
+
+test("google uses the Web tab, and swaps it for as_qdr when time-limited", () => {
+ const plain = urlFor("google");
+ assert.equal(plain.origin + plain.pathname, "https://www.google.com/search");
+ assert.equal(plain.searchParams.get("udm"), "14");
+ assert.equal(plain.searchParams.get("num"), "7");
+ assert.equal(plain.searchParams.get("as_qdr"), null);
+
+ const dated = urlFor("google", "month");
+ // Both findings in one assertion pair: tbs is never used, and udm must be
+ // dropped when a date filter is present or the page renders empty.
+ assert.equal(dated.searchParams.get("as_qdr"), "m");
+ assert.equal(dated.searchParams.get("udm"), null);
+ assert.equal(dated.searchParams.get("tbs"), null);
+});
+
+test("duckduckgo uses the no-JS html endpoint and df=", () => {
+ const plain = urlFor("duckduckgo");
+ assert.equal(plain.origin + plain.pathname, "https://html.duckduckgo.com/html/");
+ assert.equal(plain.searchParams.get("q"), "sqlite wal mode");
+ assert.equal(plain.searchParams.get("df"), null);
+
+ for (const [recency, code] of [
+ ["day", "d"],
+ ["week", "w"],
+ ["month", "m"],
+ ["year", "y"],
+ ] as const) {
+ assert.equal(urlFor("duckduckgo", recency).searchParams.get("df"), code, recency);
+ }
+});
+
+test("bing uses count=, and drops it when filtering because count cancels filters", () => {
+ const plain = urlFor("bing");
+ assert.equal(plain.origin + plain.pathname, "https://www.bing.com/search");
+ assert.equal(plain.searchParams.get("count"), "7");
+ assert.equal(plain.searchParams.get("filters"), null);
+
+ for (const [recency, code] of [
+ ["day", "ez1"],
+ ["week", "ez2"],
+ ["month", "ez3"],
+ ] as const) {
+ const dated = urlFor("bing", recency);
+ assert.equal(dated.searchParams.get("filters"), `ex1:"${code}"`, recency);
+ // Observed: sending count alongside filters silently returns unfiltered
+ // results that look entirely plausible. This assertion is the regression.
+ assert.equal(dated.searchParams.get("count"), null, `${recency}: count must be dropped`);
+ }
+});
+
+test("brave is a plain query — it advertises no time filter at all", () => {
+ const plain = urlFor("brave");
+ assert.equal(plain.origin + plain.pathname, "https://search.brave.com/search");
+ assert.equal(plain.searchParams.get("q"), "sqlite wal mode");
+ assert.deepEqual(getEngine("brave").recency, {});
+});
+
+test("every advertised recency window maps to a non-empty parameter value", () => {
+ // Guards the failure this whole design exists to prevent: an engine claiming
+ // support for a window it then silently drops from the URL.
+ for (const engine of allEngines()) {
+ for (const recency of RECENCY_VALUES) {
+ const code = engine.recency[recency];
+ if (code === undefined) continue;
+ assert.ok(code.length > 0, `${engine.id}/${recency} maps to an empty value`);
+ const url = buildSearchUrl(engine, { query: "q", numResults: 5, recency });
+ assert.ok(url.includes(encodeURIComponent(code)) || url.includes(code), `${engine.id}/${recency} lost its value`);
+ }
+ }
+});
+
+test("queries with characters that would break a URL are encoded, on every engine", () => {
+ const query = 'site:example.com "exact phrase" a&b?c=d #frag +plus/slash';
+ for (const engine of allEngines()) {
+ const url = buildSearchUrl(engine, { query, numResults: 3 });
+ assert.equal(new URL(url).searchParams.get("q"), query, engine.id);
+ assert.ok(!url.includes(" "), `${engine.id}: no raw spaces in the URL`);
+ }
+});