aboutsummaryrefslogtreecommitdiff
path: root/castle-search/test/engine-target.test.ts
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-08-04 15:08:47 +0100
committerIgor Soarez <igor@soarez.org>2026-08-04 15:08:47 +0100
commitbd03d3b8fff9ed18d3cdb104c55908ba5be6c424 (patch)
tree659afa1dd7e80c7ac97fb9c142026bf43c1ef8a0 /castle-search/test/engine-target.test.ts
parent08c0c3a8b4fe755f7fe65ea8d258d77971503882 (diff)
Write up what AI answer blocks would cost before building one
Google, Bing and Brave all put a generated answer above their results now, and none of it is being read. Investigated against castle's Chrome; nothing in src/ changed, because the cost turned out to be worth deciding on rather than absorbing. Two things make it expensive. Google's AI Overview exists only on the classic layout — udm=14 is the "Web" tab, and having no overview is part of what that tab is — so enabling it means giving back the clean page the extractor was built around. And all three stream in after the results are already parseable: Google settles ~2.3s in, Brave was still growing at 5s, against an extractor that returns at ~0.5s. The wait is often wasted, since the blocks are query-dependent — no Google overview for "how does postgres MVCC vacuum work" after 15s, no Bing answer for "who founded anthropic". Selectors, the traps behind them (#m-x-content briefly holds ~29k characters of unrelated page; Google's "Show more" hides only the disclaimer), the measured timings, three possible designs and their tradeoffs are all in the doc, so this does not have to be re-measured if the answer turns out to be yes.
Diffstat (limited to 'castle-search/test/engine-target.test.ts')
-rw-r--r--castle-search/test/engine-target.test.ts89
1 files changed, 89 insertions, 0 deletions
diff --git a/castle-search/test/engine-target.test.ts b/castle-search/test/engine-target.test.ts
new file mode 100644
index 0000000..e5ced6d
--- /dev/null
+++ b/castle-search/test/engine-target.test.ts
@@ -0,0 +1,89 @@
+/**
+ * Engine selection, and the promise that an unsupported recency window is
+ * refused rather than dropped.
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { ENGINE_ENV_VAR, DEFAULT_ENGINE, resolveEngine } from "../src/engine-target.ts";
+import { enginesSupportingRecency, parseEngineId } from "../src/engines.ts";
+import { selectEngine } from "../src/search.ts";
+
+const ok = (r: ReturnType<typeof resolveEngine>) => {
+ assert.equal(r.kind, "ok");
+ if (r.kind !== "ok") throw new Error("unreachable");
+ return r;
+};
+
+test("nothing configured means the built-in default", () => {
+ const r = ok(resolveEngine(null, {}, null));
+ assert.equal(r.id, DEFAULT_ENGINE);
+ assert.equal(r.source, "default");
+});
+
+test("precedence is parameter, then env, then stored", () => {
+ assert.equal(ok(resolveEngine(null, {}, "bing")).id, "bing");
+ assert.equal(ok(resolveEngine(null, { [ENGINE_ENV_VAR]: "brave" }, "bing")).id, "brave");
+ // The parameter beating the env var is the deliberate difference from the CDP
+ // target: it is what lets the agent fall back when an engine is blocked.
+ const r = ok(resolveEngine("duckduckgo", { [ENGINE_ENV_VAR]: "brave" }, "bing"));
+ assert.equal(r.id, "duckduckgo");
+ assert.equal(r.source, "parameter");
+});
+
+test("aliases resolve, including the short ones a human would type", () => {
+ for (const [raw, expected] of [
+ ["ddg", "duckduckgo"],
+ ["duck", "duckduckgo"],
+ ["DuckDuckGo", "duckduckgo"],
+ [" bing ", "bing"],
+ ["g", "google"],
+ ["BRAVE", "brave"],
+ ] as const) {
+ assert.equal(parseEngineId(raw), expected, raw);
+ }
+});
+
+test("an unknown name is reported, never quietly skipped for the next source", () => {
+ // A typo in the env var must not silently search Google instead.
+ const r = resolveEngine(null, { [ENGINE_ENV_VAR]: "gooogle" }, "bing");
+ assert.equal(r.kind, "invalid");
+ if (r.kind !== "invalid") throw new Error("unreachable");
+ assert.equal(r.source, "env");
+ assert.equal(r.raw, "gooogle");
+});
+
+test("blank values fall through rather than counting as a choice", () => {
+ assert.equal(ok(resolveEngine(" ", { [ENGINE_ENV_VAR]: " " }, "bing")).id, "bing");
+});
+
+test("an unsupported recency window is refused, and names engines that support it", async () => {
+ // Bing has no year window and Brave has no time filter; both must say so
+ // rather than returning unfiltered results that look filtered.
+ for (const engine of ["bing", "brave"] as const) {
+ const error = await selectEngine({ query: "q", numResults: 5, recency: "year", engine }).then(
+ () => null,
+ (e: Error) => e,
+ );
+ assert.ok(error, `${engine} should have refused`);
+ assert.equal(error.name, "RecencyUnsupportedError", engine);
+ for (const alt of enginesSupportingRecency("year")) assert.match(error.message, new RegExp(alt), engine);
+ }
+});
+
+test("windows an engine does support are accepted", async () => {
+ for (const [engine, recency] of [
+ ["google", "year"],
+ ["duckduckgo", "year"],
+ ["bing", "month"],
+ ] as const) {
+ const chosen = await selectEngine({ query: "q", numResults: 5, recency, engine });
+ assert.equal(chosen.engine.id, engine);
+ }
+});
+
+test("brave with no recency is fine — the refusal is about the window, not the engine", async () => {
+ const chosen = await selectEngine({ query: "q", numResults: 5, engine: "brave" });
+ assert.equal(chosen.engine.id, "brave");
+});