diff options
| author | Igor Soarez <igor@soarez.org> | 2026-08-04 15:08:47 +0100 |
|---|---|---|
| committer | Igor Soarez <igor@soarez.org> | 2026-08-04 15:08:47 +0100 |
| commit | bd03d3b8fff9ed18d3cdb104c55908ba5be6c424 (patch) | |
| tree | 659afa1dd7e80c7ac97fb9c142026bf43c1ef8a0 /castle-search/test/engine-target.test.ts | |
| parent | 08c0c3a8b4fe755f7fe65ea8d258d77971503882 (diff) | |
Write up what AI answer blocks would cost before building one
Google, Bing and Brave all put a generated answer above their results now,
and none of it is being read. Investigated against castle's Chrome; nothing
in src/ changed, because the cost turned out to be worth deciding on rather
than absorbing.
Two things make it expensive. Google's AI Overview exists only on the
classic layout — udm=14 is the "Web" tab, and having no overview is part of
what that tab is — so enabling it means giving back the clean page the
extractor was built around. And all three stream in after the results are
already parseable: Google settles ~2.3s in, Brave was still growing at 5s,
against an extractor that returns at ~0.5s. The wait is often wasted, since
the blocks are query-dependent — no Google overview for "how does postgres
MVCC vacuum work" after 15s, no Bing answer for "who founded anthropic".
Selectors, the traps behind them (#m-x-content briefly holds ~29k
characters of unrelated page; Google's "Show more" hides only the
disclaimer), the measured timings, three possible designs and their
tradeoffs are all in the doc, so this does not have to be re-measured if
the answer turns out to be yes.
Diffstat (limited to 'castle-search/test/engine-target.test.ts')
| -rw-r--r-- | castle-search/test/engine-target.test.ts | 89 |
1 files changed, 89 insertions, 0 deletions
diff --git a/castle-search/test/engine-target.test.ts b/castle-search/test/engine-target.test.ts new file mode 100644 index 0000000..e5ced6d --- /dev/null +++ b/castle-search/test/engine-target.test.ts @@ -0,0 +1,89 @@ +/** + * Engine selection, and the promise that an unsupported recency window is + * refused rather than dropped. + */ + +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { ENGINE_ENV_VAR, DEFAULT_ENGINE, resolveEngine } from "../src/engine-target.ts"; +import { enginesSupportingRecency, parseEngineId } from "../src/engines.ts"; +import { selectEngine } from "../src/search.ts"; + +const ok = (r: ReturnType<typeof resolveEngine>) => { + assert.equal(r.kind, "ok"); + if (r.kind !== "ok") throw new Error("unreachable"); + return r; +}; + +test("nothing configured means the built-in default", () => { + const r = ok(resolveEngine(null, {}, null)); + assert.equal(r.id, DEFAULT_ENGINE); + assert.equal(r.source, "default"); +}); + +test("precedence is parameter, then env, then stored", () => { + assert.equal(ok(resolveEngine(null, {}, "bing")).id, "bing"); + assert.equal(ok(resolveEngine(null, { [ENGINE_ENV_VAR]: "brave" }, "bing")).id, "brave"); + // The parameter beating the env var is the deliberate difference from the CDP + // target: it is what lets the agent fall back when an engine is blocked. + const r = ok(resolveEngine("duckduckgo", { [ENGINE_ENV_VAR]: "brave" }, "bing")); + assert.equal(r.id, "duckduckgo"); + assert.equal(r.source, "parameter"); +}); + +test("aliases resolve, including the short ones a human would type", () => { + for (const [raw, expected] of [ + ["ddg", "duckduckgo"], + ["duck", "duckduckgo"], + ["DuckDuckGo", "duckduckgo"], + [" bing ", "bing"], + ["g", "google"], + ["BRAVE", "brave"], + ] as const) { + assert.equal(parseEngineId(raw), expected, raw); + } +}); + +test("an unknown name is reported, never quietly skipped for the next source", () => { + // A typo in the env var must not silently search Google instead. + const r = resolveEngine(null, { [ENGINE_ENV_VAR]: "gooogle" }, "bing"); + assert.equal(r.kind, "invalid"); + if (r.kind !== "invalid") throw new Error("unreachable"); + assert.equal(r.source, "env"); + assert.equal(r.raw, "gooogle"); +}); + +test("blank values fall through rather than counting as a choice", () => { + assert.equal(ok(resolveEngine(" ", { [ENGINE_ENV_VAR]: " " }, "bing")).id, "bing"); +}); + +test("an unsupported recency window is refused, and names engines that support it", async () => { + // Bing has no year window and Brave has no time filter; both must say so + // rather than returning unfiltered results that look filtered. + for (const engine of ["bing", "brave"] as const) { + const error = await selectEngine({ query: "q", numResults: 5, recency: "year", engine }).then( + () => null, + (e: Error) => e, + ); + assert.ok(error, `${engine} should have refused`); + assert.equal(error.name, "RecencyUnsupportedError", engine); + for (const alt of enginesSupportingRecency("year")) assert.match(error.message, new RegExp(alt), engine); + } +}); + +test("windows an engine does support are accepted", async () => { + for (const [engine, recency] of [ + ["google", "year"], + ["duckduckgo", "year"], + ["bing", "month"], + ] as const) { + const chosen = await selectEngine({ query: "q", numResults: 5, recency, engine }); + assert.equal(chosen.engine.id, engine); + } +}); + +test("brave with no recency is fine — the refusal is about the window, not the engine", async () => { + const chosen = await selectEngine({ query: "q", numResults: 5, engine: "brave" }); + assert.equal(chosen.engine.id, "brave"); +}); |
