aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-11-17 08:46:39 +0000
committerIgor Soarez <igor@soarez.org>2026-11-17 08:46:39 +0000
commit4c1f9b4ea943d03e9610f13344c372e630db5b9d (patch)
tree95bf9424161c98af6d384ecbc73379379373e546
parent561f0ee076c684a79fd5671af1bad0894f33e4de (diff)
castle-search: serve searches over HTTP for clients that are not piHEADmaster
src/http.ts speaks Open WebUI's external web search protocol. A failed search answers 502 with its reason, never an empty list, so a captcha cannot pass for nothing found.
-rw-r--r--castle-search/README.md18
-rw-r--r--castle-search/src/http.ts147
-rw-r--r--castle-search/test/http.test.ts67
3 files changed, 232 insertions, 0 deletions
diff --git a/castle-search/README.md b/castle-search/README.md
index f3fb5e9..7e4fd5c 100644
--- a/castle-search/README.md
+++ b/castle-search/README.md
@@ -29,6 +29,24 @@ the model picks.
It returns titles, URLs and snippets — **search results only**. It does not fetch
the linked pages; follow up with a read/fetch tool for full content.
+## Over HTTP, for clients that are not pi
+
+`src/http.ts` serves the same search as a small HTTP service, in Open WebUI's
+"external" web search protocol: POST `{"query": "...", "count": 5}` to `/`,
+get back `[{"link", "title", "snippet"}]`.
+
+```
+CCS_SEARCH_ENGINE=duckduckgo node --experimental-strip-types src/http.ts 127.0.0.1:11158
+```
+
+Engine and browser are chosen as for the pi tool. A failed search answers
+HTTP 502 with `{"error": "<why>"}`, never an empty list, so a captcha cannot
+pass for "nothing found"; zero hits is reported as an error too, since a
+rate-limited engine serves exactly that. The message adds that the caller cannot
+switch engines, because this protocol has no field for one. There is no
+authentication: bind it to loopback (the default) or a trusted address.
+`GET /health` answers `{"status": true}`.
+
## Engines
Four, with independent indexes and independent rate limits. That last part is
diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts
new file mode 100644
index 0000000..31b1eaa
--- /dev/null
+++ b/castle-search/src/http.ts
@@ -0,0 +1,147 @@
+/**
+ * castle_cdp_search as a plain HTTP service, for clients that are not pi.
+ *
+ * It speaks Open WebUI's "external" web search protocol: POST a JSON body
+ * `{ "query": "...", "count": 5 }`, get back `[{ "link", "title", "snippet" }]`.
+ * Any client that can send that can use it.
+ *
+ * node --experimental-strip-types src/http.ts [host:port] (default 127.0.0.1:11158)
+ *
+ * The engine and the browser are chosen exactly as for the pi tool:
+ * CCS_SEARCH_ENGINE and BU_CDP_HTTP, then the stored choices, then the defaults.
+ *
+ * A failed search is answered with HTTP 502 and `{ "error": "<why>" }`, never
+ * with an empty list: an empty list reads as "the web has nothing on this", and
+ * a captcha or a rate limit is not that. A search that genuinely finds nothing
+ * is a failure too here (NoResultsError), because an engine that is quietly
+ * rate-limiting serves exactly that page, and the message says which it may be.
+ *
+ * It has no authentication, so bind it to loopback, as the default does, or to
+ * an address only trusted peers reach. Each search opens and closes one
+ * background tab in a browser someone may be looking at.
+ */
+
+import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
+
+import { messageOf } from "./errors.ts";
+import { disconnect, search, type SearchOutcome, type SearchRequest } from "./search.ts";
+
+export const DEFAULT_LISTEN = "127.0.0.1:11158";
+const MAX_BODY_BYTES = 64 * 1024;
+const DEFAULT_COUNT = 5;
+const MAX_COUNT = 20;
+
+// Said after every error. The castle-search messages tell the caller to retry
+// with another `engine`, which a client of this protocol cannot do: it sends a
+// query and a count, nothing else.
+const ENGINE_NOTE =
+ "This search service cannot switch engines per call. Tell the user web search failed and why; " +
+ "the operator can change the engine (CCS_SEARCH_ENGINE) or clear the check in the browser.";
+
+export type Reply = { status: number; body: unknown };
+
+export type Searcher = (request: SearchRequest, signal: AbortSignal) => Promise<SearchOutcome>;
+
+/** Everything but the socket: parse a request body, run the search, shape the reply. */
+export const handleSearch = async (raw: string, run: Searcher, signal: AbortSignal): Promise<Reply> => {
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(raw);
+ } catch {
+ return { status: 400, body: { error: "request body is not JSON" } };
+ }
+ const { query, count } = (parsed ?? {}) as { query?: unknown; count?: unknown };
+ if (typeof query !== "string" || query.trim() === "") {
+ return { status: 400, body: { error: "query must be a non-empty string" } };
+ }
+ const n = typeof count === "number" && Number.isFinite(count) ? Math.trunc(count) : DEFAULT_COUNT;
+ const numResults = Math.min(MAX_COUNT, Math.max(1, n));
+
+ try {
+ const outcome = await run({ query: query.trim(), numResults }, signal);
+ return {
+ status: 200,
+ body: outcome.results.map((r) => ({ link: r.url, title: r.title, snippet: r.snippet })),
+ };
+ } catch (e) {
+ return { status: 502, body: { error: `${messageOf(e)} ${ENGINE_NOTE}` } };
+ }
+};
+
+const readBody = (req: IncomingMessage): Promise<string> =>
+ new Promise((resolve, reject) => {
+ let size = 0;
+ const chunks: Buffer[] = [];
+ req.on("data", (chunk: Buffer) => {
+ size += chunk.length;
+ if (size > MAX_BODY_BYTES) {
+ reject(new Error("request body too large"));
+ req.destroy();
+ return;
+ }
+ chunks.push(chunk);
+ });
+ req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
+ req.on("error", reject);
+ });
+
+const send = (res: ServerResponse, reply: Reply): void => {
+ res.writeHead(reply.status, { "Content-Type": "application/json; charset=utf-8" });
+ res.end(JSON.stringify(reply.body));
+};
+
+export const parseListen = (value: string): { host: string; port: number } => {
+ const at = value.lastIndexOf(":");
+ const host = value.slice(0, at);
+ const port = Number(value.slice(at + 1));
+ if (at <= 0 || !Number.isInteger(port) || port < 1 || port > 65535) {
+ throw new Error(`listen address must be host:port, got "${value}"`);
+ }
+ return { host, port };
+};
+
+const main = (): void => {
+ const { host, port } = parseListen(process.argv[2] ?? DEFAULT_LISTEN);
+ const runSearch: Searcher = (request, signal) => search(request, signal, () => {});
+
+ const server = createServer(async (req, res) => {
+ const path = (req.url ?? "/").split("?")[0];
+ if (req.method === "GET" && path === "/health") {
+ send(res, { status: 200, body: { status: true } });
+ return;
+ }
+ if (req.method !== "POST" || (path !== "/" && path !== "/search")) {
+ send(res, { status: 404, body: { error: "POST a search to / or /search" } });
+ return;
+ }
+ // Stop the search, and free its tab, if the client goes away first.
+ const controller = new AbortController();
+ res.on("close", () => {
+ if (!res.writableEnded) controller.abort();
+ });
+ let raw: string;
+ try {
+ raw = await readBody(req);
+ } catch (e) {
+ send(res, { status: 413, body: { error: messageOf(e) } });
+ return;
+ }
+ const started = Date.now();
+ const reply = await handleSearch(raw, runSearch, controller.signal);
+ const outcome = reply.status === 200 ? `${(reply.body as unknown[]).length} results` : `error ${reply.status}`;
+ console.log(`search ${outcome} in ${Date.now() - started} ms`);
+ if (reply.status !== 200) console.log(` ${(reply.body as { error: string }).error}`);
+ send(res, reply);
+ });
+
+ const stop = (): void => {
+ disconnect();
+ server.close(() => process.exit(0));
+ };
+ process.on("SIGTERM", stop);
+ process.on("SIGINT", stop);
+
+ server.listen(port, host, () => console.log(`castle-search HTTP listening on ${host}:${port}`));
+};
+
+if (import.meta.url === `file://${process.argv[1]}`) main();
diff --git a/castle-search/test/http.test.ts b/castle-search/test/http.test.ts
new file mode 100644
index 0000000..6b3045b
--- /dev/null
+++ b/castle-search/test/http.test.ts
@@ -0,0 +1,67 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { NoResultsError, SearchChallengeError } from "../src/errors.ts";
+import { handleSearch, parseListen, type Searcher } from "../src/http.ts";
+
+const signal = new AbortController().signal;
+
+const returning = (results: { title: string; url: string; snippet: string }[]): Searcher =>
+ async () => ({ results, searchUrl: "", finalUrl: "", endpoint: "", engine: {} as never, engineSource: "" });
+
+const throwing = (e: Error): Searcher => async () => {
+ throw e;
+};
+
+test("results come back in Open WebUI's shape", async () => {
+ const reply = await handleSearch(
+ JSON.stringify({ query: "x", count: 2 }),
+ returning([{ title: "T", url: "https://e.com/", snippet: "S" }]),
+ signal,
+ );
+ assert.equal(reply.status, 200);
+ assert.deepEqual(reply.body, [{ link: "https://e.com/", title: "T", snippet: "S" }]);
+});
+
+test("count is clamped and defaulted", async () => {
+ const seen: number[] = [];
+ const run: Searcher = async (request) => {
+ seen.push(request.numResults);
+ return returning([])(request, signal);
+ };
+ await handleSearch(JSON.stringify({ query: "x", count: 500 }), run, signal);
+ await handleSearch(JSON.stringify({ query: "x", count: 0 }), run, signal);
+ await handleSearch(JSON.stringify({ query: "x" }), run, signal);
+ assert.deepEqual(seen, [20, 1, 5]);
+});
+
+test("a challenge is an error with its reason, never an empty list", async () => {
+ const reply = await handleSearch(
+ JSON.stringify({ query: "x" }),
+ throwing(new SearchChallengeError({
+ challenge: "captcha", pageUrl: "https://duckduckgo.com/", endpoint: "e", engine: "DuckDuckGo", alternatives: ["bing"],
+ })),
+ signal,
+ );
+ assert.equal(reply.status, 502);
+ assert.match((reply.body as { error: string }).error, /DuckDuckGo is blocked/);
+ assert.match((reply.body as { error: string }).error, /cannot switch engines/);
+});
+
+test("no results is an error too", async () => {
+ const reply = await handleSearch(JSON.stringify({ query: "x" }), throwing(new NoResultsError("nothing")), signal);
+ assert.equal(reply.status, 502);
+});
+
+test("bad requests are refused", async () => {
+ const run = returning([]);
+ assert.equal((await handleSearch("not json", run, signal)).status, 400);
+ assert.equal((await handleSearch(JSON.stringify({ query: " " }), run, signal)).status, 400);
+ assert.equal((await handleSearch(JSON.stringify({}), run, signal)).status, 400);
+});
+
+test("listen addresses parse", () => {
+ assert.deepEqual(parseListen("127.0.0.1:11158"), { host: "127.0.0.1", port: 11158 });
+ assert.throws(() => parseListen("11158"));
+ assert.throws(() => parseListen("h:99999"));
+});