/** * castle_cdp_search as a plain HTTP service, for clients that are not pi. * * It speaks Open WebUI's "external" web search protocol: POST a JSON body * `{ "query": "...", "count": 5 }`, get back `[{ "link", "title", "snippet" }]`. * Any client that can send that can use it. * * node --experimental-strip-types src/http.ts [host:port] (default 127.0.0.1:11158) * * The engine and the browser are chosen exactly as for the pi tool: * CCS_SEARCH_ENGINE and BU_CDP_HTTP, then the stored choices, then the defaults. * * A failed search is answered with HTTP 502 and `{ "error": "" }`, never * with an empty list: an empty list reads as "the web has nothing on this", and * a captcha or a rate limit is not that. A search that genuinely finds nothing * is a failure too here (NoResultsError), because an engine that is quietly * rate-limiting serves exactly that page, and the message says which it may be. * * It has no authentication, so bind it to loopback, as the default does, or to * an address only trusted peers reach. Each search opens and closes one * background tab in a browser someone may be looking at. * * POST /x-search, /x-post, /x-user and /x-trends read X instead (x-search.ts), * through the browser's signed-in X session, and GET /openapi.json describes * them, so that Open WebUI can offer them as tools ("X.com"). Each answers 502 * with `{ "error" }` on failure. */ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { messageOf } from "./errors.ts"; import { disconnect, search, type SearchOutcome, type SearchRequest } from "./search.ts"; import { handleOf, postIdOf, TREND_TABS, type XPostOutcome, type XPostRequest, xPost, xSearch, type XSearchOutcome, type XSearchRequest, type XTrendsOutcome, type XTrendTab, xTrends, xUser, type XUserOutcome, type XUserRequest, } from "./x-search.ts"; export const DEFAULT_LISTEN = "127.0.0.1:11158"; const MAX_BODY_BYTES = 64 * 1024; const DEFAULT_COUNT = 5; const MAX_COUNT = 20; // Said after every error. The castle-search messages tell the caller to retry // with another `engine`, which a client of this protocol cannot do: it sends a // query and a count, nothing else. const ENGINE_NOTE = "This search service cannot switch engines per call. Tell the user web search failed and why; " + "the operator can change the engine (CCS_SEARCH_ENGINE) or clear the check in the browser."; export type Reply = { status: number; body: unknown }; export type Searcher = (request: SearchRequest, signal: AbortSignal) => Promise; /** Everything but the socket: parse a request body, run the search, shape the reply. */ export const handleSearch = async (raw: string, run: Searcher, signal: AbortSignal): Promise => { let parsed: unknown; try { parsed = JSON.parse(raw); } catch { return { status: 400, body: { error: "request body is not JSON" } }; } const { query, count } = (parsed ?? {}) as { query?: unknown; count?: unknown }; if (typeof query !== "string" || query.trim() === "") { return { status: 400, body: { error: "query must be a non-empty string" } }; } const n = typeof count === "number" && Number.isFinite(count) ? Math.trunc(count) : DEFAULT_COUNT; const numResults = Math.min(MAX_COUNT, Math.max(1, n)); try { const outcome = await run({ query: query.trim(), numResults }, signal); return { status: 200, body: outcome.results.map((r) => ({ link: r.url, title: r.title, snippet: r.snippet })), }; } catch (e) { return { status: 502, body: { error: `${messageOf(e)} ${ENGINE_NOTE}` } }; } }; export type XSearcher = (request: XSearchRequest, signal: AbortSignal) => Promise; export type XPostReader = (request: XPostRequest, signal: AbortSignal) => Promise; export type XUserReader = (request: XUserRequest, signal: AbortSignal) => Promise; export type XTrendsReader = (tab: XTrendTab, signal: AbortSignal) => Promise; const X_DEFAULT_COUNT = 10; const jsonBody = (raw: string): Record | Reply => { try { const parsed: unknown = JSON.parse(raw || "{}"); if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as Record; } catch { // fall through } return { status: 400, body: { error: "request body is not a JSON object" } }; }; const isReply = (v: unknown): v is Reply => typeof (v as Reply).status === "number" && "body" in (v as Reply); const countOf = (v: unknown, fallback: number, min = 1): number => { const n = typeof v === "number" && Number.isFinite(v) ? Math.trunc(v) : fallback; return Math.min(MAX_COUNT, Math.max(min, n)); }; const answer = async (work: () => Promise): Promise => { try { return { status: 200, body: await work() }; } catch (e) { return { status: 502, body: { error: messageOf(e) } }; } }; /** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */ export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise => { const body = jsonBody(raw); if (isReply(body)) return body; const { query, count, sort } = body; if (typeof query !== "string" || query.trim() === "") { return { status: 400, body: { error: "query must be a non-empty string" } }; } if (sort !== undefined && sort !== null && sort !== "latest" && sort !== "top") { return { status: 400, body: { error: 'sort must be "latest" or "top"' } }; } return answer(async () => { const outcome = await run({ query: query.trim(), count: countOf(count, X_DEFAULT_COUNT), sort: sort ?? "latest" }, signal); return { query: outcome.query, sort: outcome.sort, posts: outcome.posts }; }); }; /** POST /x-post: `{ url, replies? }` in, `{ post, thread, replies }` out. */ export const handleXPost = async (raw: string, run: XPostReader, signal: AbortSignal): Promise => { const body = jsonBody(raw); if (isReply(body)) return body; if (typeof body["url"] !== "string" || !postIdOf(body["url"])) { return { status: 400, body: { error: "url must be a link to a post on X (x.com//status/) or its id" } }; } const url = body["url"]; return answer(() => run({ url, replies: countOf(body["replies"], X_DEFAULT_COUNT, 0) }, signal)); }; /** POST /x-user: `{ handle, count? }` in, `{ profile, posts }` out. */ export const handleXUser = async (raw: string, run: XUserReader, signal: AbortSignal): Promise => { const body = jsonBody(raw); if (isReply(body)) return body; if (typeof body["handle"] !== "string" || !handleOf(body["handle"])) { return { status: 400, body: { error: "handle must be an X handle, such as nasa or @nasa" } }; } const handle = body["handle"]; return answer(() => run({ handle, count: countOf(body["count"], X_DEFAULT_COUNT, 0) }, signal)); }; /** POST /x-trends: `{ tab? }` in, `{ tab, trends }` out. */ export const handleXTrends = async (raw: string, run: XTrendsReader, signal: AbortSignal): Promise => { const body = jsonBody(raw); if (isReply(body)) return body; const tab = body["tab"] ?? "trending"; if (!TREND_TABS.includes(tab as XTrendTab)) { return { status: 400, body: { error: `tab must be one of ${TREND_TABS.join(", ")}` } }; } return answer(() => run(tab as XTrendTab, signal)); }; const jsonSchema = (properties: Record, required: string[]) => ({ required: true, content: { "application/json": { schema: { type: "object", required, properties } } }, }); const RESPONSES = { "200": { description: "What X shows", content: { "application/json": { schema: { type: "object" } } } }, "502": { description: "X could not be read; error says why", content: { "application/json": { schema: { type: "object" } } } }, }; /** * The OpenAPI description Open WebUI reads to offer the X reads as tools. Its * descriptions are what the model sees, so they say what each is for. */ export const OPENAPI = { openapi: "3.1.0", info: { title: "X.com", version: "2.0.0", description: "Reads X (Twitter) through a signed-in browser: search, posts with replies, profiles, trends. Free.", }, paths: { "/x-search": { post: { operationId: "search_x", summary: "Search posts on X (Twitter)", description: "Search X for posts, newest first by default. Use it for anything on X: what someone posted, " + "reactions to news, posts about a topic. The query uses X's search syntax: from:handle, to:handle, " + '"exact phrase", OR, -word, since:YYYY-MM-DD, until:YYYY-MM-DD, min_faves:N, filter:media, ' + "-filter:replies, lang:en. A handle is written without @ after from:. Each post comes with its " + "link: cite it. A post marked truncated was cut short by X: read_x_post gives it whole. Photos " + "come with their image link and alt text.", requestBody: jsonSchema( { query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" }, count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" }, sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" }, }, ["query"], ), responses: RESPONSES, }, }, "/x-post": { post: { operationId: "read_x_post", summary: "Read one post on X, whole, with its thread and replies", description: "Read a post on X from its link: its full text (never truncated), photos with their image link " + "and alt text, its link card, the posts above it when it is a reply (thread, oldest first), and " + "the top replies under it. Use it to read a long post, a thread, or what people answered.", requestBody: jsonSchema( { url: { type: "string", description: "The post's link, x.com//status/, or the id" }, replies: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many replies" }, }, ["url"], ), responses: RESPONSES, }, }, "/x-user": { post: { operationId: "read_x_profile", summary: "Read an X profile and its latest posts", description: "Read an X account's profile: name, bio, location, website, when it joined, exact follower, " + "following and post counts, then its latest posts as its profile shows them (a pinned post and " + "reposts included, marked in context).", requestBody: jsonSchema( { handle: { type: "string", description: "The account's handle, such as nasa or @nasa" }, count: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many of its posts" }, }, ["handle"], ), responses: RESPONSES, }, }, "/x-trends": { post: { operationId: "x_trends", summary: "What is trending on X now", description: "List what X shows as trending now, for the signed-in account's region (the United Kingdom). " + "Each trend has a query: pass it to search_x for the posts.", requestBody: jsonSchema( { tab: { type: "string", enum: [...TREND_TABS], default: "trending", description: "Which explore tab" } }, [], ), responses: RESPONSES, }, }, }, } as const; const readBody = (req: IncomingMessage): Promise => new Promise((resolve, reject) => { let size = 0; const chunks: Buffer[] = []; req.on("data", (chunk: Buffer) => { size += chunk.length; if (size > MAX_BODY_BYTES) { reject(new Error("request body too large")); req.destroy(); return; } chunks.push(chunk); }); req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); req.on("error", reject); }); const send = (res: ServerResponse, reply: Reply): void => { res.writeHead(reply.status, { "Content-Type": "application/json; charset=utf-8" }); res.end(JSON.stringify(reply.body)); }; export const parseListen = (value: string): { host: string; port: number } => { const at = value.lastIndexOf(":"); const host = value.slice(0, at); const port = Number(value.slice(at + 1)); if (at <= 0 || !Number.isInteger(port) || port < 1 || port > 65535) { throw new Error(`listen address must be host:port, got "${value}"`); } return { host, port }; }; const main = (): void => { const { host, port } = parseListen(process.argv[2] ?? DEFAULT_LISTEN); const runSearch: Searcher = (request, signal) => search(request, signal, () => {}); const server = createServer(async (req, res) => { const path = (req.url ?? "/").split("?")[0]; if (req.method === "GET" && path === "/health") { send(res, { status: 200, body: { status: true } }); return; } if (req.method === "GET" && path === "/openapi.json") { send(res, { status: 200, body: OPENAPI }); return; } const xRoutes: Record Promise> = { "/x-search": (raw, signal) => handleXSearch(raw, xSearch, signal), "/x-post": (raw, signal) => handleXPost(raw, xPost, signal), "/x-user": (raw, signal) => handleXUser(raw, xUser, signal), "/x-trends": (raw, signal) => handleXTrends(raw, xTrends, signal), }; const xRoute = xRoutes[path ?? ""]; if (req.method !== "POST" || (path !== "/" && path !== "/search" && !xRoute)) { send(res, { status: 404, body: { error: "POST a search to / or /search, or an X read to /x-search, /x-post, /x-user or /x-trends" } }); return; } // Stop the search, and free its tab, if the client goes away first. const controller = new AbortController(); res.on("close", () => { if (!res.writableEnded) controller.abort(); }); let raw: string; try { raw = await readBody(req); } catch (e) { send(res, { status: 413, body: { error: messageOf(e) } }); return; } const started = Date.now(); const reply = xRoute ? await xRoute(raw, controller.signal) : await handleSearch(raw, runSearch, controller.signal); const outcome = reply.status !== 200 ? `error ${reply.status}` : xRoute ? "ok" : `${(reply.body as unknown[]).length} results`; console.log(`${xRoute ? path!.slice(1) : "search"} ${outcome} in ${Date.now() - started} ms`); if (reply.status !== 200) console.log(` ${(reply.body as { error: string }).error}`); send(res, reply); }); const stop = (): void => { disconnect(); server.close(() => process.exit(0)); }; process.on("SIGTERM", stop); process.on("SIGINT", stop); server.listen(port, host, () => console.log(`castle-search HTTP listening on ${host}:${port}`)); }; if (import.meta.url === `file://${process.argv[1]}`) main();