From b03415aafeea985561f3980f8b484173530a05f8 Mon Sep 17 00:00:00 2001 From: Igor Soarez Date: Wed, 18 Nov 2026 04:55:45 +0000 Subject: castle-search: search X over HTTP, through the browser's signed-in session POST /x-search reads x.com/search in a background tab: posts with their link, author, time, text, counts and quoted post, scrolling for up to 20. GET /openapi.json describes it for Open WebUI's tool servers. Signed out, an X error page or a stuck page are errors, never an empty list. --- castle-search/README.md | 13 +++ castle-search/src/http.ts | 107 ++++++++++++++++++- castle-search/src/search.ts | 24 +++++ castle-search/src/x-search.ts | 229 ++++++++++++++++++++++++++++++++++++++++ castle-search/test/http.test.ts | 51 +++++++++ 5 files changed, 419 insertions(+), 5 deletions(-) create mode 100644 castle-search/src/x-search.ts diff --git a/castle-search/README.md b/castle-search/README.md index 7e4fd5c..0e45aaf 100644 --- a/castle-search/README.md +++ b/castle-search/README.md @@ -47,6 +47,19 @@ switch engines, because this protocol has no field for one. There is no authentication: bind it to loopback (the default) or a trusted address. `GET /health` answers `{"status": true}`. +### X search + +`POST /x-search` searches X (Twitter) instead: `{"query": "from:nasa", "count": +10, "sort": "latest"}` in, `{"query", "sort", "posts": [...]}` out, each post +with its link, author, time, text, counts and any quoted post. The query is X's +own search syntax. It needs the browser signed in to X — X serves search to +nobody else — and it says so when it is not, rather than returning nothing. +`GET /openapi.json` describes it, so Open WebUI can offer it as a tool. Free per +search, where xAI's own X search charges per post returned. + +X keeps only the posts near the screen in the page, so it scrolls and collects +by link, up to 20 posts. A search takes 4–7 s. + ## Engines Four, with independent indexes and independent rate limits. That last part is diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts index 31b1eaa..17c8c70 100644 --- a/castle-search/src/http.ts +++ b/castle-search/src/http.ts @@ -19,12 +19,18 @@ * It has no authentication, so bind it to loopback, as the default does, or to * an address only trusted peers reach. Each search opens and closes one * background tab in a browser someone may be looking at. + * + * POST /x-search searches X instead (x-search.ts), through the browser's + * signed-in X session, and GET /openapi.json describes it, so that Open WebUI + * can offer it as a tool: `{ "query", "count", "sort" }` in, `{ "query", "sort", + * "posts": [...] }` out, 502 with `{ "error" }` on failure. */ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { messageOf } from "./errors.ts"; import { disconnect, search, type SearchOutcome, type SearchRequest } from "./search.ts"; +import { xSearch, type XSearchOutcome, type XSearchRequest } from "./x-search.ts"; export const DEFAULT_LISTEN = "127.0.0.1:11158"; const MAX_BODY_BYTES = 64 * 1024; @@ -68,6 +74,85 @@ export const handleSearch = async (raw: string, run: Searcher, signal: AbortSign } }; +export type XSearcher = (request: XSearchRequest, signal: AbortSignal) => Promise; + +const X_DEFAULT_COUNT = 10; + +/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */ +export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise => { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return { status: 400, body: { error: "request body is not JSON" } }; + } + const { query, count, sort } = (parsed ?? {}) as { query?: unknown; count?: unknown; sort?: unknown }; + if (typeof query !== "string" || query.trim() === "") { + return { status: 400, body: { error: "query must be a non-empty string" } }; + } + if (sort !== undefined && sort !== null && sort !== "latest" && sort !== "top") { + return { status: 400, body: { error: 'sort must be "latest" or "top"' } }; + } + const n = typeof count === "number" && Number.isFinite(count) ? Math.trunc(count) : X_DEFAULT_COUNT; + try { + const outcome = await run( + { query: query.trim(), count: Math.min(MAX_COUNT, Math.max(1, n)), sort: sort ?? "latest" }, + signal, + ); + return { status: 200, body: { query: outcome.query, sort: outcome.sort, posts: outcome.posts } }; + } catch (e) { + return { status: 502, body: { error: messageOf(e) } }; + } +}; + +/** + * The OpenAPI description Open WebUI reads to offer /x-search as a tool. Its + * descriptions are what the model sees, so they say what the tool is for and + * how to write a query. + */ +export const OPENAPI = { + openapi: "3.1.0", + info: { + title: "castle X search", + version: "1.0.0", + description: "Searches posts on X (Twitter) through a signed-in browser. Free per search.", + }, + paths: { + "/x-search": { + post: { + operationId: "search_x", + summary: "Search posts on X (Twitter)", + description: + "Search X for posts, newest first by default. Use it for anything on X: what someone posted, " + + "reactions to news, posts about a topic. The query uses X's search syntax: from:handle, to:handle, " + + '"exact phrase", OR, -word, since:YYYY-MM-DD, until:YYYY-MM-DD, min_faves:N, filter:media, ' + + "-filter:replies, lang:en. A handle is written without @ after from:. Each post comes with its " + + "link: cite it. Text cut short by X is marked truncated.", + requestBody: { + required: true, + content: { + "application/json": { + schema: { + type: "object", + required: ["query"], + properties: { + query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" }, + count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" }, + sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" }, + }, + }, + }, + }, + }, + responses: { + "200": { description: "Matching posts", content: { "application/json": { schema: { type: "object" } } } }, + "502": { description: "The search failed; error says why", content: { "application/json": { schema: { type: "object" } } } }, + }, + }, + }, + }, +} as const; + const readBody = (req: IncomingMessage): Promise => new Promise((resolve, reject) => { let size = 0; @@ -110,8 +195,13 @@ const main = (): void => { send(res, { status: 200, body: { status: true } }); return; } - if (req.method !== "POST" || (path !== "/" && path !== "/search")) { - send(res, { status: 404, body: { error: "POST a search to / or /search" } }); + if (req.method === "GET" && path === "/openapi.json") { + send(res, { status: 200, body: OPENAPI }); + return; + } + const isX = path === "/x-search"; + if (req.method !== "POST" || (path !== "/" && path !== "/search" && !isX)) { + send(res, { status: 404, body: { error: "POST a search to / or /search, or an X search to /x-search" } }); return; } // Stop the search, and free its tab, if the client goes away first. @@ -127,9 +217,16 @@ const main = (): void => { return; } const started = Date.now(); - const reply = await handleSearch(raw, runSearch, controller.signal); - const outcome = reply.status === 200 ? `${(reply.body as unknown[]).length} results` : `error ${reply.status}`; - console.log(`search ${outcome} in ${Date.now() - started} ms`); + const reply = isX + ? await handleXSearch(raw, xSearch, controller.signal) + : await handleSearch(raw, runSearch, controller.signal); + const outcome = + reply.status !== 200 + ? `error ${reply.status}` + : isX + ? `${(reply.body as { posts: unknown[] }).posts.length} posts` + : `${(reply.body as unknown[]).length} results`; + console.log(`${isX ? "x-search" : "search"} ${outcome} in ${Date.now() - started} ms`); if (reply.status !== 200) console.log(` ${(reply.body as { error: string }).error}`); send(res, reply); }); diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts index b5effb8..0178349 100644 --- a/castle-search/src/search.ts +++ b/castle-search/src/search.ts @@ -393,3 +393,27 @@ export const resolveLink = async (url: string, signal: AbortSignal): Promise /^https?:\/\/([a-z0-9-]+\.)*google\.(?:com?\.)?[a-z]{2,3}\//.test(url); + +/** + * Runs `fn` on a fresh background tab of the shared browser, under the same + * concurrency cap and connection as a search, and closes the tab on every path. + * For page work that is not an engine search, such as x-search.ts. + */ +export const withPage = async ( + signal: AbortSignal, + fn: (page: PageSession, endpoint: string) => Promise, +): Promise => { + const release = await acquire(signal); + try { + const conn = await ensureConnection(signal); + const endpoint = describeCdpTarget(conn.target); + const page = await PageSession.open(conn, signal); + try { + return await fn(page, endpoint); + } finally { + await page.close(); + } + } finally { + release(); + } +}; diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts new file mode 100644 index 0000000..6ea086c --- /dev/null +++ b/castle-search/src/x-search.ts @@ -0,0 +1,229 @@ +/** + * Search X (x.com) through the browser's own signed-in session. + * + * X serves search only to a signed-in session and renders it from JavaScript, + * so the page is the only interface there is: open x.com/search in a background + * tab, wait for posts to render, read them, scroll for more, close the tab. + * Free per search, unlike xAI's x_search ($5 per 1,000 posts returned). + * + * The query is X's own search syntax, passed through untouched: from:handle, + * to:handle, since:2026-11-01, until:..., min_faves:100, filter:media, + * -filter:replies, lang:en, "exact phrase", OR. + * + * Every failure throws, as for the engine searches (errors.ts): signed out, an + * X error page, or nothing rendering are reported as such, never as "no posts". + * Only X's own "No results" page is an empty answer, and even that is reported + * as an error naming the query, since an over-narrow query is the usual cause. + */ + +import { BrowserUnavailableError, NoResultsError } from "./errors.ts"; +import { withPage } from "./search.ts"; + +export type XSort = "latest" | "top"; + +export type XPost = { + url: string; + author: string; + handle: string; + /** ISO 8601, from the post's