aboutsummaryrefslogtreecommitdiff
path: root/castle-search
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-11-18 04:55:45 +0000
committerIgor Soarez <igor@soarez.org>2026-11-18 04:55:45 +0000
commitb03415aafeea985561f3980f8b484173530a05f8 (patch)
tree3748f62ae52a5c4d7906ec3268bb8b2863a5114a /castle-search
parent4c1f9b4ea943d03e9610f13344c372e630db5b9d (diff)
castle-search: search X over HTTP, through the browser's signed-in session
POST /x-search reads x.com/search in a background tab: posts with their link, author, time, text, counts and quoted post, scrolling for up to 20. GET /openapi.json describes it for Open WebUI's tool servers. Signed out, an X error page or a stuck page are errors, never an empty list.
Diffstat (limited to 'castle-search')
-rw-r--r--castle-search/README.md13
-rw-r--r--castle-search/src/http.ts107
-rw-r--r--castle-search/src/search.ts24
-rw-r--r--castle-search/src/x-search.ts229
-rw-r--r--castle-search/test/http.test.ts51
5 files changed, 419 insertions, 5 deletions
diff --git a/castle-search/README.md b/castle-search/README.md
index 7e4fd5c..0e45aaf 100644
--- a/castle-search/README.md
+++ b/castle-search/README.md
@@ -47,6 +47,19 @@ switch engines, because this protocol has no field for one. There is no
authentication: bind it to loopback (the default) or a trusted address.
`GET /health` answers `{"status": true}`.
+### X search
+
+`POST /x-search` searches X (Twitter) instead: `{"query": "from:nasa", "count":
+10, "sort": "latest"}` in, `{"query", "sort", "posts": [...]}` out, each post
+with its link, author, time, text, counts and any quoted post. The query is X's
+own search syntax. It needs the browser signed in to X — X serves search to
+nobody else — and it says so when it is not, rather than returning nothing.
+`GET /openapi.json` describes it, so Open WebUI can offer it as a tool. Free per
+search, where xAI's own X search charges per post returned.
+
+X keeps only the posts near the screen in the page, so it scrolls and collects
+by link, up to 20 posts. A search takes 4–7 s.
+
## Engines
Four, with independent indexes and independent rate limits. That last part is
diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts
index 31b1eaa..17c8c70 100644
--- a/castle-search/src/http.ts
+++ b/castle-search/src/http.ts
@@ -19,12 +19,18 @@
* It has no authentication, so bind it to loopback, as the default does, or to
* an address only trusted peers reach. Each search opens and closes one
* background tab in a browser someone may be looking at.
+ *
+ * POST /x-search searches X instead (x-search.ts), through the browser's
+ * signed-in X session, and GET /openapi.json describes it, so that Open WebUI
+ * can offer it as a tool: `{ "query", "count", "sort" }` in, `{ "query", "sort",
+ * "posts": [...] }` out, 502 with `{ "error" }` on failure.
*/
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { messageOf } from "./errors.ts";
import { disconnect, search, type SearchOutcome, type SearchRequest } from "./search.ts";
+import { xSearch, type XSearchOutcome, type XSearchRequest } from "./x-search.ts";
export const DEFAULT_LISTEN = "127.0.0.1:11158";
const MAX_BODY_BYTES = 64 * 1024;
@@ -68,6 +74,85 @@ export const handleSearch = async (raw: string, run: Searcher, signal: AbortSign
}
};
+export type XSearcher = (request: XSearchRequest, signal: AbortSignal) => Promise<XSearchOutcome>;
+
+const X_DEFAULT_COUNT = 10;
+
+/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */
+export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise<Reply> => {
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(raw);
+ } catch {
+ return { status: 400, body: { error: "request body is not JSON" } };
+ }
+ const { query, count, sort } = (parsed ?? {}) as { query?: unknown; count?: unknown; sort?: unknown };
+ if (typeof query !== "string" || query.trim() === "") {
+ return { status: 400, body: { error: "query must be a non-empty string" } };
+ }
+ if (sort !== undefined && sort !== null && sort !== "latest" && sort !== "top") {
+ return { status: 400, body: { error: 'sort must be "latest" or "top"' } };
+ }
+ const n = typeof count === "number" && Number.isFinite(count) ? Math.trunc(count) : X_DEFAULT_COUNT;
+ try {
+ const outcome = await run(
+ { query: query.trim(), count: Math.min(MAX_COUNT, Math.max(1, n)), sort: sort ?? "latest" },
+ signal,
+ );
+ return { status: 200, body: { query: outcome.query, sort: outcome.sort, posts: outcome.posts } };
+ } catch (e) {
+ return { status: 502, body: { error: messageOf(e) } };
+ }
+};
+
+/**
+ * The OpenAPI description Open WebUI reads to offer /x-search as a tool. Its
+ * descriptions are what the model sees, so they say what the tool is for and
+ * how to write a query.
+ */
+export const OPENAPI = {
+ openapi: "3.1.0",
+ info: {
+ title: "castle X search",
+ version: "1.0.0",
+ description: "Searches posts on X (Twitter) through a signed-in browser. Free per search.",
+ },
+ paths: {
+ "/x-search": {
+ post: {
+ operationId: "search_x",
+ summary: "Search posts on X (Twitter)",
+ description:
+ "Search X for posts, newest first by default. Use it for anything on X: what someone posted, " +
+ "reactions to news, posts about a topic. The query uses X's search syntax: from:handle, to:handle, " +
+ '"exact phrase", OR, -word, since:YYYY-MM-DD, until:YYYY-MM-DD, min_faves:N, filter:media, ' +
+ "-filter:replies, lang:en. A handle is written without @ after from:. Each post comes with its " +
+ "link: cite it. Text cut short by X is marked truncated.",
+ requestBody: {
+ required: true,
+ content: {
+ "application/json": {
+ schema: {
+ type: "object",
+ required: ["query"],
+ properties: {
+ query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" },
+ count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" },
+ sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" },
+ },
+ },
+ },
+ },
+ },
+ responses: {
+ "200": { description: "Matching posts", content: { "application/json": { schema: { type: "object" } } } },
+ "502": { description: "The search failed; error says why", content: { "application/json": { schema: { type: "object" } } } },
+ },
+ },
+ },
+ },
+} as const;
+
const readBody = (req: IncomingMessage): Promise<string> =>
new Promise((resolve, reject) => {
let size = 0;
@@ -110,8 +195,13 @@ const main = (): void => {
send(res, { status: 200, body: { status: true } });
return;
}
- if (req.method !== "POST" || (path !== "/" && path !== "/search")) {
- send(res, { status: 404, body: { error: "POST a search to / or /search" } });
+ if (req.method === "GET" && path === "/openapi.json") {
+ send(res, { status: 200, body: OPENAPI });
+ return;
+ }
+ const isX = path === "/x-search";
+ if (req.method !== "POST" || (path !== "/" && path !== "/search" && !isX)) {
+ send(res, { status: 404, body: { error: "POST a search to / or /search, or an X search to /x-search" } });
return;
}
// Stop the search, and free its tab, if the client goes away first.
@@ -127,9 +217,16 @@ const main = (): void => {
return;
}
const started = Date.now();
- const reply = await handleSearch(raw, runSearch, controller.signal);
- const outcome = reply.status === 200 ? `${(reply.body as unknown[]).length} results` : `error ${reply.status}`;
- console.log(`search ${outcome} in ${Date.now() - started} ms`);
+ const reply = isX
+ ? await handleXSearch(raw, xSearch, controller.signal)
+ : await handleSearch(raw, runSearch, controller.signal);
+ const outcome =
+ reply.status !== 200
+ ? `error ${reply.status}`
+ : isX
+ ? `${(reply.body as { posts: unknown[] }).posts.length} posts`
+ : `${(reply.body as unknown[]).length} results`;
+ console.log(`${isX ? "x-search" : "search"} ${outcome} in ${Date.now() - started} ms`);
if (reply.status !== 200) console.log(` ${(reply.body as { error: string }).error}`);
send(res, reply);
});
diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts
index b5effb8..0178349 100644
--- a/castle-search/src/search.ts
+++ b/castle-search/src/search.ts
@@ -393,3 +393,27 @@ export const resolveLink = async (url: string, signal: AbortSignal): Promise<Res
};
const isGoogleHost = (url: string): boolean => /^https?:\/\/([a-z0-9-]+\.)*google\.(?:com?\.)?[a-z]{2,3}\//.test(url);
+
+/**
+ * Runs `fn` on a fresh background tab of the shared browser, under the same
+ * concurrency cap and connection as a search, and closes the tab on every path.
+ * For page work that is not an engine search, such as x-search.ts.
+ */
+export const withPage = async <T>(
+ signal: AbortSignal,
+ fn: (page: PageSession, endpoint: string) => Promise<T>,
+): Promise<T> => {
+ const release = await acquire(signal);
+ try {
+ const conn = await ensureConnection(signal);
+ const endpoint = describeCdpTarget(conn.target);
+ const page = await PageSession.open(conn, signal);
+ try {
+ return await fn(page, endpoint);
+ } finally {
+ await page.close();
+ }
+ } finally {
+ release();
+ }
+};
diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts
new file mode 100644
index 0000000..6ea086c
--- /dev/null
+++ b/castle-search/src/x-search.ts
@@ -0,0 +1,229 @@
+/**
+ * Search X (x.com) through the browser's own signed-in session.
+ *
+ * X serves search only to a signed-in session and renders it from JavaScript,
+ * so the page is the only interface there is: open x.com/search in a background
+ * tab, wait for posts to render, read them, scroll for more, close the tab.
+ * Free per search, unlike xAI's x_search ($5 per 1,000 posts returned).
+ *
+ * The query is X's own search syntax, passed through untouched: from:handle,
+ * to:handle, since:2026-11-01, until:..., min_faves:100, filter:media,
+ * -filter:replies, lang:en, "exact phrase", OR.
+ *
+ * Every failure throws, as for the engine searches (errors.ts): signed out, an
+ * X error page, or nothing rendering are reported as such, never as "no posts".
+ * Only X's own "No results" page is an empty answer, and even that is reported
+ * as an error naming the query, since an over-narrow query is the usual cause.
+ */
+
+import { BrowserUnavailableError, NoResultsError } from "./errors.ts";
+import { withPage } from "./search.ts";
+
+export type XSort = "latest" | "top";
+
+export type XPost = {
+ url: string;
+ author: string;
+ handle: string;
+ /** ISO 8601, from the post's <time datetime>. */
+ time: string;
+ text: string;
+ /** X cut the text short behind "Show more"; the full post is at url. */
+ truncated?: true;
+ /** "@someone reposted", "Pinned" and the like. */
+ context?: string;
+ replies?: number;
+ reposts?: number;
+ likes?: number;
+ views?: number;
+ /** Count of photos and videos attached. */
+ media?: number;
+ quoted?: { author: string; handle: string; text: string };
+};
+
+export type XSearchRequest = { query: string; count: number; sort?: XSort | undefined };
+
+export type XSearchOutcome = { query: string; sort: XSort; searchUrl: string; posts: XPost[] };
+
+/** X shows ~5-10 posts per screen; scroll this many times at most for more. */
+const MAX_SCROLLS = 6;
+/** Wall clock for one search: render, scrolls and all. */
+export const X_SEARCH_TIMEOUT_MS = 25_000;
+const POLL_MS = 400;
+
+export const buildXSearchUrl = (query: string, sort: XSort): string => {
+ const params = new URLSearchParams({ q: query, src: "typed_query" });
+ // f=live is the Latest tab. Top is the default tab, no parameter.
+ if (sort === "latest") params.set("f", "live");
+ return `https://x.com/search?${params.toString()}`;
+};
+
+/** Parses X's action-bar label: "441 replies, 971 reposts, 8852 likes, 1247 bookmarks, 16029782 views". */
+export const parseMetrics = (label: string | null | undefined): Partial<Record<"replies" | "reposts" | "likes" | "views", number>> => {
+ const out: Partial<Record<"replies" | "reposts" | "likes" | "views", number>> = {};
+ for (const m of (label ?? "").matchAll(/(\d+)\s+(repl(?:y|ies)|reposts?|likes?|views?)/g)) {
+ const n = Number(m[1]);
+ const word = m[2] ?? "";
+ if (word.startsWith("repl")) out.replies = n;
+ else if (word.startsWith("repost")) out.reposts = n;
+ else if (word.startsWith("like")) out.likes = n;
+ else if (word.startsWith("view")) out.views = n;
+ }
+ return out;
+};
+
+type RawPost = Omit<XPost, "replies" | "reposts" | "likes" | "views"> & { metrics: string | null };
+
+type XProbe = {
+ url: string;
+ state: "signed-out" | "posts" | "empty" | "error" | "loading";
+ posts: RawPost[];
+ note: string;
+};
+
+/**
+ * Reads the page as it stands. Runs in the page; must stay self-contained.
+ * Ads are skipped by having no <time>: every real post has one.
+ */
+const PROBE = `(() => {
+ const url = location.href;
+ if (/^\\/(login|i\\/flow\\/login|i\\/jf\\/onboarding|account\\/access)/.test(location.pathname)) {
+ return { url, state: "signed-out", posts: [], note: location.pathname };
+ }
+ const txt = (el) => (el ? el.innerText.trim() : "");
+ const nameOf = (scope) => {
+ const lines = txt(scope.querySelector('[data-testid="User-Name"]')).split("\\n");
+ return { author: lines[0] || "", handle: lines.find((l) => l.startsWith("@")) || "" };
+ };
+ const posts = [];
+ for (const a of document.querySelectorAll('article[data-testid="tweet"]')) {
+ const time = a.querySelector("time");
+ const link = time && time.closest("a");
+ if (!time || !link) continue;
+ const quote = a.querySelector('div[role="link"]');
+ const own = [...a.querySelectorAll('[data-testid="tweetText"]')].filter((t) => !(quote && quote.contains(t)));
+ const post = {
+ url: link.href.split("?")[0].replace(/\\/(analytics|photo\\/\\d+)$/, ""),
+ ...nameOf(a),
+ time: time.getAttribute("datetime") || "",
+ text: own.map(txt).join("\\n"),
+ metrics: null,
+ };
+ const bar = a.querySelector('[role="group"][aria-label]');
+ if (bar) post.metrics = bar.getAttribute("aria-label");
+ if (a.querySelector('[data-testid="tweet-text-show-more-link"]')) post.truncated = true;
+ const ctx = txt(a.querySelector('[data-testid="socialContext"]'));
+ if (ctx) post.context = ctx;
+ const media = [...a.querySelectorAll('[data-testid="tweetPhoto"], video')].filter((m) => !(quote && quote.contains(m))).length;
+ if (media) post.media = media;
+ if (quote) {
+ const qt = txt(quote.querySelector('[data-testid="tweetText"]'));
+ if (qt) post.quoted = { ...nameOf(quote), text: qt };
+ }
+ posts.push(post);
+ }
+ if (posts.length) return { url, state: "posts", posts, note: "" };
+ const empty = document.querySelector('[data-testid="emptyState"]');
+ if (empty) return { url, state: "empty", posts, note: txt(empty).slice(0, 300) };
+ const retry = [...document.querySelectorAll('button, [role="button"]')].find((b) => /^Retry$/.test(txt(b)));
+ if (retry) {
+ const col = document.querySelector('[data-testid="primaryColumn"]');
+ return { url, state: "error", posts, note: txt(col).slice(0, 300) };
+ }
+ return { url, state: "loading", posts, note: "" };
+})()`;
+
+const SCROLL = `window.scrollBy(0, window.innerHeight * 2), true`;
+
+const sleep = (ms: number, signal: AbortSignal): Promise<void> =>
+ new Promise((resolve, reject) => {
+ if (signal.aborted) return reject(signal.reason);
+ const t = setTimeout(resolve, ms);
+ signal.addEventListener("abort", () => {
+ clearTimeout(t);
+ reject(signal.reason);
+ }, { once: true });
+ });
+
+const finish = (raw: RawPost): XPost => {
+ const { metrics, ...rest } = raw;
+ return { ...rest, ...parseMetrics(metrics) };
+};
+
+export const xSearch = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => {
+ const sort: XSort = request.sort ?? "latest";
+ const searchUrl = buildXSearchUrl(request.query, sort);
+ const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]);
+
+ let last = "nothing yet";
+ try {
+ return await run(request, sort, searchUrl, signal, (state) => (last = state));
+ } catch (e) {
+ if (signal.aborted && !outer.aborted) {
+ throw new BrowserUnavailableError(
+ `X search did not finish within ${X_SEARCH_TIMEOUT_MS / 1000} s (${searchUrl}; last seen: ${last}). ` +
+ `X may be slow or stuck on a page this tool does not recognise.`,
+ );
+ }
+ throw e;
+ }
+};
+
+const run = (
+ request: XSearchRequest,
+ sort: XSort,
+ searchUrl: string,
+ signal: AbortSignal,
+ saw: (state: string) => void,
+): Promise<XSearchOutcome> =>
+ withPage(signal, async (page, endpoint) => {
+ await page.navigate(searchUrl, signal);
+
+ // X renders after the load event: poll until posts, an empty or error
+ // page, or a sign-in redirect shows up.
+ let probe: XProbe;
+ for (;;) {
+ probe = await page.evaluate<XProbe>(PROBE, signal);
+ saw(`${probe.state} at ${probe.url}`);
+ if (probe.state !== "loading") break;
+ await sleep(POLL_MS, signal);
+ }
+
+ if (probe.state === "signed-out") {
+ throw new BrowserUnavailableError(
+ `X search needs a signed-in X session in the browser (${endpoint}), and it is signed out ` +
+ `(X sent it to ${probe.url}). Tell the user: someone has to sign in to X in that browser again.`,
+ );
+ }
+ if (probe.state === "error") {
+ throw new BrowserUnavailableError(
+ `X answered with an error page instead of results (${probe.url}): "${probe.note}". This is ` +
+ `usually X rate-limiting the account; wait a few minutes rather than retrying in a loop.`,
+ );
+ }
+ if (probe.state === "empty") {
+ throw new NoResultsError(
+ `X found no posts for ${JSON.stringify(request.query)} (${sort}). X said: "${probe.note}". ` +
+ `The page loaded normally, so the query is the cause: check the handle, loosen the terms, ` +
+ `or try sort "top" or a wider date range.`,
+ );
+ }
+
+ // X keeps only the posts near the viewport in the page, so collect by URL
+ // while scrolling rather than reading once at the end.
+ const seen = new Map<string, RawPost>();
+ const take = (p: XProbe) => p.posts.forEach((post) => seen.has(post.url) || seen.set(post.url, post));
+ take(probe);
+ for (let i = 0; i < MAX_SCROLLS && seen.size < request.count; i++) {
+ const before = seen.size;
+ await page.evaluate(SCROLL, signal);
+ // New posts arrive over the next second or so.
+ for (let w = 0; w < 4 && seen.size === before; w++) {
+ await sleep(POLL_MS, signal);
+ take(await page.evaluate<XProbe>(PROBE, signal));
+ }
+ if (seen.size === before) break; // the end of the results
+ }
+
+ return { query: request.query, sort, searchUrl, posts: [...seen.values()].slice(0, request.count).map(finish) };
+ });
diff --git a/castle-search/test/http.test.ts b/castle-search/test/http.test.ts
index 6b3045b..b2c90ab 100644
--- a/castle-search/test/http.test.ts
+++ b/castle-search/test/http.test.ts
@@ -65,3 +65,54 @@ test("listen addresses parse", () => {
assert.throws(() => parseListen("11158"));
assert.throws(() => parseListen("h:99999"));
});
+
+import { buildXSearchUrl, parseMetrics } from "../src/x-search.ts";
+import { handleXSearch, OPENAPI, type XSearcher } from "../src/http.ts";
+
+const xReturning = (seen: unknown[] = []): XSearcher => async (request) => {
+ seen.push(request);
+ return { query: request.query, sort: request.sort ?? "latest", searchUrl: "", posts: [] };
+};
+
+test("x-search defaults to the 10 latest posts and clamps count", async () => {
+ const seen: unknown[] = [];
+ await handleXSearch(JSON.stringify({ query: " from:nasa " }), xReturning(seen), signal);
+ await handleXSearch(JSON.stringify({ query: "x", count: 99, sort: "top" }), xReturning(seen), signal);
+ assert.deepEqual(seen, [
+ { query: "from:nasa", count: 10, sort: "latest" },
+ { query: "x", count: 20, sort: "top" },
+ ]);
+});
+
+test("x-search refuses a bad sort or query rather than guessing", async () => {
+ assert.equal((await handleXSearch(JSON.stringify({ query: "x", sort: "new" }), xReturning(), signal)).status, 400);
+ assert.equal((await handleXSearch(JSON.stringify({ query: "" }), xReturning(), signal)).status, 400);
+});
+
+test("a failed x-search is a 502 with its reason", async () => {
+ const reply = await handleXSearch(
+ JSON.stringify({ query: "x" }),
+ async () => {
+ throw new Error("signed out");
+ },
+ signal,
+ );
+ assert.equal(reply.status, 502);
+ assert.deepEqual(reply.body, { error: "signed out" });
+});
+
+test("x search URLs: latest is f=live, top has no tab parameter", () => {
+ assert.equal(buildXSearchUrl("from:a b", "latest"), "https://x.com/search?q=from%3Aa+b&src=typed_query&f=live");
+ assert.equal(buildXSearchUrl("from:a b", "top"), "https://x.com/search?q=from%3Aa+b&src=typed_query");
+});
+
+test("X's action-bar label parses, singular and plural", () => {
+ assert.deepEqual(parseMetrics("1 reply, 971 reposts, 8852 likes, 1247 bookmarks, 16029782 views"), {
+ replies: 1, reposts: 971, likes: 8852, views: 16029782,
+ });
+ assert.deepEqual(parseMetrics(null), {});
+});
+
+test("the OpenAPI description names the operation Open WebUI will offer", () => {
+ assert.equal(OPENAPI.paths["/x-search"].post.operationId, "search_x");
+});