From f8835d53b96bb6e4152bc5bf84e61b8b54396008 Mon Sep 17 00:00:00 2001 From: Igor Soarez Date: Wed, 18 Nov 2026 12:11:11 +0000 Subject: castle-search: X.com: read posts whole with thread and replies, profiles, trends /x-post, /x-user and /x-trends beside /x-search, sharing its limits and its sign-in. Posts now carry photos (image link, alt text), videos, link cards and the quoted post's time. A post's own page lists the quoted post's time first; the probe now takes the post's own linked time. --- castle-search/src/http.ts | 231 +++++++++++++++++------ castle-search/src/x-search.ts | 419 ++++++++++++++++++++++++++++++++---------- 2 files changed, 495 insertions(+), 155 deletions(-) (limited to 'castle-search/src') diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts index 17c8c70..8efc4c5 100644 --- a/castle-search/src/http.ts +++ b/castle-search/src/http.ts @@ -20,17 +20,33 @@ * an address only trusted peers reach. Each search opens and closes one * background tab in a browser someone may be looking at. * - * POST /x-search searches X instead (x-search.ts), through the browser's - * signed-in X session, and GET /openapi.json describes it, so that Open WebUI - * can offer it as a tool: `{ "query", "count", "sort" }` in, `{ "query", "sort", - * "posts": [...] }` out, 502 with `{ "error" }` on failure. + * POST /x-search, /x-post, /x-user and /x-trends read X instead (x-search.ts), + * through the browser's signed-in X session, and GET /openapi.json describes + * them, so that Open WebUI can offer them as tools ("X.com"). Each answers 502 + * with `{ "error" }` on failure. */ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { messageOf } from "./errors.ts"; import { disconnect, search, type SearchOutcome, type SearchRequest } from "./search.ts"; -import { xSearch, type XSearchOutcome, type XSearchRequest } from "./x-search.ts"; +import { + handleOf, + postIdOf, + TREND_TABS, + type XPostOutcome, + type XPostRequest, + xPost, + xSearch, + type XSearchOutcome, + type XSearchRequest, + type XTrendsOutcome, + type XTrendTab, + xTrends, + xUser, + type XUserOutcome, + type XUserRequest, +} from "./x-search.ts"; export const DEFAULT_LISTEN = "127.0.0.1:11158"; const MAX_BODY_BYTES = 64 * 1024; @@ -75,47 +91,105 @@ export const handleSearch = async (raw: string, run: Searcher, signal: AbortSign }; export type XSearcher = (request: XSearchRequest, signal: AbortSignal) => Promise; +export type XPostReader = (request: XPostRequest, signal: AbortSignal) => Promise; +export type XUserReader = (request: XUserRequest, signal: AbortSignal) => Promise; +export type XTrendsReader = (tab: XTrendTab, signal: AbortSignal) => Promise; const X_DEFAULT_COUNT = 10; -/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */ -export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise => { - let parsed: unknown; +const jsonBody = (raw: string): Record | Reply => { try { - parsed = JSON.parse(raw); + const parsed: unknown = JSON.parse(raw || "{}"); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as Record; } catch { - return { status: 400, body: { error: "request body is not JSON" } }; + // fall through } - const { query, count, sort } = (parsed ?? {}) as { query?: unknown; count?: unknown; sort?: unknown }; + return { status: 400, body: { error: "request body is not a JSON object" } }; +}; +const isReply = (v: unknown): v is Reply => typeof (v as Reply).status === "number" && "body" in (v as Reply); + +const countOf = (v: unknown, fallback: number, min = 1): number => { + const n = typeof v === "number" && Number.isFinite(v) ? Math.trunc(v) : fallback; + return Math.min(MAX_COUNT, Math.max(min, n)); +}; + +const answer = async (work: () => Promise): Promise => { + try { + return { status: 200, body: await work() }; + } catch (e) { + return { status: 502, body: { error: messageOf(e) } }; + } +}; + +/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */ +export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise => { + const body = jsonBody(raw); + if (isReply(body)) return body; + const { query, count, sort } = body; if (typeof query !== "string" || query.trim() === "") { return { status: 400, body: { error: "query must be a non-empty string" } }; } if (sort !== undefined && sort !== null && sort !== "latest" && sort !== "top") { return { status: 400, body: { error: 'sort must be "latest" or "top"' } }; } - const n = typeof count === "number" && Number.isFinite(count) ? Math.trunc(count) : X_DEFAULT_COUNT; - try { - const outcome = await run( - { query: query.trim(), count: Math.min(MAX_COUNT, Math.max(1, n)), sort: sort ?? "latest" }, - signal, - ); - return { status: 200, body: { query: outcome.query, sort: outcome.sort, posts: outcome.posts } }; - } catch (e) { - return { status: 502, body: { error: messageOf(e) } }; + return answer(async () => { + const outcome = await run({ query: query.trim(), count: countOf(count, X_DEFAULT_COUNT), sort: sort ?? "latest" }, signal); + return { query: outcome.query, sort: outcome.sort, posts: outcome.posts }; + }); +}; + +/** POST /x-post: `{ url, replies? }` in, `{ post, thread, replies }` out. */ +export const handleXPost = async (raw: string, run: XPostReader, signal: AbortSignal): Promise => { + const body = jsonBody(raw); + if (isReply(body)) return body; + if (typeof body["url"] !== "string" || !postIdOf(body["url"])) { + return { status: 400, body: { error: "url must be a link to a post on X (x.com//status/) or its id" } }; } + const url = body["url"]; + return answer(() => run({ url, replies: countOf(body["replies"], X_DEFAULT_COUNT, 0) }, signal)); +}; + +/** POST /x-user: `{ handle, count? }` in, `{ profile, posts }` out. */ +export const handleXUser = async (raw: string, run: XUserReader, signal: AbortSignal): Promise => { + const body = jsonBody(raw); + if (isReply(body)) return body; + if (typeof body["handle"] !== "string" || !handleOf(body["handle"])) { + return { status: 400, body: { error: "handle must be an X handle, such as nasa or @nasa" } }; + } + const handle = body["handle"]; + return answer(() => run({ handle, count: countOf(body["count"], X_DEFAULT_COUNT, 0) }, signal)); +}; + +/** POST /x-trends: `{ tab? }` in, `{ tab, trends }` out. */ +export const handleXTrends = async (raw: string, run: XTrendsReader, signal: AbortSignal): Promise => { + const body = jsonBody(raw); + if (isReply(body)) return body; + const tab = body["tab"] ?? "trending"; + if (!TREND_TABS.includes(tab as XTrendTab)) { + return { status: 400, body: { error: `tab must be one of ${TREND_TABS.join(", ")}` } }; + } + return answer(() => run(tab as XTrendTab, signal)); +}; + +const jsonSchema = (properties: Record, required: string[]) => ({ + required: true, + content: { "application/json": { schema: { type: "object", required, properties } } }, +}); +const RESPONSES = { + "200": { description: "What X shows", content: { "application/json": { schema: { type: "object" } } } }, + "502": { description: "X could not be read; error says why", content: { "application/json": { schema: { type: "object" } } } }, }; /** - * The OpenAPI description Open WebUI reads to offer /x-search as a tool. Its - * descriptions are what the model sees, so they say what the tool is for and - * how to write a query. + * The OpenAPI description Open WebUI reads to offer the X reads as tools. Its + * descriptions are what the model sees, so they say what each is for. */ export const OPENAPI = { openapi: "3.1.0", info: { - title: "castle X search", - version: "1.0.0", - description: "Searches posts on X (Twitter) through a signed-in browser. Free per search.", + title: "X.com", + version: "2.0.0", + description: "Reads X (Twitter) through a signed-in browser: search, posts with replies, profiles, trends. Free.", }, paths: { "/x-search": { @@ -127,27 +201,67 @@ export const OPENAPI = { "reactions to news, posts about a topic. The query uses X's search syntax: from:handle, to:handle, " + '"exact phrase", OR, -word, since:YYYY-MM-DD, until:YYYY-MM-DD, min_faves:N, filter:media, ' + "-filter:replies, lang:en. A handle is written without @ after from:. Each post comes with its " + - "link: cite it. Text cut short by X is marked truncated.", - requestBody: { - required: true, - content: { - "application/json": { - schema: { - type: "object", - required: ["query"], - properties: { - query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" }, - count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" }, - sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" }, - }, - }, - }, + "link: cite it. A post marked truncated was cut short by X: read_x_post gives it whole. Photos " + + "come with their image link and alt text.", + requestBody: jsonSchema( + { + query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" }, + count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" }, + sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" }, }, - }, - responses: { - "200": { description: "Matching posts", content: { "application/json": { schema: { type: "object" } } } }, - "502": { description: "The search failed; error says why", content: { "application/json": { schema: { type: "object" } } } }, - }, + ["query"], + ), + responses: RESPONSES, + }, + }, + "/x-post": { + post: { + operationId: "read_x_post", + summary: "Read one post on X, whole, with its thread and replies", + description: + "Read a post on X from its link: its full text (never truncated), photos with their image link " + + "and alt text, its link card, the posts above it when it is a reply (thread, oldest first), and " + + "the top replies under it. Use it to read a long post, a thread, or what people answered.", + requestBody: jsonSchema( + { + url: { type: "string", description: "The post's link, x.com//status/, or the id" }, + replies: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many replies" }, + }, + ["url"], + ), + responses: RESPONSES, + }, + }, + "/x-user": { + post: { + operationId: "read_x_profile", + summary: "Read an X profile and its latest posts", + description: + "Read an X account's profile: name, bio, location, website, when it joined, exact follower, " + + "following and post counts, then its latest posts as its profile shows them (a pinned post and " + + "reposts included, marked in context).", + requestBody: jsonSchema( + { + handle: { type: "string", description: "The account's handle, such as nasa or @nasa" }, + count: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many of its posts" }, + }, + ["handle"], + ), + responses: RESPONSES, + }, + }, + "/x-trends": { + post: { + operationId: "x_trends", + summary: "What is trending on X now", + description: + "List what X shows as trending now, for the signed-in account's region (the United Kingdom). " + + "Each trend has a query: pass it to search_x for the posts.", + requestBody: jsonSchema( + { tab: { type: "string", enum: [...TREND_TABS], default: "trending", description: "Which explore tab" } }, + [], + ), + responses: RESPONSES, }, }, }, @@ -199,9 +313,15 @@ const main = (): void => { send(res, { status: 200, body: OPENAPI }); return; } - const isX = path === "/x-search"; - if (req.method !== "POST" || (path !== "/" && path !== "/search" && !isX)) { - send(res, { status: 404, body: { error: "POST a search to / or /search, or an X search to /x-search" } }); + const xRoutes: Record Promise> = { + "/x-search": (raw, signal) => handleXSearch(raw, xSearch, signal), + "/x-post": (raw, signal) => handleXPost(raw, xPost, signal), + "/x-user": (raw, signal) => handleXUser(raw, xUser, signal), + "/x-trends": (raw, signal) => handleXTrends(raw, xTrends, signal), + }; + const xRoute = xRoutes[path ?? ""]; + if (req.method !== "POST" || (path !== "/" && path !== "/search" && !xRoute)) { + send(res, { status: 404, body: { error: "POST a search to / or /search, or an X read to /x-search, /x-post, /x-user or /x-trends" } }); return; } // Stop the search, and free its tab, if the client goes away first. @@ -217,16 +337,9 @@ const main = (): void => { return; } const started = Date.now(); - const reply = isX - ? await handleXSearch(raw, xSearch, controller.signal) - : await handleSearch(raw, runSearch, controller.signal); - const outcome = - reply.status !== 200 - ? `error ${reply.status}` - : isX - ? `${(reply.body as { posts: unknown[] }).posts.length} posts` - : `${(reply.body as unknown[]).length} results`; - console.log(`${isX ? "x-search" : "search"} ${outcome} in ${Date.now() - started} ms`); + const reply = xRoute ? await xRoute(raw, controller.signal) : await handleSearch(raw, runSearch, controller.signal); + const outcome = reply.status !== 200 ? `error ${reply.status}` : xRoute ? "ok" : `${(reply.body as unknown[]).length} results`; + console.log(`${xRoute ? path!.slice(1) : "search"} ${outcome} in ${Date.now() - started} ms`); if (reply.status !== 200) console.log(` ${(reply.body as { error: string }).error}`); send(res, reply); }); diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts index 7c8badf..174906e 100644 --- a/castle-search/src/x-search.ts +++ b/castle-search/src/x-search.ts @@ -1,35 +1,38 @@ /** - * Search X (x.com) through the browser's own signed-in session. + * Read X (x.com) through the browser's own signed-in session: search posts, + * read one post with its thread and replies, read a profile, list trends. * - * X serves search only to a signed-in session and renders it from JavaScript, - * so the page is the only interface there is: open x.com/search in a background - * tab, wait for posts to render, read them, scroll for more, close the tab. - * Free per search, unlike xAI's x_search ($5 per 1,000 posts returned). + * X serves all of this only to a signed-in session and renders it from + * JavaScript, so the page is the only interface there is: open the page in a + * background tab, wait for it to render, read it, scroll for more, close the + * tab. Free, unlike xAI's x_search ($5 per 1,000 posts returned). * - * The query is X's own search syntax, passed through untouched: from:handle, - * to:handle, since:2026-11-01, until:..., min_faves:100, filter:media, - * -filter:replies, lang:en, "exact phrase", OR. + * A search query is X's own search syntax, passed through untouched: + * from:handle, to:handle, since:2026-11-01, until:..., min_faves:100, + * filter:media, -filter:replies, lang:en, "exact phrase", OR. * - * Every failure throws, as for the engine searches (errors.ts): signed out, an - * X error page, or nothing rendering are reported as such, never as "no posts". - * Only X's own "No results" page is an empty answer, and even that is reported - * as an error naming the query, since an over-narrow query is the usual cause. + * Every read shares one set of limits (x-limits.ts) and, signed out, signs the + * browser back in once (x-login.ts). Every failure throws, as for the engine + * searches (errors.ts): signed out, an X error page, or nothing rendering are + * reported as such, never as an empty answer. X's own "No results" or "doesn't + * exist" page is reported as an error naming what was asked for. */ +import type { PageSession } from "./cdp.ts"; import { BrowserUnavailableError, NoResultsError } from "./errors.ts"; import { withPage } from "./search.ts"; import { XLimiter } from "./x-limits.ts"; import { loadCredentials, X_LOGIN_TIMEOUT_MS, xLogin } from "./x-login.ts"; -/** Shared by every X search in this process: see x-limits.ts. */ +/** Shared by every X read in this process: see x-limits.ts. */ const limiter = new XLimiter(); /** - * Signed out, a search signs the browser back in (x-login.ts) when - * CCS_X_LOGIN_FILE names the credentials, then searches again, once. At most - * one sign-in attempt per LOGIN_COOLDOWN_MS: a sign-in that keeps failing - * is what gets an account locked, so until the cooldown has passed every - * search fails with the reason the last attempt gave. + * Signed out, a read signs the browser back in (x-login.ts) when + * CCS_X_LOGIN_FILE names the credentials, then reads again, once. At most one + * sign-in attempt per LOGIN_COOLDOWN_MS: a sign-in that keeps failing is what + * gets an account locked, so until the cooldown has passed every read fails + * with the reason the last attempt gave. */ export const LOGIN_COOLDOWN_MS = 30 * 60_000; let lastLogin: { at: number; error?: string } | undefined; @@ -44,6 +47,8 @@ class SignedOut extends Error { export type XSort = "latest" | "top"; +export type XMedia = { type: "photo" | "video"; url?: string; alt?: string }; + export type XPost = { url: string; author: string; @@ -51,7 +56,7 @@ export type XPost = { /** ISO 8601, from the post's