diff options
| -rw-r--r-- | castle-search/README.md | 36 | ||||
| -rw-r--r-- | castle-search/src/http.ts | 231 | ||||
| -rw-r--r-- | castle-search/src/x-search.ts | 419 | ||||
| -rw-r--r-- | castle-search/test/http.test.ts | 70 |
4 files changed, 588 insertions, 168 deletions
diff --git a/castle-search/README.md b/castle-search/README.md index 1b74884..e8b204a 100644 --- a/castle-search/README.md +++ b/castle-search/README.md @@ -47,23 +47,33 @@ switch engines, because this protocol has no field for one. There is no authentication: bind it to loopback (the default) or a trusted address. `GET /health` answers `{"status": true}`. -### X search +### X.com: search, posts, profiles, trends -`POST /x-search` searches X (Twitter) instead: `{"query": "from:nasa", "count": -10, "sort": "latest"}` in, `{"query", "sort", "posts": [...]}` out, each post -with its link, author, time, text, counts and any quoted post. The query is X's -own search syntax. It needs the browser signed in to X — X serves search to -nobody else — and it says so when it is not, rather than returning nothing. -`GET /openapi.json` describes it, so Open WebUI can offer it as a tool. Free per -search, where xAI's own X search charges per post returned. +Four reads of X (Twitter), each `POST` a JSON object and get JSON back: + +| Route | In | Out | +|---|---|---| +| `/x-search` | `query` (X's own search syntax), `count`, `sort` (`latest` or `top`) | the posts | +| `/x-post` | `url` (a post link or its id), `replies` | the post whole, the `thread` above it when it is a reply, its top `replies` | +| `/x-user` | `handle`, `count` | the `profile` (bio, location, website, joined, exact follower, following and post counts) and its latest posts | +| `/x-trends` | `tab`: `trending`, `news`, `sports`, `entertainment`, `for_you` | the trends, each with a `query` for `/x-search` | + +Each post comes with its link, author, time, text, counts, photos (X's image +link and alt text), videos (a still), a link card and any quoted post. A search +result X cut short is marked `truncated`; `/x-post` reads it whole. It needs the +browser signed in to X, since X serves none of this to anybody else, and it +says so when it is not, rather than returning nothing. `GET /openapi.json` +describes all four, as "X.com", so Open WebUI can offer them as tools. Free, +where xAI's own X search charges per post returned. Trends are for the +signed-in account's region. X keeps only the posts near the screen in the page, so it scrolls and collects -by link, up to 20 posts. A search takes 4–7 s. +by link, up to 20 posts (or replies). A read takes 4–7 s. -It is limited so the account reads X like a person, not a scraper -(`src/x-limits.ts`): one search at a time, at least 10 s between them (a search -that comes sooner waits), and at most 30 an hour and 150 a day, beyond which a -search fails saying when the next may run. The counts live in memory, so a +All four share one set of limits, so the account reads X like a person, not a +scraper (`src/x-limits.ts`): one read at a time, at least 10 s between them (a +read that comes sooner waits), and at most 30 an hour and 150 a day, beyond which a +read fails saying when the next may run. The counts live in memory, so a restart starts them afresh. Signed out, it signs back in by itself when `CCS_X_LOGIN_FILE` names a JSON diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts index 17c8c70..8efc4c5 100644 --- a/castle-search/src/http.ts +++ b/castle-search/src/http.ts @@ -20,17 +20,33 @@ * an address only trusted peers reach. Each search opens and closes one * background tab in a browser someone may be looking at. * - * POST /x-search searches X instead (x-search.ts), through the browser's - * signed-in X session, and GET /openapi.json describes it, so that Open WebUI - * can offer it as a tool: `{ "query", "count", "sort" }` in, `{ "query", "sort", - * "posts": [...] }` out, 502 with `{ "error" }` on failure. + * POST /x-search, /x-post, /x-user and /x-trends read X instead (x-search.ts), + * through the browser's signed-in X session, and GET /openapi.json describes + * them, so that Open WebUI can offer them as tools ("X.com"). Each answers 502 + * with `{ "error" }` on failure. */ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { messageOf } from "./errors.ts"; import { disconnect, search, type SearchOutcome, type SearchRequest } from "./search.ts"; -import { xSearch, type XSearchOutcome, type XSearchRequest } from "./x-search.ts"; +import { + handleOf, + postIdOf, + TREND_TABS, + type XPostOutcome, + type XPostRequest, + xPost, + xSearch, + type XSearchOutcome, + type XSearchRequest, + type XTrendsOutcome, + type XTrendTab, + xTrends, + xUser, + type XUserOutcome, + type XUserRequest, +} from "./x-search.ts"; export const DEFAULT_LISTEN = "127.0.0.1:11158"; const MAX_BODY_BYTES = 64 * 1024; @@ -75,47 +91,105 @@ export const handleSearch = async (raw: string, run: Searcher, signal: AbortSign }; export type XSearcher = (request: XSearchRequest, signal: AbortSignal) => Promise<XSearchOutcome>; +export type XPostReader = (request: XPostRequest, signal: AbortSignal) => Promise<XPostOutcome>; +export type XUserReader = (request: XUserRequest, signal: AbortSignal) => Promise<XUserOutcome>; +export type XTrendsReader = (tab: XTrendTab, signal: AbortSignal) => Promise<XTrendsOutcome>; const X_DEFAULT_COUNT = 10; -/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */ -export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise<Reply> => { - let parsed: unknown; +const jsonBody = (raw: string): Record<string, unknown> | Reply => { try { - parsed = JSON.parse(raw); + const parsed: unknown = JSON.parse(raw || "{}"); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as Record<string, unknown>; } catch { - return { status: 400, body: { error: "request body is not JSON" } }; + // fall through } - const { query, count, sort } = (parsed ?? {}) as { query?: unknown; count?: unknown; sort?: unknown }; + return { status: 400, body: { error: "request body is not a JSON object" } }; +}; +const isReply = (v: unknown): v is Reply => typeof (v as Reply).status === "number" && "body" in (v as Reply); + +const countOf = (v: unknown, fallback: number, min = 1): number => { + const n = typeof v === "number" && Number.isFinite(v) ? Math.trunc(v) : fallback; + return Math.min(MAX_COUNT, Math.max(min, n)); +}; + +const answer = async (work: () => Promise<unknown>): Promise<Reply> => { + try { + return { status: 200, body: await work() }; + } catch (e) { + return { status: 502, body: { error: messageOf(e) } }; + } +}; + +/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */ +export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise<Reply> => { + const body = jsonBody(raw); + if (isReply(body)) return body; + const { query, count, sort } = body; if (typeof query !== "string" || query.trim() === "") { return { status: 400, body: { error: "query must be a non-empty string" } }; } if (sort !== undefined && sort !== null && sort !== "latest" && sort !== "top") { return { status: 400, body: { error: 'sort must be "latest" or "top"' } }; } - const n = typeof count === "number" && Number.isFinite(count) ? Math.trunc(count) : X_DEFAULT_COUNT; - try { - const outcome = await run( - { query: query.trim(), count: Math.min(MAX_COUNT, Math.max(1, n)), sort: sort ?? "latest" }, - signal, - ); - return { status: 200, body: { query: outcome.query, sort: outcome.sort, posts: outcome.posts } }; - } catch (e) { - return { status: 502, body: { error: messageOf(e) } }; + return answer(async () => { + const outcome = await run({ query: query.trim(), count: countOf(count, X_DEFAULT_COUNT), sort: sort ?? "latest" }, signal); + return { query: outcome.query, sort: outcome.sort, posts: outcome.posts }; + }); +}; + +/** POST /x-post: `{ url, replies? }` in, `{ post, thread, replies }` out. */ +export const handleXPost = async (raw: string, run: XPostReader, signal: AbortSignal): Promise<Reply> => { + const body = jsonBody(raw); + if (isReply(body)) return body; + if (typeof body["url"] !== "string" || !postIdOf(body["url"])) { + return { status: 400, body: { error: "url must be a link to a post on X (x.com/<handle>/status/<id>) or its id" } }; } + const url = body["url"]; + return answer(() => run({ url, replies: countOf(body["replies"], X_DEFAULT_COUNT, 0) }, signal)); +}; + +/** POST /x-user: `{ handle, count? }` in, `{ profile, posts }` out. */ +export const handleXUser = async (raw: string, run: XUserReader, signal: AbortSignal): Promise<Reply> => { + const body = jsonBody(raw); + if (isReply(body)) return body; + if (typeof body["handle"] !== "string" || !handleOf(body["handle"])) { + return { status: 400, body: { error: "handle must be an X handle, such as nasa or @nasa" } }; + } + const handle = body["handle"]; + return answer(() => run({ handle, count: countOf(body["count"], X_DEFAULT_COUNT, 0) }, signal)); +}; + +/** POST /x-trends: `{ tab? }` in, `{ tab, trends }` out. */ +export const handleXTrends = async (raw: string, run: XTrendsReader, signal: AbortSignal): Promise<Reply> => { + const body = jsonBody(raw); + if (isReply(body)) return body; + const tab = body["tab"] ?? "trending"; + if (!TREND_TABS.includes(tab as XTrendTab)) { + return { status: 400, body: { error: `tab must be one of ${TREND_TABS.join(", ")}` } }; + } + return answer(() => run(tab as XTrendTab, signal)); +}; + +const jsonSchema = (properties: Record<string, unknown>, required: string[]) => ({ + required: true, + content: { "application/json": { schema: { type: "object", required, properties } } }, +}); +const RESPONSES = { + "200": { description: "What X shows", content: { "application/json": { schema: { type: "object" } } } }, + "502": { description: "X could not be read; error says why", content: { "application/json": { schema: { type: "object" } } } }, }; /** - * The OpenAPI description Open WebUI reads to offer /x-search as a tool. Its - * descriptions are what the model sees, so they say what the tool is for and - * how to write a query. + * The OpenAPI description Open WebUI reads to offer the X reads as tools. Its + * descriptions are what the model sees, so they say what each is for. */ export const OPENAPI = { openapi: "3.1.0", info: { - title: "castle X search", - version: "1.0.0", - description: "Searches posts on X (Twitter) through a signed-in browser. Free per search.", + title: "X.com", + version: "2.0.0", + description: "Reads X (Twitter) through a signed-in browser: search, posts with replies, profiles, trends. Free.", }, paths: { "/x-search": { @@ -127,27 +201,67 @@ export const OPENAPI = { "reactions to news, posts about a topic. The query uses X's search syntax: from:handle, to:handle, " + '"exact phrase", OR, -word, since:YYYY-MM-DD, until:YYYY-MM-DD, min_faves:N, filter:media, ' + "-filter:replies, lang:en. A handle is written without @ after from:. Each post comes with its " + - "link: cite it. Text cut short by X is marked truncated.", - requestBody: { - required: true, - content: { - "application/json": { - schema: { - type: "object", - required: ["query"], - properties: { - query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" }, - count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" }, - sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" }, - }, - }, - }, + "link: cite it. A post marked truncated was cut short by X: read_x_post gives it whole. Photos " + + "come with their image link and alt text.", + requestBody: jsonSchema( + { + query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" }, + count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" }, + sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" }, }, - }, - responses: { - "200": { description: "Matching posts", content: { "application/json": { schema: { type: "object" } } } }, - "502": { description: "The search failed; error says why", content: { "application/json": { schema: { type: "object" } } } }, - }, + ["query"], + ), + responses: RESPONSES, + }, + }, + "/x-post": { + post: { + operationId: "read_x_post", + summary: "Read one post on X, whole, with its thread and replies", + description: + "Read a post on X from its link: its full text (never truncated), photos with their image link " + + "and alt text, its link card, the posts above it when it is a reply (thread, oldest first), and " + + "the top replies under it. Use it to read a long post, a thread, or what people answered.", + requestBody: jsonSchema( + { + url: { type: "string", description: "The post's link, x.com/<handle>/status/<id>, or the id" }, + replies: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many replies" }, + }, + ["url"], + ), + responses: RESPONSES, + }, + }, + "/x-user": { + post: { + operationId: "read_x_profile", + summary: "Read an X profile and its latest posts", + description: + "Read an X account's profile: name, bio, location, website, when it joined, exact follower, " + + "following and post counts, then its latest posts as its profile shows them (a pinned post and " + + "reposts included, marked in context).", + requestBody: jsonSchema( + { + handle: { type: "string", description: "The account's handle, such as nasa or @nasa" }, + count: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many of its posts" }, + }, + ["handle"], + ), + responses: RESPONSES, + }, + }, + "/x-trends": { + post: { + operationId: "x_trends", + summary: "What is trending on X now", + description: + "List what X shows as trending now, for the signed-in account's region (the United Kingdom). " + + "Each trend has a query: pass it to search_x for the posts.", + requestBody: jsonSchema( + { tab: { type: "string", enum: [...TREND_TABS], default: "trending", description: "Which explore tab" } }, + [], + ), + responses: RESPONSES, }, }, }, @@ -199,9 +313,15 @@ const main = (): void => { send(res, { status: 200, body: OPENAPI }); return; } - const isX = path === "/x-search"; - if (req.method !== "POST" || (path !== "/" && path !== "/search" && !isX)) { - send(res, { status: 404, body: { error: "POST a search to / or /search, or an X search to /x-search" } }); + const xRoutes: Record<string, (raw: string, signal: AbortSignal) => Promise<Reply>> = { + "/x-search": (raw, signal) => handleXSearch(raw, xSearch, signal), + "/x-post": (raw, signal) => handleXPost(raw, xPost, signal), + "/x-user": (raw, signal) => handleXUser(raw, xUser, signal), + "/x-trends": (raw, signal) => handleXTrends(raw, xTrends, signal), + }; + const xRoute = xRoutes[path ?? ""]; + if (req.method !== "POST" || (path !== "/" && path !== "/search" && !xRoute)) { + send(res, { status: 404, body: { error: "POST a search to / or /search, or an X read to /x-search, /x-post, /x-user or /x-trends" } }); return; } // Stop the search, and free its tab, if the client goes away first. @@ -217,16 +337,9 @@ const main = (): void => { return; } const started = Date.now(); - const reply = isX - ? await handleXSearch(raw, xSearch, controller.signal) - : await handleSearch(raw, runSearch, controller.signal); - const outcome = - reply.status !== 200 - ? `error ${reply.status}` - : isX - ? `${(reply.body as { posts: unknown[] }).posts.length} posts` - : `${(reply.body as unknown[]).length} results`; - console.log(`${isX ? "x-search" : "search"} ${outcome} in ${Date.now() - started} ms`); + const reply = xRoute ? await xRoute(raw, controller.signal) : await handleSearch(raw, runSearch, controller.signal); + const outcome = reply.status !== 200 ? `error ${reply.status}` : xRoute ? "ok" : `${(reply.body as unknown[]).length} results`; + console.log(`${xRoute ? path!.slice(1) : "search"} ${outcome} in ${Date.now() - started} ms`); if (reply.status !== 200) console.log(` ${(reply.body as { error: string }).error}`); send(res, reply); }); diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts index 7c8badf..174906e 100644 --- a/castle-search/src/x-search.ts +++ b/castle-search/src/x-search.ts @@ -1,35 +1,38 @@ /** - * Search X (x.com) through the browser's own signed-in session. + * Read X (x.com) through the browser's own signed-in session: search posts, + * read one post with its thread and replies, read a profile, list trends. * - * X serves search only to a signed-in session and renders it from JavaScript, - * so the page is the only interface there is: open x.com/search in a background - * tab, wait for posts to render, read them, scroll for more, close the tab. - * Free per search, unlike xAI's x_search ($5 per 1,000 posts returned). + * X serves all of this only to a signed-in session and renders it from + * JavaScript, so the page is the only interface there is: open the page in a + * background tab, wait for it to render, read it, scroll for more, close the + * tab. Free, unlike xAI's x_search ($5 per 1,000 posts returned). * - * The query is X's own search syntax, passed through untouched: from:handle, - * to:handle, since:2026-11-01, until:..., min_faves:100, filter:media, - * -filter:replies, lang:en, "exact phrase", OR. + * A search query is X's own search syntax, passed through untouched: + * from:handle, to:handle, since:2026-11-01, until:..., min_faves:100, + * filter:media, -filter:replies, lang:en, "exact phrase", OR. * - * Every failure throws, as for the engine searches (errors.ts): signed out, an - * X error page, or nothing rendering are reported as such, never as "no posts". - * Only X's own "No results" page is an empty answer, and even that is reported - * as an error naming the query, since an over-narrow query is the usual cause. + * Every read shares one set of limits (x-limits.ts) and, signed out, signs the + * browser back in once (x-login.ts). Every failure throws, as for the engine + * searches (errors.ts): signed out, an X error page, or nothing rendering are + * reported as such, never as an empty answer. X's own "No results" or "doesn't + * exist" page is reported as an error naming what was asked for. */ +import type { PageSession } from "./cdp.ts"; import { BrowserUnavailableError, NoResultsError } from "./errors.ts"; import { withPage } from "./search.ts"; import { XLimiter } from "./x-limits.ts"; import { loadCredentials, X_LOGIN_TIMEOUT_MS, xLogin } from "./x-login.ts"; -/** Shared by every X search in this process: see x-limits.ts. */ +/** Shared by every X read in this process: see x-limits.ts. */ const limiter = new XLimiter(); /** - * Signed out, a search signs the browser back in (x-login.ts) when - * CCS_X_LOGIN_FILE names the credentials, then searches again, once. At most - * one sign-in attempt per LOGIN_COOLDOWN_MS: a sign-in that keeps failing - * is what gets an account locked, so until the cooldown has passed every - * search fails with the reason the last attempt gave. + * Signed out, a read signs the browser back in (x-login.ts) when + * CCS_X_LOGIN_FILE names the credentials, then reads again, once. At most one + * sign-in attempt per LOGIN_COOLDOWN_MS: a sign-in that keeps failing is what + * gets an account locked, so until the cooldown has passed every read fails + * with the reason the last attempt gave. */ export const LOGIN_COOLDOWN_MS = 30 * 60_000; let lastLogin: { at: number; error?: string } | undefined; @@ -44,6 +47,8 @@ class SignedOut extends Error { export type XSort = "latest" | "top"; +export type XMedia = { type: "photo" | "video"; url?: string; alt?: string }; + export type XPost = { url: string; author: string; @@ -51,7 +56,7 @@ export type XPost = { /** ISO 8601, from the post's <time datetime>. */ time: string; text: string; - /** X cut the text short behind "Show more"; the full post is at url. */ + /** X cut the text short behind "Show more"; x_post reads it whole. */ truncated?: true; /** "@someone reposted", "Pinned" and the like. */ context?: string; @@ -59,18 +64,48 @@ export type XPost = { reposts?: number; likes?: number; views?: number; - /** Count of photos and videos attached. */ - media?: number; - quoted?: { author: string; handle: string; text: string }; + /** Photos (with X's image URL and alt text) and videos (with a still). */ + media?: XMedia[]; + /** A link card: the page the post links to. */ + card?: { url: string; title?: string }; + quoted?: { author: string; handle: string; text: string; time?: string; url?: string }; }; export type XSearchRequest = { query: string; count: number; sort?: XSort | undefined }; - export type XSearchOutcome = { query: string; sort: XSort; searchUrl: string; posts: XPost[] }; +export type XPostRequest = { url: string; replies: number }; +export type XPostOutcome = { + post: XPost; + /** The posts above it in its thread, oldest first, when it is a reply. */ + thread: XPost[]; + replies: XPost[]; +}; + +export type XUserRequest = { handle: string; count: number }; +export type XProfile = { + name: string; + handle: string; + url: string; + bio?: string; + location?: string; + website?: string; + joined?: string; + followers?: number; + following?: number; + posts?: number; + avatar?: string; +}; +export type XUserOutcome = { profile: XProfile; posts: XPost[] }; + +export const TREND_TABS = ["trending", "news", "sports", "entertainment", "for_you"] as const; +export type XTrendTab = (typeof TREND_TABS)[number]; +export type XTrend = { rank?: number; name: string; context?: string; detail?: string; query: string }; +export type XTrendsOutcome = { tab: XTrendTab; trends: XTrend[] }; + /** X shows ~5-10 posts per screen; scroll this many times at most for more. */ const MAX_SCROLLS = 6; -/** Wall clock for one search: render, scrolls and all. */ +/** Wall clock for one read: render, scrolls and all. */ export const X_SEARCH_TIMEOUT_MS = 25_000; const POLL_MS = 400; @@ -81,6 +116,16 @@ export const buildXSearchUrl = (query: string, sort: XSort): string => { return `https://x.com/search?${params.toString()}`; }; +/** The post id in an X or Twitter post link, or a bare id. */ +export const postIdOf = (url: string): string | undefined => + /^\d{5,25}$/.test(url.trim()) ? url.trim() : /(?:x|twitter)\.com\/(?:[^/]+|i(?:\/web)?)\/status(?:es)?\/(\d+)/i.exec(url)?.[1]; + +/** A handle as X writes it, from "@nasa", "nasa" or a profile link. */ +export const handleOf = (s: string): string | undefined => { + const m = /^(?:https?:\/\/(?:www\.)?(?:x|twitter)\.com\/)?@?([A-Za-z0-9_]{1,15})\/?$/.exec(s.trim()); + return m?.[1]; +}; + /** Parses X's action-bar label: "441 replies, 971 reposts, 8852 likes, 1247 bookmarks, 16029782 views". */ export const parseMetrics = (label: string | null | undefined): Partial<Record<"replies" | "reposts" | "likes" | "views", number>> => { const out: Partial<Record<"replies" | "reposts" | "likes" | "views", number>> = {}; @@ -95,6 +140,21 @@ export const parseMetrics = (label: string | null | undefined): Partial<Record<" return out; }; +/** One trend cell's lines: "2", "·", "Gaming · Trending", "GTA 6", "Trending with #GTAVI". */ +export const parseTrend = (text: string): XTrend | undefined => { + const lines = text.split("\n").map((l) => l.trim()).filter((l) => l && l !== "·"); + let rank: number | undefined; + if (lines[0] && /^\d+$/.test(lines[0])) rank = Number(lines.shift()); + const context = lines.shift(); + const name = lines.shift(); + if (!name) return undefined; + const trend: XTrend = { name, query: name }; + if (rank !== undefined) trend.rank = rank; + if (context) trend.context = context; + if (lines.length) trend.detail = lines.join("; "); + return trend; +}; + type RawPost = Omit<XPost, "replies" | "reposts" | "likes" | "views"> & { metrics: string | null }; type XProbe = { @@ -102,6 +162,8 @@ type XProbe = { state: "signed-out" | "posts" | "empty" | "error" | "loading"; posts: RawPost[]; note: string; + /** The profile's JSON-LD, on a profile page. */ + profile: string | null; }; /** @@ -111,23 +173,30 @@ type XProbe = { const PROBE = `(() => { const url = location.href; if (/^\\/(login|i\\/flow\\/login|i\\/jf\\/onboarding|account\\/access)/.test(location.pathname)) { - return { url, state: "signed-out", posts: [], note: location.pathname }; + return { url, state: "signed-out", posts: [], note: location.pathname, profile: null }; } const txt = (el) => (el ? el.innerText.trim() : ""); const nameOf = (scope) => { const lines = txt(scope.querySelector('[data-testid="User-Name"]')).split("\\n"); return { author: lines[0] || "", handle: lines.find((l) => l.startsWith("@")) || "" }; }; + const clean = (href) => href.split("?")[0].replace(/\\/(analytics|photo\\/\\d+|video\\/\\d+)$/, ""); const posts = []; for (const a of document.querySelectorAll('article[data-testid="tweet"]')) { - const time = a.querySelector("time"); + // The quoted post: the clickable box holding post text. Not simply the + // first div[role=link], which can be something else in the post. + const quote = [...a.querySelectorAll('div[role="link"]')].find((d) => d.querySelector('[data-testid="tweetText"], [data-testid="User-Name"]')) || null; + const mine = (el) => !(quote && quote.contains(el)); + // The post's own time, a link to it. On a post's own page that sits below + // the text, after the quoted post's unlinked one. + const time = [...a.querySelectorAll("time")].find((t) => mine(t) && t.closest("a")); const link = time && time.closest("a"); if (!time || !link) continue; - const quote = a.querySelector('div[role="link"]'); - const own = [...a.querySelectorAll('[data-testid="tweetText"]')].filter((t) => !(quote && quote.contains(t))); + const own = [...a.querySelectorAll('[data-testid="tweetText"]')].filter(mine); + const byline = [...a.querySelectorAll('[data-testid="User-Name"]')].find(mine); const post = { - url: link.href.split("?")[0].replace(/\\/(analytics|photo\\/\\d+)$/, ""), - ...nameOf(a), + url: clean(link.href), + ...nameOf(byline ? byline.parentElement : a), time: time.getAttribute("datetime") || "", text: own.map(txt).join("\\n"), metrics: null, @@ -137,23 +206,57 @@ const PROBE = `(() => { if (a.querySelector('[data-testid="tweet-text-show-more-link"]')) post.truncated = true; const ctx = txt(a.querySelector('[data-testid="socialContext"]')); if (ctx) post.context = ctx; - const media = [...a.querySelectorAll('[data-testid="tweetPhoto"], video')].filter((m) => !(quote && quote.contains(m))).length; - if (media) post.media = media; + const media = []; + // A video's still is an <img> in the same box: listed with the video, not as a photo. + for (const img of [...a.querySelectorAll('[data-testid="tweetPhoto"] img')].filter((i) => mine(i) && !i.closest('[data-testid="tweetPhoto"]').querySelector("video"))) { + const m = { type: "photo", url: img.src.replace(/name=\\w+/, "name=large") }; + if (img.alt && img.alt !== "Image") m.alt = img.alt; + media.push(m); + } + for (const v of [...a.querySelectorAll("video")].filter(mine)) { + const m = { type: "video" }; + if (v.poster) m.url = v.poster; + const label = v.getAttribute("aria-label"); + if (label && label !== "Embedded video") m.alt = label; + media.push(m); + } + if (media.length) post.media = media; + const card = [...a.querySelectorAll('[data-testid="card.wrapper"] a[href]')].find(mine); + if (card) { + post.card = { url: card.href }; + const title = txt(card).split("\\n").filter(Boolean).pop(); + if (title) post.card.title = title.slice(0, 200); + } if (quote) { const qt = txt(quote.querySelector('[data-testid="tweetText"]')); - if (qt) post.quoted = { ...nameOf(quote), text: qt }; + if (qt) { + post.quoted = { ...nameOf(quote), text: qt }; + const qtime = quote.querySelector("time"); + if (qtime) post.quoted.time = qtime.getAttribute("datetime") || ""; + const qlink = quote.querySelector('a[href*="/status/"]'); + if (qlink) post.quoted.url = clean(qlink.href); + } } posts.push(post); } - if (posts.length) return { url, state: "posts", posts, note: "" }; + const schema = document.querySelector('[data-testid="UserProfileSchema-test"]'); + const profile = schema ? schema.textContent : null; + if (posts.length) return { url, state: "posts", posts, note: "", profile }; const empty = document.querySelector('[data-testid="emptyState"]'); - if (empty) return { url, state: "empty", posts, note: txt(empty).slice(0, 300) }; - const retry = [...document.querySelectorAll('button, [role="button"]')].find((b) => /^Retry$/.test(txt(b))); - if (retry) { - const col = document.querySelector('[data-testid="primaryColumn"]'); - return { url, state: "error", posts, note: txt(col).slice(0, 300) }; + if (empty) return { url, state: "empty", posts, note: txt(empty).slice(0, 300), profile }; + const col = document.querySelector('[data-testid="primaryColumn"]'); + const said = txt(col); + if (/doesn.t exist|was deleted|account is suspended|these posts are protected|you.re unable to view this post/i.test(said)) { + return { url, state: "empty", posts, note: said.slice(0, 300), profile }; } - return { url, state: "loading", posts, note: "" }; + const retry = [...document.querySelectorAll('button, [role="button"]')].find((b) => /^Retry$/.test(txt(b))); + if (retry) return { url, state: "error", posts, note: said.slice(0, 300), profile }; + return { url, state: "loading", posts, note: "", profile }; +})()`; + +const TRENDS = `(() => { + if (/^\\/(login|i\\/flow\\/login|i\\/jf\\/onboarding|account\\/access)/.test(location.pathname)) return null; + return [...document.querySelectorAll('[data-testid="trend"]')].map((t) => t.innerText); })()`; const SCROLL = `window.scrollBy(0, window.innerHeight * 2), true`; @@ -173,34 +276,41 @@ const finish = (raw: RawPost): XPost => { return { ...rest, ...parseMetrics(metrics) }; }; -export const xSearch = (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => - // The search's own deadline starts once the limiter lets it through, so - // waiting its turn does not eat into it. - limiter.run(outer, () => searchNow(request, outer)); +type Saw = (state: string) => void; -const searchNow = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => { - try { - return await attempt(request, outer); - } catch (e) { - if (!(e instanceof SignedOut)) throw e; - // The sign-in has its own deadline, and the search after it a fresh one. - await signIn(e.at, outer); - return await attempt(request, outer); - } -}; +// --- Shared: limits, sign-in, deadline ----------------------------------- -const attempt = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => { - const sort: XSort = request.sort ?? "latest"; - const searchUrl = buildXSearchUrl(request.query, sort); - const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]); +/** + * Runs one X read: in turn with every other (the limits), within its own + * deadline, signing in and reading again once if X has signed the browser out. + */ +const xRead = <T>(outer: AbortSignal, what: string, read: (page: PageSession, signal: AbortSignal, saw: Saw) => Promise<T>): Promise<T> => + // The deadline starts once the limiter lets the read through, so waiting + // its turn does not eat into it. + limiter.run(outer, async () => { + try { + return await attempt(outer, what, read); + } catch (e) { + if (!(e instanceof SignedOut)) throw e; + // The sign-in has its own deadline, and the read after it a fresh one. + await signIn(e.at, outer); + return await attempt(outer, what, read); + } + }); +const attempt = async <T>( + outer: AbortSignal, + what: string, + read: (page: PageSession, signal: AbortSignal, saw: Saw) => Promise<T>, +): Promise<T> => { + const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]); let last = "nothing yet"; try { - return await run(request, sort, searchUrl, signal, (state) => (last = state)); + return await withPage(signal, (page) => read(page, signal, (state) => (last = state))); } catch (e) { if (signal.aborted && !outer.aborted) { throw new BrowserUnavailableError( - `X search did not finish within ${X_SEARCH_TIMEOUT_MS / 1000} s (${searchUrl}; last seen: ${last}). ` + + `X did not finish ${what} within ${X_SEARCH_TIMEOUT_MS / 1000} s (last seen: ${last}). ` + `X may be slow or stuck on a page this tool does not recognise.`, ); } @@ -210,7 +320,7 @@ const attempt = async (request: XSearchRequest, outer: AbortSignal): Promise<XSe const signedOutError = (at: string, why: string): BrowserUnavailableError => new BrowserUnavailableError( - `X search needs a signed-in X session in castle Chrome, and it is signed out (X sent it to ${at}). ` + + `X needs a signed-in X session in castle Chrome, and it is signed out (X sent it to ${at}). ` + `${why} Tell the user: someone has to sign in to X in that browser again.`, ); @@ -239,33 +349,68 @@ const signIn = async (at: string, outer: AbortSignal): Promise<void> => { } }; -const run = ( - request: XSearchRequest, - sort: XSort, - searchUrl: string, +/** + * Opens `url` and waits for X to render: posts, an empty or error page, or a + * sign-in redirect. `ready` can ask for more before posts count, such as a + * profile's header. + */ +const open = async ( + page: PageSession, + url: string, signal: AbortSignal, - saw: (state: string) => void, -): Promise<XSearchOutcome> => - withPage(signal, async (page) => { - await page.navigate(searchUrl, signal); - - // X renders after the load event: poll until posts, an empty or error - // page, or a sign-in redirect shows up. - let probe: XProbe; - for (;;) { - probe = await page.evaluate<XProbe>(PROBE, signal); - saw(`${probe.state} at ${probe.url}`); - if (probe.state !== "loading") break; - await sleep(POLL_MS, signal); - } - + saw: Saw, + ready: (p: XProbe) => boolean = (p) => p.state !== "loading", +): Promise<XProbe> => { + await page.navigate(url, signal); + for (;;) { + const probe = await page.evaluate<XProbe>(PROBE, signal); + saw(`${probe.state} at ${probe.url}`); if (probe.state === "signed-out") throw new SignedOut(probe.url); if (probe.state === "error") { throw new BrowserUnavailableError( - `X answered with an error page instead of results (${probe.url}): "${probe.note}". This is ` + - `usually X rate-limiting the account; wait a few minutes rather than retrying in a loop.`, + `X answered with an error page (${probe.url}): "${probe.note}". This is usually X rate-limiting ` + + `the account; wait a few minutes rather than retrying in a loop.`, ); } + if (probe.state === "empty" || ready(probe)) return probe; + await sleep(POLL_MS, signal); + } +}; + +/** + * X keeps only the posts near the viewport in the page, so collect by URL + * while scrolling rather than reading once at the end. Stops at `enough`, or + * when scrolling brings nothing new. + */ +const collect = async (page: PageSession, first: XProbe, enough: (seen: Map<string, RawPost>) => boolean, signal: AbortSignal): Promise<RawPost[]> => { + // The newest reading of a post wins: photos and cards render after the + // text, so the first reading of a post can lack them. A Map keeps the order + // posts were first seen in. + const seen = new Map<string, RawPost>(); + const take = (p: XProbe) => p.posts.forEach((post) => seen.set(post.url, post)); + take(first); + await sleep(POLL_MS * 2, signal); + take(await page.evaluate<XProbe>(PROBE, signal)); + for (let i = 0; i < MAX_SCROLLS && !enough(seen); i++) { + const before = seen.size; + await page.evaluate(SCROLL, signal); + // New posts arrive over the next second or so. + for (let w = 0; w < 4 && seen.size === before; w++) { + await sleep(POLL_MS, signal); + take(await page.evaluate<XProbe>(PROBE, signal)); + } + if (seen.size === before) break; // the end of the page + } + return [...seen.values()]; +}; + +// --- The reads --------------------------------------------------------------- + +export const xSearch = (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => { + const sort: XSort = request.sort ?? "latest"; + const searchUrl = buildXSearchUrl(request.query, sort); + return xRead(outer, `the search ${searchUrl}`, async (page, signal, saw) => { + const probe = await open(page, searchUrl, signal, saw); if (probe.state === "empty") { throw new NoResultsError( `X found no posts for ${JSON.stringify(request.query)} (${sort}). X said: "${probe.note}". ` + @@ -273,22 +418,104 @@ const run = ( `or try sort "top" or a wider date range.`, ); } + const posts = await collect(page, probe, (s) => s.size >= request.count, signal); + return { query: request.query, sort, searchUrl, posts: posts.slice(0, request.count).map(finish) }; + }); +}; - // X keeps only the posts near the viewport in the page, so collect by URL - // while scrolling rather than reading once at the end. - const seen = new Map<string, RawPost>(); - const take = (p: XProbe) => p.posts.forEach((post) => seen.has(post.url) || seen.set(post.url, post)); - take(probe); - for (let i = 0; i < MAX_SCROLLS && seen.size < request.count; i++) { - const before = seen.size; - await page.evaluate(SCROLL, signal); - // New posts arrive over the next second or so. - for (let w = 0; w < 4 && seen.size === before; w++) { +export const xPost = (request: XPostRequest, outer: AbortSignal): Promise<XPostOutcome> => { + const id = postIdOf(request.url); + if (!id) return Promise.reject(new BrowserUnavailableError(`not an X post link: ${request.url}`)); + // /i/status/<id> redirects to the post under its author's handle. + const url = `https://x.com/i/status/${id}`; + const isMain = (p: RawPost) => p.url.endsWith(`/status/${id}`); + return xRead(outer, `reading ${url}`, async (page, signal, saw) => { + const probe = await open(page, url, signal, saw, (p) => p.posts.some(isMain)); + if (probe.state === "empty") { + throw new NoResultsError(`X shows no post at ${url}: "${probe.note}". It may be deleted, or its account protected or suspended.`); + } + const posts = await collect(page, probe, (s) => s.size >= request.replies + 1 + [...s.values()].findIndex(isMain), signal); + const at = posts.findIndex(isMain); + if (at < 0) throw new BrowserUnavailableError(`X's page for ${url} did not show the post itself (${probe.url})`); + return { + post: finish(posts[at]!), + thread: posts.slice(0, at).map(finish), + replies: posts.slice(at + 1, at + 1 + request.replies).map(finish), + }; + }); +}; + +type ProfileSchema = { + dateCreated?: string; + relatedLink?: string[]; + mainEntity?: { + additionalName?: string; + name?: string; + description?: string; + homeLocation?: { name?: string }; + image?: { contentUrl?: string }; + url?: string; + interactionStatistic?: { name?: string; userInteractionCount?: number }[]; + }; +}; + +/** The profile from X's JSON-LD block: exact counts, where the page shows "92.2M". */ +export const parseProfile = (json: string, handle: string): XProfile => { + const s = JSON.parse(json) as ProfileSchema; + const e = s.mainEntity ?? {}; + const count = (name: string) => e.interactionStatistic?.find((i) => i.name === name)?.userInteractionCount; + const profile: XProfile = { name: e.name ?? handle, handle: `@${e.additionalName ?? handle}`, url: e.url ?? `https://x.com/${handle}` }; + if (e.description) profile.bio = e.description; + if (e.homeLocation?.name) profile.location = e.homeLocation.name; + const site = s.relatedLink?.find((l) => !/^https?:\/\/t\.co\//.test(l)); + if (site) profile.website = site; + if (s.dateCreated) profile.joined = s.dateCreated; + const followers = count("Follows"); + const following = count("Friends"); + const posts = count("Tweets"); + if (followers !== undefined) profile.followers = followers; + if (following !== undefined) profile.following = following; + if (posts !== undefined) profile.posts = posts; + if (e.image?.contentUrl) profile.avatar = e.image.contentUrl; + return profile; +}; + +export const xUser = (request: XUserRequest, outer: AbortSignal): Promise<XUserOutcome> => { + const handle = handleOf(request.handle); + if (!handle) return Promise.reject(new BrowserUnavailableError(`not an X handle: ${request.handle}`)); + const url = `https://x.com/${handle}`; + return xRead(outer, `reading ${url}`, async (page, signal, saw) => { + // The header renders before the posts; wait for both, or for X to say + // the account has none to show. + const probe = await open(page, url, signal, saw, (p) => p.profile !== null && p.state !== "loading"); + if (!probe.profile) { + throw new NoResultsError(`X shows no account @${handle}: "${probe.note}". Check the handle.`); + } + const profile = parseProfile(probe.profile, handle); + const posts = probe.state === "posts" && request.count > 0 ? await collect(page, probe, (s) => s.size >= request.count, signal) : []; + return { profile, posts: posts.slice(0, request.count).map(finish) }; + }); +}; + +export const xTrends = (tab: XTrendTab, outer: AbortSignal): Promise<XTrendsOutcome> => { + const url = `https://x.com/explore/tabs/${tab}`; + return xRead(outer, `reading ${url}`, async (page, signal, saw) => { + await page.navigate(url, signal); + for (;;) { + const cells = await page.evaluate<string[] | null>(TRENDS, signal); + if (cells === null) throw new SignedOut(url); + saw(`${cells.length} trends at ${url}`); + if (cells.length) { + // The list renders in one go; a moment more lets the last cells fill. await sleep(POLL_MS, signal); - take(await page.evaluate<XProbe>(PROBE, signal)); + const all = (await page.evaluate<string[] | null>(TRENDS, signal)) ?? cells; + return { tab, trends: all.map(parseTrend).filter((t): t is XTrend => t !== undefined) }; } - if (seen.size === before) break; // the end of the results + const probe = await page.evaluate<XProbe>(PROBE, signal); + if (probe.state === "error") { + throw new BrowserUnavailableError(`X answered with an error page (${url}): "${probe.note}".`); + } + await sleep(POLL_MS, signal); } - - return { query: request.query, sort, searchUrl, posts: [...seen.values()].slice(0, request.count).map(finish) }; }); +}; diff --git a/castle-search/test/http.test.ts b/castle-search/test/http.test.ts index b2c90ab..c076863 100644 --- a/castle-search/test/http.test.ts +++ b/castle-search/test/http.test.ts @@ -116,3 +116,73 @@ test("X's action-bar label parses, singular and plural", () => { test("the OpenAPI description names the operation Open WebUI will offer", () => { assert.equal(OPENAPI.paths["/x-search"].post.operationId, "search_x"); }); + +import { handleXPost, handleXTrends, handleXUser } from "../src/http.ts"; +import { handleOf, parseProfile, parseTrend, postIdOf } from "../src/x-search.ts"; + +test("post links of every shape give the post id", () => { + assert.equal(postIdOf("https://x.com/NASA/status/2122740072386609201"), "2122740072386609201"); + assert.equal(postIdOf("https://twitter.com/nasa/status/2122740072386609201?s=20"), "2122740072386609201"); + assert.equal(postIdOf("https://x.com/i/web/status/2122740072386609201"), "2122740072386609201"); + assert.equal(postIdOf("2122740072386609201"), "2122740072386609201"); + assert.equal(postIdOf("https://x.com/NASA"), undefined); +}); + +test("handles are read from @name, name or a profile link", () => { + assert.equal(handleOf("@NASA"), "NASA"); + assert.equal(handleOf("https://x.com/nasa/"), "nasa"); + assert.equal(handleOf("not a handle"), undefined); + assert.equal(handleOf("waytoolonghandle_123"), undefined); +}); + +test("a trend cell's lines become a trend", () => { + assert.deepEqual(parseTrend("2\n·\nVideo game genres · Trending\nGTA 6\nTrending with #GTAVI, Metacritic"), { + rank: 2, context: "Video game genres · Trending", name: "GTA 6", query: "GTA 6", detail: "Trending with #GTAVI, Metacritic", + }); + assert.deepEqual(parseTrend("Trending in United Kingdom\nGlastonbury"), { context: "Trending in United Kingdom", name: "Glastonbury", query: "Glastonbury" }); + assert.equal(parseTrend("1"), undefined); +}); + +test("a profile's JSON-LD gives exact counts", () => { + const p = parseProfile( + JSON.stringify({ + dateCreated: "2007-12-19T20:20:32.000Z", + relatedLink: ["https://t.co/9NkQJKAnuU", "http://www.nasa.gov/"], + mainEntity: { + additionalName: "NASA", name: "NASA", description: "Making the seemingly impossible, possible.", + homeLocation: { name: "Pale Blue Dot" }, url: "https://x.com/NASA", + interactionStatistic: [ + { name: "Follows", userInteractionCount: 92225170 }, + { name: "Friends", userInteractionCount: 115 }, + { name: "Tweets", userInteractionCount: 74553 }, + ], + }, + }), + "nasa", + ); + assert.equal(p.handle, "@NASA"); + assert.equal(p.followers, 92225170); + assert.equal(p.following, 115); + assert.equal(p.posts, 74553); + assert.equal(p.website, "http://www.nasa.gov/"); + assert.equal(p.location, "Pale Blue Dot"); +}); + +test("the X reads check what they are given before touching the browser", async () => { + const never = async (): Promise<never> => { + throw new Error("must not run"); + }; + assert.equal((await handleXPost(JSON.stringify({ url: "https://x.com/NASA" }), never, signal)).status, 400); + assert.equal((await handleXUser(JSON.stringify({ handle: "two words" }), never, signal)).status, 400); + assert.equal((await handleXTrends(JSON.stringify({ tab: "music" }), never, signal)).status, 400); + assert.equal((await handleXPost("[1]", never, signal)).status, 400); + const seen: unknown[] = []; + await handleXPost(JSON.stringify({ url: "https://x.com/a/status/12345678", replies: 99 }), async (r) => (seen.push(r), { post: {}, thread: [], replies: [] }) as never, signal); + await handleXTrends("", async (t) => (seen.push(t), { tab: t, trends: [] }), signal); + assert.deepEqual(seen, [{ url: "https://x.com/a/status/12345678", replies: 20 }, "trending"]); +}); + +test("every X read is in the OpenAPI description", () => { + assert.deepEqual(Object.keys(OPENAPI.paths), ["/x-search", "/x-post", "/x-user", "/x-trends"]); + assert.equal(OPENAPI.info.title, "X.com"); +}); |
