aboutsummaryrefslogtreecommitdiff
path: root/castle-search/src
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-11-18 12:11:11 +0000
committerIgor Soarez <igor@soarez.org>2026-11-18 12:11:11 +0000
commitf8835d53b96bb6e4152bc5bf84e61b8b54396008 (patch)
treed3815441efcb5a87498dfff8054c0786c84afffd /castle-search/src
parent910ac7cd424f6401ea042a84603c42ba055994f8 (diff)
castle-search: X.com: read posts whole with thread and replies, profiles, trends
/x-post, /x-user and /x-trends beside /x-search, sharing its limits and its sign-in. Posts now carry photos (image link, alt text), videos, link cards and the quoted post's time. A post's own page lists the quoted post's time first; the probe now takes the post's own linked time.
Diffstat (limited to 'castle-search/src')
-rw-r--r--castle-search/src/http.ts231
-rw-r--r--castle-search/src/x-search.ts419
2 files changed, 495 insertions, 155 deletions
diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts
index 17c8c70..8efc4c5 100644
--- a/castle-search/src/http.ts
+++ b/castle-search/src/http.ts
@@ -20,17 +20,33 @@
* an address only trusted peers reach. Each search opens and closes one
* background tab in a browser someone may be looking at.
*
- * POST /x-search searches X instead (x-search.ts), through the browser's
- * signed-in X session, and GET /openapi.json describes it, so that Open WebUI
- * can offer it as a tool: `{ "query", "count", "sort" }` in, `{ "query", "sort",
- * "posts": [...] }` out, 502 with `{ "error" }` on failure.
+ * POST /x-search, /x-post, /x-user and /x-trends read X instead (x-search.ts),
+ * through the browser's signed-in X session, and GET /openapi.json describes
+ * them, so that Open WebUI can offer them as tools ("X.com"). Each answers 502
+ * with `{ "error" }` on failure.
*/
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { messageOf } from "./errors.ts";
import { disconnect, search, type SearchOutcome, type SearchRequest } from "./search.ts";
-import { xSearch, type XSearchOutcome, type XSearchRequest } from "./x-search.ts";
+import {
+ handleOf,
+ postIdOf,
+ TREND_TABS,
+ type XPostOutcome,
+ type XPostRequest,
+ xPost,
+ xSearch,
+ type XSearchOutcome,
+ type XSearchRequest,
+ type XTrendsOutcome,
+ type XTrendTab,
+ xTrends,
+ xUser,
+ type XUserOutcome,
+ type XUserRequest,
+} from "./x-search.ts";
export const DEFAULT_LISTEN = "127.0.0.1:11158";
const MAX_BODY_BYTES = 64 * 1024;
@@ -75,47 +91,105 @@ export const handleSearch = async (raw: string, run: Searcher, signal: AbortSign
};
export type XSearcher = (request: XSearchRequest, signal: AbortSignal) => Promise<XSearchOutcome>;
+export type XPostReader = (request: XPostRequest, signal: AbortSignal) => Promise<XPostOutcome>;
+export type XUserReader = (request: XUserRequest, signal: AbortSignal) => Promise<XUserOutcome>;
+export type XTrendsReader = (tab: XTrendTab, signal: AbortSignal) => Promise<XTrendsOutcome>;
const X_DEFAULT_COUNT = 10;
-/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */
-export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise<Reply> => {
- let parsed: unknown;
+const jsonBody = (raw: string): Record<string, unknown> | Reply => {
try {
- parsed = JSON.parse(raw);
+ const parsed: unknown = JSON.parse(raw || "{}");
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as Record<string, unknown>;
} catch {
- return { status: 400, body: { error: "request body is not JSON" } };
+ // fall through
}
- const { query, count, sort } = (parsed ?? {}) as { query?: unknown; count?: unknown; sort?: unknown };
+ return { status: 400, body: { error: "request body is not a JSON object" } };
+};
+const isReply = (v: unknown): v is Reply => typeof (v as Reply).status === "number" && "body" in (v as Reply);
+
+const countOf = (v: unknown, fallback: number, min = 1): number => {
+ const n = typeof v === "number" && Number.isFinite(v) ? Math.trunc(v) : fallback;
+ return Math.min(MAX_COUNT, Math.max(min, n));
+};
+
+const answer = async (work: () => Promise<unknown>): Promise<Reply> => {
+ try {
+ return { status: 200, body: await work() };
+ } catch (e) {
+ return { status: 502, body: { error: messageOf(e) } };
+ }
+};
+
+/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */
+export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ const { query, count, sort } = body;
if (typeof query !== "string" || query.trim() === "") {
return { status: 400, body: { error: "query must be a non-empty string" } };
}
if (sort !== undefined && sort !== null && sort !== "latest" && sort !== "top") {
return { status: 400, body: { error: 'sort must be "latest" or "top"' } };
}
- const n = typeof count === "number" && Number.isFinite(count) ? Math.trunc(count) : X_DEFAULT_COUNT;
- try {
- const outcome = await run(
- { query: query.trim(), count: Math.min(MAX_COUNT, Math.max(1, n)), sort: sort ?? "latest" },
- signal,
- );
- return { status: 200, body: { query: outcome.query, sort: outcome.sort, posts: outcome.posts } };
- } catch (e) {
- return { status: 502, body: { error: messageOf(e) } };
+ return answer(async () => {
+ const outcome = await run({ query: query.trim(), count: countOf(count, X_DEFAULT_COUNT), sort: sort ?? "latest" }, signal);
+ return { query: outcome.query, sort: outcome.sort, posts: outcome.posts };
+ });
+};
+
+/** POST /x-post: `{ url, replies? }` in, `{ post, thread, replies }` out. */
+export const handleXPost = async (raw: string, run: XPostReader, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ if (typeof body["url"] !== "string" || !postIdOf(body["url"])) {
+ return { status: 400, body: { error: "url must be a link to a post on X (x.com/<handle>/status/<id>) or its id" } };
}
+ const url = body["url"];
+ return answer(() => run({ url, replies: countOf(body["replies"], X_DEFAULT_COUNT, 0) }, signal));
+};
+
+/** POST /x-user: `{ handle, count? }` in, `{ profile, posts }` out. */
+export const handleXUser = async (raw: string, run: XUserReader, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ if (typeof body["handle"] !== "string" || !handleOf(body["handle"])) {
+ return { status: 400, body: { error: "handle must be an X handle, such as nasa or @nasa" } };
+ }
+ const handle = body["handle"];
+ return answer(() => run({ handle, count: countOf(body["count"], X_DEFAULT_COUNT, 0) }, signal));
+};
+
+/** POST /x-trends: `{ tab? }` in, `{ tab, trends }` out. */
+export const handleXTrends = async (raw: string, run: XTrendsReader, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ const tab = body["tab"] ?? "trending";
+ if (!TREND_TABS.includes(tab as XTrendTab)) {
+ return { status: 400, body: { error: `tab must be one of ${TREND_TABS.join(", ")}` } };
+ }
+ return answer(() => run(tab as XTrendTab, signal));
+};
+
+const jsonSchema = (properties: Record<string, unknown>, required: string[]) => ({
+ required: true,
+ content: { "application/json": { schema: { type: "object", required, properties } } },
+});
+const RESPONSES = {
+ "200": { description: "What X shows", content: { "application/json": { schema: { type: "object" } } } },
+ "502": { description: "X could not be read; error says why", content: { "application/json": { schema: { type: "object" } } } },
};
/**
- * The OpenAPI description Open WebUI reads to offer /x-search as a tool. Its
- * descriptions are what the model sees, so they say what the tool is for and
- * how to write a query.
+ * The OpenAPI description Open WebUI reads to offer the X reads as tools. Its
+ * descriptions are what the model sees, so they say what each is for.
*/
export const OPENAPI = {
openapi: "3.1.0",
info: {
- title: "castle X search",
- version: "1.0.0",
- description: "Searches posts on X (Twitter) through a signed-in browser. Free per search.",
+ title: "X.com",
+ version: "2.0.0",
+ description: "Reads X (Twitter) through a signed-in browser: search, posts with replies, profiles, trends. Free.",
},
paths: {
"/x-search": {
@@ -127,27 +201,67 @@ export const OPENAPI = {
"reactions to news, posts about a topic. The query uses X's search syntax: from:handle, to:handle, " +
'"exact phrase", OR, -word, since:YYYY-MM-DD, until:YYYY-MM-DD, min_faves:N, filter:media, ' +
"-filter:replies, lang:en. A handle is written without @ after from:. Each post comes with its " +
- "link: cite it. Text cut short by X is marked truncated.",
- requestBody: {
- required: true,
- content: {
- "application/json": {
- schema: {
- type: "object",
- required: ["query"],
- properties: {
- query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" },
- count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" },
- sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" },
- },
- },
- },
+ "link: cite it. A post marked truncated was cut short by X: read_x_post gives it whole. Photos " +
+ "come with their image link and alt text.",
+ requestBody: jsonSchema(
+ {
+ query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" },
+ count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" },
+ sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" },
},
- },
- responses: {
- "200": { description: "Matching posts", content: { "application/json": { schema: { type: "object" } } } },
- "502": { description: "The search failed; error says why", content: { "application/json": { schema: { type: "object" } } } },
- },
+ ["query"],
+ ),
+ responses: RESPONSES,
+ },
+ },
+ "/x-post": {
+ post: {
+ operationId: "read_x_post",
+ summary: "Read one post on X, whole, with its thread and replies",
+ description:
+ "Read a post on X from its link: its full text (never truncated), photos with their image link " +
+ "and alt text, its link card, the posts above it when it is a reply (thread, oldest first), and " +
+ "the top replies under it. Use it to read a long post, a thread, or what people answered.",
+ requestBody: jsonSchema(
+ {
+ url: { type: "string", description: "The post's link, x.com/<handle>/status/<id>, or the id" },
+ replies: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many replies" },
+ },
+ ["url"],
+ ),
+ responses: RESPONSES,
+ },
+ },
+ "/x-user": {
+ post: {
+ operationId: "read_x_profile",
+ summary: "Read an X profile and its latest posts",
+ description:
+ "Read an X account's profile: name, bio, location, website, when it joined, exact follower, " +
+ "following and post counts, then its latest posts as its profile shows them (a pinned post and " +
+ "reposts included, marked in context).",
+ requestBody: jsonSchema(
+ {
+ handle: { type: "string", description: "The account's handle, such as nasa or @nasa" },
+ count: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many of its posts" },
+ },
+ ["handle"],
+ ),
+ responses: RESPONSES,
+ },
+ },
+ "/x-trends": {
+ post: {
+ operationId: "x_trends",
+ summary: "What is trending on X now",
+ description:
+ "List what X shows as trending now, for the signed-in account's region (the United Kingdom). " +
+ "Each trend has a query: pass it to search_x for the posts.",
+ requestBody: jsonSchema(
+ { tab: { type: "string", enum: [...TREND_TABS], default: "trending", description: "Which explore tab" } },
+ [],
+ ),
+ responses: RESPONSES,
},
},
},
@@ -199,9 +313,15 @@ const main = (): void => {
send(res, { status: 200, body: OPENAPI });
return;
}
- const isX = path === "/x-search";
- if (req.method !== "POST" || (path !== "/" && path !== "/search" && !isX)) {
- send(res, { status: 404, body: { error: "POST a search to / or /search, or an X search to /x-search" } });
+ const xRoutes: Record<string, (raw: string, signal: AbortSignal) => Promise<Reply>> = {
+ "/x-search": (raw, signal) => handleXSearch(raw, xSearch, signal),
+ "/x-post": (raw, signal) => handleXPost(raw, xPost, signal),
+ "/x-user": (raw, signal) => handleXUser(raw, xUser, signal),
+ "/x-trends": (raw, signal) => handleXTrends(raw, xTrends, signal),
+ };
+ const xRoute = xRoutes[path ?? ""];
+ if (req.method !== "POST" || (path !== "/" && path !== "/search" && !xRoute)) {
+ send(res, { status: 404, body: { error: "POST a search to / or /search, or an X read to /x-search, /x-post, /x-user or /x-trends" } });
return;
}
// Stop the search, and free its tab, if the client goes away first.
@@ -217,16 +337,9 @@ const main = (): void => {
return;
}
const started = Date.now();
- const reply = isX
- ? await handleXSearch(raw, xSearch, controller.signal)
- : await handleSearch(raw, runSearch, controller.signal);
- const outcome =
- reply.status !== 200
- ? `error ${reply.status}`
- : isX
- ? `${(reply.body as { posts: unknown[] }).posts.length} posts`
- : `${(reply.body as unknown[]).length} results`;
- console.log(`${isX ? "x-search" : "search"} ${outcome} in ${Date.now() - started} ms`);
+ const reply = xRoute ? await xRoute(raw, controller.signal) : await handleSearch(raw, runSearch, controller.signal);
+ const outcome = reply.status !== 200 ? `error ${reply.status}` : xRoute ? "ok" : `${(reply.body as unknown[]).length} results`;
+ console.log(`${xRoute ? path!.slice(1) : "search"} ${outcome} in ${Date.now() - started} ms`);
if (reply.status !== 200) console.log(` ${(reply.body as { error: string }).error}`);
send(res, reply);
});
diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts
index 7c8badf..174906e 100644
--- a/castle-search/src/x-search.ts
+++ b/castle-search/src/x-search.ts
@@ -1,35 +1,38 @@
/**
- * Search X (x.com) through the browser's own signed-in session.
+ * Read X (x.com) through the browser's own signed-in session: search posts,
+ * read one post with its thread and replies, read a profile, list trends.
*
- * X serves search only to a signed-in session and renders it from JavaScript,
- * so the page is the only interface there is: open x.com/search in a background
- * tab, wait for posts to render, read them, scroll for more, close the tab.
- * Free per search, unlike xAI's x_search ($5 per 1,000 posts returned).
+ * X serves all of this only to a signed-in session and renders it from
+ * JavaScript, so the page is the only interface there is: open the page in a
+ * background tab, wait for it to render, read it, scroll for more, close the
+ * tab. Free, unlike xAI's x_search ($5 per 1,000 posts returned).
*
- * The query is X's own search syntax, passed through untouched: from:handle,
- * to:handle, since:2026-11-01, until:..., min_faves:100, filter:media,
- * -filter:replies, lang:en, "exact phrase", OR.
+ * A search query is X's own search syntax, passed through untouched:
+ * from:handle, to:handle, since:2026-11-01, until:..., min_faves:100,
+ * filter:media, -filter:replies, lang:en, "exact phrase", OR.
*
- * Every failure throws, as for the engine searches (errors.ts): signed out, an
- * X error page, or nothing rendering are reported as such, never as "no posts".
- * Only X's own "No results" page is an empty answer, and even that is reported
- * as an error naming the query, since an over-narrow query is the usual cause.
+ * Every read shares one set of limits (x-limits.ts) and, signed out, signs the
+ * browser back in once (x-login.ts). Every failure throws, as for the engine
+ * searches (errors.ts): signed out, an X error page, or nothing rendering are
+ * reported as such, never as an empty answer. X's own "No results" or "doesn't
+ * exist" page is reported as an error naming what was asked for.
*/
+import type { PageSession } from "./cdp.ts";
import { BrowserUnavailableError, NoResultsError } from "./errors.ts";
import { withPage } from "./search.ts";
import { XLimiter } from "./x-limits.ts";
import { loadCredentials, X_LOGIN_TIMEOUT_MS, xLogin } from "./x-login.ts";
-/** Shared by every X search in this process: see x-limits.ts. */
+/** Shared by every X read in this process: see x-limits.ts. */
const limiter = new XLimiter();
/**
- * Signed out, a search signs the browser back in (x-login.ts) when
- * CCS_X_LOGIN_FILE names the credentials, then searches again, once. At most
- * one sign-in attempt per LOGIN_COOLDOWN_MS: a sign-in that keeps failing
- * is what gets an account locked, so until the cooldown has passed every
- * search fails with the reason the last attempt gave.
+ * Signed out, a read signs the browser back in (x-login.ts) when
+ * CCS_X_LOGIN_FILE names the credentials, then reads again, once. At most one
+ * sign-in attempt per LOGIN_COOLDOWN_MS: a sign-in that keeps failing is what
+ * gets an account locked, so until the cooldown has passed every read fails
+ * with the reason the last attempt gave.
*/
export const LOGIN_COOLDOWN_MS = 30 * 60_000;
let lastLogin: { at: number; error?: string } | undefined;
@@ -44,6 +47,8 @@ class SignedOut extends Error {
export type XSort = "latest" | "top";
+export type XMedia = { type: "photo" | "video"; url?: string; alt?: string };
+
export type XPost = {
url: string;
author: string;
@@ -51,7 +56,7 @@ export type XPost = {
/** ISO 8601, from the post's <time datetime>. */
time: string;
text: string;
- /** X cut the text short behind "Show more"; the full post is at url. */
+ /** X cut the text short behind "Show more"; x_post reads it whole. */
truncated?: true;
/** "@someone reposted", "Pinned" and the like. */
context?: string;
@@ -59,18 +64,48 @@ export type XPost = {
reposts?: number;
likes?: number;
views?: number;
- /** Count of photos and videos attached. */
- media?: number;
- quoted?: { author: string; handle: string; text: string };
+ /** Photos (with X's image URL and alt text) and videos (with a still). */
+ media?: XMedia[];
+ /** A link card: the page the post links to. */
+ card?: { url: string; title?: string };
+ quoted?: { author: string; handle: string; text: string; time?: string; url?: string };
};
export type XSearchRequest = { query: string; count: number; sort?: XSort | undefined };
-
export type XSearchOutcome = { query: string; sort: XSort; searchUrl: string; posts: XPost[] };
+export type XPostRequest = { url: string; replies: number };
+export type XPostOutcome = {
+ post: XPost;
+ /** The posts above it in its thread, oldest first, when it is a reply. */
+ thread: XPost[];
+ replies: XPost[];
+};
+
+export type XUserRequest = { handle: string; count: number };
+export type XProfile = {
+ name: string;
+ handle: string;
+ url: string;
+ bio?: string;
+ location?: string;
+ website?: string;
+ joined?: string;
+ followers?: number;
+ following?: number;
+ posts?: number;
+ avatar?: string;
+};
+export type XUserOutcome = { profile: XProfile; posts: XPost[] };
+
+export const TREND_TABS = ["trending", "news", "sports", "entertainment", "for_you"] as const;
+export type XTrendTab = (typeof TREND_TABS)[number];
+export type XTrend = { rank?: number; name: string; context?: string; detail?: string; query: string };
+export type XTrendsOutcome = { tab: XTrendTab; trends: XTrend[] };
+
/** X shows ~5-10 posts per screen; scroll this many times at most for more. */
const MAX_SCROLLS = 6;
-/** Wall clock for one search: render, scrolls and all. */
+/** Wall clock for one read: render, scrolls and all. */
export const X_SEARCH_TIMEOUT_MS = 25_000;
const POLL_MS = 400;
@@ -81,6 +116,16 @@ export const buildXSearchUrl = (query: string, sort: XSort): string => {
return `https://x.com/search?${params.toString()}`;
};
+/** The post id in an X or Twitter post link, or a bare id. */
+export const postIdOf = (url: string): string | undefined =>
+ /^\d{5,25}$/.test(url.trim()) ? url.trim() : /(?:x|twitter)\.com\/(?:[^/]+|i(?:\/web)?)\/status(?:es)?\/(\d+)/i.exec(url)?.[1];
+
+/** A handle as X writes it, from "@nasa", "nasa" or a profile link. */
+export const handleOf = (s: string): string | undefined => {
+ const m = /^(?:https?:\/\/(?:www\.)?(?:x|twitter)\.com\/)?@?([A-Za-z0-9_]{1,15})\/?$/.exec(s.trim());
+ return m?.[1];
+};
+
/** Parses X's action-bar label: "441 replies, 971 reposts, 8852 likes, 1247 bookmarks, 16029782 views". */
export const parseMetrics = (label: string | null | undefined): Partial<Record<"replies" | "reposts" | "likes" | "views", number>> => {
const out: Partial<Record<"replies" | "reposts" | "likes" | "views", number>> = {};
@@ -95,6 +140,21 @@ export const parseMetrics = (label: string | null | undefined): Partial<Record<"
return out;
};
+/** One trend cell's lines: "2", "·", "Gaming · Trending", "GTA 6", "Trending with #GTAVI". */
+export const parseTrend = (text: string): XTrend | undefined => {
+ const lines = text.split("\n").map((l) => l.trim()).filter((l) => l && l !== "·");
+ let rank: number | undefined;
+ if (lines[0] && /^\d+$/.test(lines[0])) rank = Number(lines.shift());
+ const context = lines.shift();
+ const name = lines.shift();
+ if (!name) return undefined;
+ const trend: XTrend = { name, query: name };
+ if (rank !== undefined) trend.rank = rank;
+ if (context) trend.context = context;
+ if (lines.length) trend.detail = lines.join("; ");
+ return trend;
+};
+
type RawPost = Omit<XPost, "replies" | "reposts" | "likes" | "views"> & { metrics: string | null };
type XProbe = {
@@ -102,6 +162,8 @@ type XProbe = {
state: "signed-out" | "posts" | "empty" | "error" | "loading";
posts: RawPost[];
note: string;
+ /** The profile's JSON-LD, on a profile page. */
+ profile: string | null;
};
/**
@@ -111,23 +173,30 @@ type XProbe = {
const PROBE = `(() => {
const url = location.href;
if (/^\\/(login|i\\/flow\\/login|i\\/jf\\/onboarding|account\\/access)/.test(location.pathname)) {
- return { url, state: "signed-out", posts: [], note: location.pathname };
+ return { url, state: "signed-out", posts: [], note: location.pathname, profile: null };
}
const txt = (el) => (el ? el.innerText.trim() : "");
const nameOf = (scope) => {
const lines = txt(scope.querySelector('[data-testid="User-Name"]')).split("\\n");
return { author: lines[0] || "", handle: lines.find((l) => l.startsWith("@")) || "" };
};
+ const clean = (href) => href.split("?")[0].replace(/\\/(analytics|photo\\/\\d+|video\\/\\d+)$/, "");
const posts = [];
for (const a of document.querySelectorAll('article[data-testid="tweet"]')) {
- const time = a.querySelector("time");
+ // The quoted post: the clickable box holding post text. Not simply the
+ // first div[role=link], which can be something else in the post.
+ const quote = [...a.querySelectorAll('div[role="link"]')].find((d) => d.querySelector('[data-testid="tweetText"], [data-testid="User-Name"]')) || null;
+ const mine = (el) => !(quote && quote.contains(el));
+ // The post's own time, a link to it. On a post's own page that sits below
+ // the text, after the quoted post's unlinked one.
+ const time = [...a.querySelectorAll("time")].find((t) => mine(t) && t.closest("a"));
const link = time && time.closest("a");
if (!time || !link) continue;
- const quote = a.querySelector('div[role="link"]');
- const own = [...a.querySelectorAll('[data-testid="tweetText"]')].filter((t) => !(quote && quote.contains(t)));
+ const own = [...a.querySelectorAll('[data-testid="tweetText"]')].filter(mine);
+ const byline = [...a.querySelectorAll('[data-testid="User-Name"]')].find(mine);
const post = {
- url: link.href.split("?")[0].replace(/\\/(analytics|photo\\/\\d+)$/, ""),
- ...nameOf(a),
+ url: clean(link.href),
+ ...nameOf(byline ? byline.parentElement : a),
time: time.getAttribute("datetime") || "",
text: own.map(txt).join("\\n"),
metrics: null,
@@ -137,23 +206,57 @@ const PROBE = `(() => {
if (a.querySelector('[data-testid="tweet-text-show-more-link"]')) post.truncated = true;
const ctx = txt(a.querySelector('[data-testid="socialContext"]'));
if (ctx) post.context = ctx;
- const media = [...a.querySelectorAll('[data-testid="tweetPhoto"], video')].filter((m) => !(quote && quote.contains(m))).length;
- if (media) post.media = media;
+ const media = [];
+ // A video's still is an <img> in the same box: listed with the video, not as a photo.
+ for (const img of [...a.querySelectorAll('[data-testid="tweetPhoto"] img')].filter((i) => mine(i) && !i.closest('[data-testid="tweetPhoto"]').querySelector("video"))) {
+ const m = { type: "photo", url: img.src.replace(/name=\\w+/, "name=large") };
+ if (img.alt && img.alt !== "Image") m.alt = img.alt;
+ media.push(m);
+ }
+ for (const v of [...a.querySelectorAll("video")].filter(mine)) {
+ const m = { type: "video" };
+ if (v.poster) m.url = v.poster;
+ const label = v.getAttribute("aria-label");
+ if (label && label !== "Embedded video") m.alt = label;
+ media.push(m);
+ }
+ if (media.length) post.media = media;
+ const card = [...a.querySelectorAll('[data-testid="card.wrapper"] a[href]')].find(mine);
+ if (card) {
+ post.card = { url: card.href };
+ const title = txt(card).split("\\n").filter(Boolean).pop();
+ if (title) post.card.title = title.slice(0, 200);
+ }
if (quote) {
const qt = txt(quote.querySelector('[data-testid="tweetText"]'));
- if (qt) post.quoted = { ...nameOf(quote), text: qt };
+ if (qt) {
+ post.quoted = { ...nameOf(quote), text: qt };
+ const qtime = quote.querySelector("time");
+ if (qtime) post.quoted.time = qtime.getAttribute("datetime") || "";
+ const qlink = quote.querySelector('a[href*="/status/"]');
+ if (qlink) post.quoted.url = clean(qlink.href);
+ }
}
posts.push(post);
}
- if (posts.length) return { url, state: "posts", posts, note: "" };
+ const schema = document.querySelector('[data-testid="UserProfileSchema-test"]');
+ const profile = schema ? schema.textContent : null;
+ if (posts.length) return { url, state: "posts", posts, note: "", profile };
const empty = document.querySelector('[data-testid="emptyState"]');
- if (empty) return { url, state: "empty", posts, note: txt(empty).slice(0, 300) };
- const retry = [...document.querySelectorAll('button, [role="button"]')].find((b) => /^Retry$/.test(txt(b)));
- if (retry) {
- const col = document.querySelector('[data-testid="primaryColumn"]');
- return { url, state: "error", posts, note: txt(col).slice(0, 300) };
+ if (empty) return { url, state: "empty", posts, note: txt(empty).slice(0, 300), profile };
+ const col = document.querySelector('[data-testid="primaryColumn"]');
+ const said = txt(col);
+ if (/doesn.t exist|was deleted|account is suspended|these posts are protected|you.re unable to view this post/i.test(said)) {
+ return { url, state: "empty", posts, note: said.slice(0, 300), profile };
}
- return { url, state: "loading", posts, note: "" };
+ const retry = [...document.querySelectorAll('button, [role="button"]')].find((b) => /^Retry$/.test(txt(b)));
+ if (retry) return { url, state: "error", posts, note: said.slice(0, 300), profile };
+ return { url, state: "loading", posts, note: "", profile };
+})()`;
+
+const TRENDS = `(() => {
+ if (/^\\/(login|i\\/flow\\/login|i\\/jf\\/onboarding|account\\/access)/.test(location.pathname)) return null;
+ return [...document.querySelectorAll('[data-testid="trend"]')].map((t) => t.innerText);
})()`;
const SCROLL = `window.scrollBy(0, window.innerHeight * 2), true`;
@@ -173,34 +276,41 @@ const finish = (raw: RawPost): XPost => {
return { ...rest, ...parseMetrics(metrics) };
};
-export const xSearch = (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> =>
- // The search's own deadline starts once the limiter lets it through, so
- // waiting its turn does not eat into it.
- limiter.run(outer, () => searchNow(request, outer));
+type Saw = (state: string) => void;
-const searchNow = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => {
- try {
- return await attempt(request, outer);
- } catch (e) {
- if (!(e instanceof SignedOut)) throw e;
- // The sign-in has its own deadline, and the search after it a fresh one.
- await signIn(e.at, outer);
- return await attempt(request, outer);
- }
-};
+// --- Shared: limits, sign-in, deadline -----------------------------------
-const attempt = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => {
- const sort: XSort = request.sort ?? "latest";
- const searchUrl = buildXSearchUrl(request.query, sort);
- const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]);
+/**
+ * Runs one X read: in turn with every other (the limits), within its own
+ * deadline, signing in and reading again once if X has signed the browser out.
+ */
+const xRead = <T>(outer: AbortSignal, what: string, read: (page: PageSession, signal: AbortSignal, saw: Saw) => Promise<T>): Promise<T> =>
+ // The deadline starts once the limiter lets the read through, so waiting
+ // its turn does not eat into it.
+ limiter.run(outer, async () => {
+ try {
+ return await attempt(outer, what, read);
+ } catch (e) {
+ if (!(e instanceof SignedOut)) throw e;
+ // The sign-in has its own deadline, and the read after it a fresh one.
+ await signIn(e.at, outer);
+ return await attempt(outer, what, read);
+ }
+ });
+const attempt = async <T>(
+ outer: AbortSignal,
+ what: string,
+ read: (page: PageSession, signal: AbortSignal, saw: Saw) => Promise<T>,
+): Promise<T> => {
+ const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]);
let last = "nothing yet";
try {
- return await run(request, sort, searchUrl, signal, (state) => (last = state));
+ return await withPage(signal, (page) => read(page, signal, (state) => (last = state)));
} catch (e) {
if (signal.aborted && !outer.aborted) {
throw new BrowserUnavailableError(
- `X search did not finish within ${X_SEARCH_TIMEOUT_MS / 1000} s (${searchUrl}; last seen: ${last}). ` +
+ `X did not finish ${what} within ${X_SEARCH_TIMEOUT_MS / 1000} s (last seen: ${last}). ` +
`X may be slow or stuck on a page this tool does not recognise.`,
);
}
@@ -210,7 +320,7 @@ const attempt = async (request: XSearchRequest, outer: AbortSignal): Promise<XSe
const signedOutError = (at: string, why: string): BrowserUnavailableError =>
new BrowserUnavailableError(
- `X search needs a signed-in X session in castle Chrome, and it is signed out (X sent it to ${at}). ` +
+ `X needs a signed-in X session in castle Chrome, and it is signed out (X sent it to ${at}). ` +
`${why} Tell the user: someone has to sign in to X in that browser again.`,
);
@@ -239,33 +349,68 @@ const signIn = async (at: string, outer: AbortSignal): Promise<void> => {
}
};
-const run = (
- request: XSearchRequest,
- sort: XSort,
- searchUrl: string,
+/**
+ * Opens `url` and waits for X to render: posts, an empty or error page, or a
+ * sign-in redirect. `ready` can ask for more before posts count, such as a
+ * profile's header.
+ */
+const open = async (
+ page: PageSession,
+ url: string,
signal: AbortSignal,
- saw: (state: string) => void,
-): Promise<XSearchOutcome> =>
- withPage(signal, async (page) => {
- await page.navigate(searchUrl, signal);
-
- // X renders after the load event: poll until posts, an empty or error
- // page, or a sign-in redirect shows up.
- let probe: XProbe;
- for (;;) {
- probe = await page.evaluate<XProbe>(PROBE, signal);
- saw(`${probe.state} at ${probe.url}`);
- if (probe.state !== "loading") break;
- await sleep(POLL_MS, signal);
- }
-
+ saw: Saw,
+ ready: (p: XProbe) => boolean = (p) => p.state !== "loading",
+): Promise<XProbe> => {
+ await page.navigate(url, signal);
+ for (;;) {
+ const probe = await page.evaluate<XProbe>(PROBE, signal);
+ saw(`${probe.state} at ${probe.url}`);
if (probe.state === "signed-out") throw new SignedOut(probe.url);
if (probe.state === "error") {
throw new BrowserUnavailableError(
- `X answered with an error page instead of results (${probe.url}): "${probe.note}". This is ` +
- `usually X rate-limiting the account; wait a few minutes rather than retrying in a loop.`,
+ `X answered with an error page (${probe.url}): "${probe.note}". This is usually X rate-limiting ` +
+ `the account; wait a few minutes rather than retrying in a loop.`,
);
}
+ if (probe.state === "empty" || ready(probe)) return probe;
+ await sleep(POLL_MS, signal);
+ }
+};
+
+/**
+ * X keeps only the posts near the viewport in the page, so collect by URL
+ * while scrolling rather than reading once at the end. Stops at `enough`, or
+ * when scrolling brings nothing new.
+ */
+const collect = async (page: PageSession, first: XProbe, enough: (seen: Map<string, RawPost>) => boolean, signal: AbortSignal): Promise<RawPost[]> => {
+ // The newest reading of a post wins: photos and cards render after the
+ // text, so the first reading of a post can lack them. A Map keeps the order
+ // posts were first seen in.
+ const seen = new Map<string, RawPost>();
+ const take = (p: XProbe) => p.posts.forEach((post) => seen.set(post.url, post));
+ take(first);
+ await sleep(POLL_MS * 2, signal);
+ take(await page.evaluate<XProbe>(PROBE, signal));
+ for (let i = 0; i < MAX_SCROLLS && !enough(seen); i++) {
+ const before = seen.size;
+ await page.evaluate(SCROLL, signal);
+ // New posts arrive over the next second or so.
+ for (let w = 0; w < 4 && seen.size === before; w++) {
+ await sleep(POLL_MS, signal);
+ take(await page.evaluate<XProbe>(PROBE, signal));
+ }
+ if (seen.size === before) break; // the end of the page
+ }
+ return [...seen.values()];
+};
+
+// --- The reads ---------------------------------------------------------------
+
+export const xSearch = (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => {
+ const sort: XSort = request.sort ?? "latest";
+ const searchUrl = buildXSearchUrl(request.query, sort);
+ return xRead(outer, `the search ${searchUrl}`, async (page, signal, saw) => {
+ const probe = await open(page, searchUrl, signal, saw);
if (probe.state === "empty") {
throw new NoResultsError(
`X found no posts for ${JSON.stringify(request.query)} (${sort}). X said: "${probe.note}". ` +
@@ -273,22 +418,104 @@ const run = (
`or try sort "top" or a wider date range.`,
);
}
+ const posts = await collect(page, probe, (s) => s.size >= request.count, signal);
+ return { query: request.query, sort, searchUrl, posts: posts.slice(0, request.count).map(finish) };
+ });
+};
- // X keeps only the posts near the viewport in the page, so collect by URL
- // while scrolling rather than reading once at the end.
- const seen = new Map<string, RawPost>();
- const take = (p: XProbe) => p.posts.forEach((post) => seen.has(post.url) || seen.set(post.url, post));
- take(probe);
- for (let i = 0; i < MAX_SCROLLS && seen.size < request.count; i++) {
- const before = seen.size;
- await page.evaluate(SCROLL, signal);
- // New posts arrive over the next second or so.
- for (let w = 0; w < 4 && seen.size === before; w++) {
+export const xPost = (request: XPostRequest, outer: AbortSignal): Promise<XPostOutcome> => {
+ const id = postIdOf(request.url);
+ if (!id) return Promise.reject(new BrowserUnavailableError(`not an X post link: ${request.url}`));
+ // /i/status/<id> redirects to the post under its author's handle.
+ const url = `https://x.com/i/status/${id}`;
+ const isMain = (p: RawPost) => p.url.endsWith(`/status/${id}`);
+ return xRead(outer, `reading ${url}`, async (page, signal, saw) => {
+ const probe = await open(page, url, signal, saw, (p) => p.posts.some(isMain));
+ if (probe.state === "empty") {
+ throw new NoResultsError(`X shows no post at ${url}: "${probe.note}". It may be deleted, or its account protected or suspended.`);
+ }
+ const posts = await collect(page, probe, (s) => s.size >= request.replies + 1 + [...s.values()].findIndex(isMain), signal);
+ const at = posts.findIndex(isMain);
+ if (at < 0) throw new BrowserUnavailableError(`X's page for ${url} did not show the post itself (${probe.url})`);
+ return {
+ post: finish(posts[at]!),
+ thread: posts.slice(0, at).map(finish),
+ replies: posts.slice(at + 1, at + 1 + request.replies).map(finish),
+ };
+ });
+};
+
+type ProfileSchema = {
+ dateCreated?: string;
+ relatedLink?: string[];
+ mainEntity?: {
+ additionalName?: string;
+ name?: string;
+ description?: string;
+ homeLocation?: { name?: string };
+ image?: { contentUrl?: string };
+ url?: string;
+ interactionStatistic?: { name?: string; userInteractionCount?: number }[];
+ };
+};
+
+/** The profile from X's JSON-LD block: exact counts, where the page shows "92.2M". */
+export const parseProfile = (json: string, handle: string): XProfile => {
+ const s = JSON.parse(json) as ProfileSchema;
+ const e = s.mainEntity ?? {};
+ const count = (name: string) => e.interactionStatistic?.find((i) => i.name === name)?.userInteractionCount;
+ const profile: XProfile = { name: e.name ?? handle, handle: `@${e.additionalName ?? handle}`, url: e.url ?? `https://x.com/${handle}` };
+ if (e.description) profile.bio = e.description;
+ if (e.homeLocation?.name) profile.location = e.homeLocation.name;
+ const site = s.relatedLink?.find((l) => !/^https?:\/\/t\.co\//.test(l));
+ if (site) profile.website = site;
+ if (s.dateCreated) profile.joined = s.dateCreated;
+ const followers = count("Follows");
+ const following = count("Friends");
+ const posts = count("Tweets");
+ if (followers !== undefined) profile.followers = followers;
+ if (following !== undefined) profile.following = following;
+ if (posts !== undefined) profile.posts = posts;
+ if (e.image?.contentUrl) profile.avatar = e.image.contentUrl;
+ return profile;
+};
+
+export const xUser = (request: XUserRequest, outer: AbortSignal): Promise<XUserOutcome> => {
+ const handle = handleOf(request.handle);
+ if (!handle) return Promise.reject(new BrowserUnavailableError(`not an X handle: ${request.handle}`));
+ const url = `https://x.com/${handle}`;
+ return xRead(outer, `reading ${url}`, async (page, signal, saw) => {
+ // The header renders before the posts; wait for both, or for X to say
+ // the account has none to show.
+ const probe = await open(page, url, signal, saw, (p) => p.profile !== null && p.state !== "loading");
+ if (!probe.profile) {
+ throw new NoResultsError(`X shows no account @${handle}: "${probe.note}". Check the handle.`);
+ }
+ const profile = parseProfile(probe.profile, handle);
+ const posts = probe.state === "posts" && request.count > 0 ? await collect(page, probe, (s) => s.size >= request.count, signal) : [];
+ return { profile, posts: posts.slice(0, request.count).map(finish) };
+ });
+};
+
+export const xTrends = (tab: XTrendTab, outer: AbortSignal): Promise<XTrendsOutcome> => {
+ const url = `https://x.com/explore/tabs/${tab}`;
+ return xRead(outer, `reading ${url}`, async (page, signal, saw) => {
+ await page.navigate(url, signal);
+ for (;;) {
+ const cells = await page.evaluate<string[] | null>(TRENDS, signal);
+ if (cells === null) throw new SignedOut(url);
+ saw(`${cells.length} trends at ${url}`);
+ if (cells.length) {
+ // The list renders in one go; a moment more lets the last cells fill.
await sleep(POLL_MS, signal);
- take(await page.evaluate<XProbe>(PROBE, signal));
+ const all = (await page.evaluate<string[] | null>(TRENDS, signal)) ?? cells;
+ return { tab, trends: all.map(parseTrend).filter((t): t is XTrend => t !== undefined) };
}
- if (seen.size === before) break; // the end of the results
+ const probe = await page.evaluate<XProbe>(PROBE, signal);
+ if (probe.state === "error") {
+ throw new BrowserUnavailableError(`X answered with an error page (${url}): "${probe.note}".`);
+ }
+ await sleep(POLL_MS, signal);
}
-
- return { query: request.query, sort, searchUrl, posts: [...seen.values()].slice(0, request.count).map(finish) };
});
+};