aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--castle-search/README.md57
-rw-r--r--castle-search/src/cdp.ts28
-rw-r--r--castle-search/src/http.ts291
-rw-r--r--castle-search/src/imap.ts148
-rw-r--r--castle-search/src/search.ts21
-rw-r--r--castle-search/src/totp.ts39
-rw-r--r--castle-search/src/x-limits.ts109
-rw-r--r--castle-search/src/x-login.ts259
-rw-r--r--castle-search/src/x-search.ts467
-rw-r--r--castle-search/src/youtube.ts303
-rw-r--r--castle-search/test/http.test.ts70
-rw-r--r--castle-search/test/x-limits.test.ts98
-rw-r--r--castle-search/test/x-login.test.ts91
-rw-r--r--castle-search/test/youtube.test.ts72
14 files changed, 1900 insertions, 153 deletions
diff --git a/castle-search/README.md b/castle-search/README.md
index 0e45aaf..1621fe3 100644
--- a/castle-search/README.md
+++ b/castle-search/README.md
@@ -47,18 +47,57 @@ switch engines, because this protocol has no field for one. There is no
authentication: bind it to loopback (the default) or a trusted address.
`GET /health` answers `{"status": true}`.
-### X search
+### X.com: search, posts, profiles, trends
-`POST /x-search` searches X (Twitter) instead: `{"query": "from:nasa", "count":
-10, "sort": "latest"}` in, `{"query", "sort", "posts": [...]}` out, each post
-with its link, author, time, text, counts and any quoted post. The query is X's
-own search syntax. It needs the browser signed in to X — X serves search to
-nobody else — and it says so when it is not, rather than returning nothing.
-`GET /openapi.json` describes it, so Open WebUI can offer it as a tool. Free per
-search, where xAI's own X search charges per post returned.
+Four reads of X (Twitter), each `POST` a JSON object and get JSON back:
+
+| Route | In | Out |
+|---|---|---|
+| `/x-search` | `query` (X's own search syntax), `count`, `sort` (`latest` or `top`) | the posts |
+| `/x-post` | `url` (a post link or its id), `replies` | the post whole, the `thread` above it when it is a reply, its top `replies` |
+| `/x-user` | `handle`, `count` | the `profile` (bio, location, website, joined, exact follower, following and post counts) and its latest posts |
+| `/x-trends` | `tab`: `trending`, `news`, `sports`, `entertainment`, `for_you` | the trends, each with a `query` for `/x-search` |
+
+Each post comes with its link, author, time, text, counts, photos (X's image
+link and alt text), videos (a still), a link card and any quoted post. A search
+result X cut short is marked `truncated`; `/x-post` reads it whole. It needs the
+browser signed in to X, since X serves none of this to anybody else, and it
+says so when it is not, rather than returning nothing. `GET /openapi.json`
+describes all four, as "X.com", so Open WebUI can offer them as tools. Free,
+where xAI's own X search charges per post returned. Trends are for the
+signed-in account's region.
X keeps only the posts near the screen in the page, so it scrolls and collects
-by link, up to 20 posts. A search takes 4–7 s.
+by link, up to 20 posts (or replies). A read takes 4–7 s.
+
+All four share one set of limits, so the account reads X like a person, not a
+scraper (`src/x-limits.ts`): one read at a time, 30 s ± a random 0–25 s between them,
+drawn afresh each time (a read that comes sooner waits), and at most 30 an hour and 150 a day, beyond which a
+read fails saying when the next may run. The counts live in memory, so a
+restart starts them afresh.
+
+Signed out, it signs back in by itself when `CCS_X_LOGIN_FILE` names a JSON
+file of the account's details (`src/x-login.ts`): `username`, `password`,
+`totpSecret` (the authenticator app's base32 secret) and, optionally, `imap`
+(`host`, `user`, `password`) for the code X emails when a sign-in looks
+unusual, read from the subject over IMAP (`src/imap.ts`). Then it searches
+again, once. It tries at most once every 30 minutes, never sends the password
+twice in one try, and stops at any step it does not recognise, such as a
+captcha, saying a person has to sign in. A sign-in takes about 6 s.
+
+### YouTube transcripts
+
+`POST /youtube-transcript` `{"url": "https://youtu.be/...", "language": "en"}`
+files a YouTube video's transcript as plain text in `CCS_TRANSCRIPT_DIR`, named
+`<upload date>-<Title-With-Dashes>.txt`, and returns it with the title,
+channel, upload date and length (`src/youtube.ts`). Captions only, never the
+video: human-written when the video has them, YouTube's automatic ones
+otherwise, with the rolling repeats of automatic captions taken out. A video
+filed before is read back from the folder. yt-dlp (`CCS_YTDLP`) updates
+itself at most daily, and again when a fetch fails; when YouTube rate-limits
+captions (HTTP 429), the text comes from youtube-transcript-api
+(`CCS_YT_PYTHON`) instead. `GET /youtube/openapi.json` describes it apart from
+X.com, as "YouTube". The recipe is Hermes's, from bambos.
## Engines
diff --git a/castle-search/src/cdp.ts b/castle-search/src/cdp.ts
index aa01b56..64c382e 100644
--- a/castle-search/src/cdp.ts
+++ b/castle-search/src/cdp.ts
@@ -242,7 +242,12 @@ export class PageSession {
this.#sessionId = sessionId;
}
- static async open(conn: BrowserConnection, signal?: AbortSignal): Promise<PageSession> {
+ /**
+ * `browserContextId`, from Target.createBrowserContext, opens the page in a
+ * context of its own, with no cookies: used to try a sign-in without
+ * touching the browser's own signed-in session.
+ */
+ static async open(conn: BrowserConnection, signal?: AbortSignal, browserContextId?: string): Promise<PageSession> {
if (signal?.aborted) throw abortError(signal);
// Deliberately NOT raced against the abort signal. raceAbort abandons the
@@ -251,7 +256,11 @@ export class PageSession {
// using, with nothing left holding its targetId to close it. It is a fast
// command already bounded by COMMAND_TIMEOUT_MS, so waiting it out costs
// little and guarantees we own whatever it created.
- const created = await conn.send("Target.createTarget", { url: "about:blank", background: true });
+ const created = await conn.send("Target.createTarget", {
+ url: "about:blank",
+ background: true,
+ ...(browserContextId ? { browserContextId } : {}),
+ });
const targetId = created["targetId"];
if (typeof targetId !== "string") {
throw new BrowserUnavailableError("Target.createTarget did not return a targetId");
@@ -331,6 +340,21 @@ export class PageSession {
}
/**
+ * Types `text` into whatever has focus, as a paste would: one input event, so
+ * a framework-controlled field (React's) sees it. Never logged.
+ */
+ async insertText(text: string, signal?: AbortSignal): Promise<void> {
+ await raceAbort(this.#conn.send("Input.insertText", { text }, this.#sessionId), signal);
+ }
+
+ /** Presses and releases Enter in whatever has focus. */
+ async pressEnter(signal?: AbortSignal): Promise<void> {
+ const key = { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 };
+ await raceAbort(this.#conn.send("Input.dispatchKeyEvent", { type: "keyDown", ...key, text: "\r" }, this.#sessionId), signal);
+ await raceAbort(this.#conn.send("Input.dispatchKeyEvent", { type: "keyUp", ...key }, this.#sessionId), signal);
+ }
+
+ /**
* Close the tab. Idempotent and never throws: it runs on error paths, where a
* second failure would mask the one worth reporting.
*/
diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts
index 17c8c70..076c624 100644
--- a/castle-search/src/http.ts
+++ b/castle-search/src/http.ts
@@ -20,17 +20,36 @@
* an address only trusted peers reach. Each search opens and closes one
* background tab in a browser someone may be looking at.
*
- * POST /x-search searches X instead (x-search.ts), through the browser's
- * signed-in X session, and GET /openapi.json describes it, so that Open WebUI
- * can offer it as a tool: `{ "query", "count", "sort" }` in, `{ "query", "sort",
- * "posts": [...] }` out, 502 with `{ "error" }` on failure.
+ * POST /x-search, /x-post, /x-user and /x-trends read X instead (x-search.ts),
+ * through the browser's signed-in X session, and GET /openapi.json describes
+ * them, so that Open WebUI can offer them as tools ("X.com"). POST
+ * /youtube-transcript files a YouTube video's transcript and returns it
+ * (youtube.ts), described at GET /youtube/openapi.json ("YouTube"). Each
+ * answers 502 with `{ "error" }` on failure.
*/
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { messageOf } from "./errors.ts";
import { disconnect, search, type SearchOutcome, type SearchRequest } from "./search.ts";
-import { xSearch, type XSearchOutcome, type XSearchRequest } from "./x-search.ts";
+import {
+ handleOf,
+ postIdOf,
+ TREND_TABS,
+ type XPostOutcome,
+ type XPostRequest,
+ xPost,
+ xSearch,
+ type XSearchOutcome,
+ type XSearchRequest,
+ type XTrendsOutcome,
+ type XTrendTab,
+ xTrends,
+ xUser,
+ type XUserOutcome,
+ type XUserRequest,
+} from "./x-search.ts";
+import { DEFAULT_MAX_CHARS, type TranscriptOutcome, type TranscriptRequest, videoIdOf, youtubeTranscript } from "./youtube.ts";
export const DEFAULT_LISTEN = "127.0.0.1:11158";
const MAX_BODY_BYTES = 64 * 1024;
@@ -75,47 +94,124 @@ export const handleSearch = async (raw: string, run: Searcher, signal: AbortSign
};
export type XSearcher = (request: XSearchRequest, signal: AbortSignal) => Promise<XSearchOutcome>;
+export type XPostReader = (request: XPostRequest, signal: AbortSignal) => Promise<XPostOutcome>;
+export type XUserReader = (request: XUserRequest, signal: AbortSignal) => Promise<XUserOutcome>;
+export type XTrendsReader = (tab: XTrendTab, signal: AbortSignal) => Promise<XTrendsOutcome>;
const X_DEFAULT_COUNT = 10;
-/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */
-export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise<Reply> => {
- let parsed: unknown;
+const jsonBody = (raw: string): Record<string, unknown> | Reply => {
try {
- parsed = JSON.parse(raw);
+ const parsed: unknown = JSON.parse(raw || "{}");
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as Record<string, unknown>;
} catch {
- return { status: 400, body: { error: "request body is not JSON" } };
+ // fall through
}
- const { query, count, sort } = (parsed ?? {}) as { query?: unknown; count?: unknown; sort?: unknown };
+ return { status: 400, body: { error: "request body is not a JSON object" } };
+};
+const isReply = (v: unknown): v is Reply => typeof (v as Reply).status === "number" && "body" in (v as Reply);
+
+const countOf = (v: unknown, fallback: number, min = 1): number => {
+ const n = typeof v === "number" && Number.isFinite(v) ? Math.trunc(v) : fallback;
+ return Math.min(MAX_COUNT, Math.max(min, n));
+};
+
+const answer = async (work: () => Promise<unknown>): Promise<Reply> => {
+ try {
+ return { status: 200, body: await work() };
+ } catch (e) {
+ return { status: 502, body: { error: messageOf(e) } };
+ }
+};
+
+/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */
+export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ const { query, count, sort } = body;
if (typeof query !== "string" || query.trim() === "") {
return { status: 400, body: { error: "query must be a non-empty string" } };
}
if (sort !== undefined && sort !== null && sort !== "latest" && sort !== "top") {
return { status: 400, body: { error: 'sort must be "latest" or "top"' } };
}
- const n = typeof count === "number" && Number.isFinite(count) ? Math.trunc(count) : X_DEFAULT_COUNT;
- try {
- const outcome = await run(
- { query: query.trim(), count: Math.min(MAX_COUNT, Math.max(1, n)), sort: sort ?? "latest" },
- signal,
- );
- return { status: 200, body: { query: outcome.query, sort: outcome.sort, posts: outcome.posts } };
- } catch (e) {
- return { status: 502, body: { error: messageOf(e) } };
+ return answer(async () => {
+ const outcome = await run({ query: query.trim(), count: countOf(count, X_DEFAULT_COUNT), sort: sort ?? "latest" }, signal);
+ return { query: outcome.query, sort: outcome.sort, posts: outcome.posts };
+ });
+};
+
+/** POST /x-post: `{ url, replies? }` in, `{ post, thread, replies }` out. */
+export const handleXPost = async (raw: string, run: XPostReader, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ if (typeof body["url"] !== "string" || !postIdOf(body["url"])) {
+ return { status: 400, body: { error: "url must be a link to a post on X (x.com/<handle>/status/<id>) or its id" } };
+ }
+ const url = body["url"];
+ return answer(() => run({ url, replies: countOf(body["replies"], X_DEFAULT_COUNT, 0) }, signal));
+};
+
+/** POST /x-user: `{ handle, count? }` in, `{ profile, posts }` out. */
+export const handleXUser = async (raw: string, run: XUserReader, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ if (typeof body["handle"] !== "string" || !handleOf(body["handle"])) {
+ return { status: 400, body: { error: "handle must be an X handle, such as nasa or @nasa" } };
+ }
+ const handle = body["handle"];
+ return answer(() => run({ handle, count: countOf(body["count"], X_DEFAULT_COUNT, 0) }, signal));
+};
+
+/** POST /x-trends: `{ tab? }` in, `{ tab, trends }` out. */
+export const handleXTrends = async (raw: string, run: XTrendsReader, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ const tab = body["tab"] ?? "trending";
+ if (!TREND_TABS.includes(tab as XTrendTab)) {
+ return { status: 400, body: { error: `tab must be one of ${TREND_TABS.join(", ")}` } };
+ }
+ return answer(() => run(tab as XTrendTab, signal));
+};
+
+export type TranscriptFetcher = (request: TranscriptRequest, signal: AbortSignal) => Promise<TranscriptOutcome>;
+
+/** POST /youtube-transcript: `{ url, language?, max_chars? }` in, the transcript and where it was filed out. */
+export const handleTranscript = async (raw: string, run: TranscriptFetcher, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ if (typeof body["url"] !== "string" || !videoIdOf(body["url"])) {
+ return { status: 400, body: { error: "url must be a YouTube video link (youtube.com/watch?v=..., youtu.be/..., shorts/...)" } };
+ }
+ const language = body["language"];
+ if (language !== undefined && (typeof language !== "string" || !/^[A-Za-z]{2,3}(-[A-Za-z0-9]+)?$/.test(language))) {
+ return { status: 400, body: { error: "language must be a language code such as en, pt or en-GB" } };
}
+ const max = body["max_chars"];
+ const maxChars = typeof max === "number" && Number.isFinite(max) ? Math.max(1000, Math.trunc(max)) : DEFAULT_MAX_CHARS;
+ const url = body["url"];
+ return answer(() => run({ url, language: language as string | undefined, maxChars }, signal));
+};
+
+const jsonSchema = (properties: Record<string, unknown>, required: string[]) => ({
+ required: true,
+ content: { "application/json": { schema: { type: "object", required, properties } } },
+});
+const RESPONSES = {
+ "200": { description: "What X shows", content: { "application/json": { schema: { type: "object" } } } },
+ "502": { description: "X could not be read; error says why", content: { "application/json": { schema: { type: "object" } } } },
};
/**
- * The OpenAPI description Open WebUI reads to offer /x-search as a tool. Its
- * descriptions are what the model sees, so they say what the tool is for and
- * how to write a query.
+ * The OpenAPI description Open WebUI reads to offer the X reads as tools. Its
+ * descriptions are what the model sees, so they say what each is for.
*/
export const OPENAPI = {
openapi: "3.1.0",
info: {
- title: "castle X search",
- version: "1.0.0",
- description: "Searches posts on X (Twitter) through a signed-in browser. Free per search.",
+ title: "X.com",
+ version: "2.0.0",
+ description: "Reads X (Twitter) through a signed-in browser: search, posts with replies, profiles, trends. Free.",
},
paths: {
"/x-search": {
@@ -127,26 +223,103 @@ export const OPENAPI = {
"reactions to news, posts about a topic. The query uses X's search syntax: from:handle, to:handle, " +
'"exact phrase", OR, -word, since:YYYY-MM-DD, until:YYYY-MM-DD, min_faves:N, filter:media, ' +
"-filter:replies, lang:en. A handle is written without @ after from:. Each post comes with its " +
- "link: cite it. Text cut short by X is marked truncated.",
- requestBody: {
- required: true,
- content: {
- "application/json": {
- schema: {
- type: "object",
- required: ["query"],
- properties: {
- query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" },
- count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" },
- sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" },
- },
- },
- },
+ "link: cite it. A post marked truncated was cut short by X: read_x_post gives it whole. Photos " +
+ "come with their image link and alt text.",
+ requestBody: jsonSchema(
+ {
+ query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" },
+ count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" },
+ sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" },
},
- },
+ ["query"],
+ ),
+ responses: RESPONSES,
+ },
+ },
+ "/x-post": {
+ post: {
+ operationId: "read_x_post",
+ summary: "Read one post on X, whole, with its thread and replies",
+ description:
+ "Read a post on X from its link: its full text (never truncated), photos with their image link " +
+ "and alt text, its link card, the posts above it when it is a reply (thread, oldest first), and " +
+ "the top replies under it. Use it to read a long post, a thread, or what people answered.",
+ requestBody: jsonSchema(
+ {
+ url: { type: "string", description: "The post's link, x.com/<handle>/status/<id>, or the id" },
+ replies: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many replies" },
+ },
+ ["url"],
+ ),
+ responses: RESPONSES,
+ },
+ },
+ "/x-user": {
+ post: {
+ operationId: "read_x_profile",
+ summary: "Read an X profile and its latest posts",
+ description:
+ "Read an X account's profile: name, bio, location, website, when it joined, exact follower, " +
+ "following and post counts, then its latest posts as its profile shows them (a pinned post and " +
+ "reposts included, marked in context).",
+ requestBody: jsonSchema(
+ {
+ handle: { type: "string", description: "The account's handle, such as nasa or @nasa" },
+ count: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many of its posts" },
+ },
+ ["handle"],
+ ),
+ responses: RESPONSES,
+ },
+ },
+ "/x-trends": {
+ post: {
+ operationId: "x_trends",
+ summary: "What is trending on X now",
+ description:
+ "List what X shows as trending now, for the signed-in account's region (the United Kingdom). " +
+ "Each trend has a query: pass it to search_x for the posts.",
+ requestBody: jsonSchema(
+ { tab: { type: "string", enum: [...TREND_TABS], default: "trending", description: "Which explore tab" } },
+ [],
+ ),
+ responses: RESPONSES,
+ },
+ },
+ },
+} as const;
+
+/** The YouTube tool's own description, so Open WebUI lists it apart from X.com. */
+export const YOUTUBE_OPENAPI = {
+ openapi: "3.1.0",
+ info: {
+ title: "YouTube",
+ version: "1.0.0",
+ description: "A YouTube video's transcript, filed in Igor's Nextcloud transcript folder and returned.",
+ },
+ paths: {
+ "/youtube-transcript": {
+ post: {
+ operationId: "youtube_transcript",
+ summary: "Get a YouTube video's transcript",
+ description:
+ "Fetch a YouTube video's transcript from its link: the video's captions (human-written when it " +
+ "has them, YouTube's automatic ones otherwise) as clean prose, with its title, channel, upload " +
+ "date and length. It is also filed in Igor's Nextcloud, in the transcript folder, as " +
+ "<upload date>-<Title>.txt; a video filed before is read from there. Use it whenever a YouTube " +
+ "link comes up and its content matters: to summarise it, answer questions about it, or quote " +
+ "it. Automatic captions garble names and numbers: say so when one matters.",
+ requestBody: jsonSchema(
+ {
+ url: { type: "string", description: "The video's link: youtube.com/watch?v=..., youtu.be/..., shorts/..." },
+ language: { type: "string", default: "en", description: "Caption language code, such as en or pt" },
+ max_chars: { type: "integer", minimum: 1000, default: DEFAULT_MAX_CHARS, description: "Most characters of text to return; the file always has all of it" },
+ },
+ ["url"],
+ ),
responses: {
- "200": { description: "Matching posts", content: { "application/json": { schema: { type: "object" } } } },
- "502": { description: "The search failed; error says why", content: { "application/json": { schema: { type: "object" } } } },
+ "200": { description: "The transcript and where it was filed", content: { "application/json": { schema: { type: "object" } } } },
+ "502": { description: "No transcript; error says why", content: { "application/json": { schema: { type: "object" } } } },
},
},
},
@@ -199,9 +372,20 @@ const main = (): void => {
send(res, { status: 200, body: OPENAPI });
return;
}
- const isX = path === "/x-search";
- if (req.method !== "POST" || (path !== "/" && path !== "/search" && !isX)) {
- send(res, { status: 404, body: { error: "POST a search to / or /search, or an X search to /x-search" } });
+ if (req.method === "GET" && path === "/youtube/openapi.json") {
+ send(res, { status: 200, body: YOUTUBE_OPENAPI });
+ return;
+ }
+ const xRoutes: Record<string, (raw: string, signal: AbortSignal) => Promise<Reply>> = {
+ "/x-search": (raw, signal) => handleXSearch(raw, xSearch, signal),
+ "/x-post": (raw, signal) => handleXPost(raw, xPost, signal),
+ "/x-user": (raw, signal) => handleXUser(raw, xUser, signal),
+ "/x-trends": (raw, signal) => handleXTrends(raw, xTrends, signal),
+ "/youtube-transcript": (raw, signal) => handleTranscript(raw, youtubeTranscript, signal),
+ };
+ const xRoute = xRoutes[path ?? ""];
+ if (req.method !== "POST" || (path !== "/" && path !== "/search" && !xRoute)) {
+ send(res, { status: 404, body: { error: "POST a search to / or /search, an X read to /x-search, /x-post, /x-user or /x-trends, or a YouTube link to /youtube-transcript" } });
return;
}
// Stop the search, and free its tab, if the client goes away first.
@@ -217,16 +401,9 @@ const main = (): void => {
return;
}
const started = Date.now();
- const reply = isX
- ? await handleXSearch(raw, xSearch, controller.signal)
- : await handleSearch(raw, runSearch, controller.signal);
- const outcome =
- reply.status !== 200
- ? `error ${reply.status}`
- : isX
- ? `${(reply.body as { posts: unknown[] }).posts.length} posts`
- : `${(reply.body as unknown[]).length} results`;
- console.log(`${isX ? "x-search" : "search"} ${outcome} in ${Date.now() - started} ms`);
+ const reply = xRoute ? await xRoute(raw, controller.signal) : await handleSearch(raw, runSearch, controller.signal);
+ const outcome = reply.status !== 200 ? `error ${reply.status}` : xRoute ? "ok" : `${(reply.body as unknown[]).length} results`;
+ console.log(`${xRoute ? path!.slice(1) : "search"} ${outcome} in ${Date.now() - started} ms`);
if (reply.status !== 200) console.log(` ${(reply.body as { error: string }).error}`);
send(res, reply);
});
diff --git a/castle-search/src/imap.ts b/castle-search/src/imap.ts
new file mode 100644
index 0000000..ef02979
--- /dev/null
+++ b/castle-search/src/imap.ts
@@ -0,0 +1,148 @@
+/**
+ * Just enough IMAP to read a login code X emailed: log in over TLS, look at the
+ * inbox read-only, read the subjects of X's recent mail. No dependency, so
+ * castle-search keeps having none.
+ *
+ * X puts the code in the subject ("Your X confirmation code is abc12def"), so
+ * only headers are fetched, and nothing is marked read or changed.
+ */
+
+import { connect, type TLSSocket } from "node:tls";
+
+export type ImapAccount = { host: string; port?: number; user: string; password: string };
+
+export type MailHeader = { received: number; from: string; subject: string };
+
+/** RFC 2047 encoded words, the B and Q forms, as subjects use them. */
+export const decodeWords = (s: string): string =>
+ s
+ .replace(/\?=\s+=\?/g, "?==?")
+ .replace(/=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g, (_, charset: string, enc: string, text: string) => {
+ const bytes =
+ enc.toUpperCase() === "B"
+ ? Buffer.from(text, "base64")
+ : Buffer.from(
+ text.replace(/_/g, " ").replace(/=([0-9A-Fa-f]{2})/g, (_m, h: string) => String.fromCharCode(parseInt(h, 16))),
+ "latin1",
+ );
+ try {
+ return new TextDecoder(charset).decode(bytes);
+ } catch {
+ return bytes.toString("utf8");
+ }
+ });
+
+/** The code in one of X's subjects, or undefined when the mail is something else. */
+export const xCodeFromSubject = (subject: string): string | undefined => {
+ const m =
+ /\bcode is:?\s*([A-Za-z0-9]{5,10})\b/i.exec(subject) ??
+ /^\s*([A-Za-z0-9]{5,10}) is your X (?:verification|confirmation|login) code\b/i.exec(subject);
+ return m?.[1];
+};
+
+/** "18-Nov-2026 10:16:23 +0000" -> epoch ms. */
+export const parseInternalDate = (s: string): number => Date.parse(s.replace(/^(\d{1,2})-(\w{3})-(\d{4})/, "$1 $2 $3"));
+
+/** Splits a FETCH response into one header set per message. */
+export const parseFetch = (raw: string): MailHeader[] => {
+ const out: MailHeader[] = [];
+ for (const part of raw.split(/\r\n(?=\* \d+ FETCH )/)) {
+ const date = /INTERNALDATE "([^"]+)"/.exec(part)?.[1];
+ if (!date) continue;
+ const unfolded = part.replace(/\r\n[ \t]+/g, " ");
+ const header = (name: string) => new RegExp(`^${name}:\\s*(.*)$`, "im").exec(unfolded)?.[1]?.trim() ?? "";
+ out.push({ received: parseInternalDate(date), from: decodeWords(header("From")), subject: decodeWords(header("Subject")) });
+ }
+ return out;
+};
+
+const quote = (s: string): string => `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+
+const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+const imapDate = (ms: number): string => {
+ const d = new Date(ms);
+ return `${d.getUTCDate()}-${MONTHS[d.getUTCMonth()]}-${d.getUTCFullYear()}`;
+};
+
+class Session {
+ readonly #socket: TLSSocket;
+ #buffer = "";
+ #tag = 0;
+ #waiter: (() => void) | undefined;
+ #error: Error | undefined;
+
+ constructor(socket: TLSSocket) {
+ this.#socket = socket;
+ socket.setEncoding("latin1");
+ socket.on("data", (chunk: string) => {
+ this.#buffer += chunk;
+ this.#waiter?.();
+ });
+ const fail = (e: Error) => {
+ this.#error ??= e;
+ this.#waiter?.();
+ };
+ socket.on("error", fail);
+ socket.on("close", () => fail(new Error("the IMAP server closed the connection")));
+ }
+
+ async #until(done: (buffer: string) => boolean, signal: AbortSignal): Promise<string> {
+ while (!done(this.#buffer)) {
+ if (this.#error) throw this.#error;
+ if (signal.aborted) throw signal.reason;
+ await new Promise<void>((resolve) => {
+ this.#waiter = resolve;
+ signal.addEventListener("abort", () => resolve(), { once: true });
+ });
+ this.#waiter = undefined;
+ }
+ const out = this.#buffer;
+ this.#buffer = "";
+ return out;
+ }
+
+ greeting(signal: AbortSignal): Promise<string> {
+ return this.#until((b) => b.includes("\r\n"), signal);
+ }
+
+ /** Sends one command; `shown` is what an error may say was sent, so a password never is. */
+ async command(line: string, signal: AbortSignal, shown = line): Promise<string> {
+ const tag = `c${++this.#tag}`;
+ this.#socket.write(`${tag} ${line}\r\n`);
+ const reply = await this.#until((b) => new RegExp(`(^|\\r\\n)${tag} (OK|NO|BAD)[^\\r\\n]*\\r\\n`).test(b), signal);
+ const status = new RegExp(`(?:^|\\r\\n)${tag} (OK|NO|BAD)([^\\r\\n]*)`).exec(reply);
+ if (status?.[1] !== "OK") throw new Error(`IMAP ${shown.split(" ")[0]} failed: ${status?.[1]}${status?.[2]}`);
+ return reply;
+ }
+
+ close(): void {
+ this.#socket.end();
+ this.#socket.destroy();
+ }
+}
+
+/** Headers of mail from `fromDomain` received at or after `sinceMs`, newest first. */
+export const recentMail = async (
+ account: ImapAccount,
+ fromDomain: string,
+ sinceMs: number,
+ signal: AbortSignal,
+): Promise<MailHeader[]> => {
+ const socket = connect({ host: account.host, port: account.port ?? 993, servername: account.host });
+ const s = new Session(socket);
+ try {
+ await s.greeting(signal);
+ await s.command(`LOGIN ${quote(account.user)} ${quote(account.password)}`, signal, "LOGIN");
+ await s.command("EXAMINE INBOX", signal);
+ const found = await s.command(`SEARCH SINCE ${imapDate(sinceMs - 86_400_000)} FROM ${quote(fromDomain)}`, signal);
+ const ids = (/\* SEARCH([\d ]*)/.exec(found)?.[1] ?? "").trim().split(/\s+/).filter(Boolean).slice(-10);
+ if (ids.length === 0) return [];
+ const raw = await s.command(`FETCH ${ids.join(",")} (INTERNALDATE BODY.PEEK[HEADER.FIELDS (FROM SUBJECT)])`, signal);
+ await s.command("LOGOUT", signal).catch(() => "");
+ return parseFetch(raw)
+ .filter((m) => m.received >= sinceMs)
+ .sort((a, b) => b.received - a.received);
+ } finally {
+ s.close();
+ }
+};
diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts
index 0178349..1ddd523 100644
--- a/castle-search/src/search.ts
+++ b/castle-search/src/search.ts
@@ -402,16 +402,31 @@ const isGoogleHost = (url: string): boolean => /^https?:\/\/([a-z0-9-]+\.)*googl
export const withPage = async <T>(
signal: AbortSignal,
fn: (page: PageSession, endpoint: string) => Promise<T>,
+ options: { isolated?: boolean } = {},
): Promise<T> => {
const release = await acquire(signal);
try {
const conn = await ensureConnection(signal);
const endpoint = describeCdpTarget(conn.target);
- const page = await PageSession.open(conn, signal);
+ // isolated: a browser context of its own, with none of the profile's
+ // cookies, thrown away afterwards with everything in it.
+ let context: string | undefined;
+ if (options.isolated) {
+ const created = await conn.send("Target.createBrowserContext", { disposeOnDetach: true });
+ if (typeof created["browserContextId"] !== "string") {
+ throw new BrowserUnavailableError("Target.createBrowserContext did not return a browserContextId");
+ }
+ context = created["browserContextId"];
+ }
try {
- return await fn(page, endpoint);
+ const page = await PageSession.open(conn, signal, context);
+ try {
+ return await fn(page, endpoint);
+ } finally {
+ await page.close();
+ }
} finally {
- await page.close();
+ if (context) await conn.send("Target.disposeBrowserContext", { browserContextId: context }).catch(() => {});
}
} finally {
release();
diff --git a/castle-search/src/totp.ts b/castle-search/src/totp.ts
new file mode 100644
index 0000000..c14a906
--- /dev/null
+++ b/castle-search/src/totp.ts
@@ -0,0 +1,39 @@
+/**
+ * TOTP (RFC 6238): the six-digit code an authenticator app shows, from the
+ * base32 secret behind its QR code. SHA-1, 30-second steps, as X uses.
+ */
+
+import { createHmac } from "node:crypto";
+
+const B32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+
+export const base32Decode = (input: string): Buffer => {
+ const clean = input.toUpperCase().replace(/[\s=-]/g, "");
+ let bits = 0;
+ let value = 0;
+ const out: number[] = [];
+ for (const ch of clean) {
+ const i = B32.indexOf(ch);
+ if (i < 0) throw new Error("the TOTP secret is not base32");
+ value = (value << 5) | i;
+ bits += 5;
+ if (bits >= 8) {
+ out.push((value >>> (bits - 8)) & 0xff);
+ bits -= 8;
+ }
+ }
+ return Buffer.from(out);
+};
+
+export const totp = (secret: string, nowMs: number = Date.now(), digits = 6, stepS = 30): string => {
+ const counter = Buffer.alloc(8);
+ counter.writeBigUInt64BE(BigInt(Math.floor(nowMs / 1000 / stepS)));
+ const h = createHmac("sha1", base32Decode(secret)).update(counter).digest();
+ const o = h[h.length - 1]! & 0x0f;
+ const n = (h.readUInt32BE(o) & 0x7fffffff) % 10 ** digits;
+ return String(n).padStart(digits, "0");
+};
+
+/** Seconds the code for `nowMs` stays valid. */
+export const totpRemaining = (nowMs: number = Date.now(), stepS = 30): number =>
+ stepS - (Math.floor(nowMs / 1000) % stepS);
diff --git a/castle-search/src/x-limits.ts b/castle-search/src/x-limits.ts
new file mode 100644
index 0000000..bdc4229
--- /dev/null
+++ b/castle-search/src/x-limits.ts
@@ -0,0 +1,109 @@
+/**
+ * Limits on X searches, so the signed-in account reads X like a person rather
+ * than a scraper, and X has no reason to lock it.
+ *
+ * - one search at a time;
+ * - GAP_MS ± up to JITTER_MS between the starts of two searches, drawn
+ * afresh each time (5-55 s), so they do not come at a machine's even
+ * beat: one that comes sooner waits its turn rather than failing;
+ * - at most PER_HOUR in any rolling hour and PER_DAY in any rolling day:
+ * beyond that a search fails at once, saying when the next one may run.
+ *
+ * Counts are kept in memory: a restart of the service starts them afresh.
+ */
+
+/** Owner's choice, 2026-11-18: 30 s plus or minus a random 0-25 s. */
+export const GAP_MS = 30_000;
+export const JITTER_MS = 25_000;
+export const PER_HOUR = 30;
+export const PER_DAY = 150;
+const HOUR_MS = 3_600_000;
+const DAY_MS = 24 * HOUR_MS;
+
+export class XRateLimitError extends Error {
+ override readonly name = "XRateLimitError";
+}
+
+export type Clock = {
+ now: () => number;
+ sleep: (ms: number, signal: AbortSignal) => Promise<void>;
+ /** Uniform in [0, 1), as Math.random. */
+ random: () => number;
+};
+
+const realSleep = (ms: number, signal: AbortSignal): Promise<void> =>
+ new Promise((resolve, reject) => {
+ if (signal.aborted) return reject(signal.reason);
+ const t = setTimeout(resolve, ms);
+ signal.addEventListener("abort", () => {
+ clearTimeout(t);
+ reject(signal.reason);
+ }, { once: true });
+ });
+
+export const realClock: Clock = { now: Date.now, sleep: realSleep, random: Math.random };
+
+export type Limits = { gap: number; jitter: number; hour: number; day: number };
+
+const at = (ms: number): string => new Date(ms).toISOString().replace("T", " ").slice(0, 16) + " UTC";
+
+export class XLimiter {
+ readonly #clock: Clock;
+ readonly #limits: Limits;
+ /** Start times of the searches in the last day, oldest first. */
+ #starts: number[] = [];
+ /** The tail of the queue: each search waits for the one before it. */
+ #tail: Promise<void> = Promise.resolve();
+
+ constructor(clock: Clock = realClock, limits: Limits = { gap: GAP_MS, jitter: JITTER_MS, hour: PER_HOUR, day: PER_DAY }) {
+ this.#clock = clock;
+ this.#limits = limits;
+ }
+
+ /** Runs `fn` once the limits allow, one at a time; throws XRateLimitError when over a cap. */
+ async run<T>(signal: AbortSignal, fn: () => Promise<T>): Promise<T> {
+ const previous = this.#tail;
+ let done!: () => void;
+ this.#tail = new Promise<void>((resolve) => (done = resolve));
+ try {
+ await Promise.race([
+ previous,
+ new Promise<never>((_, reject) => {
+ if (signal.aborted) reject(signal.reason);
+ signal.addEventListener("abort", () => reject(signal.reason), { once: true });
+ }),
+ ]);
+ // An abandoned search ahead of us must not hold the queue: `previous`
+ // settles when it finishes or is aborted, whichever comes first.
+ await this.#admit(signal);
+ return await fn();
+ } finally {
+ done();
+ }
+ }
+
+ async #admit(signal: AbortSignal): Promise<void> {
+ const now = this.#clock.now();
+ this.#starts = this.#starts.filter((t) => t > now - DAY_MS);
+ const lastHour = this.#starts.filter((t) => t > now - HOUR_MS);
+ if (this.#starts.length >= this.#limits.day) {
+ throw new XRateLimitError(
+ `X search limit reached: ${this.#limits.day} searches in the last 24 hours, to keep the account ` +
+ `from being locked. The next one can run after ${at(this.#starts[0]! + DAY_MS)}. Tell the user.`,
+ );
+ }
+ if (lastHour.length >= this.#limits.hour) {
+ throw new XRateLimitError(
+ `X search limit reached: ${this.#limits.hour} searches in the last hour, to keep the account ` +
+ `from being locked. The next one can run after ${at(lastHour[0]! + HOUR_MS)}. Tell the user, ` +
+ `and do not retry before then.`,
+ );
+ }
+ const last = this.#starts[this.#starts.length - 1];
+ const gap = Math.max(0, this.#limits.gap + (this.#clock.random() * 2 - 1) * this.#limits.jitter);
+ if (last !== undefined && now - last < gap) {
+ await this.#clock.sleep(gap - (now - last), signal);
+ }
+ this.#starts.push(this.#clock.now());
+ }
+}
diff --git a/castle-search/src/x-login.ts b/castle-search/src/x-login.ts
new file mode 100644
index 0000000..d4d33d1
--- /dev/null
+++ b/castle-search/src/x-login.ts
@@ -0,0 +1,259 @@
+/**
+ * Signing castle Chrome back in to X, when an X search finds it signed out.
+ *
+ * X's sign-in is a series of steps on x.com/i/jf/onboarding/web (2026-11), each
+ * named in the URL's hash: the username, `login_enter_password`,
+ * `two_factor_code` (the authenticator app's code), and sometimes a code X
+ * emails when the sign-in looks unusual. Each step is recognised by its
+ * fields and words rather than its name alone, since X renames them, and
+ * answered from the credentials file:
+ *
+ * {"username": "...", "password": "...", "totpSecret": "BASE32...",
+ * "imap": {"host": "imap.gmail.com", "user": "...", "password": "..."}}
+ *
+ * `imap` is optional: without it an emailed code fails the sign-in, saying so.
+ * The file is named by CCS_X_LOGIN_FILE; cloud-setup's macos/castle-search.sh
+ * writes it from szecret, readable by its owner only.
+ *
+ * A step it does not recognise (a captcha, "verify your phone") fails the
+ * sign-in with what the page said: that needs a person. Nothing secret is
+ * ever put in an error or a log line.
+ */
+
+import { readFileSync } from "node:fs";
+
+import type { PageSession } from "./cdp.ts";
+import { BrowserUnavailableError } from "./errors.ts";
+import { type ImapAccount, recentMail, xCodeFromSubject } from "./imap.ts";
+import { totp, totpRemaining } from "./totp.ts";
+
+export type XCredentials = { username: string; password: string; totpSecret?: string; imap?: ImapAccount };
+
+export const loadCredentials = (path: string | undefined): XCredentials | undefined => {
+ if (!path) return undefined;
+ let parsed: Partial<XCredentials>;
+ try {
+ parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<XCredentials>;
+ } catch (e) {
+ throw new BrowserUnavailableError(`cannot read the X sign-in file ${path}: ${(e as Error).message}`);
+ }
+ if (typeof parsed.username !== "string" || typeof parsed.password !== "string") {
+ throw new BrowserUnavailableError(`the X sign-in file ${path} needs "username" and "password"`);
+ }
+ return parsed as XCredentials;
+};
+
+export const LOGIN_URL = "https://x.com/i/flow/login";
+/** Whole sign-in, emailed code included: X's mail can take a minute. */
+export const X_LOGIN_TIMEOUT_MS = 150_000;
+const MAX_STEPS = 10;
+const EMAIL_WAIT_MS = 90_000;
+const EMAIL_POLL_MS = 5_000;
+
+export type LoginStep =
+ | { kind: "done" }
+ | { kind: "username" }
+ | { kind: "password" }
+ | { kind: "totp" }
+ | { kind: "email-code" }
+ | { kind: "wrong-password" }
+ | { kind: "loading" }
+ | { kind: "unknown" };
+
+export type LoginProbe = {
+ url: string;
+ path: string;
+ hash: string;
+ text: string;
+ /** Visible inputs: name, type, autocomplete. */
+ inputs: { name: string; type: string; autocomplete: string }[];
+ signedIn: boolean;
+};
+
+/**
+ * An input a person could see. X keeps a password field on its first page for
+ * autofill, laid out but at opacity 0 under aria-hidden: not a step to answer.
+ */
+const SHOWN = `((i) => i.type !== "hidden" && i.offsetParent !== null &&
+ !i.closest('[aria-hidden="true"], [inert]') && getComputedStyle(i).opacity !== "0")`;
+
+/** Reads the sign-in page as it stands. Runs in the page; self-contained. */
+export const LOGIN_PROBE = `(() => {
+ const scope = document.querySelector('[role="dialog"]') || document.querySelector("main") || document.body;
+ const inputs = [...document.querySelectorAll("input")]
+ .filter(${SHOWN})
+ .map((i) => ({ name: i.name || "", type: i.type || "", autocomplete: i.autocomplete || "" }));
+ return {
+ url: location.href,
+ path: location.pathname,
+ hash: location.hash,
+ text: (scope ? scope.innerText : "").trim().slice(0, 600),
+ inputs,
+ signedIn: !!document.querySelector('[data-testid="SideNav_AccountSwitcher_Button"], [data-testid="AppTabBar_Profile_Link"]'),
+ };
+})()`;
+
+const onSignInPage = (path: string): boolean => /^\/(login|i\/flow\/login|i\/jf\/onboarding|account\/access)/.test(path);
+
+/** Which step the page is on. Pure, so it can be tested on recorded probes. */
+export const classify = (p: LoginProbe): LoginStep => {
+ if (p.signedIn && !onSignInPage(p.path)) return { kind: "done" };
+ const has = (f: (i: LoginProbe["inputs"][number]) => boolean) => p.inputs.some(f);
+ const text = p.text;
+ if (/wrong password|incorrect password/i.test(text)) return { kind: "wrong-password" };
+ if (has((i) => i.type === "password")) return { kind: "password" };
+ const codeField = has((i) => i.autocomplete.includes("one-time-code") || i.name === "text" || i.name === "code" || i.name === "verfication_code");
+ if (codeField && /authenticat(or|ion) app|two[- ]factor/i.test(text) && !/email/i.test(text)) return { kind: "totp" };
+ if (codeField && /email/i.test(text) && /code/i.test(text)) return { kind: "email-code" };
+ if (has((i) => i.name === "username_or_email" || i.autocomplete.startsWith("username"))) return { kind: "username" };
+ // "Enter your phone number or username": X asks again when it is unsure.
+ if (has((i) => i.name === "text") && /phone number or username|username/i.test(text)) return { kind: "username" };
+ if (p.inputs.length === 0 && text === "") return { kind: "loading" };
+ return { kind: "unknown" };
+};
+
+const sleep = (ms: number, signal: AbortSignal): Promise<void> =>
+ new Promise((resolve, reject) => {
+ if (signal.aborted) return reject(signal.reason);
+ const t = setTimeout(resolve, ms);
+ signal.addEventListener("abort", () => {
+ clearTimeout(t);
+ reject(signal.reason);
+ }, { once: true });
+ });
+
+/** Focuses the step's text field: the password field on the password step, else the first one. */
+const FOCUS = (password: boolean) => `(() => {
+ const inputs = [...document.querySelectorAll("input")].filter(${SHOWN});
+ const el = ${password ? `inputs.find((i) => i.type === "password")` : `inputs.find((i) => i.type !== "password" && i.type !== "checkbox")`};
+ if (!el) return false;
+ el.focus();
+ if (el.value) { el.select(); }
+ return document.activeElement === el;
+})()`;
+
+/** False when the field is gone, the page moving between steps: look again. */
+const answer = async (page: PageSession, text: () => string, password: boolean, signal: AbortSignal): Promise<boolean> => {
+ if (!(await page.evaluate<boolean>(FOCUS(password), signal).catch(() => false))) return false;
+ await page.insertText(text(), signal);
+ await sleep(300, signal);
+ await page.pressEnter(signal);
+ return true;
+};
+
+/** What makes one step differ from the next: not its text, which changes as it is typed into. */
+const shape = (p: LoginProbe): string => `${p.path}${p.hash}|${p.inputs.map((i) => `${i.name}:${i.type}`).join(",")}`;
+
+/** Waits for X's emailed code, sent at or after `since`. */
+const emailedCode = async (imap: ImapAccount, since: number, signal: AbortSignal): Promise<string> => {
+ const deadline = Date.now() + EMAIL_WAIT_MS;
+ let seen = 0;
+ while (Date.now() < deadline) {
+ // A minute's slack for the two clocks; an older code is from an earlier sign-in.
+ const mail = await recentMail(imap, "x.com", since - 60_000, signal);
+ seen = mail.length;
+ for (const m of mail) {
+ const code = xCodeFromSubject(m.subject);
+ if (code) return code;
+ }
+ await sleep(EMAIL_POLL_MS, signal);
+ }
+ throw new BrowserUnavailableError(
+ `X sign-in: X asked for the code it emailed, and none arrived at ${imap.user} within ` +
+ `${EMAIL_WAIT_MS / 1000} s (${seen} recent mails from x.com, none with a code in the subject). ` +
+ `The account's email may not be that mailbox.`,
+ );
+};
+
+/**
+ * Signs in on `page`. Resolves when X shows the signed-in app; throws
+ * BrowserUnavailableError naming the step it could not get past.
+ */
+export const xLogin = async (
+ page: PageSession,
+ creds: XCredentials,
+ signal: AbortSignal,
+ log: (line: string) => void = () => {},
+): Promise<void> => {
+ const started = Date.now();
+ await page.navigate(LOGIN_URL, signal);
+ const answered = new Map<string, number>();
+ let last: LoginProbe | undefined;
+
+ for (let steps = 0; steps < MAX_STEPS; ) {
+ // A step's Enter navigates, and a probe caught mid-navigation fails: retry.
+ const probe = await page.evaluate<LoginProbe>(LOGIN_PROBE, signal).catch((e: unknown) => {
+ if (signal.aborted) throw e;
+ return undefined;
+ });
+ if (!probe) {
+ await sleep(400, signal);
+ continue;
+ }
+ last = probe;
+ const step = classify(probe);
+ if (step.kind === "done") {
+ log(`x-login signed in as ${creds.username}`);
+ return;
+ }
+ if (step.kind === "loading") {
+ await sleep(400, signal);
+ continue;
+ }
+ // Coming back to a step means the answer was refused: stop rather than
+ // hammer X with a wrong password or a stale code. The password is never
+ // sent twice; a code gets one more go, in case it expired on the way.
+ const key = step.kind;
+ if (step.kind === "wrong-password" || (answered.get(key) ?? 0) >= (key === "password" ? 1 : 2)) {
+ throw new BrowserUnavailableError(
+ `X sign-in refused at the ${step.kind} step (${probe.url}): "${probe.text.slice(0, 200)}". ` +
+ `Tell the user: someone has to sign in to X in castle Chrome by hand.`,
+ );
+ }
+ if (step.kind === "unknown") {
+ throw new BrowserUnavailableError(
+ `X sign-in stopped at a step this tool does not handle (${probe.url}): "${probe.text.slice(0, 300)}". ` +
+ `Tell the user: someone has to sign in to X in castle Chrome by hand.`,
+ );
+ }
+
+ let code: (() => string) | undefined;
+ if (step.kind === "username") code = () => creds.username;
+ else if (step.kind === "password") code = () => creds.password;
+ else if (step.kind === "totp") {
+ const secret = creds.totpSecret;
+ if (!secret) throw new BrowserUnavailableError("X sign-in wants an authenticator code and the sign-in file has no totpSecret");
+ // A code about to expire may be stale by the time X checks it.
+ if (totpRemaining() < 5) await sleep(totpRemaining() * 1000 + 500, signal);
+ code = () => totp(secret);
+ } else if (step.kind === "email-code") {
+ if (!creds.imap) {
+ throw new BrowserUnavailableError(
+ `X sign-in wants a code it emailed, and the sign-in file has no mailbox to read it from ` +
+ `(${probe.url}). Tell the user: someone has to sign in to X in castle Chrome by hand.`,
+ );
+ }
+ const emailed = await emailedCode(creds.imap, started, signal);
+ code = () => emailed;
+ }
+ if (!code || !(await answer(page, code, step.kind === "password", signal))) {
+ await sleep(400, signal);
+ continue;
+ }
+ log(`x-login answered ${step.kind} at ${probe.hash || probe.path}`);
+ answered.set(key, (answered.get(key) ?? 0) + 1);
+ steps++;
+
+ // Wait for the next step, up to ~10 s; a refused answer stays on this one.
+ const before = shape(probe);
+ for (let w = 0; w < 25; w++) {
+ await sleep(400, signal);
+ const now = await page.evaluate<LoginProbe>(LOGIN_PROBE, signal).catch(() => undefined);
+ if (!now || shape(now) !== before || classify(now).kind !== step.kind) break;
+ }
+ }
+ throw new BrowserUnavailableError(
+ `X sign-in did not finish within ${MAX_STEPS} steps (last at ${last?.url}). Tell the user: someone has ` +
+ `to sign in to X in castle Chrome by hand.`,
+ );
+};
diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts
index 6ea086c..93e8b56 100644
--- a/castle-search/src/x-search.ts
+++ b/castle-search/src/x-search.ts
@@ -1,26 +1,54 @@
/**
- * Search X (x.com) through the browser's own signed-in session.
+ * Read X (x.com) through the browser's own signed-in session: search posts,
+ * read one post with its thread and replies, read a profile, list trends.
*
- * X serves search only to a signed-in session and renders it from JavaScript,
- * so the page is the only interface there is: open x.com/search in a background
- * tab, wait for posts to render, read them, scroll for more, close the tab.
- * Free per search, unlike xAI's x_search ($5 per 1,000 posts returned).
+ * X serves all of this only to a signed-in session and renders it from
+ * JavaScript, so the page is the only interface there is: open the page in a
+ * background tab, wait for it to render, read it, scroll for more, close the
+ * tab. Free, unlike xAI's x_search ($5 per 1,000 posts returned).
*
- * The query is X's own search syntax, passed through untouched: from:handle,
- * to:handle, since:2026-11-01, until:..., min_faves:100, filter:media,
- * -filter:replies, lang:en, "exact phrase", OR.
+ * A search query is X's own search syntax, passed through untouched:
+ * from:handle, to:handle, since:2026-11-01, until:..., min_faves:100,
+ * filter:media, -filter:replies, lang:en, "exact phrase", OR.
*
- * Every failure throws, as for the engine searches (errors.ts): signed out, an
- * X error page, or nothing rendering are reported as such, never as "no posts".
- * Only X's own "No results" page is an empty answer, and even that is reported
- * as an error naming the query, since an over-narrow query is the usual cause.
+ * Every read shares one set of limits (x-limits.ts) and, signed out, signs the
+ * browser back in once (x-login.ts). Every failure throws, as for the engine
+ * searches (errors.ts): signed out, an X error page, or nothing rendering are
+ * reported as such, never as an empty answer. X's own "No results" or "doesn't
+ * exist" page is reported as an error naming what was asked for.
*/
+import type { PageSession } from "./cdp.ts";
import { BrowserUnavailableError, NoResultsError } from "./errors.ts";
import { withPage } from "./search.ts";
+import { XLimiter } from "./x-limits.ts";
+import { loadCredentials, X_LOGIN_TIMEOUT_MS, xLogin } from "./x-login.ts";
+
+/** Shared by every X read in this process: see x-limits.ts. */
+const limiter = new XLimiter();
+
+/**
+ * Signed out, a read signs the browser back in (x-login.ts) when
+ * CCS_X_LOGIN_FILE names the credentials, then reads again, once. At most one
+ * sign-in attempt per LOGIN_COOLDOWN_MS: a sign-in that keeps failing is what
+ * gets an account locked, so until the cooldown has passed every read fails
+ * with the reason the last attempt gave.
+ */
+export const LOGIN_COOLDOWN_MS = 30 * 60_000;
+let lastLogin: { at: number; error?: string } | undefined;
+
+class SignedOut extends Error {
+ readonly at: string;
+ constructor(at: string) {
+ super(`signed out at ${at}`);
+ this.at = at;
+ }
+}
export type XSort = "latest" | "top";
+export type XMedia = { type: "photo" | "video"; url?: string; alt?: string };
+
export type XPost = {
url: string;
author: string;
@@ -28,7 +56,7 @@ export type XPost = {
/** ISO 8601, from the post's <time datetime>. */
time: string;
text: string;
- /** X cut the text short behind "Show more"; the full post is at url. */
+ /** X cut the text short behind "Show more"; x_post reads it whole. */
truncated?: true;
/** "@someone reposted", "Pinned" and the like. */
context?: string;
@@ -36,18 +64,48 @@ export type XPost = {
reposts?: number;
likes?: number;
views?: number;
- /** Count of photos and videos attached. */
- media?: number;
- quoted?: { author: string; handle: string; text: string };
+ /** Photos (with X's image URL and alt text) and videos (with a still). */
+ media?: XMedia[];
+ /** A link card: the page the post links to. */
+ card?: { url: string; title?: string };
+ quoted?: { author: string; handle: string; text: string; time?: string; url?: string };
};
export type XSearchRequest = { query: string; count: number; sort?: XSort | undefined };
-
export type XSearchOutcome = { query: string; sort: XSort; searchUrl: string; posts: XPost[] };
+export type XPostRequest = { url: string; replies: number };
+export type XPostOutcome = {
+ post: XPost;
+ /** The posts above it in its thread, oldest first, when it is a reply. */
+ thread: XPost[];
+ replies: XPost[];
+};
+
+export type XUserRequest = { handle: string; count: number };
+export type XProfile = {
+ name: string;
+ handle: string;
+ url: string;
+ bio?: string;
+ location?: string;
+ website?: string;
+ joined?: string;
+ followers?: number;
+ following?: number;
+ posts?: number;
+ avatar?: string;
+};
+export type XUserOutcome = { profile: XProfile; posts: XPost[] };
+
+export const TREND_TABS = ["trending", "news", "sports", "entertainment", "for_you"] as const;
+export type XTrendTab = (typeof TREND_TABS)[number];
+export type XTrend = { rank?: number; name: string; context?: string; detail?: string; query: string };
+export type XTrendsOutcome = { tab: XTrendTab; trends: XTrend[] };
+
/** X shows ~5-10 posts per screen; scroll this many times at most for more. */
const MAX_SCROLLS = 6;
-/** Wall clock for one search: render, scrolls and all. */
+/** Wall clock for one read: render, scrolls and all. */
export const X_SEARCH_TIMEOUT_MS = 25_000;
const POLL_MS = 400;
@@ -58,6 +116,16 @@ export const buildXSearchUrl = (query: string, sort: XSort): string => {
return `https://x.com/search?${params.toString()}`;
};
+/** The post id in an X or Twitter post link, or a bare id. */
+export const postIdOf = (url: string): string | undefined =>
+ /^\d{5,25}$/.test(url.trim()) ? url.trim() : /(?:x|twitter)\.com\/(?:[^/]+|i(?:\/web)?)\/status(?:es)?\/(\d+)/i.exec(url)?.[1];
+
+/** A handle as X writes it, from "@nasa", "nasa" or a profile link. */
+export const handleOf = (s: string): string | undefined => {
+ const m = /^(?:https?:\/\/(?:www\.)?(?:x|twitter)\.com\/)?@?([A-Za-z0-9_]{1,15})\/?$/.exec(s.trim());
+ return m?.[1];
+};
+
/** Parses X's action-bar label: "441 replies, 971 reposts, 8852 likes, 1247 bookmarks, 16029782 views". */
export const parseMetrics = (label: string | null | undefined): Partial<Record<"replies" | "reposts" | "likes" | "views", number>> => {
const out: Partial<Record<"replies" | "reposts" | "likes" | "views", number>> = {};
@@ -72,6 +140,21 @@ export const parseMetrics = (label: string | null | undefined): Partial<Record<"
return out;
};
+/** One trend cell's lines: "2", "·", "Gaming · Trending", "GTA 6", "Trending with #GTAVI". */
+export const parseTrend = (text: string): XTrend | undefined => {
+ const lines = text.split("\n").map((l) => l.trim()).filter((l) => l && l !== "·");
+ let rank: number | undefined;
+ if (lines[0] && /^\d+$/.test(lines[0])) rank = Number(lines.shift());
+ const context = lines.shift();
+ const name = lines.shift();
+ if (!name) return undefined;
+ const trend: XTrend = { name, query: name };
+ if (rank !== undefined) trend.rank = rank;
+ if (context) trend.context = context;
+ if (lines.length) trend.detail = lines.join("; ");
+ return trend;
+};
+
type RawPost = Omit<XPost, "replies" | "reposts" | "likes" | "views"> & { metrics: string | null };
type XProbe = {
@@ -79,32 +162,41 @@ type XProbe = {
state: "signed-out" | "posts" | "empty" | "error" | "loading";
posts: RawPost[];
note: string;
+ /** The profile's JSON-LD, on a profile page. */
+ profile: string | null;
};
/**
* Reads the page as it stands. Runs in the page; must stay self-contained.
* Ads are skipped by having no <time>: every real post has one.
*/
-const PROBE = `(() => {
+export const PROBE = `(() => {
const url = location.href;
if (/^\\/(login|i\\/flow\\/login|i\\/jf\\/onboarding|account\\/access)/.test(location.pathname)) {
- return { url, state: "signed-out", posts: [], note: location.pathname };
+ return { url, state: "signed-out", posts: [], note: location.pathname, profile: null };
}
const txt = (el) => (el ? el.innerText.trim() : "");
const nameOf = (scope) => {
const lines = txt(scope.querySelector('[data-testid="User-Name"]')).split("\\n");
return { author: lines[0] || "", handle: lines.find((l) => l.startsWith("@")) || "" };
};
+ const clean = (href) => href.split("?")[0].replace(/\\/(analytics|photo\\/\\d+|video\\/\\d+)$/, "");
const posts = [];
for (const a of document.querySelectorAll('article[data-testid="tweet"]')) {
- const time = a.querySelector("time");
+ // The quoted post: the clickable box holding post text. Not simply the
+ // first div[role=link], which can be something else in the post.
+ const quote = [...a.querySelectorAll('div[role="link"]')].find((d) => d.querySelector('[data-testid="tweetText"], [data-testid="User-Name"]')) || null;
+ const mine = (el) => !(quote && quote.contains(el));
+ // The post's own time, a link to it. On a post's own page that sits below
+ // the text, after the quoted post's unlinked one.
+ const time = [...a.querySelectorAll("time")].find((t) => mine(t) && t.closest("a"));
const link = time && time.closest("a");
if (!time || !link) continue;
- const quote = a.querySelector('div[role="link"]');
- const own = [...a.querySelectorAll('[data-testid="tweetText"]')].filter((t) => !(quote && quote.contains(t)));
+ const own = [...a.querySelectorAll('[data-testid="tweetText"]')].filter(mine);
+ const byline = [...a.querySelectorAll('[data-testid="User-Name"]')].find(mine);
const post = {
- url: link.href.split("?")[0].replace(/\\/(analytics|photo\\/\\d+)$/, ""),
- ...nameOf(a),
+ url: clean(link.href),
+ ...nameOf(byline ? byline.parentElement : a),
time: time.getAttribute("datetime") || "",
text: own.map(txt).join("\\n"),
metrics: null,
@@ -114,23 +206,57 @@ const PROBE = `(() => {
if (a.querySelector('[data-testid="tweet-text-show-more-link"]')) post.truncated = true;
const ctx = txt(a.querySelector('[data-testid="socialContext"]'));
if (ctx) post.context = ctx;
- const media = [...a.querySelectorAll('[data-testid="tweetPhoto"], video')].filter((m) => !(quote && quote.contains(m))).length;
- if (media) post.media = media;
+ const media = [];
+ // A video's still is an <img> in the same box: listed with the video, not as a photo.
+ for (const img of [...a.querySelectorAll('[data-testid="tweetPhoto"] img')].filter((i) => mine(i) && !i.closest('[data-testid="tweetPhoto"]').querySelector("video"))) {
+ const m = { type: "photo", url: img.src.replace(/name=\\w+/, "name=large") };
+ if (img.alt && img.alt !== "Image") m.alt = img.alt;
+ media.push(m);
+ }
+ for (const v of [...a.querySelectorAll("video")].filter(mine)) {
+ const m = { type: "video" };
+ if (v.poster) m.url = v.poster;
+ const label = v.getAttribute("aria-label");
+ if (label && label !== "Embedded video") m.alt = label;
+ media.push(m);
+ }
+ if (media.length) post.media = media;
+ const card = [...a.querySelectorAll('[data-testid="card.wrapper"] a[href]')].find(mine);
+ if (card) {
+ post.card = { url: card.href };
+ const title = txt(card).split("\\n").filter(Boolean).pop();
+ if (title) post.card.title = title.slice(0, 200);
+ }
if (quote) {
const qt = txt(quote.querySelector('[data-testid="tweetText"]'));
- if (qt) post.quoted = { ...nameOf(quote), text: qt };
+ if (qt) {
+ post.quoted = { ...nameOf(quote), text: qt };
+ const qtime = quote.querySelector("time");
+ if (qtime) post.quoted.time = qtime.getAttribute("datetime") || "";
+ const qlink = quote.querySelector('a[href*="/status/"]');
+ if (qlink) post.quoted.url = clean(qlink.href);
+ }
}
posts.push(post);
}
- if (posts.length) return { url, state: "posts", posts, note: "" };
+ const schema = document.querySelector('[data-testid="UserProfileSchema-test"]');
+ const profile = schema ? schema.textContent : null;
+ if (posts.length) return { url, state: "posts", posts, note: "", profile };
const empty = document.querySelector('[data-testid="emptyState"]');
- if (empty) return { url, state: "empty", posts, note: txt(empty).slice(0, 300) };
- const retry = [...document.querySelectorAll('button, [role="button"]')].find((b) => /^Retry$/.test(txt(b)));
- if (retry) {
- const col = document.querySelector('[data-testid="primaryColumn"]');
- return { url, state: "error", posts, note: txt(col).slice(0, 300) };
+ if (empty) return { url, state: "empty", posts, note: txt(empty).slice(0, 300), profile };
+ const col = document.querySelector('[data-testid="primaryColumn"]');
+ const said = txt(col);
+ if (/doesn.t exist|was deleted|account is suspended|these posts are protected|you.re unable to view this post/i.test(said)) {
+ return { url, state: "empty", posts, note: said.slice(0, 300), profile };
}
- return { url, state: "loading", posts, note: "" };
+ const retry = [...document.querySelectorAll('button, [role="button"]')].find((b) => /^Retry$/.test(txt(b)));
+ if (retry) return { url, state: "error", posts, note: said.slice(0, 300), profile };
+ return { url, state: "loading", posts, note: "", profile };
+})()`;
+
+const TRENDS = `(() => {
+ if (/^\\/(login|i\\/flow\\/login|i\\/jf\\/onboarding|account\\/access)/.test(location.pathname)) return null;
+ return [...document.querySelectorAll('[data-testid="trend"]')].map((t) => t.innerText);
})()`;
const SCROLL = `window.scrollBy(0, window.innerHeight * 2), true`;
@@ -150,18 +276,41 @@ const finish = (raw: RawPost): XPost => {
return { ...rest, ...parseMetrics(metrics) };
};
-export const xSearch = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => {
- const sort: XSort = request.sort ?? "latest";
- const searchUrl = buildXSearchUrl(request.query, sort);
- const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]);
+type Saw = (state: string) => void;
+
+// --- Shared: limits, sign-in, deadline -----------------------------------
+
+/**
+ * Runs one X read: in turn with every other (the limits), within its own
+ * deadline, signing in and reading again once if X has signed the browser out.
+ */
+const xRead = <T>(outer: AbortSignal, what: string, read: (page: PageSession, signal: AbortSignal, saw: Saw) => Promise<T>): Promise<T> =>
+ // The deadline starts once the limiter lets the read through, so waiting
+ // its turn does not eat into it.
+ limiter.run(outer, async () => {
+ try {
+ return await attempt(outer, what, read);
+ } catch (e) {
+ if (!(e instanceof SignedOut)) throw e;
+ // The sign-in has its own deadline, and the read after it a fresh one.
+ await signIn(e.at, outer);
+ return await attempt(outer, what, read);
+ }
+ });
+const attempt = async <T>(
+ outer: AbortSignal,
+ what: string,
+ read: (page: PageSession, signal: AbortSignal, saw: Saw) => Promise<T>,
+): Promise<T> => {
+ const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]);
let last = "nothing yet";
try {
- return await run(request, sort, searchUrl, signal, (state) => (last = state));
+ return await withPage(signal, (page) => read(page, signal, (state) => (last = state)));
} catch (e) {
if (signal.aborted && !outer.aborted) {
throw new BrowserUnavailableError(
- `X search did not finish within ${X_SEARCH_TIMEOUT_MS / 1000} s (${searchUrl}; last seen: ${last}). ` +
+ `X did not finish ${what} within ${X_SEARCH_TIMEOUT_MS / 1000} s (last seen: ${last}). ` +
`X may be slow or stuck on a page this tool does not recognise.`,
);
}
@@ -169,38 +318,110 @@ export const xSearch = async (request: XSearchRequest, outer: AbortSignal): Prom
}
};
-const run = (
- request: XSearchRequest,
- sort: XSort,
- searchUrl: string,
- signal: AbortSignal,
- saw: (state: string) => void,
-): Promise<XSearchOutcome> =>
- withPage(signal, async (page, endpoint) => {
- await page.navigate(searchUrl, signal);
-
- // X renders after the load event: poll until posts, an empty or error
- // page, or a sign-in redirect shows up.
- let probe: XProbe;
- for (;;) {
- probe = await page.evaluate<XProbe>(PROBE, signal);
- saw(`${probe.state} at ${probe.url}`);
- if (probe.state !== "loading") break;
- await sleep(POLL_MS, signal);
- }
+const signedOutError = (at: string, why: string): BrowserUnavailableError =>
+ new BrowserUnavailableError(
+ `X needs a signed-in X session in castle Chrome, and it is signed out (X sent it to ${at}). ` +
+ `${why} Tell the user: someone has to sign in to X in that browser again.`,
+ );
- if (probe.state === "signed-out") {
- throw new BrowserUnavailableError(
- `X search needs a signed-in X session in the browser (${endpoint}), and it is signed out ` +
- `(X sent it to ${probe.url}). Tell the user: someone has to sign in to X in that browser again.`,
- );
- }
+/** Signs the browser in, within its own deadline, or throws saying why not. */
+const signIn = async (at: string, outer: AbortSignal): Promise<void> => {
+ const creds = loadCredentials(process.env["CCS_X_LOGIN_FILE"]);
+ if (!creds) throw signedOutError(at, "No sign-in details are configured (CCS_X_LOGIN_FILE).");
+ if (lastLogin && Date.now() - lastLogin.at < LOGIN_COOLDOWN_MS) {
+ const next = new Date(lastLogin.at + LOGIN_COOLDOWN_MS).toISOString().slice(11, 16);
+ throw signedOutError(
+ at,
+ `Signing in again was tried at ${new Date(lastLogin.at).toISOString().slice(11, 16)} UTC` +
+ (lastLogin.error ? ` and failed: ${lastLogin.error}` : ", yet it is signed out again.") +
+ ` The next try is after ${next} UTC.`,
+ );
+ }
+ lastLogin = { at: Date.now() };
+ const signal = AbortSignal.any([outer, AbortSignal.timeout(X_LOGIN_TIMEOUT_MS)]);
+ try {
+ await withPage(signal, (page) => xLogin(page, creds, signal, (line) => console.log(line)));
+ } catch (e) {
+ const why = signal.aborted && !outer.aborted ? `it took over ${X_LOGIN_TIMEOUT_MS / 1000} s` : (e as Error).message;
+ lastLogin.error = why;
+ console.log(`x-login failed: ${why}`);
+ throw signedOutError(at, `Signing in again failed: ${why}`);
+ }
+};
+
+/**
+ * Opens `url` and waits for X to render: posts, an empty or error page, or a
+ * sign-in redirect. `ready` can ask for more before posts count, such as a
+ * profile's header.
+ */
+const open = async (
+ page: PageSession,
+ url: string,
+ signal: AbortSignal,
+ saw: Saw,
+ ready: (p: XProbe) => boolean = (p) => p.state !== "loading",
+): Promise<XProbe> => {
+ await page.navigate(url, signal);
+ for (;;) {
+ const probe = await page.evaluate<XProbe>(PROBE, signal);
+ saw(`${probe.state} at ${probe.url}`);
+ if (probe.state === "signed-out") throw new SignedOut(probe.url);
if (probe.state === "error") {
throw new BrowserUnavailableError(
- `X answered with an error page instead of results (${probe.url}): "${probe.note}". This is ` +
- `usually X rate-limiting the account; wait a few minutes rather than retrying in a loop.`,
+ `X answered with an error page (${probe.url}): "${probe.note}". This is usually X rate-limiting ` +
+ `the account; wait a few minutes rather than retrying in a loop.`,
);
}
+ if (probe.state === "empty" || ready(probe)) return probe;
+ await sleep(POLL_MS, signal);
+ }
+};
+
+/**
+ * X keeps only the posts near the viewport in the page, so collect by URL
+ * while scrolling rather than reading once at the end. Stops at `enough`, or
+ * when scrolling brings nothing new.
+ */
+const collect = async (page: PageSession, first: XProbe, enough: (seen: Map<string, RawPost>) => boolean, signal: AbortSignal): Promise<RawPost[]> => {
+ // The newest reading of a post wins: photos and cards render after the
+ // text, so the first reading of a post can lack them. A Map keeps the order
+ // posts were first seen in.
+ const seen = new Map<string, RawPost>();
+ const take = (p: XProbe) => p.posts.forEach((post) => seen.set(post.url, post));
+ take(first);
+ // Let the first screen settle: photos, cards and a post's replies arrive
+ // after the post itself. Settled means two readings in a row agree, and
+ // there are enough posts or ~3 s have passed.
+ const shape = (p: XProbe) => p.posts.map((x) => `${x.url}:${x.media?.length ?? 0}:${x.card ? 1 : 0}`).join(" ");
+ let previous = shape(first);
+ for (let w = 0; w < 8; w++) {
+ await sleep(POLL_MS, signal);
+ const now = await page.evaluate<XProbe>(PROBE, signal);
+ take(now);
+ const current = shape(now);
+ if (current === previous && (enough(seen) || w >= 3)) break;
+ previous = current;
+ }
+ for (let i = 0; i < MAX_SCROLLS && !enough(seen); i++) {
+ const before = seen.size;
+ await page.evaluate(SCROLL, signal);
+ // New posts arrive over the next second or so.
+ for (let w = 0; w < 4 && seen.size === before; w++) {
+ await sleep(POLL_MS, signal);
+ take(await page.evaluate<XProbe>(PROBE, signal));
+ }
+ if (seen.size === before) break; // the end of the page
+ }
+ return [...seen.values()];
+};
+
+// --- The reads ---------------------------------------------------------------
+
+export const xSearch = (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => {
+ const sort: XSort = request.sort ?? "latest";
+ const searchUrl = buildXSearchUrl(request.query, sort);
+ return xRead(outer, `the search ${searchUrl}`, async (page, signal, saw) => {
+ const probe = await open(page, searchUrl, signal, saw);
if (probe.state === "empty") {
throw new NoResultsError(
`X found no posts for ${JSON.stringify(request.query)} (${sort}). X said: "${probe.note}". ` +
@@ -208,22 +429,104 @@ const run = (
`or try sort "top" or a wider date range.`,
);
}
+ const posts = await collect(page, probe, (s) => s.size >= request.count, signal);
+ return { query: request.query, sort, searchUrl, posts: posts.slice(0, request.count).map(finish) };
+ });
+};
+
+export const xPost = (request: XPostRequest, outer: AbortSignal): Promise<XPostOutcome> => {
+ const id = postIdOf(request.url);
+ if (!id) return Promise.reject(new BrowserUnavailableError(`not an X post link: ${request.url}`));
+ // /i/status/<id> redirects to the post under its author's handle.
+ const url = `https://x.com/i/status/${id}`;
+ const isMain = (p: RawPost) => p.url.endsWith(`/status/${id}`);
+ return xRead(outer, `reading ${url}`, async (page, signal, saw) => {
+ const probe = await open(page, url, signal, saw, (p) => p.posts.some(isMain));
+ if (probe.state === "empty") {
+ throw new NoResultsError(`X shows no post at ${url}: "${probe.note}". It may be deleted, or its account protected or suspended.`);
+ }
+ const posts = await collect(page, probe, (s) => s.size >= request.replies + 1 + [...s.values()].findIndex(isMain), signal);
+ const at = posts.findIndex(isMain);
+ if (at < 0) throw new BrowserUnavailableError(`X's page for ${url} did not show the post itself (${probe.url})`);
+ return {
+ post: finish(posts[at]!),
+ thread: posts.slice(0, at).map(finish),
+ replies: posts.slice(at + 1, at + 1 + request.replies).map(finish),
+ };
+ });
+};
+
+type ProfileSchema = {
+ dateCreated?: string;
+ relatedLink?: string[];
+ mainEntity?: {
+ additionalName?: string;
+ name?: string;
+ description?: string;
+ homeLocation?: { name?: string };
+ image?: { contentUrl?: string };
+ url?: string;
+ interactionStatistic?: { name?: string; userInteractionCount?: number }[];
+ };
+};
- // X keeps only the posts near the viewport in the page, so collect by URL
- // while scrolling rather than reading once at the end.
- const seen = new Map<string, RawPost>();
- const take = (p: XProbe) => p.posts.forEach((post) => seen.has(post.url) || seen.set(post.url, post));
- take(probe);
- for (let i = 0; i < MAX_SCROLLS && seen.size < request.count; i++) {
- const before = seen.size;
- await page.evaluate(SCROLL, signal);
- // New posts arrive over the next second or so.
- for (let w = 0; w < 4 && seen.size === before; w++) {
+/** The profile from X's JSON-LD block: exact counts, where the page shows "92.2M". */
+export const parseProfile = (json: string, handle: string): XProfile => {
+ const s = JSON.parse(json) as ProfileSchema;
+ const e = s.mainEntity ?? {};
+ const count = (name: string) => e.interactionStatistic?.find((i) => i.name === name)?.userInteractionCount;
+ const profile: XProfile = { name: e.name ?? handle, handle: `@${e.additionalName ?? handle}`, url: e.url ?? `https://x.com/${handle}` };
+ if (e.description) profile.bio = e.description;
+ if (e.homeLocation?.name) profile.location = e.homeLocation.name;
+ const site = s.relatedLink?.find((l) => !/^https?:\/\/t\.co\//.test(l));
+ if (site) profile.website = site;
+ if (s.dateCreated) profile.joined = s.dateCreated;
+ const followers = count("Follows");
+ const following = count("Friends");
+ const posts = count("Tweets");
+ if (followers !== undefined) profile.followers = followers;
+ if (following !== undefined) profile.following = following;
+ if (posts !== undefined) profile.posts = posts;
+ if (e.image?.contentUrl) profile.avatar = e.image.contentUrl;
+ return profile;
+};
+
+export const xUser = (request: XUserRequest, outer: AbortSignal): Promise<XUserOutcome> => {
+ const handle = handleOf(request.handle);
+ if (!handle) return Promise.reject(new BrowserUnavailableError(`not an X handle: ${request.handle}`));
+ const url = `https://x.com/${handle}`;
+ return xRead(outer, `reading ${url}`, async (page, signal, saw) => {
+ // The header renders before the posts; wait for both, or for X to say
+ // the account has none to show.
+ const probe = await open(page, url, signal, saw, (p) => p.profile !== null && p.state !== "loading");
+ if (!probe.profile) {
+ throw new NoResultsError(`X shows no account @${handle}: "${probe.note}". Check the handle.`);
+ }
+ const profile = parseProfile(probe.profile, handle);
+ const posts = probe.state === "posts" && request.count > 0 ? await collect(page, probe, (s) => s.size >= request.count, signal) : [];
+ return { profile, posts: posts.slice(0, request.count).map(finish) };
+ });
+};
+
+export const xTrends = (tab: XTrendTab, outer: AbortSignal): Promise<XTrendsOutcome> => {
+ const url = `https://x.com/explore/tabs/${tab}`;
+ return xRead(outer, `reading ${url}`, async (page, signal, saw) => {
+ await page.navigate(url, signal);
+ for (;;) {
+ const cells = await page.evaluate<string[] | null>(TRENDS, signal);
+ if (cells === null) throw new SignedOut(url);
+ saw(`${cells.length} trends at ${url}`);
+ if (cells.length) {
+ // The list renders in one go; a moment more lets the last cells fill.
await sleep(POLL_MS, signal);
- take(await page.evaluate<XProbe>(PROBE, signal));
+ const all = (await page.evaluate<string[] | null>(TRENDS, signal)) ?? cells;
+ return { tab, trends: all.map(parseTrend).filter((t): t is XTrend => t !== undefined) };
+ }
+ const probe = await page.evaluate<XProbe>(PROBE, signal);
+ if (probe.state === "error") {
+ throw new BrowserUnavailableError(`X answered with an error page (${url}): "${probe.note}".`);
}
- if (seen.size === before) break; // the end of the results
+ await sleep(POLL_MS, signal);
}
-
- return { query: request.query, sort, searchUrl, posts: [...seen.values()].slice(0, request.count).map(finish) };
});
+};
diff --git a/castle-search/src/youtube.ts b/castle-search/src/youtube.ts
new file mode 100644
index 0000000..18b1d49
--- /dev/null
+++ b/castle-search/src/youtube.ts
@@ -0,0 +1,303 @@
+/**
+ * A YouTube video's transcript, filed as plain text in a folder (castle's
+ * synced Nextcloud "transcript" folder) and handed back to the caller.
+ *
+ * The recipe is the one Hermes worked out on bambos (its skill
+ * youtube-transcript-to-nextcloud), so files from either read the same:
+ *
+ * 1. the bare video link: share-link junk (?si=, ?t=, &list=) dropped;
+ * 2. title, channel, upload date and length from yt-dlp;
+ * 3. subtitles only, never the video: human-written ones when the video has
+ * them, YouTube's automatic ones otherwise, as VTT;
+ * 4. YouTube's rolling captions repeat the end of the previous cue, so each
+ * cue keeps only the words the text so far does not already end with;
+ * 5. filed as <upload date>-<Title-With-Dashes>.txt, paragraphs of 90 words.
+ *
+ * yt-dlp breaks whenever YouTube changes, so it updates itself (`yt-dlp -U`)
+ * at most once a day, and once more when a fetch fails. YouTube rate-limits
+ * its caption downloads (HTTP 429); then the transcript comes from
+ * youtube-transcript-api instead, which uses another endpoint.
+ *
+ * Configured by environment: CCS_YTDLP (the yt-dlp binary, one it can update),
+ * CCS_YT_PYTHON (a python with youtube_transcript_api, optional),
+ * CCS_TRANSCRIPT_DIR (where files go). One transcript at a time.
+ */
+
+import { execFile } from "node:child_process";
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { BrowserUnavailableError, NoResultsError } from "./errors.ts";
+
+export type TranscriptRequest = { url: string; language?: string | undefined; maxChars?: number | undefined };
+
+export type TranscriptOutcome = {
+ file: string;
+ /** Already in the folder from an earlier run, so not fetched again. */
+ existing: boolean;
+ url: string;
+ title: string;
+ channel: string;
+ /** YYYY-MM-DD */
+ uploaded: string;
+ /** Seconds. */
+ duration?: number;
+ /** "manual", "automatic", or "youtube-transcript-api". */
+ source: string;
+ words: number;
+ text: string;
+ /** The text was longer than maxChars; the file has all of it. */
+ truncated?: true;
+};
+
+const FETCH_TIMEOUT_MS = 120_000;
+const UPDATE_EVERY_MS = 24 * 3_600_000;
+export const DEFAULT_MAX_CHARS = 60_000;
+
+// --- Pure parts, tested --------------------------------------------------
+
+/** The video id in any YouTube link: watch?v=, youtu.be/, shorts/, live/, embed/. */
+export const videoIdOf = (url: string): string | undefined => {
+ const s = url.trim();
+ if (/^[A-Za-z0-9_-]{11}$/.test(s)) return s;
+ const m =
+ /(?:youtube\.com\/(?:watch\?(?:.*&)?v=|shorts\/|live\/|embed\/|v\/)|youtu\.be\/|youtube-nocookie\.com\/embed\/)([A-Za-z0-9_-]{11})/i.exec(s);
+ return m?.[1];
+};
+
+/** "Dr. Becky: Parenting 101 | Call Her Daddy" -> "Dr-Becky-Parenting-101-Call-Her-Daddy". */
+export const titleSlug = (title: string): string => {
+ const words = title
+ .normalize("NFKD")
+ .replace(/[\u0300-\u036f]/g, "")
+ .replace(/['’]/g, "")
+ .match(/[A-Za-z0-9]+/g);
+ return (words ?? []).join("-").slice(0, 100).replace(/-+$/, "") || "transcript";
+};
+
+/** "20051024" -> "2005-10-24". */
+export const isoDate = (yyyymmdd: string): string =>
+ /^\d{8}$/.test(yyyymmdd) ? `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}` : new Date().toISOString().slice(0, 10);
+
+const SKIP = ["WEBVTT", "NOTE", "Kind:", "Language:", "STYLE", "REGION", "X-TIMESTAMP"];
+
+/** Each cue's text, timings and markup removed. */
+export const cueTexts = (vtt: string): string[] => {
+ const out: string[] = [];
+ for (const block of vtt.replace(/\r/g, "").split(/\n\s*\n/)) {
+ const lines = block
+ .split("\n")
+ .map((l) => l.trim())
+ .filter((l) => l && !l.includes("-->") && !SKIP.some((p) => l.startsWith(p)) && !/^\d+$/.test(l));
+ if (!lines.length) continue;
+ const text = lines
+ .join(" ")
+ .replace(/<[^>]*>/g, "")
+ .replace(/\[[^\]]*\]/g, " ") // [Music], [Applause]
+ .replace(/&gt;/g, " ")
+ .replace(/&lt;/g, "<")
+ .replace(/&amp;/g, "&")
+ .replace(/&#39;/g, "'")
+ .replace(/&quot;/g, '"')
+ .replace(/&nbsp;/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+ if (text) out.push(text);
+ }
+ return out;
+};
+
+/**
+ * Rolling captions: each cue starts with the end of what came before. Keep
+ * only the words past the longest overlap with the text so far.
+ */
+export const dedupeRolling = (cues: string[], window = 80): string[] => {
+ const emitted: string[] = [];
+ for (const cue of cues) {
+ const words = cue.split(" ").filter(Boolean);
+ const tail = emitted.slice(-window);
+ let overlap = 0;
+ for (let k = Math.min(tail.length, words.length); k > 0; k--) {
+ if (tail.slice(-k).join(" ") === words.slice(0, k).join(" ")) {
+ overlap = k;
+ break;
+ }
+ }
+ emitted.push(...words.slice(overlap));
+ }
+ return emitted;
+};
+
+export const paragraphs = (words: string[], per = 90): string => {
+ const out: string[] = [];
+ for (let i = 0; i < words.length; i += per) out.push(words.slice(i, i + per).join(" "));
+ return out.join("\n\n");
+};
+
+// --- Running yt-dlp ------------------------------------------------------
+
+const run = (cmd: string, args: string[], signal: AbortSignal, timeout = FETCH_TIMEOUT_MS): Promise<{ code: number; stdout: string; stderr: string }> =>
+ new Promise((resolve) => {
+ execFile(cmd, args, { signal, timeout, maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => {
+ const code = error ? (typeof (error as { code?: unknown }).code === "number" ? (error as { code: number }).code : 1) : 0;
+ resolve({ code, stdout: String(stdout), stderr: String(stderr) });
+ });
+ });
+
+const ytdlp = (): string => process.env["CCS_YTDLP"] ?? "yt-dlp";
+
+let lastUpdate = 0;
+const update = async (signal: AbortSignal, force = false): Promise<void> => {
+ if (!force && Date.now() - lastUpdate < UPDATE_EVERY_MS) return;
+ lastUpdate = Date.now();
+ const r = await run(ytdlp(), ["-U"], signal, 60_000);
+ console.log(`yt-dlp -U: ${(r.stdout + r.stderr).trim().split("\n").pop() ?? ""}`);
+};
+
+type Meta = { title: string; channel: string; uploaded: string; duration?: number; id: string };
+
+const metadata = async (url: string, signal: AbortSignal): Promise<Meta | { error: string }> => {
+ const r = await run(ytdlp(), ["--no-warnings", "--skip-download", "--no-playlist", "--print", "%(title)s\t%(uploader)s\t%(upload_date)s\t%(duration)s\t%(id)s", url], signal);
+ const line = r.stdout.trim().split("\n").pop() ?? "";
+ const [title, channel, date, duration, id] = line.split("\t");
+ if (r.code !== 0 || !title || !id) return { error: (r.stderr.trim().split("\n").pop() ?? `yt-dlp exited ${r.code}`).slice(0, 300) };
+ const meta: Meta = { title, channel: channel && channel !== "NA" ? channel : "", uploaded: isoDate(date ?? ""), id };
+ const d = Number(duration);
+ if (Number.isFinite(d) && d > 0) meta.duration = d;
+ return meta;
+};
+
+/** The subtitle file yt-dlp wrote: human-written beats automatic (".en-orig" is automatic). */
+const pickSubs = (dir: string, lang: string): { path: string; source: string } | undefined => {
+ const files = readdirSync(dir).filter((f) => f.endsWith(".vtt") && statSync(join(dir, f)).size > 0);
+ const auto = files.find((f) => f.includes(`.${lang}-orig.`));
+ const manual = files.find((f) => f !== auto);
+ if (manual) {
+ const path = join(dir, manual);
+ const text = readFileSync(path, "utf8");
+ // Beside an -orig track, the plain one is human-written unless it is the
+ // same file. Alone, its word-timing tags say whether it is automatic.
+ const automatic = auto !== undefined ? text === readFileSync(join(dir, auto), "utf8") : /<\d\d:\d\d:\d\d\.\d{3}><c>/.test(text);
+ return { path, source: automatic ? "automatic" : "manual" };
+ }
+ return auto ? { path: join(dir, auto), source: "automatic" } : undefined;
+};
+
+const subtitles = async (id: string, lang: string, dir: string, signal: AbortSignal): Promise<{ words: string[]; source: string } | { error: string; rateLimited: boolean }> => {
+ // Only the language itself and its regional variants: a pattern like
+ // "en-.*" also takes YouTube's machine translations ("en-de"), which are
+ // what its rate limit falls on first.
+ const langs = [lang, `${lang}-orig`, ...(lang === "en" ? ["en-GB", "en-US"] : [])].join(",");
+ const r = await run(
+ ytdlp(),
+ ["--no-warnings", "--skip-download", "--no-playlist", "--write-subs", "--write-auto-subs", "--sub-langs", langs, "--sub-format", "vtt", "-o", join(dir, "%(id)s"), `https://www.youtube.com/watch?v=${id}`],
+ signal,
+ );
+ // yt-dlp fails the whole run when one of several tracks fails; a track it
+ // did write is still good.
+ const picked = pickSubs(dir, lang);
+ if (picked) return { words: dedupeRolling(cueTexts(readFileSync(picked.path, "utf8"))), source: picked.source };
+ const err = r.stderr.trim().split("\n").filter((l) => l.startsWith("ERROR")).pop() ?? "";
+ return { error: err || `no ${lang} subtitles`, rateLimited: /429|Too Many Requests/.test(err) };
+};
+
+const TRANSCRIPT_API = `
+import json, sys
+from youtube_transcript_api import YouTubeTranscriptApi
+vid, lang = sys.argv[1], sys.argv[2]
+langs = [lang] + ([f"{lang}-US", f"{lang}-GB"] if lang == "en" else [])
+f = YouTubeTranscriptApi().fetch(vid, languages=langs)
+print(json.dumps([s.text for s in f]))
+`;
+
+const transcriptApi = async (id: string, lang: string, signal: AbortSignal): Promise<string[] | { error: string }> => {
+ const py = process.env["CCS_YT_PYTHON"];
+ if (!py) return { error: "no youtube-transcript-api configured (CCS_YT_PYTHON)" };
+ const r = await run(py, ["-I", "-c", TRANSCRIPT_API, id, lang], signal);
+ if (r.code !== 0) return { error: (r.stderr.trim().split("\n").pop() ?? "").slice(0, 300) };
+ // Discrete snippets, not rolling cues: no dedupe needed.
+ return (JSON.parse(r.stdout) as string[]).join(" ").replace(/\[[^\]]*\]/g, " ").split(/\s+/).filter(Boolean);
+};
+
+// --- The tool ------------------------------------------------------------
+
+let tail: Promise<unknown> = Promise.resolve();
+
+/** One at a time: yt-dlp updating itself under a running fetch breaks both. */
+export const youtubeTranscript = (request: TranscriptRequest, signal: AbortSignal): Promise<TranscriptOutcome> => {
+ const next = tail.then(() => fetchTranscript(request, signal));
+ tail = next.catch(() => {});
+ return next;
+};
+
+const fetchTranscript = async (request: TranscriptRequest, signal: AbortSignal): Promise<TranscriptOutcome> => {
+ const id = videoIdOf(request.url);
+ if (!id) throw new NoResultsError(`not a YouTube video link: ${request.url}`);
+ const lang = (request.language ?? "en").trim() || "en";
+ const maxChars = request.maxChars ?? DEFAULT_MAX_CHARS;
+ const url = `https://www.youtube.com/watch?v=${id}`;
+ const folder = process.env["CCS_TRANSCRIPT_DIR"];
+ if (!folder || !existsSync(folder)) {
+ throw new BrowserUnavailableError(`the transcript folder (CCS_TRANSCRIPT_DIR=${folder ?? "unset"}) does not exist`);
+ }
+
+ await update(signal);
+ let meta = await metadata(url, signal);
+ if ("error" in meta) {
+ // An extractor broken by a YouTube change: a newer yt-dlp is nearly always the fix.
+ await update(signal, true);
+ meta = await metadata(url, signal);
+ if ("error" in meta) throw new BrowserUnavailableError(`yt-dlp could not read ${url}: ${meta.error}`);
+ }
+
+ const name = `${meta.uploaded}-${titleSlug(meta.title)}.txt`;
+ const file = join(folder, name);
+ const shape = (text: string, source: string, existing: boolean): TranscriptOutcome => {
+ const out: TranscriptOutcome = {
+ file: `transcript/${name}`,
+ existing,
+ url,
+ title: meta.title,
+ channel: meta.channel,
+ uploaded: meta.uploaded,
+ source,
+ words: text.split(/\s+/).filter(Boolean).length,
+ text: text.length > maxChars ? text.slice(0, maxChars) : text,
+ };
+ if ("duration" in meta && meta.duration !== undefined) out.duration = meta.duration;
+ if (text.length > maxChars) out.truncated = true;
+ return out;
+ };
+ if (existsSync(file)) return shape(readFileSync(file, "utf8").trim(), "earlier run", true);
+
+ const scratch = mkdtempSync(join(tmpdir(), "yt-"));
+ try {
+ const subs = await subtitles(id, lang, scratch, signal);
+ let words: string[];
+ let source: string;
+ if ("words" in subs) {
+ ({ words, source } = subs);
+ } else {
+ // A 429 is YouTube limiting the caption endpoint: retrying yt-dlp gets
+ // another. The transcript API is a different endpoint.
+ const alt = await transcriptApi(id, lang, signal);
+ if (!Array.isArray(alt)) {
+ throw new NoResultsError(
+ `no ${lang} transcript for ${url}: yt-dlp said "${subs.error}"` +
+ `${subs.rateLimited ? " (YouTube rate-limiting captions)" : ""}; youtube-transcript-api said "${alt.error}". ` +
+ `The video may have no captions in that language: try another language code.`,
+ );
+ }
+ words = alt;
+ source = "youtube-transcript-api";
+ }
+ if (words.length === 0) throw new NoResultsError(`the ${lang} captions of ${url} are empty`);
+ const text = paragraphs(words);
+ writeFileSync(file, text + "\n", "utf8");
+ console.log(`youtube ${id} -> ${name} (${words.length} words, ${source})`);
+ return shape(text, source, false);
+ } finally {
+ rmSync(scratch, { recursive: true, force: true });
+ }
+};
diff --git a/castle-search/test/http.test.ts b/castle-search/test/http.test.ts
index b2c90ab..c076863 100644
--- a/castle-search/test/http.test.ts
+++ b/castle-search/test/http.test.ts
@@ -116,3 +116,73 @@ test("X's action-bar label parses, singular and plural", () => {
test("the OpenAPI description names the operation Open WebUI will offer", () => {
assert.equal(OPENAPI.paths["/x-search"].post.operationId, "search_x");
});
+
+import { handleXPost, handleXTrends, handleXUser } from "../src/http.ts";
+import { handleOf, parseProfile, parseTrend, postIdOf } from "../src/x-search.ts";
+
+test("post links of every shape give the post id", () => {
+ assert.equal(postIdOf("https://x.com/NASA/status/2122740072386609201"), "2122740072386609201");
+ assert.equal(postIdOf("https://twitter.com/nasa/status/2122740072386609201?s=20"), "2122740072386609201");
+ assert.equal(postIdOf("https://x.com/i/web/status/2122740072386609201"), "2122740072386609201");
+ assert.equal(postIdOf("2122740072386609201"), "2122740072386609201");
+ assert.equal(postIdOf("https://x.com/NASA"), undefined);
+});
+
+test("handles are read from @name, name or a profile link", () => {
+ assert.equal(handleOf("@NASA"), "NASA");
+ assert.equal(handleOf("https://x.com/nasa/"), "nasa");
+ assert.equal(handleOf("not a handle"), undefined);
+ assert.equal(handleOf("waytoolonghandle_123"), undefined);
+});
+
+test("a trend cell's lines become a trend", () => {
+ assert.deepEqual(parseTrend("2\n·\nVideo game genres · Trending\nGTA 6\nTrending with #GTAVI, Metacritic"), {
+ rank: 2, context: "Video game genres · Trending", name: "GTA 6", query: "GTA 6", detail: "Trending with #GTAVI, Metacritic",
+ });
+ assert.deepEqual(parseTrend("Trending in United Kingdom\nGlastonbury"), { context: "Trending in United Kingdom", name: "Glastonbury", query: "Glastonbury" });
+ assert.equal(parseTrend("1"), undefined);
+});
+
+test("a profile's JSON-LD gives exact counts", () => {
+ const p = parseProfile(
+ JSON.stringify({
+ dateCreated: "2007-12-19T20:20:32.000Z",
+ relatedLink: ["https://t.co/9NkQJKAnuU", "http://www.nasa.gov/"],
+ mainEntity: {
+ additionalName: "NASA", name: "NASA", description: "Making the seemingly impossible, possible.",
+ homeLocation: { name: "Pale Blue Dot" }, url: "https://x.com/NASA",
+ interactionStatistic: [
+ { name: "Follows", userInteractionCount: 92225170 },
+ { name: "Friends", userInteractionCount: 115 },
+ { name: "Tweets", userInteractionCount: 74553 },
+ ],
+ },
+ }),
+ "nasa",
+ );
+ assert.equal(p.handle, "@NASA");
+ assert.equal(p.followers, 92225170);
+ assert.equal(p.following, 115);
+ assert.equal(p.posts, 74553);
+ assert.equal(p.website, "http://www.nasa.gov/");
+ assert.equal(p.location, "Pale Blue Dot");
+});
+
+test("the X reads check what they are given before touching the browser", async () => {
+ const never = async (): Promise<never> => {
+ throw new Error("must not run");
+ };
+ assert.equal((await handleXPost(JSON.stringify({ url: "https://x.com/NASA" }), never, signal)).status, 400);
+ assert.equal((await handleXUser(JSON.stringify({ handle: "two words" }), never, signal)).status, 400);
+ assert.equal((await handleXTrends(JSON.stringify({ tab: "music" }), never, signal)).status, 400);
+ assert.equal((await handleXPost("[1]", never, signal)).status, 400);
+ const seen: unknown[] = [];
+ await handleXPost(JSON.stringify({ url: "https://x.com/a/status/12345678", replies: 99 }), async (r) => (seen.push(r), { post: {}, thread: [], replies: [] }) as never, signal);
+ await handleXTrends("", async (t) => (seen.push(t), { tab: t, trends: [] }), signal);
+ assert.deepEqual(seen, [{ url: "https://x.com/a/status/12345678", replies: 20 }, "trending"]);
+});
+
+test("every X read is in the OpenAPI description", () => {
+ assert.deepEqual(Object.keys(OPENAPI.paths), ["/x-search", "/x-post", "/x-user", "/x-trends"]);
+ assert.equal(OPENAPI.info.title, "X.com");
+});
diff --git a/castle-search/test/x-limits.test.ts b/castle-search/test/x-limits.test.ts
new file mode 100644
index 0000000..6be5f7d
--- /dev/null
+++ b/castle-search/test/x-limits.test.ts
@@ -0,0 +1,98 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { type Clock, XLimiter, XRateLimitError } from "../src/x-limits.ts";
+
+/** A clock that only moves when told to, recording the waits asked of it. */
+const fakeClock = () => {
+ let now = 1_000_000;
+ const waits: number[] = [];
+ let draw = 0.5;
+ const clock: Clock = {
+ random: () => draw,
+ now: () => now,
+ sleep: async (ms) => {
+ waits.push(ms);
+ now += ms;
+ },
+ };
+ return { clock, waits, advance: (ms: number) => (now += ms), setDraw: (r: number) => (draw = r) };
+};
+
+const signal = new AbortController().signal;
+const ok = async () => "ok";
+
+test("a search sooner than the gap waits out the rest of it", async () => {
+ const { clock, waits, advance } = fakeClock();
+ const l = new XLimiter(clock, { gap: 10_000, jitter: 0, hour: 30, day: 150 });
+ await l.run(signal, ok);
+ advance(3_000);
+ await l.run(signal, ok);
+ assert.deepEqual(waits, [7_000]);
+});
+
+test("over the hourly cap a search fails at once, naming when it can run", async () => {
+ const { clock, advance } = fakeClock();
+ const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 2, day: 150 });
+ await l.run(signal, ok);
+ await l.run(signal, ok);
+ await assert.rejects(l.run(signal, ok), (e) => e instanceof XRateLimitError && /last hour/.test(e.message));
+ advance(3_600_001);
+ assert.equal(await l.run(signal, ok), "ok");
+});
+
+test("the daily cap holds across hours", async () => {
+ const { clock, advance } = fakeClock();
+ const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 100, day: 3 });
+ for (let i = 0; i < 3; i++) {
+ await l.run(signal, ok);
+ advance(3_600_001);
+ }
+ await assert.rejects(l.run(signal, ok), (e) => e instanceof XRateLimitError && /24 hours/.test(e.message));
+});
+
+test("one at a time: the second starts only after the first finishes", async () => {
+ const { clock } = fakeClock();
+ const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 30, day: 150 });
+ const order: string[] = [];
+ let release!: () => void;
+ const first = l.run(signal, async () => {
+ order.push("first start");
+ await new Promise<void>((r) => (release = r));
+ order.push("first end");
+ });
+ const second = l.run(signal, async () => {
+ order.push("second start");
+ });
+ await new Promise((r) => setTimeout(r, 10));
+ release();
+ await Promise.all([first, second]);
+ assert.deepEqual(order, ["first start", "first end", "second start"]);
+});
+
+test("a failed search does not block the queue", async () => {
+ const { clock } = fakeClock();
+ const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 30, day: 150 });
+ await assert.rejects(l.run(signal, async () => {
+ throw new Error("boom");
+ }));
+ assert.equal(await l.run(signal, ok), "ok");
+});
+
+test("the gap is drawn afresh each time, within gap ± jitter", async () => {
+ const { clock, waits, setDraw } = fakeClock();
+ const l = new XLimiter(clock, { gap: 30_000, jitter: 25_000, hour: 30, day: 150 });
+ await l.run(signal, ok);
+ setDraw(0); // the shortest: 30 - 25 s
+ await l.run(signal, ok);
+ setDraw(0.999999); // close to the longest: 30 + 25 s
+ await l.run(signal, ok);
+ assert.equal(waits[0], 5_000);
+ assert.ok(waits[1]! > 54_999 && waits[1]! <= 55_000);
+});
+
+test("the defaults are 30 s ± 25 s", async () => {
+ const { GAP_MS, JITTER_MS } = await import("../src/x-limits.ts");
+ assert.equal(GAP_MS, 30_000);
+ assert.equal(JITTER_MS, 25_000);
+});
diff --git a/castle-search/test/x-login.test.ts b/castle-search/test/x-login.test.ts
new file mode 100644
index 0000000..11468d5
--- /dev/null
+++ b/castle-search/test/x-login.test.ts
@@ -0,0 +1,91 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { decodeWords, parseFetch, parseInternalDate, xCodeFromSubject } from "../src/imap.ts";
+import { classify, type LoginProbe } from "../src/x-login.ts";
+import { base32Decode, totp, totpRemaining } from "../src/totp.ts";
+
+// RFC 6238 appendix B, SHA-1: the secret "12345678901234567890".
+const RFC_SECRET = "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ";
+
+test("totp matches RFC 6238's SHA-1 vectors", () => {
+ assert.equal(totp(RFC_SECRET, 59_000, 8), "94287082");
+ assert.equal(totp(RFC_SECRET, 1_111_111_109_000, 8), "07081804");
+ assert.equal(totp(RFC_SECRET, 1_234_567_890_000, 8), "89005924");
+ assert.equal(totp(RFC_SECRET, 59_000), "287082");
+});
+
+test("base32 ignores spaces and case, and refuses other characters", () => {
+ assert.deepEqual(base32Decode("gezd gnbv"), base32Decode("GEZDGNBV"));
+ assert.throws(() => base32Decode("GEZD1"));
+});
+
+test("totpRemaining counts down within the step", () => {
+ assert.equal(totpRemaining(0), 30);
+ assert.equal(totpRemaining(29_000), 1);
+});
+
+test("X's code is read from the subjects X uses, and nothing else", () => {
+ assert.equal(xCodeFromSubject("Your X confirmation code is abc12def"), "abc12def");
+ assert.equal(xCodeFromSubject("Your X verification code is: 123456"), "123456");
+ assert.equal(xCodeFromSubject("123456 is your X verification code"), "123456");
+ assert.equal(xCodeFromSubject("New login to X from Chrome on Mac"), undefined);
+ assert.equal(xCodeFromSubject("Security alert"), undefined);
+});
+
+test("encoded subjects are decoded", () => {
+ assert.equal(decodeWords("=?UTF-8?B?WW91ciBYIGNvbmZpcm1hdGlvbiBjb2RlIGlzIGFiYzEyZGVm?="), "Your X confirmation code is abc12def");
+ assert.equal(decodeWords("=?UTF-8?Q?Caf=C3=A9_menu?="), "Café menu");
+});
+
+test("a FETCH reply splits into one header set per message", () => {
+ const raw =
+ '* 1 FETCH (INTERNALDATE "18-Nov-2026 10:16:23 +0000" BODY[HEADER.FIELDS (FROM SUBJECT)] {60}\r\n' +
+ "From: X <verify@x.com>\r\nSubject: Your X confirmation code is\r\n abc12def\r\n\r\n)\r\n" +
+ '* 2 FETCH (INTERNALDATE "18-Nov-2026 10:20:00 +0000" BODY[HEADER.FIELDS (FROM SUBJECT)] {40}\r\n' +
+ "From: X <info@x.com>\r\nSubject: New login\r\n\r\n)\r\nc4 OK Success\r\n";
+ const mails = parseFetch(raw);
+ assert.equal(mails.length, 2);
+ assert.equal(mails[0]!.subject, "Your X confirmation code is abc12def");
+ assert.equal(mails[0]!.received, parseInternalDate("18-Nov-2026 10:16:23 +0000"));
+ assert.equal(mails[0]!.received, Date.UTC(2026, 10, 18, 10, 16, 23));
+ assert.equal(mails[1]!.from, "X <info@x.com>");
+});
+
+const probe = (over: Partial<LoginProbe>): LoginProbe => ({
+ url: "https://x.com/i/jf/onboarding/web?mode=login",
+ path: "/i/jf/onboarding/web",
+ hash: "",
+ text: "",
+ inputs: [],
+ signedIn: false,
+ ...over,
+});
+
+// The steps as castle Chrome saw them on 2026-11-18.
+test("each step of X's sign-in is recognised", () => {
+ assert.equal(
+ classify(probe({ text: "See what's happening\n\nEmail or username\n\nContinue", inputs: [{ name: "username_or_email", type: "text", autocomplete: "username webauthn" }] })).kind,
+ "username",
+ );
+ assert.equal(
+ classify(probe({ hash: "#/s/login_enter_password/r-1", text: "Login\n\nUsername\nPassword", inputs: [{ name: "username", type: "text", autocomplete: "" }, { name: "password", type: "password", autocomplete: "" }] })).kind,
+ "password",
+ );
+ assert.equal(
+ classify(probe({ hash: "#/s/two_factor_code/r-2", text: "Enter your two factor code\n\nUse your authenticator app to generate the code.", inputs: [{ name: "", type: "text", autocomplete: "one-time-code" }] })).kind,
+ "totp",
+ );
+ assert.equal(
+ classify(probe({ text: "Check your email\n\nYou'll receive a code to verify here so you can sign in.", inputs: [{ name: "text", type: "text", autocomplete: "" }] })).kind,
+ "email-code",
+ );
+ assert.equal(
+ classify(probe({ text: "Enter your phone number or username\n\nThere was unusual login activity on your account.", inputs: [{ name: "text", type: "text", autocomplete: "" }] })).kind,
+ "username",
+ );
+ assert.equal(classify(probe({ url: "https://x.com/home", path: "/home", signedIn: true })).kind, "done");
+ assert.equal(classify(probe({})).kind, "loading");
+ assert.equal(classify(probe({ text: "Authenticate your account\n\nSolve this puzzle" })).kind, "unknown");
+ assert.equal(classify(probe({ text: "Wrong password!", inputs: [{ name: "password", type: "password", autocomplete: "" }] })).kind, "wrong-password");
+});
diff --git a/castle-search/test/youtube.test.ts b/castle-search/test/youtube.test.ts
new file mode 100644
index 0000000..c5530fb
--- /dev/null
+++ b/castle-search/test/youtube.test.ts
@@ -0,0 +1,72 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { handleTranscript } from "../src/http.ts";
+import { cueTexts, dedupeRolling, isoDate, paragraphs, titleSlug, videoIdOf } from "../src/youtube.ts";
+
+test("every shape of YouTube link gives the video id, share junk ignored", () => {
+ for (const url of [
+ "https://www.youtube.com/watch?v=jNQXAC9IVRw",
+ "https://youtube.com/watch?feature=share&v=jNQXAC9IVRw&t=30s",
+ "https://youtu.be/jNQXAC9IVRw?si=T-d5TUXNFjmR1Uad",
+ "https://www.youtube.com/shorts/jNQXAC9IVRw",
+ "https://www.youtube.com/live/jNQXAC9IVRw?feature=shared",
+ "https://m.youtube.com/watch?v=jNQXAC9IVRw&list=PL123",
+ "jNQXAC9IVRw",
+ ]) {
+ assert.equal(videoIdOf(url), "jNQXAC9IVRw", url);
+ }
+ assert.equal(videoIdOf("https://vimeo.com/123"), undefined);
+});
+
+test("titles become the archive's dashed names", () => {
+ assert.equal(titleSlug("How to stay calm when you know you'll be stressed | Daniel Levitin | TED"), "How-to-stay-calm-when-you-know-youll-be-stressed-Daniel-Levitin-TED");
+ assert.equal(titleSlug("Anthropic Files for IPO: Nasdaq!"), "Anthropic-Files-for-IPO-Nasdaq");
+ assert.equal(titleSlug("Café Müller"), "Cafe-Muller");
+ assert.equal(titleSlug("!!!"), "transcript");
+ assert.equal(isoDate("20051024"), "2005-10-24");
+});
+
+// The shape of YouTube's automatic captions: each cue repeats the last one's words.
+const ROLLING = `WEBVTT
+Kind: captions
+Language: en
+
+00:00:00.000 --> 00:00:02.000 align:start position:0%
+the filing is out<00:00:01.000><c> ENT</c><00:00:01.500><c> is</c>
+
+00:00:02.000 --> 00:00:02.010 align:start position:0%
+the filing is out ENT is
+
+00:00:02.010 --> 00:00:04.000 align:start position:0%
+the filing is out ENT is
+going to be [Music] big &amp; bold
+
+00:00:04.000 --> 00:00:06.000
+going to be big & bold for everyone
+`;
+
+test("rolling captions come out once, markup and noise gone", () => {
+ const cues = cueTexts(ROLLING);
+ assert.equal(cues[0], "the filing is out ENT is");
+ assert.equal(dedupeRolling(cues).join(" "), "the filing is out ENT is going to be big & bold for everyone");
+});
+
+test("words are set in paragraphs of 90", () => {
+ const words = Array.from({ length: 200 }, (_, i) => `w${i}`);
+ const text = paragraphs(words);
+ assert.equal(text.split("\n\n").length, 3);
+ assert.equal(text.split("\n\n")[0]!.split(" ").length, 90);
+});
+
+test("the transcript route checks the link and language first", async () => {
+ const signal = new AbortController().signal;
+ const never = async (): Promise<never> => {
+ throw new Error("must not run");
+ };
+ assert.equal((await handleTranscript(JSON.stringify({ url: "https://vimeo.com/1" }), never, signal)).status, 400);
+ assert.equal((await handleTranscript(JSON.stringify({ url: "https://youtu.be/jNQXAC9IVRw", language: "english!" }), never, signal)).status, 400);
+ const seen: unknown[] = [];
+ await handleTranscript(JSON.stringify({ url: "https://youtu.be/jNQXAC9IVRw", language: "pt", max_chars: 10 }), async (r) => (seen.push(r), {}) as never, signal);
+ assert.deepEqual(seen, [{ url: "https://youtu.be/jNQXAC9IVRw", language: "pt", maxChars: 1000 }]);
+});