aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-11-18 10:50:14 +0000
committerIgor Soarez <igor@soarez.org>2026-11-18 10:50:14 +0000
commit910ac7cd424f6401ea042a84603c42ba055994f8 (patch)
tree3ff40da7d14c644428c71cab1b005eca0b186d13
parent39f887e07721c5b3ddd31657e5e8e25e4486d2e5 (diff)
castle-search: sign X back in when a search finds it signed out
Username, password, the authenticator code (TOTP) and, when X asks, the code it emails, read over IMAP. At most one try per 30 minutes, the password never sent twice, and any unknown step (a captcha) fails saying a person has to sign in. Details from the file CCS_X_LOGIN_FILE names.
-rw-r--r--castle-search/README.md9
-rw-r--r--castle-search/src/cdp.ts28
-rw-r--r--castle-search/src/imap.ts148
-rw-r--r--castle-search/src/search.ts21
-rw-r--r--castle-search/src/totp.ts39
-rw-r--r--castle-search/src/x-login.ts259
-rw-r--r--castle-search/src/x-search.ts70
-rw-r--r--castle-search/test/x-login.test.ts91
8 files changed, 653 insertions, 12 deletions
diff --git a/castle-search/README.md b/castle-search/README.md
index 1b80994..1b74884 100644
--- a/castle-search/README.md
+++ b/castle-search/README.md
@@ -66,6 +66,15 @@ that comes sooner waits), and at most 30 an hour and 150 a day, beyond which a
search fails saying when the next may run. The counts live in memory, so a
restart starts them afresh.
+Signed out, it signs back in by itself when `CCS_X_LOGIN_FILE` names a JSON
+file of the account's details (`src/x-login.ts`): `username`, `password`,
+`totpSecret` (the authenticator app's base32 secret) and, optionally, `imap`
+(`host`, `user`, `password`) for the code X emails when a sign-in looks
+unusual, read from the subject over IMAP (`src/imap.ts`). Then it searches
+again, once. It tries at most once every 30 minutes, never sends the password
+twice in one try, and stops at any step it does not recognise, such as a
+captcha, saying a person has to sign in. A sign-in takes about 6 s.
+
## Engines
Four, with independent indexes and independent rate limits. That last part is
diff --git a/castle-search/src/cdp.ts b/castle-search/src/cdp.ts
index aa01b56..64c382e 100644
--- a/castle-search/src/cdp.ts
+++ b/castle-search/src/cdp.ts
@@ -242,7 +242,12 @@ export class PageSession {
this.#sessionId = sessionId;
}
- static async open(conn: BrowserConnection, signal?: AbortSignal): Promise<PageSession> {
+ /**
+ * `browserContextId`, from Target.createBrowserContext, opens the page in a
+ * context of its own, with no cookies: used to try a sign-in without
+ * touching the browser's own signed-in session.
+ */
+ static async open(conn: BrowserConnection, signal?: AbortSignal, browserContextId?: string): Promise<PageSession> {
if (signal?.aborted) throw abortError(signal);
// Deliberately NOT raced against the abort signal. raceAbort abandons the
@@ -251,7 +256,11 @@ export class PageSession {
// using, with nothing left holding its targetId to close it. It is a fast
// command already bounded by COMMAND_TIMEOUT_MS, so waiting it out costs
// little and guarantees we own whatever it created.
- const created = await conn.send("Target.createTarget", { url: "about:blank", background: true });
+ const created = await conn.send("Target.createTarget", {
+ url: "about:blank",
+ background: true,
+ ...(browserContextId ? { browserContextId } : {}),
+ });
const targetId = created["targetId"];
if (typeof targetId !== "string") {
throw new BrowserUnavailableError("Target.createTarget did not return a targetId");
@@ -331,6 +340,21 @@ export class PageSession {
}
/**
+ * Types `text` into whatever has focus, as a paste would: one input event, so
+ * a framework-controlled field (React's) sees it. Never logged.
+ */
+ async insertText(text: string, signal?: AbortSignal): Promise<void> {
+ await raceAbort(this.#conn.send("Input.insertText", { text }, this.#sessionId), signal);
+ }
+
+ /** Presses and releases Enter in whatever has focus. */
+ async pressEnter(signal?: AbortSignal): Promise<void> {
+ const key = { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 };
+ await raceAbort(this.#conn.send("Input.dispatchKeyEvent", { type: "keyDown", ...key, text: "\r" }, this.#sessionId), signal);
+ await raceAbort(this.#conn.send("Input.dispatchKeyEvent", { type: "keyUp", ...key }, this.#sessionId), signal);
+ }
+
+ /**
* Close the tab. Idempotent and never throws: it runs on error paths, where a
* second failure would mask the one worth reporting.
*/
diff --git a/castle-search/src/imap.ts b/castle-search/src/imap.ts
new file mode 100644
index 0000000..ef02979
--- /dev/null
+++ b/castle-search/src/imap.ts
@@ -0,0 +1,148 @@
+/**
+ * Just enough IMAP to read a login code X emailed: log in over TLS, look at the
+ * inbox read-only, read the subjects of X's recent mail. No dependency, so
+ * castle-search keeps having none.
+ *
+ * X puts the code in the subject ("Your X confirmation code is abc12def"), so
+ * only headers are fetched, and nothing is marked read or changed.
+ */
+
+import { connect, type TLSSocket } from "node:tls";
+
+export type ImapAccount = { host: string; port?: number; user: string; password: string };
+
+export type MailHeader = { received: number; from: string; subject: string };
+
+/** RFC 2047 encoded words, the B and Q forms, as subjects use them. */
+export const decodeWords = (s: string): string =>
+ s
+ .replace(/\?=\s+=\?/g, "?==?")
+ .replace(/=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g, (_, charset: string, enc: string, text: string) => {
+ const bytes =
+ enc.toUpperCase() === "B"
+ ? Buffer.from(text, "base64")
+ : Buffer.from(
+ text.replace(/_/g, " ").replace(/=([0-9A-Fa-f]{2})/g, (_m, h: string) => String.fromCharCode(parseInt(h, 16))),
+ "latin1",
+ );
+ try {
+ return new TextDecoder(charset).decode(bytes);
+ } catch {
+ return bytes.toString("utf8");
+ }
+ });
+
+/** The code in one of X's subjects, or undefined when the mail is something else. */
+export const xCodeFromSubject = (subject: string): string | undefined => {
+ const m =
+ /\bcode is:?\s*([A-Za-z0-9]{5,10})\b/i.exec(subject) ??
+ /^\s*([A-Za-z0-9]{5,10}) is your X (?:verification|confirmation|login) code\b/i.exec(subject);
+ return m?.[1];
+};
+
+/** "18-Nov-2026 10:16:23 +0000" -> epoch ms. */
+export const parseInternalDate = (s: string): number => Date.parse(s.replace(/^(\d{1,2})-(\w{3})-(\d{4})/, "$1 $2 $3"));
+
+/** Splits a FETCH response into one header set per message. */
+export const parseFetch = (raw: string): MailHeader[] => {
+ const out: MailHeader[] = [];
+ for (const part of raw.split(/\r\n(?=\* \d+ FETCH )/)) {
+ const date = /INTERNALDATE "([^"]+)"/.exec(part)?.[1];
+ if (!date) continue;
+ const unfolded = part.replace(/\r\n[ \t]+/g, " ");
+ const header = (name: string) => new RegExp(`^${name}:\\s*(.*)$`, "im").exec(unfolded)?.[1]?.trim() ?? "";
+ out.push({ received: parseInternalDate(date), from: decodeWords(header("From")), subject: decodeWords(header("Subject")) });
+ }
+ return out;
+};
+
+const quote = (s: string): string => `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
+
+const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+const imapDate = (ms: number): string => {
+ const d = new Date(ms);
+ return `${d.getUTCDate()}-${MONTHS[d.getUTCMonth()]}-${d.getUTCFullYear()}`;
+};
+
+class Session {
+ readonly #socket: TLSSocket;
+ #buffer = "";
+ #tag = 0;
+ #waiter: (() => void) | undefined;
+ #error: Error | undefined;
+
+ constructor(socket: TLSSocket) {
+ this.#socket = socket;
+ socket.setEncoding("latin1");
+ socket.on("data", (chunk: string) => {
+ this.#buffer += chunk;
+ this.#waiter?.();
+ });
+ const fail = (e: Error) => {
+ this.#error ??= e;
+ this.#waiter?.();
+ };
+ socket.on("error", fail);
+ socket.on("close", () => fail(new Error("the IMAP server closed the connection")));
+ }
+
+ async #until(done: (buffer: string) => boolean, signal: AbortSignal): Promise<string> {
+ while (!done(this.#buffer)) {
+ if (this.#error) throw this.#error;
+ if (signal.aborted) throw signal.reason;
+ await new Promise<void>((resolve) => {
+ this.#waiter = resolve;
+ signal.addEventListener("abort", () => resolve(), { once: true });
+ });
+ this.#waiter = undefined;
+ }
+ const out = this.#buffer;
+ this.#buffer = "";
+ return out;
+ }
+
+ greeting(signal: AbortSignal): Promise<string> {
+ return this.#until((b) => b.includes("\r\n"), signal);
+ }
+
+ /** Sends one command; `shown` is what an error may say was sent, so a password never is. */
+ async command(line: string, signal: AbortSignal, shown = line): Promise<string> {
+ const tag = `c${++this.#tag}`;
+ this.#socket.write(`${tag} ${line}\r\n`);
+ const reply = await this.#until((b) => new RegExp(`(^|\\r\\n)${tag} (OK|NO|BAD)[^\\r\\n]*\\r\\n`).test(b), signal);
+ const status = new RegExp(`(?:^|\\r\\n)${tag} (OK|NO|BAD)([^\\r\\n]*)`).exec(reply);
+ if (status?.[1] !== "OK") throw new Error(`IMAP ${shown.split(" ")[0]} failed: ${status?.[1]}${status?.[2]}`);
+ return reply;
+ }
+
+ close(): void {
+ this.#socket.end();
+ this.#socket.destroy();
+ }
+}
+
+/** Headers of mail from `fromDomain` received at or after `sinceMs`, newest first. */
+export const recentMail = async (
+ account: ImapAccount,
+ fromDomain: string,
+ sinceMs: number,
+ signal: AbortSignal,
+): Promise<MailHeader[]> => {
+ const socket = connect({ host: account.host, port: account.port ?? 993, servername: account.host });
+ const s = new Session(socket);
+ try {
+ await s.greeting(signal);
+ await s.command(`LOGIN ${quote(account.user)} ${quote(account.password)}`, signal, "LOGIN");
+ await s.command("EXAMINE INBOX", signal);
+ const found = await s.command(`SEARCH SINCE ${imapDate(sinceMs - 86_400_000)} FROM ${quote(fromDomain)}`, signal);
+ const ids = (/\* SEARCH([\d ]*)/.exec(found)?.[1] ?? "").trim().split(/\s+/).filter(Boolean).slice(-10);
+ if (ids.length === 0) return [];
+ const raw = await s.command(`FETCH ${ids.join(",")} (INTERNALDATE BODY.PEEK[HEADER.FIELDS (FROM SUBJECT)])`, signal);
+ await s.command("LOGOUT", signal).catch(() => "");
+ return parseFetch(raw)
+ .filter((m) => m.received >= sinceMs)
+ .sort((a, b) => b.received - a.received);
+ } finally {
+ s.close();
+ }
+};
diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts
index 0178349..1ddd523 100644
--- a/castle-search/src/search.ts
+++ b/castle-search/src/search.ts
@@ -402,16 +402,31 @@ const isGoogleHost = (url: string): boolean => /^https?:\/\/([a-z0-9-]+\.)*googl
export const withPage = async <T>(
signal: AbortSignal,
fn: (page: PageSession, endpoint: string) => Promise<T>,
+ options: { isolated?: boolean } = {},
): Promise<T> => {
const release = await acquire(signal);
try {
const conn = await ensureConnection(signal);
const endpoint = describeCdpTarget(conn.target);
- const page = await PageSession.open(conn, signal);
+ // isolated: a browser context of its own, with none of the profile's
+ // cookies, thrown away afterwards with everything in it.
+ let context: string | undefined;
+ if (options.isolated) {
+ const created = await conn.send("Target.createBrowserContext", { disposeOnDetach: true });
+ if (typeof created["browserContextId"] !== "string") {
+ throw new BrowserUnavailableError("Target.createBrowserContext did not return a browserContextId");
+ }
+ context = created["browserContextId"];
+ }
try {
- return await fn(page, endpoint);
+ const page = await PageSession.open(conn, signal, context);
+ try {
+ return await fn(page, endpoint);
+ } finally {
+ await page.close();
+ }
} finally {
- await page.close();
+ if (context) await conn.send("Target.disposeBrowserContext", { browserContextId: context }).catch(() => {});
}
} finally {
release();
diff --git a/castle-search/src/totp.ts b/castle-search/src/totp.ts
new file mode 100644
index 0000000..c14a906
--- /dev/null
+++ b/castle-search/src/totp.ts
@@ -0,0 +1,39 @@
+/**
+ * TOTP (RFC 6238): the six-digit code an authenticator app shows, from the
+ * base32 secret behind its QR code. SHA-1, 30-second steps, as X uses.
+ */
+
+import { createHmac } from "node:crypto";
+
+const B32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+
+export const base32Decode = (input: string): Buffer => {
+ const clean = input.toUpperCase().replace(/[\s=-]/g, "");
+ let bits = 0;
+ let value = 0;
+ const out: number[] = [];
+ for (const ch of clean) {
+ const i = B32.indexOf(ch);
+ if (i < 0) throw new Error("the TOTP secret is not base32");
+ value = (value << 5) | i;
+ bits += 5;
+ if (bits >= 8) {
+ out.push((value >>> (bits - 8)) & 0xff);
+ bits -= 8;
+ }
+ }
+ return Buffer.from(out);
+};
+
+export const totp = (secret: string, nowMs: number = Date.now(), digits = 6, stepS = 30): string => {
+ const counter = Buffer.alloc(8);
+ counter.writeBigUInt64BE(BigInt(Math.floor(nowMs / 1000 / stepS)));
+ const h = createHmac("sha1", base32Decode(secret)).update(counter).digest();
+ const o = h[h.length - 1]! & 0x0f;
+ const n = (h.readUInt32BE(o) & 0x7fffffff) % 10 ** digits;
+ return String(n).padStart(digits, "0");
+};
+
+/** Seconds the code for `nowMs` stays valid. */
+export const totpRemaining = (nowMs: number = Date.now(), stepS = 30): number =>
+ stepS - (Math.floor(nowMs / 1000) % stepS);
diff --git a/castle-search/src/x-login.ts b/castle-search/src/x-login.ts
new file mode 100644
index 0000000..d4d33d1
--- /dev/null
+++ b/castle-search/src/x-login.ts
@@ -0,0 +1,259 @@
+/**
+ * Signing castle Chrome back in to X, when an X search finds it signed out.
+ *
+ * X's sign-in is a series of steps on x.com/i/jf/onboarding/web (2026-11), each
+ * named in the URL's hash: the username, `login_enter_password`,
+ * `two_factor_code` (the authenticator app's code), and sometimes a code X
+ * emails when the sign-in looks unusual. Each step is recognised by its
+ * fields and words rather than its name alone, since X renames them, and
+ * answered from the credentials file:
+ *
+ * {"username": "...", "password": "...", "totpSecret": "BASE32...",
+ * "imap": {"host": "imap.gmail.com", "user": "...", "password": "..."}}
+ *
+ * `imap` is optional: without it an emailed code fails the sign-in, saying so.
+ * The file is named by CCS_X_LOGIN_FILE; cloud-setup's macos/castle-search.sh
+ * writes it from szecret, readable by its owner only.
+ *
+ * A step it does not recognise (a captcha, "verify your phone") fails the
+ * sign-in with what the page said: that needs a person. Nothing secret is
+ * ever put in an error or a log line.
+ */
+
+import { readFileSync } from "node:fs";
+
+import type { PageSession } from "./cdp.ts";
+import { BrowserUnavailableError } from "./errors.ts";
+import { type ImapAccount, recentMail, xCodeFromSubject } from "./imap.ts";
+import { totp, totpRemaining } from "./totp.ts";
+
+export type XCredentials = { username: string; password: string; totpSecret?: string; imap?: ImapAccount };
+
+export const loadCredentials = (path: string | undefined): XCredentials | undefined => {
+ if (!path) return undefined;
+ let parsed: Partial<XCredentials>;
+ try {
+ parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<XCredentials>;
+ } catch (e) {
+ throw new BrowserUnavailableError(`cannot read the X sign-in file ${path}: ${(e as Error).message}`);
+ }
+ if (typeof parsed.username !== "string" || typeof parsed.password !== "string") {
+ throw new BrowserUnavailableError(`the X sign-in file ${path} needs "username" and "password"`);
+ }
+ return parsed as XCredentials;
+};
+
+export const LOGIN_URL = "https://x.com/i/flow/login";
+/** Whole sign-in, emailed code included: X's mail can take a minute. */
+export const X_LOGIN_TIMEOUT_MS = 150_000;
+const MAX_STEPS = 10;
+const EMAIL_WAIT_MS = 90_000;
+const EMAIL_POLL_MS = 5_000;
+
+export type LoginStep =
+ | { kind: "done" }
+ | { kind: "username" }
+ | { kind: "password" }
+ | { kind: "totp" }
+ | { kind: "email-code" }
+ | { kind: "wrong-password" }
+ | { kind: "loading" }
+ | { kind: "unknown" };
+
+export type LoginProbe = {
+ url: string;
+ path: string;
+ hash: string;
+ text: string;
+ /** Visible inputs: name, type, autocomplete. */
+ inputs: { name: string; type: string; autocomplete: string }[];
+ signedIn: boolean;
+};
+
+/**
+ * An input a person could see. X keeps a password field on its first page for
+ * autofill, laid out but at opacity 0 under aria-hidden: not a step to answer.
+ */
+const SHOWN = `((i) => i.type !== "hidden" && i.offsetParent !== null &&
+ !i.closest('[aria-hidden="true"], [inert]') && getComputedStyle(i).opacity !== "0")`;
+
+/** Reads the sign-in page as it stands. Runs in the page; self-contained. */
+export const LOGIN_PROBE = `(() => {
+ const scope = document.querySelector('[role="dialog"]') || document.querySelector("main") || document.body;
+ const inputs = [...document.querySelectorAll("input")]
+ .filter(${SHOWN})
+ .map((i) => ({ name: i.name || "", type: i.type || "", autocomplete: i.autocomplete || "" }));
+ return {
+ url: location.href,
+ path: location.pathname,
+ hash: location.hash,
+ text: (scope ? scope.innerText : "").trim().slice(0, 600),
+ inputs,
+ signedIn: !!document.querySelector('[data-testid="SideNav_AccountSwitcher_Button"], [data-testid="AppTabBar_Profile_Link"]'),
+ };
+})()`;
+
+const onSignInPage = (path: string): boolean => /^\/(login|i\/flow\/login|i\/jf\/onboarding|account\/access)/.test(path);
+
+/** Which step the page is on. Pure, so it can be tested on recorded probes. */
+export const classify = (p: LoginProbe): LoginStep => {
+ if (p.signedIn && !onSignInPage(p.path)) return { kind: "done" };
+ const has = (f: (i: LoginProbe["inputs"][number]) => boolean) => p.inputs.some(f);
+ const text = p.text;
+ if (/wrong password|incorrect password/i.test(text)) return { kind: "wrong-password" };
+ if (has((i) => i.type === "password")) return { kind: "password" };
+ const codeField = has((i) => i.autocomplete.includes("one-time-code") || i.name === "text" || i.name === "code" || i.name === "verfication_code");
+ if (codeField && /authenticat(or|ion) app|two[- ]factor/i.test(text) && !/email/i.test(text)) return { kind: "totp" };
+ if (codeField && /email/i.test(text) && /code/i.test(text)) return { kind: "email-code" };
+ if (has((i) => i.name === "username_or_email" || i.autocomplete.startsWith("username"))) return { kind: "username" };
+ // "Enter your phone number or username": X asks again when it is unsure.
+ if (has((i) => i.name === "text") && /phone number or username|username/i.test(text)) return { kind: "username" };
+ if (p.inputs.length === 0 && text === "") return { kind: "loading" };
+ return { kind: "unknown" };
+};
+
+const sleep = (ms: number, signal: AbortSignal): Promise<void> =>
+ new Promise((resolve, reject) => {
+ if (signal.aborted) return reject(signal.reason);
+ const t = setTimeout(resolve, ms);
+ signal.addEventListener("abort", () => {
+ clearTimeout(t);
+ reject(signal.reason);
+ }, { once: true });
+ });
+
+/** Focuses the step's text field: the password field on the password step, else the first one. */
+const FOCUS = (password: boolean) => `(() => {
+ const inputs = [...document.querySelectorAll("input")].filter(${SHOWN});
+ const el = ${password ? `inputs.find((i) => i.type === "password")` : `inputs.find((i) => i.type !== "password" && i.type !== "checkbox")`};
+ if (!el) return false;
+ el.focus();
+ if (el.value) { el.select(); }
+ return document.activeElement === el;
+})()`;
+
+/** False when the field is gone, the page moving between steps: look again. */
+const answer = async (page: PageSession, text: () => string, password: boolean, signal: AbortSignal): Promise<boolean> => {
+ if (!(await page.evaluate<boolean>(FOCUS(password), signal).catch(() => false))) return false;
+ await page.insertText(text(), signal);
+ await sleep(300, signal);
+ await page.pressEnter(signal);
+ return true;
+};
+
+/** What makes one step differ from the next: not its text, which changes as it is typed into. */
+const shape = (p: LoginProbe): string => `${p.path}${p.hash}|${p.inputs.map((i) => `${i.name}:${i.type}`).join(",")}`;
+
+/** Waits for X's emailed code, sent at or after `since`. */
+const emailedCode = async (imap: ImapAccount, since: number, signal: AbortSignal): Promise<string> => {
+ const deadline = Date.now() + EMAIL_WAIT_MS;
+ let seen = 0;
+ while (Date.now() < deadline) {
+ // A minute's slack for the two clocks; an older code is from an earlier sign-in.
+ const mail = await recentMail(imap, "x.com", since - 60_000, signal);
+ seen = mail.length;
+ for (const m of mail) {
+ const code = xCodeFromSubject(m.subject);
+ if (code) return code;
+ }
+ await sleep(EMAIL_POLL_MS, signal);
+ }
+ throw new BrowserUnavailableError(
+ `X sign-in: X asked for the code it emailed, and none arrived at ${imap.user} within ` +
+ `${EMAIL_WAIT_MS / 1000} s (${seen} recent mails from x.com, none with a code in the subject). ` +
+ `The account's email may not be that mailbox.`,
+ );
+};
+
+/**
+ * Signs in on `page`. Resolves when X shows the signed-in app; throws
+ * BrowserUnavailableError naming the step it could not get past.
+ */
+export const xLogin = async (
+ page: PageSession,
+ creds: XCredentials,
+ signal: AbortSignal,
+ log: (line: string) => void = () => {},
+): Promise<void> => {
+ const started = Date.now();
+ await page.navigate(LOGIN_URL, signal);
+ const answered = new Map<string, number>();
+ let last: LoginProbe | undefined;
+
+ for (let steps = 0; steps < MAX_STEPS; ) {
+ // A step's Enter navigates, and a probe caught mid-navigation fails: retry.
+ const probe = await page.evaluate<LoginProbe>(LOGIN_PROBE, signal).catch((e: unknown) => {
+ if (signal.aborted) throw e;
+ return undefined;
+ });
+ if (!probe) {
+ await sleep(400, signal);
+ continue;
+ }
+ last = probe;
+ const step = classify(probe);
+ if (step.kind === "done") {
+ log(`x-login signed in as ${creds.username}`);
+ return;
+ }
+ if (step.kind === "loading") {
+ await sleep(400, signal);
+ continue;
+ }
+ // Coming back to a step means the answer was refused: stop rather than
+ // hammer X with a wrong password or a stale code. The password is never
+ // sent twice; a code gets one more go, in case it expired on the way.
+ const key = step.kind;
+ if (step.kind === "wrong-password" || (answered.get(key) ?? 0) >= (key === "password" ? 1 : 2)) {
+ throw new BrowserUnavailableError(
+ `X sign-in refused at the ${step.kind} step (${probe.url}): "${probe.text.slice(0, 200)}". ` +
+ `Tell the user: someone has to sign in to X in castle Chrome by hand.`,
+ );
+ }
+ if (step.kind === "unknown") {
+ throw new BrowserUnavailableError(
+ `X sign-in stopped at a step this tool does not handle (${probe.url}): "${probe.text.slice(0, 300)}". ` +
+ `Tell the user: someone has to sign in to X in castle Chrome by hand.`,
+ );
+ }
+
+ let code: (() => string) | undefined;
+ if (step.kind === "username") code = () => creds.username;
+ else if (step.kind === "password") code = () => creds.password;
+ else if (step.kind === "totp") {
+ const secret = creds.totpSecret;
+ if (!secret) throw new BrowserUnavailableError("X sign-in wants an authenticator code and the sign-in file has no totpSecret");
+ // A code about to expire may be stale by the time X checks it.
+ if (totpRemaining() < 5) await sleep(totpRemaining() * 1000 + 500, signal);
+ code = () => totp(secret);
+ } else if (step.kind === "email-code") {
+ if (!creds.imap) {
+ throw new BrowserUnavailableError(
+ `X sign-in wants a code it emailed, and the sign-in file has no mailbox to read it from ` +
+ `(${probe.url}). Tell the user: someone has to sign in to X in castle Chrome by hand.`,
+ );
+ }
+ const emailed = await emailedCode(creds.imap, started, signal);
+ code = () => emailed;
+ }
+ if (!code || !(await answer(page, code, step.kind === "password", signal))) {
+ await sleep(400, signal);
+ continue;
+ }
+ log(`x-login answered ${step.kind} at ${probe.hash || probe.path}`);
+ answered.set(key, (answered.get(key) ?? 0) + 1);
+ steps++;
+
+ // Wait for the next step, up to ~10 s; a refused answer stays on this one.
+ const before = shape(probe);
+ for (let w = 0; w < 25; w++) {
+ await sleep(400, signal);
+ const now = await page.evaluate<LoginProbe>(LOGIN_PROBE, signal).catch(() => undefined);
+ if (!now || shape(now) !== before || classify(now).kind !== step.kind) break;
+ }
+ }
+ throw new BrowserUnavailableError(
+ `X sign-in did not finish within ${MAX_STEPS} steps (last at ${last?.url}). Tell the user: someone has ` +
+ `to sign in to X in castle Chrome by hand.`,
+ );
+};
diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts
index 38dc053..7c8badf 100644
--- a/castle-search/src/x-search.ts
+++ b/castle-search/src/x-search.ts
@@ -19,10 +19,29 @@
import { BrowserUnavailableError, NoResultsError } from "./errors.ts";
import { withPage } from "./search.ts";
import { XLimiter } from "./x-limits.ts";
+import { loadCredentials, X_LOGIN_TIMEOUT_MS, xLogin } from "./x-login.ts";
/** Shared by every X search in this process: see x-limits.ts. */
const limiter = new XLimiter();
+/**
+ * Signed out, a search signs the browser back in (x-login.ts) when
+ * CCS_X_LOGIN_FILE names the credentials, then searches again, once. At most
+ * one sign-in attempt per LOGIN_COOLDOWN_MS: a sign-in that keeps failing
+ * is what gets an account locked, so until the cooldown has passed every
+ * search fails with the reason the last attempt gave.
+ */
+export const LOGIN_COOLDOWN_MS = 30 * 60_000;
+let lastLogin: { at: number; error?: string } | undefined;
+
+class SignedOut extends Error {
+ readonly at: string;
+ constructor(at: string) {
+ super(`signed out at ${at}`);
+ this.at = at;
+ }
+}
+
export type XSort = "latest" | "top";
export type XPost = {
@@ -160,6 +179,17 @@ export const xSearch = (request: XSearchRequest, outer: AbortSignal): Promise<XS
limiter.run(outer, () => searchNow(request, outer));
const searchNow = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => {
+ try {
+ return await attempt(request, outer);
+ } catch (e) {
+ if (!(e instanceof SignedOut)) throw e;
+ // The sign-in has its own deadline, and the search after it a fresh one.
+ await signIn(e.at, outer);
+ return await attempt(request, outer);
+ }
+};
+
+const attempt = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => {
const sort: XSort = request.sort ?? "latest";
const searchUrl = buildXSearchUrl(request.query, sort);
const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]);
@@ -178,6 +208,37 @@ const searchNow = async (request: XSearchRequest, outer: AbortSignal): Promise<X
}
};
+const signedOutError = (at: string, why: string): BrowserUnavailableError =>
+ new BrowserUnavailableError(
+ `X search needs a signed-in X session in castle Chrome, and it is signed out (X sent it to ${at}). ` +
+ `${why} Tell the user: someone has to sign in to X in that browser again.`,
+ );
+
+/** Signs the browser in, within its own deadline, or throws saying why not. */
+const signIn = async (at: string, outer: AbortSignal): Promise<void> => {
+ const creds = loadCredentials(process.env["CCS_X_LOGIN_FILE"]);
+ if (!creds) throw signedOutError(at, "No sign-in details are configured (CCS_X_LOGIN_FILE).");
+ if (lastLogin && Date.now() - lastLogin.at < LOGIN_COOLDOWN_MS) {
+ const next = new Date(lastLogin.at + LOGIN_COOLDOWN_MS).toISOString().slice(11, 16);
+ throw signedOutError(
+ at,
+ `Signing in again was tried at ${new Date(lastLogin.at).toISOString().slice(11, 16)} UTC` +
+ (lastLogin.error ? ` and failed: ${lastLogin.error}` : ", yet it is signed out again.") +
+ ` The next try is after ${next} UTC.`,
+ );
+ }
+ lastLogin = { at: Date.now() };
+ const signal = AbortSignal.any([outer, AbortSignal.timeout(X_LOGIN_TIMEOUT_MS)]);
+ try {
+ await withPage(signal, (page) => xLogin(page, creds, signal, (line) => console.log(line)));
+ } catch (e) {
+ const why = signal.aborted && !outer.aborted ? `it took over ${X_LOGIN_TIMEOUT_MS / 1000} s` : (e as Error).message;
+ lastLogin.error = why;
+ console.log(`x-login failed: ${why}`);
+ throw signedOutError(at, `Signing in again failed: ${why}`);
+ }
+};
+
const run = (
request: XSearchRequest,
sort: XSort,
@@ -185,7 +246,7 @@ const run = (
signal: AbortSignal,
saw: (state: string) => void,
): Promise<XSearchOutcome> =>
- withPage(signal, async (page, endpoint) => {
+ withPage(signal, async (page) => {
await page.navigate(searchUrl, signal);
// X renders after the load event: poll until posts, an empty or error
@@ -198,12 +259,7 @@ const run = (
await sleep(POLL_MS, signal);
}
- if (probe.state === "signed-out") {
- throw new BrowserUnavailableError(
- `X search needs a signed-in X session in the browser (${endpoint}), and it is signed out ` +
- `(X sent it to ${probe.url}). Tell the user: someone has to sign in to X in that browser again.`,
- );
- }
+ if (probe.state === "signed-out") throw new SignedOut(probe.url);
if (probe.state === "error") {
throw new BrowserUnavailableError(
`X answered with an error page instead of results (${probe.url}): "${probe.note}". This is ` +
diff --git a/castle-search/test/x-login.test.ts b/castle-search/test/x-login.test.ts
new file mode 100644
index 0000000..11468d5
--- /dev/null
+++ b/castle-search/test/x-login.test.ts
@@ -0,0 +1,91 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { decodeWords, parseFetch, parseInternalDate, xCodeFromSubject } from "../src/imap.ts";
+import { classify, type LoginProbe } from "../src/x-login.ts";
+import { base32Decode, totp, totpRemaining } from "../src/totp.ts";
+
+// RFC 6238 appendix B, SHA-1: the secret "12345678901234567890".
+const RFC_SECRET = "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ";
+
+test("totp matches RFC 6238's SHA-1 vectors", () => {
+ assert.equal(totp(RFC_SECRET, 59_000, 8), "94287082");
+ assert.equal(totp(RFC_SECRET, 1_111_111_109_000, 8), "07081804");
+ assert.equal(totp(RFC_SECRET, 1_234_567_890_000, 8), "89005924");
+ assert.equal(totp(RFC_SECRET, 59_000), "287082");
+});
+
+test("base32 ignores spaces and case, and refuses other characters", () => {
+ assert.deepEqual(base32Decode("gezd gnbv"), base32Decode("GEZDGNBV"));
+ assert.throws(() => base32Decode("GEZD1"));
+});
+
+test("totpRemaining counts down within the step", () => {
+ assert.equal(totpRemaining(0), 30);
+ assert.equal(totpRemaining(29_000), 1);
+});
+
+test("X's code is read from the subjects X uses, and nothing else", () => {
+ assert.equal(xCodeFromSubject("Your X confirmation code is abc12def"), "abc12def");
+ assert.equal(xCodeFromSubject("Your X verification code is: 123456"), "123456");
+ assert.equal(xCodeFromSubject("123456 is your X verification code"), "123456");
+ assert.equal(xCodeFromSubject("New login to X from Chrome on Mac"), undefined);
+ assert.equal(xCodeFromSubject("Security alert"), undefined);
+});
+
+test("encoded subjects are decoded", () => {
+ assert.equal(decodeWords("=?UTF-8?B?WW91ciBYIGNvbmZpcm1hdGlvbiBjb2RlIGlzIGFiYzEyZGVm?="), "Your X confirmation code is abc12def");
+ assert.equal(decodeWords("=?UTF-8?Q?Caf=C3=A9_menu?="), "Café menu");
+});
+
+test("a FETCH reply splits into one header set per message", () => {
+ const raw =
+ '* 1 FETCH (INTERNALDATE "18-Nov-2026 10:16:23 +0000" BODY[HEADER.FIELDS (FROM SUBJECT)] {60}\r\n' +
+ "From: X <verify@x.com>\r\nSubject: Your X confirmation code is\r\n abc12def\r\n\r\n)\r\n" +
+ '* 2 FETCH (INTERNALDATE "18-Nov-2026 10:20:00 +0000" BODY[HEADER.FIELDS (FROM SUBJECT)] {40}\r\n' +
+ "From: X <info@x.com>\r\nSubject: New login\r\n\r\n)\r\nc4 OK Success\r\n";
+ const mails = parseFetch(raw);
+ assert.equal(mails.length, 2);
+ assert.equal(mails[0]!.subject, "Your X confirmation code is abc12def");
+ assert.equal(mails[0]!.received, parseInternalDate("18-Nov-2026 10:16:23 +0000"));
+ assert.equal(mails[0]!.received, Date.UTC(2026, 10, 18, 10, 16, 23));
+ assert.equal(mails[1]!.from, "X <info@x.com>");
+});
+
+const probe = (over: Partial<LoginProbe>): LoginProbe => ({
+ url: "https://x.com/i/jf/onboarding/web?mode=login",
+ path: "/i/jf/onboarding/web",
+ hash: "",
+ text: "",
+ inputs: [],
+ signedIn: false,
+ ...over,
+});
+
+// The steps as castle Chrome saw them on 2026-11-18.
+test("each step of X's sign-in is recognised", () => {
+ assert.equal(
+ classify(probe({ text: "See what's happening\n\nEmail or username\n\nContinue", inputs: [{ name: "username_or_email", type: "text", autocomplete: "username webauthn" }] })).kind,
+ "username",
+ );
+ assert.equal(
+ classify(probe({ hash: "#/s/login_enter_password/r-1", text: "Login\n\nUsername\nPassword", inputs: [{ name: "username", type: "text", autocomplete: "" }, { name: "password", type: "password", autocomplete: "" }] })).kind,
+ "password",
+ );
+ assert.equal(
+ classify(probe({ hash: "#/s/two_factor_code/r-2", text: "Enter your two factor code\n\nUse your authenticator app to generate the code.", inputs: [{ name: "", type: "text", autocomplete: "one-time-code" }] })).kind,
+ "totp",
+ );
+ assert.equal(
+ classify(probe({ text: "Check your email\n\nYou'll receive a code to verify here so you can sign in.", inputs: [{ name: "text", type: "text", autocomplete: "" }] })).kind,
+ "email-code",
+ );
+ assert.equal(
+ classify(probe({ text: "Enter your phone number or username\n\nThere was unusual login activity on your account.", inputs: [{ name: "text", type: "text", autocomplete: "" }] })).kind,
+ "username",
+ );
+ assert.equal(classify(probe({ url: "https://x.com/home", path: "/home", signedIn: true })).kind, "done");
+ assert.equal(classify(probe({})).kind, "loading");
+ assert.equal(classify(probe({ text: "Authenticate your account\n\nSolve this puzzle" })).kind, "unknown");
+ assert.equal(classify(probe({ text: "Wrong password!", inputs: [{ name: "password", type: "password", autocomplete: "" }] })).kind, "wrong-password");
+});