diff options
| author | Igor Soarez <igor@soarez.org> | 2026-11-18 10:50:14 +0000 |
|---|---|---|
| committer | Igor Soarez <igor@soarez.org> | 2026-11-18 10:50:14 +0000 |
| commit | 910ac7cd424f6401ea042a84603c42ba055994f8 (patch) | |
| tree | 3ff40da7d14c644428c71cab1b005eca0b186d13 | |
| parent | 39f887e07721c5b3ddd31657e5e8e25e4486d2e5 (diff) | |
castle-search: sign X back in when a search finds it signed out
Username, password, the authenticator code (TOTP) and, when X asks, the
code it emails, read over IMAP. At most one try per 30 minutes, the
password never sent twice, and any unknown step (a captcha) fails saying
a person has to sign in. Details from the file CCS_X_LOGIN_FILE names.
| -rw-r--r-- | castle-search/README.md | 9 | ||||
| -rw-r--r-- | castle-search/src/cdp.ts | 28 | ||||
| -rw-r--r-- | castle-search/src/imap.ts | 148 | ||||
| -rw-r--r-- | castle-search/src/search.ts | 21 | ||||
| -rw-r--r-- | castle-search/src/totp.ts | 39 | ||||
| -rw-r--r-- | castle-search/src/x-login.ts | 259 | ||||
| -rw-r--r-- | castle-search/src/x-search.ts | 70 | ||||
| -rw-r--r-- | castle-search/test/x-login.test.ts | 91 |
8 files changed, 653 insertions, 12 deletions
diff --git a/castle-search/README.md b/castle-search/README.md index 1b80994..1b74884 100644 --- a/castle-search/README.md +++ b/castle-search/README.md @@ -66,6 +66,15 @@ that comes sooner waits), and at most 30 an hour and 150 a day, beyond which a search fails saying when the next may run. The counts live in memory, so a restart starts them afresh. +Signed out, it signs back in by itself when `CCS_X_LOGIN_FILE` names a JSON +file of the account's details (`src/x-login.ts`): `username`, `password`, +`totpSecret` (the authenticator app's base32 secret) and, optionally, `imap` +(`host`, `user`, `password`) for the code X emails when a sign-in looks +unusual, read from the subject over IMAP (`src/imap.ts`). Then it searches +again, once. It tries at most once every 30 minutes, never sends the password +twice in one try, and stops at any step it does not recognise, such as a +captcha, saying a person has to sign in. A sign-in takes about 6 s. + ## Engines Four, with independent indexes and independent rate limits. That last part is diff --git a/castle-search/src/cdp.ts b/castle-search/src/cdp.ts index aa01b56..64c382e 100644 --- a/castle-search/src/cdp.ts +++ b/castle-search/src/cdp.ts @@ -242,7 +242,12 @@ export class PageSession { this.#sessionId = sessionId; } - static async open(conn: BrowserConnection, signal?: AbortSignal): Promise<PageSession> { + /** + * `browserContextId`, from Target.createBrowserContext, opens the page in a + * context of its own, with no cookies: used to try a sign-in without + * touching the browser's own signed-in session. + */ + static async open(conn: BrowserConnection, signal?: AbortSignal, browserContextId?: string): Promise<PageSession> { if (signal?.aborted) throw abortError(signal); // Deliberately NOT raced against the abort signal. raceAbort abandons the @@ -251,7 +256,11 @@ export class PageSession { // using, with nothing left holding its targetId to close it. It is a fast // command already bounded by COMMAND_TIMEOUT_MS, so waiting it out costs // little and guarantees we own whatever it created. - const created = await conn.send("Target.createTarget", { url: "about:blank", background: true }); + const created = await conn.send("Target.createTarget", { + url: "about:blank", + background: true, + ...(browserContextId ? { browserContextId } : {}), + }); const targetId = created["targetId"]; if (typeof targetId !== "string") { throw new BrowserUnavailableError("Target.createTarget did not return a targetId"); @@ -331,6 +340,21 @@ export class PageSession { } /** + * Types `text` into whatever has focus, as a paste would: one input event, so + * a framework-controlled field (React's) sees it. Never logged. + */ + async insertText(text: string, signal?: AbortSignal): Promise<void> { + await raceAbort(this.#conn.send("Input.insertText", { text }, this.#sessionId), signal); + } + + /** Presses and releases Enter in whatever has focus. */ + async pressEnter(signal?: AbortSignal): Promise<void> { + const key = { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13 }; + await raceAbort(this.#conn.send("Input.dispatchKeyEvent", { type: "keyDown", ...key, text: "\r" }, this.#sessionId), signal); + await raceAbort(this.#conn.send("Input.dispatchKeyEvent", { type: "keyUp", ...key }, this.#sessionId), signal); + } + + /** * Close the tab. Idempotent and never throws: it runs on error paths, where a * second failure would mask the one worth reporting. */ diff --git a/castle-search/src/imap.ts b/castle-search/src/imap.ts new file mode 100644 index 0000000..ef02979 --- /dev/null +++ b/castle-search/src/imap.ts @@ -0,0 +1,148 @@ +/** + * Just enough IMAP to read a login code X emailed: log in over TLS, look at the + * inbox read-only, read the subjects of X's recent mail. No dependency, so + * castle-search keeps having none. + * + * X puts the code in the subject ("Your X confirmation code is abc12def"), so + * only headers are fetched, and nothing is marked read or changed. + */ + +import { connect, type TLSSocket } from "node:tls"; + +export type ImapAccount = { host: string; port?: number; user: string; password: string }; + +export type MailHeader = { received: number; from: string; subject: string }; + +/** RFC 2047 encoded words, the B and Q forms, as subjects use them. */ +export const decodeWords = (s: string): string => + s + .replace(/\?=\s+=\?/g, "?==?") + .replace(/=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g, (_, charset: string, enc: string, text: string) => { + const bytes = + enc.toUpperCase() === "B" + ? Buffer.from(text, "base64") + : Buffer.from( + text.replace(/_/g, " ").replace(/=([0-9A-Fa-f]{2})/g, (_m, h: string) => String.fromCharCode(parseInt(h, 16))), + "latin1", + ); + try { + return new TextDecoder(charset).decode(bytes); + } catch { + return bytes.toString("utf8"); + } + }); + +/** The code in one of X's subjects, or undefined when the mail is something else. */ +export const xCodeFromSubject = (subject: string): string | undefined => { + const m = + /\bcode is:?\s*([A-Za-z0-9]{5,10})\b/i.exec(subject) ?? + /^\s*([A-Za-z0-9]{5,10}) is your X (?:verification|confirmation|login) code\b/i.exec(subject); + return m?.[1]; +}; + +/** "18-Nov-2026 10:16:23 +0000" -> epoch ms. */ +export const parseInternalDate = (s: string): number => Date.parse(s.replace(/^(\d{1,2})-(\w{3})-(\d{4})/, "$1 $2 $3")); + +/** Splits a FETCH response into one header set per message. */ +export const parseFetch = (raw: string): MailHeader[] => { + const out: MailHeader[] = []; + for (const part of raw.split(/\r\n(?=\* \d+ FETCH )/)) { + const date = /INTERNALDATE "([^"]+)"/.exec(part)?.[1]; + if (!date) continue; + const unfolded = part.replace(/\r\n[ \t]+/g, " "); + const header = (name: string) => new RegExp(`^${name}:\\s*(.*)$`, "im").exec(unfolded)?.[1]?.trim() ?? ""; + out.push({ received: parseInternalDate(date), from: decodeWords(header("From")), subject: decodeWords(header("Subject")) }); + } + return out; +}; + +const quote = (s: string): string => `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; + +const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; +const imapDate = (ms: number): string => { + const d = new Date(ms); + return `${d.getUTCDate()}-${MONTHS[d.getUTCMonth()]}-${d.getUTCFullYear()}`; +}; + +class Session { + readonly #socket: TLSSocket; + #buffer = ""; + #tag = 0; + #waiter: (() => void) | undefined; + #error: Error | undefined; + + constructor(socket: TLSSocket) { + this.#socket = socket; + socket.setEncoding("latin1"); + socket.on("data", (chunk: string) => { + this.#buffer += chunk; + this.#waiter?.(); + }); + const fail = (e: Error) => { + this.#error ??= e; + this.#waiter?.(); + }; + socket.on("error", fail); + socket.on("close", () => fail(new Error("the IMAP server closed the connection"))); + } + + async #until(done: (buffer: string) => boolean, signal: AbortSignal): Promise<string> { + while (!done(this.#buffer)) { + if (this.#error) throw this.#error; + if (signal.aborted) throw signal.reason; + await new Promise<void>((resolve) => { + this.#waiter = resolve; + signal.addEventListener("abort", () => resolve(), { once: true }); + }); + this.#waiter = undefined; + } + const out = this.#buffer; + this.#buffer = ""; + return out; + } + + greeting(signal: AbortSignal): Promise<string> { + return this.#until((b) => b.includes("\r\n"), signal); + } + + /** Sends one command; `shown` is what an error may say was sent, so a password never is. */ + async command(line: string, signal: AbortSignal, shown = line): Promise<string> { + const tag = `c${++this.#tag}`; + this.#socket.write(`${tag} ${line}\r\n`); + const reply = await this.#until((b) => new RegExp(`(^|\\r\\n)${tag} (OK|NO|BAD)[^\\r\\n]*\\r\\n`).test(b), signal); + const status = new RegExp(`(?:^|\\r\\n)${tag} (OK|NO|BAD)([^\\r\\n]*)`).exec(reply); + if (status?.[1] !== "OK") throw new Error(`IMAP ${shown.split(" ")[0]} failed: ${status?.[1]}${status?.[2]}`); + return reply; + } + + close(): void { + this.#socket.end(); + this.#socket.destroy(); + } +} + +/** Headers of mail from `fromDomain` received at or after `sinceMs`, newest first. */ +export const recentMail = async ( + account: ImapAccount, + fromDomain: string, + sinceMs: number, + signal: AbortSignal, +): Promise<MailHeader[]> => { + const socket = connect({ host: account.host, port: account.port ?? 993, servername: account.host }); + const s = new Session(socket); + try { + await s.greeting(signal); + await s.command(`LOGIN ${quote(account.user)} ${quote(account.password)}`, signal, "LOGIN"); + await s.command("EXAMINE INBOX", signal); + const found = await s.command(`SEARCH SINCE ${imapDate(sinceMs - 86_400_000)} FROM ${quote(fromDomain)}`, signal); + const ids = (/\* SEARCH([\d ]*)/.exec(found)?.[1] ?? "").trim().split(/\s+/).filter(Boolean).slice(-10); + if (ids.length === 0) return []; + const raw = await s.command(`FETCH ${ids.join(",")} (INTERNALDATE BODY.PEEK[HEADER.FIELDS (FROM SUBJECT)])`, signal); + await s.command("LOGOUT", signal).catch(() => ""); + return parseFetch(raw) + .filter((m) => m.received >= sinceMs) + .sort((a, b) => b.received - a.received); + } finally { + s.close(); + } +}; diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts index 0178349..1ddd523 100644 --- a/castle-search/src/search.ts +++ b/castle-search/src/search.ts @@ -402,16 +402,31 @@ const isGoogleHost = (url: string): boolean => /^https?:\/\/([a-z0-9-]+\.)*googl export const withPage = async <T>( signal: AbortSignal, fn: (page: PageSession, endpoint: string) => Promise<T>, + options: { isolated?: boolean } = {}, ): Promise<T> => { const release = await acquire(signal); try { const conn = await ensureConnection(signal); const endpoint = describeCdpTarget(conn.target); - const page = await PageSession.open(conn, signal); + // isolated: a browser context of its own, with none of the profile's + // cookies, thrown away afterwards with everything in it. + let context: string | undefined; + if (options.isolated) { + const created = await conn.send("Target.createBrowserContext", { disposeOnDetach: true }); + if (typeof created["browserContextId"] !== "string") { + throw new BrowserUnavailableError("Target.createBrowserContext did not return a browserContextId"); + } + context = created["browserContextId"]; + } try { - return await fn(page, endpoint); + const page = await PageSession.open(conn, signal, context); + try { + return await fn(page, endpoint); + } finally { + await page.close(); + } } finally { - await page.close(); + if (context) await conn.send("Target.disposeBrowserContext", { browserContextId: context }).catch(() => {}); } } finally { release(); diff --git a/castle-search/src/totp.ts b/castle-search/src/totp.ts new file mode 100644 index 0000000..c14a906 --- /dev/null +++ b/castle-search/src/totp.ts @@ -0,0 +1,39 @@ +/** + * TOTP (RFC 6238): the six-digit code an authenticator app shows, from the + * base32 secret behind its QR code. SHA-1, 30-second steps, as X uses. + */ + +import { createHmac } from "node:crypto"; + +const B32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + +export const base32Decode = (input: string): Buffer => { + const clean = input.toUpperCase().replace(/[\s=-]/g, ""); + let bits = 0; + let value = 0; + const out: number[] = []; + for (const ch of clean) { + const i = B32.indexOf(ch); + if (i < 0) throw new Error("the TOTP secret is not base32"); + value = (value << 5) | i; + bits += 5; + if (bits >= 8) { + out.push((value >>> (bits - 8)) & 0xff); + bits -= 8; + } + } + return Buffer.from(out); +}; + +export const totp = (secret: string, nowMs: number = Date.now(), digits = 6, stepS = 30): string => { + const counter = Buffer.alloc(8); + counter.writeBigUInt64BE(BigInt(Math.floor(nowMs / 1000 / stepS))); + const h = createHmac("sha1", base32Decode(secret)).update(counter).digest(); + const o = h[h.length - 1]! & 0x0f; + const n = (h.readUInt32BE(o) & 0x7fffffff) % 10 ** digits; + return String(n).padStart(digits, "0"); +}; + +/** Seconds the code for `nowMs` stays valid. */ +export const totpRemaining = (nowMs: number = Date.now(), stepS = 30): number => + stepS - (Math.floor(nowMs / 1000) % stepS); diff --git a/castle-search/src/x-login.ts b/castle-search/src/x-login.ts new file mode 100644 index 0000000..d4d33d1 --- /dev/null +++ b/castle-search/src/x-login.ts @@ -0,0 +1,259 @@ +/** + * Signing castle Chrome back in to X, when an X search finds it signed out. + * + * X's sign-in is a series of steps on x.com/i/jf/onboarding/web (2026-11), each + * named in the URL's hash: the username, `login_enter_password`, + * `two_factor_code` (the authenticator app's code), and sometimes a code X + * emails when the sign-in looks unusual. Each step is recognised by its + * fields and words rather than its name alone, since X renames them, and + * answered from the credentials file: + * + * {"username": "...", "password": "...", "totpSecret": "BASE32...", + * "imap": {"host": "imap.gmail.com", "user": "...", "password": "..."}} + * + * `imap` is optional: without it an emailed code fails the sign-in, saying so. + * The file is named by CCS_X_LOGIN_FILE; cloud-setup's macos/castle-search.sh + * writes it from szecret, readable by its owner only. + * + * A step it does not recognise (a captcha, "verify your phone") fails the + * sign-in with what the page said: that needs a person. Nothing secret is + * ever put in an error or a log line. + */ + +import { readFileSync } from "node:fs"; + +import type { PageSession } from "./cdp.ts"; +import { BrowserUnavailableError } from "./errors.ts"; +import { type ImapAccount, recentMail, xCodeFromSubject } from "./imap.ts"; +import { totp, totpRemaining } from "./totp.ts"; + +export type XCredentials = { username: string; password: string; totpSecret?: string; imap?: ImapAccount }; + +export const loadCredentials = (path: string | undefined): XCredentials | undefined => { + if (!path) return undefined; + let parsed: Partial<XCredentials>; + try { + parsed = JSON.parse(readFileSync(path, "utf8")) as Partial<XCredentials>; + } catch (e) { + throw new BrowserUnavailableError(`cannot read the X sign-in file ${path}: ${(e as Error).message}`); + } + if (typeof parsed.username !== "string" || typeof parsed.password !== "string") { + throw new BrowserUnavailableError(`the X sign-in file ${path} needs "username" and "password"`); + } + return parsed as XCredentials; +}; + +export const LOGIN_URL = "https://x.com/i/flow/login"; +/** Whole sign-in, emailed code included: X's mail can take a minute. */ +export const X_LOGIN_TIMEOUT_MS = 150_000; +const MAX_STEPS = 10; +const EMAIL_WAIT_MS = 90_000; +const EMAIL_POLL_MS = 5_000; + +export type LoginStep = + | { kind: "done" } + | { kind: "username" } + | { kind: "password" } + | { kind: "totp" } + | { kind: "email-code" } + | { kind: "wrong-password" } + | { kind: "loading" } + | { kind: "unknown" }; + +export type LoginProbe = { + url: string; + path: string; + hash: string; + text: string; + /** Visible inputs: name, type, autocomplete. */ + inputs: { name: string; type: string; autocomplete: string }[]; + signedIn: boolean; +}; + +/** + * An input a person could see. X keeps a password field on its first page for + * autofill, laid out but at opacity 0 under aria-hidden: not a step to answer. + */ +const SHOWN = `((i) => i.type !== "hidden" && i.offsetParent !== null && + !i.closest('[aria-hidden="true"], [inert]') && getComputedStyle(i).opacity !== "0")`; + +/** Reads the sign-in page as it stands. Runs in the page; self-contained. */ +export const LOGIN_PROBE = `(() => { + const scope = document.querySelector('[role="dialog"]') || document.querySelector("main") || document.body; + const inputs = [...document.querySelectorAll("input")] + .filter(${SHOWN}) + .map((i) => ({ name: i.name || "", type: i.type || "", autocomplete: i.autocomplete || "" })); + return { + url: location.href, + path: location.pathname, + hash: location.hash, + text: (scope ? scope.innerText : "").trim().slice(0, 600), + inputs, + signedIn: !!document.querySelector('[data-testid="SideNav_AccountSwitcher_Button"], [data-testid="AppTabBar_Profile_Link"]'), + }; +})()`; + +const onSignInPage = (path: string): boolean => /^\/(login|i\/flow\/login|i\/jf\/onboarding|account\/access)/.test(path); + +/** Which step the page is on. Pure, so it can be tested on recorded probes. */ +export const classify = (p: LoginProbe): LoginStep => { + if (p.signedIn && !onSignInPage(p.path)) return { kind: "done" }; + const has = (f: (i: LoginProbe["inputs"][number]) => boolean) => p.inputs.some(f); + const text = p.text; + if (/wrong password|incorrect password/i.test(text)) return { kind: "wrong-password" }; + if (has((i) => i.type === "password")) return { kind: "password" }; + const codeField = has((i) => i.autocomplete.includes("one-time-code") || i.name === "text" || i.name === "code" || i.name === "verfication_code"); + if (codeField && /authenticat(or|ion) app|two[- ]factor/i.test(text) && !/email/i.test(text)) return { kind: "totp" }; + if (codeField && /email/i.test(text) && /code/i.test(text)) return { kind: "email-code" }; + if (has((i) => i.name === "username_or_email" || i.autocomplete.startsWith("username"))) return { kind: "username" }; + // "Enter your phone number or username": X asks again when it is unsure. + if (has((i) => i.name === "text") && /phone number or username|username/i.test(text)) return { kind: "username" }; + if (p.inputs.length === 0 && text === "") return { kind: "loading" }; + return { kind: "unknown" }; +}; + +const sleep = (ms: number, signal: AbortSignal): Promise<void> => + new Promise((resolve, reject) => { + if (signal.aborted) return reject(signal.reason); + const t = setTimeout(resolve, ms); + signal.addEventListener("abort", () => { + clearTimeout(t); + reject(signal.reason); + }, { once: true }); + }); + +/** Focuses the step's text field: the password field on the password step, else the first one. */ +const FOCUS = (password: boolean) => `(() => { + const inputs = [...document.querySelectorAll("input")].filter(${SHOWN}); + const el = ${password ? `inputs.find((i) => i.type === "password")` : `inputs.find((i) => i.type !== "password" && i.type !== "checkbox")`}; + if (!el) return false; + el.focus(); + if (el.value) { el.select(); } + return document.activeElement === el; +})()`; + +/** False when the field is gone, the page moving between steps: look again. */ +const answer = async (page: PageSession, text: () => string, password: boolean, signal: AbortSignal): Promise<boolean> => { + if (!(await page.evaluate<boolean>(FOCUS(password), signal).catch(() => false))) return false; + await page.insertText(text(), signal); + await sleep(300, signal); + await page.pressEnter(signal); + return true; +}; + +/** What makes one step differ from the next: not its text, which changes as it is typed into. */ +const shape = (p: LoginProbe): string => `${p.path}${p.hash}|${p.inputs.map((i) => `${i.name}:${i.type}`).join(",")}`; + +/** Waits for X's emailed code, sent at or after `since`. */ +const emailedCode = async (imap: ImapAccount, since: number, signal: AbortSignal): Promise<string> => { + const deadline = Date.now() + EMAIL_WAIT_MS; + let seen = 0; + while (Date.now() < deadline) { + // A minute's slack for the two clocks; an older code is from an earlier sign-in. + const mail = await recentMail(imap, "x.com", since - 60_000, signal); + seen = mail.length; + for (const m of mail) { + const code = xCodeFromSubject(m.subject); + if (code) return code; + } + await sleep(EMAIL_POLL_MS, signal); + } + throw new BrowserUnavailableError( + `X sign-in: X asked for the code it emailed, and none arrived at ${imap.user} within ` + + `${EMAIL_WAIT_MS / 1000} s (${seen} recent mails from x.com, none with a code in the subject). ` + + `The account's email may not be that mailbox.`, + ); +}; + +/** + * Signs in on `page`. Resolves when X shows the signed-in app; throws + * BrowserUnavailableError naming the step it could not get past. + */ +export const xLogin = async ( + page: PageSession, + creds: XCredentials, + signal: AbortSignal, + log: (line: string) => void = () => {}, +): Promise<void> => { + const started = Date.now(); + await page.navigate(LOGIN_URL, signal); + const answered = new Map<string, number>(); + let last: LoginProbe | undefined; + + for (let steps = 0; steps < MAX_STEPS; ) { + // A step's Enter navigates, and a probe caught mid-navigation fails: retry. + const probe = await page.evaluate<LoginProbe>(LOGIN_PROBE, signal).catch((e: unknown) => { + if (signal.aborted) throw e; + return undefined; + }); + if (!probe) { + await sleep(400, signal); + continue; + } + last = probe; + const step = classify(probe); + if (step.kind === "done") { + log(`x-login signed in as ${creds.username}`); + return; + } + if (step.kind === "loading") { + await sleep(400, signal); + continue; + } + // Coming back to a step means the answer was refused: stop rather than + // hammer X with a wrong password or a stale code. The password is never + // sent twice; a code gets one more go, in case it expired on the way. + const key = step.kind; + if (step.kind === "wrong-password" || (answered.get(key) ?? 0) >= (key === "password" ? 1 : 2)) { + throw new BrowserUnavailableError( + `X sign-in refused at the ${step.kind} step (${probe.url}): "${probe.text.slice(0, 200)}". ` + + `Tell the user: someone has to sign in to X in castle Chrome by hand.`, + ); + } + if (step.kind === "unknown") { + throw new BrowserUnavailableError( + `X sign-in stopped at a step this tool does not handle (${probe.url}): "${probe.text.slice(0, 300)}". ` + + `Tell the user: someone has to sign in to X in castle Chrome by hand.`, + ); + } + + let code: (() => string) | undefined; + if (step.kind === "username") code = () => creds.username; + else if (step.kind === "password") code = () => creds.password; + else if (step.kind === "totp") { + const secret = creds.totpSecret; + if (!secret) throw new BrowserUnavailableError("X sign-in wants an authenticator code and the sign-in file has no totpSecret"); + // A code about to expire may be stale by the time X checks it. + if (totpRemaining() < 5) await sleep(totpRemaining() * 1000 + 500, signal); + code = () => totp(secret); + } else if (step.kind === "email-code") { + if (!creds.imap) { + throw new BrowserUnavailableError( + `X sign-in wants a code it emailed, and the sign-in file has no mailbox to read it from ` + + `(${probe.url}). Tell the user: someone has to sign in to X in castle Chrome by hand.`, + ); + } + const emailed = await emailedCode(creds.imap, started, signal); + code = () => emailed; + } + if (!code || !(await answer(page, code, step.kind === "password", signal))) { + await sleep(400, signal); + continue; + } + log(`x-login answered ${step.kind} at ${probe.hash || probe.path}`); + answered.set(key, (answered.get(key) ?? 0) + 1); + steps++; + + // Wait for the next step, up to ~10 s; a refused answer stays on this one. + const before = shape(probe); + for (let w = 0; w < 25; w++) { + await sleep(400, signal); + const now = await page.evaluate<LoginProbe>(LOGIN_PROBE, signal).catch(() => undefined); + if (!now || shape(now) !== before || classify(now).kind !== step.kind) break; + } + } + throw new BrowserUnavailableError( + `X sign-in did not finish within ${MAX_STEPS} steps (last at ${last?.url}). Tell the user: someone has ` + + `to sign in to X in castle Chrome by hand.`, + ); +}; diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts index 38dc053..7c8badf 100644 --- a/castle-search/src/x-search.ts +++ b/castle-search/src/x-search.ts @@ -19,10 +19,29 @@ import { BrowserUnavailableError, NoResultsError } from "./errors.ts"; import { withPage } from "./search.ts"; import { XLimiter } from "./x-limits.ts"; +import { loadCredentials, X_LOGIN_TIMEOUT_MS, xLogin } from "./x-login.ts"; /** Shared by every X search in this process: see x-limits.ts. */ const limiter = new XLimiter(); +/** + * Signed out, a search signs the browser back in (x-login.ts) when + * CCS_X_LOGIN_FILE names the credentials, then searches again, once. At most + * one sign-in attempt per LOGIN_COOLDOWN_MS: a sign-in that keeps failing + * is what gets an account locked, so until the cooldown has passed every + * search fails with the reason the last attempt gave. + */ +export const LOGIN_COOLDOWN_MS = 30 * 60_000; +let lastLogin: { at: number; error?: string } | undefined; + +class SignedOut extends Error { + readonly at: string; + constructor(at: string) { + super(`signed out at ${at}`); + this.at = at; + } +} + export type XSort = "latest" | "top"; export type XPost = { @@ -160,6 +179,17 @@ export const xSearch = (request: XSearchRequest, outer: AbortSignal): Promise<XS limiter.run(outer, () => searchNow(request, outer)); const searchNow = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => { + try { + return await attempt(request, outer); + } catch (e) { + if (!(e instanceof SignedOut)) throw e; + // The sign-in has its own deadline, and the search after it a fresh one. + await signIn(e.at, outer); + return await attempt(request, outer); + } +}; + +const attempt = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => { const sort: XSort = request.sort ?? "latest"; const searchUrl = buildXSearchUrl(request.query, sort); const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]); @@ -178,6 +208,37 @@ const searchNow = async (request: XSearchRequest, outer: AbortSignal): Promise<X } }; +const signedOutError = (at: string, why: string): BrowserUnavailableError => + new BrowserUnavailableError( + `X search needs a signed-in X session in castle Chrome, and it is signed out (X sent it to ${at}). ` + + `${why} Tell the user: someone has to sign in to X in that browser again.`, + ); + +/** Signs the browser in, within its own deadline, or throws saying why not. */ +const signIn = async (at: string, outer: AbortSignal): Promise<void> => { + const creds = loadCredentials(process.env["CCS_X_LOGIN_FILE"]); + if (!creds) throw signedOutError(at, "No sign-in details are configured (CCS_X_LOGIN_FILE)."); + if (lastLogin && Date.now() - lastLogin.at < LOGIN_COOLDOWN_MS) { + const next = new Date(lastLogin.at + LOGIN_COOLDOWN_MS).toISOString().slice(11, 16); + throw signedOutError( + at, + `Signing in again was tried at ${new Date(lastLogin.at).toISOString().slice(11, 16)} UTC` + + (lastLogin.error ? ` and failed: ${lastLogin.error}` : ", yet it is signed out again.") + + ` The next try is after ${next} UTC.`, + ); + } + lastLogin = { at: Date.now() }; + const signal = AbortSignal.any([outer, AbortSignal.timeout(X_LOGIN_TIMEOUT_MS)]); + try { + await withPage(signal, (page) => xLogin(page, creds, signal, (line) => console.log(line))); + } catch (e) { + const why = signal.aborted && !outer.aborted ? `it took over ${X_LOGIN_TIMEOUT_MS / 1000} s` : (e as Error).message; + lastLogin.error = why; + console.log(`x-login failed: ${why}`); + throw signedOutError(at, `Signing in again failed: ${why}`); + } +}; + const run = ( request: XSearchRequest, sort: XSort, @@ -185,7 +246,7 @@ const run = ( signal: AbortSignal, saw: (state: string) => void, ): Promise<XSearchOutcome> => - withPage(signal, async (page, endpoint) => { + withPage(signal, async (page) => { await page.navigate(searchUrl, signal); // X renders after the load event: poll until posts, an empty or error @@ -198,12 +259,7 @@ const run = ( await sleep(POLL_MS, signal); } - if (probe.state === "signed-out") { - throw new BrowserUnavailableError( - `X search needs a signed-in X session in the browser (${endpoint}), and it is signed out ` + - `(X sent it to ${probe.url}). Tell the user: someone has to sign in to X in that browser again.`, - ); - } + if (probe.state === "signed-out") throw new SignedOut(probe.url); if (probe.state === "error") { throw new BrowserUnavailableError( `X answered with an error page instead of results (${probe.url}): "${probe.note}". This is ` + diff --git a/castle-search/test/x-login.test.ts b/castle-search/test/x-login.test.ts new file mode 100644 index 0000000..11468d5 --- /dev/null +++ b/castle-search/test/x-login.test.ts @@ -0,0 +1,91 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { decodeWords, parseFetch, parseInternalDate, xCodeFromSubject } from "../src/imap.ts"; +import { classify, type LoginProbe } from "../src/x-login.ts"; +import { base32Decode, totp, totpRemaining } from "../src/totp.ts"; + +// RFC 6238 appendix B, SHA-1: the secret "12345678901234567890". +const RFC_SECRET = "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ"; + +test("totp matches RFC 6238's SHA-1 vectors", () => { + assert.equal(totp(RFC_SECRET, 59_000, 8), "94287082"); + assert.equal(totp(RFC_SECRET, 1_111_111_109_000, 8), "07081804"); + assert.equal(totp(RFC_SECRET, 1_234_567_890_000, 8), "89005924"); + assert.equal(totp(RFC_SECRET, 59_000), "287082"); +}); + +test("base32 ignores spaces and case, and refuses other characters", () => { + assert.deepEqual(base32Decode("gezd gnbv"), base32Decode("GEZDGNBV")); + assert.throws(() => base32Decode("GEZD1")); +}); + +test("totpRemaining counts down within the step", () => { + assert.equal(totpRemaining(0), 30); + assert.equal(totpRemaining(29_000), 1); +}); + +test("X's code is read from the subjects X uses, and nothing else", () => { + assert.equal(xCodeFromSubject("Your X confirmation code is abc12def"), "abc12def"); + assert.equal(xCodeFromSubject("Your X verification code is: 123456"), "123456"); + assert.equal(xCodeFromSubject("123456 is your X verification code"), "123456"); + assert.equal(xCodeFromSubject("New login to X from Chrome on Mac"), undefined); + assert.equal(xCodeFromSubject("Security alert"), undefined); +}); + +test("encoded subjects are decoded", () => { + assert.equal(decodeWords("=?UTF-8?B?WW91ciBYIGNvbmZpcm1hdGlvbiBjb2RlIGlzIGFiYzEyZGVm?="), "Your X confirmation code is abc12def"); + assert.equal(decodeWords("=?UTF-8?Q?Caf=C3=A9_menu?="), "Café menu"); +}); + +test("a FETCH reply splits into one header set per message", () => { + const raw = + '* 1 FETCH (INTERNALDATE "18-Nov-2026 10:16:23 +0000" BODY[HEADER.FIELDS (FROM SUBJECT)] {60}\r\n' + + "From: X <verify@x.com>\r\nSubject: Your X confirmation code is\r\n abc12def\r\n\r\n)\r\n" + + '* 2 FETCH (INTERNALDATE "18-Nov-2026 10:20:00 +0000" BODY[HEADER.FIELDS (FROM SUBJECT)] {40}\r\n' + + "From: X <info@x.com>\r\nSubject: New login\r\n\r\n)\r\nc4 OK Success\r\n"; + const mails = parseFetch(raw); + assert.equal(mails.length, 2); + assert.equal(mails[0]!.subject, "Your X confirmation code is abc12def"); + assert.equal(mails[0]!.received, parseInternalDate("18-Nov-2026 10:16:23 +0000")); + assert.equal(mails[0]!.received, Date.UTC(2026, 10, 18, 10, 16, 23)); + assert.equal(mails[1]!.from, "X <info@x.com>"); +}); + +const probe = (over: Partial<LoginProbe>): LoginProbe => ({ + url: "https://x.com/i/jf/onboarding/web?mode=login", + path: "/i/jf/onboarding/web", + hash: "", + text: "", + inputs: [], + signedIn: false, + ...over, +}); + +// The steps as castle Chrome saw them on 2026-11-18. +test("each step of X's sign-in is recognised", () => { + assert.equal( + classify(probe({ text: "See what's happening\n\nEmail or username\n\nContinue", inputs: [{ name: "username_or_email", type: "text", autocomplete: "username webauthn" }] })).kind, + "username", + ); + assert.equal( + classify(probe({ hash: "#/s/login_enter_password/r-1", text: "Login\n\nUsername\nPassword", inputs: [{ name: "username", type: "text", autocomplete: "" }, { name: "password", type: "password", autocomplete: "" }] })).kind, + "password", + ); + assert.equal( + classify(probe({ hash: "#/s/two_factor_code/r-2", text: "Enter your two factor code\n\nUse your authenticator app to generate the code.", inputs: [{ name: "", type: "text", autocomplete: "one-time-code" }] })).kind, + "totp", + ); + assert.equal( + classify(probe({ text: "Check your email\n\nYou'll receive a code to verify here so you can sign in.", inputs: [{ name: "text", type: "text", autocomplete: "" }] })).kind, + "email-code", + ); + assert.equal( + classify(probe({ text: "Enter your phone number or username\n\nThere was unusual login activity on your account.", inputs: [{ name: "text", type: "text", autocomplete: "" }] })).kind, + "username", + ); + assert.equal(classify(probe({ url: "https://x.com/home", path: "/home", signedIn: true })).kind, "done"); + assert.equal(classify(probe({})).kind, "loading"); + assert.equal(classify(probe({ text: "Authenticate your account\n\nSolve this puzzle" })).kind, "unknown"); + assert.equal(classify(probe({ text: "Wrong password!", inputs: [{ name: "password", type: "password", autocomplete: "" }] })).kind, "wrong-password"); +}); |
