diff options
| author | Igor Soarez <igor@soarez.org> | 2026-11-18 05:00:44 +0000 |
|---|---|---|
| committer | Igor Soarez <igor@soarez.org> | 2026-11-18 05:00:44 +0000 |
| commit | 39f887e07721c5b3ddd31657e5e8e25e4486d2e5 (patch) | |
| tree | 95e0e6f98db7cfe3923f9bd8d7718f29e38f65b0 /castle-search/src | |
| parent | b03415aafeea985561f3980f8b484173530a05f8 (diff) | |
castle-search: limit X searches so the account reads X like a person
One at a time, 10 s apart (a sooner one waits), at most 30 an hour and 150 a
day; over a cap a search fails at once, saying when the next may run.
Diffstat (limited to 'castle-search/src')
| -rw-r--r-- | castle-search/src/x-limits.ts | 98 | ||||
| -rw-r--r-- | castle-search/src/x-search.ts | 11 |
2 files changed, 108 insertions, 1 deletions
diff --git a/castle-search/src/x-limits.ts b/castle-search/src/x-limits.ts new file mode 100644 index 0000000..77e6234 --- /dev/null +++ b/castle-search/src/x-limits.ts @@ -0,0 +1,98 @@ +/** + * Limits on X searches, so the signed-in account reads X like a person rather + * than a scraper, and X has no reason to lock it. + * + * - one search at a time; + * - at least MIN_GAP_MS between the starts of two searches: one that comes + * sooner waits its turn rather than failing; + * - at most PER_HOUR in any rolling hour and PER_DAY in any rolling day: + * beyond that a search fails at once, saying when the next one may run. + * + * Counts are kept in memory: a restart of the service starts them afresh. + */ + +export const MIN_GAP_MS = 10_000; +export const PER_HOUR = 30; +export const PER_DAY = 150; +const HOUR_MS = 3_600_000; +const DAY_MS = 24 * HOUR_MS; + +export class XRateLimitError extends Error { + override readonly name = "XRateLimitError"; +} + +export type Clock = { now: () => number; sleep: (ms: number, signal: AbortSignal) => Promise<void> }; + +const realSleep = (ms: number, signal: AbortSignal): Promise<void> => + new Promise((resolve, reject) => { + if (signal.aborted) return reject(signal.reason); + const t = setTimeout(resolve, ms); + signal.addEventListener("abort", () => { + clearTimeout(t); + reject(signal.reason); + }, { once: true }); + }); + +export const realClock: Clock = { now: Date.now, sleep: realSleep }; + +const at = (ms: number): string => new Date(ms).toISOString().replace("T", " ").slice(0, 16) + " UTC"; + +export class XLimiter { + readonly #clock: Clock; + readonly #limits: { gap: number; hour: number; day: number }; + /** Start times of the searches in the last day, oldest first. */ + #starts: number[] = []; + /** The tail of the queue: each search waits for the one before it. */ + #tail: Promise<void> = Promise.resolve(); + + constructor(clock: Clock = realClock, limits = { gap: MIN_GAP_MS, hour: PER_HOUR, day: PER_DAY }) { + this.#clock = clock; + this.#limits = limits; + } + + /** Runs `fn` once the limits allow, one at a time; throws XRateLimitError when over a cap. */ + async run<T>(signal: AbortSignal, fn: () => Promise<T>): Promise<T> { + const previous = this.#tail; + let done!: () => void; + this.#tail = new Promise<void>((resolve) => (done = resolve)); + try { + await Promise.race([ + previous, + new Promise<never>((_, reject) => { + if (signal.aborted) reject(signal.reason); + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }), + ]); + // An abandoned search ahead of us must not hold the queue: `previous` + // settles when it finishes or is aborted, whichever comes first. + await this.#admit(signal); + return await fn(); + } finally { + done(); + } + } + + async #admit(signal: AbortSignal): Promise<void> { + const now = this.#clock.now(); + this.#starts = this.#starts.filter((t) => t > now - DAY_MS); + const lastHour = this.#starts.filter((t) => t > now - HOUR_MS); + if (this.#starts.length >= this.#limits.day) { + throw new XRateLimitError( + `X search limit reached: ${this.#limits.day} searches in the last 24 hours, to keep the account ` + + `from being locked. The next one can run after ${at(this.#starts[0]! + DAY_MS)}. Tell the user.`, + ); + } + if (lastHour.length >= this.#limits.hour) { + throw new XRateLimitError( + `X search limit reached: ${this.#limits.hour} searches in the last hour, to keep the account ` + + `from being locked. The next one can run after ${at(lastHour[0]! + HOUR_MS)}. Tell the user, ` + + `and do not retry before then.`, + ); + } + const last = this.#starts[this.#starts.length - 1]; + if (last !== undefined && now - last < this.#limits.gap) { + await this.#clock.sleep(this.#limits.gap - (now - last), signal); + } + this.#starts.push(this.#clock.now()); + } +} diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts index 6ea086c..38dc053 100644 --- a/castle-search/src/x-search.ts +++ b/castle-search/src/x-search.ts @@ -18,6 +18,10 @@ import { BrowserUnavailableError, NoResultsError } from "./errors.ts"; import { withPage } from "./search.ts"; +import { XLimiter } from "./x-limits.ts"; + +/** Shared by every X search in this process: see x-limits.ts. */ +const limiter = new XLimiter(); export type XSort = "latest" | "top"; @@ -150,7 +154,12 @@ const finish = (raw: RawPost): XPost => { return { ...rest, ...parseMetrics(metrics) }; }; -export const xSearch = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => { +export const xSearch = (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => + // The search's own deadline starts once the limiter lets it through, so + // waiting its turn does not eat into it. + limiter.run(outer, () => searchNow(request, outer)); + +const searchNow = async (request: XSearchRequest, outer: AbortSignal): Promise<XSearchOutcome> => { const sort: XSort = request.sort ?? "latest"; const searchUrl = buildXSearchUrl(request.query, sort); const signal = AbortSignal.any([outer, AbortSignal.timeout(X_SEARCH_TIMEOUT_MS)]); |
