aboutsummaryrefslogtreecommitdiff
path: root/castle-search/src
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-11-18 12:37:11 +0000
committerIgor Soarez <igor@soarez.org>2026-11-18 12:37:11 +0000
commit358cf43172f0028b079c49fa16f376e400d6cc25 (patch)
treebfcd8a9ca5f2bff967a01e756794c9c318e10a90 /castle-search/src
parentea10a2f1c5f7a9aae2fea99894df21b76d126d27 (diff)
castle-search: X reads 30 s ± a random 0-25 s apart, not a fixed 10 s
An even 10 s beat allowed 30 page loads in five minutes, which reads as a machine. Drawn afresh for each read.
Diffstat (limited to 'castle-search/src')
-rw-r--r--castle-search/src/x-limits.ts29
1 files changed, 20 insertions, 9 deletions
diff --git a/castle-search/src/x-limits.ts b/castle-search/src/x-limits.ts
index 77e6234..bdc4229 100644
--- a/castle-search/src/x-limits.ts
+++ b/castle-search/src/x-limits.ts
@@ -3,15 +3,18 @@
* than a scraper, and X has no reason to lock it.
*
* - one search at a time;
- * - at least MIN_GAP_MS between the starts of two searches: one that comes
- * sooner waits its turn rather than failing;
+ * - GAP_MS ± up to JITTER_MS between the starts of two searches, drawn
+ * afresh each time (5-55 s), so they do not come at a machine's even
+ * beat: one that comes sooner waits its turn rather than failing;
* - at most PER_HOUR in any rolling hour and PER_DAY in any rolling day:
* beyond that a search fails at once, saying when the next one may run.
*
* Counts are kept in memory: a restart of the service starts them afresh.
*/
-export const MIN_GAP_MS = 10_000;
+/** Owner's choice, 2026-11-18: 30 s plus or minus a random 0-25 s. */
+export const GAP_MS = 30_000;
+export const JITTER_MS = 25_000;
export const PER_HOUR = 30;
export const PER_DAY = 150;
const HOUR_MS = 3_600_000;
@@ -21,7 +24,12 @@ export class XRateLimitError extends Error {
override readonly name = "XRateLimitError";
}
-export type Clock = { now: () => number; sleep: (ms: number, signal: AbortSignal) => Promise<void> };
+export type Clock = {
+ now: () => number;
+ sleep: (ms: number, signal: AbortSignal) => Promise<void>;
+ /** Uniform in [0, 1), as Math.random. */
+ random: () => number;
+};
const realSleep = (ms: number, signal: AbortSignal): Promise<void> =>
new Promise((resolve, reject) => {
@@ -33,19 +41,21 @@ const realSleep = (ms: number, signal: AbortSignal): Promise<void> =>
}, { once: true });
});
-export const realClock: Clock = { now: Date.now, sleep: realSleep };
+export const realClock: Clock = { now: Date.now, sleep: realSleep, random: Math.random };
+
+export type Limits = { gap: number; jitter: number; hour: number; day: number };
const at = (ms: number): string => new Date(ms).toISOString().replace("T", " ").slice(0, 16) + " UTC";
export class XLimiter {
readonly #clock: Clock;
- readonly #limits: { gap: number; hour: number; day: number };
+ readonly #limits: Limits;
/** Start times of the searches in the last day, oldest first. */
#starts: number[] = [];
/** The tail of the queue: each search waits for the one before it. */
#tail: Promise<void> = Promise.resolve();
- constructor(clock: Clock = realClock, limits = { gap: MIN_GAP_MS, hour: PER_HOUR, day: PER_DAY }) {
+ constructor(clock: Clock = realClock, limits: Limits = { gap: GAP_MS, jitter: JITTER_MS, hour: PER_HOUR, day: PER_DAY }) {
this.#clock = clock;
this.#limits = limits;
}
@@ -90,8 +100,9 @@ export class XLimiter {
);
}
const last = this.#starts[this.#starts.length - 1];
- if (last !== undefined && now - last < this.#limits.gap) {
- await this.#clock.sleep(this.#limits.gap - (now - last), signal);
+ const gap = Math.max(0, this.#limits.gap + (this.#clock.random() * 2 - 1) * this.#limits.jitter);
+ if (last !== undefined && now - last < gap) {
+ await this.#clock.sleep(gap - (now - last), signal);
}
this.#starts.push(this.#clock.now());
}