diff options
Diffstat (limited to 'castle-search')
| -rw-r--r-- | castle-search/README.md | 4 | ||||
| -rw-r--r-- | castle-search/src/x-limits.ts | 29 | ||||
| -rw-r--r-- | castle-search/test/x-limits.test.ts | 32 |
3 files changed, 48 insertions, 17 deletions
diff --git a/castle-search/README.md b/castle-search/README.md index e8b204a..cf15aa1 100644 --- a/castle-search/README.md +++ b/castle-search/README.md @@ -71,8 +71,8 @@ X keeps only the posts near the screen in the page, so it scrolls and collects by link, up to 20 posts (or replies). A read takes 4–7 s. All four share one set of limits, so the account reads X like a person, not a -scraper (`src/x-limits.ts`): one read at a time, at least 10 s between them (a -read that comes sooner waits), and at most 30 an hour and 150 a day, beyond which a +scraper (`src/x-limits.ts`): one read at a time, 30 s ± a random 0–25 s between them, +drawn afresh each time (a read that comes sooner waits), and at most 30 an hour and 150 a day, beyond which a read fails saying when the next may run. The counts live in memory, so a restart starts them afresh. diff --git a/castle-search/src/x-limits.ts b/castle-search/src/x-limits.ts index 77e6234..bdc4229 100644 --- a/castle-search/src/x-limits.ts +++ b/castle-search/src/x-limits.ts @@ -3,15 +3,18 @@ * than a scraper, and X has no reason to lock it. * * - one search at a time; - * - at least MIN_GAP_MS between the starts of two searches: one that comes - * sooner waits its turn rather than failing; + * - GAP_MS ± up to JITTER_MS between the starts of two searches, drawn + * afresh each time (5-55 s), so they do not come at a machine's even + * beat: one that comes sooner waits its turn rather than failing; * - at most PER_HOUR in any rolling hour and PER_DAY in any rolling day: * beyond that a search fails at once, saying when the next one may run. * * Counts are kept in memory: a restart of the service starts them afresh. */ -export const MIN_GAP_MS = 10_000; +/** Owner's choice, 2026-11-18: 30 s plus or minus a random 0-25 s. */ +export const GAP_MS = 30_000; +export const JITTER_MS = 25_000; export const PER_HOUR = 30; export const PER_DAY = 150; const HOUR_MS = 3_600_000; @@ -21,7 +24,12 @@ export class XRateLimitError extends Error { override readonly name = "XRateLimitError"; } -export type Clock = { now: () => number; sleep: (ms: number, signal: AbortSignal) => Promise<void> }; +export type Clock = { + now: () => number; + sleep: (ms: number, signal: AbortSignal) => Promise<void>; + /** Uniform in [0, 1), as Math.random. */ + random: () => number; +}; const realSleep = (ms: number, signal: AbortSignal): Promise<void> => new Promise((resolve, reject) => { @@ -33,19 +41,21 @@ const realSleep = (ms: number, signal: AbortSignal): Promise<void> => }, { once: true }); }); -export const realClock: Clock = { now: Date.now, sleep: realSleep }; +export const realClock: Clock = { now: Date.now, sleep: realSleep, random: Math.random }; + +export type Limits = { gap: number; jitter: number; hour: number; day: number }; const at = (ms: number): string => new Date(ms).toISOString().replace("T", " ").slice(0, 16) + " UTC"; export class XLimiter { readonly #clock: Clock; - readonly #limits: { gap: number; hour: number; day: number }; + readonly #limits: Limits; /** Start times of the searches in the last day, oldest first. */ #starts: number[] = []; /** The tail of the queue: each search waits for the one before it. */ #tail: Promise<void> = Promise.resolve(); - constructor(clock: Clock = realClock, limits = { gap: MIN_GAP_MS, hour: PER_HOUR, day: PER_DAY }) { + constructor(clock: Clock = realClock, limits: Limits = { gap: GAP_MS, jitter: JITTER_MS, hour: PER_HOUR, day: PER_DAY }) { this.#clock = clock; this.#limits = limits; } @@ -90,8 +100,9 @@ export class XLimiter { ); } const last = this.#starts[this.#starts.length - 1]; - if (last !== undefined && now - last < this.#limits.gap) { - await this.#clock.sleep(this.#limits.gap - (now - last), signal); + const gap = Math.max(0, this.#limits.gap + (this.#clock.random() * 2 - 1) * this.#limits.jitter); + if (last !== undefined && now - last < gap) { + await this.#clock.sleep(gap - (now - last), signal); } this.#starts.push(this.#clock.now()); } diff --git a/castle-search/test/x-limits.test.ts b/castle-search/test/x-limits.test.ts index b239f5b..6be5f7d 100644 --- a/castle-search/test/x-limits.test.ts +++ b/castle-search/test/x-limits.test.ts @@ -7,14 +7,16 @@ import { type Clock, XLimiter, XRateLimitError } from "../src/x-limits.ts"; const fakeClock = () => { let now = 1_000_000; const waits: number[] = []; + let draw = 0.5; const clock: Clock = { + random: () => draw, now: () => now, sleep: async (ms) => { waits.push(ms); now += ms; }, }; - return { clock, waits, advance: (ms: number) => (now += ms) }; + return { clock, waits, advance: (ms: number) => (now += ms), setDraw: (r: number) => (draw = r) }; }; const signal = new AbortController().signal; @@ -22,7 +24,7 @@ const ok = async () => "ok"; test("a search sooner than the gap waits out the rest of it", async () => { const { clock, waits, advance } = fakeClock(); - const l = new XLimiter(clock, { gap: 10_000, hour: 30, day: 150 }); + const l = new XLimiter(clock, { gap: 10_000, jitter: 0, hour: 30, day: 150 }); await l.run(signal, ok); advance(3_000); await l.run(signal, ok); @@ -31,7 +33,7 @@ test("a search sooner than the gap waits out the rest of it", async () => { test("over the hourly cap a search fails at once, naming when it can run", async () => { const { clock, advance } = fakeClock(); - const l = new XLimiter(clock, { gap: 0, hour: 2, day: 150 }); + const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 2, day: 150 }); await l.run(signal, ok); await l.run(signal, ok); await assert.rejects(l.run(signal, ok), (e) => e instanceof XRateLimitError && /last hour/.test(e.message)); @@ -41,7 +43,7 @@ test("over the hourly cap a search fails at once, naming when it can run", async test("the daily cap holds across hours", async () => { const { clock, advance } = fakeClock(); - const l = new XLimiter(clock, { gap: 0, hour: 100, day: 3 }); + const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 100, day: 3 }); for (let i = 0; i < 3; i++) { await l.run(signal, ok); advance(3_600_001); @@ -51,7 +53,7 @@ test("the daily cap holds across hours", async () => { test("one at a time: the second starts only after the first finishes", async () => { const { clock } = fakeClock(); - const l = new XLimiter(clock, { gap: 0, hour: 30, day: 150 }); + const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 30, day: 150 }); const order: string[] = []; let release!: () => void; const first = l.run(signal, async () => { @@ -70,9 +72,27 @@ test("one at a time: the second starts only after the first finishes", async () test("a failed search does not block the queue", async () => { const { clock } = fakeClock(); - const l = new XLimiter(clock, { gap: 0, hour: 30, day: 150 }); + const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 30, day: 150 }); await assert.rejects(l.run(signal, async () => { throw new Error("boom"); })); assert.equal(await l.run(signal, ok), "ok"); }); + +test("the gap is drawn afresh each time, within gap ± jitter", async () => { + const { clock, waits, setDraw } = fakeClock(); + const l = new XLimiter(clock, { gap: 30_000, jitter: 25_000, hour: 30, day: 150 }); + await l.run(signal, ok); + setDraw(0); // the shortest: 30 - 25 s + await l.run(signal, ok); + setDraw(0.999999); // close to the longest: 30 + 25 s + await l.run(signal, ok); + assert.equal(waits[0], 5_000); + assert.ok(waits[1]! > 54_999 && waits[1]! <= 55_000); +}); + +test("the defaults are 30 s ± 25 s", async () => { + const { GAP_MS, JITTER_MS } = await import("../src/x-limits.ts"); + assert.equal(GAP_MS, 30_000); + assert.equal(JITTER_MS, 25_000); +}); |
