aboutsummaryrefslogtreecommitdiff
path: root/castle-search
diff options
context:
space:
mode:
Diffstat (limited to 'castle-search')
-rw-r--r--castle-search/README.md4
-rw-r--r--castle-search/src/x-limits.ts29
-rw-r--r--castle-search/test/x-limits.test.ts32
3 files changed, 48 insertions, 17 deletions
diff --git a/castle-search/README.md b/castle-search/README.md
index e8b204a..cf15aa1 100644
--- a/castle-search/README.md
+++ b/castle-search/README.md
@@ -71,8 +71,8 @@ X keeps only the posts near the screen in the page, so it scrolls and collects
by link, up to 20 posts (or replies). A read takes 4–7 s.
All four share one set of limits, so the account reads X like a person, not a
-scraper (`src/x-limits.ts`): one read at a time, at least 10 s between them (a
-read that comes sooner waits), and at most 30 an hour and 150 a day, beyond which a
+scraper (`src/x-limits.ts`): one read at a time, 30 s ± a random 0–25 s between them,
+drawn afresh each time (a read that comes sooner waits), and at most 30 an hour and 150 a day, beyond which a
read fails saying when the next may run. The counts live in memory, so a
restart starts them afresh.
diff --git a/castle-search/src/x-limits.ts b/castle-search/src/x-limits.ts
index 77e6234..bdc4229 100644
--- a/castle-search/src/x-limits.ts
+++ b/castle-search/src/x-limits.ts
@@ -3,15 +3,18 @@
* than a scraper, and X has no reason to lock it.
*
* - one search at a time;
- * - at least MIN_GAP_MS between the starts of two searches: one that comes
- * sooner waits its turn rather than failing;
+ * - GAP_MS ± up to JITTER_MS between the starts of two searches, drawn
+ * afresh each time (5-55 s), so they do not come at a machine's even
+ * beat: one that comes sooner waits its turn rather than failing;
* - at most PER_HOUR in any rolling hour and PER_DAY in any rolling day:
* beyond that a search fails at once, saying when the next one may run.
*
* Counts are kept in memory: a restart of the service starts them afresh.
*/
-export const MIN_GAP_MS = 10_000;
+/** Owner's choice, 2026-11-18: 30 s plus or minus a random 0-25 s. */
+export const GAP_MS = 30_000;
+export const JITTER_MS = 25_000;
export const PER_HOUR = 30;
export const PER_DAY = 150;
const HOUR_MS = 3_600_000;
@@ -21,7 +24,12 @@ export class XRateLimitError extends Error {
override readonly name = "XRateLimitError";
}
-export type Clock = { now: () => number; sleep: (ms: number, signal: AbortSignal) => Promise<void> };
+export type Clock = {
+ now: () => number;
+ sleep: (ms: number, signal: AbortSignal) => Promise<void>;
+ /** Uniform in [0, 1), as Math.random. */
+ random: () => number;
+};
const realSleep = (ms: number, signal: AbortSignal): Promise<void> =>
new Promise((resolve, reject) => {
@@ -33,19 +41,21 @@ const realSleep = (ms: number, signal: AbortSignal): Promise<void> =>
}, { once: true });
});
-export const realClock: Clock = { now: Date.now, sleep: realSleep };
+export const realClock: Clock = { now: Date.now, sleep: realSleep, random: Math.random };
+
+export type Limits = { gap: number; jitter: number; hour: number; day: number };
const at = (ms: number): string => new Date(ms).toISOString().replace("T", " ").slice(0, 16) + " UTC";
export class XLimiter {
readonly #clock: Clock;
- readonly #limits: { gap: number; hour: number; day: number };
+ readonly #limits: Limits;
/** Start times of the searches in the last day, oldest first. */
#starts: number[] = [];
/** The tail of the queue: each search waits for the one before it. */
#tail: Promise<void> = Promise.resolve();
- constructor(clock: Clock = realClock, limits = { gap: MIN_GAP_MS, hour: PER_HOUR, day: PER_DAY }) {
+ constructor(clock: Clock = realClock, limits: Limits = { gap: GAP_MS, jitter: JITTER_MS, hour: PER_HOUR, day: PER_DAY }) {
this.#clock = clock;
this.#limits = limits;
}
@@ -90,8 +100,9 @@ export class XLimiter {
);
}
const last = this.#starts[this.#starts.length - 1];
- if (last !== undefined && now - last < this.#limits.gap) {
- await this.#clock.sleep(this.#limits.gap - (now - last), signal);
+ const gap = Math.max(0, this.#limits.gap + (this.#clock.random() * 2 - 1) * this.#limits.jitter);
+ if (last !== undefined && now - last < gap) {
+ await this.#clock.sleep(gap - (now - last), signal);
}
this.#starts.push(this.#clock.now());
}
diff --git a/castle-search/test/x-limits.test.ts b/castle-search/test/x-limits.test.ts
index b239f5b..6be5f7d 100644
--- a/castle-search/test/x-limits.test.ts
+++ b/castle-search/test/x-limits.test.ts
@@ -7,14 +7,16 @@ import { type Clock, XLimiter, XRateLimitError } from "../src/x-limits.ts";
const fakeClock = () => {
let now = 1_000_000;
const waits: number[] = [];
+ let draw = 0.5;
const clock: Clock = {
+ random: () => draw,
now: () => now,
sleep: async (ms) => {
waits.push(ms);
now += ms;
},
};
- return { clock, waits, advance: (ms: number) => (now += ms) };
+ return { clock, waits, advance: (ms: number) => (now += ms), setDraw: (r: number) => (draw = r) };
};
const signal = new AbortController().signal;
@@ -22,7 +24,7 @@ const ok = async () => "ok";
test("a search sooner than the gap waits out the rest of it", async () => {
const { clock, waits, advance } = fakeClock();
- const l = new XLimiter(clock, { gap: 10_000, hour: 30, day: 150 });
+ const l = new XLimiter(clock, { gap: 10_000, jitter: 0, hour: 30, day: 150 });
await l.run(signal, ok);
advance(3_000);
await l.run(signal, ok);
@@ -31,7 +33,7 @@ test("a search sooner than the gap waits out the rest of it", async () => {
test("over the hourly cap a search fails at once, naming when it can run", async () => {
const { clock, advance } = fakeClock();
- const l = new XLimiter(clock, { gap: 0, hour: 2, day: 150 });
+ const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 2, day: 150 });
await l.run(signal, ok);
await l.run(signal, ok);
await assert.rejects(l.run(signal, ok), (e) => e instanceof XRateLimitError && /last hour/.test(e.message));
@@ -41,7 +43,7 @@ test("over the hourly cap a search fails at once, naming when it can run", async
test("the daily cap holds across hours", async () => {
const { clock, advance } = fakeClock();
- const l = new XLimiter(clock, { gap: 0, hour: 100, day: 3 });
+ const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 100, day: 3 });
for (let i = 0; i < 3; i++) {
await l.run(signal, ok);
advance(3_600_001);
@@ -51,7 +53,7 @@ test("the daily cap holds across hours", async () => {
test("one at a time: the second starts only after the first finishes", async () => {
const { clock } = fakeClock();
- const l = new XLimiter(clock, { gap: 0, hour: 30, day: 150 });
+ const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 30, day: 150 });
const order: string[] = [];
let release!: () => void;
const first = l.run(signal, async () => {
@@ -70,9 +72,27 @@ test("one at a time: the second starts only after the first finishes", async ()
test("a failed search does not block the queue", async () => {
const { clock } = fakeClock();
- const l = new XLimiter(clock, { gap: 0, hour: 30, day: 150 });
+ const l = new XLimiter(clock, { gap: 0, jitter: 0, hour: 30, day: 150 });
await assert.rejects(l.run(signal, async () => {
throw new Error("boom");
}));
assert.equal(await l.run(signal, ok), "ok");
});
+
+test("the gap is drawn afresh each time, within gap ± jitter", async () => {
+ const { clock, waits, setDraw } = fakeClock();
+ const l = new XLimiter(clock, { gap: 30_000, jitter: 25_000, hour: 30, day: 150 });
+ await l.run(signal, ok);
+ setDraw(0); // the shortest: 30 - 25 s
+ await l.run(signal, ok);
+ setDraw(0.999999); // close to the longest: 30 + 25 s
+ await l.run(signal, ok);
+ assert.equal(waits[0], 5_000);
+ assert.ok(waits[1]! > 54_999 && waits[1]! <= 55_000);
+});
+
+test("the defaults are 30 s ± 25 s", async () => {
+ const { GAP_MS, JITTER_MS } = await import("../src/x-limits.ts");
+ assert.equal(GAP_MS, 30_000);
+ assert.equal(JITTER_MS, 25_000);
+});