aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorIgor Soarez <igor@soarez.org>2026-11-18 12:18:44 +0000
committerIgor Soarez <igor@soarez.org>2026-11-18 12:18:44 +0000
commitea10a2f1c5f7a9aae2fea99894df21b76d126d27 (patch)
tree72fd059f431152f9b7e9a0da4236f458888e61e2
parentf8835d53b96bb6e4152bc5bf84e61b8b54396008 (diff)
castle-search: let an X page's first screen settle before reading it
Photos, link cards and a post's replies render after the post itself; a read taken at once could miss them. Wait until two readings agree.
-rw-r--r--castle-search/src/x-search.ts17
1 files changed, 14 insertions, 3 deletions
diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts
index 174906e..93e8b56 100644
--- a/castle-search/src/x-search.ts
+++ b/castle-search/src/x-search.ts
@@ -170,7 +170,7 @@ type XProbe = {
* Reads the page as it stands. Runs in the page; must stay self-contained.
* Ads are skipped by having no <time>: every real post has one.
*/
-const PROBE = `(() => {
+export const PROBE = `(() => {
const url = location.href;
if (/^\\/(login|i\\/flow\\/login|i\\/jf\\/onboarding|account\\/access)/.test(location.pathname)) {
return { url, state: "signed-out", posts: [], note: location.pathname, profile: null };
@@ -389,8 +389,19 @@ const collect = async (page: PageSession, first: XProbe, enough: (seen: Map<stri
const seen = new Map<string, RawPost>();
const take = (p: XProbe) => p.posts.forEach((post) => seen.set(post.url, post));
take(first);
- await sleep(POLL_MS * 2, signal);
- take(await page.evaluate<XProbe>(PROBE, signal));
+ // Let the first screen settle: photos, cards and a post's replies arrive
+ // after the post itself. Settled means two readings in a row agree, and
+ // there are enough posts or ~3 s have passed.
+ const shape = (p: XProbe) => p.posts.map((x) => `${x.url}:${x.media?.length ?? 0}:${x.card ? 1 : 0}`).join(" ");
+ let previous = shape(first);
+ for (let w = 0; w < 8; w++) {
+ await sleep(POLL_MS, signal);
+ const now = await page.evaluate<XProbe>(PROBE, signal);
+ take(now);
+ const current = shape(now);
+ if (current === previous && (enough(seen) || w >= 3)) break;
+ previous = current;
+ }
for (let i = 0; i < MAX_SCROLLS && !enough(seen); i++) {
const before = seen.size;
await page.evaluate(SCROLL, signal);