diff options
| -rw-r--r-- | castle-search/src/x-search.ts | 17 |
1 files changed, 14 insertions, 3 deletions
diff --git a/castle-search/src/x-search.ts b/castle-search/src/x-search.ts index 174906e..93e8b56 100644 --- a/castle-search/src/x-search.ts +++ b/castle-search/src/x-search.ts @@ -170,7 +170,7 @@ type XProbe = { * Reads the page as it stands. Runs in the page; must stay self-contained. * Ads are skipped by having no <time>: every real post has one. */ -const PROBE = `(() => { +export const PROBE = `(() => { const url = location.href; if (/^\\/(login|i\\/flow\\/login|i\\/jf\\/onboarding|account\\/access)/.test(location.pathname)) { return { url, state: "signed-out", posts: [], note: location.pathname, profile: null }; @@ -389,8 +389,19 @@ const collect = async (page: PageSession, first: XProbe, enough: (seen: Map<stri const seen = new Map<string, RawPost>(); const take = (p: XProbe) => p.posts.forEach((post) => seen.set(post.url, post)); take(first); - await sleep(POLL_MS * 2, signal); - take(await page.evaluate<XProbe>(PROBE, signal)); + // Let the first screen settle: photos, cards and a post's replies arrive + // after the post itself. Settled means two readings in a row agree, and + // there are enough posts or ~3 s have passed. + const shape = (p: XProbe) => p.posts.map((x) => `${x.url}:${x.media?.length ?? 0}:${x.card ? 1 : 0}`).join(" "); + let previous = shape(first); + for (let w = 0; w < 8; w++) { + await sleep(POLL_MS, signal); + const now = await page.evaluate<XProbe>(PROBE, signal); + take(now); + const current = shape(now); + if (current === previous && (enough(seen) || w >= 3)) break; + previous = current; + } for (let i = 0; i < MAX_SCROLLS && !enough(seen); i++) { const before = seen.size; await page.evaluate(SCROLL, signal); |
