/** * The script that runs inside the search results page. * * It is built as a string rather than shipped as a function because it goes to * Chrome via `Runtime.evaluate` — nothing in here is typechecked, so it is kept * defensive and free of anything that could throw on an unexpected DOM. The * per-engine configuration is injected as JSON by {@link buildProbeScript}. * * ## Two modes, because the engines genuinely differ * * **"items"** — DuckDuckGo, Bing and Brave each have a clean per-result * container, so results are read directly out of it. * * **"headings"** — Google has no stable result container; its class names * (`MjjYud`, `kb0PBd`, `yuRUbf`) are generated and change without notice. So * results are found from each `

` outward: * * - anchor = nearest enclosing `` * - header = nearest `[data-snhf]`, else the anchor * - container = climb from the header until the text grows past the header's, * abandoning the climb if a second `

` comes into scope * (that would mean the next result had been swallowed) * - snippet = `[data-sncf]` if present, else the container's lines minus the * header's lines, minus URL and breadcrumb noise * * Verified against Google's classic SERP and its `udm=14` layout, DuckDuckGo's * no-JS endpoint, Bing's `li.b_algo`, and Brave's `.snippet[data-type=web]`. * * ## Link unwrapping * * DuckDuckGo and Bing both route outbound links through a redirector, so the * raw href is useless to the agent. Both are unwrapped in the page, where the * URL and base64 primitives already exist: * * - DuckDuckGo: `//duckduckgo.com/l/?uddg=` * - Bing: `//bing.com/ck/a?…&u=a1` * * Google and Brave link straight out and need no unwrapping. */ import type { ExtractionConfig } from "./engines.ts"; /** One search hit, as the page script reports it. */ export type SearchResult = { title: string; url: string; snippet: string; }; /** What the page script returns in a single round trip. */ export type PageProbe = { /** Where the page actually ended up — redirects and consent walls move it. */ url: string; title: string; readyState: string; /** Non-null when a human check is in the way. */ challenge: { kind: string; detail: string } | null; /** * Whether the engine's results container was present at all. "none" means a * page shell with no results area — a different failure from an empty one, * and worth telling apart when diagnosing a zero-hit search. */ container: "found" | "none"; results: SearchResult[]; }; /** * Challenge detection, shared across engines. * * Every phrase here has been seen on a real page during development: Google's * `/sorry/` interstitial, and Brave's "Verifying you're not a bot / Quick check * before you continue searching" — the latter is why the wording list is broad * rather than just matching Google's "unusual traffic". Ordering matters: a * `/sorry/` page also contains a recaptcha iframe, and naming the page beats * naming the widget sitting on it. */ const CHALLENGE_DETECTION = String.raw` const detectChallenge = () => { if (/\/sorry\//.test(location.href)) return { kind: "google-block-page", detail: bodyText.slice(0, 300) }; if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname)) return { kind: "consent-wall", detail: bodyText.slice(0, 300) }; if (document.querySelector("#captcha-form, form#captcha-form")) return { kind: "captcha-form", detail: bodyText.slice(0, 300) }; if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running, [id^=cf-chl]")) return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) }; if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[src*="turnstile"], iframe[title*="challenge"]')) return { kind: "captcha-widget", detail: bodyText.slice(0, 300) }; if (/verifying (that )?you('| a)?re( not)? a? ?(human|bot|robot)|quick check before you continue|verify you are human|are you a robot|not a robot/i.test(bodyText)) return { kind: "bot-check", detail: bodyText.slice(0, 300) }; if (/unusual traffic|automated queries|suspicious activity from your/i.test(bodyText)) return { kind: "rate-limit-block", detail: bodyText.slice(0, 300) }; return null; }; `; /** * Build the probe for one engine. The configuration is embedded as a JSON * literal so the script stays a single self-contained expression. */ export const buildProbeScript = (config: ExtractionConfig): string => String.raw`(() => { var CFG = ${JSON.stringify(config)}; var norm = function (s) { return (s || "").replace(/\s+/g, " ").trim(); }; var linesOf = function (el) { return el && el.innerText ? el.innerText.split("\n").map(function (l) { return l.trim(); }).filter(Boolean) : []; }; var bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : ""; var selfHost = new RegExp(CFG.selfHostPattern); ${CHALLENGE_DETECTION} // DuckDuckGo and Bing both hide the real destination behind a redirector. var unwrap = function (href) { try { var u = new URL(href, location.href); if (CFG.unwrap === "ddg") { if (!/(^|\.)duckduckgo\.com$/.test(u.hostname)) return href; return u.searchParams.get("uddg") || href; } if (CFG.unwrap === "bing") { if (!/(^|\.)bing\.com$/.test(u.hostname)) return href; var p = u.searchParams.get("u"); if (!p) return href; var b64 = p.replace(/^a1/, "").replace(/-/g, "+").replace(/_/g, "/"); b64 += "=".repeat((4 - (b64.length % 4)) % 4); var bin = atob(b64); var bytes = new Uint8Array(bin.length); for (var i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); return new TextDecoder().decode(bytes); } return href; } catch (e) { // An unwrap that fails leaves the redirector URL in place rather than // dropping the result: a working link the agent has to follow twice beats // no link at all. return href; } }; var findRoot = function () { for (var i = 0; i < CFG.roots.length; i++) { var el = document.querySelector(CFG.roots[i]); if (el) return el; } return null; }; var acceptable = function (url) { return /^https?:/.test(url) && !selfHost.test(url); }; // ── items mode ─────────────────────────────────────────────────────────── var extractItems = function (root) { var out = []; var seen = {}; var items = root.querySelectorAll(CFG.item); for (var i = 0; i < items.length; i++) { var item = items[i]; var skip = false; for (var x = 0; CFG.exclude && x < CFG.exclude.length; x++) { if (item.matches(CFG.exclude[x]) || item.querySelector(CFG.exclude[x])) { skip = true; break; } } if (skip) continue; var anchor = item.querySelector(CFG.link); if (!anchor) continue; var url = unwrap(anchor.href); if (!acceptable(url) || seen[url]) continue; var titleEl = CFG.title ? item.querySelector(CFG.title) : null; var title = norm(titleEl ? titleEl.innerText : anchor.innerText); if (!title) continue; var snippet = ""; var snippetEl = CFG.snippet ? item.querySelector(CFG.snippet) : null; if (snippetEl) { snippet = norm(snippetEl.innerText); } else { // No dedicated description element: subtract the parts we can name // (title, breadcrumb, source) and keep what is left. var drop = {}; for (var s = 0; CFG.subtract && s < CFG.subtract.length; s++) { var parts = item.querySelectorAll(CFG.subtract[s]); for (var p = 0; p < parts.length; p++) { var pl = linesOf(parts[p]); for (var q = 0; q < pl.length; q++) drop[norm(pl[q])] = true; } } var kept = linesOf(item).filter(function (l) { var n = norm(l); return !drop[n] && n !== title && !/^https?:\/\//.test(n) && n.indexOf("›") === -1; }); snippet = norm(kept.join(" ")); } seen[url] = true; out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); } return out; }; // ── headings mode (Google) ─────────────────────────────────────────────── var extractHeadings = function (root) { var out = []; var seen = {}; var headings = root.querySelectorAll("h3"); for (var i = 0; i < headings.length; i++) { var h3 = headings[i]; var anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]")); if (!anchor) continue; var url = unwrap(anchor.href); if (!acceptable(url) || seen[url]) continue; var title = norm(h3.innerText); if (!title) continue; var header = h3.closest("[data-snhf]") || anchor; var headerLines = {}; var hl = linesOf(header); for (var k = 0; k < hl.length; k++) headerLines[norm(hl[k])] = true; var baseline = norm(header.innerText || "").length; var container = header.parentElement; for (var depth = 0; depth < 6 && container && container !== root; depth++) { if (container.querySelectorAll("h3").length > 1) { container = null; break; } if (norm(container.innerText || "").length > baseline + 40) break; container = container.parentElement; } var snippet = ""; if (container && container !== root) { var explicit = container.querySelector("[data-sncf]"); var body = explicit ? linesOf(explicit) : linesOf(container).filter(function (l) { return !headerLines[norm(l)]; }); snippet = norm( body.filter(function (l) { return !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results"; }).join(" "), ).replace(/Read more$/, "").trim(); } seen[url] = true; out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); } return out; }; var challenge = null; var results = []; var container = "none"; try { challenge = detectChallenge(); } catch (e) { challenge = null; } try { var root = findRoot(); container = root ? "found" : "none"; if (root && !challenge) results = CFG.mode === "items" ? extractItems(root) : extractHeadings(root); } catch (e) { results = []; } return { url: location.href, title: document.title || "", readyState: document.readyState, challenge: challenge, container: container, results: results, }; })()`;