diff options
Diffstat (limited to 'src/extract.ts')
| -rw-r--r-- | src/extract.ts | 276 |
1 files changed, 194 insertions, 82 deletions
diff --git a/src/extract.ts b/src/extract.ts index c4d263a..282fa07 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -1,30 +1,45 @@ /** * The script that runs inside the search results page. * - * It is a string rather than a function because it is shipped to Chrome via - * `Runtime.evaluate` — nothing in here is typechecked, so it is kept small, - * defensive, and free of anything that could throw on an unexpected DOM. + * It is built as a string rather than shipped as a function because it goes to + * Chrome via `Runtime.evaluate` — nothing in here is typechecked, so it is kept + * defensive and free of anything that could throw on an unexpected DOM. The + * per-engine configuration is injected as JSON by {@link buildProbeScript}. * - * ## Why it does not select on class names + * ## Two modes, because the engines genuinely differ * - * Google's result classes (`MjjYud`, `kb0PBd`, `yuRUbf`, …) are generated and - * change without notice; the `data-` hooks (`data-snhf` for the title/source - * header, `data-sncf` for the description) are more stable but not promised - * either. So the extractor uses them when present and otherwise falls back to a - * structural walk that only assumes "an <h3> inside a link, with the - * description somewhere in a shared ancestor": + * **"items"** — DuckDuckGo, Bing and Brave each have a clean per-result + * container, so results are read directly out of it. * - * - anchor = nearest enclosing <a href> - * - header = nearest [data-snhf], else the anchor itself + * **"headings"** — Google has no stable result container; its class names + * (`MjjYud`, `kb0PBd`, `yuRUbf`) are generated and change without notice. So + * results are found from each `<h3>` outward: + * + * - anchor = nearest enclosing `<a href>` + * - header = nearest `[data-snhf]`, else the anchor * - container = climb from the header until the text grows past the header's, - * abandoning the climb if a second <h3> comes into scope - * (that would mean we had swallowed the next result) - * - snippet = [data-sncf] if present, else the container's lines minus the - * header's lines, minus URL/breadcrumb noise + * abandoning the climb if a second `<h3>` comes into scope + * (that would mean the next result had been swallowed) + * - snippet = `[data-sncf]` if present, else the container's lines minus the + * header's lines, minus URL and breadcrumb noise + * + * Verified against Google's classic SERP and its `udm=14` layout, DuckDuckGo's + * no-JS endpoint, Bing's `li.b_algo`, and Brave's `.snippet[data-type=web]`. + * + * ## Link unwrapping * - * Verified against both the classic SERP and the `udm=14` "Web" layout. + * DuckDuckGo and Bing both route outbound links through a redirector, so the + * raw href is useless to the agent. Both are unwrapped in the page, where the + * URL and base64 primitives already exist: + * + * - DuckDuckGo: `//duckduckgo.com/l/?uddg=<percent-encoded target>` + * - Bing: `//bing.com/ck/a?…&u=a1<base64url of target>` + * + * Google and Brave link straight out and need no unwrapping. */ +import type { ExtractionConfig } from "./engines.ts"; + /** One search hit, as the page script reports it. */ export type SearchResult = { title: string; @@ -41,113 +56,210 @@ export type PageProbe = { /** Non-null when a human check is in the way. */ challenge: { kind: string; detail: string } | null; /** - * Which results container the page rendered. "none" means Google served a - * shell with no results area at all — a different failure from an empty one, + * Whether the engine's results container was present at all. "none" means a + * page shell with no results area — a different failure from an empty one, * and worth telling apart when diagnosing a zero-hit search. */ - container: "rso" | "search" | "none"; + container: "found" | "none"; results: SearchResult[]; }; /** - * Detection order matters: a `/sorry/` interstitial also contains a recaptcha - * iframe, and reporting the specific page beats reporting the widget on it. + * Challenge detection, shared across engines. + * + * Every phrase here has been seen on a real page during development: Google's + * `/sorry/` interstitial, and Brave's "Verifying you're not a bot / Quick check + * before you continue searching" — the latter is why the wording list is broad + * rather than just matching Google's "unusual traffic". Ordering matters: a + * `/sorry/` page also contains a recaptcha iframe, and naming the page beats + * naming the widget sitting on it. */ -export const PROBE_SCRIPT = String.raw`(() => { - const norm = (s) => (s || "").replace(/\s+/g, " ").trim(); - const linesOf = (el) => - el && el.innerText ? el.innerText.split("\n").map((l) => l.trim()).filter(Boolean) : []; - - const bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : ""; - +const CHALLENGE_DETECTION = String.raw` const detectChallenge = () => { - const href = location.href; - if (/\/sorry\//.test(href)) { - return { kind: "google-block-page", detail: bodyText.slice(0, 300) }; - } - if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname)) { + if (/\/sorry\//.test(location.href)) return { kind: "google-block-page", detail: bodyText.slice(0, 300) }; + if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname)) return { kind: "consent-wall", detail: bodyText.slice(0, 300) }; - } - if (document.querySelector("#captcha-form, form#captcha-form")) { + if (document.querySelector("#captcha-form, form#captcha-form")) return { kind: "captcha-form", detail: bodyText.slice(0, 300) }; - } - if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[title*="challenge"]')) { - return { kind: "captcha-widget", detail: bodyText.slice(0, 300) }; - } - if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running")) { + if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running, [id^=cf-chl]")) return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) }; - } - if (/unusual traffic|are you a robot|verify (that )?you('| a)?re human|not a robot|automated queries/i.test(bodyText)) { + if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[src*="turnstile"], iframe[title*="challenge"]')) + return { kind: "captcha-widget", detail: bodyText.slice(0, 300) }; + if (/verifying (that )?you('| a)?re( not)? a? ?(human|bot|robot)|quick check before you continue|verify you are human|are you a robot|not a robot/i.test(bodyText)) return { kind: "bot-check", detail: bodyText.slice(0, 300) }; + if (/unusual traffic|automated queries|suspicious activity from your/i.test(bodyText)) + return { kind: "rate-limit-block", detail: bodyText.slice(0, 300) }; + return null; + }; +`; + +/** + * Build the probe for one engine. The configuration is embedded as a JSON + * literal so the script stays a single self-contained expression. + */ +export const buildProbeScript = (config: ExtractionConfig): string => String.raw`(() => { + var CFG = ${JSON.stringify(config)}; + + var norm = function (s) { return (s || "").replace(/\s+/g, " ").trim(); }; + var linesOf = function (el) { + return el && el.innerText ? el.innerText.split("\n").map(function (l) { return l.trim(); }).filter(Boolean) : []; + }; + var bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : ""; + var selfHost = new RegExp(CFG.selfHostPattern); + + ${CHALLENGE_DETECTION} + + // DuckDuckGo and Bing both hide the real destination behind a redirector. + var unwrap = function (href) { + try { + var u = new URL(href, location.href); + if (CFG.unwrap === "ddg") { + if (!/(^|\.)duckduckgo\.com$/.test(u.hostname)) return href; + return u.searchParams.get("uddg") || href; + } + if (CFG.unwrap === "bing") { + if (!/(^|\.)bing\.com$/.test(u.hostname)) return href; + var p = u.searchParams.get("u"); + if (!p) return href; + var b64 = p.replace(/^a1/, "").replace(/-/g, "+").replace(/_/g, "/"); + b64 += "=".repeat((4 - (b64.length % 4)) % 4); + var bin = atob(b64); + var bytes = new Uint8Array(bin.length); + for (var i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + return new TextDecoder().decode(bytes); + } + return href; + } catch (e) { + // An unwrap that fails leaves the redirector URL in place rather than + // dropping the result: a working link the agent has to follow twice beats + // no link at all. + return href; + } + }; + + var findRoot = function () { + for (var i = 0; i < CFG.roots.length; i++) { + var el = document.querySelector(CFG.roots[i]); + if (el) return el; } return null; }; - const extract = () => { - const root = document.querySelector("#rso") || document.querySelector("#search"); - if (!root) return []; - const out = []; - const seen = new Set(); + var acceptable = function (url) { + return /^https?:/.test(url) && !selfHost.test(url); + }; + + // ── items mode ─────────────────────────────────────────────────────────── + var extractItems = function (root) { + var out = []; + var seen = {}; + var items = root.querySelectorAll(CFG.item); + for (var i = 0; i < items.length; i++) { + var item = items[i]; + + var skip = false; + for (var x = 0; CFG.exclude && x < CFG.exclude.length; x++) { + if (item.matches(CFG.exclude[x]) || item.querySelector(CFG.exclude[x])) { skip = true; break; } + } + if (skip) continue; - const headings = root.querySelectorAll("h3"); - for (let i = 0; i < headings.length; i++) { - const h3 = headings[i]; - const anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]")); + var anchor = item.querySelector(CFG.link); if (!anchor) continue; + var url = unwrap(anchor.href); + if (!acceptable(url) || seen[url]) continue; - const url = anchor.href; - if (!/^https?:/.test(url)) continue; - // Drop Google's own links (image search, cached copies, "more results"). - if (/^https?:\/\/(www\.)?google\.[a-z.]+\//.test(url)) continue; - if (seen.has(url)) continue; + var titleEl = CFG.title ? item.querySelector(CFG.title) : null; + var title = norm(titleEl ? titleEl.innerText : anchor.innerText); + if (!title) continue; - const title = norm(h3.innerText); + var snippet = ""; + var snippetEl = CFG.snippet ? item.querySelector(CFG.snippet) : null; + if (snippetEl) { + snippet = norm(snippetEl.innerText); + } else { + // No dedicated description element: subtract the parts we can name + // (title, breadcrumb, source) and keep what is left. + var drop = {}; + for (var s = 0; CFG.subtract && s < CFG.subtract.length; s++) { + var parts = item.querySelectorAll(CFG.subtract[s]); + for (var p = 0; p < parts.length; p++) { + var pl = linesOf(parts[p]); + for (var q = 0; q < pl.length; q++) drop[norm(pl[q])] = true; + } + } + var kept = linesOf(item).filter(function (l) { + var n = norm(l); + return !drop[n] && n !== title && !/^https?:\/\//.test(n) && n.indexOf("›") === -1; + }); + snippet = norm(kept.join(" ")); + } + + seen[url] = true; + out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); + } + return out; + }; + + // ── headings mode (Google) ─────────────────────────────────────────────── + var extractHeadings = function (root) { + var out = []; + var seen = {}; + var headings = root.querySelectorAll("h3"); + for (var i = 0; i < headings.length; i++) { + var h3 = headings[i]; + var anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]")); + if (!anchor) continue; + + var url = unwrap(anchor.href); + if (!acceptable(url) || seen[url]) continue; + + var title = norm(h3.innerText); if (!title) continue; - const header = h3.closest("[data-snhf]") || anchor; - const headerLines = new Set(linesOf(header).map(norm)); - const baseline = norm(header.innerText || "").length; + var header = h3.closest("[data-snhf]") || anchor; + var headerLines = {}; + var hl = linesOf(header); + for (var k = 0; k < hl.length; k++) headerLines[norm(hl[k])] = true; + var baseline = norm(header.innerText || "").length; - let container = header.parentElement; - for (let depth = 0; depth < 6 && container && container !== root; depth++) { - if (container.querySelectorAll("h3").length > 1) { - container = null; - break; - } + var container = header.parentElement; + for (var depth = 0; depth < 6 && container && container !== root; depth++) { + if (container.querySelectorAll("h3").length > 1) { container = null; break; } if (norm(container.innerText || "").length > baseline + 40) break; container = container.parentElement; } - let snippet = ""; + var snippet = ""; if (container && container !== root) { - const explicit = container.querySelector("[data-sncf]"); - const body = explicit ? linesOf(explicit) : linesOf(container).filter((l) => !headerLines.has(norm(l))); + var explicit = container.querySelector("[data-sncf]"); + var body = explicit + ? linesOf(explicit) + : linesOf(container).filter(function (l) { return !headerLines[norm(l)]; }); snippet = norm( - body - .filter((l) => !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results") - .join(" "), - ) - .replace(/Read more$/, "") - .trim(); + body.filter(function (l) { + return !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results"; + }).join(" "), + ).replace(/Read more$/, "").trim(); } - seen.add(url); + seen[url] = true; out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); } return out; }; - let challenge = null; - let results = []; - let container = "none"; + var challenge = null; + var results = []; + var container = "none"; try { challenge = detectChallenge(); } catch (e) { challenge = null; } try { - container = document.querySelector("#rso") ? "rso" : document.querySelector("#search") ? "search" : "none"; - results = challenge ? [] : extract(); + var root = findRoot(); + container = root ? "found" : "none"; + if (root && !challenge) results = CFG.mode === "items" ? extractItems(root) : extractHeadings(root); } catch (e) { results = []; } |