From db69207c06e8d5233bff4996e3ef5b43332d533f Mon Sep 17 00:00:00 2001 From: Igor Soarez Date: Mon, 3 Aug 2026 21:43:56 +0100 Subject: Support DuckDuckGo, Bing and Brave, switchable like the CDP host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine resolution mirrors the browser target: CCS_SEARCH_ENGINE, then a /search-engine choice persisted machine-wide, then Google. A per-call `engine` parameter sits above both so the agent can fall back when one engine starts serving captchas — the one case where the model, not the operator, has to make the call. Unlike the CDP target there is no safety argument for the environment winning: driving the wrong browser means automating someone's signed-in Chrome, choosing a different index does not. An unsupported recency window is refused, naming the engines that support it, rather than dropped. Silently returning unfiltered results is indistinguishable from success, which is the failure this whole design is trying to avoid. Extraction grows a second mode. DuckDuckGo, Bing and Brave have clean per-result containers; Google does not, so its heading-walk stays as its own path rather than being bent into the item shape. DuckDuckGo and Bing route links through redirectors, unwrapped in the page. Third silent-failure trap found, alongside Google's two: on Bing, `count` cancels `filters`. With ex1:"ez1" alone every result is hours old; add count in either order and months-old results return, looking perfectly ordinary. Bing now drops count whenever a date filter is present. Challenge detection widened to Brave's "Verifying you're not a bot" and "Quick check before you continue searching", which the previous Google- shaped matcher missed entirely — found by tripping it. --- src/extract.ts | 276 ++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 194 insertions(+), 82 deletions(-) (limited to 'src/extract.ts') diff --git a/src/extract.ts b/src/extract.ts index c4d263a..282fa07 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -1,30 +1,45 @@ /** * The script that runs inside the search results page. * - * It is a string rather than a function because it is shipped to Chrome via - * `Runtime.evaluate` — nothing in here is typechecked, so it is kept small, - * defensive, and free of anything that could throw on an unexpected DOM. + * It is built as a string rather than shipped as a function because it goes to + * Chrome via `Runtime.evaluate` — nothing in here is typechecked, so it is kept + * defensive and free of anything that could throw on an unexpected DOM. The + * per-engine configuration is injected as JSON by {@link buildProbeScript}. * - * ## Why it does not select on class names + * ## Two modes, because the engines genuinely differ * - * Google's result classes (`MjjYud`, `kb0PBd`, `yuRUbf`, …) are generated and - * change without notice; the `data-` hooks (`data-snhf` for the title/source - * header, `data-sncf` for the description) are more stable but not promised - * either. So the extractor uses them when present and otherwise falls back to a - * structural walk that only assumes "an

inside a link, with the - * description somewhere in a shared ancestor": + * **"items"** — DuckDuckGo, Bing and Brave each have a clean per-result + * container, so results are read directly out of it. * - * - anchor = nearest enclosing - * - header = nearest [data-snhf], else the anchor itself + * **"headings"** — Google has no stable result container; its class names + * (`MjjYud`, `kb0PBd`, `yuRUbf`) are generated and change without notice. So + * results are found from each `

` outward: + * + * - anchor = nearest enclosing `` + * - header = nearest `[data-snhf]`, else the anchor * - container = climb from the header until the text grows past the header's, - * abandoning the climb if a second

comes into scope - * (that would mean we had swallowed the next result) - * - snippet = [data-sncf] if present, else the container's lines minus the - * header's lines, minus URL/breadcrumb noise + * abandoning the climb if a second `

` comes into scope + * (that would mean the next result had been swallowed) + * - snippet = `[data-sncf]` if present, else the container's lines minus the + * header's lines, minus URL and breadcrumb noise + * + * Verified against Google's classic SERP and its `udm=14` layout, DuckDuckGo's + * no-JS endpoint, Bing's `li.b_algo`, and Brave's `.snippet[data-type=web]`. + * + * ## Link unwrapping * - * Verified against both the classic SERP and the `udm=14` "Web" layout. + * DuckDuckGo and Bing both route outbound links through a redirector, so the + * raw href is useless to the agent. Both are unwrapped in the page, where the + * URL and base64 primitives already exist: + * + * - DuckDuckGo: `//duckduckgo.com/l/?uddg=` + * - Bing: `//bing.com/ck/a?…&u=a1` + * + * Google and Brave link straight out and need no unwrapping. */ +import type { ExtractionConfig } from "./engines.ts"; + /** One search hit, as the page script reports it. */ export type SearchResult = { title: string; @@ -41,113 +56,210 @@ export type PageProbe = { /** Non-null when a human check is in the way. */ challenge: { kind: string; detail: string } | null; /** - * Which results container the page rendered. "none" means Google served a - * shell with no results area at all — a different failure from an empty one, + * Whether the engine's results container was present at all. "none" means a + * page shell with no results area — a different failure from an empty one, * and worth telling apart when diagnosing a zero-hit search. */ - container: "rso" | "search" | "none"; + container: "found" | "none"; results: SearchResult[]; }; /** - * Detection order matters: a `/sorry/` interstitial also contains a recaptcha - * iframe, and reporting the specific page beats reporting the widget on it. + * Challenge detection, shared across engines. + * + * Every phrase here has been seen on a real page during development: Google's + * `/sorry/` interstitial, and Brave's "Verifying you're not a bot / Quick check + * before you continue searching" — the latter is why the wording list is broad + * rather than just matching Google's "unusual traffic". Ordering matters: a + * `/sorry/` page also contains a recaptcha iframe, and naming the page beats + * naming the widget sitting on it. */ -export const PROBE_SCRIPT = String.raw`(() => { - const norm = (s) => (s || "").replace(/\s+/g, " ").trim(); - const linesOf = (el) => - el && el.innerText ? el.innerText.split("\n").map((l) => l.trim()).filter(Boolean) : []; - - const bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : ""; - +const CHALLENGE_DETECTION = String.raw` const detectChallenge = () => { - const href = location.href; - if (/\/sorry\//.test(href)) { - return { kind: "google-block-page", detail: bodyText.slice(0, 300) }; - } - if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname)) { + if (/\/sorry\//.test(location.href)) return { kind: "google-block-page", detail: bodyText.slice(0, 300) }; + if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname)) return { kind: "consent-wall", detail: bodyText.slice(0, 300) }; - } - if (document.querySelector("#captcha-form, form#captcha-form")) { + if (document.querySelector("#captcha-form, form#captcha-form")) return { kind: "captcha-form", detail: bodyText.slice(0, 300) }; - } - if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[title*="challenge"]')) { - return { kind: "captcha-widget", detail: bodyText.slice(0, 300) }; - } - if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running")) { + if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running, [id^=cf-chl]")) return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) }; - } - if (/unusual traffic|are you a robot|verify (that )?you('| a)?re human|not a robot|automated queries/i.test(bodyText)) { + if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[src*="turnstile"], iframe[title*="challenge"]')) + return { kind: "captcha-widget", detail: bodyText.slice(0, 300) }; + if (/verifying (that )?you('| a)?re( not)? a? ?(human|bot|robot)|quick check before you continue|verify you are human|are you a robot|not a robot/i.test(bodyText)) return { kind: "bot-check", detail: bodyText.slice(0, 300) }; + if (/unusual traffic|automated queries|suspicious activity from your/i.test(bodyText)) + return { kind: "rate-limit-block", detail: bodyText.slice(0, 300) }; + return null; + }; +`; + +/** + * Build the probe for one engine. The configuration is embedded as a JSON + * literal so the script stays a single self-contained expression. + */ +export const buildProbeScript = (config: ExtractionConfig): string => String.raw`(() => { + var CFG = ${JSON.stringify(config)}; + + var norm = function (s) { return (s || "").replace(/\s+/g, " ").trim(); }; + var linesOf = function (el) { + return el && el.innerText ? el.innerText.split("\n").map(function (l) { return l.trim(); }).filter(Boolean) : []; + }; + var bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : ""; + var selfHost = new RegExp(CFG.selfHostPattern); + + ${CHALLENGE_DETECTION} + + // DuckDuckGo and Bing both hide the real destination behind a redirector. + var unwrap = function (href) { + try { + var u = new URL(href, location.href); + if (CFG.unwrap === "ddg") { + if (!/(^|\.)duckduckgo\.com$/.test(u.hostname)) return href; + return u.searchParams.get("uddg") || href; + } + if (CFG.unwrap === "bing") { + if (!/(^|\.)bing\.com$/.test(u.hostname)) return href; + var p = u.searchParams.get("u"); + if (!p) return href; + var b64 = p.replace(/^a1/, "").replace(/-/g, "+").replace(/_/g, "/"); + b64 += "=".repeat((4 - (b64.length % 4)) % 4); + var bin = atob(b64); + var bytes = new Uint8Array(bin.length); + for (var i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + return new TextDecoder().decode(bytes); + } + return href; + } catch (e) { + // An unwrap that fails leaves the redirector URL in place rather than + // dropping the result: a working link the agent has to follow twice beats + // no link at all. + return href; + } + }; + + var findRoot = function () { + for (var i = 0; i < CFG.roots.length; i++) { + var el = document.querySelector(CFG.roots[i]); + if (el) return el; } return null; }; - const extract = () => { - const root = document.querySelector("#rso") || document.querySelector("#search"); - if (!root) return []; - const out = []; - const seen = new Set(); + var acceptable = function (url) { + return /^https?:/.test(url) && !selfHost.test(url); + }; + + // ── items mode ─────────────────────────────────────────────────────────── + var extractItems = function (root) { + var out = []; + var seen = {}; + var items = root.querySelectorAll(CFG.item); + for (var i = 0; i < items.length; i++) { + var item = items[i]; + + var skip = false; + for (var x = 0; CFG.exclude && x < CFG.exclude.length; x++) { + if (item.matches(CFG.exclude[x]) || item.querySelector(CFG.exclude[x])) { skip = true; break; } + } + if (skip) continue; - const headings = root.querySelectorAll("h3"); - for (let i = 0; i < headings.length; i++) { - const h3 = headings[i]; - const anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]")); + var anchor = item.querySelector(CFG.link); if (!anchor) continue; + var url = unwrap(anchor.href); + if (!acceptable(url) || seen[url]) continue; - const url = anchor.href; - if (!/^https?:/.test(url)) continue; - // Drop Google's own links (image search, cached copies, "more results"). - if (/^https?:\/\/(www\.)?google\.[a-z.]+\//.test(url)) continue; - if (seen.has(url)) continue; + var titleEl = CFG.title ? item.querySelector(CFG.title) : null; + var title = norm(titleEl ? titleEl.innerText : anchor.innerText); + if (!title) continue; - const title = norm(h3.innerText); + var snippet = ""; + var snippetEl = CFG.snippet ? item.querySelector(CFG.snippet) : null; + if (snippetEl) { + snippet = norm(snippetEl.innerText); + } else { + // No dedicated description element: subtract the parts we can name + // (title, breadcrumb, source) and keep what is left. + var drop = {}; + for (var s = 0; CFG.subtract && s < CFG.subtract.length; s++) { + var parts = item.querySelectorAll(CFG.subtract[s]); + for (var p = 0; p < parts.length; p++) { + var pl = linesOf(parts[p]); + for (var q = 0; q < pl.length; q++) drop[norm(pl[q])] = true; + } + } + var kept = linesOf(item).filter(function (l) { + var n = norm(l); + return !drop[n] && n !== title && !/^https?:\/\//.test(n) && n.indexOf("›") === -1; + }); + snippet = norm(kept.join(" ")); + } + + seen[url] = true; + out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); + } + return out; + }; + + // ── headings mode (Google) ─────────────────────────────────────────────── + var extractHeadings = function (root) { + var out = []; + var seen = {}; + var headings = root.querySelectorAll("h3"); + for (var i = 0; i < headings.length; i++) { + var h3 = headings[i]; + var anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]")); + if (!anchor) continue; + + var url = unwrap(anchor.href); + if (!acceptable(url) || seen[url]) continue; + + var title = norm(h3.innerText); if (!title) continue; - const header = h3.closest("[data-snhf]") || anchor; - const headerLines = new Set(linesOf(header).map(norm)); - const baseline = norm(header.innerText || "").length; + var header = h3.closest("[data-snhf]") || anchor; + var headerLines = {}; + var hl = linesOf(header); + for (var k = 0; k < hl.length; k++) headerLines[norm(hl[k])] = true; + var baseline = norm(header.innerText || "").length; - let container = header.parentElement; - for (let depth = 0; depth < 6 && container && container !== root; depth++) { - if (container.querySelectorAll("h3").length > 1) { - container = null; - break; - } + var container = header.parentElement; + for (var depth = 0; depth < 6 && container && container !== root; depth++) { + if (container.querySelectorAll("h3").length > 1) { container = null; break; } if (norm(container.innerText || "").length > baseline + 40) break; container = container.parentElement; } - let snippet = ""; + var snippet = ""; if (container && container !== root) { - const explicit = container.querySelector("[data-sncf]"); - const body = explicit ? linesOf(explicit) : linesOf(container).filter((l) => !headerLines.has(norm(l))); + var explicit = container.querySelector("[data-sncf]"); + var body = explicit + ? linesOf(explicit) + : linesOf(container).filter(function (l) { return !headerLines[norm(l)]; }); snippet = norm( - body - .filter((l) => !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results") - .join(" "), - ) - .replace(/Read more$/, "") - .trim(); + body.filter(function (l) { + return !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results"; + }).join(" "), + ).replace(/Read more$/, "").trim(); } - seen.add(url); + seen[url] = true; out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); } return out; }; - let challenge = null; - let results = []; - let container = "none"; + var challenge = null; + var results = []; + var container = "none"; try { challenge = detectChallenge(); } catch (e) { challenge = null; } try { - container = document.querySelector("#rso") ? "rso" : document.querySelector("#search") ? "search" : "none"; - results = challenge ? [] : extract(); + var root = findRoot(); + container = root ? "found" : "none"; + if (root && !challenge) results = CFG.mode === "items" ? extractItems(root) : extractHeadings(root); } catch (e) { results = []; } -- cgit v1.3.1