From 495de0d5283dd3e4a6ef715b596c4a2892e95915 Mon Sep 17 00:00:00 2001 From: Igor Soarez Date: Mon, 3 Aug 2026 21:18:02 +0100 Subject: Web search for pi through an existing Chrome over CDP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Attaches to a browser that is already running — never launches one — using the same endpoint configuration as pi-browser-harness, so a single /browser-target choice governs both packages. One tool, castle_cdp_search, deliberately not named web_search so it coexists with pi-web-access rather than shadowing it. Notes from validating against castle's Chrome: - tbs=qdr:*, the parameter Google's own Tools menu writes, renders an empty page on this profile; the older as_qdr=* works. Any date filter combined with udm=14 is also empty, so recency drops udm. - Target.createTarget must not be raced against the abort signal: raceAbort abandons the promise but cannot cancel the command, and the command's side effect is a tab nothing is left holding. - A search cancelled while queued has to be removed from the semaphore queue, or the slot handed to it later is never counted back. - A decaying rate-limit block stops serving /sorry/ and returns an empty results page instead, indistinguishable from a genuine zero-hit search. --- src/extract.ts | 163 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 src/extract.ts (limited to 'src/extract.ts') diff --git a/src/extract.ts b/src/extract.ts new file mode 100644 index 0000000..c4d263a --- /dev/null +++ b/src/extract.ts @@ -0,0 +1,163 @@ +/** + * The script that runs inside the search results page. + * + * It is a string rather than a function because it is shipped to Chrome via + * `Runtime.evaluate` — nothing in here is typechecked, so it is kept small, + * defensive, and free of anything that could throw on an unexpected DOM. + * + * ## Why it does not select on class names + * + * Google's result classes (`MjjYud`, `kb0PBd`, `yuRUbf`, …) are generated and + * change without notice; the `data-` hooks (`data-snhf` for the title/source + * header, `data-sncf` for the description) are more stable but not promised + * either. So the extractor uses them when present and otherwise falls back to a + * structural walk that only assumes "an

inside a link, with the + * description somewhere in a shared ancestor": + * + * - anchor = nearest enclosing + * - header = nearest [data-snhf], else the anchor itself + * - container = climb from the header until the text grows past the header's, + * abandoning the climb if a second

comes into scope + * (that would mean we had swallowed the next result) + * - snippet = [data-sncf] if present, else the container's lines minus the + * header's lines, minus URL/breadcrumb noise + * + * Verified against both the classic SERP and the `udm=14` "Web" layout. + */ + +/** One search hit, as the page script reports it. */ +export type SearchResult = { + title: string; + url: string; + snippet: string; +}; + +/** What the page script returns in a single round trip. */ +export type PageProbe = { + /** Where the page actually ended up — redirects and consent walls move it. */ + url: string; + title: string; + readyState: string; + /** Non-null when a human check is in the way. */ + challenge: { kind: string; detail: string } | null; + /** + * Which results container the page rendered. "none" means Google served a + * shell with no results area at all — a different failure from an empty one, + * and worth telling apart when diagnosing a zero-hit search. + */ + container: "rso" | "search" | "none"; + results: SearchResult[]; +}; + +/** + * Detection order matters: a `/sorry/` interstitial also contains a recaptcha + * iframe, and reporting the specific page beats reporting the widget on it. + */ +export const PROBE_SCRIPT = String.raw`(() => { + const norm = (s) => (s || "").replace(/\s+/g, " ").trim(); + const linesOf = (el) => + el && el.innerText ? el.innerText.split("\n").map((l) => l.trim()).filter(Boolean) : []; + + const bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : ""; + + const detectChallenge = () => { + const href = location.href; + if (/\/sorry\//.test(href)) { + return { kind: "google-block-page", detail: bodyText.slice(0, 300) }; + } + if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname)) { + return { kind: "consent-wall", detail: bodyText.slice(0, 300) }; + } + if (document.querySelector("#captcha-form, form#captcha-form")) { + return { kind: "captcha-form", detail: bodyText.slice(0, 300) }; + } + if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[title*="challenge"]')) { + return { kind: "captcha-widget", detail: bodyText.slice(0, 300) }; + } + if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running")) { + return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) }; + } + if (/unusual traffic|are you a robot|verify (that )?you('| a)?re human|not a robot|automated queries/i.test(bodyText)) { + return { kind: "bot-check", detail: bodyText.slice(0, 300) }; + } + return null; + }; + + const extract = () => { + const root = document.querySelector("#rso") || document.querySelector("#search"); + if (!root) return []; + const out = []; + const seen = new Set(); + + const headings = root.querySelectorAll("h3"); + for (let i = 0; i < headings.length; i++) { + const h3 = headings[i]; + const anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]")); + if (!anchor) continue; + + const url = anchor.href; + if (!/^https?:/.test(url)) continue; + // Drop Google's own links (image search, cached copies, "more results"). + if (/^https?:\/\/(www\.)?google\.[a-z.]+\//.test(url)) continue; + if (seen.has(url)) continue; + + const title = norm(h3.innerText); + if (!title) continue; + + const header = h3.closest("[data-snhf]") || anchor; + const headerLines = new Set(linesOf(header).map(norm)); + const baseline = norm(header.innerText || "").length; + + let container = header.parentElement; + for (let depth = 0; depth < 6 && container && container !== root; depth++) { + if (container.querySelectorAll("h3").length > 1) { + container = null; + break; + } + if (norm(container.innerText || "").length > baseline + 40) break; + container = container.parentElement; + } + + let snippet = ""; + if (container && container !== root) { + const explicit = container.querySelector("[data-sncf]"); + const body = explicit ? linesOf(explicit) : linesOf(container).filter((l) => !headerLines.has(norm(l))); + snippet = norm( + body + .filter((l) => !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results") + .join(" "), + ) + .replace(/Read more$/, "") + .trim(); + } + + seen.add(url); + out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); + } + return out; + }; + + let challenge = null; + let results = []; + let container = "none"; + try { + challenge = detectChallenge(); + } catch (e) { + challenge = null; + } + try { + container = document.querySelector("#rso") ? "rso" : document.querySelector("#search") ? "search" : "none"; + results = challenge ? [] : extract(); + } catch (e) { + results = []; + } + + return { + url: location.href, + title: document.title || "", + readyState: document.readyState, + challenge: challenge, + container: container, + results: results, + }; +})()`; -- cgit v1.3.1