1 files changed, 163 insertions, 0 deletions
diff --git a/src/extract.ts b/src/extract.ts
new file mode 100644
index 0000000..c4d263a
--- /dev/null
+++ b/src/extract.ts
@@ -0,0 +1,163 @@
+/**
+ * The script that runs inside the search results page.
+ *
+ * It is a string rather than a function because it is shipped to Chrome via
+ * `Runtime.evaluate` — nothing in here is typechecked, so it is kept small,
+ * defensive, and free of anything that could throw on an unexpected DOM.
+ *
+ * ## Why it does not select on class names
+ *
+ * Google's result classes (`MjjYud`, `kb0PBd`, `yuRUbf`, …) are generated and
+ * change without notice; the `data-` hooks (`data-snhf` for the title/source
+ * header, `data-sncf` for the description) are more stable but not promised
+ * either. So the extractor uses them when present and otherwise falls back to a
+ * structural walk that only assumes "an <h3> inside a link, with the
+ * description somewhere in a shared ancestor":
+ *
+ *   - anchor  = nearest enclosing <a href>
+ *   - header  = nearest [data-snhf], else the anchor itself
+ *   - container = climb from the header until the text grows past the header's,
+ *                 abandoning the climb if a second <h3> comes into scope
+ *                 (that would mean we had swallowed the next result)
+ *   - snippet = [data-sncf] if present, else the container's lines minus the
+ *               header's lines, minus URL/breadcrumb noise
+ *
+ * Verified against both the classic SERP and the `udm=14` "Web" layout.
+ */
+
+/** One search hit, as the page script reports it. */
+export type SearchResult = {
+  title: string;
+  url: string;
+  snippet: string;
+};
+
+/** What the page script returns in a single round trip. */
+export type PageProbe = {
+  /** Where the page actually ended up — redirects and consent walls move it. */
+  url: string;
+  title: string;
+  readyState: string;
+  /** Non-null when a human check is in the way. */
+  challenge: { kind: string; detail: string } | null;
+  /**
+   * Which results container the page rendered. "none" means Google served a
+   * shell with no results area at all — a different failure from an empty one,
+   * and worth telling apart when diagnosing a zero-hit search.
+   */
+  container: "rso" | "search" | "none";
+  results: SearchResult[];
+};
+
+/**
+ * Detection order matters: a `/sorry/` interstitial also contains a recaptcha
+ * iframe, and reporting the specific page beats reporting the widget on it.
+ */
+export const PROBE_SCRIPT = String.raw`(() => {
+  const norm = (s) => (s || "").replace(/\s+/g, " ").trim();
+  const linesOf = (el) =>
+    el && el.innerText ? el.innerText.split("\n").map((l) => l.trim()).filter(Boolean) : [];
+
+  const bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : "";
+
+  const detectChallenge = () => {
+    const href = location.href;
+    if (/\/sorry\//.test(href)) {
+      return { kind: "google-block-page", detail: bodyText.slice(0, 300) };
+    }
+    if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname)) {
+      return { kind: "consent-wall", detail: bodyText.slice(0, 300) };
+    }
+    if (document.querySelector("#captcha-form, form#captcha-form")) {
+      return { kind: "captcha-form", detail: bodyText.slice(0, 300) };
+    }
+    if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[title*="challenge"]')) {
+      return { kind: "captcha-widget", detail: bodyText.slice(0, 300) };
+    }
+    if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running")) {
+      return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) };
+    }
+    if (/unusual traffic|are you a robot|verify (that )?you('| a)?re human|not a robot|automated queries/i.test(bodyText)) {
+      return { kind: "bot-check", detail: bodyText.slice(0, 300) };
+    }
+    return null;
+  };
+
+  const extract = () => {
+    const root = document.querySelector("#rso") || document.querySelector("#search");
+    if (!root) return [];
+    const out = [];
+    const seen = new Set();
+
+    const headings = root.querySelectorAll("h3");
+    for (let i = 0; i < headings.length; i++) {
+      const h3 = headings[i];
+      const anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]"));
+      if (!anchor) continue;
+
+      const url = anchor.href;
+      if (!/^https?:/.test(url)) continue;
+      // Drop Google's own links (image search, cached copies, "more results").
+      if (/^https?:\/\/(www\.)?google\.[a-z.]+\//.test(url)) continue;
+      if (seen.has(url)) continue;
+
+      const title = norm(h3.innerText);
+      if (!title) continue;
+
+      const header = h3.closest("[data-snhf]") || anchor;
+      const headerLines = new Set(linesOf(header).map(norm));
+      const baseline = norm(header.innerText || "").length;
+
+      let container = header.parentElement;
+      for (let depth = 0; depth < 6 && container && container !== root; depth++) {
+        if (container.querySelectorAll("h3").length > 1) {
+          container = null;
+          break;
+        }
+        if (norm(container.innerText || "").length > baseline + 40) break;
+        container = container.parentElement;
+      }
+
+      let snippet = "";
+      if (container && container !== root) {
+        const explicit = container.querySelector("[data-sncf]");
+        const body = explicit ? linesOf(explicit) : linesOf(container).filter((l) => !headerLines.has(norm(l)));
+        snippet = norm(
+          body
+            .filter((l) => !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results")
+            .join(" "),
+        )
+          .replace(/Read more$/, "")
+          .trim();
+      }
+
+      seen.add(url);
+      out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
+    }
+    return out;
+  };
+
+  let challenge = null;
+  let results = [];
+  let container = "none";
+  try {
+    challenge = detectChallenge();
+  } catch (e) {
+    challenge = null;
+  }
+  try {
+    container = document.querySelector("#rso") ? "rso" : document.querySelector("#search") ? "search" : "none";
+    results = challenge ? [] : extract();
+  } catch (e) {
+    results = [];
+  }
+
+  return {
+    url: location.href,
+    title: document.title || "",
+    readyState: document.readyState,
+    challenge: challenge,
+    container: container,
+    results: results,
+  };
+})()`;