path: root/src/extract.ts

/**
 * The script that runs inside the search results page.
 *
 * It is built as a string rather than shipped as a function because it goes to
 * Chrome via `Runtime.evaluate` — nothing in here is typechecked, so it is kept
 * defensive and free of anything that could throw on an unexpected DOM. The
 * per-engine configuration is injected as JSON by {@link buildProbeScript}.
 *
 * ## Two modes, because the engines genuinely differ
 *
 * **"items"** — DuckDuckGo, Bing and Brave each have a clean per-result
 * container, so results are read directly out of it.
 *
 * **"headings"** — Google has no stable result container; its class names
 * (`MjjYud`, `kb0PBd`, `yuRUbf`) are generated and change without notice. So
 * results are found from each `<h3>` outward:
 *
 *   - anchor    = nearest enclosing `<a href>`
 *   - header    = nearest `[data-snhf]`, else the anchor
 *   - container = climb from the header until the text grows past the header's,
 *                 abandoning the climb if a second `<h3>` comes into scope
 *                 (that would mean the next result had been swallowed)
 *   - snippet   = `[data-sncf]` if present, else the container's lines minus the
 *                 header's lines, minus URL and breadcrumb noise
 *
 * Verified against Google's classic SERP and its `udm=14` layout, DuckDuckGo's
 * no-JS endpoint, Bing's `li.b_algo`, and Brave's `.snippet[data-type=web]`.
 *
 * ## Link unwrapping
 *
 * DuckDuckGo and Bing both route outbound links through a redirector, so the
 * raw href is useless to the agent. Both are unwrapped in the page, where the
 * URL and base64 primitives already exist:
 *
 *   - DuckDuckGo: `//duckduckgo.com/l/?uddg=<percent-encoded target>`
 *   - Bing:       `//bing.com/ck/a?…&u=a1<base64url of target>`
 *
 * Google and Brave link straight out and need no unwrapping.
 */

import type { ExtractionConfig } from "./engines.ts";

/** One search hit, as the page script reports it. */
export type SearchResult = {
  title: string;
  url: string;
  snippet: string;
};

/** What the page script returns in a single round trip. */
export type PageProbe = {
  /** Where the page actually ended up — redirects and consent walls move it. */
  url: string;
  title: string;
  readyState: string;
  /** Non-null when a human check is in the way. */
  challenge: { kind: string; detail: string } | null;
  /**
   * Whether the engine's results container was present at all. "none" means a
   * page shell with no results area — a different failure from an empty one,
   * and worth telling apart when diagnosing a zero-hit search.
   */
  container: "found" | "none";
  results: SearchResult[];
};

/**
 * Challenge detection, shared across engines.
 *
 * Every phrase here has been seen on a real page during development: Google's
 * `/sorry/` interstitial, and Brave's "Verifying you're not a bot / Quick check
 * before you continue searching" — the latter is why the wording list is broad
 * rather than just matching Google's "unusual traffic". Ordering matters: a
 * `/sorry/` page also contains a recaptcha iframe, and naming the page beats
 * naming the widget sitting on it.
 */
const CHALLENGE_DETECTION = String.raw`
  const detectChallenge = () => {
    if (/\/sorry\//.test(location.href)) return { kind: "google-block-page", detail: bodyText.slice(0, 300) };
    if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname))
      return { kind: "consent-wall", detail: bodyText.slice(0, 300) };
    if (document.querySelector("#captcha-form, form#captcha-form"))
      return { kind: "captcha-form", detail: bodyText.slice(0, 300) };
    if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running, [id^=cf-chl]"))
      return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) };
    if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[src*="turnstile"], iframe[title*="challenge"]'))
      return { kind: "captcha-widget", detail: bodyText.slice(0, 300) };
    if (/verifying (that )?you('| a)?re( not)? a? ?(human|bot|robot)|quick check before you continue|verify you are human|are you a robot|not a robot/i.test(bodyText))
      return { kind: "bot-check", detail: bodyText.slice(0, 300) };
    if (/unusual traffic|automated queries|suspicious activity from your/i.test(bodyText))
      return { kind: "rate-limit-block", detail: bodyText.slice(0, 300) };
    return null;
  };
`;

/**
 * Build the probe for one engine. The configuration is embedded as a JSON
 * literal so the script stays a single self-contained expression.
 */
export const buildProbeScript = (config: ExtractionConfig): string => String.raw`(() => {
  var CFG = ${JSON.stringify(config)};

  var norm = function (s) { return (s || "").replace(/\s+/g, " ").trim(); };
  var linesOf = function (el) {
    return el && el.innerText ? el.innerText.split("\n").map(function (l) { return l.trim(); }).filter(Boolean) : [];
  };
  var bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : "";
  var selfHost = new RegExp(CFG.selfHostPattern);

  ${CHALLENGE_DETECTION}

  // DuckDuckGo and Bing both hide the real destination behind a redirector.
  var unwrap = function (href) {
    try {
      var u = new URL(href, location.href);
      if (CFG.unwrap === "ddg") {
        if (!/(^|\.)duckduckgo\.com$/.test(u.hostname)) return href;
        return u.searchParams.get("uddg") || href;
      }
      if (CFG.unwrap === "bing") {
        if (!/(^|\.)bing\.com$/.test(u.hostname)) return href;
        var p = u.searchParams.get("u");
        if (!p) return href;
        var b64 = p.replace(/^a1/, "").replace(/-/g, "+").replace(/_/g, "/");
        b64 += "=".repeat((4 - (b64.length % 4)) % 4);
        var bin = atob(b64);
        var bytes = new Uint8Array(bin.length);
        for (var i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
        return new TextDecoder().decode(bytes);
      }
      return href;
    } catch (e) {
      // An unwrap that fails leaves the redirector URL in place rather than
      // dropping the result: a working link the agent has to follow twice beats
      // no link at all.
      return href;
    }
  };

  var findRoot = function () {
    for (var i = 0; i < CFG.roots.length; i++) {
      var el = document.querySelector(CFG.roots[i]);
      if (el) return el;
    }
    return null;
  };

  var acceptable = function (url) {
    return /^https?:/.test(url) && !selfHost.test(url);
  };

  // ── items mode ───────────────────────────────────────────────────────────
  var extractItems = function (root) {
    var out = [];
    var seen = {};
    var items = root.querySelectorAll(CFG.item);
    for (var i = 0; i < items.length; i++) {
      var item = items[i];

      var skip = false;
      for (var x = 0; CFG.exclude && x < CFG.exclude.length; x++) {
        if (item.matches(CFG.exclude[x]) || item.querySelector(CFG.exclude[x])) { skip = true; break; }
      }
      if (skip) continue;

      var anchor = item.querySelector(CFG.link);
      if (!anchor) continue;
      var url = unwrap(anchor.href);
      if (!acceptable(url) || seen[url]) continue;

      var titleEl = CFG.title ? item.querySelector(CFG.title) : null;
      var title = norm(titleEl ? titleEl.innerText : anchor.innerText);
      if (!title) continue;

      var snippet = "";
      var snippetEl = CFG.snippet ? item.querySelector(CFG.snippet) : null;
      if (snippetEl) {
        snippet = norm(snippetEl.innerText);
      } else {
        // No dedicated description element. Rather than guess at class names
        // for the source and breadcrumb, use their position: an engine that
        // renders "source / breadcrumb / title / description" puts every piece
        // of metadata *before* the title, so everything after the title line is
        // the description. Class names churn; that ordering does not.
        var itemLines = linesOf(item);
        var titleIndex = -1;
        for (var t = 0; t < itemLines.length; t++) {
          if (norm(itemLines[t]) === title) { titleIndex = t; break; }
        }

        var candidate;
        if (titleIndex >= 0) {
          candidate = itemLines.slice(titleIndex + 1);
        } else {
          // Title is not its own line (it may be inline with other text).
          // Fall back to subtracting the parts we can name.
          var drop = {};
          for (var s = 0; CFG.subtract && s < CFG.subtract.length; s++) {
            var parts = item.querySelectorAll(CFG.subtract[s]);
            for (var p = 0; p < parts.length; p++) {
              var pl = linesOf(parts[p]);
              for (var q = 0; q < pl.length; q++) drop[norm(pl[q])] = true;
            }
          }
          candidate = itemLines.filter(function (l) { return !drop[norm(l)]; });
        }

        snippet = norm(
          candidate.filter(function (l) {
            var n = norm(l);
            return n !== title && !/^https?:\/\//.test(n) && n.indexOf("›") === -1;
          }).join(" "),
        );
      }

      seen[url] = true;
      out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
    }
    return out;
  };

  // ── headings mode (Google) ───────────────────────────────────────────────
  var extractHeadings = function (root) {
    var out = [];
    var seen = {};
    var headings = root.querySelectorAll("h3");
    for (var i = 0; i < headings.length; i++) {
      var h3 = headings[i];
      var anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]"));
      if (!anchor) continue;

      var url = unwrap(anchor.href);
      if (!acceptable(url) || seen[url]) continue;

      var title = norm(h3.innerText);
      if (!title) continue;

      var header = h3.closest("[data-snhf]") || anchor;
      var headerLines = {};
      var hl = linesOf(header);
      for (var k = 0; k < hl.length; k++) headerLines[norm(hl[k])] = true;
      var baseline = norm(header.innerText || "").length;

      var container = header.parentElement;
      for (var depth = 0; depth < 6 && container && container !== root; depth++) {
        if (container.querySelectorAll("h3").length > 1) { container = null; break; }
        if (norm(container.innerText || "").length > baseline + 40) break;
        container = container.parentElement;
      }

      var snippet = "";
      if (container && container !== root) {
        var explicit = container.querySelector("[data-sncf]");
        var body = explicit
          ? linesOf(explicit)
          : linesOf(container).filter(function (l) { return !headerLines[norm(l)]; });
        snippet = norm(
          body.filter(function (l) {
            return !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results";
          }).join(" "),
        ).replace(/Read more$/, "").trim();
      }

      seen[url] = true;
      out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
    }
    return out;
  };

  var challenge = null;
  var results = [];
  var container = "none";
  try {
    challenge = detectChallenge();
  } catch (e) {
    challenge = null;
  }
  try {
    var root = findRoot();
    container = root ? "found" : "none";
    if (root && !challenge) results = CFG.mode === "items" ? extractItems(root) : extractHeadings(root);
  } catch (e) {
    results = [];
  }

  return {
    url: location.href,
    title: document.title || "",
    readyState: document.readyState,
    challenge: challenge,
    container: container,
    results: results,
  };
})()`;