1 files changed, 294 insertions, 0 deletions
diff --git a/castle-search/src/extract.ts b/castle-search/src/extract.ts
new file mode 100644
index 0000000..8b5c57f
--- /dev/null
+++ b/castle-search/src/extract.ts
@@ -0,0 +1,294 @@
+/**
+ * The script that runs inside the search results page.
+ *
+ * It is built as a string rather than shipped as a function because it goes to
+ * Chrome via `Runtime.evaluate` — nothing in here is typechecked, so it is kept
+ * defensive and free of anything that could throw on an unexpected DOM. The
+ * per-engine configuration is injected as JSON by {@link buildProbeScript}.
+ *
+ * ## Two modes, because the engines genuinely differ
+ *
+ * **"items"** — DuckDuckGo, Bing and Brave each have a clean per-result
+ * container, so results are read directly out of it.
+ *
+ * **"headings"** — Google has no stable result container; its class names
+ * (`MjjYud`, `kb0PBd`, `yuRUbf`) are generated and change without notice. So
+ * results are found from each `<h3>` outward:
+ *
+ *   - anchor    = nearest enclosing `<a href>`
+ *   - header    = nearest `[data-snhf]`, else the anchor
+ *   - container = climb from the header until the text grows past the header's,
+ *                 abandoning the climb if a second `<h3>` comes into scope
+ *                 (that would mean the next result had been swallowed)
+ *   - snippet   = `[data-sncf]` if present, else the container's lines minus the
+ *                 header's lines, minus URL and breadcrumb noise
+ *
+ * Verified against Google's classic SERP and its `udm=14` layout, DuckDuckGo's
+ * no-JS endpoint, Bing's `li.b_algo`, and Brave's `.snippet[data-type=web]`.
+ *
+ * ## Link unwrapping
+ *
+ * DuckDuckGo and Bing both route outbound links through a redirector, so the
+ * raw href is useless to the agent. Both are unwrapped in the page, where the
+ * URL and base64 primitives already exist:
+ *
+ *   - DuckDuckGo: `//duckduckgo.com/l/?uddg=<percent-encoded target>`
+ *   - Bing:       `//bing.com/ck/a?…&u=a1<base64url of target>`
+ *
+ * Google and Brave link straight out and need no unwrapping.
+ */
+
+import type { ExtractionConfig } from "./engines.ts";
+
+/** One search hit, as the page script reports it. */
+export type SearchResult = {
+  title: string;
+  url: string;
+  snippet: string;
+};
+
+/** What the page script returns in a single round trip. */
+export type PageProbe = {
+  /** Where the page actually ended up — redirects and consent walls move it. */
+  url: string;
+  title: string;
+  readyState: string;
+  /** Non-null when a human check is in the way. */
+  challenge: { kind: string; detail: string } | null;
+  /**
+   * Whether the engine's results container was present at all. "none" means a
+   * page shell with no results area — a different failure from an empty one,
+   * and worth telling apart when diagnosing a zero-hit search.
+   */
+  container: "found" | "none";
+  results: SearchResult[];
+};
+
+/**
+ * Challenge detection, shared across engines.
+ *
+ * Every phrase here has been seen on a real page during development: Google's
+ * `/sorry/` interstitial, and Brave's "Verifying you're not a bot / Quick check
+ * before you continue searching" — the latter is why the wording list is broad
+ * rather than just matching Google's "unusual traffic". Ordering matters: a
+ * `/sorry/` page also contains a recaptcha iframe, and naming the page beats
+ * naming the widget sitting on it.
+ */
+const CHALLENGE_DETECTION = String.raw`
+  const detectChallenge = () => {
+    if (/\/sorry\//.test(location.href)) return { kind: "google-block-page", detail: bodyText.slice(0, 300) };
+    if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname))
+      return { kind: "consent-wall", detail: bodyText.slice(0, 300) };
+    if (document.querySelector("#captcha-form, form#captcha-form"))
+      return { kind: "captcha-form", detail: bodyText.slice(0, 300) };
+    if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running, [id^=cf-chl]"))
+      return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) };
+    if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[src*="turnstile"], iframe[title*="challenge"]'))
+      return { kind: "captcha-widget", detail: bodyText.slice(0, 300) };
+    if (/verifying (that )?you('| a)?re( not)? a? ?(human|bot|robot)|quick check before you continue|verify you are human|are you a robot|not a robot/i.test(bodyText))
+      return { kind: "bot-check", detail: bodyText.slice(0, 300) };
+    if (/unusual traffic|automated queries|suspicious activity from your/i.test(bodyText))
+      return { kind: "rate-limit-block", detail: bodyText.slice(0, 300) };
+    return null;
+  };
+`;
+
+/**
+ * Build the probe for one engine. The configuration is embedded as a JSON
+ * literal so the script stays a single self-contained expression.
+ */
+export const buildProbeScript = (config: ExtractionConfig): string => String.raw`(() => {
+  var CFG = ${JSON.stringify(config)};
+
+  var norm = function (s) { return (s || "").replace(/\s+/g, " ").trim(); };
+  var linesOf = function (el) {
+    return el && el.innerText ? el.innerText.split("\n").map(function (l) { return l.trim(); }).filter(Boolean) : [];
+  };
+  var bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : "";
+  var selfHost = new RegExp(CFG.selfHostPattern);
+
+  ${CHALLENGE_DETECTION}
+
+  // DuckDuckGo and Bing both hide the real destination behind a redirector.
+  var unwrap = function (href) {
+    try {
+      var u = new URL(href, location.href);
+      if (CFG.unwrap === "ddg") {
+        if (!/(^|\.)duckduckgo\.com$/.test(u.hostname)) return href;
+        return u.searchParams.get("uddg") || href;
+      }
+      if (CFG.unwrap === "bing") {
+        if (!/(^|\.)bing\.com$/.test(u.hostname)) return href;
+        var p = u.searchParams.get("u");
+        if (!p) return href;
+        var b64 = p.replace(/^a1/, "").replace(/-/g, "+").replace(/_/g, "/");
+        b64 += "=".repeat((4 - (b64.length % 4)) % 4);
+        var bin = atob(b64);
+        var bytes = new Uint8Array(bin.length);
+        for (var i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+        return new TextDecoder().decode(bytes);
+      }
+      return href;
+    } catch (e) {
+      // An unwrap that fails leaves the redirector URL in place rather than
+      // dropping the result: a working link the agent has to follow twice beats
+      // no link at all.
+      return href;
+    }
+  };
+
+  var findRoot = function () {
+    for (var i = 0; i < CFG.roots.length; i++) {
+      var el = document.querySelector(CFG.roots[i]);
+      if (el) return el;
+    }
+    return null;
+  };
+
+  var acceptable = function (url) {
+    return /^https?:/.test(url) && !selfHost.test(url);
+  };
+
+  // ── items mode ───────────────────────────────────────────────────────────
+  var extractItems = function (root) {
+    var out = [];
+    var seen = {};
+    var items = root.querySelectorAll(CFG.item);
+    for (var i = 0; i < items.length; i++) {
+      var item = items[i];
+
+      var skip = false;
+      for (var x = 0; CFG.exclude && x < CFG.exclude.length; x++) {
+        if (item.matches(CFG.exclude[x]) || item.querySelector(CFG.exclude[x])) { skip = true; break; }
+      }
+      if (skip) continue;
+
+      var anchor = item.querySelector(CFG.link);
+      if (!anchor) continue;
+      var url = unwrap(anchor.href);
+      if (!acceptable(url) || seen[url]) continue;
+
+      var titleEl = CFG.title ? item.querySelector(CFG.title) : null;
+      var title = norm(titleEl ? titleEl.innerText : anchor.innerText);
+      if (!title) continue;
+
+      var snippet = "";
+      var snippetEl = CFG.snippet ? item.querySelector(CFG.snippet) : null;
+      if (snippetEl) {
+        snippet = norm(snippetEl.innerText);
+      } else {
+        // No dedicated description element. Rather than guess at class names
+        // for the source and breadcrumb, use their position: an engine that
+        // renders "source / breadcrumb / title / description" puts every piece
+        // of metadata *before* the title, so everything after the title line is
+        // the description. Class names churn; that ordering does not.
+        var itemLines = linesOf(item);
+        var titleIndex = -1;
+        for (var t = 0; t < itemLines.length; t++) {
+          if (norm(itemLines[t]) === title) { titleIndex = t; break; }
+        }
+
+        var candidate;
+        if (titleIndex >= 0) {
+          candidate = itemLines.slice(titleIndex + 1);
+        } else {
+          // Title is not its own line (it may be inline with other text).
+          // Fall back to subtracting the parts we can name.
+          var drop = {};
+          for (var s = 0; CFG.subtract && s < CFG.subtract.length; s++) {
+            var parts = item.querySelectorAll(CFG.subtract[s]);
+            for (var p = 0; p < parts.length; p++) {
+              var pl = linesOf(parts[p]);
+              for (var q = 0; q < pl.length; q++) drop[norm(pl[q])] = true;
+            }
+          }
+          candidate = itemLines.filter(function (l) { return !drop[norm(l)]; });
+        }
+
+        snippet = norm(
+          candidate.filter(function (l) {
+            var n = norm(l);
+            return n !== title && !/^https?:\/\//.test(n) && n.indexOf("›") === -1;
+          }).join(" "),
+        );
+      }
+
+      seen[url] = true;
+      out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
+    }
+    return out;
+  };
+
+  // ── headings mode (Google) ───────────────────────────────────────────────
+  var extractHeadings = function (root) {
+    var out = [];
+    var seen = {};
+    var headings = root.querySelectorAll("h3");
+    for (var i = 0; i < headings.length; i++) {
+      var h3 = headings[i];
+      var anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]"));
+      if (!anchor) continue;
+
+      var url = unwrap(anchor.href);
+      if (!acceptable(url) || seen[url]) continue;
+
+      var title = norm(h3.innerText);
+      if (!title) continue;
+
+      var header = h3.closest("[data-snhf]") || anchor;
+      var headerLines = {};
+      var hl = linesOf(header);
+      for (var k = 0; k < hl.length; k++) headerLines[norm(hl[k])] = true;
+      var baseline = norm(header.innerText || "").length;
+
+      var container = header.parentElement;
+      for (var depth = 0; depth < 6 && container && container !== root; depth++) {
+        if (container.querySelectorAll("h3").length > 1) { container = null; break; }
+        if (norm(container.innerText || "").length > baseline + 40) break;
+        container = container.parentElement;
+      }
+
+      var snippet = "";
+      if (container && container !== root) {
+        var explicit = container.querySelector("[data-sncf]");
+        var body = explicit
+          ? linesOf(explicit)
+          : linesOf(container).filter(function (l) { return !headerLines[norm(l)]; });
+        snippet = norm(
+          body.filter(function (l) {
+            return !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results";
+          }).join(" "),
+        ).replace(/Read more$/, "").trim();
+      }
+
+      seen[url] = true;
+      out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
+    }
+    return out;
+  };
+
+  var challenge = null;
+  var results = [];
+  var container = "none";
+  try {
+    challenge = detectChallenge();
+  } catch (e) {
+    challenge = null;
+  }
+  try {
+    var root = findRoot();
+    container = root ? "found" : "none";
+    if (root && !challenge) results = CFG.mode === "items" ? extractItems(root) : extractHeadings(root);
+  } catch (e) {
+    results = [];
+  }
+
+  return {
+    url: location.href,
+    title: document.title || "",
+    readyState: document.readyState,
+    challenge: challenge,
+    container: container,
+    results: results,
+  };
+})()`;