castle-search: read Google results again by resolving its /goto links

Google now wraps every result as google.com/goto?url=<opaque token>. The
self-host filter discarded all of them, so every Google search failed with
'nothing could be parsed'. Accept /goto, resolve it node-side from the
302 Location, and when Google answers its 'Verifying your request'
interstitial instead, keep the wrapped link and put the displayed
breadcrumb in the snippet.

5 files changed, 122 insertions, 6 deletions

diff --git a/castle-search/src/engines.ts b/castle-search/src/engines.ts
index ef8baef..76716c6 100644
--- a/castle-search/src/engines.ts
+++ b/castle-search/src/engines.ts
@@ -61,7 +61,7 @@ export type ExtractionConfig = {
   /** Items matching any of these are ads or non-web cards, and are skipped. */
   exclude?: string[];
   /** How outbound links are wrapped, if they are. */
-  unwrap: "none" | "ddg" | "bing";
+  unwrap: "none" | "ddg" | "bing" | "google";
   /** Hosts belonging to the engine itself; links to them are not results. */
   selfHostPattern: string;
 };
@@ -99,8 +99,11 @@ const GOOGLE: EngineDefinition = {
   extraction: {
     mode: "headings",
     roots: ["#rso", "#search"],
-    unwrap: "none",
-    selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/`,
+    // Every result is `google.com/goto?url=<opaque token>` (since at least
+    // 2026-11-06). The page cannot unwrap it; search.ts resolves it afterwards.
+    // It must not count as Google's own link, hence the lookahead.
+    unwrap: "google",
+    selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/(?!goto\?)`,
   },
 };
 
diff --git a/castle-search/src/extract.ts b/castle-search/src/extract.ts
index 8b5c57f..653942b 100644
--- a/castle-search/src/extract.ts
+++ b/castle-search/src/extract.ts
@@ -35,7 +35,9 @@
  *   - DuckDuckGo: `//duckduckgo.com/l/?uddg=<percent-encoded target>`
  *   - Bing:       `//bing.com/ck/a?…&u=a1<base64url of target>`
  *
- * Google and Brave link straight out and need no unwrapping.
+ * Brave links straight out. Google wraps results in `/goto?url=<opaque token>`
+ * that only Google can resolve; those are left alone here and resolved
+ * node-side by resolve.ts.
  */
 
 import type { ExtractionConfig } from "./engines.ts";
@@ -45,6 +47,8 @@ export type SearchResult = {
   title: string;
   url: string;
   snippet: string;
+  /** Google only: the site and breadcrumb as displayed, kept for links that cannot be resolved. */
+  display?: string;
 };
 
 /** What the page script returns in a single round trip. */
@@ -261,8 +265,13 @@ export const buildProbeScript = (config: ExtractionConfig): string => String.raw
         ).replace(/Read more$/, "").trim();
       }
 
+      var display = "";
+      var siteEl = anchor.querySelector("span.VuuXrf");
+      var citeEl = anchor.querySelector("cite");
+      display = norm([siteEl && siteEl.innerText, citeEl && citeEl.innerText].filter(Boolean).join(" — "));
+
       seen[url] = true;
-      out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
+      out.push({ title: title, url: url, snippet: snippet.slice(0, 600), display: display });
     }
     return out;
   };
diff --git a/castle-search/src/resolve.ts b/castle-search/src/resolve.ts
new file mode 100644
index 0000000..f432f7d
--- /dev/null
+++ b/castle-search/src/resolve.ts
@@ -0,0 +1,53 @@
+/**
+ * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque
+ * token>`. The token is a protobuf blob with no URL in it, and the real target
+ * appears nowhere in the DOM, so the only way to learn it is to ask Google: an
+ * unauthenticated GET answers `302 Location: <target>`.
+ *
+ * That cannot be done from the page (cross-origin fetches see an opaque
+ * redirect), so it is done here, one request per result, in parallel.
+ */
+
+import type { SearchResult } from "./extract.ts";
+
+const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/;
+const PER_LINK_TIMEOUT_MS = 4_000;
+
+export const isGoogleGoto = (url: string): boolean => GOTO.test(url);
+
+const resolveOne = async (url: string, signal?: AbortSignal): Promise<string> => {
+  const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS);
+  try {
+    const res = await fetch(url, {
+      redirect: "manual",
+      signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+      headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" },
+    });
+    const location = res.headers.get("location");
+    return location && /^https?:/.test(location) ? location : url;
+  } catch {
+    // Left wrapped rather than dropped: the wrapped link still redirects to the
+    // right page when followed, so the agent has a working link either way.
+    return url;
+  }
+};
+
+/**
+ * Replaces Google `/goto` links with their targets; anything else passes through.
+ *
+ * Best effort, and it degrades with volume: after a handful of requests Google
+ * answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s
+ * countdown) instead of 302, and every link then stays wrapped. Waiting that out
+ * or skipping it is not on the table. A link that stays wrapped keeps the page's
+ * own breadcrumb in its snippet, so the agent can still tell where it points.
+ */
+export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise<SearchResult[]> =>
+  Promise.all(
+    results.map(async (r) => {
+      if (!isGoogleGoto(r.url)) return r;
+      const url = await resolveOne(r.url, signal);
+      if (!isGoogleGoto(url) || !r.display) return { ...r, url };
+      return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() };
+    }),
+  );
+
diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts
index 1306443..1c84392 100644
--- a/castle-search/src/search.ts
+++ b/castle-search/src/search.ts
@@ -8,6 +8,7 @@
  * first search and closed by an idempotent `session_shutdown`.
  */
 
+import { resolveGoogleLinks } from "./resolve.ts";
 import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts";
 import {
   type CdpTarget,
@@ -319,7 +320,7 @@ export const search = async (
       }
 
       return {
-        results: probe.results.slice(0, request.numResults),
+        results: await resolveGoogleLinks(probe.results.slice(0, request.numResults), signal),
         searchUrl,
         finalUrl: probe.url || searchUrl,
         endpoint,
diff --git a/castle-search/test/resolve.test.ts b/castle-search/test/resolve.test.ts
new file mode 100644
index 0000000..bcd4ef4
--- /dev/null
+++ b/castle-search/test/resolve.test.ts
@@ -0,0 +1,50 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { isGoogleGoto, resolveGoogleLinks } from "../src/resolve.ts";
+import { getEngine } from "../src/engines.ts";
+
+test("google /goto links are recognised, other google links are not", () => {
+  assert.equal(isGoogleGoto("https://www.google.com/goto?url=CAES"), true);
+  assert.equal(isGoogleGoto("https://www.google.co.uk/goto?url=CAES"), true);
+  assert.equal(isGoogleGoto("https://www.google.com/search?q=x"), false);
+  assert.equal(isGoogleGoto("https://example.com/goto?url=x"), false);
+});
+
+test("the google self-host filter lets /goto through but still blocks google's own pages", () => {
+  const re = new RegExp(getEngine("google").extraction.selfHostPattern);
+  assert.equal(re.test("https://www.google.com/goto?url=CAES"), false);
+  assert.equal(re.test("https://www.google.com/search?q=x"), true);
+  assert.equal(re.test("https://accounts.google.com/ServiceLogin"), false);
+});
+
+test("non-google results pass through untouched", async () => {
+  const input = [{ title: "t", url: "https://example.com/", snippet: "" }];
+  assert.deepEqual(await resolveGoogleLinks(input), input);
+});
+
+test("a link that cannot be resolved keeps the page's breadcrumb in its snippet", async () => {
+  const wrapped = "https://www.google.com/goto?url=CAES";
+  const real = globalThis.fetch;
+  // Google's interstitial: 200 with no Location header.
+  globalThis.fetch = (async () => new Response("<html>Verifying your request</html>", { status: 200 })) as typeof fetch;
+  try {
+    const [r] = await resolveGoogleLinks([{ title: "t", url: wrapped, snippet: "s", display: "GitHub — https://github.com › a › b" }]);
+    assert.equal(r?.url, wrapped);
+    assert.match(r?.snippet ?? "", /^\[link unresolved; Google shows it as: GitHub — https:\/\/github\.com › a › b\] s$/);
+  } finally {
+    globalThis.fetch = real;
+  }
+});
+
+test("a link that resolves is replaced and the snippet is left alone", async () => {
+  const real = globalThis.fetch;
+  globalThis.fetch = (async () => new Response(null, { status: 302, headers: { location: "https://github.com/a/b" } })) as typeof fetch;
+  try {
+    const [r] = await resolveGoogleLinks([{ title: "t", url: "https://www.google.com/goto?url=CAES", snippet: "s", display: "x" }]);
+    assert.equal(r?.url, "https://github.com/a/b");
+    assert.equal(r?.snippet, "s");
+  } finally {
+    globalThis.fetch = real;
+  }
+});