castle-search: read Google results again by resolving its /goto links

Google now wraps every result as google.com/goto?url=<opaque token>. The
self-host filter discarded all of them, so every Google search failed with
'nothing could be parsed'. Accept /goto, resolve it node-side from the
302 Location, and when Google answers its 'Verifying your request'
interstitial instead, keep the wrapped link and put the displayed
breadcrumb in the snippet.

1 files changed, 53 insertions, 0 deletions

diff --git a/castle-search/src/resolve.ts b/castle-search/src/resolve.ts
new file mode 100644
index 0000000..f432f7d
--- /dev/null
+++ b/castle-search/src/resolve.ts
@@ -0,0 +1,53 @@
+/**
+ * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque
+ * token>`. The token is a protobuf blob with no URL in it, and the real target
+ * appears nowhere in the DOM, so the only way to learn it is to ask Google: an
+ * unauthenticated GET answers `302 Location: <target>`.
+ *
+ * That cannot be done from the page (cross-origin fetches see an opaque
+ * redirect), so it is done here, one request per result, in parallel.
+ */
+
+import type { SearchResult } from "./extract.ts";
+
+const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/;
+const PER_LINK_TIMEOUT_MS = 4_000;
+
+export const isGoogleGoto = (url: string): boolean => GOTO.test(url);
+
+const resolveOne = async (url: string, signal?: AbortSignal): Promise<string> => {
+  const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS);
+  try {
+    const res = await fetch(url, {
+      redirect: "manual",
+      signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+      headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" },
+    });
+    const location = res.headers.get("location");
+    return location && /^https?:/.test(location) ? location : url;
+  } catch {
+    // Left wrapped rather than dropped: the wrapped link still redirects to the
+    // right page when followed, so the agent has a working link either way.
+    return url;
+  }
+};
+
+/**
+ * Replaces Google `/goto` links with their targets; anything else passes through.
+ *
+ * Best effort, and it degrades with volume: after a handful of requests Google
+ * answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s
+ * countdown) instead of 302, and every link then stays wrapped. Waiting that out
+ * or skipping it is not on the table. A link that stays wrapped keeps the page's
+ * own breadcrumb in its snippet, so the agent can still tell where it points.
+ */
+export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise<SearchResult[]> =>
+  Promise.all(
+    results.map(async (r) => {
+      if (!isGoogleGoto(r.url)) return r;
+      const url = await resolveOne(r.url, signal);
+      if (!isGoogleGoto(url) || !r.display) return { ...r, url };
+      return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() };
+    }),
+  );
+