/** * Google wraps every outbound result in `https://www.google.com/goto?url=`. The token is a protobuf blob with no URL in it, and the real target * appears nowhere in the DOM, so the only way to learn it is to ask Google: an * unauthenticated GET answers `302 Location: `. * * That cannot be done from the page (cross-origin fetches see an opaque * redirect), so it is done here, one request per result, in parallel. */ import type { SearchResult } from "./extract.ts"; const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/; const PER_LINK_TIMEOUT_MS = 4_000; export const isGoogleGoto = (url: string): boolean => GOTO.test(url); const resolveOne = async (url: string, signal?: AbortSignal): Promise => { const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS); try { const res = await fetch(url, { redirect: "manual", signal: signal ? AbortSignal.any([signal, timeout]) : timeout, headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" }, }); const location = res.headers.get("location"); return location && /^https?:/.test(location) ? location : url; } catch { // Left wrapped rather than dropped: the wrapped link still redirects to the // right page when followed, so the agent has a working link either way. return url; } }; /** * Replaces Google `/goto` links with their targets; anything else passes through. * * Best effort, and it degrades with volume: after a handful of requests Google * answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s * countdown) instead of 302, and every link then stays wrapped. Waiting that out * or skipping it is not on the table. A link that stays wrapped keeps the page's * own breadcrumb in its snippet, so the agent can still tell where it points. */ export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise => Promise.all( results.map(async (r) => { if (!isGoogleGoto(r.url)) return r; const url = await resolveOne(r.url, signal); if (!isGoogleGoto(url) || !r.display) return { ...r, url }; return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() }; }), );