From 74ff9fdcb8b49a021b63ac2e6b0191e29754d0f3 Mon Sep 17 00:00:00 2001 From: Igor Soarez Date: Fri, 6 Nov 2026 03:17:13 +0000 Subject: castle-search: read Google results again by resolving its /goto links Google now wraps every result as google.com/goto?url=. The self-host filter discarded all of them, so every Google search failed with 'nothing could be parsed'. Accept /goto, resolve it node-side from the 302 Location, and when Google answers its 'Verifying your request' interstitial instead, keep the wrapped link and put the displayed breadcrumb in the snippet. --- castle-search/src/engines.ts | 9 ++++--- castle-search/src/extract.ts | 13 ++++++++-- castle-search/src/resolve.ts | 53 ++++++++++++++++++++++++++++++++++++++ castle-search/src/search.ts | 3 ++- castle-search/test/resolve.test.ts | 50 +++++++++++++++++++++++++++++++++++ 5 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 castle-search/src/resolve.ts create mode 100644 castle-search/test/resolve.test.ts diff --git a/castle-search/src/engines.ts b/castle-search/src/engines.ts index ef8baef..76716c6 100644 --- a/castle-search/src/engines.ts +++ b/castle-search/src/engines.ts @@ -61,7 +61,7 @@ export type ExtractionConfig = { /** Items matching any of these are ads or non-web cards, and are skipped. */ exclude?: string[]; /** How outbound links are wrapped, if they are. */ - unwrap: "none" | "ddg" | "bing"; + unwrap: "none" | "ddg" | "bing" | "google"; /** Hosts belonging to the engine itself; links to them are not results. */ selfHostPattern: string; }; @@ -99,8 +99,11 @@ const GOOGLE: EngineDefinition = { extraction: { mode: "headings", roots: ["#rso", "#search"], - unwrap: "none", - selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/`, + // Every result is `google.com/goto?url=` (since at least + // 2026-11-06). The page cannot unwrap it; search.ts resolves it afterwards. + // It must not count as Google's own link, hence the lookahead. + unwrap: "google", + selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/(?!goto\?)`, }, }; diff --git a/castle-search/src/extract.ts b/castle-search/src/extract.ts index 8b5c57f..653942b 100644 --- a/castle-search/src/extract.ts +++ b/castle-search/src/extract.ts @@ -35,7 +35,9 @@ * - DuckDuckGo: `//duckduckgo.com/l/?uddg=` * - Bing: `//bing.com/ck/a?…&u=a1` * - * Google and Brave link straight out and need no unwrapping. + * Brave links straight out. Google wraps results in `/goto?url=` + * that only Google can resolve; those are left alone here and resolved + * node-side by resolve.ts. */ import type { ExtractionConfig } from "./engines.ts"; @@ -45,6 +47,8 @@ export type SearchResult = { title: string; url: string; snippet: string; + /** Google only: the site and breadcrumb as displayed, kept for links that cannot be resolved. */ + display?: string; }; /** What the page script returns in a single round trip. */ @@ -261,8 +265,13 @@ export const buildProbeScript = (config: ExtractionConfig): string => String.raw ).replace(/Read more$/, "").trim(); } + var display = ""; + var siteEl = anchor.querySelector("span.VuuXrf"); + var citeEl = anchor.querySelector("cite"); + display = norm([siteEl && siteEl.innerText, citeEl && citeEl.innerText].filter(Boolean).join(" — ")); + seen[url] = true; - out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); + out.push({ title: title, url: url, snippet: snippet.slice(0, 600), display: display }); } return out; }; diff --git a/castle-search/src/resolve.ts b/castle-search/src/resolve.ts new file mode 100644 index 0000000..f432f7d --- /dev/null +++ b/castle-search/src/resolve.ts @@ -0,0 +1,53 @@ +/** + * Google wraps every outbound result in `https://www.google.com/goto?url=`. The token is a protobuf blob with no URL in it, and the real target + * appears nowhere in the DOM, so the only way to learn it is to ask Google: an + * unauthenticated GET answers `302 Location: `. + * + * That cannot be done from the page (cross-origin fetches see an opaque + * redirect), so it is done here, one request per result, in parallel. + */ + +import type { SearchResult } from "./extract.ts"; + +const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/; +const PER_LINK_TIMEOUT_MS = 4_000; + +export const isGoogleGoto = (url: string): boolean => GOTO.test(url); + +const resolveOne = async (url: string, signal?: AbortSignal): Promise => { + const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS); + try { + const res = await fetch(url, { + redirect: "manual", + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, + headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" }, + }); + const location = res.headers.get("location"); + return location && /^https?:/.test(location) ? location : url; + } catch { + // Left wrapped rather than dropped: the wrapped link still redirects to the + // right page when followed, so the agent has a working link either way. + return url; + } +}; + +/** + * Replaces Google `/goto` links with their targets; anything else passes through. + * + * Best effort, and it degrades with volume: after a handful of requests Google + * answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s + * countdown) instead of 302, and every link then stays wrapped. Waiting that out + * or skipping it is not on the table. A link that stays wrapped keeps the page's + * own breadcrumb in its snippet, so the agent can still tell where it points. + */ +export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise => + Promise.all( + results.map(async (r) => { + if (!isGoogleGoto(r.url)) return r; + const url = await resolveOne(r.url, signal); + if (!isGoogleGoto(url) || !r.display) return { ...r, url }; + return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() }; + }), + ); + diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts index 1306443..1c84392 100644 --- a/castle-search/src/search.ts +++ b/castle-search/src/search.ts @@ -8,6 +8,7 @@ * first search and closed by an idempotent `session_shutdown`. */ +import { resolveGoogleLinks } from "./resolve.ts"; import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts"; import { type CdpTarget, @@ -319,7 +320,7 @@ export const search = async ( } return { - results: probe.results.slice(0, request.numResults), + results: await resolveGoogleLinks(probe.results.slice(0, request.numResults), signal), searchUrl, finalUrl: probe.url || searchUrl, endpoint, diff --git a/castle-search/test/resolve.test.ts b/castle-search/test/resolve.test.ts new file mode 100644 index 0000000..bcd4ef4 --- /dev/null +++ b/castle-search/test/resolve.test.ts @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { isGoogleGoto, resolveGoogleLinks } from "../src/resolve.ts"; +import { getEngine } from "../src/engines.ts"; + +test("google /goto links are recognised, other google links are not", () => { + assert.equal(isGoogleGoto("https://www.google.com/goto?url=CAES"), true); + assert.equal(isGoogleGoto("https://www.google.co.uk/goto?url=CAES"), true); + assert.equal(isGoogleGoto("https://www.google.com/search?q=x"), false); + assert.equal(isGoogleGoto("https://example.com/goto?url=x"), false); +}); + +test("the google self-host filter lets /goto through but still blocks google's own pages", () => { + const re = new RegExp(getEngine("google").extraction.selfHostPattern); + assert.equal(re.test("https://www.google.com/goto?url=CAES"), false); + assert.equal(re.test("https://www.google.com/search?q=x"), true); + assert.equal(re.test("https://accounts.google.com/ServiceLogin"), false); +}); + +test("non-google results pass through untouched", async () => { + const input = [{ title: "t", url: "https://example.com/", snippet: "" }]; + assert.deepEqual(await resolveGoogleLinks(input), input); +}); + +test("a link that cannot be resolved keeps the page's breadcrumb in its snippet", async () => { + const wrapped = "https://www.google.com/goto?url=CAES"; + const real = globalThis.fetch; + // Google's interstitial: 200 with no Location header. + globalThis.fetch = (async () => new Response("Verifying your request", { status: 200 })) as typeof fetch; + try { + const [r] = await resolveGoogleLinks([{ title: "t", url: wrapped, snippet: "s", display: "GitHub — https://github.com › a › b" }]); + assert.equal(r?.url, wrapped); + assert.match(r?.snippet ?? "", /^\[link unresolved; Google shows it as: GitHub — https:\/\/github\.com › a › b\] s$/); + } finally { + globalThis.fetch = real; + } +}); + +test("a link that resolves is replaced and the snippet is left alone", async () => { + const real = globalThis.fetch; + globalThis.fetch = (async () => new Response(null, { status: 302, headers: { location: "https://github.com/a/b" } })) as typeof fetch; + try { + const [r] = await resolveGoogleLinks([{ title: "t", url: "https://www.google.com/goto?url=CAES", snippet: "s", display: "x" }]); + assert.equal(r?.url, "https://github.com/a/b"); + assert.equal(r?.snippet, "s"); + } finally { + globalThis.fetch = real; + } +}); -- cgit v1.2.3