diff options
Diffstat (limited to 'castle-search/src')
| -rw-r--r-- | castle-search/src/engines.ts | 9 | ||||
| -rw-r--r-- | castle-search/src/extract.ts | 13 | ||||
| -rw-r--r-- | castle-search/src/resolve.ts | 53 | ||||
| -rw-r--r-- | castle-search/src/search.ts | 3 |
4 files changed, 72 insertions, 6 deletions
diff --git a/castle-search/src/engines.ts b/castle-search/src/engines.ts index ef8baef..76716c6 100644 --- a/castle-search/src/engines.ts +++ b/castle-search/src/engines.ts @@ -61,7 +61,7 @@ export type ExtractionConfig = { /** Items matching any of these are ads or non-web cards, and are skipped. */ exclude?: string[]; /** How outbound links are wrapped, if they are. */ - unwrap: "none" | "ddg" | "bing"; + unwrap: "none" | "ddg" | "bing" | "google"; /** Hosts belonging to the engine itself; links to them are not results. */ selfHostPattern: string; }; @@ -99,8 +99,11 @@ const GOOGLE: EngineDefinition = { extraction: { mode: "headings", roots: ["#rso", "#search"], - unwrap: "none", - selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/`, + // Every result is `google.com/goto?url=<opaque token>` (since at least + // 2026-11-06). The page cannot unwrap it; search.ts resolves it afterwards. + // It must not count as Google's own link, hence the lookahead. + unwrap: "google", + selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/(?!goto\?)`, }, }; diff --git a/castle-search/src/extract.ts b/castle-search/src/extract.ts index 8b5c57f..653942b 100644 --- a/castle-search/src/extract.ts +++ b/castle-search/src/extract.ts @@ -35,7 +35,9 @@ * - DuckDuckGo: `//duckduckgo.com/l/?uddg=<percent-encoded target>` * - Bing: `//bing.com/ck/a?…&u=a1<base64url of target>` * - * Google and Brave link straight out and need no unwrapping. + * Brave links straight out. Google wraps results in `/goto?url=<opaque token>` + * that only Google can resolve; those are left alone here and resolved + * node-side by resolve.ts. */ import type { ExtractionConfig } from "./engines.ts"; @@ -45,6 +47,8 @@ export type SearchResult = { title: string; url: string; snippet: string; + /** Google only: the site and breadcrumb as displayed, kept for links that cannot be resolved. */ + display?: string; }; /** What the page script returns in a single round trip. */ @@ -261,8 +265,13 @@ export const buildProbeScript = (config: ExtractionConfig): string => String.raw ).replace(/Read more$/, "").trim(); } + var display = ""; + var siteEl = anchor.querySelector("span.VuuXrf"); + var citeEl = anchor.querySelector("cite"); + display = norm([siteEl && siteEl.innerText, citeEl && citeEl.innerText].filter(Boolean).join(" — ")); + seen[url] = true; - out.push({ title: title, url: url, snippet: snippet.slice(0, 600) }); + out.push({ title: title, url: url, snippet: snippet.slice(0, 600), display: display }); } return out; }; diff --git a/castle-search/src/resolve.ts b/castle-search/src/resolve.ts new file mode 100644 index 0000000..f432f7d --- /dev/null +++ b/castle-search/src/resolve.ts @@ -0,0 +1,53 @@ +/** + * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque + * token>`. The token is a protobuf blob with no URL in it, and the real target + * appears nowhere in the DOM, so the only way to learn it is to ask Google: an + * unauthenticated GET answers `302 Location: <target>`. + * + * That cannot be done from the page (cross-origin fetches see an opaque + * redirect), so it is done here, one request per result, in parallel. + */ + +import type { SearchResult } from "./extract.ts"; + +const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/; +const PER_LINK_TIMEOUT_MS = 4_000; + +export const isGoogleGoto = (url: string): boolean => GOTO.test(url); + +const resolveOne = async (url: string, signal?: AbortSignal): Promise<string> => { + const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS); + try { + const res = await fetch(url, { + redirect: "manual", + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, + headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" }, + }); + const location = res.headers.get("location"); + return location && /^https?:/.test(location) ? location : url; + } catch { + // Left wrapped rather than dropped: the wrapped link still redirects to the + // right page when followed, so the agent has a working link either way. + return url; + } +}; + +/** + * Replaces Google `/goto` links with their targets; anything else passes through. + * + * Best effort, and it degrades with volume: after a handful of requests Google + * answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s + * countdown) instead of 302, and every link then stays wrapped. Waiting that out + * or skipping it is not on the table. A link that stays wrapped keeps the page's + * own breadcrumb in its snippet, so the agent can still tell where it points. + */ +export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise<SearchResult[]> => + Promise.all( + results.map(async (r) => { + if (!isGoogleGoto(r.url)) return r; + const url = await resolveOne(r.url, signal); + if (!isGoogleGoto(url) || !r.display) return { ...r, url }; + return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() }; + }), + ); + diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts index 1306443..1c84392 100644 --- a/castle-search/src/search.ts +++ b/castle-search/src/search.ts @@ -8,6 +8,7 @@ * first search and closed by an idempotent `session_shutdown`. */ +import { resolveGoogleLinks } from "./resolve.ts"; import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts"; import { type CdpTarget, @@ -319,7 +320,7 @@ export const search = async ( } return { - results: probe.results.slice(0, request.numResults), + results: await resolveGoogleLinks(probe.results.slice(0, request.numResults), signal), searchUrl, finalUrl: probe.url || searchUrl, endpoint, |
