diff options
| author | Igor Soarez <igor@soarez.org> | 2026-11-06 03:17:13 +0000 |
|---|---|---|
| committer | Igor Soarez <igor@soarez.org> | 2026-11-06 03:17:16 +0000 |
| commit | 74ff9fdcb8b49a021b63ac2e6b0191e29754d0f3 (patch) | |
| tree | bab1e3995f1a19762086bdf5aaac0751fdf7ff2e /castle-search/src/engines.ts | |
| parent | f61c43418d704cccb67132540ed7feceacef1d2a (diff) | |
castle-search: read Google results again by resolving its /goto links
Google now wraps every result as google.com/goto?url=<opaque token>. The
self-host filter discarded all of them, so every Google search failed with
'nothing could be parsed'. Accept /goto, resolve it node-side from the
302 Location, and when Google answers its 'Verifying your request'
interstitial instead, keep the wrapped link and put the displayed
breadcrumb in the snippet.
Diffstat (limited to 'castle-search/src/engines.ts')
| -rw-r--r-- | castle-search/src/engines.ts | 9 |
1 files changed, 6 insertions, 3 deletions
diff --git a/castle-search/src/engines.ts b/castle-search/src/engines.ts index ef8baef..76716c6 100644 --- a/castle-search/src/engines.ts +++ b/castle-search/src/engines.ts @@ -61,7 +61,7 @@ export type ExtractionConfig = { /** Items matching any of these are ads or non-web cards, and are skipped. */ exclude?: string[]; /** How outbound links are wrapped, if they are. */ - unwrap: "none" | "ddg" | "bing"; + unwrap: "none" | "ddg" | "bing" | "google"; /** Hosts belonging to the engine itself; links to them are not results. */ selfHostPattern: string; }; @@ -99,8 +99,11 @@ const GOOGLE: EngineDefinition = { extraction: { mode: "headings", roots: ["#rso", "#search"], - unwrap: "none", - selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/`, + // Every result is `google.com/goto?url=<opaque token>` (since at least + // 2026-11-06). The page cannot unwrap it; search.ts resolves it afterwards. + // It must not count as Google's own link, hence the lookahead. + unwrap: "google", + selfHostPattern: String.raw`^https?://(www\.)?google\.[a-z.]+/(?!goto\?)`, }, }; |
