diff options
| -rw-r--r-- | castle-search/README.md | 20 | ||||
| -rw-r--r-- | castle-search/src/errors.ts | 19 | ||||
| -rw-r--r-- | castle-search/src/format.ts | 8 | ||||
| -rw-r--r-- | castle-search/src/google-links.ts | 15 | ||||
| -rw-r--r-- | castle-search/src/index.ts | 37 | ||||
| -rw-r--r-- | castle-search/src/resolve.ts | 53 | ||||
| -rw-r--r-- | castle-search/src/search.ts | 47 | ||||
| -rw-r--r-- | castle-search/test/google-links.test.ts | 41 | ||||
| -rw-r--r-- | castle-search/test/resolve.test.ts | 50 |
9 files changed, 182 insertions, 108 deletions
diff --git a/castle-search/README.md b/castle-search/README.md index 23a36e6..f3fb5e9 100644 --- a/castle-search/README.md +++ b/castle-search/README.md @@ -184,8 +184,24 @@ Two modes, because the engines genuinely differ: DuckDuckGo and Bing both route outbound links through redirectors (`duckduckgo.com/l/?uddg=…`, `bing.com/ck/a?…&u=a1<base64url>`); both are -unwrapped in the page so the agent gets real URLs. Google and Brave link -straight out. +unwrapped in the page so the agent gets real URLs. Brave links straight out. + +**Google wraps every result** as `google.com/goto?url=<opaque token>` (rolled +out June–August 2026; Google called it "technical measures against evolving +forms of abuse" and has not documented it). The token is not reversible and the +target is not in the DOM. So search leaves Google links wrapped, prints the +breadcrumb Google shows for each, and `castle_cdp_resolve` follows one link in +Chrome — one tab, closed afterwards — when the agent decides it wants it. +Resolve lazily: a plain HTTP client (curl, Node `fetch`) is answered with a +"Verifying your request" page and a 17–19 s countdown after a handful of +requests, while a click or a direct navigation in Chrome redirects at once +(measured 2026-11-06). If Chrome does get the countdown, `GoogleVerificationError` +says so; it is never waited out or clicked through. Never resolve links with +plain HTTP: that was tried first, and the testing itself triggered the check. + +An engine's own "no results" page is not an extractor failure. DuckDuckGo's +notice is itself a `.result`, so `emptyMarker` identifies it and the error says +the query was too narrow rather than blaming markup or a rate limit. Markup will change. When it does, `NoResultsError` distinguishes "container rendered but unparseable" (that engine's extractor needs updating) from "no diff --git a/castle-search/src/errors.ts b/castle-search/src/errors.ts index 2318b27..95fa7ef 100644 --- a/castle-search/src/errors.ts +++ b/castle-search/src/errors.ts @@ -81,3 +81,22 @@ export class SearchAbortedError extends Error { } export const messageOf = (e: unknown): string => (e instanceof Error ? e.message : String(e)); + +/** + * Google answered a `/goto` link with its "Verifying your request" page instead + * of redirecting. It is a bot check with a countdown; it is reported, never + * waited out or clicked through. + */ +export class GoogleVerificationError extends Error { + override readonly name = "GoogleVerificationError"; + readonly pageUrl: string; + constructor(pageUrl: string) { + super( + "Google showed its \"Verifying your request\" page instead of redirecting, so the real URL could not be " + + "read. This is a rate-based bot check: opening many links in a row triggers it, and it eases after a few " + + "minutes. Do not retry in a loop. Use the breadcrumb printed under the result to judge where it points, " + + "pick fewer links, or search again with another engine (duckduckgo, brave, bing), which return real URLs.", + ); + this.pageUrl = pageUrl; + } +} diff --git a/castle-search/src/format.ts b/castle-search/src/format.ts index aef6c3b..9c7c3cc 100644 --- a/castle-search/src/format.ts +++ b/castle-search/src/format.ts @@ -8,6 +8,7 @@ */ import type { SearchResult } from "./extract.ts"; +import { isGoogleGoto } from "./google-links.ts"; export type FormatInput = { query: string; @@ -36,6 +37,13 @@ export const formatResults = (input: FormatInput): string => { input.results.forEach((result, index) => { lines.push(`${index + 1}. ${result.title}`); lines.push(` ${result.url}`); + // Google wraps every link and only reveals the target when followed. Say so + // here, next to the link, with what Google displays for it — that is often + // enough to decide without resolving anything. + if (isGoogleGoto(result.url)) { + const shown = result.display ? ` Google shows it as: ${result.display}.` : ""; + lines.push(` [Google-wrapped link, real URL not yet known.${shown} Pass it to castle_cdp_resolve if you need to open it.]`); + } if (result.snippet) lines.push(` ${result.snippet}`); lines.push(""); }); diff --git a/castle-search/src/google-links.ts b/castle-search/src/google-links.ts new file mode 100644 index 0000000..7500e2d --- /dev/null +++ b/castle-search/src/google-links.ts @@ -0,0 +1,15 @@ +/** + * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque + * token>`. The token is a protobuf blob with no URL in it and the target appears + * nowhere in the page, so the only way to learn it is to follow the link. + * + * It is followed in Chrome, lazily, one link at a time, when the caller asks + * (see `resolveLink` in search.ts) — not here, and not for every result. A + * plain HTTP client gets Google's "Verifying your request" countdown after a + * handful of requests; the browser, with its own profile, gets the redirect. + */ + +const GOTO = /^https?:\/\/(www\.)?google\.(?:com?\.)?[a-z]{2,3}\/goto\?url=[A-Za-z0-9_-]+$/; + +/** Only these may be opened by castle_cdp_resolve — it is not a general navigator. */ +export const isGoogleGoto = (url: string): boolean => GOTO.test(url); diff --git a/castle-search/src/index.ts b/castle-search/src/index.ts index 29cdaaf..9b7c143 100644 --- a/castle-search/src/index.ts +++ b/castle-search/src/index.ts @@ -43,6 +43,7 @@ import { describeConfiguredEndpoint, describeConfiguredEngine, disconnect, + resolveLink, search, } from "./search.ts"; @@ -202,6 +203,38 @@ export default function (pi: ExtensionAPI): void { }); pi.registerTool({ + name: "castle_cdp_resolve", + label: "Castle Resolve", + description: + "Turn one Google-wrapped result link (https://www.google.com/goto?url=…, as printed by castle_cdp_search " + + "with engine google) into the real destination URL, by following it in the same Chrome. Opens one tab, " + + "reads where it lands, closes it. It does not read the page. Resolve only the results you mean to open: " + + "Google answers a burst of links with a \"Verifying your request\" countdown, which this tool reports " + + "instead of waiting out. Other engines already return real URLs and need no resolving.", + promptSnippet: "Resolve one Google-wrapped result link to its real URL (castle_cdp_resolve)", + promptGuidelines: [ + "castle_cdp_search on Google returns wrapped links. Read the titles, snippets and the breadcrumb Google " + + "shows for each, decide which result you actually need, and resolve only that one with castle_cdp_resolve. " + + "Do not resolve every result: that burst is what triggers Google's verification page.", + ], + parameters: Type.Object({ + url: Type.String({ + description: "A https://www.google.com/goto?url=… link exactly as castle_cdp_search printed it.", + minLength: 1, + }), + }), + async execute(_toolCallId, params, signal, _onUpdate, _ctx) { + const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS); + const combined = signal ? AbortSignal.any([signal, timeout]) : timeout; + const out = await resolveLink(params.url.trim(), combined); + return { + content: [{ type: "text", text: `${out.to}\n(resolved via Chrome at ${out.endpoint})` }], + details: { from: out.from, to: out.to, endpoint: out.endpoint }, + }; + }, + }); + + pi.registerTool({ name: "castle_cdp_search", label: "Castle Search", description: @@ -209,7 +242,9 @@ export default function (pi: ExtensionAPI): void { "DevTools Protocol. Returns ranked results with titles, URLs and snippets. Because it uses a real " + "logged-in browser rather than a search API, it reaches pages that block datacentre traffic. It " + "returns search results only — it does not fetch or read the linked pages, so follow up with a " + - "fetch/read tool for full page content. Supports four engines with independent indexes and " + + "fetch/read tool for full page content. Google results come back as wrapped links with the " + + "breadcrumb Google shows; resolve the ones you need with castle_cdp_resolve. " + + "Supports four engines with independent indexes and " + `independent rate limits (${ENGINE_IDS.join(", ")}); pass engine to switch when one is blocked. ` + `Output is truncated at ${formatSize(DEFAULT_MAX_BYTES)} or ${DEFAULT_MAX_LINES} lines.`, promptSnippet: diff --git a/castle-search/src/resolve.ts b/castle-search/src/resolve.ts deleted file mode 100644 index f432f7d..0000000 --- a/castle-search/src/resolve.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque - * token>`. The token is a protobuf blob with no URL in it, and the real target - * appears nowhere in the DOM, so the only way to learn it is to ask Google: an - * unauthenticated GET answers `302 Location: <target>`. - * - * That cannot be done from the page (cross-origin fetches see an opaque - * redirect), so it is done here, one request per result, in parallel. - */ - -import type { SearchResult } from "./extract.ts"; - -const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/; -const PER_LINK_TIMEOUT_MS = 4_000; - -export const isGoogleGoto = (url: string): boolean => GOTO.test(url); - -const resolveOne = async (url: string, signal?: AbortSignal): Promise<string> => { - const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS); - try { - const res = await fetch(url, { - redirect: "manual", - signal: signal ? AbortSignal.any([signal, timeout]) : timeout, - headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" }, - }); - const location = res.headers.get("location"); - return location && /^https?:/.test(location) ? location : url; - } catch { - // Left wrapped rather than dropped: the wrapped link still redirects to the - // right page when followed, so the agent has a working link either way. - return url; - } -}; - -/** - * Replaces Google `/goto` links with their targets; anything else passes through. - * - * Best effort, and it degrades with volume: after a handful of requests Google - * answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s - * countdown) instead of 302, and every link then stays wrapped. Waiting that out - * or skipping it is not on the table. A link that stays wrapped keeps the page's - * own breadcrumb in its snippet, so the agent can still tell where it points. - */ -export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise<SearchResult[]> => - Promise.all( - results.map(async (r) => { - if (!isGoogleGoto(r.url)) return r; - const url = await resolveOne(r.url, signal); - if (!isGoogleGoto(url) || !r.display) return { ...r, url }; - return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() }; - }), - ); - diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts index c64735d..b5effb8 100644 --- a/castle-search/src/search.ts +++ b/castle-search/src/search.ts @@ -8,7 +8,8 @@ * first search and closed by an idempotent `session_shutdown`. */ -import { resolveGoogleLinks } from "./resolve.ts"; +import { isGoogleGoto } from "./google-links.ts"; +import { GoogleVerificationError } from "./errors.ts"; import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts"; import { type CdpTarget, @@ -332,7 +333,7 @@ export const search = async ( } return { - results: await resolveGoogleLinks(probe.results.slice(0, request.numResults), signal), + results: probe.results.slice(0, request.numResults), searchUrl, finalUrl: probe.url || searchUrl, endpoint, @@ -350,3 +351,45 @@ export const search = async ( release(); } }; + +export type ResolveOutcome = { from: string; to: string; endpoint: string }; + +/** + * Follows one Google `/goto` link in Chrome and reports where it lands. + * + * Lazy by design: search leaves the links wrapped, and the caller resolves only + * the ones it means to open. One tab, one navigation, closed afterwards — the + * pattern of a person clicking a result, which Google serves directly. + * + * If Google answers with its "Verifying your request" page instead, the tab is + * still on google.com when the page loads. That is reported as such. The + * countdown is not waited out and nothing is clicked. + */ +export const resolveLink = async (url: string, signal: AbortSignal): Promise<ResolveOutcome> => { + if (!isGoogleGoto(url)) { + throw new Error( + "castle_cdp_resolve only follows Google result links of the form https://www.google.com/goto?url=… " + + "as printed by castle_cdp_search. Results from other engines already carry their real URL.", + ); + } + const release = await acquire(signal); + try { + const conn = await ensureConnection(signal); + const endpoint = describeCdpTarget(conn.target); + const page = await PageSession.open(conn, signal); + try { + await page.navigate(url, signal); + const landed = await page.evaluate<string>("location.href", signal); + if (!landed || isGoogleHost(landed)) { + throw new GoogleVerificationError(landed || url); + } + return { from: url, to: landed, endpoint }; + } finally { + await page.close(); + } + } finally { + release(); + } +}; + +const isGoogleHost = (url: string): boolean => /^https?:\/\/([a-z0-9-]+\.)*google\.(?:com?\.)?[a-z]{2,3}\//.test(url); diff --git a/castle-search/test/google-links.test.ts b/castle-search/test/google-links.test.ts new file mode 100644 index 0000000..f31f112 --- /dev/null +++ b/castle-search/test/google-links.test.ts @@ -0,0 +1,41 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { formatResults } from "../src/format.ts"; +import { isGoogleGoto } from "../src/google-links.ts"; +import { getEngine } from "../src/engines.ts"; +import { resolveLink } from "../src/search.ts"; + +test("only well-formed google /goto links count", () => { + assert.equal(isGoogleGoto("https://www.google.com/goto?url=CAESabc_-9"), true); + assert.equal(isGoogleGoto("https://www.google.co.uk/goto?url=CAESabc"), true); + assert.equal(isGoogleGoto("https://www.google.com/search?q=x"), false); + assert.equal(isGoogleGoto("https://example.com/goto?url=CAES"), false); + // Not a general navigator: nothing may ride along after the token. + assert.equal(isGoogleGoto("https://www.google.com/goto?url=CAES&x=https://evil.example"), false); + assert.equal(isGoogleGoto("https://www.google.com.evil.example/goto?url=CAES"), false); +}); + +test("the google self-host filter lets /goto through but still blocks google's own pages", () => { + const re = new RegExp(getEngine("google").extraction.selfHostPattern); + assert.equal(re.test("https://www.google.com/goto?url=CAES"), false); + assert.equal(re.test("https://www.google.com/search?q=x"), true); + assert.equal(re.test("https://accounts.google.com/ServiceLogin"), false); +}); + +test("resolveLink refuses anything that is not a google /goto link, before touching the browser", async () => { + await assert.rejects(() => resolveLink("https://example.com/", AbortSignal.timeout(1000)), /only follows Google result links/); + await assert.rejects(() => resolveLink("file:///etc/passwd", AbortSignal.timeout(1000)), /only follows Google result links/); +}); + +test("a wrapped result says so, shows Google's breadcrumb, and names the tool to resolve it", () => { + const text = formatResults({ + query: "q", searchUrl: "u", finalUrl: "u", endpoint: "e", engine: "Google", engineSource: "s", + results: [ + { title: "Wrapped", url: "https://www.google.com/goto?url=CAES", snippet: "snip", display: "GitHub — https://github.com › a › b" }, + { title: "Direct", url: "https://example.com/", snippet: "" }, + ], + }); + assert.match(text, /\[Google-wrapped link, real URL not yet known\. Google shows it as: GitHub — https:\/\/github\.com › a › b\. Pass it to castle_cdp_resolve/); + assert.equal((text.match(/Google-wrapped/g) ?? []).length, 1); +}); diff --git a/castle-search/test/resolve.test.ts b/castle-search/test/resolve.test.ts deleted file mode 100644 index bcd4ef4..0000000 --- a/castle-search/test/resolve.test.ts +++ /dev/null @@ -1,50 +0,0 @@ -import assert from "node:assert/strict"; -import { test } from "node:test"; - -import { isGoogleGoto, resolveGoogleLinks } from "../src/resolve.ts"; -import { getEngine } from "../src/engines.ts"; - -test("google /goto links are recognised, other google links are not", () => { - assert.equal(isGoogleGoto("https://www.google.com/goto?url=CAES"), true); - assert.equal(isGoogleGoto("https://www.google.co.uk/goto?url=CAES"), true); - assert.equal(isGoogleGoto("https://www.google.com/search?q=x"), false); - assert.equal(isGoogleGoto("https://example.com/goto?url=x"), false); -}); - -test("the google self-host filter lets /goto through but still blocks google's own pages", () => { - const re = new RegExp(getEngine("google").extraction.selfHostPattern); - assert.equal(re.test("https://www.google.com/goto?url=CAES"), false); - assert.equal(re.test("https://www.google.com/search?q=x"), true); - assert.equal(re.test("https://accounts.google.com/ServiceLogin"), false); -}); - -test("non-google results pass through untouched", async () => { - const input = [{ title: "t", url: "https://example.com/", snippet: "" }]; - assert.deepEqual(await resolveGoogleLinks(input), input); -}); - -test("a link that cannot be resolved keeps the page's breadcrumb in its snippet", async () => { - const wrapped = "https://www.google.com/goto?url=CAES"; - const real = globalThis.fetch; - // Google's interstitial: 200 with no Location header. - globalThis.fetch = (async () => new Response("<html>Verifying your request</html>", { status: 200 })) as typeof fetch; - try { - const [r] = await resolveGoogleLinks([{ title: "t", url: wrapped, snippet: "s", display: "GitHub — https://github.com › a › b" }]); - assert.equal(r?.url, wrapped); - assert.match(r?.snippet ?? "", /^\[link unresolved; Google shows it as: GitHub — https:\/\/github\.com › a › b\] s$/); - } finally { - globalThis.fetch = real; - } -}); - -test("a link that resolves is replaced and the snippet is left alone", async () => { - const real = globalThis.fetch; - globalThis.fetch = (async () => new Response(null, { status: 302, headers: { location: "https://github.com/a/b" } })) as typeof fetch; - try { - const [r] = await resolveGoogleLinks([{ title: "t", url: "https://www.google.com/goto?url=CAES", snippet: "s", display: "x" }]); - assert.equal(r?.url, "https://github.com/a/b"); - assert.equal(r?.snippet, "s"); - } finally { - globalThis.fetch = real; - } -}); |