diff options
Diffstat (limited to 'castle-search/src')
| -rw-r--r-- | castle-search/src/errors.ts | 19 | ||||
| -rw-r--r-- | castle-search/src/format.ts | 8 | ||||
| -rw-r--r-- | castle-search/src/google-links.ts | 15 | ||||
| -rw-r--r-- | castle-search/src/index.ts | 37 | ||||
| -rw-r--r-- | castle-search/src/resolve.ts | 53 | ||||
| -rw-r--r-- | castle-search/src/search.ts | 47 |
6 files changed, 123 insertions, 56 deletions
diff --git a/castle-search/src/errors.ts b/castle-search/src/errors.ts index 2318b27..95fa7ef 100644 --- a/castle-search/src/errors.ts +++ b/castle-search/src/errors.ts @@ -81,3 +81,22 @@ export class SearchAbortedError extends Error { } export const messageOf = (e: unknown): string => (e instanceof Error ? e.message : String(e)); + +/** + * Google answered a `/goto` link with its "Verifying your request" page instead + * of redirecting. It is a bot check with a countdown; it is reported, never + * waited out or clicked through. + */ +export class GoogleVerificationError extends Error { + override readonly name = "GoogleVerificationError"; + readonly pageUrl: string; + constructor(pageUrl: string) { + super( + "Google showed its \"Verifying your request\" page instead of redirecting, so the real URL could not be " + + "read. This is a rate-based bot check: opening many links in a row triggers it, and it eases after a few " + + "minutes. Do not retry in a loop. Use the breadcrumb printed under the result to judge where it points, " + + "pick fewer links, or search again with another engine (duckduckgo, brave, bing), which return real URLs.", + ); + this.pageUrl = pageUrl; + } +} diff --git a/castle-search/src/format.ts b/castle-search/src/format.ts index aef6c3b..9c7c3cc 100644 --- a/castle-search/src/format.ts +++ b/castle-search/src/format.ts @@ -8,6 +8,7 @@ */ import type { SearchResult } from "./extract.ts"; +import { isGoogleGoto } from "./google-links.ts"; export type FormatInput = { query: string; @@ -36,6 +37,13 @@ export const formatResults = (input: FormatInput): string => { input.results.forEach((result, index) => { lines.push(`${index + 1}. ${result.title}`); lines.push(` ${result.url}`); + // Google wraps every link and only reveals the target when followed. Say so + // here, next to the link, with what Google displays for it — that is often + // enough to decide without resolving anything. + if (isGoogleGoto(result.url)) { + const shown = result.display ? ` Google shows it as: ${result.display}.` : ""; + lines.push(` [Google-wrapped link, real URL not yet known.${shown} Pass it to castle_cdp_resolve if you need to open it.]`); + } if (result.snippet) lines.push(` ${result.snippet}`); lines.push(""); }); diff --git a/castle-search/src/google-links.ts b/castle-search/src/google-links.ts new file mode 100644 index 0000000..7500e2d --- /dev/null +++ b/castle-search/src/google-links.ts @@ -0,0 +1,15 @@ +/** + * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque + * token>`. The token is a protobuf blob with no URL in it and the target appears + * nowhere in the page, so the only way to learn it is to follow the link. + * + * It is followed in Chrome, lazily, one link at a time, when the caller asks + * (see `resolveLink` in search.ts) — not here, and not for every result. A + * plain HTTP client gets Google's "Verifying your request" countdown after a + * handful of requests; the browser, with its own profile, gets the redirect. + */ + +const GOTO = /^https?:\/\/(www\.)?google\.(?:com?\.)?[a-z]{2,3}\/goto\?url=[A-Za-z0-9_-]+$/; + +/** Only these may be opened by castle_cdp_resolve — it is not a general navigator. */ +export const isGoogleGoto = (url: string): boolean => GOTO.test(url); diff --git a/castle-search/src/index.ts b/castle-search/src/index.ts index 29cdaaf..9b7c143 100644 --- a/castle-search/src/index.ts +++ b/castle-search/src/index.ts @@ -43,6 +43,7 @@ import { describeConfiguredEndpoint, describeConfiguredEngine, disconnect, + resolveLink, search, } from "./search.ts"; @@ -202,6 +203,38 @@ export default function (pi: ExtensionAPI): void { }); pi.registerTool({ + name: "castle_cdp_resolve", + label: "Castle Resolve", + description: + "Turn one Google-wrapped result link (https://www.google.com/goto?url=…, as printed by castle_cdp_search " + + "with engine google) into the real destination URL, by following it in the same Chrome. Opens one tab, " + + "reads where it lands, closes it. It does not read the page. Resolve only the results you mean to open: " + + "Google answers a burst of links with a \"Verifying your request\" countdown, which this tool reports " + + "instead of waiting out. Other engines already return real URLs and need no resolving.", + promptSnippet: "Resolve one Google-wrapped result link to its real URL (castle_cdp_resolve)", + promptGuidelines: [ + "castle_cdp_search on Google returns wrapped links. Read the titles, snippets and the breadcrumb Google " + + "shows for each, decide which result you actually need, and resolve only that one with castle_cdp_resolve. " + + "Do not resolve every result: that burst is what triggers Google's verification page.", + ], + parameters: Type.Object({ + url: Type.String({ + description: "A https://www.google.com/goto?url=… link exactly as castle_cdp_search printed it.", + minLength: 1, + }), + }), + async execute(_toolCallId, params, signal, _onUpdate, _ctx) { + const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS); + const combined = signal ? AbortSignal.any([signal, timeout]) : timeout; + const out = await resolveLink(params.url.trim(), combined); + return { + content: [{ type: "text", text: `${out.to}\n(resolved via Chrome at ${out.endpoint})` }], + details: { from: out.from, to: out.to, endpoint: out.endpoint }, + }; + }, + }); + + pi.registerTool({ name: "castle_cdp_search", label: "Castle Search", description: @@ -209,7 +242,9 @@ export default function (pi: ExtensionAPI): void { "DevTools Protocol. Returns ranked results with titles, URLs and snippets. Because it uses a real " + "logged-in browser rather than a search API, it reaches pages that block datacentre traffic. It " + "returns search results only — it does not fetch or read the linked pages, so follow up with a " + - "fetch/read tool for full page content. Supports four engines with independent indexes and " + + "fetch/read tool for full page content. Google results come back as wrapped links with the " + + "breadcrumb Google shows; resolve the ones you need with castle_cdp_resolve. " + + "Supports four engines with independent indexes and " + `independent rate limits (${ENGINE_IDS.join(", ")}); pass engine to switch when one is blocked. ` + `Output is truncated at ${formatSize(DEFAULT_MAX_BYTES)} or ${DEFAULT_MAX_LINES} lines.`, promptSnippet: diff --git a/castle-search/src/resolve.ts b/castle-search/src/resolve.ts deleted file mode 100644 index f432f7d..0000000 --- a/castle-search/src/resolve.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque - * token>`. The token is a protobuf blob with no URL in it, and the real target - * appears nowhere in the DOM, so the only way to learn it is to ask Google: an - * unauthenticated GET answers `302 Location: <target>`. - * - * That cannot be done from the page (cross-origin fetches see an opaque - * redirect), so it is done here, one request per result, in parallel. - */ - -import type { SearchResult } from "./extract.ts"; - -const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/; -const PER_LINK_TIMEOUT_MS = 4_000; - -export const isGoogleGoto = (url: string): boolean => GOTO.test(url); - -const resolveOne = async (url: string, signal?: AbortSignal): Promise<string> => { - const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS); - try { - const res = await fetch(url, { - redirect: "manual", - signal: signal ? AbortSignal.any([signal, timeout]) : timeout, - headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" }, - }); - const location = res.headers.get("location"); - return location && /^https?:/.test(location) ? location : url; - } catch { - // Left wrapped rather than dropped: the wrapped link still redirects to the - // right page when followed, so the agent has a working link either way. - return url; - } -}; - -/** - * Replaces Google `/goto` links with their targets; anything else passes through. - * - * Best effort, and it degrades with volume: after a handful of requests Google - * answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s - * countdown) instead of 302, and every link then stays wrapped. Waiting that out - * or skipping it is not on the table. A link that stays wrapped keeps the page's - * own breadcrumb in its snippet, so the agent can still tell where it points. - */ -export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise<SearchResult[]> => - Promise.all( - results.map(async (r) => { - if (!isGoogleGoto(r.url)) return r; - const url = await resolveOne(r.url, signal); - if (!isGoogleGoto(url) || !r.display) return { ...r, url }; - return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() }; - }), - ); - diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts index c64735d..b5effb8 100644 --- a/castle-search/src/search.ts +++ b/castle-search/src/search.ts @@ -8,7 +8,8 @@ * first search and closed by an idempotent `session_shutdown`. */ -import { resolveGoogleLinks } from "./resolve.ts"; +import { isGoogleGoto } from "./google-links.ts"; +import { GoogleVerificationError } from "./errors.ts"; import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts"; import { type CdpTarget, @@ -332,7 +333,7 @@ export const search = async ( } return { - results: await resolveGoogleLinks(probe.results.slice(0, request.numResults), signal), + results: probe.results.slice(0, request.numResults), searchUrl, finalUrl: probe.url || searchUrl, endpoint, @@ -350,3 +351,45 @@ export const search = async ( release(); } }; + +export type ResolveOutcome = { from: string; to: string; endpoint: string }; + +/** + * Follows one Google `/goto` link in Chrome and reports where it lands. + * + * Lazy by design: search leaves the links wrapped, and the caller resolves only + * the ones it means to open. One tab, one navigation, closed afterwards — the + * pattern of a person clicking a result, which Google serves directly. + * + * If Google answers with its "Verifying your request" page instead, the tab is + * still on google.com when the page loads. That is reported as such. The + * countdown is not waited out and nothing is clicked. + */ +export const resolveLink = async (url: string, signal: AbortSignal): Promise<ResolveOutcome> => { + if (!isGoogleGoto(url)) { + throw new Error( + "castle_cdp_resolve only follows Google result links of the form https://www.google.com/goto?url=… " + + "as printed by castle_cdp_search. Results from other engines already carry their real URL.", + ); + } + const release = await acquire(signal); + try { + const conn = await ensureConnection(signal); + const endpoint = describeCdpTarget(conn.target); + const page = await PageSession.open(conn, signal); + try { + await page.navigate(url, signal); + const landed = await page.evaluate<string>("location.href", signal); + if (!landed || isGoogleHost(landed)) { + throw new GoogleVerificationError(landed || url); + } + return { from: url, to: landed, endpoint }; + } finally { + await page.close(); + } + } finally { + release(); + } +}; + +const isGoogleHost = (url: string): boolean => /^https?:\/\/([a-z0-9-]+\.)*google\.(?:com?\.)?[a-z]{2,3}\//.test(url); |