castle-search: resolve Google links lazily, in Chrome, not eagerly over plain HTTPHEADmaster

The Node-side resolver sent one curl-like request per result with no
cookies, JS or browser; Google answers that with its 'Verifying your
request' countdown after a handful, and the testing itself triggered it.
A click or direct navigation in Chrome redirects at once (measured).

Search now leaves Google links wrapped and prints the breadcrumb Google
shows for each. castle_cdp_resolve follows one wrapped link in a Chrome tab
when the agent picks it, accepts only google.com/goto?url=<token>, and
reports the verification page instead of waiting it out.

6 files changed, 123 insertions, 56 deletions

diff --git a/castle-search/src/errors.ts b/castle-search/src/errors.ts
index 2318b27..95fa7ef 100644
--- a/castle-search/src/errors.ts
+++ b/castle-search/src/errors.ts
@@ -81,3 +81,22 @@ export class SearchAbortedError extends Error {
 }
 
 export const messageOf = (e: unknown): string => (e instanceof Error ? e.message : String(e));
+
+/**
+ * Google answered a `/goto` link with its "Verifying your request" page instead
+ * of redirecting. It is a bot check with a countdown; it is reported, never
+ * waited out or clicked through.
+ */
+export class GoogleVerificationError extends Error {
+  override readonly name = "GoogleVerificationError";
+  readonly pageUrl: string;
+  constructor(pageUrl: string) {
+    super(
+      "Google showed its \"Verifying your request\" page instead of redirecting, so the real URL could not be " +
+        "read. This is a rate-based bot check: opening many links in a row triggers it, and it eases after a few " +
+        "minutes. Do not retry in a loop. Use the breadcrumb printed under the result to judge where it points, " +
+        "pick fewer links, or search again with another engine (duckduckgo, brave, bing), which return real URLs.",
+    );
+    this.pageUrl = pageUrl;
+  }
+}
diff --git a/castle-search/src/format.ts b/castle-search/src/format.ts
index aef6c3b..9c7c3cc 100644
--- a/castle-search/src/format.ts
+++ b/castle-search/src/format.ts
@@ -8,6 +8,7 @@
  */
 
 import type { SearchResult } from "./extract.ts";
+import { isGoogleGoto } from "./google-links.ts";
 
 export type FormatInput = {
   query: string;
@@ -36,6 +37,13 @@ export const formatResults = (input: FormatInput): string => {
   input.results.forEach((result, index) => {
     lines.push(`${index + 1}. ${result.title}`);
     lines.push(`   ${result.url}`);
+    // Google wraps every link and only reveals the target when followed. Say so
+    // here, next to the link, with what Google displays for it — that is often
+    // enough to decide without resolving anything.
+    if (isGoogleGoto(result.url)) {
+      const shown = result.display ? ` Google shows it as: ${result.display}.` : "";
+      lines.push(`   [Google-wrapped link, real URL not yet known.${shown} Pass it to castle_cdp_resolve if you need to open it.]`);
+    }
     if (result.snippet) lines.push(`   ${result.snippet}`);
     lines.push("");
   });
diff --git a/castle-search/src/google-links.ts b/castle-search/src/google-links.ts
new file mode 100644
index 0000000..7500e2d
--- /dev/null
+++ b/castle-search/src/google-links.ts
@@ -0,0 +1,15 @@
+/**
+ * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque
+ * token>`. The token is a protobuf blob with no URL in it and the target appears
+ * nowhere in the page, so the only way to learn it is to follow the link.
+ *
+ * It is followed in Chrome, lazily, one link at a time, when the caller asks
+ * (see `resolveLink` in search.ts) — not here, and not for every result. A
+ * plain HTTP client gets Google's "Verifying your request" countdown after a
+ * handful of requests; the browser, with its own profile, gets the redirect.
+ */
+
+const GOTO = /^https?:\/\/(www\.)?google\.(?:com?\.)?[a-z]{2,3}\/goto\?url=[A-Za-z0-9_-]+$/;
+
+/** Only these may be opened by castle_cdp_resolve — it is not a general navigator. */
+export const isGoogleGoto = (url: string): boolean => GOTO.test(url);
diff --git a/castle-search/src/index.ts b/castle-search/src/index.ts
index 29cdaaf..9b7c143 100644
--- a/castle-search/src/index.ts
+++ b/castle-search/src/index.ts
@@ -43,6 +43,7 @@ import {
   describeConfiguredEndpoint,
   describeConfiguredEngine,
   disconnect,
+  resolveLink,
   search,
 } from "./search.ts";
 
@@ -202,6 +203,38 @@ export default function (pi: ExtensionAPI): void {
   });
 
   pi.registerTool({
+    name: "castle_cdp_resolve",
+    label: "Castle Resolve",
+    description:
+      "Turn one Google-wrapped result link (https://www.google.com/goto?url=…, as printed by castle_cdp_search " +
+      "with engine google) into the real destination URL, by following it in the same Chrome. Opens one tab, " +
+      "reads where it lands, closes it. It does not read the page. Resolve only the results you mean to open: " +
+      "Google answers a burst of links with a \"Verifying your request\" countdown, which this tool reports " +
+      "instead of waiting out. Other engines already return real URLs and need no resolving.",
+    promptSnippet: "Resolve one Google-wrapped result link to its real URL (castle_cdp_resolve)",
+    promptGuidelines: [
+      "castle_cdp_search on Google returns wrapped links. Read the titles, snippets and the breadcrumb Google " +
+        "shows for each, decide which result you actually need, and resolve only that one with castle_cdp_resolve. " +
+        "Do not resolve every result: that burst is what triggers Google's verification page.",
+    ],
+    parameters: Type.Object({
+      url: Type.String({
+        description: "A https://www.google.com/goto?url=… link exactly as castle_cdp_search printed it.",
+        minLength: 1,
+      }),
+    }),
+    async execute(_toolCallId, params, signal, _onUpdate, _ctx) {
+      const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+      const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
+      const out = await resolveLink(params.url.trim(), combined);
+      return {
+        content: [{ type: "text", text: `${out.to}\n(resolved via Chrome at ${out.endpoint})` }],
+        details: { from: out.from, to: out.to, endpoint: out.endpoint },
+      };
+    },
+  });
+
+  pi.registerTool({
     name: "castle_cdp_search",
     label: "Castle Search",
     description:
@@ -209,7 +242,9 @@ export default function (pi: ExtensionAPI): void {
       "DevTools Protocol. Returns ranked results with titles, URLs and snippets. Because it uses a real " +
       "logged-in browser rather than a search API, it reaches pages that block datacentre traffic. It " +
       "returns search results only — it does not fetch or read the linked pages, so follow up with a " +
-      "fetch/read tool for full page content. Supports four engines with independent indexes and " +
+      "fetch/read tool for full page content. Google results come back as wrapped links with the " +
+      "breadcrumb Google shows; resolve the ones you need with castle_cdp_resolve. " +
+      "Supports four engines with independent indexes and " +
       `independent rate limits (${ENGINE_IDS.join(", ")}); pass engine to switch when one is blocked. ` +
       `Output is truncated at ${formatSize(DEFAULT_MAX_BYTES)} or ${DEFAULT_MAX_LINES} lines.`,
     promptSnippet:
diff --git a/castle-search/src/resolve.ts b/castle-search/src/resolve.ts
deleted file mode 100644
index f432f7d..0000000
--- a/castle-search/src/resolve.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque
- * token>`. The token is a protobuf blob with no URL in it, and the real target
- * appears nowhere in the DOM, so the only way to learn it is to ask Google: an
- * unauthenticated GET answers `302 Location: <target>`.
- *
- * That cannot be done from the page (cross-origin fetches see an opaque
- * redirect), so it is done here, one request per result, in parallel.
- */
-
-import type { SearchResult } from "./extract.ts";
-
-const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/;
-const PER_LINK_TIMEOUT_MS = 4_000;
-
-export const isGoogleGoto = (url: string): boolean => GOTO.test(url);
-
-const resolveOne = async (url: string, signal?: AbortSignal): Promise<string> => {
-  const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS);
-  try {
-    const res = await fetch(url, {
-      redirect: "manual",
-      signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
-      headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" },
-    });
-    const location = res.headers.get("location");
-    return location && /^https?:/.test(location) ? location : url;
-  } catch {
-    // Left wrapped rather than dropped: the wrapped link still redirects to the
-    // right page when followed, so the agent has a working link either way.
-    return url;
-  }
-};
-
-/**
- * Replaces Google `/goto` links with their targets; anything else passes through.
- *
- * Best effort, and it degrades with volume: after a handful of requests Google
- * answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s
- * countdown) instead of 302, and every link then stays wrapped. Waiting that out
- * or skipping it is not on the table. A link that stays wrapped keeps the page's
- * own breadcrumb in its snippet, so the agent can still tell where it points.
- */
-export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise<SearchResult[]> =>
-  Promise.all(
-    results.map(async (r) => {
-      if (!isGoogleGoto(r.url)) return r;
-      const url = await resolveOne(r.url, signal);
-      if (!isGoogleGoto(url) || !r.display) return { ...r, url };
-      return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() };
-    }),
-  );
-
diff --git a/castle-search/src/search.ts b/castle-search/src/search.ts
index c64735d..b5effb8 100644
--- a/castle-search/src/search.ts
+++ b/castle-search/src/search.ts
@@ -8,7 +8,8 @@
  * first search and closed by an idempotent `session_shutdown`.
  */
 
-import { resolveGoogleLinks } from "./resolve.ts";
+import { isGoogleGoto } from "./google-links.ts";
+import { GoogleVerificationError } from "./errors.ts";
 import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts";
 import {
   type CdpTarget,
@@ -332,7 +333,7 @@ export const search = async (
       }
 
       return {
-        results: await resolveGoogleLinks(probe.results.slice(0, request.numResults), signal),
+        results: probe.results.slice(0, request.numResults),
         searchUrl,
         finalUrl: probe.url || searchUrl,
         endpoint,
@@ -350,3 +351,45 @@ export const search = async (
     release();
   }
 };
+
+export type ResolveOutcome = { from: string; to: string; endpoint: string };
+
+/**
+ * Follows one Google `/goto` link in Chrome and reports where it lands.
+ *
+ * Lazy by design: search leaves the links wrapped, and the caller resolves only
+ * the ones it means to open. One tab, one navigation, closed afterwards — the
+ * pattern of a person clicking a result, which Google serves directly.
+ *
+ * If Google answers with its "Verifying your request" page instead, the tab is
+ * still on google.com when the page loads. That is reported as such. The
+ * countdown is not waited out and nothing is clicked.
+ */
+export const resolveLink = async (url: string, signal: AbortSignal): Promise<ResolveOutcome> => {
+  if (!isGoogleGoto(url)) {
+    throw new Error(
+      "castle_cdp_resolve only follows Google result links of the form https://www.google.com/goto?url=… " +
+        "as printed by castle_cdp_search. Results from other engines already carry their real URL.",
+    );
+  }
+  const release = await acquire(signal);
+  try {
+    const conn = await ensureConnection(signal);
+    const endpoint = describeCdpTarget(conn.target);
+    const page = await PageSession.open(conn, signal);
+    try {
+      await page.navigate(url, signal);
+      const landed = await page.evaluate<string>("location.href", signal);
+      if (!landed || isGoogleHost(landed)) {
+        throw new GoogleVerificationError(landed || url);
+      }
+      return { from: url, to: landed, endpoint };
+    } finally {
+      await page.close();
+    }
+  } finally {
+    release();
+  }
+};
+
+const isGoogleHost = (url: string): boolean => /^https?:\/\/([a-z0-9-]+\.)*google\.(?:com?\.)?[a-z]{2,3}\//.test(url);