castle-search: resolve Google links lazily, in Chrome, not eagerly over plain HTTPHEADmaster

The Node-side resolver sent one curl-like request per result with no
cookies, JS or browser; Google answers that with its 'Verifying your
request' countdown after a handful, and the testing itself triggered it.
A click or direct navigation in Chrome redirects at once (measured).

Search now leaves Google links wrapped and prints the breadcrumb Google
shows for each. castle_cdp_resolve follows one wrapped link in a Chrome tab
when the agent picks it, accepts only google.com/goto?url=<token>, and
reports the verification page instead of waiting it out.

1 files changed, 0 insertions, 53 deletions

diff --git a/castle-search/src/resolve.ts b/castle-search/src/resolve.ts
deleted file mode 100644
index f432f7d..0000000
--- a/castle-search/src/resolve.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Google wraps every outbound result in `https://www.google.com/goto?url=<opaque
- * token>`. The token is a protobuf blob with no URL in it, and the real target
- * appears nowhere in the DOM, so the only way to learn it is to ask Google: an
- * unauthenticated GET answers `302 Location: <target>`.
- *
- * That cannot be done from the page (cross-origin fetches see an opaque
- * redirect), so it is done here, one request per result, in parallel.
- */
-
-import type { SearchResult } from "./extract.ts";
-
-const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/;
-const PER_LINK_TIMEOUT_MS = 4_000;
-
-export const isGoogleGoto = (url: string): boolean => GOTO.test(url);
-
-const resolveOne = async (url: string, signal?: AbortSignal): Promise<string> => {
-  const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS);
-  try {
-    const res = await fetch(url, {
-      redirect: "manual",
-      signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
-      headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" },
-    });
-    const location = res.headers.get("location");
-    return location && /^https?:/.test(location) ? location : url;
-  } catch {
-    // Left wrapped rather than dropped: the wrapped link still redirects to the
-    // right page when followed, so the agent has a working link either way.
-    return url;
-  }
-};
-
-/**
- * Replaces Google `/goto` links with their targets; anything else passes through.
- *
- * Best effort, and it degrades with volume: after a handful of requests Google
- * answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s
- * countdown) instead of 302, and every link then stays wrapped. Waiting that out
- * or skipping it is not on the table. A link that stays wrapped keeps the page's
- * own breadcrumb in its snippet, so the agent can still tell where it points.
- */
-export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise<SearchResult[]> =>
-  Promise.all(
-    results.map(async (r) => {
-      if (!isGoogleGoto(r.url)) return r;
-      const url = await resolveOne(r.url, signal);
-      if (!isGoogleGoto(url) || !r.display) return { ...r, url };
-      return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() };
-    }),
-  );
-