1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
|
/**
* Google wraps every outbound result in `https://www.google.com/goto?url=<opaque
* token>`. The token is a protobuf blob with no URL in it, and the real target
* appears nowhere in the DOM, so the only way to learn it is to ask Google: an
* unauthenticated GET answers `302 Location: <target>`.
*
* That cannot be done from the page (cross-origin fetches see an opaque
* redirect), so it is done here, one request per result, in parallel.
*/
import type { SearchResult } from "./extract.ts";
const GOTO = /^https?:\/\/(www\.)?google\.[a-z.]+\/goto\?/;
const PER_LINK_TIMEOUT_MS = 4_000;
export const isGoogleGoto = (url: string): boolean => GOTO.test(url);
const resolveOne = async (url: string, signal?: AbortSignal): Promise<string> => {
const timeout = AbortSignal.timeout(PER_LINK_TIMEOUT_MS);
try {
const res = await fetch(url, {
redirect: "manual",
signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/157.0.0.0 Safari/537.36" },
});
const location = res.headers.get("location");
return location && /^https?:/.test(location) ? location : url;
} catch {
// Left wrapped rather than dropped: the wrapped link still redirects to the
// right page when followed, so the agent has a working link either way.
return url;
}
};
/**
* Replaces Google `/goto` links with their targets; anything else passes through.
*
* Best effort, and it degrades with volume: after a handful of requests Google
* answers 200 "Verifying your request… continue in a few seconds" (a 17-19 s
* countdown) instead of 302, and every link then stays wrapped. Waiting that out
* or skipping it is not on the table. A link that stays wrapped keeps the page's
* own breadcrumb in its snippet, so the agent can still tell where it points.
*/
export const resolveGoogleLinks = (results: SearchResult[], signal?: AbortSignal): Promise<SearchResult[]> =>
Promise.all(
results.map(async (r) => {
if (!isGoogleGoto(r.url)) return r;
const url = await resolveOne(r.url, signal);
if (!isGoogleGoto(url) || !r.display) return { ...r, url };
return { ...r, url, snippet: `[link unresolved; Google shows it as: ${r.display}] ${r.snippet}`.trim() };
}),
);
|
