aboutsummaryrefslogtreecommitdiff
path: root/src/domains/search/google-serp.ts
blob: 7e49bea61c72fdd1cf0e3b3acc9b5d551005cb4d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
/**
 * Pure Google SERP parsing — no CDP, no browser. Everything here is a pure
 * function over data captured from the page, so it is unit-testable against
 * saved fixtures (see test/deep-research/serp-parser-test.ts).
 *
 * ADR-0001: results are extracted by structure/semantics (anchors carrying an
 * <h3> heading in the main results column), never by brittle Google CSS class
 * names, and Google redirect wrappers are unwound to the real target URL.
 */

/** One raw anchor captured in-page: a link with a heading and nearby snippet. */
export type SerpAnchor = {
  readonly href: string;
  readonly heading: string;
  readonly snippet: string;
};

/** The JSON payload the in-page extraction expression returns. */
export type SerpExtraction = {
  readonly anchors: ReadonlyArray<SerpAnchor>;
  readonly pageText: string;
};

/** A parsed, ranked search result — the tool's public output shape. */
export type SearchResult = {
  readonly title: string;
  readonly url: string;
  readonly snippet: string;
  readonly rank: number;
};

/** Why a SERP yielded no usable results, or that it did. */
export type SerpVerdict = "ok" | "captcha" | "no_results";

const CAPTCHA_MARKERS: ReadonlyArray<string> = [
  "unusual traffic",
  "not a robot",
  "recaptcha",
  "detected unusual",
  "systems have detected",
  "before you continue to google",
  "our systems have detected unusual traffic",
];

/**
 * Decode a Google redirect wrapper (`/url?q=<real>&sa=...`) to its target.
 * Returns the input unchanged when it is not a wrapper.
 */
const unwrapRedirect = (href: string): string => {
  const wrapper = href.match(/^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i);
  if (!wrapper) return href;
  const query = href.slice(href.indexOf("?") + 1);
  const target = new URLSearchParams(query).get("q");
  return target ?? href;
};

const isExternalResult = (url: string): boolean => {
  if (!/^https?:\/\//i.test(url)) return false;
  try {
    const host = new URL(url).hostname.toLowerCase();
    if (host === "google.com" || host.endsWith(".google.com")) return false;
    if (host === "googleusercontent.com" || host.endsWith(".googleusercontent.com")) return false;
    return true;
  } catch {
    return false;
  }
};

/** Normalized key for dedupe: origin + pathname, trailing slash and hash removed. */
const dedupeKey = (url: string): string => {
  try {
    const u = new URL(url);
    return `${u.origin}${u.pathname.replace(/\/$/, "")}`.toLowerCase();
  } catch {
    return url.toLowerCase();
  }
};

/**
 * Parse captured anchors into ranked results: unwrap redirects, keep only
 * external http(s) links, dedupe by normalized url (first wins), rank by input
 * order, cap to `limit`.
 */
export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: number): ReadonlyArray<SearchResult> => {
  const seen = new Set<string>();
  const results: SearchResult[] = [];
  for (const anchor of anchors) {
    if (results.length >= limit) break;
    const url = unwrapRedirect(anchor.href);
    if (!isExternalResult(url)) continue;
    const key = dedupeKey(url);
    if (seen.has(key)) continue;
    seen.add(key);
    results.push({
      title: anchor.heading.trim(),
      url,
      snippet: anchor.snippet.trim(),
      rank: results.length + 1,
    });
  }
  return results;
};

/**
 * Decide whether the SERP is usable, blocked by a CAPTCHA, or genuinely empty.
 * `resultCount` is the number of parsed results; `pageText` is a bounded slice
 * of the page's visible text.
 */
export const classifySerp = (pageText: string, resultCount: number): SerpVerdict => {
  if (resultCount > 0) return "ok";
  const lower = pageText.toLowerCase();
  if (CAPTCHA_MARKERS.some((marker) => lower.includes(marker))) return "captcha";
  return "no_results";
};