diff options
| author | amankumarsingh77 <amankumarsingh7702@gmail.com> | 2026-07-08 21:13:15 +0530 |
|---|---|---|
| committer | amankumarsingh77 <amankumarsingh7702@gmail.com> | 2026-07-08 21:13:15 +0530 |
| commit | c5c42df5976ed63faac2adcf3b04a4ad2f519b32 (patch) | |
| tree | 77d389a590864eaf5c2408c9bc99ed07f55343d8 /src/domains/search/google-serp.ts | |
| parent | 7826c950adee89512162185d07fee565ab0e96cc (diff) | |
feat: add browser_web_search tool (real-Chrome SERP scrape, no API key)
Thin query→ranked-links primitive. Isolated-attach tab lifecycle mirroring
browser_open_urls (never touches the user's current tab); pure Google SERP
parser with semantic (a h3) extraction and redirect unwrapping; CAPTCHA and
no-results surfaced as invalid_state + details.reason. Fixture-based parser
tests (9 scenarios). Establishes the isolated-tab lifecycle and tsx+fixture
test convention that browser_read_page and deep-research build on.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Diffstat (limited to 'src/domains/search/google-serp.ts')
| -rw-r--r-- | src/domains/search/google-serp.ts | 114 |
1 files changed, 114 insertions, 0 deletions
diff --git a/src/domains/search/google-serp.ts b/src/domains/search/google-serp.ts new file mode 100644 index 0000000..7e49bea --- /dev/null +++ b/src/domains/search/google-serp.ts @@ -0,0 +1,114 @@ +/** + * Pure Google SERP parsing — no CDP, no browser. Everything here is a pure + * function over data captured from the page, so it is unit-testable against + * saved fixtures (see test/deep-research/serp-parser-test.ts). + * + * ADR-0001: results are extracted by structure/semantics (anchors carrying an + * <h3> heading in the main results column), never by brittle Google CSS class + * names, and Google redirect wrappers are unwound to the real target URL. + */ + +/** One raw anchor captured in-page: a link with a heading and nearby snippet. */ +export type SerpAnchor = { + readonly href: string; + readonly heading: string; + readonly snippet: string; +}; + +/** The JSON payload the in-page extraction expression returns. */ +export type SerpExtraction = { + readonly anchors: ReadonlyArray<SerpAnchor>; + readonly pageText: string; +}; + +/** A parsed, ranked search result — the tool's public output shape. */ +export type SearchResult = { + readonly title: string; + readonly url: string; + readonly snippet: string; + readonly rank: number; +}; + +/** Why a SERP yielded no usable results, or that it did. */ +export type SerpVerdict = "ok" | "captcha" | "no_results"; + +const CAPTCHA_MARKERS: ReadonlyArray<string> = [ + "unusual traffic", + "not a robot", + "recaptcha", + "detected unusual", + "systems have detected", + "before you continue to google", + "our systems have detected unusual traffic", +]; + +/** + * Decode a Google redirect wrapper (`/url?q=<real>&sa=...`) to its target. + * Returns the input unchanged when it is not a wrapper. + */ +const unwrapRedirect = (href: string): string => { + const wrapper = href.match(/^(?:https?:\/\/[^/]*google\.[^/]*)?\/url\?/i); + if (!wrapper) return href; + const query = href.slice(href.indexOf("?") + 1); + const target = new URLSearchParams(query).get("q"); + return target ?? href; +}; + +const isExternalResult = (url: string): boolean => { + if (!/^https?:\/\//i.test(url)) return false; + try { + const host = new URL(url).hostname.toLowerCase(); + if (host === "google.com" || host.endsWith(".google.com")) return false; + if (host === "googleusercontent.com" || host.endsWith(".googleusercontent.com")) return false; + return true; + } catch { + return false; + } +}; + +/** Normalized key for dedupe: origin + pathname, trailing slash and hash removed. */ +const dedupeKey = (url: string): string => { + try { + const u = new URL(url); + return `${u.origin}${u.pathname.replace(/\/$/, "")}`.toLowerCase(); + } catch { + return url.toLowerCase(); + } +}; + +/** + * Parse captured anchors into ranked results: unwrap redirects, keep only + * external http(s) links, dedupe by normalized url (first wins), rank by input + * order, cap to `limit`. + */ +export const parseGoogleSerp = (anchors: ReadonlyArray<SerpAnchor>, limit: number): ReadonlyArray<SearchResult> => { + const seen = new Set<string>(); + const results: SearchResult[] = []; + for (const anchor of anchors) { + if (results.length >= limit) break; + const url = unwrapRedirect(anchor.href); + if (!isExternalResult(url)) continue; + const key = dedupeKey(url); + if (seen.has(key)) continue; + seen.add(key); + results.push({ + title: anchor.heading.trim(), + url, + snippet: anchor.snippet.trim(), + rank: results.length + 1, + }); + } + return results; +}; + +/** + * Decide whether the SERP is usable, blocked by a CAPTCHA, or genuinely empty. + * `resultCount` is the number of parsed results; `pageText` is a bounded slice + * of the page's visible text. + */ +export const classifySerp = (pageText: string, resultCount: number): SerpVerdict => { + if (resultCount > 0) return "ok"; + const lower = pageText.toLowerCase(); + if (CAPTCHA_MARKERS.some((marker) => lower.includes(marker))) return "captcha"; + return "no_results"; +}; |
