/** * Search orchestration: resolve a browser, borrow a page, navigate, extract, * hand the page back. * * The connection is process-wide and lazy. It is *not* opened by the extension * factory — pi runs factories in invocations that never start a session, and * the docs are explicit that sockets must not start there. It is opened on the * first search and closed by an idempotent `session_shutdown`. */ import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts"; import { type CdpTarget, describeCdpTarget, describeResolutionFailure, resolveCdpTarget, } from "./endpoint.ts"; import { BrowserUnavailableError, NoResultsError, SearchChallengeError } from "./errors.ts"; import { PROBE_SCRIPT, type PageProbe, type SearchResult } from "./extract.ts"; import { readStoredTarget } from "./target-store.ts"; /** Per-search wall clock, covering connect, navigate and extract together. */ export const SEARCH_TIMEOUT_MS = 20_000; /** How many searches may drive the shared browser at once. */ export const MAX_CONCURRENT_SEARCHES = 2; /** Results sometimes land a beat after the load event; re-read a couple of times. */ const EXTRACT_RETRIES = 2; const EXTRACT_RETRY_DELAY_MS = 700; export type Recency = "day" | "week" | "month" | "year"; export type SearchRequest = { query: string; numResults: number; recency?: Recency | undefined; }; export type SearchOutcome = { results: SearchResult[]; searchUrl: string; /** The URL the page settled on, which differs from searchUrl after a redirect. */ finalUrl: string; endpoint: string; }; /** Google's date-restrict vocabulary. */ const RECENCY_CODE: Record = { day: "d", week: "w", month: "m", year: "y" }; /** * `udm=14` asks for the plain "Web" tab: no AI overview, no carousels, just * ranked links. That is both cheaper to parse and closer to what the agent * asked for. `hl=en` pins the result language so snippets do not change shape * with whatever locale the browser profile happens to carry. * * Two findings from testing against the real browser, both counter-intuitive * enough to be worth writing down: * * - `tbs=qdr:*` — the parameter the Tools menu puts in the URL — renders an * empty page in this profile, with `#search` present but no `#rso` and no * results at all. `as_qdr=*`, the older advanced-search parameter, works and * genuinely filters (verified: d/w/m/y all return dated results). * - Any date restriction combined with `udm=14` also renders that empty page. * * So a time-limited search drops `udm` and uses `as_qdr`. The classic layout it * falls back to parses fine — the extractor was checked against both. */ export const buildSearchUrl = (request: SearchRequest): string => { const params = new URLSearchParams({ q: request.query, num: String(request.numResults), hl: "en", }); if (request.recency) params.set("as_qdr", RECENCY_CODE[request.recency]); else params.set("udm", "14"); return `https://www.google.com/search?${params.toString()}`; }; // ── connection lifecycle ─────────────────────────────────────────────────── let connection: BrowserConnection | null = null; /** In-flight connect, so two concurrent searches share one dial rather than racing. */ let connecting: Promise | null = null; const resolveTarget = async (): Promise => { const resolution = resolveCdpTarget(process.env, await readStoredTarget()); if (resolution.kind === "invalid") { throw new BrowserUnavailableError(describeResolutionFailure(resolution)); } return resolution.target; }; /** * Return a live connection, dialling one if there isn't one or the last one * died. Reconnecting transparently matters here: castle recycles Chrome hourly, * so a session that searched an hour ago is holding a dead socket. */ const ensureConnection = async (signal?: AbortSignal): Promise => { if (connection && connection.isOpen) return connection; if (connection) { connection.close(); connection = null; } if (!connecting) { // Assigned synchronously, before the first await inside: two searches // starting in the same tick must share one dial, not open two sockets and // leak whichever loses the assignment race. connecting = (async () => { const target = await resolveTarget(); const conn = await BrowserConnection.connect(target); connection = conn; return conn; })().finally(() => { connecting = null; }); } // The caller's signal is raced here rather than passed into connect(). A // shared dial must not be cancelled by whichever caller happened to start it // — one search pressing Esc would otherwise fail the other search waiting on // the same connection. connect() is bounded by its own timeouts regardless. return raceAbort(connecting, signal); }; /** Idempotent: safe to call from `session_shutdown` however many times it fires. */ export const disconnect = (): void => { connection?.close(); connection = null; }; /** For status reporting — never dials. */ export const describeConfiguredEndpoint = async (): Promise => { const resolution = resolveCdpTarget(process.env, await readStoredTarget()); return resolution.kind === "invalid" ? describeResolutionFailure(resolution) : describeCdpTarget(resolution.target); }; // ── concurrency cap ──────────────────────────────────────────────────────── let active = 0; const waiting: Array<() => void> = []; /** Give a held slot to the next waiter, or return it to the pool. */ const handOff = (): void => { const next = waiting.shift(); // Hand the slot straight over rather than decrementing and letting the waiter // re-check, so a queued search cannot be overtaken by a newly arriving one. if (next) next(); else active--; }; const acquire = async (signal?: AbortSignal): Promise<() => void> => { if (active < MAX_CONCURRENT_SEARCHES) { active++; } else { let resolver!: () => void; const queued = new Promise((resolve) => { resolver = resolve; }); waiting.push(resolver); try { await raceAbort(queued, signal); // The slot was handed over by handOff(), which already counted it. } catch (e) { // An abort while queued must not strand the slot. Either we are still in // the queue — drop out of it — or the slot was handed to us in the same // tick the abort landed, in which case give it back. Without this, two // cancelled-while-queued searches would wedge the semaphore permanently. const index = waiting.indexOf(resolver); if (index !== -1) waiting.splice(index, 1); else handOff(); throw e; } } let released = false; return () => { if (released) return; released = true; handOff(); }; }; // ── the search itself ────────────────────────────────────────────────────── export type ProgressReporter = (phase: SearchPhase, note: string) => void; export type SearchPhase = "queued" | "connecting" | "opening" | "navigating" | "extracting"; const sleep = (ms: number, signal?: AbortSignal): Promise => raceAbort(new Promise((resolve) => setTimeout(resolve, ms)), signal); /** * Run one search. Throws on every failure — see errors.ts for why a returned * error object would be worse than useless here. */ export const search = async ( request: SearchRequest, signal: AbortSignal, report: ProgressReporter, ): Promise => { // Reported before acquiring, so a search that dies waiting for a slot is // distinguishable from one that died reaching the browser. The deadline is // the caller's and covers the queue wait too — with a cap of two and searches // that take a couple of seconds, waiting is the rare case, and a tool call // that silently takes twice its stated deadline would be worse. report("queued", "waiting for a search slot"); const release = await acquire(signal); try { report("connecting", "resolving browser"); const conn = await ensureConnection(signal); const endpoint = describeCdpTarget(conn.target); report("opening", `opening a page on ${endpoint}`); const page = await PageSession.open(conn, signal); const searchUrl = buildSearchUrl(request); try { report("navigating", `searching for "${request.query}"`); await page.navigate(searchUrl, signal); report("extracting", "reading results"); let probe = await page.evaluate(PROBE_SCRIPT, signal); for (let attempt = 0; attempt < EXTRACT_RETRIES; attempt++) { if (probe?.challenge || (probe?.results?.length ?? 0) > 0) break; await sleep(EXTRACT_RETRY_DELAY_MS, signal); probe = await page.evaluate(PROBE_SCRIPT, signal); } if (!probe) { throw new BrowserUnavailableError("the page returned nothing — the extraction script did not run"); } if (probe.challenge) { throw new SearchChallengeError({ challenge: probe.challenge.kind, // The page the human must clear is where the browser *ended up*, which // after a consent redirect is not the URL we asked for. pageUrl: probe.url || searchUrl, endpoint, detail: probe.challenge.detail || undefined, }); } if (probe.results.length === 0) { // "#rso missing entirely" and "#rso present but empty" are different // faults, and saying which saves the next person a browser session. const diagnosis = probe.container === "rso" ? "The results container was rendered but nothing could be parsed out of it, which suggests Google's " + "result markup changed and castle_cdp_search's extractor needs updating." : "Google served a page with no results area at all. Most likely this is a soft rate-limit: as a " + "block decays Google stops serving the /sorry/ challenge page and returns an empty result page " + "instead, which is indistinguishable from a genuine zero-hit search (observed lasting ~90 minutes " + "after heavy querying). It can also mean the query truly has no hits, or that this combination of " + "search parameters is not honoured for this browser profile."; throw new NoResultsError( `No results could be read from ${probe.url || searchUrl} (page title: "${probe.title}"). ${diagnosis} ` + `Before concluding the extractor is broken, try a broader query and the same query without recency; ` + `if those are also empty, treat it as rate limiting and wait rather than retrying in a loop.`, ); } return { results: probe.results.slice(0, request.numResults), searchUrl, finalUrl: probe.url || searchUrl, endpoint, }; } finally { // Always, including the challenge path. Leaving the tab open would let the // operator solve the captcha in place, but it would also litter a browser // someone else is using with abandoned tabs on every failure; the error // carries the URL instead, and the cookie it sets is profile-wide. await page.close(); } } finally { release(); } };