1 files changed, 279 insertions, 0 deletions
diff --git a/src/search.ts b/src/search.ts
new file mode 100644
index 0000000..fa7ce35
--- /dev/null
+++ b/src/search.ts
@@ -0,0 +1,279 @@
+/**
+ * Search orchestration: resolve a browser, borrow a page, navigate, extract,
+ * hand the page back.
+ *
+ * The connection is process-wide and lazy. It is *not* opened by the extension
+ * factory — pi runs factories in invocations that never start a session, and
+ * the docs are explicit that sockets must not start there. It is opened on the
+ * first search and closed by an idempotent `session_shutdown`.
+ */
+
+import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts";
+import {
+  type CdpTarget,
+  describeCdpTarget,
+  describeResolutionFailure,
+  resolveCdpTarget,
+} from "./endpoint.ts";
+import { BrowserUnavailableError, NoResultsError, SearchChallengeError } from "./errors.ts";
+import { PROBE_SCRIPT, type PageProbe, type SearchResult } from "./extract.ts";
+import { readStoredTarget } from "./target-store.ts";
+
+/** Per-search wall clock, covering connect, navigate and extract together. */
+export const SEARCH_TIMEOUT_MS = 20_000;
+/** How many searches may drive the shared browser at once. */
+export const MAX_CONCURRENT_SEARCHES = 2;
+/** Results sometimes land a beat after the load event; re-read a couple of times. */
+const EXTRACT_RETRIES = 2;
+const EXTRACT_RETRY_DELAY_MS = 700;
+
+export type Recency = "day" | "week" | "month" | "year";
+
+export type SearchRequest = {
+  query: string;
+  numResults: number;
+  recency?: Recency | undefined;
+};
+
+export type SearchOutcome = {
+  results: SearchResult[];
+  searchUrl: string;
+  /** The URL the page settled on, which differs from searchUrl after a redirect. */
+  finalUrl: string;
+  endpoint: string;
+};
+
+/** Google's date-restrict vocabulary. */
+const RECENCY_CODE: Record<Recency, string> = { day: "d", week: "w", month: "m", year: "y" };
+
+/**
+ * `udm=14` asks for the plain "Web" tab: no AI overview, no carousels, just
+ * ranked links. That is both cheaper to parse and closer to what the agent
+ * asked for. `hl=en` pins the result language so snippets do not change shape
+ * with whatever locale the browser profile happens to carry.
+ *
+ * Two findings from testing against the real browser, both counter-intuitive
+ * enough to be worth writing down:
+ *
+ *   - `tbs=qdr:*` — the parameter the Tools menu puts in the URL — renders an
+ *     empty page in this profile, with `#search` present but no `#rso` and no
+ *     results at all. `as_qdr=*`, the older advanced-search parameter, works and
+ *     genuinely filters (verified: d/w/m/y all return dated results).
+ *   - Any date restriction combined with `udm=14` also renders that empty page.
+ *
+ * So a time-limited search drops `udm` and uses `as_qdr`. The classic layout it
+ * falls back to parses fine — the extractor was checked against both.
+ */
+export const buildSearchUrl = (request: SearchRequest): string => {
+  const params = new URLSearchParams({
+    q: request.query,
+    num: String(request.numResults),
+    hl: "en",
+  });
+  if (request.recency) params.set("as_qdr", RECENCY_CODE[request.recency]);
+  else params.set("udm", "14");
+  return `https://www.google.com/search?${params.toString()}`;
+};
+
+// ── connection lifecycle ───────────────────────────────────────────────────
+
+let connection: BrowserConnection | null = null;
+/** In-flight connect, so two concurrent searches share one dial rather than racing. */
+let connecting: Promise<BrowserConnection> | null = null;
+
+const resolveTarget = async (): Promise<CdpTarget> => {
+  const resolution = resolveCdpTarget(process.env, await readStoredTarget());
+  if (resolution.kind === "invalid") {
+    throw new BrowserUnavailableError(describeResolutionFailure(resolution));
+  }
+  return resolution.target;
+};
+
+/**
+ * Return a live connection, dialling one if there isn't one or the last one
+ * died. Reconnecting transparently matters here: castle recycles Chrome hourly,
+ * so a session that searched an hour ago is holding a dead socket.
+ */
+const ensureConnection = async (signal?: AbortSignal): Promise<BrowserConnection> => {
+  if (connection && connection.isOpen) return connection;
+  if (connection) {
+    connection.close();
+    connection = null;
+  }
+
+  if (!connecting) {
+    // Assigned synchronously, before the first await inside: two searches
+    // starting in the same tick must share one dial, not open two sockets and
+    // leak whichever loses the assignment race.
+    connecting = (async () => {
+      const target = await resolveTarget();
+      const conn = await BrowserConnection.connect(target);
+      connection = conn;
+      return conn;
+    })().finally(() => {
+      connecting = null;
+    });
+  }
+
+  // The caller's signal is raced here rather than passed into connect(). A
+  // shared dial must not be cancelled by whichever caller happened to start it
+  // — one search pressing Esc would otherwise fail the other search waiting on
+  // the same connection. connect() is bounded by its own timeouts regardless.
+  return raceAbort(connecting, signal);
+};
+
+/** Idempotent: safe to call from `session_shutdown` however many times it fires. */
+export const disconnect = (): void => {
+  connection?.close();
+  connection = null;
+};
+
+/** For status reporting — never dials. */
+export const describeConfiguredEndpoint = async (): Promise<string> => {
+  const resolution = resolveCdpTarget(process.env, await readStoredTarget());
+  return resolution.kind === "invalid"
+    ? describeResolutionFailure(resolution)
+    : describeCdpTarget(resolution.target);
+};
+
+// ── concurrency cap ────────────────────────────────────────────────────────
+
+let active = 0;
+const waiting: Array<() => void> = [];
+
+/** Give a held slot to the next waiter, or return it to the pool. */
+const handOff = (): void => {
+  const next = waiting.shift();
+  // Hand the slot straight over rather than decrementing and letting the waiter
+  // re-check, so a queued search cannot be overtaken by a newly arriving one.
+  if (next) next();
+  else active--;
+};
+
+const acquire = async (signal?: AbortSignal): Promise<() => void> => {
+  if (active < MAX_CONCURRENT_SEARCHES) {
+    active++;
+  } else {
+    let resolver!: () => void;
+    const queued = new Promise<void>((resolve) => {
+      resolver = resolve;
+    });
+    waiting.push(resolver);
+    try {
+      await raceAbort(queued, signal);
+      // The slot was handed over by handOff(), which already counted it.
+    } catch (e) {
+      // An abort while queued must not strand the slot. Either we are still in
+      // the queue — drop out of it — or the slot was handed to us in the same
+      // tick the abort landed, in which case give it back. Without this, two
+      // cancelled-while-queued searches would wedge the semaphore permanently.
+      const index = waiting.indexOf(resolver);
+      if (index !== -1) waiting.splice(index, 1);
+      else handOff();
+      throw e;
+    }
+  }
+  let released = false;
+  return () => {
+    if (released) return;
+    released = true;
+    handOff();
+  };
+};
+
+// ── the search itself ──────────────────────────────────────────────────────
+
+export type ProgressReporter = (phase: SearchPhase, note: string) => void;
+export type SearchPhase = "queued" | "connecting" | "opening" | "navigating" | "extracting";
+
+const sleep = (ms: number, signal?: AbortSignal): Promise<void> =>
+  raceAbort(new Promise<void>((resolve) => setTimeout(resolve, ms)), signal);
+
+/**
+ * Run one search. Throws on every failure — see errors.ts for why a returned
+ * error object would be worse than useless here.
+ */
+export const search = async (
+  request: SearchRequest,
+  signal: AbortSignal,
+  report: ProgressReporter,
+): Promise<SearchOutcome> => {
+  // Reported before acquiring, so a search that dies waiting for a slot is
+  // distinguishable from one that died reaching the browser. The deadline is
+  // the caller's and covers the queue wait too — with a cap of two and searches
+  // that take a couple of seconds, waiting is the rare case, and a tool call
+  // that silently takes twice its stated deadline would be worse.
+  report("queued", "waiting for a search slot");
+  const release = await acquire(signal);
+  try {
+    report("connecting", "resolving browser");
+    const conn = await ensureConnection(signal);
+    const endpoint = describeCdpTarget(conn.target);
+
+    report("opening", `opening a page on ${endpoint}`);
+    const page = await PageSession.open(conn, signal);
+    const searchUrl = buildSearchUrl(request);
+
+    try {
+      report("navigating", `searching for "${request.query}"`);
+      await page.navigate(searchUrl, signal);
+
+      report("extracting", "reading results");
+      let probe = await page.evaluate<PageProbe>(PROBE_SCRIPT, signal);
+      for (let attempt = 0; attempt < EXTRACT_RETRIES; attempt++) {
+        if (probe?.challenge || (probe?.results?.length ?? 0) > 0) break;
+        await sleep(EXTRACT_RETRY_DELAY_MS, signal);
+        probe = await page.evaluate<PageProbe>(PROBE_SCRIPT, signal);
+      }
+
+      if (!probe) {
+        throw new BrowserUnavailableError("the page returned nothing — the extraction script did not run");
+      }
+
+      if (probe.challenge) {
+        throw new SearchChallengeError({
+          challenge: probe.challenge.kind,
+          // The page the human must clear is where the browser *ended up*, which
+          // after a consent redirect is not the URL we asked for.
+          pageUrl: probe.url || searchUrl,
+          endpoint,
+          detail: probe.challenge.detail || undefined,
+        });
+      }
+
+      if (probe.results.length === 0) {
+        // "#rso missing entirely" and "#rso present but empty" are different
+        // faults, and saying which saves the next person a browser session.
+        const diagnosis =
+          probe.container === "rso"
+            ? "The results container was rendered but nothing could be parsed out of it, which suggests Google's " +
+              "result markup changed and castle_cdp_search's extractor needs updating."
+            : "Google served a page with no results area at all. Most likely this is a soft rate-limit: as a " +
+              "block decays Google stops serving the /sorry/ challenge page and returns an empty result page " +
+              "instead, which is indistinguishable from a genuine zero-hit search (observed lasting ~90 minutes " +
+              "after heavy querying). It can also mean the query truly has no hits, or that this combination of " +
+              "search parameters is not honoured for this browser profile.";
+        throw new NoResultsError(
+          `No results could be read from ${probe.url || searchUrl} (page title: "${probe.title}"). ${diagnosis} ` +
+            `Before concluding the extractor is broken, try a broader query and the same query without recency; ` +
+            `if those are also empty, treat it as rate limiting and wait rather than retrying in a loop.`,
+        );
+      }
+
+      return {
+        results: probe.results.slice(0, request.numResults),
+        searchUrl,
+        finalUrl: probe.url || searchUrl,
+        endpoint,
+      };
+    } finally {
+      // Always, including the challenge path. Leaving the tab open would let the
+      // operator solve the captcha in place, but it would also litter a browser
+      // someone else is using with abandoned tabs on every failure; the error
+      // carries the URL instead, and the cookie it sets is profile-wide.
+      await page.close();
+    }
+  } finally {
+    release();
+  }
+};