path: root/src/search.ts

/**
 * Search orchestration: resolve a browser, borrow a page, navigate, extract,
 * hand the page back.
 *
 * The connection is process-wide and lazy. It is *not* opened by the extension
 * factory — pi runs factories in invocations that never start a session, and
 * the docs are explicit that sockets must not start there. It is opened on the
 * first search and closed by an idempotent `session_shutdown`.
 */

import { BrowserConnection, PageSession, raceAbort } from "./cdp.ts";
import {
  type CdpTarget,
  describeCdpTarget,
  describeResolutionFailure,
  resolveCdpTarget,
} from "./endpoint.ts";
import { BrowserUnavailableError, NoResultsError, SearchChallengeError } from "./errors.ts";
import { PROBE_SCRIPT, type PageProbe, type SearchResult } from "./extract.ts";
import { readStoredTarget } from "./target-store.ts";

/** Per-search wall clock, covering connect, navigate and extract together. */
export const SEARCH_TIMEOUT_MS = 20_000;
/** How many searches may drive the shared browser at once. */
export const MAX_CONCURRENT_SEARCHES = 2;
/** Results sometimes land a beat after the load event; re-read a couple of times. */
const EXTRACT_RETRIES = 2;
const EXTRACT_RETRY_DELAY_MS = 700;

export type Recency = "day" | "week" | "month" | "year";

export type SearchRequest = {
  query: string;
  numResults: number;
  recency?: Recency | undefined;
};

export type SearchOutcome = {
  results: SearchResult[];
  searchUrl: string;
  /** The URL the page settled on, which differs from searchUrl after a redirect. */
  finalUrl: string;
  endpoint: string;
};

/** Google's date-restrict vocabulary. */
const RECENCY_CODE: Record<Recency, string> = { day: "d", week: "w", month: "m", year: "y" };

/**
 * `udm=14` asks for the plain "Web" tab: no AI overview, no carousels, just
 * ranked links. That is both cheaper to parse and closer to what the agent
 * asked for. `hl=en` pins the result language so snippets do not change shape
 * with whatever locale the browser profile happens to carry.
 *
 * Two findings from testing against the real browser, both counter-intuitive
 * enough to be worth writing down:
 *
 *   - `tbs=qdr:*` — the parameter the Tools menu puts in the URL — renders an
 *     empty page in this profile, with `#search` present but no `#rso` and no
 *     results at all. `as_qdr=*`, the older advanced-search parameter, works and
 *     genuinely filters (verified: d/w/m/y all return dated results).
 *   - Any date restriction combined with `udm=14` also renders that empty page.
 *
 * So a time-limited search drops `udm` and uses `as_qdr`. The classic layout it
 * falls back to parses fine — the extractor was checked against both.
 */
export const buildSearchUrl = (request: SearchRequest): string => {
  const params = new URLSearchParams({
    q: request.query,
    num: String(request.numResults),
    hl: "en",
  });
  if (request.recency) params.set("as_qdr", RECENCY_CODE[request.recency]);
  else params.set("udm", "14");
  return `https://www.google.com/search?${params.toString()}`;
};

// ── connection lifecycle ───────────────────────────────────────────────────

let connection: BrowserConnection | null = null;
/** In-flight connect, so two concurrent searches share one dial rather than racing. */
let connecting: Promise<BrowserConnection> | null = null;

const resolveTarget = async (): Promise<CdpTarget> => {
  const resolution = resolveCdpTarget(process.env, await readStoredTarget());
  if (resolution.kind === "invalid") {
    throw new BrowserUnavailableError(describeResolutionFailure(resolution));
  }
  return resolution.target;
};

/**
 * Return a live connection, dialling one if there isn't one or the last one
 * died. Reconnecting transparently matters here: castle recycles Chrome hourly,
 * so a session that searched an hour ago is holding a dead socket.
 */
const ensureConnection = async (signal?: AbortSignal): Promise<BrowserConnection> => {
  if (connection && connection.isOpen) return connection;
  if (connection) {
    connection.close();
    connection = null;
  }

  if (!connecting) {
    // Assigned synchronously, before the first await inside: two searches
    // starting in the same tick must share one dial, not open two sockets and
    // leak whichever loses the assignment race.
    connecting = (async () => {
      const target = await resolveTarget();
      const conn = await BrowserConnection.connect(target);
      connection = conn;
      return conn;
    })().finally(() => {
      connecting = null;
    });
  }

  // The caller's signal is raced here rather than passed into connect(). A
  // shared dial must not be cancelled by whichever caller happened to start it
  // — one search pressing Esc would otherwise fail the other search waiting on
  // the same connection. connect() is bounded by its own timeouts regardless.
  return raceAbort(connecting, signal);
};

/** Idempotent: safe to call from `session_shutdown` however many times it fires. */
export const disconnect = (): void => {
  connection?.close();
  connection = null;
};

/** For status reporting — never dials. */
export const describeConfiguredEndpoint = async (): Promise<string> => {
  const resolution = resolveCdpTarget(process.env, await readStoredTarget());
  return resolution.kind === "invalid"
    ? describeResolutionFailure(resolution)
    : describeCdpTarget(resolution.target);
};

// ── concurrency cap ────────────────────────────────────────────────────────

let active = 0;
const waiting: Array<() => void> = [];

/** Give a held slot to the next waiter, or return it to the pool. */
const handOff = (): void => {
  const next = waiting.shift();
  // Hand the slot straight over rather than decrementing and letting the waiter
  // re-check, so a queued search cannot be overtaken by a newly arriving one.
  if (next) next();
  else active--;
};

const acquire = async (signal?: AbortSignal): Promise<() => void> => {
  if (active < MAX_CONCURRENT_SEARCHES) {
    active++;
  } else {
    let resolver!: () => void;
    const queued = new Promise<void>((resolve) => {
      resolver = resolve;
    });
    waiting.push(resolver);
    try {
      await raceAbort(queued, signal);
      // The slot was handed over by handOff(), which already counted it.
    } catch (e) {
      // An abort while queued must not strand the slot. Either we are still in
      // the queue — drop out of it — or the slot was handed to us in the same
      // tick the abort landed, in which case give it back. Without this, two
      // cancelled-while-queued searches would wedge the semaphore permanently.
      const index = waiting.indexOf(resolver);
      if (index !== -1) waiting.splice(index, 1);
      else handOff();
      throw e;
    }
  }
  let released = false;
  return () => {
    if (released) return;
    released = true;
    handOff();
  };
};

// ── the search itself ──────────────────────────────────────────────────────

export type ProgressReporter = (phase: SearchPhase, note: string) => void;
export type SearchPhase = "queued" | "connecting" | "opening" | "navigating" | "extracting";

const sleep = (ms: number, signal?: AbortSignal): Promise<void> =>
  raceAbort(new Promise<void>((resolve) => setTimeout(resolve, ms)), signal);

/**
 * Run one search. Throws on every failure — see errors.ts for why a returned
 * error object would be worse than useless here.
 */
export const search = async (
  request: SearchRequest,
  signal: AbortSignal,
  report: ProgressReporter,
): Promise<SearchOutcome> => {
  // Reported before acquiring, so a search that dies waiting for a slot is
  // distinguishable from one that died reaching the browser. The deadline is
  // the caller's and covers the queue wait too — with a cap of two and searches
  // that take a couple of seconds, waiting is the rare case, and a tool call
  // that silently takes twice its stated deadline would be worse.
  report("queued", "waiting for a search slot");
  const release = await acquire(signal);
  try {
    report("connecting", "resolving browser");
    const conn = await ensureConnection(signal);
    const endpoint = describeCdpTarget(conn.target);

    report("opening", `opening a page on ${endpoint}`);
    const page = await PageSession.open(conn, signal);
    const searchUrl = buildSearchUrl(request);

    try {
      report("navigating", `searching for "${request.query}"`);
      await page.navigate(searchUrl, signal);

      report("extracting", "reading results");
      let probe = await page.evaluate<PageProbe>(PROBE_SCRIPT, signal);
      for (let attempt = 0; attempt < EXTRACT_RETRIES; attempt++) {
        if (probe?.challenge || (probe?.results?.length ?? 0) > 0) break;
        await sleep(EXTRACT_RETRY_DELAY_MS, signal);
        probe = await page.evaluate<PageProbe>(PROBE_SCRIPT, signal);
      }

      if (!probe) {
        throw new BrowserUnavailableError("the page returned nothing — the extraction script did not run");
      }

      if (probe.challenge) {
        throw new SearchChallengeError({
          challenge: probe.challenge.kind,
          // The page the human must clear is where the browser *ended up*, which
          // after a consent redirect is not the URL we asked for.
          pageUrl: probe.url || searchUrl,
          endpoint,
          detail: probe.challenge.detail || undefined,
        });
      }

      if (probe.results.length === 0) {
        // "#rso missing entirely" and "#rso present but empty" are different
        // faults, and saying which saves the next person a browser session.
        const diagnosis =
          probe.container === "rso"
            ? "The results container was rendered but nothing could be parsed out of it, which suggests Google's " +
              "result markup changed and castle_cdp_search's extractor needs updating."
            : "Google served a page with no results area at all. Most likely this is a soft rate-limit: as a " +
              "block decays Google stops serving the /sorry/ challenge page and returns an empty result page " +
              "instead, which is indistinguishable from a genuine zero-hit search (observed lasting ~90 minutes " +
              "after heavy querying). It can also mean the query truly has no hits, or that this combination of " +
              "search parameters is not honoured for this browser profile.";
        throw new NoResultsError(
          `No results could be read from ${probe.url || searchUrl} (page title: "${probe.title}"). ${diagnosis} ` +
            `Before concluding the extractor is broken, try a broader query and the same query without recency; ` +
            `if those are also empty, treat it as rate limiting and wait rather than retrying in a loop.`,
        );
      }

      return {
        results: probe.results.slice(0, request.numResults),
        searchUrl,
        finalUrl: probe.url || searchUrl,
        endpoint,
      };
    } finally {
      // Always, including the challenge path. Leaving the tab open would let the
      // operator solve the captcha in place, but it would also litter a browser
      // someone else is using with abandoned tabs on every failure; the error
      // carries the URL instead, and the cookie it sets is profile-wide.
      await page.close();
    }
  } finally {
    release();
  }
};