aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/domains/readpage/capture.ts46
-rw-r--r--src/domains/readpage/read-page.ts109
-rw-r--r--src/domains/readpage/readability.ts89
-rw-r--r--src/registry.ts3
-rw-r--r--test/deep-research/fixtures/article-capture.json17
-rw-r--r--test/deep-research/fixtures/no-article-capture.json10
-rw-r--r--test/deep-research/readability-test.ts46
7 files changed, 320 insertions, 0 deletions
diff --git a/src/domains/readpage/capture.ts b/src/domains/readpage/capture.ts
new file mode 100644
index 0000000..1ee4500
--- /dev/null
+++ b/src/domains/readpage/capture.ts
@@ -0,0 +1,46 @@
+/**
+ * The JavaScript source evaluated in a loaded page to capture candidate content
+ * blocks + metadata for the pure readability extractor (readability.ts). Crosses
+ * the CDP boundary via Runtime.evaluate (returnByValue) and returns a JSON
+ * string shaped like PageCapture.
+ *
+ * The DOM walk lives here (unavoidable); all scoring/selection is pure and lives
+ * in readability.ts, so the decision logic stays unit-testable.
+ */
+
+/** Max bytes of body-text fallback captured for thin/structure-less pages. */
+const BODY_TEXT_LIMIT = 20_000;
+
+/** Build the capture expression. No interpolated values ⇒ no injection surface. */
+export const buildPageCaptureExpr = (): string => `
+ (() => {
+ const BOILERPLATE = new Set(['NAV', 'HEADER', 'FOOTER', 'ASIDE']);
+ const inBoilerplate = (el) => {
+ for (let n = el; n; n = n.parentElement) {
+ if (BOILERPLATE.has(n.tagName)) return true;
+ const role = n.getAttribute && n.getAttribute('role');
+ if (role === 'navigation' || role === 'banner' || role === 'contentinfo') return true;
+ }
+ return false;
+ };
+ const kindOf = (tag) => {
+ if (/^H[1-6]$/.test(tag)) return 'heading';
+ if (tag === 'LI') return 'listitem';
+ if (tag === 'BLOCKQUOTE') return 'blockquote';
+ if (tag === 'P') return 'paragraph';
+ return 'other';
+ };
+ const selector = 'p, h1, h2, h3, h4, h5, h6, li, blockquote';
+ const root = document.querySelector('article, main, [role=main]') || document.body;
+ const blocks = [];
+ for (const el of root.querySelectorAll(selector)) {
+ const text = (el.innerText || '').replace(/\\s+/g, ' ').trim();
+ if (!text) continue;
+ let linkTextLength = 0;
+ for (const a of el.querySelectorAll('a')) linkTextLength += (a.innerText || '').length;
+ blocks.push({ kind: kindOf(el.tagName), text, linkTextLength, inBoilerplate: inBoilerplate(el) });
+ }
+ const bodyText = (document.body.innerText || '').replace(/\\n{3,}/g, '\\n\\n').slice(0, ${BODY_TEXT_LIMIT});
+ return JSON.stringify({ url: location.href, title: document.title, blocks, bodyText });
+ })()
+`;
diff --git a/src/domains/readpage/read-page.ts b/src/domains/readpage/read-page.ts
new file mode 100644
index 0000000..7bf2fdb
--- /dev/null
+++ b/src/domains/readpage/read-page.ts
@@ -0,0 +1,109 @@
+import { Type } from "typebox";
+import { type Result, err, ok } from "../../util/result";
+import { applyTruncation } from "../../util/truncate";
+import { defineBrowserTool, type ToolErr, type ToolOk } from "../../util/tool";
+import {
+ closeIsolatedTab,
+ evalInIsolatedTab,
+ type IsolatedTab,
+ navigateIsolatedTab,
+ openIsolatedTab,
+ waitForIsolatedLoad,
+} from "../isolated-tab";
+import { buildPageCaptureExpr } from "./capture";
+import { extractReadable, type PageCapture, type ReadablePage } from "./readability";
+
+const LOAD_TIMEOUT_MS = 15_000;
+
+const ReadPageArgs = Type.Object({
+ url: Type.Optional(Type.String({ description: "URL to open in an isolated tab, read, then close." })),
+ targetId: Type.Optional(
+ Type.String({ description: "Read an already-open tab this session owns instead of opening a URL." }),
+ ),
+});
+
+const isPageCapture = (value: unknown): value is PageCapture => {
+ if (typeof value !== "object" || value === null) return false;
+ const v = value as Record<string, unknown>;
+ return typeof v["url"] === "string" && typeof v["title"] === "string" && Array.isArray(v["blocks"]) && typeof v["bodyText"] === "string";
+};
+
+const safeParse = (text: string): unknown => {
+ try {
+ return JSON.parse(text) as unknown;
+ } catch {
+ return undefined;
+ }
+};
+
+const captureToResult = async (raw: unknown): Promise<Result<ToolOk, ToolErr>> => {
+ const parsed = typeof raw === "string" ? safeParse(raw) : raw;
+ if (!isPageCapture(parsed)) return err({ kind: "internal", message: "page capture returned an unexpected shape" });
+ const page: ReadablePage = extractReadable(parsed);
+ const header = `# ${page.title}\n${page.url}\n(${page.wordCount} words)\n\n`;
+ const truncated = await applyTruncation(header + page.text, "readpage");
+ return ok({
+ text: truncated.text,
+ details: {
+ title: page.title,
+ url: page.url,
+ wordCount: page.wordCount,
+ ...(truncated.fullOutputPath !== undefined ? { fullOutputPath: truncated.fullOutputPath } : {}),
+ },
+ });
+};
+
+const readOpenedUrl = async (
+ client: Parameters<typeof openIsolatedTab>[0],
+ url: string,
+ signal: AbortSignal | undefined,
+): Promise<Result<ToolOk, ToolErr>> => {
+ const opened = await openIsolatedTab(client);
+ if (!opened.success) return err(opened.error);
+ const tab: IsolatedTab = opened.data;
+ try {
+ const navigated = await navigateIsolatedTab(client, tab, url);
+ if (!navigated.success) return err(navigated.error);
+ const loaded = await waitForIsolatedLoad(client, tab, LOAD_TIMEOUT_MS, signal);
+ if (!loaded.success) return err(loaded.error);
+ const captured = await evalInIsolatedTab(client, tab, buildPageCaptureExpr());
+ if (!captured.success) return err(captured.error);
+ return captureToResult(captured.data);
+ } finally {
+ await closeIsolatedTab(client, tab);
+ }
+};
+
+const readOwnedTab = async (
+ client: Parameters<typeof openIsolatedTab>[0],
+ targetId: string,
+): Promise<Result<ToolOk, ToolErr>> => {
+ if (!client.owns(targetId)) {
+ return err({ kind: "invalid_state", message: `Tab ${targetId} is not owned by this session.` });
+ }
+ const attached = await client.session().callBrowser("Target.attachToTarget", { targetId, flatten: true });
+ if (!attached.success) return err({ kind: "cdp_error", message: attached.error.message });
+ const sessionId = (attached.data as { sessionId: string }).sessionId;
+ const captured = await client.evaluateJs(buildPageCaptureExpr(), sessionId);
+ if (!captured.success) return err({ kind: "cdp_error", message: captured.error.message });
+ return captureToResult(captured.data);
+};
+
+export const readPageTool = defineBrowserTool({
+ name: "browser_read_page",
+ label: "Browser Read Page",
+ description:
+ "Read a page as clean article text — main content with nav/ads/boilerplate stripped. Pass a url (opened in an isolated tab, read, then closed) or a targetId of an owned tab. Reader-mode counterpart to browser_web_search.",
+ promptSnippet: "Read a page's main content as clean text",
+ promptGuidelines: [
+ "Pass url to read an arbitrary page (opened + closed in its own tab, never disturbing your current tab), or targetId to read an already-open owned tab.",
+ "Returns readable main-article text — use this over browser_snapshot/browser_execute_js when you want an article's content for reading or research.",
+ "Boilerplate-heavy or structure-less pages fall back to bounded body text rather than erroring.",
+ ],
+ parameters: ReadPageArgs,
+ async handler(args, { client, signal }): Promise<Result<ToolOk, ToolErr>> {
+ if (args.url !== undefined && args.url.length > 0) return readOpenedUrl(client, args.url, signal);
+ if (args.targetId !== undefined && args.targetId.length > 0) return readOwnedTab(client, args.targetId);
+ return err({ kind: "invalid_state", message: "provide either url or targetId" });
+ },
+});
diff --git a/src/domains/readpage/readability.ts b/src/domains/readpage/readability.ts
new file mode 100644
index 0000000..37f2a38
--- /dev/null
+++ b/src/domains/readpage/readability.ts
@@ -0,0 +1,89 @@
+/**
+ * Pure reader-mode extraction — no CDP, no DOM. The in-page script
+ * (see capture.ts) serializes candidate content blocks; these pure functions
+ * score them and assemble clean article text, so the selection logic is
+ * unit-testable against saved fixtures (test/deep-research/readability-test.ts).
+ *
+ * Dependency-free by design (no jsdom / Readability): the harness ships no new
+ * runtime deps. The heuristic favors blocks with high text density and low link
+ * density, which is what separates article body from nav/ads/boilerplate.
+ */
+
+/** One candidate content block captured in-page. */
+export type ContentBlock = {
+ /** Semantic tag: paragraph | heading | listitem | blockquote | other. */
+ readonly kind: "paragraph" | "heading" | "listitem" | "blockquote" | "other";
+ readonly text: string;
+ /** Chars of text that sit inside <a> tags — high ratio ⇒ nav/menu, not prose. */
+ readonly linkTextLength: number;
+ /** True when an ancestor is nav/header/footer/aside — structural boilerplate. */
+ readonly inBoilerplate: boolean;
+};
+
+/** The JSON payload capture.ts returns from the page. */
+export type PageCapture = {
+ readonly url: string;
+ readonly title: string;
+ readonly blocks: ReadonlyArray<ContentBlock>;
+ /** Bounded innerText fallback when block extraction finds nothing usable. */
+ readonly bodyText: string;
+};
+
+/** The tool's public output shape. */
+export type ReadablePage = {
+ readonly title: string;
+ readonly url: string;
+ readonly text: string;
+ readonly wordCount: number;
+};
+
+const MIN_PARAGRAPH_CHARS = 25;
+const MAX_LINK_DENSITY = 0.5;
+const MIN_ARTICLE_WORDS = 40;
+
+const linkDensity = (block: ContentBlock): number => {
+ if (block.text.length === 0) return 1;
+ return block.linkTextLength / block.text.length;
+};
+
+/** A block is article prose if it is substantive, text-dense, and not chrome. */
+const isArticleBlock = (block: ContentBlock): boolean => {
+ if (block.inBoilerplate) return false;
+ if (linkDensity(block) > MAX_LINK_DENSITY) return false;
+ if (block.kind === "heading" || block.kind === "blockquote" || block.kind === "listitem") {
+ return block.text.trim().length > 0;
+ }
+ return block.text.trim().length >= MIN_PARAGRAPH_CHARS;
+};
+
+const render = (block: ContentBlock): string => {
+ const text = block.text.trim();
+ if (block.kind === "heading") return `\n## ${text}\n`;
+ if (block.kind === "listitem") return `- ${text}`;
+ if (block.kind === "blockquote") return `> ${text}`;
+ return text;
+};
+
+const wordCountOf = (text: string): number => {
+ const trimmed = text.trim();
+ return trimmed.length === 0 ? 0 : trimmed.split(/\s+/).length;
+};
+
+/**
+ * Assemble readable article text from a page capture. Keeps text-dense,
+ * non-boilerplate blocks; falls back to the bounded body text when the article
+ * is too thin (e.g. a page with no clear content structure).
+ */
+export const extractReadable = (capture: PageCapture): ReadablePage => {
+ const articleText = capture.blocks.filter(isArticleBlock).map(render).join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
+
+ const useFallback = wordCountOf(articleText) < MIN_ARTICLE_WORDS;
+ const text = useFallback ? capture.bodyText.trim() : articleText;
+
+ return {
+ title: capture.title.trim(),
+ url: capture.url,
+ text,
+ wordCount: wordCountOf(text),
+ };
+};
diff --git a/src/registry.ts b/src/registry.ts
index c7ff5c8..a063c9f 100644
--- a/src/registry.ts
+++ b/src/registry.ts
@@ -20,6 +20,7 @@ import { snapshotTool } from "./domains/snapshot";
import { executeJsTool, runScriptTool } from "./domains/js";
import { setupTool } from "./domains/setup";
import { webSearchTool } from "./domains/search/web-search";
+import { readPageTool } from "./domains/readpage/read-page";
// Mutation tools that modify page / browser state are marked `serialized: true`
// so they run through a shared async mutex. Observation / read-only tools are
@@ -64,6 +65,8 @@ const TOOLS: ReadonlyArray<AnyBrowserToolDefinition> = [
runScriptTool,
// Opens and owns an isolated tab to scrape a SERP — same class as openUrlsTool.
{ ...webSearchTool, serialized: true },
+ // Pure isolated read (own sessionId, own tab) — safe concurrent, like executeJsTool.
+ readPageTool,
];
export const registerAllTools = (pi: ExtensionAPI, client: BrowserClient): void => {
diff --git a/test/deep-research/fixtures/article-capture.json b/test/deep-research/fixtures/article-capture.json
new file mode 100644
index 0000000..df9c67c
--- /dev/null
+++ b/test/deep-research/fixtures/article-capture.json
@@ -0,0 +1,17 @@
+{
+ "url": "https://blog.example.com/how-cdp-works",
+ "title": "How the Chrome DevTools Protocol Works",
+ "blocks": [
+ { "kind": "other", "text": "Home Products Blog About Contact", "linkTextLength": 33, "inBoilerplate": true },
+ { "kind": "heading", "text": "How the Chrome DevTools Protocol Works", "linkTextLength": 0, "inBoilerplate": false },
+ { "kind": "paragraph", "text": "The Chrome DevTools Protocol lets tools instrument, inspect, and control Chromium-based browsers over a WebSocket connection.", "linkTextLength": 0, "inBoilerplate": false },
+ { "kind": "paragraph", "text": "Each command targets a domain such as Page, Runtime, or Network, and receives a structured JSON response that the client correlates by id.", "linkTextLength": 0, "inBoilerplate": false },
+ { "kind": "heading", "text": "Sessions and Targets", "linkTextLength": 0, "inBoilerplate": false },
+ { "kind": "paragraph", "text": "A target is a page or worker; attaching to one yields a session id that scopes subsequent commands to that target without disturbing others.", "linkTextLength": 0, "inBoilerplate": false },
+ { "kind": "listitem", "text": "Attach with Target.attachToTarget", "linkTextLength": 0, "inBoilerplate": false },
+ { "kind": "listitem", "text": "Drive with Runtime.evaluate", "linkTextLength": 0, "inBoilerplate": false },
+ { "kind": "paragraph", "text": "Subscribe here for our newsletter and follow us on social media for updates", "linkTextLength": 55, "inBoilerplate": true },
+ { "kind": "other", "text": "Copyright 2026 Example Inc. Privacy Terms Cookies", "linkTextLength": 24, "inBoilerplate": true }
+ ],
+ "bodyText": "Home Products Blog How the Chrome DevTools Protocol Works The Chrome DevTools Protocol lets tools instrument inspect and control browsers Copyright 2026"
+}
diff --git a/test/deep-research/fixtures/no-article-capture.json b/test/deep-research/fixtures/no-article-capture.json
new file mode 100644
index 0000000..d367b7e
--- /dev/null
+++ b/test/deep-research/fixtures/no-article-capture.json
@@ -0,0 +1,10 @@
+{
+ "url": "https://app.example.com/dashboard",
+ "title": "Dashboard",
+ "blocks": [
+ { "kind": "other", "text": "Menu", "linkTextLength": 4, "inBoilerplate": true },
+ { "kind": "listitem", "text": "Widget A", "linkTextLength": 8, "inBoilerplate": true },
+ { "kind": "listitem", "text": "Widget B", "linkTextLength": 8, "inBoilerplate": true }
+ ],
+ "bodyText": "Dashboard Widget A Widget B Your account is active. Usage this month is within limits and no action is required at this time from you."
+}
diff --git a/test/deep-research/readability-test.ts b/test/deep-research/readability-test.ts
new file mode 100644
index 0000000..10f9a8b
--- /dev/null
+++ b/test/deep-research/readability-test.ts
@@ -0,0 +1,46 @@
+/**
+ * Unit tests for the pure readability extractor — no browser required.
+ *
+ * Run: npx tsx test/deep-research/readability-test.ts
+ */
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { extractReadable, type PageCapture } from "../../src/domains/readpage/readability";
+
+let passed = 0;
+let failed = 0;
+const check = (cond: boolean, label: string): void => {
+ if (cond) {
+ passed++;
+ console.log(` ✓ ${label}`);
+ } else {
+ failed++;
+ console.error(` ✗ ${label}`);
+ }
+};
+
+const fixture = (name: string): PageCapture =>
+ JSON.parse(readFileSync(join(import.meta.dirname, "fixtures", name), "utf8")) as PageCapture;
+
+// S10: boilerplate stripped, article content preserved
+{
+ const page = extractReadable(fixture("article-capture.json"));
+ check(page.text.includes("Chrome DevTools Protocol lets tools instrument"), "S10: article body text is present");
+ check(page.text.includes("Sessions and Targets"), "S10: article headings are preserved");
+ check(!page.text.includes("Home Products Blog"), "S10: nav boilerplate is dropped");
+ check(!page.text.includes("Copyright 2026"), "S10: footer boilerplate is dropped");
+ check(!page.text.includes("Subscribe here for our newsletter"), "S10: high-link-density promo block is dropped");
+ check(page.text.includes("## Sessions and Targets"), "S10: paragraph/heading structure preserved as markdown");
+ check(page.title === "How the Chrome DevTools Protocol Works" && page.url.startsWith("https://"), "S10: title and url passed through");
+ check(page.wordCount > 40, "S10: word count reflects the article body");
+}
+
+// S11: no clear article node falls back to bounded body text
+{
+ const page = extractReadable(fixture("no-article-capture.json"));
+ check(page.text.length > 0, "S11: structure-less page falls back to non-empty body text");
+ check(page.text.includes("Your account is active"), "S11: fallback body text carries the real content");
+}
+
+console.log(`\n${passed} passed, ${failed} failed`);
+if (failed > 0) process.exit(1);