aboutsummaryrefslogtreecommitdiff
path: root/src/domains/readpage/readability.ts
diff options
context:
space:
mode:
authoramankumarsingh77 <amankumarsingh7702@gmail.com>2026-07-08 21:24:32 +0530
committeramankumarsingh77 <amankumarsingh7702@gmail.com>2026-07-08 21:24:32 +0530
commit68bf7fd2988e42343833be09e4115cd7175029d0 (patch)
tree52b8eaf99fa7e4d383ff169dd3bdccfe13911217 /src/domains/readpage/readability.ts
parentc5c42df5976ed63faac2adcf3b04a4ad2f519b32 (diff)
feat: add browser_read_page reader-mode tool
URL (or owned targetId) → clean main-article text with nav/ads/footer boilerplate stripped, via a dependency-free readability heuristic. DOM walk lives in an in-page capture expression; scoring/selection is a pure, fixture- tested function (link-density + boilerplate-ancestor filtering, body-text fallback for structure-less pages). Reuses the isolated-attach lifecycle; registered unserialized (safe concurrent read). 10 extractor unit scenarios. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Diffstat (limited to 'src/domains/readpage/readability.ts')
-rw-r--r--src/domains/readpage/readability.ts89
1 files changed, 89 insertions, 0 deletions
diff --git a/src/domains/readpage/readability.ts b/src/domains/readpage/readability.ts
new file mode 100644
index 0000000..37f2a38
--- /dev/null
+++ b/src/domains/readpage/readability.ts
@@ -0,0 +1,89 @@
+/**
+ * Pure reader-mode extraction — no CDP, no DOM. The in-page script
+ * (see capture.ts) serializes candidate content blocks; these pure functions
+ * score them and assemble clean article text, so the selection logic is
+ * unit-testable against saved fixtures (test/deep-research/readability-test.ts).
+ *
+ * Dependency-free by design (no jsdom / Readability): the harness ships no new
+ * runtime deps. The heuristic favors blocks with high text density and low link
+ * density, which is what separates article body from nav/ads/boilerplate.
+ */
+
+/** One candidate content block captured in-page. */
+export type ContentBlock = {
+ /** Semantic tag: paragraph | heading | listitem | blockquote | other. */
+ readonly kind: "paragraph" | "heading" | "listitem" | "blockquote" | "other";
+ readonly text: string;
+ /** Chars of text that sit inside <a> tags — high ratio ⇒ nav/menu, not prose. */
+ readonly linkTextLength: number;
+ /** True when an ancestor is nav/header/footer/aside — structural boilerplate. */
+ readonly inBoilerplate: boolean;
+};
+
+/** The JSON payload capture.ts returns from the page. */
+export type PageCapture = {
+ readonly url: string;
+ readonly title: string;
+ readonly blocks: ReadonlyArray<ContentBlock>;
+ /** Bounded innerText fallback when block extraction finds nothing usable. */
+ readonly bodyText: string;
+};
+
+/** The tool's public output shape. */
+export type ReadablePage = {
+ readonly title: string;
+ readonly url: string;
+ readonly text: string;
+ readonly wordCount: number;
+};
+
+const MIN_PARAGRAPH_CHARS = 25;
+const MAX_LINK_DENSITY = 0.5;
+const MIN_ARTICLE_WORDS = 40;
+
+const linkDensity = (block: ContentBlock): number => {
+ if (block.text.length === 0) return 1;
+ return block.linkTextLength / block.text.length;
+};
+
+/** A block is article prose if it is substantive, text-dense, and not chrome. */
+const isArticleBlock = (block: ContentBlock): boolean => {
+ if (block.inBoilerplate) return false;
+ if (linkDensity(block) > MAX_LINK_DENSITY) return false;
+ if (block.kind === "heading" || block.kind === "blockquote" || block.kind === "listitem") {
+ return block.text.trim().length > 0;
+ }
+ return block.text.trim().length >= MIN_PARAGRAPH_CHARS;
+};
+
+const render = (block: ContentBlock): string => {
+ const text = block.text.trim();
+ if (block.kind === "heading") return `\n## ${text}\n`;
+ if (block.kind === "listitem") return `- ${text}`;
+ if (block.kind === "blockquote") return `> ${text}`;
+ return text;
+};
+
+const wordCountOf = (text: string): number => {
+ const trimmed = text.trim();
+ return trimmed.length === 0 ? 0 : trimmed.split(/\s+/).length;
+};
+
+/**
+ * Assemble readable article text from a page capture. Keeps text-dense,
+ * non-boilerplate blocks; falls back to the bounded body text when the article
+ * is too thin (e.g. a page with no clear content structure).
+ */
+export const extractReadable = (capture: PageCapture): ReadablePage => {
+ const articleText = capture.blocks.filter(isArticleBlock).map(render).join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
+
+ const useFallback = wordCountOf(articleText) < MIN_ARTICLE_WORDS;
+ const text = useFallback ? capture.bodyText.trim() : articleText;
+
+ return {
+ title: capture.title.trim(),
+ url: capture.url,
+ text,
+ wordCount: wordCountOf(text),
+ };
+};