diff options
Diffstat (limited to 'src/domains/readpage')
| -rw-r--r-- | src/domains/readpage/capture.ts | 11 | ||||
| -rw-r--r-- | src/domains/readpage/readability.ts | 23 |
2 files changed, 0 insertions, 34 deletions
diff --git a/src/domains/readpage/capture.ts b/src/domains/readpage/capture.ts index 1ee4500..cf9d055 100644 --- a/src/domains/readpage/capture.ts +++ b/src/domains/readpage/capture.ts @@ -1,17 +1,6 @@ -/** - * The JavaScript source evaluated in a loaded page to capture candidate content - * blocks + metadata for the pure readability extractor (readability.ts). Crosses - * the CDP boundary via Runtime.evaluate (returnByValue) and returns a JSON - * string shaped like PageCapture. - * - * The DOM walk lives here (unavoidable); all scoring/selection is pure and lives - * in readability.ts, so the decision logic stays unit-testable. - */ -/** Max bytes of body-text fallback captured for thin/structure-less pages. */ const BODY_TEXT_LIMIT = 20_000; -/** Build the capture expression. No interpolated values ⇒ no injection surface. */ export const buildPageCaptureExpr = (): string => ` (() => { const BOILERPLATE = new Set(['NAV', 'HEADER', 'FOOTER', 'ASIDE']); diff --git a/src/domains/readpage/readability.ts b/src/domains/readpage/readability.ts index 37f2a38..b20cacd 100644 --- a/src/domains/readpage/readability.ts +++ b/src/domains/readpage/readability.ts @@ -1,35 +1,18 @@ -/** - * Pure reader-mode extraction — no CDP, no DOM. The in-page script - * (see capture.ts) serializes candidate content blocks; these pure functions - * score them and assemble clean article text, so the selection logic is - * unit-testable against saved fixtures (test/deep-research/readability-test.ts). - * - * Dependency-free by design (no jsdom / Readability): the harness ships no new - * runtime deps. The heuristic favors blocks with high text density and low link - * density, which is what separates article body from nav/ads/boilerplate. - */ -/** One candidate content block captured in-page. */ export type ContentBlock = { - /** Semantic tag: paragraph | heading | listitem | blockquote | other. */ readonly kind: "paragraph" | "heading" | "listitem" | "blockquote" | "other"; readonly text: string; - /** Chars of text that sit inside <a> tags — high ratio ⇒ nav/menu, not prose. */ readonly linkTextLength: number; - /** True when an ancestor is nav/header/footer/aside — structural boilerplate. */ readonly inBoilerplate: boolean; }; -/** The JSON payload capture.ts returns from the page. */ export type PageCapture = { readonly url: string; readonly title: string; readonly blocks: ReadonlyArray<ContentBlock>; - /** Bounded innerText fallback when block extraction finds nothing usable. */ readonly bodyText: string; }; -/** The tool's public output shape. */ export type ReadablePage = { readonly title: string; readonly url: string; @@ -46,7 +29,6 @@ const linkDensity = (block: ContentBlock): number => { return block.linkTextLength / block.text.length; }; -/** A block is article prose if it is substantive, text-dense, and not chrome. */ const isArticleBlock = (block: ContentBlock): boolean => { if (block.inBoilerplate) return false; if (linkDensity(block) > MAX_LINK_DENSITY) return false; @@ -69,11 +51,6 @@ const wordCountOf = (text: string): number => { return trimmed.length === 0 ? 0 : trimmed.split(/\s+/).length; }; -/** - * Assemble readable article text from a page capture. Keeps text-dense, - * non-boilerplate blocks; falls back to the bounded body text when the article - * is too thin (e.g. a page with no clear content structure). - */ export const extractReadable = (capture: PageCapture): ReadablePage => { const articleText = capture.blocks.filter(isArticleBlock).map(render).join("\n\n").replace(/\n{3,}/g, "\n\n").trim(); |
