blob: 32413e6adcb3d583d99002dcf1d3ffc1a408901b (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
|
export type ContentBlock = {
readonly kind: "paragraph" | "heading" | "listitem" | "blockquote" | "other";
readonly text: string;
readonly linkTextLength: number;
readonly inBoilerplate: boolean;
};
export type PageCapture = {
readonly url: string;
readonly title: string;
readonly blocks: ReadonlyArray<ContentBlock>;
readonly bodyText: string;
};
export type ReadablePage = {
readonly title: string;
readonly url: string;
readonly text: string;
readonly wordCount: number;
};
const MIN_PARAGRAPH_CHARS = 25;
const MAX_LINK_DENSITY = 0.5;
const MIN_ARTICLE_WORDS = 40;
const linkDensity = (block: ContentBlock): number => {
if (block.text.length === 0) return 1;
return block.linkTextLength / block.text.length;
};
const isArticleBlock = (block: ContentBlock): boolean => {
if (block.inBoilerplate) return false;
if (linkDensity(block) > MAX_LINK_DENSITY) return false;
if (block.kind === "heading" || block.kind === "blockquote" || block.kind === "listitem") {
return block.text.trim().length > 0;
}
return block.text.trim().length >= MIN_PARAGRAPH_CHARS;
};
const render = (block: ContentBlock): string => {
const text = block.text.trim();
if (block.kind === "heading") return `\n## ${text}\n`;
if (block.kind === "listitem") return `- ${text}`;
if (block.kind === "blockquote") return `> ${text}`;
return text;
};
const wordCountOf = (text: string): number => {
const trimmed = text.trim();
return trimmed.length === 0 ? 0 : trimmed.split(/\s+/).length;
};
export const extractReadable = (capture: PageCapture): ReadablePage => {
const articleText = capture.blocks.filter(isArticleBlock).map(render).join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
const useFallback = wordCountOf(articleText) < MIN_ARTICLE_WORDS;
const text = useFallback ? capture.bodyText.trim() : articleText;
return {
title: capture.title.trim(),
url: capture.url,
text,
wordCount: wordCountOf(text),
};
};
|
