From ea11182a0e8cb62d208b5b53a60c98c8bdef9e9a Mon Sep 17 00:00:00 2001 From: amankumarsingh77 Date: Tue, 28 Jul 2026 21:17:13 +0530 Subject: feat!: remove the deep-research feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness ships browser tools; orchestrating a multi-agent research workflow on top of them is the agent's job, not this extension's. Drops the skill, the /deep-research command, and the web-search-researcher subagent config. browser_web_search and browser_read_page are untouched — the two tests that covered their pure parsers move to test/domains/, where the test glob still picks them up. Co-Authored-By: Claude Opus 5 (1M context) --- .pi/agents/.rpiv-managed.json | 3 +- .pi/agents/web-search-researcher.md | 58 ------------------- CHANGELOG.md | 6 ++ README.md | 7 +-- docs/ARCHITECTURE.md | 2 +- package.json | 2 +- skills/deep-research/SKILL.md | 66 ---------------------- skills/pi-browser-harness/SKILL.md | 2 +- src/deep-research.ts | 25 -------- src/index.ts | 2 - test/deep-research/agent-config-test.ts | 43 -------------- test/deep-research/fixtures/article-capture.json | 17 ------ .../fixtures/google-serp-captcha.json | 4 -- test/deep-research/fixtures/google-serp-empty.json | 4 -- test/deep-research/fixtures/google-serp-ok.json | 40 ------------- .../deep-research/fixtures/no-article-capture.json | 10 ---- test/deep-research/readability-test.ts | 60 -------------------- test/deep-research/serp-parser-test.ts | 60 -------------------- test/domains/fixtures/article-capture.json | 17 ++++++ test/domains/fixtures/google-serp-captcha.json | 4 ++ test/domains/fixtures/google-serp-empty.json | 4 ++ test/domains/fixtures/google-serp-ok.json | 40 +++++++++++++ test/domains/fixtures/no-article-capture.json | 10 ++++ test/domains/readability-test.ts | 60 ++++++++++++++++++++ test/domains/serp-parser-test.ts | 60 ++++++++++++++++++++ 25 files changed, 206 insertions(+), 400 deletions(-) delete mode 100644 .pi/agents/web-search-researcher.md delete mode 100644 skills/deep-research/SKILL.md delete mode 100644 src/deep-research.ts delete mode 100644 test/deep-research/agent-config-test.ts delete mode 100644 test/deep-research/fixtures/article-capture.json delete mode 100644 test/deep-research/fixtures/google-serp-captcha.json delete mode 100644 test/deep-research/fixtures/google-serp-empty.json delete mode 100644 test/deep-research/fixtures/google-serp-ok.json delete mode 100644 test/deep-research/fixtures/no-article-capture.json delete mode 100644 test/deep-research/readability-test.ts delete mode 100644 test/deep-research/serp-parser-test.ts create mode 100644 test/domains/fixtures/article-capture.json create mode 100644 test/domains/fixtures/google-serp-captcha.json create mode 100644 test/domains/fixtures/google-serp-empty.json create mode 100644 test/domains/fixtures/google-serp-ok.json create mode 100644 test/domains/fixtures/no-article-capture.json create mode 100644 test/domains/readability-test.ts create mode 100644 test/domains/serp-parser-test.ts diff --git a/.pi/agents/.rpiv-managed.json b/.pi/agents/.rpiv-managed.json index 51543a4..9a26529 100644 --- a/.pi/agents/.rpiv-managed.json +++ b/.pi/agents/.rpiv-managed.json @@ -11,6 +11,5 @@ "scope-tracer.md": "2a2064465dd81c27278dc8c0a7e7b549f9cc3e31487e77ad3d35a9f9b2827fdc", "test-case-locator.md": "c67c95442ffafb66ededa5f0fbfb11efe5f6cce5a7a61adcf29895465535115a", "thoughts-analyzer.md": "ab693f876c6076cfb141ebf30a99b4e4617078eb90058ba9e1276bf5e4de29d2", - "thoughts-locator.md": "5f175e1c177c6e8446a4771c858772c18fc383005719d96ecc45511b4823476f", - "web-search-researcher.md": "621963a90b58d6cea58be52dbfe51a40964e89ef3c7ae08737b2a074b453d62d" + "thoughts-locator.md": "5f175e1c177c6e8446a4771c858772c18fc383005719d96ecc45511b4823476f" } diff --git a/.pi/agents/web-search-researcher.md b/.pi/agents/web-search-researcher.md deleted file mode 100644 index 0ab6a5a..0000000 --- a/.pi/agents/web-search-researcher.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -name: web-search-researcher -description: Researches ONE focused sub-question against the live web using the real browser. Give it a single, well-scoped question; it runs browser_web_search, reads the most relevant results with browser_read_page, and returns a distilled, source-linked finding (claims + urls) — never raw page dumps. Re-run with a sharpened question if the first pass leaves gaps. -tools: browser_web_search, browser_read_page, read, grep, find, ls -isolated: true ---- - -You are an expert web research specialist. You take ONE focused sub-question and return a **distilled, source-linked finding** — the specific claims that answer it, each carrying the URL it came from. You do the reading so the caller doesn't have to: never dump raw pages back, only the distilled answer. - -Your only web tools are **browser_web_search** (ranked SERP results — links only) and **browser_read_page** (a URL or an owned tab → clean readable article text). Both run in their own isolated browser tabs and never disturb the user's tabs. - -## Workflow - -For the single sub-question you were given: - -1. **Frame the search.** Identify the key terms and the kind of source most likely to answer it (official docs, a reputable blog, a spec, a Q&A thread). Draft 1–3 focused queries. - -2. **Search.** Call `browser_web_search({ query, limit })`. It returns ranked `{ title, url, snippet, rank }` — **links only, no page content.** - - On `kind: "invalid_state"` with `details.reason: "captcha"` — a bot wall. **Do not retry in a loop.** Stop and report this clearly in your findings so the caller can surface it to the user. - - On `details.reason: "no_results"` — rephrase the query once with different terms and search again. If still empty, report the gap. - -3. **Pick the few best results.** From the ranked list choose the 3–5 most relevant, most authoritative URLs. Prefer official documentation, primary sources, and recent, reputable material. Skip the obvious spam and SEO filler. - -4. **Read them.** Call `browser_read_page({ url })` on each chosen result. It returns clean main-article text with nav/ads/boilerplate stripped. Extract the specific sentences, values, and code that bear on the sub-question. Note publication dates and version numbers when they matter. - -5. **Distill.** Turn what you read into a small set of claims that directly answer the sub-question, each tagged with its source URL. Drop everything that doesn't answer it. If sources conflict, say so and cite both. If the question can't be fully answered from what you found, name the gap — do not pad or invent. - -Be efficient: 1–3 searches, then read only the most promising 3–5 pages. Refine and re-read only if the first pass genuinely leaves the sub-question unanswered. - -## Output Format - -Return your finding in this structure — distilled, every claim linked, no raw page text: - -``` -## Summary -{2–4 sentences answering the sub-question, at the level of "here is what the web says".} - -## Findings - -- {A specific claim that answers the sub-question.} — [{source title}]({url}) -- {Another claim, possibly from a different source.} — [{source title}]({url}) -- {A version-specific or dated detail, if relevant.} — [{source title}]({url}) (as of {date/version}) - -## Conflicts / Caveats -{Any disagreement between sources, outdated info, or uncertainty. Omit if none.} - -## Gaps -{What part of the sub-question could not be answered from available sources, and why. Write "None" if fully answered.} -``` - -## Quality Guidelines - -- **Distill, don't dump.** The caller wants the answer, not the pages. Every line is a claim or a caveat, each with a link. -- **Every claim carries a source URL.** No unsourced assertions. -- **Authority first.** Prefer official docs, specs, and primary sources over aggregators. -- **Currency.** Note dates and versions when the answer depends on them. -- **Honesty about gaps.** Report what you couldn't find rather than filling it with guesses. -- **Stay in scope.** Answer the one sub-question you were given; don't wander into adjacent topics. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ce251b..5db9367 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to pi-browser-harness will be documented in this file. +## Unreleased + +### Removed + +- **The `deep-research` skill, the `/deep-research` command, and the `web-search-researcher` subagent.** The harness ships browser tools; orchestrating multi-agent research on top of them belongs to the agent, not to this extension. `browser_web_search` and `browser_read_page` are unaffected — search then read still works, it is just no longer wrapped in a fan-out workflow. + ## 0.10.3 — 2026-07-26 ### Fixed diff --git a/README.md b/README.md index 53208ff..f85effc 100644 --- a/README.md +++ b/README.md @@ -238,11 +238,7 @@ browser_web_search({ query: "chrome devtools protocol overview", limit: 5 }) browser_read_page({ url: "" }) ``` -For a question that needs several sources synthesized into a **cited report**, use -`/deep-research `. It decomposes the question, fans out one isolated -`web-search-researcher` subagent per sub-question, loops for coverage under a hard ceiling, and -writes a Markdown report where every claim carries a source link. The skill also auto-triggers on -research-shaped requests without the command. +Both tools open their own isolated tab, so neither disturbs the tab you are working in. ### Keyboard modifiers @@ -310,7 +306,6 @@ re-runnable. | `/browser-setup` | Connect pi to your browser. Run once; idempotent. | | `/browser-profile` | Pick which browser profile the agent works in. Remembered across sessions. | | `/browser-status` | Connection state, selected profile, and current page. | -| `/deep-research ` | Multi-source web research → cited Markdown report. | | `/browser-reload-daemon` | Force a client restart. Rarely needed — the transport reconnects on its own. | | Flag / variable | Effect | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c1efae2..e09e4a7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -339,7 +339,7 @@ of it: `@mariozechner/pi-coding-agent` publishes no `"require"` condition, and t unresolvable. It also *matches* production — pi loads the extension with jiti and aliases that package to `dist/index.js` by absolute path, which is what the `import` condition selects. -**The test glob is `test/{profile,deep-research,domains,cdp,util}/**/*-test.ts`.** A test placed +**The test glob is `test/{profile,domains,cdp,util}/**/*-test.ts`.** A test placed in any other directory — `test/schemas/`, `test/daemon/` — **silently never runs.** `test/manual/` is excluded on purpose: those need live Chrome. diff --git a/package.json b/package.json index 2507382..1ea453e 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ }, "scripts": { "typecheck": "tsc --noEmit && tsc -p tsconfig.test.json", - "test": "node --conditions=import --import tsx --test \"test/{profile,deep-research,domains,cdp,util}/**/*-test.ts\"", + "test": "node --conditions=import --import tsx --test \"test/{profile,domains,cdp,util}/**/*-test.ts\"", "boundaries": "tsx scripts/check-boundaries.ts", "check": "npm run typecheck && npm run boundaries && npm test", "prepublishOnly": "tsc --noEmit" diff --git a/skills/deep-research/SKILL.md b/skills/deep-research/SKILL.md deleted file mode 100644 index 846c522..0000000 --- a/skills/deep-research/SKILL.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -name: deep-research -description: Use when the user asks you to research a topic on the web, do deep research, investigate something across multiple sources, or produce a cited report / literature review from the live web. Decomposes the question, fans out isolated web-search-researcher subagents (one per sub-question) that scrape the real browser, then synthesizes a source-cited Markdown report. Trigger phrases include "research", "deep research", "look this up and cite sources", "investigate", "compare X and Y with sources". ---- - -# Deep Research - -Turn a research question into a **cited Markdown report** by fanning out to isolated `web-search-researcher` subagents that use the real browser, then synthesizing their distilled findings. Every claim in the final report carries a source link. - -This runs on two browser tools — `browser_web_search` (ranked SERP, links only) and `browser_read_page` (URL → clean article text) — used *inside the subagents*, not by you directly. Both tools open their own isolated tabs, so **the fan-out never disturbs the user's tabs or current page.** You do not need `/browser-setup` to have been run by you; if the browser isn't connected the subagents' first tool call will report `not_connected` — relay that and ask the user to run `/browser-setup`. - -## Workflow - -### 1. Decompose - -Break the question into **3–6 focused sub-questions**, each independently answerable from the web. Cover the distinct facets — definitions, current state, competing options, trade-offs, recent changes — not slight rewordings of the same thing. Aim for coverage: if you synthesized perfect answers to all sub-questions, the original question should be fully answered. - -State the sub-questions to the user before dispatching (a short numbered list) so the plan is visible. - -### 2. Fan out - -Dispatch **one `web-search-researcher` subagent per sub-question**, in parallel where the runtime allows. Each subagent runs in isolated context and returns a distilled, source-linked finding (claims + urls) — not raw pages. Give each subagent exactly one sub-question plus any constraints (recency, specific ecosystem, authoritative sources to prefer). - -Do not do the searching yourself — that is the subagents' job, and their isolation is what keeps your context clean enough to synthesize well. - -### 3. Assess coverage — loop with a hard ceiling - -After each round of findings, self-assess: **is every sub-question answered, or did findings surface new gaps worth a follow-up search?** - -- If gaps remain and you are under the ceiling, dispatch a second round of researchers targeting only the gaps (rephrased or newly discovered sub-questions). -- **Hard ceiling:** at most **2 rounds** and **8 researcher subagents total**. When you hit either limit, stop searching, synthesize with what you have, and list the unresolved gaps explicitly in the report. Never loop indefinitely chasing completeness. - -Handle subagent-reported tool failures: -- A subagent reporting a `browser_web_search` `invalid_state` with `reason: "captcha"` means a bot wall. **Surface this to the user immediately** and do not spawn a tight retry loop — ask whether to continue, wait, or narrow scope. -- `reason: "no_results"` is already handled inside the subagent (one rephrase); if it still comes back empty, treat that sub-question as a gap. - -### 4. Synthesize a cited report - -Merge the distilled findings into a **Markdown report written to a file** (e.g. `research-.md` in the working directory — tell the user the path). Structure: - -``` -# {The research question} - -## Summary -{A few paragraphs answering the question, synthesized across sources.} - -## {Sub-topic / sub-question 1} -{Findings, in prose. Every claim carries a source link inline: … as shown in [Title](url).} - -## {Sub-topic 2} -... - -## Open questions / gaps -{Anything the ceiling left unresolved, or where sources conflicted.} - -## Sources -{Deduplicated list of every URL cited, with titles.} -``` - -Rules for the report: -- **Every factual claim carries a source link** — inline `[title](url)`. No unsourced assertions. -- **Attribute conflicts.** Where sources disagree, present both and cite each. -- **Distinguish synthesis from quotation.** Don't fabricate precision the sources didn't provide. -- **Be honest about gaps.** The gaps section is mandatory when the ceiling cut off coverage — an incomplete-but-honest report beats a complete-looking-but-padded one. - -After writing the file, give the user a short chat summary and the file path — don't paste the whole report into the conversation. diff --git a/skills/pi-browser-harness/SKILL.md b/skills/pi-browser-harness/SKILL.md index fa76a2e..36d0346 100644 --- a/skills/pi-browser-harness/SKILL.md +++ b/skills/pi-browser-harness/SKILL.md @@ -36,7 +36,7 @@ What do you need to know? Pass `@(x,y)` from `browser_snapshot` straight to `browser_click`. No screenshot round-trip. -`browser_web_search` and `browser_read_page` each run in their own isolated tab and never touch the user's current tab. For a multi-source question that needs a synthesized, cited report, use the **deep-research** skill (or `/deep-research `): it fans out isolated `web-search-researcher` subagents over both tools and writes a source-cited Markdown report. +`browser_web_search` and `browser_read_page` each run in their own isolated tab and never touch the user's current tab. Pair them for a research question: search for candidate URLs, then read the promising ones. ## Connection Setup diff --git a/src/deep-research.ts b/src/deep-research.ts deleted file mode 100644 index 9445cf8..0000000 --- a/src/deep-research.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"; - -export function buildDeepResearchPrompt(question: string): string { - return ( - `Do deep research on the following question and produce a cited Markdown report. ` + - `Use the deep-research skill: decompose it into sub-questions, fan out web-search-researcher ` + - `subagents to research each against the live web, then synthesize a source-cited report file.\n\n` + - `Question: ${question}` - ); -} - -export function registerDeepResearchCommand(pi: ExtensionAPI): void { - pi.registerCommand("deep-research", { - description: "Research a question on the web and write a cited report", - handler: async (args, ctx) => { - const question = args.trim(); - if (question.length === 0) { - ctx.ui.notify('Usage: /deep-research . Example: /deep-research "compare CDP vs WebDriver"', "warning"); - return; - } - ctx.ui.notify(`Starting deep research: ${question}`, "info"); - pi.sendUserMessage(buildDeepResearchPrompt(question)); - }, - }); -} diff --git a/src/index.ts b/src/index.ts index e0fa750..1573605 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,7 +4,6 @@ import { getBrowserSystemPrompt } from "./prompt"; import { registerSetupCommand } from "./setup"; import { registerProfileCommand } from "./profile/command"; import { readPin } from "./profile/store"; -import { registerDeepResearchCommand } from "./deep-research"; import { type BrowserState, defaultState, persistState, restoreState } from "./state"; import { registerAllTools } from "./registry"; import { cleanupTempDirs } from "./util/truncate"; @@ -124,7 +123,6 @@ export default function browserHarnessExtension(pi: ExtensionAPI): void { } registerSetupCommand(pi, client); registerProfileCommand(pi, client); - registerDeepResearchCommand(pi); }); pi.on("session_shutdown", async () => { diff --git a/test/deep-research/agent-config-test.ts b/test/deep-research/agent-config-test.ts deleted file mode 100644 index 025a844..0000000 --- a/test/deep-research/agent-config-test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { test, before, describe } from "node:test"; -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; - -describe("web-search-researcher subagent config references only real tools", () => { - let frontmatter: string; - let body: string; - let toolsLine: string; - - before(() => { - const agentPath = join(import.meta.dirname, "..", "..", ".pi", "agents", "web-search-researcher.md"); - const raw = readFileSync(agentPath, "utf8"); - const fmMatch = raw.match(/^---\n([\s\S]*?)\n---/); - frontmatter = fmMatch ? (fmMatch[1] ?? "") : ""; - body = fmMatch ? raw.slice(fmMatch[0].length) : raw; - toolsLine = frontmatter.split("\n").find((l) => l.startsWith("tools:")) ?? ""; - }); - - test("S20: tools includes browser_web_search", () => { - assert.ok(toolsLine.includes("browser_web_search")); - }); - - test("S20: tools includes browser_read_page", () => { - assert.ok(toolsLine.includes("browser_read_page")); - }); - - test("S20: tools does NOT include web_search", () => { - assert.ok(!/\bweb_search\b/.test(toolsLine)); - }); - - test("S20: tools does NOT include web_fetch", () => { - assert.ok(!/\bweb_fetch\b/.test(toolsLine)); - }); - - test("S20: isolated: true is present", () => { - assert.ok(/^\s*isolated:\s*true\s*$/m.test(frontmatter)); - }); - - test("S20: body has no WebSearch/WebFetch references", () => { - assert.ok(!/WebSearch|WebFetch/.test(body)); - }); -}); diff --git a/test/deep-research/fixtures/article-capture.json b/test/deep-research/fixtures/article-capture.json deleted file mode 100644 index df9c67c..0000000 --- a/test/deep-research/fixtures/article-capture.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "url": "https://blog.example.com/how-cdp-works", - "title": "How the Chrome DevTools Protocol Works", - "blocks": [ - { "kind": "other", "text": "Home Products Blog About Contact", "linkTextLength": 33, "inBoilerplate": true }, - { "kind": "heading", "text": "How the Chrome DevTools Protocol Works", "linkTextLength": 0, "inBoilerplate": false }, - { "kind": "paragraph", "text": "The Chrome DevTools Protocol lets tools instrument, inspect, and control Chromium-based browsers over a WebSocket connection.", "linkTextLength": 0, "inBoilerplate": false }, - { "kind": "paragraph", "text": "Each command targets a domain such as Page, Runtime, or Network, and receives a structured JSON response that the client correlates by id.", "linkTextLength": 0, "inBoilerplate": false }, - { "kind": "heading", "text": "Sessions and Targets", "linkTextLength": 0, "inBoilerplate": false }, - { "kind": "paragraph", "text": "A target is a page or worker; attaching to one yields a session id that scopes subsequent commands to that target without disturbing others.", "linkTextLength": 0, "inBoilerplate": false }, - { "kind": "listitem", "text": "Attach with Target.attachToTarget", "linkTextLength": 0, "inBoilerplate": false }, - { "kind": "listitem", "text": "Drive with Runtime.evaluate", "linkTextLength": 0, "inBoilerplate": false }, - { "kind": "paragraph", "text": "Subscribe here for our newsletter and follow us on social media for updates", "linkTextLength": 55, "inBoilerplate": true }, - { "kind": "other", "text": "Copyright 2026 Example Inc. Privacy Terms Cookies", "linkTextLength": 24, "inBoilerplate": true } - ], - "bodyText": "Home Products Blog How the Chrome DevTools Protocol Works The Chrome DevTools Protocol lets tools instrument inspect and control browsers Copyright 2026" -} diff --git a/test/deep-research/fixtures/google-serp-captcha.json b/test/deep-research/fixtures/google-serp-captcha.json deleted file mode 100644 index 7c6610f..0000000 --- a/test/deep-research/fixtures/google-serp-captcha.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "anchors": [], - "pageText": "About this page Our systems have detected unusual traffic from your computer network. This page checks to see if it's really you sending the requests and not a robot. Why did this happen? To continue, please type the characters below. reCAPTCHA." -} diff --git a/test/deep-research/fixtures/google-serp-empty.json b/test/deep-research/fixtures/google-serp-empty.json deleted file mode 100644 index 34bb1dc..0000000 --- a/test/deep-research/fixtures/google-serp-empty.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "anchors": [], - "pageText": "Your search - asdkfjhqwlekjrhqwlekjrhqwe - did not match any documents. Suggestions: Make sure that all words are spelled correctly. Try different keywords." -} diff --git a/test/deep-research/fixtures/google-serp-ok.json b/test/deep-research/fixtures/google-serp-ok.json deleted file mode 100644 index 61aa483..0000000 --- a/test/deep-research/fixtures/google-serp-ok.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "anchors": [ - { - "href": "https://www.google.com/url?q=https://real.example/x&sa=U&ved=abc", - "heading": "Real Example X — the definitive guide", - "snippet": "Real Example X — the definitive guide. Everything about x, explained clearly." - }, - { - "href": "https://docs.example.org/guide", - "heading": "Official Guide", - "snippet": "Official Guide to the thing you searched for." - }, - { - "href": "https://www.google.com/search?q=related", - "heading": "Related searches on Google", - "snippet": "internal google navigation" - }, - { - "href": "javascript:void(0)", - "heading": "Expand", - "snippet": "ui control" - }, - { - "href": "https://real.example/x/", - "heading": "Real Example X (duplicate with trailing slash)", - "snippet": "should be deduped against the first result" - }, - { - "href": "https://second.example/a", - "heading": "Second Example A", - "snippet": "Second result body." - }, - { - "href": "https://third.example/b", - "heading": "Third Example B", - "snippet": "Third result body." - } - ], - "pageText": "Real Example X the definitive guide Official Guide Second Example Third Example About 12,300 results" -} diff --git a/test/deep-research/fixtures/no-article-capture.json b/test/deep-research/fixtures/no-article-capture.json deleted file mode 100644 index d367b7e..0000000 --- a/test/deep-research/fixtures/no-article-capture.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "url": "https://app.example.com/dashboard", - "title": "Dashboard", - "blocks": [ - { "kind": "other", "text": "Menu", "linkTextLength": 4, "inBoilerplate": true }, - { "kind": "listitem", "text": "Widget A", "linkTextLength": 8, "inBoilerplate": true }, - { "kind": "listitem", "text": "Widget B", "linkTextLength": 8, "inBoilerplate": true } - ], - "bodyText": "Dashboard Widget A Widget B Your account is active. Usage this month is within limits and no action is required at this time from you." -} diff --git a/test/deep-research/readability-test.ts b/test/deep-research/readability-test.ts deleted file mode 100644 index 211229d..0000000 --- a/test/deep-research/readability-test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { test, describe } from "node:test"; -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; -import { extractReadable, type PageCapture } from "../../src/domains/readpage/readability"; - -const fixture = (name: string): PageCapture => - JSON.parse(readFileSync(join(import.meta.dirname, "fixtures", name), "utf8")) as PageCapture; - -describe("pure readability extractor", () => { - test("S10: article body text is present", () => { - const page = extractReadable(fixture("article-capture.json")); - assert.ok(page.text.includes("Chrome DevTools Protocol lets tools instrument")); - }); - - test("S10: article headings are preserved", () => { - const page = extractReadable(fixture("article-capture.json")); - assert.ok(page.text.includes("Sessions and Targets")); - }); - - test("S10: nav boilerplate is dropped", () => { - const page = extractReadable(fixture("article-capture.json")); - assert.ok(!page.text.includes("Home Products Blog")); - }); - - test("S10: footer boilerplate is dropped", () => { - const page = extractReadable(fixture("article-capture.json")); - assert.ok(!page.text.includes("Copyright 2026")); - }); - - test("S10: high-link-density promo block is dropped", () => { - const page = extractReadable(fixture("article-capture.json")); - assert.ok(!page.text.includes("Subscribe here for our newsletter")); - }); - - test("S10: paragraph/heading structure preserved as markdown", () => { - const page = extractReadable(fixture("article-capture.json")); - assert.ok(page.text.includes("## Sessions and Targets")); - }); - - test("S10: title and url passed through", () => { - const page = extractReadable(fixture("article-capture.json")); - assert.ok(page.title === "How the Chrome DevTools Protocol Works" && page.url.startsWith("https://")); - }); - - test("S10: word count reflects the article body", () => { - const page = extractReadable(fixture("article-capture.json")); - assert.ok(page.wordCount > 40); - }); - - test("S11: structure-less page falls back to non-empty body text", () => { - const page = extractReadable(fixture("no-article-capture.json")); - assert.ok(page.text.length > 0); - }); - - test("S11: fallback body text carries the real content", () => { - const page = extractReadable(fixture("no-article-capture.json")); - assert.ok(page.text.includes("Your account is active")); - }); -}); diff --git a/test/deep-research/serp-parser-test.ts b/test/deep-research/serp-parser-test.ts deleted file mode 100644 index 433c650..0000000 --- a/test/deep-research/serp-parser-test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { test, describe } from "node:test"; -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import { join } from "node:path"; -import { classifySerp, parseGoogleSerp, type SerpExtraction } from "../../src/domains/search/google-serp"; - -const fixture = (name: string): SerpExtraction => - JSON.parse(readFileSync(join(import.meta.dirname, "fixtures", name), "utf8")) as SerpExtraction; - -const ok = fixture("google-serp-ok.json"); -const captcha = fixture("google-serp-captcha.json"); -const empty = fixture("google-serp-empty.json"); - -describe("pure Google SERP parser", () => { - test("S1: /url?q= redirect is unwrapped to the real target", () => { - const results = parseGoogleSerp(ok.anchors, 10); - const urls = results.map((r) => r.url); - assert.ok(urls.includes("https://real.example/x")); - }); - - test("S1: google-internal anchors are dropped", () => { - const results = parseGoogleSerp(ok.anchors, 10); - const urls = results.map((r) => r.url); - assert.ok(!urls.some((u) => u.includes("google.com"))); - }); - - test("S1: javascript: anchors are dropped", () => { - const results = parseGoogleSerp(ok.anchors, 10); - const urls = results.map((r) => r.url); - assert.ok(!urls.some((u) => u.startsWith("javascript:"))); - }); - - test("S2: trailing-slash duplicate collapses to one result", () => { - const results = parseGoogleSerp(ok.anchors, 10); - const realExampleCount = results.filter((r) => r.url.startsWith("https://real.example/x")).length; - assert.equal(realExampleCount, 1); - }); - - test("S2: rank runs 1..n in input order", () => { - const results = parseGoogleSerp(ok.anchors, 10); - assert.ok(results.every((r, i) => r.rank === i + 1)); - }); - - test("S2: limit truncates and ranks stay 1..limit", () => { - const capped = parseGoogleSerp(ok.anchors, 2); - assert.ok(capped.length === 2 && capped[capped.length - 1]?.rank === 2); - }); - - test("S3: populated SERP classifies as ok", () => { - assert.equal(classifySerp(ok.pageText, parseGoogleSerp(ok.anchors, 10).length), "ok"); - }); - - test("S3: captcha page classifies as captcha", () => { - assert.equal(classifySerp(captcha.pageText, 0), "captcha"); - }); - - test("S3: empty results classify as no_results", () => { - assert.equal(classifySerp(empty.pageText, 0), "no_results"); - }); -}); diff --git a/test/domains/fixtures/article-capture.json b/test/domains/fixtures/article-capture.json new file mode 100644 index 0000000..df9c67c --- /dev/null +++ b/test/domains/fixtures/article-capture.json @@ -0,0 +1,17 @@ +{ + "url": "https://blog.example.com/how-cdp-works", + "title": "How the Chrome DevTools Protocol Works", + "blocks": [ + { "kind": "other", "text": "Home Products Blog About Contact", "linkTextLength": 33, "inBoilerplate": true }, + { "kind": "heading", "text": "How the Chrome DevTools Protocol Works", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "paragraph", "text": "The Chrome DevTools Protocol lets tools instrument, inspect, and control Chromium-based browsers over a WebSocket connection.", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "paragraph", "text": "Each command targets a domain such as Page, Runtime, or Network, and receives a structured JSON response that the client correlates by id.", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "heading", "text": "Sessions and Targets", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "paragraph", "text": "A target is a page or worker; attaching to one yields a session id that scopes subsequent commands to that target without disturbing others.", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "listitem", "text": "Attach with Target.attachToTarget", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "listitem", "text": "Drive with Runtime.evaluate", "linkTextLength": 0, "inBoilerplate": false }, + { "kind": "paragraph", "text": "Subscribe here for our newsletter and follow us on social media for updates", "linkTextLength": 55, "inBoilerplate": true }, + { "kind": "other", "text": "Copyright 2026 Example Inc. Privacy Terms Cookies", "linkTextLength": 24, "inBoilerplate": true } + ], + "bodyText": "Home Products Blog How the Chrome DevTools Protocol Works The Chrome DevTools Protocol lets tools instrument inspect and control browsers Copyright 2026" +} diff --git a/test/domains/fixtures/google-serp-captcha.json b/test/domains/fixtures/google-serp-captcha.json new file mode 100644 index 0000000..7c6610f --- /dev/null +++ b/test/domains/fixtures/google-serp-captcha.json @@ -0,0 +1,4 @@ +{ + "anchors": [], + "pageText": "About this page Our systems have detected unusual traffic from your computer network. This page checks to see if it's really you sending the requests and not a robot. Why did this happen? To continue, please type the characters below. reCAPTCHA." +} diff --git a/test/domains/fixtures/google-serp-empty.json b/test/domains/fixtures/google-serp-empty.json new file mode 100644 index 0000000..34bb1dc --- /dev/null +++ b/test/domains/fixtures/google-serp-empty.json @@ -0,0 +1,4 @@ +{ + "anchors": [], + "pageText": "Your search - asdkfjhqwlekjrhqwlekjrhqwe - did not match any documents. Suggestions: Make sure that all words are spelled correctly. Try different keywords." +} diff --git a/test/domains/fixtures/google-serp-ok.json b/test/domains/fixtures/google-serp-ok.json new file mode 100644 index 0000000..61aa483 --- /dev/null +++ b/test/domains/fixtures/google-serp-ok.json @@ -0,0 +1,40 @@ +{ + "anchors": [ + { + "href": "https://www.google.com/url?q=https://real.example/x&sa=U&ved=abc", + "heading": "Real Example X — the definitive guide", + "snippet": "Real Example X — the definitive guide. Everything about x, explained clearly." + }, + { + "href": "https://docs.example.org/guide", + "heading": "Official Guide", + "snippet": "Official Guide to the thing you searched for." + }, + { + "href": "https://www.google.com/search?q=related", + "heading": "Related searches on Google", + "snippet": "internal google navigation" + }, + { + "href": "javascript:void(0)", + "heading": "Expand", + "snippet": "ui control" + }, + { + "href": "https://real.example/x/", + "heading": "Real Example X (duplicate with trailing slash)", + "snippet": "should be deduped against the first result" + }, + { + "href": "https://second.example/a", + "heading": "Second Example A", + "snippet": "Second result body." + }, + { + "href": "https://third.example/b", + "heading": "Third Example B", + "snippet": "Third result body." + } + ], + "pageText": "Real Example X the definitive guide Official Guide Second Example Third Example About 12,300 results" +} diff --git a/test/domains/fixtures/no-article-capture.json b/test/domains/fixtures/no-article-capture.json new file mode 100644 index 0000000..d367b7e --- /dev/null +++ b/test/domains/fixtures/no-article-capture.json @@ -0,0 +1,10 @@ +{ + "url": "https://app.example.com/dashboard", + "title": "Dashboard", + "blocks": [ + { "kind": "other", "text": "Menu", "linkTextLength": 4, "inBoilerplate": true }, + { "kind": "listitem", "text": "Widget A", "linkTextLength": 8, "inBoilerplate": true }, + { "kind": "listitem", "text": "Widget B", "linkTextLength": 8, "inBoilerplate": true } + ], + "bodyText": "Dashboard Widget A Widget B Your account is active. Usage this month is within limits and no action is required at this time from you." +} diff --git a/test/domains/readability-test.ts b/test/domains/readability-test.ts new file mode 100644 index 0000000..211229d --- /dev/null +++ b/test/domains/readability-test.ts @@ -0,0 +1,60 @@ +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { extractReadable, type PageCapture } from "../../src/domains/readpage/readability"; + +const fixture = (name: string): PageCapture => + JSON.parse(readFileSync(join(import.meta.dirname, "fixtures", name), "utf8")) as PageCapture; + +describe("pure readability extractor", () => { + test("S10: article body text is present", () => { + const page = extractReadable(fixture("article-capture.json")); + assert.ok(page.text.includes("Chrome DevTools Protocol lets tools instrument")); + }); + + test("S10: article headings are preserved", () => { + const page = extractReadable(fixture("article-capture.json")); + assert.ok(page.text.includes("Sessions and Targets")); + }); + + test("S10: nav boilerplate is dropped", () => { + const page = extractReadable(fixture("article-capture.json")); + assert.ok(!page.text.includes("Home Products Blog")); + }); + + test("S10: footer boilerplate is dropped", () => { + const page = extractReadable(fixture("article-capture.json")); + assert.ok(!page.text.includes("Copyright 2026")); + }); + + test("S10: high-link-density promo block is dropped", () => { + const page = extractReadable(fixture("article-capture.json")); + assert.ok(!page.text.includes("Subscribe here for our newsletter")); + }); + + test("S10: paragraph/heading structure preserved as markdown", () => { + const page = extractReadable(fixture("article-capture.json")); + assert.ok(page.text.includes("## Sessions and Targets")); + }); + + test("S10: title and url passed through", () => { + const page = extractReadable(fixture("article-capture.json")); + assert.ok(page.title === "How the Chrome DevTools Protocol Works" && page.url.startsWith("https://")); + }); + + test("S10: word count reflects the article body", () => { + const page = extractReadable(fixture("article-capture.json")); + assert.ok(page.wordCount > 40); + }); + + test("S11: structure-less page falls back to non-empty body text", () => { + const page = extractReadable(fixture("no-article-capture.json")); + assert.ok(page.text.length > 0); + }); + + test("S11: fallback body text carries the real content", () => { + const page = extractReadable(fixture("no-article-capture.json")); + assert.ok(page.text.includes("Your account is active")); + }); +}); diff --git a/test/domains/serp-parser-test.ts b/test/domains/serp-parser-test.ts new file mode 100644 index 0000000..433c650 --- /dev/null +++ b/test/domains/serp-parser-test.ts @@ -0,0 +1,60 @@ +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { classifySerp, parseGoogleSerp, type SerpExtraction } from "../../src/domains/search/google-serp"; + +const fixture = (name: string): SerpExtraction => + JSON.parse(readFileSync(join(import.meta.dirname, "fixtures", name), "utf8")) as SerpExtraction; + +const ok = fixture("google-serp-ok.json"); +const captcha = fixture("google-serp-captcha.json"); +const empty = fixture("google-serp-empty.json"); + +describe("pure Google SERP parser", () => { + test("S1: /url?q= redirect is unwrapped to the real target", () => { + const results = parseGoogleSerp(ok.anchors, 10); + const urls = results.map((r) => r.url); + assert.ok(urls.includes("https://real.example/x")); + }); + + test("S1: google-internal anchors are dropped", () => { + const results = parseGoogleSerp(ok.anchors, 10); + const urls = results.map((r) => r.url); + assert.ok(!urls.some((u) => u.includes("google.com"))); + }); + + test("S1: javascript: anchors are dropped", () => { + const results = parseGoogleSerp(ok.anchors, 10); + const urls = results.map((r) => r.url); + assert.ok(!urls.some((u) => u.startsWith("javascript:"))); + }); + + test("S2: trailing-slash duplicate collapses to one result", () => { + const results = parseGoogleSerp(ok.anchors, 10); + const realExampleCount = results.filter((r) => r.url.startsWith("https://real.example/x")).length; + assert.equal(realExampleCount, 1); + }); + + test("S2: rank runs 1..n in input order", () => { + const results = parseGoogleSerp(ok.anchors, 10); + assert.ok(results.every((r, i) => r.rank === i + 1)); + }); + + test("S2: limit truncates and ranks stay 1..limit", () => { + const capped = parseGoogleSerp(ok.anchors, 2); + assert.ok(capped.length === 2 && capped[capped.length - 1]?.rank === 2); + }); + + test("S3: populated SERP classifies as ok", () => { + assert.equal(classifySerp(ok.pageText, parseGoogleSerp(ok.anchors, 10).length), "ok"); + }); + + test("S3: captcha page classifies as captcha", () => { + assert.equal(classifySerp(captcha.pageText, 0), "captcha"); + }); + + test("S3: empty results classify as no_results", () => { + assert.equal(classifySerp(empty.pageText, 0), "no_results"); + }); +}); -- cgit v1.2.3