1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
/**
* The script that runs inside the search results page.
*
* It is a string rather than a function because it is shipped to Chrome via
* `Runtime.evaluate` — nothing in here is typechecked, so it is kept small,
* defensive, and free of anything that could throw on an unexpected DOM.
*
* ## Why it does not select on class names
*
* Google's result classes (`MjjYud`, `kb0PBd`, `yuRUbf`, …) are generated and
* change without notice; the `data-` hooks (`data-snhf` for the title/source
* header, `data-sncf` for the description) are more stable but not promised
* either. So the extractor uses them when present and otherwise falls back to a
* structural walk that only assumes "an <h3> inside a link, with the
* description somewhere in a shared ancestor":
*
* - anchor = nearest enclosing <a href>
* - header = nearest [data-snhf], else the anchor itself
* - container = climb from the header until the text grows past the header's,
* abandoning the climb if a second <h3> comes into scope
* (that would mean we had swallowed the next result)
* - snippet = [data-sncf] if present, else the container's lines minus the
* header's lines, minus URL/breadcrumb noise
*
* Verified against both the classic SERP and the `udm=14` "Web" layout.
*/
/** One search hit, as the page script reports it. */
export type SearchResult = {
title: string;
url: string;
snippet: string;
};
/** What the page script returns in a single round trip. */
export type PageProbe = {
/** Where the page actually ended up — redirects and consent walls move it. */
url: string;
title: string;
readyState: string;
/** Non-null when a human check is in the way. */
challenge: { kind: string; detail: string } | null;
/**
* Which results container the page rendered. "none" means Google served a
* shell with no results area at all — a different failure from an empty one,
* and worth telling apart when diagnosing a zero-hit search.
*/
container: "rso" | "search" | "none";
results: SearchResult[];
};
/**
* Detection order matters: a `/sorry/` interstitial also contains a recaptcha
* iframe, and reporting the specific page beats reporting the widget on it.
*/
export const PROBE_SCRIPT = String.raw`(() => {
const norm = (s) => (s || "").replace(/\s+/g, " ").trim();
const linesOf = (el) =>
el && el.innerText ? el.innerText.split("\n").map((l) => l.trim()).filter(Boolean) : [];
const bodyText = document.body ? norm(document.body.innerText).slice(0, 4000) : "";
const detectChallenge = () => {
const href = location.href;
if (/\/sorry\//.test(href)) {
return { kind: "google-block-page", detail: bodyText.slice(0, 300) };
}
if (location.hostname.indexOf("consent.") === 0 || /\/consent\b/.test(location.pathname)) {
return { kind: "consent-wall", detail: bodyText.slice(0, 300) };
}
if (document.querySelector("#captcha-form, form#captcha-form")) {
return { kind: "captcha-form", detail: bodyText.slice(0, 300) };
}
if (document.querySelector('iframe[src*="recaptcha"], iframe[src*="hcaptcha"], iframe[title*="challenge"]')) {
return { kind: "captcha-widget", detail: bodyText.slice(0, 300) };
}
if (document.querySelector("#challenge-form, #cf-chl-widget, #cf-challenge-running")) {
return { kind: "cloudflare-challenge", detail: bodyText.slice(0, 300) };
}
if (/unusual traffic|are you a robot|verify (that )?you('| a)?re human|not a robot|automated queries/i.test(bodyText)) {
return { kind: "bot-check", detail: bodyText.slice(0, 300) };
}
return null;
};
const extract = () => {
const root = document.querySelector("#rso") || document.querySelector("#search");
if (!root) return [];
const out = [];
const seen = new Set();
const headings = root.querySelectorAll("h3");
for (let i = 0; i < headings.length; i++) {
const h3 = headings[i];
const anchor = h3.closest("a[href]") || (h3.parentElement && h3.parentElement.querySelector("a[href]"));
if (!anchor) continue;
const url = anchor.href;
if (!/^https?:/.test(url)) continue;
// Drop Google's own links (image search, cached copies, "more results").
if (/^https?:\/\/(www\.)?google\.[a-z.]+\//.test(url)) continue;
if (seen.has(url)) continue;
const title = norm(h3.innerText);
if (!title) continue;
const header = h3.closest("[data-snhf]") || anchor;
const headerLines = new Set(linesOf(header).map(norm));
const baseline = norm(header.innerText || "").length;
let container = header.parentElement;
for (let depth = 0; depth < 6 && container && container !== root; depth++) {
if (container.querySelectorAll("h3").length > 1) {
container = null;
break;
}
if (norm(container.innerText || "").length > baseline + 40) break;
container = container.parentElement;
}
let snippet = "";
if (container && container !== root) {
const explicit = container.querySelector("[data-sncf]");
const body = explicit ? linesOf(explicit) : linesOf(container).filter((l) => !headerLines.has(norm(l)));
snippet = norm(
body
.filter((l) => !/^https?:\/\//.test(l) && l.indexOf("›") === -1 && l !== "Web results")
.join(" "),
)
.replace(/Read more$/, "")
.trim();
}
seen.add(url);
out.push({ title: title, url: url, snippet: snippet.slice(0, 600) });
}
return out;
};
let challenge = null;
let results = [];
let container = "none";
try {
challenge = detectChallenge();
} catch (e) {
challenge = null;
}
try {
container = document.querySelector("#rso") ? "rso" : document.querySelector("#search") ? "search" : "none";
results = challenge ? [] : extract();
} catch (e) {
results = [];
}
return {
url: location.href,
title: document.title || "",
readyState: document.readyState,
challenge: challenge,
container: container,
results: results,
};
})()`;
|
