castle-search: read Google results again by resolving its /goto links

Google now wraps every result as google.com/goto?url=<opaque token>. The
self-host filter discarded all of them, so every Google search failed with
'nothing could be parsed'. Accept /goto, resolve it node-side from the
302 Location, and when Google answers its 'Verifying your request'
interstitial instead, keep the wrapped link and put the displayed
breadcrumb in the snippet.

1 files changed, 50 insertions, 0 deletions

diff --git a/castle-search/test/resolve.test.ts b/castle-search/test/resolve.test.ts
new file mode 100644
index 0000000..bcd4ef4
--- /dev/null
+++ b/castle-search/test/resolve.test.ts
@@ -0,0 +1,50 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { isGoogleGoto, resolveGoogleLinks } from "../src/resolve.ts";
+import { getEngine } from "../src/engines.ts";
+
+test("google /goto links are recognised, other google links are not", () => {
+  assert.equal(isGoogleGoto("https://www.google.com/goto?url=CAES"), true);
+  assert.equal(isGoogleGoto("https://www.google.co.uk/goto?url=CAES"), true);
+  assert.equal(isGoogleGoto("https://www.google.com/search?q=x"), false);
+  assert.equal(isGoogleGoto("https://example.com/goto?url=x"), false);
+});
+
+test("the google self-host filter lets /goto through but still blocks google's own pages", () => {
+  const re = new RegExp(getEngine("google").extraction.selfHostPattern);
+  assert.equal(re.test("https://www.google.com/goto?url=CAES"), false);
+  assert.equal(re.test("https://www.google.com/search?q=x"), true);
+  assert.equal(re.test("https://accounts.google.com/ServiceLogin"), false);
+});
+
+test("non-google results pass through untouched", async () => {
+  const input = [{ title: "t", url: "https://example.com/", snippet: "" }];
+  assert.deepEqual(await resolveGoogleLinks(input), input);
+});
+
+test("a link that cannot be resolved keeps the page's breadcrumb in its snippet", async () => {
+  const wrapped = "https://www.google.com/goto?url=CAES";
+  const real = globalThis.fetch;
+  // Google's interstitial: 200 with no Location header.
+  globalThis.fetch = (async () => new Response("<html>Verifying your request</html>", { status: 200 })) as typeof fetch;
+  try {
+    const [r] = await resolveGoogleLinks([{ title: "t", url: wrapped, snippet: "s", display: "GitHub — https://github.com › a › b" }]);
+    assert.equal(r?.url, wrapped);
+    assert.match(r?.snippet ?? "", /^\[link unresolved; Google shows it as: GitHub — https:\/\/github\.com › a › b\] s$/);
+  } finally {
+    globalThis.fetch = real;
+  }
+});
+
+test("a link that resolves is replaced and the snippet is left alone", async () => {
+  const real = globalThis.fetch;
+  globalThis.fetch = (async () => new Response(null, { status: 302, headers: { location: "https://github.com/a/b" } })) as typeof fetch;
+  try {
+    const [r] = await resolveGoogleLinks([{ title: "t", url: "https://www.google.com/goto?url=CAES", snippet: "s", display: "x" }]);
+    assert.equal(r?.url, "https://github.com/a/b");
+    assert.equal(r?.snippet, "s");
+  } finally {
+    globalThis.fetch = real;
+  }
+});