aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--castle-search/README.md14
-rw-r--r--castle-search/src/http.ts70
-rw-r--r--castle-search/src/youtube.ts303
-rw-r--r--castle-search/test/youtube.test.ts72
4 files changed, 456 insertions, 3 deletions
diff --git a/castle-search/README.md b/castle-search/README.md
index cf15aa1..1621fe3 100644
--- a/castle-search/README.md
+++ b/castle-search/README.md
@@ -85,6 +85,20 @@ again, once. It tries at most once every 30 minutes, never sends the password
twice in one try, and stops at any step it does not recognise, such as a
captcha, saying a person has to sign in. A sign-in takes about 6 s.
+### YouTube transcripts
+
+`POST /youtube-transcript` `{"url": "https://youtu.be/...", "language": "en"}`
+files a YouTube video's transcript as plain text in `CCS_TRANSCRIPT_DIR`, named
+`<upload date>-<Title-With-Dashes>.txt`, and returns it with the title,
+channel, upload date and length (`src/youtube.ts`). Captions only, never the
+video: human-written when the video has them, YouTube's automatic ones
+otherwise, with the rolling repeats of automatic captions taken out. A video
+filed before is read back from the folder. yt-dlp (`CCS_YTDLP`) updates
+itself at most daily, and again when a fetch fails; when YouTube rate-limits
+captions (HTTP 429), the text comes from youtube-transcript-api
+(`CCS_YT_PYTHON`) instead. `GET /youtube/openapi.json` describes it apart from
+X.com, as "YouTube". The recipe is Hermes's, from bambos.
+
## Engines
Four, with independent indexes and independent rate limits. That last part is
diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts
index 8efc4c5..076c624 100644
--- a/castle-search/src/http.ts
+++ b/castle-search/src/http.ts
@@ -22,8 +22,10 @@
*
* POST /x-search, /x-post, /x-user and /x-trends read X instead (x-search.ts),
* through the browser's signed-in X session, and GET /openapi.json describes
- * them, so that Open WebUI can offer them as tools ("X.com"). Each answers 502
- * with `{ "error" }` on failure.
+ * them, so that Open WebUI can offer them as tools ("X.com"). POST
+ * /youtube-transcript files a YouTube video's transcript and returns it
+ * (youtube.ts), described at GET /youtube/openapi.json ("YouTube"). Each
+ * answers 502 with `{ "error" }` on failure.
*/
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
@@ -47,6 +49,7 @@ import {
type XUserOutcome,
type XUserRequest,
} from "./x-search.ts";
+import { DEFAULT_MAX_CHARS, type TranscriptOutcome, type TranscriptRequest, videoIdOf, youtubeTranscript } from "./youtube.ts";
export const DEFAULT_LISTEN = "127.0.0.1:11158";
const MAX_BODY_BYTES = 64 * 1024;
@@ -171,6 +174,25 @@ export const handleXTrends = async (raw: string, run: XTrendsReader, signal: Abo
return answer(() => run(tab as XTrendTab, signal));
};
+export type TranscriptFetcher = (request: TranscriptRequest, signal: AbortSignal) => Promise<TranscriptOutcome>;
+
+/** POST /youtube-transcript: `{ url, language?, max_chars? }` in, the transcript and where it was filed out. */
+export const handleTranscript = async (raw: string, run: TranscriptFetcher, signal: AbortSignal): Promise<Reply> => {
+ const body = jsonBody(raw);
+ if (isReply(body)) return body;
+ if (typeof body["url"] !== "string" || !videoIdOf(body["url"])) {
+ return { status: 400, body: { error: "url must be a YouTube video link (youtube.com/watch?v=..., youtu.be/..., shorts/...)" } };
+ }
+ const language = body["language"];
+ if (language !== undefined && (typeof language !== "string" || !/^[A-Za-z]{2,3}(-[A-Za-z0-9]+)?$/.test(language))) {
+ return { status: 400, body: { error: "language must be a language code such as en, pt or en-GB" } };
+ }
+ const max = body["max_chars"];
+ const maxChars = typeof max === "number" && Number.isFinite(max) ? Math.max(1000, Math.trunc(max)) : DEFAULT_MAX_CHARS;
+ const url = body["url"];
+ return answer(() => run({ url, language: language as string | undefined, maxChars }, signal));
+};
+
const jsonSchema = (properties: Record<string, unknown>, required: string[]) => ({
required: true,
content: { "application/json": { schema: { type: "object", required, properties } } },
@@ -267,6 +289,43 @@ export const OPENAPI = {
},
} as const;
+/** The YouTube tool's own description, so Open WebUI lists it apart from X.com. */
+export const YOUTUBE_OPENAPI = {
+ openapi: "3.1.0",
+ info: {
+ title: "YouTube",
+ version: "1.0.0",
+ description: "A YouTube video's transcript, filed in Igor's Nextcloud transcript folder and returned.",
+ },
+ paths: {
+ "/youtube-transcript": {
+ post: {
+ operationId: "youtube_transcript",
+ summary: "Get a YouTube video's transcript",
+ description:
+ "Fetch a YouTube video's transcript from its link: the video's captions (human-written when it " +
+ "has them, YouTube's automatic ones otherwise) as clean prose, with its title, channel, upload " +
+ "date and length. It is also filed in Igor's Nextcloud, in the transcript folder, as " +
+ "<upload date>-<Title>.txt; a video filed before is read from there. Use it whenever a YouTube " +
+ "link comes up and its content matters: to summarise it, answer questions about it, or quote " +
+ "it. Automatic captions garble names and numbers: say so when one matters.",
+ requestBody: jsonSchema(
+ {
+ url: { type: "string", description: "The video's link: youtube.com/watch?v=..., youtu.be/..., shorts/..." },
+ language: { type: "string", default: "en", description: "Caption language code, such as en or pt" },
+ max_chars: { type: "integer", minimum: 1000, default: DEFAULT_MAX_CHARS, description: "Most characters of text to return; the file always has all of it" },
+ },
+ ["url"],
+ ),
+ responses: {
+ "200": { description: "The transcript and where it was filed", content: { "application/json": { schema: { type: "object" } } } },
+ "502": { description: "No transcript; error says why", content: { "application/json": { schema: { type: "object" } } } },
+ },
+ },
+ },
+ },
+} as const;
+
const readBody = (req: IncomingMessage): Promise<string> =>
new Promise((resolve, reject) => {
let size = 0;
@@ -313,15 +372,20 @@ const main = (): void => {
send(res, { status: 200, body: OPENAPI });
return;
}
+ if (req.method === "GET" && path === "/youtube/openapi.json") {
+ send(res, { status: 200, body: YOUTUBE_OPENAPI });
+ return;
+ }
const xRoutes: Record<string, (raw: string, signal: AbortSignal) => Promise<Reply>> = {
"/x-search": (raw, signal) => handleXSearch(raw, xSearch, signal),
"/x-post": (raw, signal) => handleXPost(raw, xPost, signal),
"/x-user": (raw, signal) => handleXUser(raw, xUser, signal),
"/x-trends": (raw, signal) => handleXTrends(raw, xTrends, signal),
+ "/youtube-transcript": (raw, signal) => handleTranscript(raw, youtubeTranscript, signal),
};
const xRoute = xRoutes[path ?? ""];
if (req.method !== "POST" || (path !== "/" && path !== "/search" && !xRoute)) {
- send(res, { status: 404, body: { error: "POST a search to / or /search, or an X read to /x-search, /x-post, /x-user or /x-trends" } });
+ send(res, { status: 404, body: { error: "POST a search to / or /search, an X read to /x-search, /x-post, /x-user or /x-trends, or a YouTube link to /youtube-transcript" } });
return;
}
// Stop the search, and free its tab, if the client goes away first.
diff --git a/castle-search/src/youtube.ts b/castle-search/src/youtube.ts
new file mode 100644
index 0000000..18b1d49
--- /dev/null
+++ b/castle-search/src/youtube.ts
@@ -0,0 +1,303 @@
+/**
+ * A YouTube video's transcript, filed as plain text in a folder (castle's
+ * synced Nextcloud "transcript" folder) and handed back to the caller.
+ *
+ * The recipe is the one Hermes worked out on bambos (its skill
+ * youtube-transcript-to-nextcloud), so files from either read the same:
+ *
+ * 1. the bare video link: share-link junk (?si=, ?t=, &list=) dropped;
+ * 2. title, channel, upload date and length from yt-dlp;
+ * 3. subtitles only, never the video: human-written ones when the video has
+ * them, YouTube's automatic ones otherwise, as VTT;
+ * 4. YouTube's rolling captions repeat the end of the previous cue, so each
+ * cue keeps only the words the text so far does not already end with;
+ * 5. filed as <upload date>-<Title-With-Dashes>.txt, paragraphs of 90 words.
+ *
+ * yt-dlp breaks whenever YouTube changes, so it updates itself (`yt-dlp -U`)
+ * at most once a day, and once more when a fetch fails. YouTube rate-limits
+ * its caption downloads (HTTP 429); then the transcript comes from
+ * youtube-transcript-api instead, which uses another endpoint.
+ *
+ * Configured by environment: CCS_YTDLP (the yt-dlp binary, one it can update),
+ * CCS_YT_PYTHON (a python with youtube_transcript_api, optional),
+ * CCS_TRANSCRIPT_DIR (where files go). One transcript at a time.
+ */
+
+import { execFile } from "node:child_process";
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { BrowserUnavailableError, NoResultsError } from "./errors.ts";
+
+export type TranscriptRequest = { url: string; language?: string | undefined; maxChars?: number | undefined };
+
+export type TranscriptOutcome = {
+ file: string;
+ /** Already in the folder from an earlier run, so not fetched again. */
+ existing: boolean;
+ url: string;
+ title: string;
+ channel: string;
+ /** YYYY-MM-DD */
+ uploaded: string;
+ /** Seconds. */
+ duration?: number;
+ /** "manual", "automatic", or "youtube-transcript-api". */
+ source: string;
+ words: number;
+ text: string;
+ /** The text was longer than maxChars; the file has all of it. */
+ truncated?: true;
+};
+
+const FETCH_TIMEOUT_MS = 120_000;
+const UPDATE_EVERY_MS = 24 * 3_600_000;
+export const DEFAULT_MAX_CHARS = 60_000;
+
+// --- Pure parts, tested --------------------------------------------------
+
+/** The video id in any YouTube link: watch?v=, youtu.be/, shorts/, live/, embed/. */
+export const videoIdOf = (url: string): string | undefined => {
+ const s = url.trim();
+ if (/^[A-Za-z0-9_-]{11}$/.test(s)) return s;
+ const m =
+ /(?:youtube\.com\/(?:watch\?(?:.*&)?v=|shorts\/|live\/|embed\/|v\/)|youtu\.be\/|youtube-nocookie\.com\/embed\/)([A-Za-z0-9_-]{11})/i.exec(s);
+ return m?.[1];
+};
+
+/** "Dr. Becky: Parenting 101 | Call Her Daddy" -> "Dr-Becky-Parenting-101-Call-Her-Daddy". */
+export const titleSlug = (title: string): string => {
+ const words = title
+ .normalize("NFKD")
+ .replace(/[\u0300-\u036f]/g, "")
+ .replace(/['’]/g, "")
+ .match(/[A-Za-z0-9]+/g);
+ return (words ?? []).join("-").slice(0, 100).replace(/-+$/, "") || "transcript";
+};
+
+/** "20051024" -> "2005-10-24". */
+export const isoDate = (yyyymmdd: string): string =>
+ /^\d{8}$/.test(yyyymmdd) ? `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}` : new Date().toISOString().slice(0, 10);
+
+const SKIP = ["WEBVTT", "NOTE", "Kind:", "Language:", "STYLE", "REGION", "X-TIMESTAMP"];
+
+/** Each cue's text, timings and markup removed. */
+export const cueTexts = (vtt: string): string[] => {
+ const out: string[] = [];
+ for (const block of vtt.replace(/\r/g, "").split(/\n\s*\n/)) {
+ const lines = block
+ .split("\n")
+ .map((l) => l.trim())
+ .filter((l) => l && !l.includes("-->") && !SKIP.some((p) => l.startsWith(p)) && !/^\d+$/.test(l));
+ if (!lines.length) continue;
+ const text = lines
+ .join(" ")
+ .replace(/<[^>]*>/g, "")
+ .replace(/\[[^\]]*\]/g, " ") // [Music], [Applause]
+ .replace(/&gt;/g, " ")
+ .replace(/&lt;/g, "<")
+ .replace(/&amp;/g, "&")
+ .replace(/&#39;/g, "'")
+ .replace(/&quot;/g, '"')
+ .replace(/&nbsp;/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+ if (text) out.push(text);
+ }
+ return out;
+};
+
+/**
+ * Rolling captions: each cue starts with the end of what came before. Keep
+ * only the words past the longest overlap with the text so far.
+ */
+export const dedupeRolling = (cues: string[], window = 80): string[] => {
+ const emitted: string[] = [];
+ for (const cue of cues) {
+ const words = cue.split(" ").filter(Boolean);
+ const tail = emitted.slice(-window);
+ let overlap = 0;
+ for (let k = Math.min(tail.length, words.length); k > 0; k--) {
+ if (tail.slice(-k).join(" ") === words.slice(0, k).join(" ")) {
+ overlap = k;
+ break;
+ }
+ }
+ emitted.push(...words.slice(overlap));
+ }
+ return emitted;
+};
+
+export const paragraphs = (words: string[], per = 90): string => {
+ const out: string[] = [];
+ for (let i = 0; i < words.length; i += per) out.push(words.slice(i, i + per).join(" "));
+ return out.join("\n\n");
+};
+
+// --- Running yt-dlp ------------------------------------------------------
+
+const run = (cmd: string, args: string[], signal: AbortSignal, timeout = FETCH_TIMEOUT_MS): Promise<{ code: number; stdout: string; stderr: string }> =>
+ new Promise((resolve) => {
+ execFile(cmd, args, { signal, timeout, maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => {
+ const code = error ? (typeof (error as { code?: unknown }).code === "number" ? (error as { code: number }).code : 1) : 0;
+ resolve({ code, stdout: String(stdout), stderr: String(stderr) });
+ });
+ });
+
+const ytdlp = (): string => process.env["CCS_YTDLP"] ?? "yt-dlp";
+
+let lastUpdate = 0;
+const update = async (signal: AbortSignal, force = false): Promise<void> => {
+ if (!force && Date.now() - lastUpdate < UPDATE_EVERY_MS) return;
+ lastUpdate = Date.now();
+ const r = await run(ytdlp(), ["-U"], signal, 60_000);
+ console.log(`yt-dlp -U: ${(r.stdout + r.stderr).trim().split("\n").pop() ?? ""}`);
+};
+
+type Meta = { title: string; channel: string; uploaded: string; duration?: number; id: string };
+
+const metadata = async (url: string, signal: AbortSignal): Promise<Meta | { error: string }> => {
+ const r = await run(ytdlp(), ["--no-warnings", "--skip-download", "--no-playlist", "--print", "%(title)s\t%(uploader)s\t%(upload_date)s\t%(duration)s\t%(id)s", url], signal);
+ const line = r.stdout.trim().split("\n").pop() ?? "";
+ const [title, channel, date, duration, id] = line.split("\t");
+ if (r.code !== 0 || !title || !id) return { error: (r.stderr.trim().split("\n").pop() ?? `yt-dlp exited ${r.code}`).slice(0, 300) };
+ const meta: Meta = { title, channel: channel && channel !== "NA" ? channel : "", uploaded: isoDate(date ?? ""), id };
+ const d = Number(duration);
+ if (Number.isFinite(d) && d > 0) meta.duration = d;
+ return meta;
+};
+
+/** The subtitle file yt-dlp wrote: human-written beats automatic (".en-orig" is automatic). */
+const pickSubs = (dir: string, lang: string): { path: string; source: string } | undefined => {
+ const files = readdirSync(dir).filter((f) => f.endsWith(".vtt") && statSync(join(dir, f)).size > 0);
+ const auto = files.find((f) => f.includes(`.${lang}-orig.`));
+ const manual = files.find((f) => f !== auto);
+ if (manual) {
+ const path = join(dir, manual);
+ const text = readFileSync(path, "utf8");
+ // Beside an -orig track, the plain one is human-written unless it is the
+ // same file. Alone, its word-timing tags say whether it is automatic.
+ const automatic = auto !== undefined ? text === readFileSync(join(dir, auto), "utf8") : /<\d\d:\d\d:\d\d\.\d{3}><c>/.test(text);
+ return { path, source: automatic ? "automatic" : "manual" };
+ }
+ return auto ? { path: join(dir, auto), source: "automatic" } : undefined;
+};
+
+const subtitles = async (id: string, lang: string, dir: string, signal: AbortSignal): Promise<{ words: string[]; source: string } | { error: string; rateLimited: boolean }> => {
+ // Only the language itself and its regional variants: a pattern like
+ // "en-.*" also takes YouTube's machine translations ("en-de"), which are
+ // what its rate limit falls on first.
+ const langs = [lang, `${lang}-orig`, ...(lang === "en" ? ["en-GB", "en-US"] : [])].join(",");
+ const r = await run(
+ ytdlp(),
+ ["--no-warnings", "--skip-download", "--no-playlist", "--write-subs", "--write-auto-subs", "--sub-langs", langs, "--sub-format", "vtt", "-o", join(dir, "%(id)s"), `https://www.youtube.com/watch?v=${id}`],
+ signal,
+ );
+ // yt-dlp fails the whole run when one of several tracks fails; a track it
+ // did write is still good.
+ const picked = pickSubs(dir, lang);
+ if (picked) return { words: dedupeRolling(cueTexts(readFileSync(picked.path, "utf8"))), source: picked.source };
+ const err = r.stderr.trim().split("\n").filter((l) => l.startsWith("ERROR")).pop() ?? "";
+ return { error: err || `no ${lang} subtitles`, rateLimited: /429|Too Many Requests/.test(err) };
+};
+
+const TRANSCRIPT_API = `
+import json, sys
+from youtube_transcript_api import YouTubeTranscriptApi
+vid, lang = sys.argv[1], sys.argv[2]
+langs = [lang] + ([f"{lang}-US", f"{lang}-GB"] if lang == "en" else [])
+f = YouTubeTranscriptApi().fetch(vid, languages=langs)
+print(json.dumps([s.text for s in f]))
+`;
+
+const transcriptApi = async (id: string, lang: string, signal: AbortSignal): Promise<string[] | { error: string }> => {
+ const py = process.env["CCS_YT_PYTHON"];
+ if (!py) return { error: "no youtube-transcript-api configured (CCS_YT_PYTHON)" };
+ const r = await run(py, ["-I", "-c", TRANSCRIPT_API, id, lang], signal);
+ if (r.code !== 0) return { error: (r.stderr.trim().split("\n").pop() ?? "").slice(0, 300) };
+ // Discrete snippets, not rolling cues: no dedupe needed.
+ return (JSON.parse(r.stdout) as string[]).join(" ").replace(/\[[^\]]*\]/g, " ").split(/\s+/).filter(Boolean);
+};
+
+// --- The tool ------------------------------------------------------------
+
+let tail: Promise<unknown> = Promise.resolve();
+
+/** One at a time: yt-dlp updating itself under a running fetch breaks both. */
+export const youtubeTranscript = (request: TranscriptRequest, signal: AbortSignal): Promise<TranscriptOutcome> => {
+ const next = tail.then(() => fetchTranscript(request, signal));
+ tail = next.catch(() => {});
+ return next;
+};
+
+const fetchTranscript = async (request: TranscriptRequest, signal: AbortSignal): Promise<TranscriptOutcome> => {
+ const id = videoIdOf(request.url);
+ if (!id) throw new NoResultsError(`not a YouTube video link: ${request.url}`);
+ const lang = (request.language ?? "en").trim() || "en";
+ const maxChars = request.maxChars ?? DEFAULT_MAX_CHARS;
+ const url = `https://www.youtube.com/watch?v=${id}`;
+ const folder = process.env["CCS_TRANSCRIPT_DIR"];
+ if (!folder || !existsSync(folder)) {
+ throw new BrowserUnavailableError(`the transcript folder (CCS_TRANSCRIPT_DIR=${folder ?? "unset"}) does not exist`);
+ }
+
+ await update(signal);
+ let meta = await metadata(url, signal);
+ if ("error" in meta) {
+ // An extractor broken by a YouTube change: a newer yt-dlp is nearly always the fix.
+ await update(signal, true);
+ meta = await metadata(url, signal);
+ if ("error" in meta) throw new BrowserUnavailableError(`yt-dlp could not read ${url}: ${meta.error}`);
+ }
+
+ const name = `${meta.uploaded}-${titleSlug(meta.title)}.txt`;
+ const file = join(folder, name);
+ const shape = (text: string, source: string, existing: boolean): TranscriptOutcome => {
+ const out: TranscriptOutcome = {
+ file: `transcript/${name}`,
+ existing,
+ url,
+ title: meta.title,
+ channel: meta.channel,
+ uploaded: meta.uploaded,
+ source,
+ words: text.split(/\s+/).filter(Boolean).length,
+ text: text.length > maxChars ? text.slice(0, maxChars) : text,
+ };
+ if ("duration" in meta && meta.duration !== undefined) out.duration = meta.duration;
+ if (text.length > maxChars) out.truncated = true;
+ return out;
+ };
+ if (existsSync(file)) return shape(readFileSync(file, "utf8").trim(), "earlier run", true);
+
+ const scratch = mkdtempSync(join(tmpdir(), "yt-"));
+ try {
+ const subs = await subtitles(id, lang, scratch, signal);
+ let words: string[];
+ let source: string;
+ if ("words" in subs) {
+ ({ words, source } = subs);
+ } else {
+ // A 429 is YouTube limiting the caption endpoint: retrying yt-dlp gets
+ // another. The transcript API is a different endpoint.
+ const alt = await transcriptApi(id, lang, signal);
+ if (!Array.isArray(alt)) {
+ throw new NoResultsError(
+ `no ${lang} transcript for ${url}: yt-dlp said "${subs.error}"` +
+ `${subs.rateLimited ? " (YouTube rate-limiting captions)" : ""}; youtube-transcript-api said "${alt.error}". ` +
+ `The video may have no captions in that language: try another language code.`,
+ );
+ }
+ words = alt;
+ source = "youtube-transcript-api";
+ }
+ if (words.length === 0) throw new NoResultsError(`the ${lang} captions of ${url} are empty`);
+ const text = paragraphs(words);
+ writeFileSync(file, text + "\n", "utf8");
+ console.log(`youtube ${id} -> ${name} (${words.length} words, ${source})`);
+ return shape(text, source, false);
+ } finally {
+ rmSync(scratch, { recursive: true, force: true });
+ }
+};
diff --git a/castle-search/test/youtube.test.ts b/castle-search/test/youtube.test.ts
new file mode 100644
index 0000000..c5530fb
--- /dev/null
+++ b/castle-search/test/youtube.test.ts
@@ -0,0 +1,72 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { handleTranscript } from "../src/http.ts";
+import { cueTexts, dedupeRolling, isoDate, paragraphs, titleSlug, videoIdOf } from "../src/youtube.ts";
+
+test("every shape of YouTube link gives the video id, share junk ignored", () => {
+ for (const url of [
+ "https://www.youtube.com/watch?v=jNQXAC9IVRw",
+ "https://youtube.com/watch?feature=share&v=jNQXAC9IVRw&t=30s",
+ "https://youtu.be/jNQXAC9IVRw?si=T-d5TUXNFjmR1Uad",
+ "https://www.youtube.com/shorts/jNQXAC9IVRw",
+ "https://www.youtube.com/live/jNQXAC9IVRw?feature=shared",
+ "https://m.youtube.com/watch?v=jNQXAC9IVRw&list=PL123",
+ "jNQXAC9IVRw",
+ ]) {
+ assert.equal(videoIdOf(url), "jNQXAC9IVRw", url);
+ }
+ assert.equal(videoIdOf("https://vimeo.com/123"), undefined);
+});
+
+test("titles become the archive's dashed names", () => {
+ assert.equal(titleSlug("How to stay calm when you know you'll be stressed | Daniel Levitin | TED"), "How-to-stay-calm-when-you-know-youll-be-stressed-Daniel-Levitin-TED");
+ assert.equal(titleSlug("Anthropic Files for IPO: Nasdaq!"), "Anthropic-Files-for-IPO-Nasdaq");
+ assert.equal(titleSlug("Café Müller"), "Cafe-Muller");
+ assert.equal(titleSlug("!!!"), "transcript");
+ assert.equal(isoDate("20051024"), "2005-10-24");
+});
+
+// The shape of YouTube's automatic captions: each cue repeats the last one's words.
+const ROLLING = `WEBVTT
+Kind: captions
+Language: en
+
+00:00:00.000 --> 00:00:02.000 align:start position:0%
+the filing is out<00:00:01.000><c> ENT</c><00:00:01.500><c> is</c>
+
+00:00:02.000 --> 00:00:02.010 align:start position:0%
+the filing is out ENT is
+
+00:00:02.010 --> 00:00:04.000 align:start position:0%
+the filing is out ENT is
+going to be [Music] big &amp; bold
+
+00:00:04.000 --> 00:00:06.000
+going to be big & bold for everyone
+`;
+
+test("rolling captions come out once, markup and noise gone", () => {
+ const cues = cueTexts(ROLLING);
+ assert.equal(cues[0], "the filing is out ENT is");
+ assert.equal(dedupeRolling(cues).join(" "), "the filing is out ENT is going to be big & bold for everyone");
+});
+
+test("words are set in paragraphs of 90", () => {
+ const words = Array.from({ length: 200 }, (_, i) => `w${i}`);
+ const text = paragraphs(words);
+ assert.equal(text.split("\n\n").length, 3);
+ assert.equal(text.split("\n\n")[0]!.split(" ").length, 90);
+});
+
+test("the transcript route checks the link and language first", async () => {
+ const signal = new AbortController().signal;
+ const never = async (): Promise<never> => {
+ throw new Error("must not run");
+ };
+ assert.equal((await handleTranscript(JSON.stringify({ url: "https://vimeo.com/1" }), never, signal)).status, 400);
+ assert.equal((await handleTranscript(JSON.stringify({ url: "https://youtu.be/jNQXAC9IVRw", language: "english!" }), never, signal)).status, 400);
+ const seen: unknown[] = [];
+ await handleTranscript(JSON.stringify({ url: "https://youtu.be/jNQXAC9IVRw", language: "pt", max_chars: 10 }), async (r) => (seen.push(r), {}) as never, signal);
+ assert.deepEqual(seen, [{ url: "https://youtu.be/jNQXAC9IVRw", language: "pt", maxChars: 1000 }]);
+});