diff options
Diffstat (limited to 'castle-search/src')
| -rw-r--r-- | castle-search/src/http.ts | 70 | ||||
| -rw-r--r-- | castle-search/src/youtube.ts | 303 |
2 files changed, 370 insertions, 3 deletions
diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts index 8efc4c5..076c624 100644 --- a/castle-search/src/http.ts +++ b/castle-search/src/http.ts @@ -22,8 +22,10 @@ * * POST /x-search, /x-post, /x-user and /x-trends read X instead (x-search.ts), * through the browser's signed-in X session, and GET /openapi.json describes - * them, so that Open WebUI can offer them as tools ("X.com"). Each answers 502 - * with `{ "error" }` on failure. + * them, so that Open WebUI can offer them as tools ("X.com"). POST + * /youtube-transcript files a YouTube video's transcript and returns it + * (youtube.ts), described at GET /youtube/openapi.json ("YouTube"). Each + * answers 502 with `{ "error" }` on failure. */ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; @@ -47,6 +49,7 @@ import { type XUserOutcome, type XUserRequest, } from "./x-search.ts"; +import { DEFAULT_MAX_CHARS, type TranscriptOutcome, type TranscriptRequest, videoIdOf, youtubeTranscript } from "./youtube.ts"; export const DEFAULT_LISTEN = "127.0.0.1:11158"; const MAX_BODY_BYTES = 64 * 1024; @@ -171,6 +174,25 @@ export const handleXTrends = async (raw: string, run: XTrendsReader, signal: Abo return answer(() => run(tab as XTrendTab, signal)); }; +export type TranscriptFetcher = (request: TranscriptRequest, signal: AbortSignal) => Promise<TranscriptOutcome>; + +/** POST /youtube-transcript: `{ url, language?, max_chars? }` in, the transcript and where it was filed out. */ +export const handleTranscript = async (raw: string, run: TranscriptFetcher, signal: AbortSignal): Promise<Reply> => { + const body = jsonBody(raw); + if (isReply(body)) return body; + if (typeof body["url"] !== "string" || !videoIdOf(body["url"])) { + return { status: 400, body: { error: "url must be a YouTube video link (youtube.com/watch?v=..., youtu.be/..., shorts/...)" } }; + } + const language = body["language"]; + if (language !== undefined && (typeof language !== "string" || !/^[A-Za-z]{2,3}(-[A-Za-z0-9]+)?$/.test(language))) { + return { status: 400, body: { error: "language must be a language code such as en, pt or en-GB" } }; + } + const max = body["max_chars"]; + const maxChars = typeof max === "number" && Number.isFinite(max) ? Math.max(1000, Math.trunc(max)) : DEFAULT_MAX_CHARS; + const url = body["url"]; + return answer(() => run({ url, language: language as string | undefined, maxChars }, signal)); +}; + const jsonSchema = (properties: Record<string, unknown>, required: string[]) => ({ required: true, content: { "application/json": { schema: { type: "object", required, properties } } }, @@ -267,6 +289,43 @@ export const OPENAPI = { }, } as const; +/** The YouTube tool's own description, so Open WebUI lists it apart from X.com. */ +export const YOUTUBE_OPENAPI = { + openapi: "3.1.0", + info: { + title: "YouTube", + version: "1.0.0", + description: "A YouTube video's transcript, filed in Igor's Nextcloud transcript folder and returned.", + }, + paths: { + "/youtube-transcript": { + post: { + operationId: "youtube_transcript", + summary: "Get a YouTube video's transcript", + description: + "Fetch a YouTube video's transcript from its link: the video's captions (human-written when it " + + "has them, YouTube's automatic ones otherwise) as clean prose, with its title, channel, upload " + + "date and length. It is also filed in Igor's Nextcloud, in the transcript folder, as " + + "<upload date>-<Title>.txt; a video filed before is read from there. Use it whenever a YouTube " + + "link comes up and its content matters: to summarise it, answer questions about it, or quote " + + "it. Automatic captions garble names and numbers: say so when one matters.", + requestBody: jsonSchema( + { + url: { type: "string", description: "The video's link: youtube.com/watch?v=..., youtu.be/..., shorts/..." }, + language: { type: "string", default: "en", description: "Caption language code, such as en or pt" }, + max_chars: { type: "integer", minimum: 1000, default: DEFAULT_MAX_CHARS, description: "Most characters of text to return; the file always has all of it" }, + }, + ["url"], + ), + responses: { + "200": { description: "The transcript and where it was filed", content: { "application/json": { schema: { type: "object" } } } }, + "502": { description: "No transcript; error says why", content: { "application/json": { schema: { type: "object" } } } }, + }, + }, + }, + }, +} as const; + const readBody = (req: IncomingMessage): Promise<string> => new Promise((resolve, reject) => { let size = 0; @@ -313,15 +372,20 @@ const main = (): void => { send(res, { status: 200, body: OPENAPI }); return; } + if (req.method === "GET" && path === "/youtube/openapi.json") { + send(res, { status: 200, body: YOUTUBE_OPENAPI }); + return; + } const xRoutes: Record<string, (raw: string, signal: AbortSignal) => Promise<Reply>> = { "/x-search": (raw, signal) => handleXSearch(raw, xSearch, signal), "/x-post": (raw, signal) => handleXPost(raw, xPost, signal), "/x-user": (raw, signal) => handleXUser(raw, xUser, signal), "/x-trends": (raw, signal) => handleXTrends(raw, xTrends, signal), + "/youtube-transcript": (raw, signal) => handleTranscript(raw, youtubeTranscript, signal), }; const xRoute = xRoutes[path ?? ""]; if (req.method !== "POST" || (path !== "/" && path !== "/search" && !xRoute)) { - send(res, { status: 404, body: { error: "POST a search to / or /search, or an X read to /x-search, /x-post, /x-user or /x-trends" } }); + send(res, { status: 404, body: { error: "POST a search to / or /search, an X read to /x-search, /x-post, /x-user or /x-trends, or a YouTube link to /youtube-transcript" } }); return; } // Stop the search, and free its tab, if the client goes away first. diff --git a/castle-search/src/youtube.ts b/castle-search/src/youtube.ts new file mode 100644 index 0000000..18b1d49 --- /dev/null +++ b/castle-search/src/youtube.ts @@ -0,0 +1,303 @@ +/** + * A YouTube video's transcript, filed as plain text in a folder (castle's + * synced Nextcloud "transcript" folder) and handed back to the caller. + * + * The recipe is the one Hermes worked out on bambos (its skill + * youtube-transcript-to-nextcloud), so files from either read the same: + * + * 1. the bare video link: share-link junk (?si=, ?t=, &list=) dropped; + * 2. title, channel, upload date and length from yt-dlp; + * 3. subtitles only, never the video: human-written ones when the video has + * them, YouTube's automatic ones otherwise, as VTT; + * 4. YouTube's rolling captions repeat the end of the previous cue, so each + * cue keeps only the words the text so far does not already end with; + * 5. filed as <upload date>-<Title-With-Dashes>.txt, paragraphs of 90 words. + * + * yt-dlp breaks whenever YouTube changes, so it updates itself (`yt-dlp -U`) + * at most once a day, and once more when a fetch fails. YouTube rate-limits + * its caption downloads (HTTP 429); then the transcript comes from + * youtube-transcript-api instead, which uses another endpoint. + * + * Configured by environment: CCS_YTDLP (the yt-dlp binary, one it can update), + * CCS_YT_PYTHON (a python with youtube_transcript_api, optional), + * CCS_TRANSCRIPT_DIR (where files go). One transcript at a time. + */ + +import { execFile } from "node:child_process"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { BrowserUnavailableError, NoResultsError } from "./errors.ts"; + +export type TranscriptRequest = { url: string; language?: string | undefined; maxChars?: number | undefined }; + +export type TranscriptOutcome = { + file: string; + /** Already in the folder from an earlier run, so not fetched again. */ + existing: boolean; + url: string; + title: string; + channel: string; + /** YYYY-MM-DD */ + uploaded: string; + /** Seconds. */ + duration?: number; + /** "manual", "automatic", or "youtube-transcript-api". */ + source: string; + words: number; + text: string; + /** The text was longer than maxChars; the file has all of it. */ + truncated?: true; +}; + +const FETCH_TIMEOUT_MS = 120_000; +const UPDATE_EVERY_MS = 24 * 3_600_000; +export const DEFAULT_MAX_CHARS = 60_000; + +// --- Pure parts, tested -------------------------------------------------- + +/** The video id in any YouTube link: watch?v=, youtu.be/, shorts/, live/, embed/. */ +export const videoIdOf = (url: string): string | undefined => { + const s = url.trim(); + if (/^[A-Za-z0-9_-]{11}$/.test(s)) return s; + const m = + /(?:youtube\.com\/(?:watch\?(?:.*&)?v=|shorts\/|live\/|embed\/|v\/)|youtu\.be\/|youtube-nocookie\.com\/embed\/)([A-Za-z0-9_-]{11})/i.exec(s); + return m?.[1]; +}; + +/** "Dr. Becky: Parenting 101 | Call Her Daddy" -> "Dr-Becky-Parenting-101-Call-Her-Daddy". */ +export const titleSlug = (title: string): string => { + const words = title + .normalize("NFKD") + .replace(/[\u0300-\u036f]/g, "") + .replace(/['’]/g, "") + .match(/[A-Za-z0-9]+/g); + return (words ?? []).join("-").slice(0, 100).replace(/-+$/, "") || "transcript"; +}; + +/** "20051024" -> "2005-10-24". */ +export const isoDate = (yyyymmdd: string): string => + /^\d{8}$/.test(yyyymmdd) ? `${yyyymmdd.slice(0, 4)}-${yyyymmdd.slice(4, 6)}-${yyyymmdd.slice(6, 8)}` : new Date().toISOString().slice(0, 10); + +const SKIP = ["WEBVTT", "NOTE", "Kind:", "Language:", "STYLE", "REGION", "X-TIMESTAMP"]; + +/** Each cue's text, timings and markup removed. */ +export const cueTexts = (vtt: string): string[] => { + const out: string[] = []; + for (const block of vtt.replace(/\r/g, "").split(/\n\s*\n/)) { + const lines = block + .split("\n") + .map((l) => l.trim()) + .filter((l) => l && !l.includes("-->") && !SKIP.some((p) => l.startsWith(p)) && !/^\d+$/.test(l)); + if (!lines.length) continue; + const text = lines + .join(" ") + .replace(/<[^>]*>/g, "") + .replace(/\[[^\]]*\]/g, " ") // [Music], [Applause] + .replace(/>/g, " ") + .replace(/</g, "<") + .replace(/&/g, "&") + .replace(/'/g, "'") + .replace(/"/g, '"') + .replace(/ /g, " ") + .replace(/\s+/g, " ") + .trim(); + if (text) out.push(text); + } + return out; +}; + +/** + * Rolling captions: each cue starts with the end of what came before. Keep + * only the words past the longest overlap with the text so far. + */ +export const dedupeRolling = (cues: string[], window = 80): string[] => { + const emitted: string[] = []; + for (const cue of cues) { + const words = cue.split(" ").filter(Boolean); + const tail = emitted.slice(-window); + let overlap = 0; + for (let k = Math.min(tail.length, words.length); k > 0; k--) { + if (tail.slice(-k).join(" ") === words.slice(0, k).join(" ")) { + overlap = k; + break; + } + } + emitted.push(...words.slice(overlap)); + } + return emitted; +}; + +export const paragraphs = (words: string[], per = 90): string => { + const out: string[] = []; + for (let i = 0; i < words.length; i += per) out.push(words.slice(i, i + per).join(" ")); + return out.join("\n\n"); +}; + +// --- Running yt-dlp ------------------------------------------------------ + +const run = (cmd: string, args: string[], signal: AbortSignal, timeout = FETCH_TIMEOUT_MS): Promise<{ code: number; stdout: string; stderr: string }> => + new Promise((resolve) => { + execFile(cmd, args, { signal, timeout, maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => { + const code = error ? (typeof (error as { code?: unknown }).code === "number" ? (error as { code: number }).code : 1) : 0; + resolve({ code, stdout: String(stdout), stderr: String(stderr) }); + }); + }); + +const ytdlp = (): string => process.env["CCS_YTDLP"] ?? "yt-dlp"; + +let lastUpdate = 0; +const update = async (signal: AbortSignal, force = false): Promise<void> => { + if (!force && Date.now() - lastUpdate < UPDATE_EVERY_MS) return; + lastUpdate = Date.now(); + const r = await run(ytdlp(), ["-U"], signal, 60_000); + console.log(`yt-dlp -U: ${(r.stdout + r.stderr).trim().split("\n").pop() ?? ""}`); +}; + +type Meta = { title: string; channel: string; uploaded: string; duration?: number; id: string }; + +const metadata = async (url: string, signal: AbortSignal): Promise<Meta | { error: string }> => { + const r = await run(ytdlp(), ["--no-warnings", "--skip-download", "--no-playlist", "--print", "%(title)s\t%(uploader)s\t%(upload_date)s\t%(duration)s\t%(id)s", url], signal); + const line = r.stdout.trim().split("\n").pop() ?? ""; + const [title, channel, date, duration, id] = line.split("\t"); + if (r.code !== 0 || !title || !id) return { error: (r.stderr.trim().split("\n").pop() ?? `yt-dlp exited ${r.code}`).slice(0, 300) }; + const meta: Meta = { title, channel: channel && channel !== "NA" ? channel : "", uploaded: isoDate(date ?? ""), id }; + const d = Number(duration); + if (Number.isFinite(d) && d > 0) meta.duration = d; + return meta; +}; + +/** The subtitle file yt-dlp wrote: human-written beats automatic (".en-orig" is automatic). */ +const pickSubs = (dir: string, lang: string): { path: string; source: string } | undefined => { + const files = readdirSync(dir).filter((f) => f.endsWith(".vtt") && statSync(join(dir, f)).size > 0); + const auto = files.find((f) => f.includes(`.${lang}-orig.`)); + const manual = files.find((f) => f !== auto); + if (manual) { + const path = join(dir, manual); + const text = readFileSync(path, "utf8"); + // Beside an -orig track, the plain one is human-written unless it is the + // same file. Alone, its word-timing tags say whether it is automatic. + const automatic = auto !== undefined ? text === readFileSync(join(dir, auto), "utf8") : /<\d\d:\d\d:\d\d\.\d{3}><c>/.test(text); + return { path, source: automatic ? "automatic" : "manual" }; + } + return auto ? { path: join(dir, auto), source: "automatic" } : undefined; +}; + +const subtitles = async (id: string, lang: string, dir: string, signal: AbortSignal): Promise<{ words: string[]; source: string } | { error: string; rateLimited: boolean }> => { + // Only the language itself and its regional variants: a pattern like + // "en-.*" also takes YouTube's machine translations ("en-de"), which are + // what its rate limit falls on first. + const langs = [lang, `${lang}-orig`, ...(lang === "en" ? ["en-GB", "en-US"] : [])].join(","); + const r = await run( + ytdlp(), + ["--no-warnings", "--skip-download", "--no-playlist", "--write-subs", "--write-auto-subs", "--sub-langs", langs, "--sub-format", "vtt", "-o", join(dir, "%(id)s"), `https://www.youtube.com/watch?v=${id}`], + signal, + ); + // yt-dlp fails the whole run when one of several tracks fails; a track it + // did write is still good. + const picked = pickSubs(dir, lang); + if (picked) return { words: dedupeRolling(cueTexts(readFileSync(picked.path, "utf8"))), source: picked.source }; + const err = r.stderr.trim().split("\n").filter((l) => l.startsWith("ERROR")).pop() ?? ""; + return { error: err || `no ${lang} subtitles`, rateLimited: /429|Too Many Requests/.test(err) }; +}; + +const TRANSCRIPT_API = ` +import json, sys +from youtube_transcript_api import YouTubeTranscriptApi +vid, lang = sys.argv[1], sys.argv[2] +langs = [lang] + ([f"{lang}-US", f"{lang}-GB"] if lang == "en" else []) +f = YouTubeTranscriptApi().fetch(vid, languages=langs) +print(json.dumps([s.text for s in f])) +`; + +const transcriptApi = async (id: string, lang: string, signal: AbortSignal): Promise<string[] | { error: string }> => { + const py = process.env["CCS_YT_PYTHON"]; + if (!py) return { error: "no youtube-transcript-api configured (CCS_YT_PYTHON)" }; + const r = await run(py, ["-I", "-c", TRANSCRIPT_API, id, lang], signal); + if (r.code !== 0) return { error: (r.stderr.trim().split("\n").pop() ?? "").slice(0, 300) }; + // Discrete snippets, not rolling cues: no dedupe needed. + return (JSON.parse(r.stdout) as string[]).join(" ").replace(/\[[^\]]*\]/g, " ").split(/\s+/).filter(Boolean); +}; + +// --- The tool ------------------------------------------------------------ + +let tail: Promise<unknown> = Promise.resolve(); + +/** One at a time: yt-dlp updating itself under a running fetch breaks both. */ +export const youtubeTranscript = (request: TranscriptRequest, signal: AbortSignal): Promise<TranscriptOutcome> => { + const next = tail.then(() => fetchTranscript(request, signal)); + tail = next.catch(() => {}); + return next; +}; + +const fetchTranscript = async (request: TranscriptRequest, signal: AbortSignal): Promise<TranscriptOutcome> => { + const id = videoIdOf(request.url); + if (!id) throw new NoResultsError(`not a YouTube video link: ${request.url}`); + const lang = (request.language ?? "en").trim() || "en"; + const maxChars = request.maxChars ?? DEFAULT_MAX_CHARS; + const url = `https://www.youtube.com/watch?v=${id}`; + const folder = process.env["CCS_TRANSCRIPT_DIR"]; + if (!folder || !existsSync(folder)) { + throw new BrowserUnavailableError(`the transcript folder (CCS_TRANSCRIPT_DIR=${folder ?? "unset"}) does not exist`); + } + + await update(signal); + let meta = await metadata(url, signal); + if ("error" in meta) { + // An extractor broken by a YouTube change: a newer yt-dlp is nearly always the fix. + await update(signal, true); + meta = await metadata(url, signal); + if ("error" in meta) throw new BrowserUnavailableError(`yt-dlp could not read ${url}: ${meta.error}`); + } + + const name = `${meta.uploaded}-${titleSlug(meta.title)}.txt`; + const file = join(folder, name); + const shape = (text: string, source: string, existing: boolean): TranscriptOutcome => { + const out: TranscriptOutcome = { + file: `transcript/${name}`, + existing, + url, + title: meta.title, + channel: meta.channel, + uploaded: meta.uploaded, + source, + words: text.split(/\s+/).filter(Boolean).length, + text: text.length > maxChars ? text.slice(0, maxChars) : text, + }; + if ("duration" in meta && meta.duration !== undefined) out.duration = meta.duration; + if (text.length > maxChars) out.truncated = true; + return out; + }; + if (existsSync(file)) return shape(readFileSync(file, "utf8").trim(), "earlier run", true); + + const scratch = mkdtempSync(join(tmpdir(), "yt-")); + try { + const subs = await subtitles(id, lang, scratch, signal); + let words: string[]; + let source: string; + if ("words" in subs) { + ({ words, source } = subs); + } else { + // A 429 is YouTube limiting the caption endpoint: retrying yt-dlp gets + // another. The transcript API is a different endpoint. + const alt = await transcriptApi(id, lang, signal); + if (!Array.isArray(alt)) { + throw new NoResultsError( + `no ${lang} transcript for ${url}: yt-dlp said "${subs.error}"` + + `${subs.rateLimited ? " (YouTube rate-limiting captions)" : ""}; youtube-transcript-api said "${alt.error}". ` + + `The video may have no captions in that language: try another language code.`, + ); + } + words = alt; + source = "youtube-transcript-api"; + } + if (words.length === 0) throw new NoResultsError(`the ${lang} captions of ${url} are empty`); + const text = paragraphs(words); + writeFileSync(file, text + "\n", "utf8"); + console.log(`youtube ${id} -> ${name} (${words.length} words, ${source})`); + return shape(text, source, false); + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +}; |
