From 8415a20c25f62aad589d62520fb0df1222c0045b Mon Sep 17 00:00:00 2001 From: Igor Soarez Date: Thu, 19 Nov 2026 21:26:39 +0000 Subject: castle-search: video clips from DeepInfra, filed in a folder --- castle-search/README.md | 14 +++ castle-search/src/http.ts | 99 ++++++++++++++++- castle-search/src/video.ts | 222 +++++++++++++++++++++++++++++++++++++++ castle-search/test/video.test.ts | 79 ++++++++++++++ 4 files changed, 411 insertions(+), 3 deletions(-) create mode 100644 castle-search/src/video.ts create mode 100644 castle-search/test/video.test.ts diff --git a/castle-search/README.md b/castle-search/README.md index 1621fe3..accdc24 100644 --- a/castle-search/README.md +++ b/castle-search/README.md @@ -99,6 +99,20 @@ captions (HTTP 429), the text comes from youtube-transcript-api (`CCS_YT_PYTHON`) instead. `GET /youtube/openapi.json` describes it apart from X.com, as "YouTube". The recipe is Hermes's, from bambos. +### Video clips + +`POST /video` `{"prompt": "...", "kind": "fast"}` has one of DeepInfra's video +models make a clip and files the `.mp4` in `CCS_VIDEO_DIR`, named by the time +and the prompt's first words (`src/video.ts`). `kind` picks the model: +`fast` (FastWan 2.2 5B, 5 s, no sound, about $0.02; the default), `quality` +(Wan 2.2 14B, 5 s, about $0.17), `long` (LTX 2.3 Fast, 6 to 20 s with sound, +can start from `image_url`, $0.03 a second) and `sound` (Veo 3.1 Fast, 8 s, +with realistic sound and speech, about $1.20). `orientation` is landscape or +portrait. Paid per clip, so at most 10 a day (counted in `CCS_VIDEO_STATE`), +one at a time; a clip whose caller hangs up is still finished and filed, +since it is billed anyway. The key is read from `CCS_DEEPINFRA_KEY_FILE`. +`GET /video/openapi.json` describes it as "Video". + ## Engines Four, with independent indexes and independent rate limits. That last part is diff --git a/castle-search/src/http.ts b/castle-search/src/http.ts index 076c624..d19b9f7 100644 --- a/castle-search/src/http.ts +++ b/castle-search/src/http.ts @@ -24,8 +24,9 @@ * through the browser's signed-in X session, and GET /openapi.json describes * them, so that Open WebUI can offer them as tools ("X.com"). POST * /youtube-transcript files a YouTube video's transcript and returns it - * (youtube.ts), described at GET /youtube/openapi.json ("YouTube"). Each - * answers 502 with `{ "error" }` on failure. + * (youtube.ts), described at GET /youtube/openapi.json ("YouTube"). POST + * /video makes a short video clip and files it (video.ts), described at GET + * /video/openapi.json ("Video"). Each answers 502 with `{ "error" }` on failure. */ import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; @@ -49,6 +50,7 @@ import { type XUserOutcome, type XUserRequest, } from "./x-search.ts"; +import { CLIPS_PER_DAY, LONG_SECONDS, makeVideo, VIDEO_KINDS, type VideoKind, type VideoOutcome, type VideoRequest } from "./video.ts"; import { DEFAULT_MAX_CHARS, type TranscriptOutcome, type TranscriptRequest, videoIdOf, youtubeTranscript } from "./youtube.ts"; export const DEFAULT_LISTEN = "127.0.0.1:11158"; @@ -193,6 +195,49 @@ export const handleTranscript = async (raw: string, run: TranscriptFetcher, sign return answer(() => run({ url, language: language as string | undefined, maxChars }, signal)); }; +export type VideoMaker = (request: VideoRequest) => Promise; + +const MAX_PROMPT_CHARS = 2000; + +/** POST /video: `{ prompt, kind?, orientation?, seconds?, image_url? }` in, where the clip was filed out. */ +export const handleVideo = async (raw: string, run: VideoMaker): Promise => { + const body = jsonBody(raw); + if (isReply(body)) return body; + const { prompt, orientation, seconds } = body; + const kind = body["kind"] ?? "fast"; + const imageUrl = body["image_url"]; + if (typeof prompt !== "string" || prompt.trim() === "" || prompt.length > MAX_PROMPT_CHARS) { + return { status: 400, body: { error: `prompt must be a description of the clip, at most ${MAX_PROMPT_CHARS} characters` } }; + } + if (!VIDEO_KINDS.includes(kind as VideoKind)) { + return { status: 400, body: { error: `kind must be one of ${VIDEO_KINDS.join(", ")}` } }; + } + if (orientation !== undefined && orientation !== null && orientation !== "landscape" && orientation !== "portrait") { + return { status: 400, body: { error: 'orientation must be "landscape" or "portrait"' } }; + } + if (seconds !== undefined && seconds !== null) { + if (kind !== "long") return { status: 400, body: { error: "seconds can only be chosen for kind long; the others have a fixed length" } }; + if (!LONG_SECONDS.includes(seconds as (typeof LONG_SECONDS)[number])) { + return { status: 400, body: { error: `seconds must be one of ${LONG_SECONDS.join(", ")}` } }; + } + } + if (imageUrl !== undefined && imageUrl !== null) { + if (kind !== "long") return { status: 400, body: { error: "image_url (animating a picture) needs kind long" } }; + if (typeof imageUrl !== "string" || !/^(https:\/\/|data:image\/)/.test(imageUrl)) { + return { status: 400, body: { error: "image_url must be an https link to the picture or a data:image URI" } }; + } + } + return answer(() => + run({ + prompt: prompt.trim(), + kind: kind as VideoKind, + portrait: orientation === "portrait", + seconds: typeof seconds === "number" ? seconds : undefined, + imageUrl: typeof imageUrl === "string" ? imageUrl : undefined, + }), + ); +}; + const jsonSchema = (properties: Record, required: string[]) => ({ required: true, content: { "application/json": { schema: { type: "object", required, properties } } }, @@ -326,6 +371,49 @@ export const YOUTUBE_OPENAPI = { }, } as const; +/** The video tool's own description, so Open WebUI lists it on its own and it can stay off by default. */ +export const VIDEO_OPENAPI = { + openapi: "3.1.0", + info: { + title: "Video", + version: "1.0.0", + description: "Makes a short video clip from a description and files it in Igor's Nextcloud video folder. Costs money per clip.", + }, + paths: { + "/video": { + post: { + operationId: "make_video", + summary: "Make a short video clip", + description: + "Make a video clip from a text description, only when asked for a video. It takes from under a " + + "minute to about three, costs money for every clip, and is limited to " + + `${CLIPS_PER_DAY} clips a day, so never make one unasked or retry one that failed. Pick kind by what ` + + "is asked: fast (5 s, no sound, about $0.02) unless the request calls for another; quality (5 s, " + + "no sound, sharper and smoother, about $0.17) when quality is asked for or a fast clip was not " + + "good enough; long (6 to 20 s with sound, $0.03 a second) for a clip longer than 5 s or to animate " + + "a picture; sound (8 s, the best quality, with realistic sound and speech, about $1.20) when the " + + "clip needs dialogue, voices, music or sound that matters. Describe the scene densely: the " + + "subject, the action, the setting, the light and the camera. The clip is filed in Igor's " + + "Nextcloud video folder: tell him the file name and give the folder link.", + requestBody: jsonSchema( + { + prompt: { type: "string", description: "What the clip shows: subject, action, setting, light, camera movement; for sound, what is heard or said" }, + kind: { type: "string", enum: [...VIDEO_KINDS], default: "fast", description: "fast, quality, long or sound: see the description" }, + orientation: { type: "string", enum: ["landscape", "portrait"], default: "landscape", description: "landscape 16:9 or portrait 9:16" }, + seconds: { type: "integer", enum: [...LONG_SECONDS], default: 6, description: "Length, for kind long only" }, + image_url: { type: "string", description: "For kind long only: a picture to start the clip from, as an https link or a data:image URI" }, + }, + ["prompt"], + ), + responses: { + "200": { description: "Where the clip was filed, and what it cost", content: { "application/json": { schema: { type: "object" } } } }, + "502": { description: "No clip; error says why", content: { "application/json": { schema: { type: "object" } } } }, + }, + }, + }, + }, +} as const; + const readBody = (req: IncomingMessage): Promise => new Promise((resolve, reject) => { let size = 0; @@ -376,16 +464,21 @@ const main = (): void => { send(res, { status: 200, body: YOUTUBE_OPENAPI }); return; } + if (req.method === "GET" && path === "/video/openapi.json") { + send(res, { status: 200, body: VIDEO_OPENAPI }); + return; + } const xRoutes: Record Promise> = { "/x-search": (raw, signal) => handleXSearch(raw, xSearch, signal), "/x-post": (raw, signal) => handleXPost(raw, xPost, signal), "/x-user": (raw, signal) => handleXUser(raw, xUser, signal), "/x-trends": (raw, signal) => handleXTrends(raw, xTrends, signal), "/youtube-transcript": (raw, signal) => handleTranscript(raw, youtubeTranscript, signal), + "/video": (raw) => handleVideo(raw, makeVideo), }; const xRoute = xRoutes[path ?? ""]; if (req.method !== "POST" || (path !== "/" && path !== "/search" && !xRoute)) { - send(res, { status: 404, body: { error: "POST a search to / or /search, an X read to /x-search, /x-post, /x-user or /x-trends, or a YouTube link to /youtube-transcript" } }); + send(res, { status: 404, body: { error: "POST a search to / or /search, an X read to /x-search, /x-post, /x-user or /x-trends, a YouTube link to /youtube-transcript, or a clip description to /video" } }); return; } // Stop the search, and free its tab, if the client goes away first. diff --git a/castle-search/src/video.ts b/castle-search/src/video.ts new file mode 100644 index 0000000..19c6729 --- /dev/null +++ b/castle-search/src/video.ts @@ -0,0 +1,222 @@ +/** + * A short video clip made from a text prompt by one of DeepInfra's video + * models, filed as an .mp4 in a folder (castle's synced Nextcloud "video" + * folder) and described to the caller. + * + * Four kinds, one model each, chosen by the caller: + * + * fast FastWan 2.2 5B: 5 s, 720p, no sound, about $0.02. The default. + * quality Wan 2.2 14B: 5 s, 720p, no sound, better motion and detail, + * about $0.17 and about 3 minutes. + * long LTX 2.3 Fast: 6 to 20 s, 720p, with sound; can start from a + * picture (image_url). $0.03 a second. + * sound Veo 3.1 Fast: 8 s, 720p, the best quality, with realistic sound + * and speech. $0.15 a second, about $1.20. + * + * A clip costs ten to fifty times an image, so at most CLIPS_PER_DAY are made + * a day (counted in a state file, so a restart does not reset it), one at a + * time. Once started a clip is finished and filed even if the caller has gone: + * DeepInfra bills it either way. + * + * Configured by environment: CCS_DEEPINFRA_KEY_FILE (a file holding the API + * key), CCS_VIDEO_DIR (where clips go), CCS_VIDEO_STATE (the daily count). + */ + +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import { BrowserUnavailableError, NoResultsError } from "./errors.ts"; + +export const VIDEO_KINDS = ["fast", "quality", "long", "sound"] as const; +export type VideoKind = (typeof VIDEO_KINDS)[number]; +export const LONG_SECONDS = [6, 8, 10, 12, 14, 16, 18, 20] as const; +export const CLIPS_PER_DAY = 10; +const CALL_TIMEOUT_MS = 15 * 60_000; +const API = "https://api.deepinfra.com"; + +export type VideoRequest = { + prompt: string; + kind: VideoKind; + portrait: boolean; + /** Only for long: one of LONG_SECONDS. */ + seconds?: number | undefined; + /** Only for long: the first frame, an https link or a data:image URI. */ + imageUrl?: string | undefined; +}; + +export type VideoOutcome = { + file: string; + folder_link: string; + kind: VideoKind; + model: string; + seconds: number; + sound: boolean; + cost_usd: number | null; + clips_left_today: number; + prompt: string; +}; + +type Plan = { model: string; seconds: number; sound: boolean; body: Record }; + +/** What to ask DeepInfra for, by kind. */ +export const planOf = (r: VideoRequest): Plan => { + const orientation = r.portrait ? "portrait" : "landscape"; + switch (r.kind) { + case "fast": + return { + model: "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers", + seconds: 5, + sound: false, + body: { prompt: r.prompt, seconds: 5, resolution: "720p", orientation }, + }; + case "quality": + return { + model: "Wan-AI/Wan2.2-T2V-A14B", + seconds: 5, + sound: false, + body: { prompt: r.prompt, seconds: 5, resolution: "720p", orientation }, + }; + case "long": { + const seconds = r.seconds ?? 6; + const body: Record = { + prompt: r.prompt, + duration: seconds, + resolution: r.portrait ? "720x1280" : "1280x720", + generate_audio: true, + }; + if (r.imageUrl) body["image_url"] = r.imageUrl; + return { model: "Lightricks/LTX-2.3-Fast", seconds, sound: true, body }; + } + case "sound": + return { + model: "google/veo-3.1-fast", + seconds: 8, + sound: true, + body: { prompt: r.prompt, aspect_ratio: r.portrait ? "9:16" : "16:9", resolution: "720p", generate_audio: true, sample_count: 1 }, + }; + } +}; + +/** A file name from the time and the prompt's first words: 2026-11-19-2130-a-red-fox-trotting.mp4 */ +export const clipName = (prompt: string, now: Date): string => { + const p = (n: number): string => String(n).padStart(2, "0"); + const stamp = `${now.getFullYear()}-${p(now.getMonth() + 1)}-${p(now.getDate())}-${p(now.getHours())}${p(now.getMinutes())}${p(now.getSeconds())}`; + const words = prompt + .toLowerCase() + .normalize("NFKD") + .replace(/[\u0300-\u036f]/g, "") + .replace(/[^a-z0-9\s-]/g, " ") + .split(/\s+/) + .filter(Boolean) + .slice(0, 6); + return `${stamp}${words.length ? "-" + words.join("-") : ""}.mp4`; +}; + +const today = (now: Date): string => now.toISOString().slice(0, 10); + +/** Clips made today, from the state file; a missing or unreadable file is none. */ +export const usedToday = (state: string | undefined, now: Date): number => { + if (!state || !existsSync(state)) return 0; + try { + const s = JSON.parse(readFileSync(state, "utf8")) as { day?: string; clips?: number }; + return s.day === today(now) && typeof s.clips === "number" ? s.clips : 0; + } catch { + return 0; + } +}; + +const recordClip = (state: string | undefined, now: Date): number => { + const clips = usedToday(state, now) + 1; + if (state) writeFileSync(state, JSON.stringify({ day: today(now), clips }) + "\n", "utf8"); + return clips; +}; + +/** The video bytes from DeepInfra's answer: a data URL, a path on its API, or a link. */ +export const videoRef = (reply: Record): string | null => { + const one = reply["video_url"]; + if (typeof one === "string" && one) return one; + const many = reply["videos"]; + if (Array.isArray(many) && typeof many[0] === "string" && many[0]) return many[0]; + return null; +}; + +const bytesOf = async (ref: string, key: string): Promise => { + if (ref.startsWith("data:")) { + const comma = ref.indexOf(","); + return Buffer.from(ref.slice(comma + 1), "base64"); + } + const url = ref.startsWith("/") ? API + ref : ref; + const res = await fetch(url, { + headers: url.startsWith(API) ? { Authorization: `bearer ${key}` } : {}, + signal: AbortSignal.timeout(120_000), + }); + if (!res.ok) throw new BrowserUnavailableError(`DeepInfra made the clip but fetching it failed: HTTP ${res.status}`); + return Buffer.from(await res.arrayBuffer()); +}; + +const make = async (r: VideoRequest): Promise => { + const folder = process.env["CCS_VIDEO_DIR"]; + if (!folder || !existsSync(folder)) { + throw new BrowserUnavailableError(`the video folder (CCS_VIDEO_DIR=${folder ?? "unset"}) does not exist`); + } + const keyFile = process.env["CCS_DEEPINFRA_KEY_FILE"]; + const key = keyFile && existsSync(keyFile) ? readFileSync(keyFile, "utf8").trim() : ""; + if (!key) throw new BrowserUnavailableError("no DeepInfra key is configured (CCS_DEEPINFRA_KEY_FILE)"); + const state = process.env["CCS_VIDEO_STATE"]; + const now = new Date(); + if (usedToday(state, now) >= CLIPS_PER_DAY) { + throw new NoResultsError( + `today's limit of ${CLIPS_PER_DAY} video clips is used up; it resets at midnight UTC. ` + + "Tell Igor rather than trying again.", + ); + } + + const plan = planOf(r); + const started = Date.now(); + // Not tied to the caller's connection: an abandoned clip is still billed, so it is still filed. + const res = await fetch(`${API}/v1/inference/${plan.model}`, { + method: "POST", + headers: { Authorization: `bearer ${key}`, "Content-Type": "application/json" }, + body: JSON.stringify(plan.body), + signal: AbortSignal.timeout(CALL_TIMEOUT_MS), + }); + const text = await res.text(); + let reply: Record = {}; + try { + reply = JSON.parse(text) as Record; + } catch { + // reported below + } + if (!res.ok) { + const why = (reply["detail"] as { error?: string } | undefined)?.error ?? reply["message"] ?? reply["detail"] ?? text.slice(0, 300); + throw new BrowserUnavailableError(`${plan.model} refused the clip: HTTP ${res.status}: ${typeof why === "string" ? why : JSON.stringify(why)}`); + } + const ref = videoRef(reply); + if (!ref) throw new BrowserUnavailableError(`${plan.model} answered without a video: ${text.slice(0, 300)}`); + const clips = recordClip(state, now); + const bytes = await bytesOf(ref, key); + const name = clipName(r.prompt, now); + writeFileSync(join(folder, name), bytes); + const cost = (reply["inference_status"] as { cost?: number } | undefined)?.cost; + console.log(`video ${r.kind} ${plan.model} -> ${name} (${bytes.length} bytes, $${cost ?? "?"}, ${Date.now() - started} ms, ${clips}/${CLIPS_PER_DAY} today)`); + return { + file: `video/${name}`, + folder_link: "https://nc.soarez.net/apps/files/files?dir=/video", + kind: r.kind, + model: plan.model, + seconds: plan.seconds, + sound: plan.sound, + cost_usd: typeof cost === "number" ? cost : null, + clips_left_today: Math.max(0, CLIPS_PER_DAY - clips), + prompt: r.prompt, + }; +}; + +let tail: Promise = Promise.resolve(); + +/** One clip at a time. */ +export const makeVideo = (r: VideoRequest): Promise => { + const next = tail.then(() => make(r)); + tail = next.catch(() => {}); + return next; +}; diff --git a/castle-search/test/video.test.ts b/castle-search/test/video.test.ts new file mode 100644 index 0000000..ece0353 --- /dev/null +++ b/castle-search/test/video.test.ts @@ -0,0 +1,79 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { handleVideo } from "../src/http.ts"; +import { clipName, planOf, usedToday, type VideoRequest, videoRef } from "../src/video.ts"; + +const req = (over: Partial = {}): VideoRequest => ({ prompt: "a fox", kind: "fast", portrait: false, ...over }); + +test("each kind asks its own model, with its own fixed length and sound", () => { + assert.equal(planOf(req()).model, "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers"); + assert.deepEqual(planOf(req()).body, { prompt: "a fox", seconds: 5, resolution: "720p", orientation: "landscape" }); + assert.equal(planOf(req({ kind: "quality", portrait: true })).body["orientation"], "portrait"); + const long = planOf(req({ kind: "long", seconds: 12, imageUrl: "https://x/y.png" })); + assert.equal(long.model, "Lightricks/LTX-2.3-Fast"); + assert.deepEqual(long.body, { prompt: "a fox", duration: 12, resolution: "1280x720", generate_audio: true, image_url: "https://x/y.png" }); + assert.equal(planOf(req({ kind: "long" })).seconds, 6); + const sound = planOf(req({ kind: "sound", portrait: true })); + assert.equal(sound.model, "google/veo-3.1-fast"); + assert.equal(sound.body["aspect_ratio"], "9:16"); + assert.equal(sound.sound, true); + assert.equal(planOf(req()).sound, false); +}); + +test("a clip is named by the time and the prompt's first words", () => { + const at = new Date(2026, 10, 19, 21, 5, 9); + assert.equal(clipName("A red fox, trotting through fresh snow in a pine forest", at), "2026-11-19-210509-a-red-fox-trotting-through-fresh.mp4"); + assert.equal(clipName("¡¿", at), "2026-11-19-210509.mp4"); + assert.equal(clipName("Café crème", at), "2026-11-19-210509-cafe-creme.mp4"); +}); + +test("the daily count is today's only, and a missing or broken file is none", () => { + const dir = mkdtempSync(join(tmpdir(), "video-")); + try { + const state = join(dir, "state.json"); + const now = new Date("2026-11-19T12:00:00Z"); + assert.equal(usedToday(state, now), 0); + assert.equal(usedToday(undefined, now), 0); + writeFileSync(state, JSON.stringify({ day: "2026-11-19", clips: 4 })); + assert.equal(usedToday(state, now), 4); + assert.equal(usedToday(state, new Date("2026-11-20T00:00:01Z")), 0); + writeFileSync(state, "{nope"); + assert.equal(usedToday(state, now), 0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("the video comes from video_url or the first of videos", () => { + assert.equal(videoRef({ video_url: "data:video/mp4;base64,AA" }), "data:video/mp4;base64,AA"); + assert.equal(videoRef({ status: "ok", videos: ["/model/inference/x.mp4"] }), "/model/inference/x.mp4"); + assert.equal(videoRef({ status: "error", videos: [] }), null); +}); + +test("/video checks its body before spending anything", async () => { + const calls: VideoRequest[] = []; + const run = async (r: VideoRequest) => { + calls.push(r); + return { file: "video/x.mp4" } as never; + }; + const bad = async (body: unknown) => (await handleVideo(JSON.stringify(body), run)).status; + assert.equal(await bad({}), 400); + assert.equal(await bad({ prompt: " " }), 400); + assert.equal(await bad({ prompt: "x".repeat(2001) }), 400); + assert.equal(await bad({ prompt: "a fox", kind: "cinema" }), 400); + assert.equal(await bad({ prompt: "a fox", orientation: "square" }), 400); + assert.equal(await bad({ prompt: "a fox", seconds: 10 }), 400); + assert.equal(await bad({ prompt: "a fox", kind: "long", seconds: 7 }), 400); + assert.equal(await bad({ prompt: "a fox", image_url: "https://x/y.png" }), 400); + assert.equal(await bad({ prompt: "a fox", kind: "long", image_url: "file:///etc/passwd" }), 400); + assert.equal(calls.length, 0); + + assert.equal(await bad({ prompt: " a fox " }), 200); + assert.deepEqual(calls.at(-1), { prompt: "a fox", kind: "fast", portrait: false, seconds: undefined, imageUrl: undefined }); + assert.equal(await bad({ prompt: "a fox", kind: "long", seconds: 10, orientation: "portrait", image_url: "data:image/png;base64,AA" }), 200); + assert.deepEqual(calls.at(-1), { prompt: "a fox", kind: "long", portrait: true, seconds: 10, imageUrl: "data:image/png;base64,AA" }); +}); -- cgit v1.2.3