aboutsummaryrefslogtreecommitdiff
path: root/castle-search/src/video.ts
diff options
context:
space:
mode:
Diffstat (limited to 'castle-search/src/video.ts')
-rw-r--r--castle-search/src/video.ts222
1 files changed, 222 insertions, 0 deletions
diff --git a/castle-search/src/video.ts b/castle-search/src/video.ts
new file mode 100644
index 0000000..19c6729
--- /dev/null
+++ b/castle-search/src/video.ts
@@ -0,0 +1,222 @@
+/**
+ * A short video clip made from a text prompt by one of DeepInfra's video
+ * models, filed as an .mp4 in a folder (castle's synced Nextcloud "video"
+ * folder) and described to the caller.
+ *
+ * Four kinds, one model each, chosen by the caller:
+ *
+ * fast FastWan 2.2 5B: 5 s, 720p, no sound, about $0.02. The default.
+ * quality Wan 2.2 14B: 5 s, 720p, no sound, better motion and detail,
+ * about $0.17 and about 3 minutes.
+ * long LTX 2.3 Fast: 6 to 20 s, 720p, with sound; can start from a
+ * picture (image_url). $0.03 a second.
+ * sound Veo 3.1 Fast: 8 s, 720p, the best quality, with realistic sound
+ * and speech. $0.15 a second, about $1.20.
+ *
+ * A clip costs ten to fifty times an image, so at most CLIPS_PER_DAY are made
+ * a day (counted in a state file, so a restart does not reset it), one at a
+ * time. Once started a clip is finished and filed even if the caller has gone:
+ * DeepInfra bills it either way.
+ *
+ * Configured by environment: CCS_DEEPINFRA_KEY_FILE (a file holding the API
+ * key), CCS_VIDEO_DIR (where clips go), CCS_VIDEO_STATE (the daily count).
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { BrowserUnavailableError, NoResultsError } from "./errors.ts";
+
+export const VIDEO_KINDS = ["fast", "quality", "long", "sound"] as const;
+export type VideoKind = (typeof VIDEO_KINDS)[number];
+export const LONG_SECONDS = [6, 8, 10, 12, 14, 16, 18, 20] as const;
+export const CLIPS_PER_DAY = 10;
+const CALL_TIMEOUT_MS = 15 * 60_000;
+const API = "https://api.deepinfra.com";
+
+export type VideoRequest = {
+ prompt: string;
+ kind: VideoKind;
+ portrait: boolean;
+ /** Only for long: one of LONG_SECONDS. */
+ seconds?: number | undefined;
+ /** Only for long: the first frame, an https link or a data:image URI. */
+ imageUrl?: string | undefined;
+};
+
+export type VideoOutcome = {
+ file: string;
+ folder_link: string;
+ kind: VideoKind;
+ model: string;
+ seconds: number;
+ sound: boolean;
+ cost_usd: number | null;
+ clips_left_today: number;
+ prompt: string;
+};
+
+type Plan = { model: string; seconds: number; sound: boolean; body: Record<string, unknown> };
+
+/** What to ask DeepInfra for, by kind. */
+export const planOf = (r: VideoRequest): Plan => {
+ const orientation = r.portrait ? "portrait" : "landscape";
+ switch (r.kind) {
+ case "fast":
+ return {
+ model: "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
+ seconds: 5,
+ sound: false,
+ body: { prompt: r.prompt, seconds: 5, resolution: "720p", orientation },
+ };
+ case "quality":
+ return {
+ model: "Wan-AI/Wan2.2-T2V-A14B",
+ seconds: 5,
+ sound: false,
+ body: { prompt: r.prompt, seconds: 5, resolution: "720p", orientation },
+ };
+ case "long": {
+ const seconds = r.seconds ?? 6;
+ const body: Record<string, unknown> = {
+ prompt: r.prompt,
+ duration: seconds,
+ resolution: r.portrait ? "720x1280" : "1280x720",
+ generate_audio: true,
+ };
+ if (r.imageUrl) body["image_url"] = r.imageUrl;
+ return { model: "Lightricks/LTX-2.3-Fast", seconds, sound: true, body };
+ }
+ case "sound":
+ return {
+ model: "google/veo-3.1-fast",
+ seconds: 8,
+ sound: true,
+ body: { prompt: r.prompt, aspect_ratio: r.portrait ? "9:16" : "16:9", resolution: "720p", generate_audio: true, sample_count: 1 },
+ };
+ }
+};
+
+/** A file name from the time and the prompt's first words: 2026-11-19-2130-a-red-fox-trotting.mp4 */
+export const clipName = (prompt: string, now: Date): string => {
+ const p = (n: number): string => String(n).padStart(2, "0");
+ const stamp = `${now.getFullYear()}-${p(now.getMonth() + 1)}-${p(now.getDate())}-${p(now.getHours())}${p(now.getMinutes())}${p(now.getSeconds())}`;
+ const words = prompt
+ .toLowerCase()
+ .normalize("NFKD")
+ .replace(/[\u0300-\u036f]/g, "")
+ .replace(/[^a-z0-9\s-]/g, " ")
+ .split(/\s+/)
+ .filter(Boolean)
+ .slice(0, 6);
+ return `${stamp}${words.length ? "-" + words.join("-") : ""}.mp4`;
+};
+
+const today = (now: Date): string => now.toISOString().slice(0, 10);
+
+/** Clips made today, from the state file; a missing or unreadable file is none. */
+export const usedToday = (state: string | undefined, now: Date): number => {
+ if (!state || !existsSync(state)) return 0;
+ try {
+ const s = JSON.parse(readFileSync(state, "utf8")) as { day?: string; clips?: number };
+ return s.day === today(now) && typeof s.clips === "number" ? s.clips : 0;
+ } catch {
+ return 0;
+ }
+};
+
+const recordClip = (state: string | undefined, now: Date): number => {
+ const clips = usedToday(state, now) + 1;
+ if (state) writeFileSync(state, JSON.stringify({ day: today(now), clips }) + "\n", "utf8");
+ return clips;
+};
+
+/** The video bytes from DeepInfra's answer: a data URL, a path on its API, or a link. */
+export const videoRef = (reply: Record<string, unknown>): string | null => {
+ const one = reply["video_url"];
+ if (typeof one === "string" && one) return one;
+ const many = reply["videos"];
+ if (Array.isArray(many) && typeof many[0] === "string" && many[0]) return many[0];
+ return null;
+};
+
+const bytesOf = async (ref: string, key: string): Promise<Buffer> => {
+ if (ref.startsWith("data:")) {
+ const comma = ref.indexOf(",");
+ return Buffer.from(ref.slice(comma + 1), "base64");
+ }
+ const url = ref.startsWith("/") ? API + ref : ref;
+ const res = await fetch(url, {
+ headers: url.startsWith(API) ? { Authorization: `bearer ${key}` } : {},
+ signal: AbortSignal.timeout(120_000),
+ });
+ if (!res.ok) throw new BrowserUnavailableError(`DeepInfra made the clip but fetching it failed: HTTP ${res.status}`);
+ return Buffer.from(await res.arrayBuffer());
+};
+
+const make = async (r: VideoRequest): Promise<VideoOutcome> => {
+ const folder = process.env["CCS_VIDEO_DIR"];
+ if (!folder || !existsSync(folder)) {
+ throw new BrowserUnavailableError(`the video folder (CCS_VIDEO_DIR=${folder ?? "unset"}) does not exist`);
+ }
+ const keyFile = process.env["CCS_DEEPINFRA_KEY_FILE"];
+ const key = keyFile && existsSync(keyFile) ? readFileSync(keyFile, "utf8").trim() : "";
+ if (!key) throw new BrowserUnavailableError("no DeepInfra key is configured (CCS_DEEPINFRA_KEY_FILE)");
+ const state = process.env["CCS_VIDEO_STATE"];
+ const now = new Date();
+ if (usedToday(state, now) >= CLIPS_PER_DAY) {
+ throw new NoResultsError(
+ `today's limit of ${CLIPS_PER_DAY} video clips is used up; it resets at midnight UTC. ` +
+ "Tell Igor rather than trying again.",
+ );
+ }
+
+ const plan = planOf(r);
+ const started = Date.now();
+ // Not tied to the caller's connection: an abandoned clip is still billed, so it is still filed.
+ const res = await fetch(`${API}/v1/inference/${plan.model}`, {
+ method: "POST",
+ headers: { Authorization: `bearer ${key}`, "Content-Type": "application/json" },
+ body: JSON.stringify(plan.body),
+ signal: AbortSignal.timeout(CALL_TIMEOUT_MS),
+ });
+ const text = await res.text();
+ let reply: Record<string, unknown> = {};
+ try {
+ reply = JSON.parse(text) as Record<string, unknown>;
+ } catch {
+ // reported below
+ }
+ if (!res.ok) {
+ const why = (reply["detail"] as { error?: string } | undefined)?.error ?? reply["message"] ?? reply["detail"] ?? text.slice(0, 300);
+ throw new BrowserUnavailableError(`${plan.model} refused the clip: HTTP ${res.status}: ${typeof why === "string" ? why : JSON.stringify(why)}`);
+ }
+ const ref = videoRef(reply);
+ if (!ref) throw new BrowserUnavailableError(`${plan.model} answered without a video: ${text.slice(0, 300)}`);
+ const clips = recordClip(state, now);
+ const bytes = await bytesOf(ref, key);
+ const name = clipName(r.prompt, now);
+ writeFileSync(join(folder, name), bytes);
+ const cost = (reply["inference_status"] as { cost?: number } | undefined)?.cost;
+ console.log(`video ${r.kind} ${plan.model} -> ${name} (${bytes.length} bytes, $${cost ?? "?"}, ${Date.now() - started} ms, ${clips}/${CLIPS_PER_DAY} today)`);
+ return {
+ file: `video/${name}`,
+ folder_link: "https://nc.soarez.net/apps/files/files?dir=/video",
+ kind: r.kind,
+ model: plan.model,
+ seconds: plan.seconds,
+ sound: plan.sound,
+ cost_usd: typeof cost === "number" ? cost : null,
+ clips_left_today: Math.max(0, CLIPS_PER_DAY - clips),
+ prompt: r.prompt,
+ };
+};
+
+let tail: Promise<unknown> = Promise.resolve();
+
+/** One clip at a time. */
+export const makeVideo = (r: VideoRequest): Promise<VideoOutcome> => {
+ const next = tail.then(() => make(r));
+ tail = next.catch(() => {});
+ return next;
+};