aboutsummaryrefslogtreecommitdiff
path: root/castle-search/src/video.ts
blob: 19c6729b078214bd73b3de66fc2b5cc546167dab (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
/**
 * A short video clip made from a text prompt by one of DeepInfra's video
 * models, filed as an .mp4 in a folder (castle's synced Nextcloud "video"
 * folder) and described to the caller.
 *
 * Four kinds, one model each, chosen by the caller:
 *
 *   fast     FastWan 2.2 5B: 5 s, 720p, no sound, about $0.02. The default.
 *   quality  Wan 2.2 14B: 5 s, 720p, no sound, better motion and detail,
 *            about $0.17 and about 3 minutes.
 *   long     LTX 2.3 Fast: 6 to 20 s, 720p, with sound; can start from a
 *            picture (image_url). $0.03 a second.
 *   sound    Veo 3.1 Fast: 8 s, 720p, the best quality, with realistic sound
 *            and speech. $0.15 a second, about $1.20.
 *
 * A clip costs ten to fifty times an image, so at most CLIPS_PER_DAY are made
 * a day (counted in a state file, so a restart does not reset it), one at a
 * time. Once started a clip is finished and filed even if the caller has gone:
 * DeepInfra bills it either way.
 *
 * Configured by environment: CCS_DEEPINFRA_KEY_FILE (a file holding the API
 * key), CCS_VIDEO_DIR (where clips go), CCS_VIDEO_STATE (the daily count).
 */

import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";

import { BrowserUnavailableError, NoResultsError } from "./errors.ts";

export const VIDEO_KINDS = ["fast", "quality", "long", "sound"] as const;
export type VideoKind = (typeof VIDEO_KINDS)[number];
export const LONG_SECONDS = [6, 8, 10, 12, 14, 16, 18, 20] as const;
export const CLIPS_PER_DAY = 10;
const CALL_TIMEOUT_MS = 15 * 60_000;
const API = "https://api.deepinfra.com";

export type VideoRequest = {
  prompt: string;
  kind: VideoKind;
  portrait: boolean;
  /** Only for long: one of LONG_SECONDS. */
  seconds?: number | undefined;
  /** Only for long: the first frame, an https link or a data:image URI. */
  imageUrl?: string | undefined;
};

export type VideoOutcome = {
  file: string;
  folder_link: string;
  kind: VideoKind;
  model: string;
  seconds: number;
  sound: boolean;
  cost_usd: number | null;
  clips_left_today: number;
  prompt: string;
};

type Plan = { model: string; seconds: number; sound: boolean; body: Record<string, unknown> };

/** What to ask DeepInfra for, by kind. */
export const planOf = (r: VideoRequest): Plan => {
  const orientation = r.portrait ? "portrait" : "landscape";
  switch (r.kind) {
    case "fast":
      return {
        model: "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
        seconds: 5,
        sound: false,
        body: { prompt: r.prompt, seconds: 5, resolution: "720p", orientation },
      };
    case "quality":
      return {
        model: "Wan-AI/Wan2.2-T2V-A14B",
        seconds: 5,
        sound: false,
        body: { prompt: r.prompt, seconds: 5, resolution: "720p", orientation },
      };
    case "long": {
      const seconds = r.seconds ?? 6;
      const body: Record<string, unknown> = {
        prompt: r.prompt,
        duration: seconds,
        resolution: r.portrait ? "720x1280" : "1280x720",
        generate_audio: true,
      };
      if (r.imageUrl) body["image_url"] = r.imageUrl;
      return { model: "Lightricks/LTX-2.3-Fast", seconds, sound: true, body };
    }
    case "sound":
      return {
        model: "google/veo-3.1-fast",
        seconds: 8,
        sound: true,
        body: { prompt: r.prompt, aspect_ratio: r.portrait ? "9:16" : "16:9", resolution: "720p", generate_audio: true, sample_count: 1 },
      };
  }
};

/** A file name from the time and the prompt's first words: 2026-11-19-2130-a-red-fox-trotting.mp4 */
export const clipName = (prompt: string, now: Date): string => {
  const p = (n: number): string => String(n).padStart(2, "0");
  const stamp = `${now.getFullYear()}-${p(now.getMonth() + 1)}-${p(now.getDate())}-${p(now.getHours())}${p(now.getMinutes())}${p(now.getSeconds())}`;
  const words = prompt
    .toLowerCase()
    .normalize("NFKD")
    .replace(/[\u0300-\u036f]/g, "")
    .replace(/[^a-z0-9\s-]/g, " ")
    .split(/\s+/)
    .filter(Boolean)
    .slice(0, 6);
  return `${stamp}${words.length ? "-" + words.join("-") : ""}.mp4`;
};

const today = (now: Date): string => now.toISOString().slice(0, 10);

/** Clips made today, from the state file; a missing or unreadable file is none. */
export const usedToday = (state: string | undefined, now: Date): number => {
  if (!state || !existsSync(state)) return 0;
  try {
    const s = JSON.parse(readFileSync(state, "utf8")) as { day?: string; clips?: number };
    return s.day === today(now) && typeof s.clips === "number" ? s.clips : 0;
  } catch {
    return 0;
  }
};

const recordClip = (state: string | undefined, now: Date): number => {
  const clips = usedToday(state, now) + 1;
  if (state) writeFileSync(state, JSON.stringify({ day: today(now), clips }) + "\n", "utf8");
  return clips;
};

/** The video bytes from DeepInfra's answer: a data URL, a path on its API, or a link. */
export const videoRef = (reply: Record<string, unknown>): string | null => {
  const one = reply["video_url"];
  if (typeof one === "string" && one) return one;
  const many = reply["videos"];
  if (Array.isArray(many) && typeof many[0] === "string" && many[0]) return many[0];
  return null;
};

const bytesOf = async (ref: string, key: string): Promise<Buffer> => {
  if (ref.startsWith("data:")) {
    const comma = ref.indexOf(",");
    return Buffer.from(ref.slice(comma + 1), "base64");
  }
  const url = ref.startsWith("/") ? API + ref : ref;
  const res = await fetch(url, {
    headers: url.startsWith(API) ? { Authorization: `bearer ${key}` } : {},
    signal: AbortSignal.timeout(120_000),
  });
  if (!res.ok) throw new BrowserUnavailableError(`DeepInfra made the clip but fetching it failed: HTTP ${res.status}`);
  return Buffer.from(await res.arrayBuffer());
};

const make = async (r: VideoRequest): Promise<VideoOutcome> => {
  const folder = process.env["CCS_VIDEO_DIR"];
  if (!folder || !existsSync(folder)) {
    throw new BrowserUnavailableError(`the video folder (CCS_VIDEO_DIR=${folder ?? "unset"}) does not exist`);
  }
  const keyFile = process.env["CCS_DEEPINFRA_KEY_FILE"];
  const key = keyFile && existsSync(keyFile) ? readFileSync(keyFile, "utf8").trim() : "";
  if (!key) throw new BrowserUnavailableError("no DeepInfra key is configured (CCS_DEEPINFRA_KEY_FILE)");
  const state = process.env["CCS_VIDEO_STATE"];
  const now = new Date();
  if (usedToday(state, now) >= CLIPS_PER_DAY) {
    throw new NoResultsError(
      `today's limit of ${CLIPS_PER_DAY} video clips is used up; it resets at midnight UTC. ` +
        "Tell Igor rather than trying again.",
    );
  }

  const plan = planOf(r);
  const started = Date.now();
  // Not tied to the caller's connection: an abandoned clip is still billed, so it is still filed.
  const res = await fetch(`${API}/v1/inference/${plan.model}`, {
    method: "POST",
    headers: { Authorization: `bearer ${key}`, "Content-Type": "application/json" },
    body: JSON.stringify(plan.body),
    signal: AbortSignal.timeout(CALL_TIMEOUT_MS),
  });
  const text = await res.text();
  let reply: Record<string, unknown> = {};
  try {
    reply = JSON.parse(text) as Record<string, unknown>;
  } catch {
    // reported below
  }
  if (!res.ok) {
    const why = (reply["detail"] as { error?: string } | undefined)?.error ?? reply["message"] ?? reply["detail"] ?? text.slice(0, 300);
    throw new BrowserUnavailableError(`${plan.model} refused the clip: HTTP ${res.status}: ${typeof why === "string" ? why : JSON.stringify(why)}`);
  }
  const ref = videoRef(reply);
  if (!ref) throw new BrowserUnavailableError(`${plan.model} answered without a video: ${text.slice(0, 300)}`);
  const clips = recordClip(state, now);
  const bytes = await bytesOf(ref, key);
  const name = clipName(r.prompt, now);
  writeFileSync(join(folder, name), bytes);
  const cost = (reply["inference_status"] as { cost?: number } | undefined)?.cost;
  console.log(`video ${r.kind} ${plan.model} -> ${name} (${bytes.length} bytes, $${cost ?? "?"}, ${Date.now() - started} ms, ${clips}/${CLIPS_PER_DAY} today)`);
  return {
    file: `video/${name}`,
    folder_link: "https://nc.soarez.net/apps/files/files?dir=/video",
    kind: r.kind,
    model: plan.model,
    seconds: plan.seconds,
    sound: plan.sound,
    cost_usd: typeof cost === "number" ? cost : null,
    clips_left_today: Math.max(0, CLIPS_PER_DAY - clips),
    prompt: r.prompt,
  };
};

let tail: Promise<unknown> = Promise.resolve();

/** One clip at a time. */
export const makeVideo = (r: VideoRequest): Promise<VideoOutcome> => {
  const next = tail.then(() => make(r));
  tail = next.catch(() => {});
  return next;
};