1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
|
/**
* castle_cdp_search as a plain HTTP service, for clients that are not pi.
*
* It speaks Open WebUI's "external" web search protocol: POST a JSON body
* `{ "query": "...", "count": 5 }`, get back `[{ "link", "title", "snippet" }]`.
* Any client that can send that can use it.
*
* node --experimental-strip-types src/http.ts [host:port] (default 127.0.0.1:11158)
*
* The engine and the browser are chosen exactly as for the pi tool:
* CCS_SEARCH_ENGINE and BU_CDP_HTTP, then the stored choices, then the defaults.
*
* A failed search is answered with HTTP 502 and `{ "error": "<why>" }`, never
* with an empty list: an empty list reads as "the web has nothing on this", and
* a captcha or a rate limit is not that. A search that genuinely finds nothing
* is a failure too here (NoResultsError), because an engine that is quietly
* rate-limiting serves exactly that page, and the message says which it may be.
*
* It has no authentication, so bind it to loopback, as the default does, or to
* an address only trusted peers reach. Each search opens and closes one
* background tab in a browser someone may be looking at.
*
* POST /x-search, /x-post, /x-user and /x-trends read X instead (x-search.ts),
* through the browser's signed-in X session, and GET /openapi.json describes
* them, so that Open WebUI can offer them as tools ("X.com"). Each answers 502
* with `{ "error" }` on failure.
*/
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { messageOf } from "./errors.ts";
import { disconnect, search, type SearchOutcome, type SearchRequest } from "./search.ts";
import {
handleOf,
postIdOf,
TREND_TABS,
type XPostOutcome,
type XPostRequest,
xPost,
xSearch,
type XSearchOutcome,
type XSearchRequest,
type XTrendsOutcome,
type XTrendTab,
xTrends,
xUser,
type XUserOutcome,
type XUserRequest,
} from "./x-search.ts";
export const DEFAULT_LISTEN = "127.0.0.1:11158";
const MAX_BODY_BYTES = 64 * 1024;
const DEFAULT_COUNT = 5;
const MAX_COUNT = 20;
// Said after every error. The castle-search messages tell the caller to retry
// with another `engine`, which a client of this protocol cannot do: it sends a
// query and a count, nothing else.
const ENGINE_NOTE =
"This search service cannot switch engines per call. Tell the user web search failed and why; " +
"the operator can change the engine (CCS_SEARCH_ENGINE) or clear the check in the browser.";
export type Reply = { status: number; body: unknown };
export type Searcher = (request: SearchRequest, signal: AbortSignal) => Promise<SearchOutcome>;
/** Everything but the socket: parse a request body, run the search, shape the reply. */
export const handleSearch = async (raw: string, run: Searcher, signal: AbortSignal): Promise<Reply> => {
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch {
return { status: 400, body: { error: "request body is not JSON" } };
}
const { query, count } = (parsed ?? {}) as { query?: unknown; count?: unknown };
if (typeof query !== "string" || query.trim() === "") {
return { status: 400, body: { error: "query must be a non-empty string" } };
}
const n = typeof count === "number" && Number.isFinite(count) ? Math.trunc(count) : DEFAULT_COUNT;
const numResults = Math.min(MAX_COUNT, Math.max(1, n));
try {
const outcome = await run({ query: query.trim(), numResults }, signal);
return {
status: 200,
body: outcome.results.map((r) => ({ link: r.url, title: r.title, snippet: r.snippet })),
};
} catch (e) {
return { status: 502, body: { error: `${messageOf(e)} ${ENGINE_NOTE}` } };
}
};
export type XSearcher = (request: XSearchRequest, signal: AbortSignal) => Promise<XSearchOutcome>;
export type XPostReader = (request: XPostRequest, signal: AbortSignal) => Promise<XPostOutcome>;
export type XUserReader = (request: XUserRequest, signal: AbortSignal) => Promise<XUserOutcome>;
export type XTrendsReader = (tab: XTrendTab, signal: AbortSignal) => Promise<XTrendsOutcome>;
const X_DEFAULT_COUNT = 10;
const jsonBody = (raw: string): Record<string, unknown> | Reply => {
try {
const parsed: unknown = JSON.parse(raw || "{}");
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed as Record<string, unknown>;
} catch {
// fall through
}
return { status: 400, body: { error: "request body is not a JSON object" } };
};
const isReply = (v: unknown): v is Reply => typeof (v as Reply).status === "number" && "body" in (v as Reply);
const countOf = (v: unknown, fallback: number, min = 1): number => {
const n = typeof v === "number" && Number.isFinite(v) ? Math.trunc(v) : fallback;
return Math.min(MAX_COUNT, Math.max(min, n));
};
const answer = async (work: () => Promise<unknown>): Promise<Reply> => {
try {
return { status: 200, body: await work() };
} catch (e) {
return { status: 502, body: { error: messageOf(e) } };
}
};
/** POST /x-search: `{ query, count?, sort? }` in, `{ query, sort, posts }` out. */
export const handleXSearch = async (raw: string, run: XSearcher, signal: AbortSignal): Promise<Reply> => {
const body = jsonBody(raw);
if (isReply(body)) return body;
const { query, count, sort } = body;
if (typeof query !== "string" || query.trim() === "") {
return { status: 400, body: { error: "query must be a non-empty string" } };
}
if (sort !== undefined && sort !== null && sort !== "latest" && sort !== "top") {
return { status: 400, body: { error: 'sort must be "latest" or "top"' } };
}
return answer(async () => {
const outcome = await run({ query: query.trim(), count: countOf(count, X_DEFAULT_COUNT), sort: sort ?? "latest" }, signal);
return { query: outcome.query, sort: outcome.sort, posts: outcome.posts };
});
};
/** POST /x-post: `{ url, replies? }` in, `{ post, thread, replies }` out. */
export const handleXPost = async (raw: string, run: XPostReader, signal: AbortSignal): Promise<Reply> => {
const body = jsonBody(raw);
if (isReply(body)) return body;
if (typeof body["url"] !== "string" || !postIdOf(body["url"])) {
return { status: 400, body: { error: "url must be a link to a post on X (x.com/<handle>/status/<id>) or its id" } };
}
const url = body["url"];
return answer(() => run({ url, replies: countOf(body["replies"], X_DEFAULT_COUNT, 0) }, signal));
};
/** POST /x-user: `{ handle, count? }` in, `{ profile, posts }` out. */
export const handleXUser = async (raw: string, run: XUserReader, signal: AbortSignal): Promise<Reply> => {
const body = jsonBody(raw);
if (isReply(body)) return body;
if (typeof body["handle"] !== "string" || !handleOf(body["handle"])) {
return { status: 400, body: { error: "handle must be an X handle, such as nasa or @nasa" } };
}
const handle = body["handle"];
return answer(() => run({ handle, count: countOf(body["count"], X_DEFAULT_COUNT, 0) }, signal));
};
/** POST /x-trends: `{ tab? }` in, `{ tab, trends }` out. */
export const handleXTrends = async (raw: string, run: XTrendsReader, signal: AbortSignal): Promise<Reply> => {
const body = jsonBody(raw);
if (isReply(body)) return body;
const tab = body["tab"] ?? "trending";
if (!TREND_TABS.includes(tab as XTrendTab)) {
return { status: 400, body: { error: `tab must be one of ${TREND_TABS.join(", ")}` } };
}
return answer(() => run(tab as XTrendTab, signal));
};
const jsonSchema = (properties: Record<string, unknown>, required: string[]) => ({
required: true,
content: { "application/json": { schema: { type: "object", required, properties } } },
});
const RESPONSES = {
"200": { description: "What X shows", content: { "application/json": { schema: { type: "object" } } } },
"502": { description: "X could not be read; error says why", content: { "application/json": { schema: { type: "object" } } } },
};
/**
* The OpenAPI description Open WebUI reads to offer the X reads as tools. Its
* descriptions are what the model sees, so they say what each is for.
*/
export const OPENAPI = {
openapi: "3.1.0",
info: {
title: "X.com",
version: "2.0.0",
description: "Reads X (Twitter) through a signed-in browser: search, posts with replies, profiles, trends. Free.",
},
paths: {
"/x-search": {
post: {
operationId: "search_x",
summary: "Search posts on X (Twitter)",
description:
"Search X for posts, newest first by default. Use it for anything on X: what someone posted, " +
"reactions to news, posts about a topic. The query uses X's search syntax: from:handle, to:handle, " +
'"exact phrase", OR, -word, since:YYYY-MM-DD, until:YYYY-MM-DD, min_faves:N, filter:media, ' +
"-filter:replies, lang:en. A handle is written without @ after from:. Each post comes with its " +
"link: cite it. A post marked truncated was cut short by X: read_x_post gives it whole. Photos " +
"come with their image link and alt text.",
requestBody: jsonSchema(
{
query: { type: "string", description: "X search query, e.g. from:nasa since:2026-11-01" },
count: { type: "integer", minimum: 1, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many posts" },
sort: { type: "string", enum: ["latest", "top"], default: "latest", description: "latest: newest first; top: most engaged" },
},
["query"],
),
responses: RESPONSES,
},
},
"/x-post": {
post: {
operationId: "read_x_post",
summary: "Read one post on X, whole, with its thread and replies",
description:
"Read a post on X from its link: its full text (never truncated), photos with their image link " +
"and alt text, its link card, the posts above it when it is a reply (thread, oldest first), and " +
"the top replies under it. Use it to read a long post, a thread, or what people answered.",
requestBody: jsonSchema(
{
url: { type: "string", description: "The post's link, x.com/<handle>/status/<id>, or the id" },
replies: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many replies" },
},
["url"],
),
responses: RESPONSES,
},
},
"/x-user": {
post: {
operationId: "read_x_profile",
summary: "Read an X profile and its latest posts",
description:
"Read an X account's profile: name, bio, location, website, when it joined, exact follower, " +
"following and post counts, then its latest posts as its profile shows them (a pinned post and " +
"reposts included, marked in context).",
requestBody: jsonSchema(
{
handle: { type: "string", description: "The account's handle, such as nasa or @nasa" },
count: { type: "integer", minimum: 0, maximum: MAX_COUNT, default: X_DEFAULT_COUNT, description: "How many of its posts" },
},
["handle"],
),
responses: RESPONSES,
},
},
"/x-trends": {
post: {
operationId: "x_trends",
summary: "What is trending on X now",
description:
"List what X shows as trending now, for the signed-in account's region (the United Kingdom). " +
"Each trend has a query: pass it to search_x for the posts.",
requestBody: jsonSchema(
{ tab: { type: "string", enum: [...TREND_TABS], default: "trending", description: "Which explore tab" } },
[],
),
responses: RESPONSES,
},
},
},
} as const;
const readBody = (req: IncomingMessage): Promise<string> =>
new Promise((resolve, reject) => {
let size = 0;
const chunks: Buffer[] = [];
req.on("data", (chunk: Buffer) => {
size += chunk.length;
if (size > MAX_BODY_BYTES) {
reject(new Error("request body too large"));
req.destroy();
return;
}
chunks.push(chunk);
});
req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
req.on("error", reject);
});
const send = (res: ServerResponse, reply: Reply): void => {
res.writeHead(reply.status, { "Content-Type": "application/json; charset=utf-8" });
res.end(JSON.stringify(reply.body));
};
export const parseListen = (value: string): { host: string; port: number } => {
const at = value.lastIndexOf(":");
const host = value.slice(0, at);
const port = Number(value.slice(at + 1));
if (at <= 0 || !Number.isInteger(port) || port < 1 || port > 65535) {
throw new Error(`listen address must be host:port, got "${value}"`);
}
return { host, port };
};
const main = (): void => {
const { host, port } = parseListen(process.argv[2] ?? DEFAULT_LISTEN);
const runSearch: Searcher = (request, signal) => search(request, signal, () => {});
const server = createServer(async (req, res) => {
const path = (req.url ?? "/").split("?")[0];
if (req.method === "GET" && path === "/health") {
send(res, { status: 200, body: { status: true } });
return;
}
if (req.method === "GET" && path === "/openapi.json") {
send(res, { status: 200, body: OPENAPI });
return;
}
const xRoutes: Record<string, (raw: string, signal: AbortSignal) => Promise<Reply>> = {
"/x-search": (raw, signal) => handleXSearch(raw, xSearch, signal),
"/x-post": (raw, signal) => handleXPost(raw, xPost, signal),
"/x-user": (raw, signal) => handleXUser(raw, xUser, signal),
"/x-trends": (raw, signal) => handleXTrends(raw, xTrends, signal),
};
const xRoute = xRoutes[path ?? ""];
if (req.method !== "POST" || (path !== "/" && path !== "/search" && !xRoute)) {
send(res, { status: 404, body: { error: "POST a search to / or /search, or an X read to /x-search, /x-post, /x-user or /x-trends" } });
return;
}
// Stop the search, and free its tab, if the client goes away first.
const controller = new AbortController();
res.on("close", () => {
if (!res.writableEnded) controller.abort();
});
let raw: string;
try {
raw = await readBody(req);
} catch (e) {
send(res, { status: 413, body: { error: messageOf(e) } });
return;
}
const started = Date.now();
const reply = xRoute ? await xRoute(raw, controller.signal) : await handleSearch(raw, runSearch, controller.signal);
const outcome = reply.status !== 200 ? `error ${reply.status}` : xRoute ? "ok" : `${(reply.body as unknown[]).length} results`;
console.log(`${xRoute ? path!.slice(1) : "search"} ${outcome} in ${Date.now() - started} ms`);
if (reply.status !== 200) console.log(` ${(reply.body as { error: string }).error}`);
send(res, reply);
});
const stop = (): void => {
disconnect();
server.close(() => process.exit(0));
};
process.on("SIGTERM", stop);
process.on("SIGINT", stop);
server.listen(port, host, () => console.log(`castle-search HTTP listening on ${host}:${port}`));
};
if (import.meta.url === `file://${process.argv[1]}`) main();
|