aboutsummaryrefslogtreecommitdiff
path: root/workers/plugin-marketplace/src
diff options
context:
space:
mode:
Diffstat (limited to 'workers/plugin-marketplace/src')
-rw-r--r--workers/plugin-marketplace/src/index.test.ts717
-rw-r--r--workers/plugin-marketplace/src/index.ts855
2 files changed, 1301 insertions, 271 deletions
diff --git a/workers/plugin-marketplace/src/index.test.ts b/workers/plugin-marketplace/src/index.test.ts
index b84ed14d..9e3df19b 100644
--- a/workers/plugin-marketplace/src/index.test.ts
+++ b/workers/plugin-marketplace/src/index.test.ts
@@ -1,20 +1,44 @@
import { describe, expect, test } from "bun:test";
-import worker, { normalizeRepositories, refreshPlugins, type Env } from "./index";
+import worker, {
+ normalizeRepositories,
+ parseManifestSummary,
+ refreshPlugins,
+ type Env,
+} from "./index";
+
+const HEAD_COMMIT = "a".repeat(40);
+const SECOND_COMMIT = "b".repeat(40);
+
+type TreeFixture = {
+ path: string;
+ content?: string;
+ mode?: string;
+ type?: string;
+ size?: number;
+};
class MemoryR2 {
objects = new Map<string, { value: string; options: unknown }>();
-
async put(key: string, value: string, options?: unknown): Promise<void> {
this.objects.set(key, { value, options });
}
+
+ async get(key: string): Promise<{ text(): Promise<string> } | null> {
+ const object = this.objects.get(key);
+ return object
+ ? {
+ async text() {
+ return object.value;
+ },
+ }
+ : null;
+ }
}
class MemoryKV {
constructor(private readonly keyNames: string[]) {}
- async list(options?: { prefix?: string }): Promise<{
- keys: Array<{ name: string }>;
- }> {
+ async list(options?: { prefix?: string }): Promise<{ keys: Array<{ name: string }> }> {
return {
keys: this.keyNames
.filter((name) => !options?.prefix || name.startsWith(options.prefix))
@@ -29,8 +53,9 @@ function repo(overrides: Record<string, unknown> = {}): Record<string, unknown>
full_name: "ogulcancelik/herdr-plugin-example",
owner: { login: "ogulcancelik" },
name: "herdr-plugin-example",
- description: "Example plugin",
+ description: "Example plugin repository",
html_url: "https://github.com/ogulcancelik/herdr-plugin-example",
+ default_branch: "main",
stargazers_count: 5,
forks_count: 1,
open_issues_count: 0,
@@ -48,6 +73,17 @@ function repo(overrides: Record<string, unknown> = {}): Record<string, unknown>
};
}
+function manifest(overrides = ""): string {
+ return `
+id = "example.plugin"
+name = "Example Plugin"
+version = "0.2.0"
+min_herdr_version = "0.7.0"
+description = "Example manifest"
+platforms = ["linux", "macos"]
+${overrides}`;
+}
+
function env(bucket = new MemoryR2(), blacklist?: MemoryKV): Env {
return {
PLUGIN_MARKETPLACE_BUCKET: bucket,
@@ -56,62 +92,137 @@ function env(bucket = new MemoryR2(), blacklist?: MemoryKV): Env {
};
}
+function repositoryFetch(options: {
+ repositories: Record<string, unknown>[];
+ trees?: Record<string, TreeFixture[]>;
+ commits?: Record<string, string>;
+ totalCount?: number;
+ incompleteResults?: boolean;
+ searchStatus?: number;
+ treeStatus?: Record<string, number>;
+ truncatedTrees?: Set<string>;
+ onRequest?: (kind: "search" | "head" | "tree" | "manifest", detail: string) => void;
+}): typeof fetch {
+ return (async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
+ const url = new URL(input.toString());
+ if (url.pathname === "/search/repositories") {
+ options.onRequest?.("search", url.toString());
+ if (options.searchStatus) return new Response("search failed", { status: options.searchStatus });
+ return Response.json({
+ total_count: options.totalCount ?? options.repositories.length,
+ incomplete_results: options.incompleteResults ?? false,
+ items: options.repositories,
+ });
+ }
+
+ const treeMatch = url.pathname.match(/^\/repos\/([^/]+)\/([^/]+)\/git\/trees\/([^/]+)$/);
+ if (treeMatch) {
+ const fullName = `${decodeURIComponent(treeMatch[1])}/${decodeURIComponent(treeMatch[2])}`;
+ options.onRequest?.("tree", fullName);
+ const status = options.treeStatus?.[fullName];
+ if (status) return new Response("tree failed", { status });
+ const fixtures = options.trees?.[fullName] ?? [
+ { path: "herdr-plugin.toml", content: manifest() },
+ ];
+ return Response.json({
+ sha: "tree-sha",
+ truncated: options.truncatedTrees?.has(fullName) ?? false,
+ tree: fixtures.map((fixture) => ({
+ path: fixture.path,
+ mode: fixture.mode ?? "100644",
+ type: fixture.type ?? "blob",
+ size:
+ fixture.size ??
+ new TextEncoder().encode(fixture.content ?? "").byteLength,
+ })),
+ });
+ }
+
+ if (url.pathname === "/graphql") {
+ const request = JSON.parse(String(init?.body ?? "{}"));
+ const query = String(request.query ?? "");
+ const data: Record<string, unknown> = {};
+ if (query.includes("PluginMarketplaceHeads")) {
+ options.onRequest?.("head", query);
+ for (const match of query.matchAll(
+ /repo(\d+): repository\(owner: "([^"]+)", name: "([^"]+)"\)/g,
+ )) {
+ const [, alias, owner, name] = match;
+ const fullName = `${owner}/${name}`;
+ const repository = options.repositories.find(
+ (candidate) => candidate.full_name === fullName,
+ );
+ data[`repo${alias}`] = repository
+ ? {
+ defaultBranchRef: {
+ name: repository.default_branch ?? "main",
+ target: { oid: options.commits?.[fullName] ?? HEAD_COMMIT },
+ },
+ }
+ : null;
+ }
+ } else if (query.includes("PluginMarketplaceManifests")) {
+ options.onRequest?.("manifest", query);
+ for (const match of query.matchAll(
+ /item(\d+): repository\(owner: "([^"]+)", name: "([^"]+)"\) \{\s+manifest: object\(expression: "[a-f0-9]+:([^"]+)"\)/g,
+ )) {
+ const [, alias, owner, name, path] = match;
+ const fullName = `${owner}/${name}`;
+ const fixture = (options.trees?.[fullName] ?? [
+ { path: "herdr-plugin.toml", content: manifest() },
+ ]).find((entry) => entry.path === path);
+ data[`item${alias}`] = fixture?.content === undefined
+ ? { manifest: null }
+ : { manifest: { text: fixture.content } };
+ }
+ } else {
+ throw new Error(`unexpected GraphQL query: ${query}`);
+ }
+ return Response.json({ data });
+ }
+
+ throw new Error(`unexpected request: ${url}`);
+ }) as typeof fetch;
+}
+
describe("normalizeRepositories", () => {
- test("normalizes repository fields into the public snapshot schema", () => {
- const [plugin] = normalizeRepositories([repo()]);
+ test("normalizes fields while preserving repository-card ordering", () => {
+ const plugins = normalizeRepositories([
+ repo({
+ id: 2,
+ full_name: "other/newer",
+ owner: { login: "other" },
+ name: "newer",
+ html_url: "https://github.com/other/newer",
+ stargazers_count: 5,
+ pushed_at: "2026-06-04T00:00:00Z",
+ }),
+ repo(),
+ ]);
- expect(plugin).toEqual({
+ expect(plugins.map((plugin) => plugin.fullName)).toEqual([
+ "other/newer",
+ "ogulcancelik/herdr-plugin-example",
+ ]);
+ expect(plugins[1]).toMatchObject({
id: 1,
- fullName: "ogulcancelik/herdr-plugin-example",
owner: "ogulcancelik",
name: "herdr-plugin-example",
- description: "Example plugin",
- url: "https://github.com/ogulcancelik/herdr-plugin-example",
+ defaultBranch: "main",
stars: 5,
- forks: 1,
- openIssues: 0,
- language: "TypeScript",
- topics: ["herdr-plugin"],
- createdAt: "2026-06-01T00:00:00Z",
- updatedAt: "2026-06-02T00:00:00Z",
- pushedAt: "2026-06-03T00:00:00Z",
});
});
- test("sorts by stars, pushed date, and full name", () => {
+ test("deduplicates repositories by immutable GitHub id", () => {
const plugins = normalizeRepositories([
- repo({
- id: 1,
- full_name: "z/z",
- owner: { login: "z" },
- name: "z",
- html_url: "https://github.com/z/z",
- stargazers_count: 3,
- pushed_at: "2026-06-01T00:00:00Z",
- }),
- repo({
- id: 2,
- full_name: "a/a",
- owner: { login: "a" },
- name: "a",
- html_url: "https://github.com/a/a",
- stargazers_count: 3,
- pushed_at: "2026-06-02T00:00:00Z",
- }),
- repo({
- id: 3,
- full_name: "m/m",
- owner: { login: "m" },
- name: "m",
- html_url: "https://github.com/m/m",
- stargazers_count: 10,
- }),
+ repo(),
+ repo({ full_name: "duplicate/name", owner: { login: "duplicate" }, name: "name", html_url: "https://github.com/duplicate/name" }),
]);
-
- expect(plugins.map((plugin) => plugin.fullName)).toEqual(["m/m", "a/a", "z/z"]);
+ expect(plugins).toHaveLength(1);
+ expect(plugins[0].fullName).toBe("ogulcancelik/herdr-plugin-example");
});
- test("drops unsafe urls, archived repositories, forks, disabled repositories, and private repositories", () => {
+ test("drops unsafe, unavailable, and default-branch-less repositories", () => {
const plugins = normalizeRepositories([
repo({ html_url: "https://example.com/ogulcancelik/herdr-plugin-example" }),
repo({ archived: true }),
@@ -119,62 +230,52 @@ describe("normalizeRepositories", () => {
repo({ disabled: true }),
repo({ private: true }),
repo({ visibility: "private" }),
+ repo({ default_branch: undefined }),
repo({ id: 5 }),
]);
-
expect(plugins.map((plugin) => plugin.id)).toEqual([5]);
});
+});
- test("uses safe defaults for missing nullable fields", () => {
- const [plugin] = normalizeRepositories([
- repo({
- id: undefined,
- description: undefined,
- stargazers_count: undefined,
- forks_count: undefined,
- open_issues_count: undefined,
- language: undefined,
- topics: undefined,
- created_at: "not a date",
- updated_at: undefined,
- pushed_at: undefined,
- }),
- ]);
+describe("parseManifestSummary", () => {
+ test("extracts metadata and accepts the UTF-8 BOM accepted by Herdr", () => {
+ expect(parseManifestSummary(`\uFEFF${manifest()}`)).toEqual({
+ id: "example.plugin",
+ name: "Example Plugin",
+ version: "0.2.0",
+ minHerdrVersion: "0.7.0",
+ description: "Example manifest",
+ platforms: ["linux", "macos"],
+ });
+ });
- expect(plugin.id).toBe(0);
- expect(plugin.description).toBeNull();
- expect(plugin.stars).toBe(0);
- expect(plugin.forks).toBe(0);
- expect(plugin.openIssues).toBe(0);
- expect(plugin.language).toBeNull();
- expect(plugin.topics).toEqual([]);
- expect(plugin.createdAt).toBeNull();
- expect(plugin.updatedAt).toBeNull();
- expect(plugin.pushedAt).toBeNull();
+ test("rejects malformed TOML and invalid required metadata", () => {
+ expect(parseManifestSummary("not = [valid")).toBeNull();
+ expect(parseManifestSummary(manifest().replace("example.plugin", "bad/plugin"))).toBeNull();
+ expect(parseManifestSummary(manifest().replace("0.7.0", "next"))).toBeNull();
+ expect(parseManifestSummary(manifest().replace('["linux", "macos"]', "[]"))).toBeNull();
+ expect(
+ parseManifestSummary(manifest().replace("Example Plugin", "n".repeat(121))),
+ ).toBeNull();
});
});
describe("refreshPlugins", () => {
- test("fetches pages and writes a sorted snapshot to R2 with cache metadata", async () => {
- const calls: string[] = [];
- const fetch = async (input: RequestInfo | URL): Promise<Response> => {
- const url = new URL(input.toString());
- calls.push(url.searchParams.get("page") ?? "");
- const page = url.searchParams.get("page");
- const item =
- page === "1"
- ? repo({ id: 1, full_name: "b/b", owner: { login: "b" }, name: "b", html_url: "https://github.com/b/b" })
- : repo({
- id: 2,
- full_name: "a/a",
- owner: { login: "a" },
- name: "a",
- html_url: "https://github.com/a/a",
- stargazers_count: 9,
- });
- return Response.json({ total_count: 2, items: [item] });
- };
+ test("publishes one backward-compatible repository card with multiple manifests", async () => {
const bucket = new MemoryR2();
+ const fullName = "ogulcancelik/herdr-plugin-example";
+ const fetch = repositoryFetch({
+ repositories: [repo()],
+ trees: {
+ [fullName]: [
+ { path: "herdr-plugin.toml", content: manifest() },
+ {
+ path: "plugins/second/herdr-plugin.toml",
+ content: manifest().replace("example.plugin", "example.second").replace("Example Plugin", "Second Plugin"),
+ },
+ ],
+ },
+ });
const result = await refreshPlugins(env(bucket), {
fetch,
@@ -183,142 +284,389 @@ describe("refreshPlugins", () => {
});
expect(result.ok).toBe(true);
- expect(calls).toEqual(["1", "2"]);
- const object = bucket.objects.get("plugins/index.json");
- expect(object?.options).toEqual({
+ const snapshotObject = bucket.objects.get("plugins/index.json");
+ const snapshot = JSON.parse(snapshotObject?.value ?? "");
+ expect(snapshotObject?.options).toEqual({
httpMetadata: {
contentType: "application/json; charset=utf-8",
cacheControl: "public, max-age=300, s-maxage=1800, stale-while-revalidate=3600",
},
});
- const snapshot = JSON.parse(object?.value ?? "");
- expect(snapshot.generatedAt).toBe("2026-06-20T12:00:00.000Z");
- expect(snapshot.source).toMatchObject({
- provider: "github",
- query: "topic:herdr-plugin is:public",
- totalCount: 2,
- collectedCount: 2,
- truncated: false,
+ expect(snapshot).toMatchObject({
+ schemaVersion: 1,
+ generatedAt: "2026-06-20T12:00:00.000Z",
+ pluginCount: 2,
+ repositoryCount: 1,
+ source: {
+ missingManifestCount: 0,
+ invalidManifestCount: 0,
+ },
});
- expect(snapshot.plugins.map((plugin: { fullName: string }) => plugin.fullName)).toEqual([
- "a/a",
- "b/b",
+ expect(snapshot.plugins[0]).toMatchObject({
+ id: 1,
+ fullName,
+ name: "herdr-plugin-example",
+ headCommit: HEAD_COMMIT,
+ manifests: [
+ { path: "herdr-plugin.toml", id: "example.plugin" },
+ { path: "plugins/second/herdr-plugin.toml", id: "example.second" },
+ ],
+ });
+ expect(snapshot.plugins[0]).not.toHaveProperty("defaultBranch");
+ expect(bucket.objects.has("plugins/scan-cache.json")).toBe(true);
+ expect(
+ [...bucket.objects.keys()].some((key) => key.includes("history")),
+ ).toBe(false);
+ });
+
+ test("reuses cached manifests without resolving or rescanning an unchanged repository", async () => {
+ const bucket = new MemoryR2();
+ const requests: string[] = [];
+ const fetch = repositoryFetch({
+ repositories: [repo()],
+ onRequest(kind) {
+ requests.push(kind);
+ },
+ });
+
+ expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true);
+ requests.length = 0;
+ expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true);
+
+ expect(requests).toEqual(["search"]);
+ });
+
+ test("rescans a repository when its pushed timestamp and head change", async () => {
+ const bucket = new MemoryR2();
+ const repository = repo();
+ const fullName = String(repository.full_name);
+ const commits = { [fullName]: HEAD_COMMIT };
+ const trees = { [fullName]: [{ path: "herdr-plugin.toml", content: manifest() }] };
+ const fetch = repositoryFetch({ repositories: [repository], commits, trees });
+
+ expect((await refreshPlugins(env(bucket), { fetch, logger: { error() {} } })).ok).toBe(true);
+ commits[fullName] = SECOND_COMMIT;
+ repository.pushed_at = "2026-06-04T00:00:00Z";
+ trees[fullName][0].content = manifest().replace("0.2.0", "0.3.0");
+
+ const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } });
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.snapshot.plugins[0].headCommit).toBe(SECOND_COMMIT);
+ expect(result.snapshot.plugins[0].manifests[0].version).toBe("0.3.0");
+ });
+
+ test("counts manifests separately from repository cards and omits empty cards", async () => {
+ const repositories = [
+ repo(),
+ repo({
+ id: 2,
+ full_name: "example/empty",
+ owner: { login: "example" },
+ name: "empty",
+ html_url: "https://github.com/example/empty",
+ }),
+ repo({
+ id: 3,
+ full_name: "example/invalid",
+ owner: { login: "example" },
+ name: "invalid",
+ html_url: "https://github.com/example/invalid",
+ }),
+ ];
+ const result = await refreshPlugins(env(), {
+ fetch: repositoryFetch({
+ repositories,
+ trees: {
+ "ogulcancelik/herdr-plugin-example": [
+ { path: "one/herdr-plugin.toml", content: manifest() },
+ { path: "two/herdr-plugin.toml", content: manifest().replace("example.plugin", "example.two") },
+ ],
+ "example/empty": [{ path: "README.md", content: "empty" }],
+ "example/invalid": [{ path: "herdr-plugin.toml", content: "id = [broken" }],
+ },
+ }),
+ logger: { error() {} },
+ });
+
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.snapshot.pluginCount).toBe(2);
+ expect(result.snapshot.repositoryCount).toBe(1);
+ expect(result.snapshot.source.missingManifestCount).toBe(1);
+ expect(result.snapshot.source.invalidManifestCount).toBe(1);
+ });
+
+ test("ignores manifest symlinks and indexes their regular target", async () => {
+ const fullName = "ogulcancelik/herdr-plugin-example";
+ const result = await refreshPlugins(env(), {
+ fetch: repositoryFetch({
+ repositories: [repo()],
+ trees: {
+ [fullName]: [
+ { path: "herdr-plugin.toml", content: "plugin/herdr-plugin.toml", mode: "120000" },
+ { path: "plugin/herdr-plugin.toml", content: manifest() },
+ ],
+ },
+ }),
+ logger: { error() {} },
+ });
+
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.snapshot.plugins[0].manifests.map((item) => item.path)).toEqual([
+ "plugin/herdr-plugin.toml",
]);
+ expect(result.snapshot.source.invalidManifestCount).toBe(0);
});
- test("excludes repositories listed in the KV blacklist", async () => {
- const fetch = async (): Promise<Response> =>
- Response.json({
- total_count: 2,
- items: [
- repo({
- id: 1,
- full_name: "example/not-a-plugin",
- owner: { login: "example" },
- name: "not-a-plugin",
- html_url: "https://github.com/example/not-a-plugin",
- }),
- repo({
- id: 2,
- full_name: "ogulcancelik/herdr-plugin-example",
- owner: { login: "ogulcancelik" },
- name: "herdr-plugin-example",
- html_url: "https://github.com/ogulcancelik/herdr-plugin-example",
- }),
- ],
- });
+ test("ignores test fixtures and deduplicates repeated plugin ids", async () => {
+ const fullName = "ogulcancelik/herdr-plugin-example";
+ const result = await refreshPlugins(env(), {
+ fetch: repositoryFetch({
+ repositories: [repo()],
+ trees: {
+ [fullName]: [
+ { path: "herdr-plugin.toml", content: manifest() },
+ { path: "platform/herdr-plugin.toml", content: manifest() },
+ {
+ path: "tests/fixtures/helper/herdr-plugin.toml",
+ content: manifest().replace("example.plugin", "example.fixture"),
+ },
+ ],
+ },
+ }),
+ logger: { error() {} },
+ });
+
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.snapshot.plugins[0].manifests.map((item) => item.path)).toEqual([
+ "herdr-plugin.toml",
+ ]);
+ expect(result.snapshot.source.duplicateManifestCount).toBe(1);
+ });
+
+ test("skips a truncated tree without blocking the marketplace", async () => {
+ const fullName = "ogulcancelik/herdr-plugin-example";
+ const result = await refreshPlugins(env(), {
+ fetch: repositoryFetch({
+ repositories: [repo()],
+ truncatedTrees: new Set([fullName]),
+ }),
+ logger: { error() {} },
+ });
+
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.snapshot.plugins).toEqual([]);
+ expect(result.snapshot.source.skippedRepositoryCount).toBe(1);
+ expect(result.snapshot.source.warnings?.[0]).toContain(fullName);
+ });
+
+ test("writes an empty snapshot when every repository is blacklisted", async () => {
const bucket = new MemoryR2();
+ const result = await refreshPlugins(
+ env(bucket, new MemoryKV(["repo:ogulcancelik/herdr-plugin-example"])),
+ { fetch: repositoryFetch({ repositories: [repo()] }), logger: { error() {} } },
+ );
- const result = await refreshPlugins(env(bucket, new MemoryKV(["repo:example/not-a-plugin"])), {
- fetch,
+ expect(result.ok).toBe(true);
+ if (!result.ok) return;
+ expect(result.snapshot.plugins).toEqual([]);
+ expect(result.snapshot.source.blacklistedCount).toBe(1);
+ });
+
+ test("pins tree and manifest reads to the resolved commit", async () => {
+ const requests: Array<[string, string]> = [];
+ const result = await refreshPlugins(env(), {
+ fetch: repositoryFetch({
+ repositories: [repo()],
+ onRequest(kind, detail) {
+ requests.push([kind, detail]);
+ },
+ }),
logger: { error() {} },
});
expect(result.ok).toBe(true);
- const snapshot = JSON.parse(bucket.objects.get("plugins/index.json")?.value ?? "");
- expect(snapshot.plugins.map((plugin: { fullName: string }) => plugin.fullName)).toEqual([
+ expect(requests.find(([kind]) => kind === "tree")?.[1]).toBe(
"ogulcancelik/herdr-plugin-example",
- ]);
+ );
+ const manifestQuery = requests.find(([kind]) => kind === "manifest")?.[1] ?? "";
+ expect(manifestQuery).toContain(`${HEAD_COMMIT}:herdr-plugin.toml`);
+ expect(manifestQuery).not.toContain("main:herdr-plugin.toml");
});
- test("writes an empty snapshot when every listable repository is blacklisted", async () => {
- const fetch = async (): Promise<Response> =>
- Response.json({
- total_count: 1,
- items: [
- repo({
- id: 1,
- full_name: "example/not-a-plugin",
- owner: { login: "example" },
- name: "not-a-plugin",
- html_url: "https://github.com/example/not-a-plugin",
- }),
- ],
- });
+ test("does not overwrite the public snapshot when scanning fails", async () => {
const bucket = new MemoryR2();
await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}');
+ const result = await refreshPlugins(env(bucket), {
+ fetch: repositoryFetch({
+ repositories: [repo()],
+ treeStatus: { "ogulcancelik/herdr-plugin-example": 429 },
+ }),
+ logger: { error() {} },
+ });
- const result = await refreshPlugins(env(bucket, new MemoryKV(["repo:example/not-a-plugin"])), {
- fetch,
+ expect(result.ok).toBe(false);
+ expect(bucket.objects.get("plugins/index.json")?.value).toBe(
+ '{"schemaVersion":1,"plugins":[{"id":1}]}',
+ );
+ });
+
+ test("discards malformed cache state and rebuilds it", async () => {
+ const bucket = new MemoryR2();
+ await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}');
+ await bucket.put("plugins/scan-cache.json", "broken");
+ const errors: string[] = [];
+
+ const result = await refreshPlugins(env(bucket), {
+ fetch: repositoryFetch({ repositories: [repo()] }),
+ logger: { error(message) { errors.push(String(message)); } },
+ });
+
+ expect(result.ok).toBe(true);
+ expect(errors[0]).toContain("discarding invalid plugin marketplace scan cache");
+ expect(JSON.parse(bucket.objects.get("plugins/scan-cache.json")?.value ?? "").entries).toHaveLength(1);
+ });
+
+ test("rejects an implausible empty search when a healthy cache exists", async () => {
+ const bucket = new MemoryR2();
+ const normalFetch = repositoryFetch({ repositories: [repo()] });
+ expect((await refreshPlugins(env(bucket), { fetch: normalFetch, logger: { error() {} } })).ok).toBe(true);
+ const previous = bucket.objects.get("plugins/index.json")?.value;
+
+ const result = await refreshPlugins(env(bucket), {
+ fetch: repositoryFetch({ repositories: [] }),
+ logger: { error() {} },
+ });
+
+ expect(result.ok).toBe(false);
+ expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous);
+ });
+
+ test("publishes an empty snapshot when the initial complete search is empty", async () => {
+ const bucket = new MemoryR2();
+ const result = await refreshPlugins(env(bucket), {
+ fetch: repositoryFetch({ repositories: [] }),
logger: { error() {} },
});
expect(result.ok).toBe(true);
- const snapshot = JSON.parse(bucket.objects.get("plugins/index.json")?.value ?? "");
- expect(snapshot.plugins).toEqual([]);
+ if (!result.ok) return;
+ expect(result.snapshot.plugins).toEqual([]);
+ expect(result.snapshot.pluginCount).toBe(0);
});
- test("marks snapshots truncated at the GitHub search cap", async () => {
- const fetch = async (): Promise<Response> => {
- const items = Array.from({ length: 100 }, (_, index) =>
- repo({
- id: index,
- full_name: `owner/repo-${index}`,
- owner: { login: "owner" },
- name: `repo-${index}`,
- html_url: `https://github.com/owner/repo-${index}`,
- }),
- );
- return Response.json({ total_count: 1200, items });
+ test("preserves the snapshot when head resolution omits a cached repository", async () => {
+ const bucket = new MemoryR2();
+ const repository = repo();
+ const baseFetch = repositoryFetch({ repositories: [repository] });
+ expect((await refreshPlugins(env(bucket), { fetch: baseFetch, logger: { error() {} } })).ok).toBe(true);
+ const previous = bucket.objects.get("plugins/index.json")?.value;
+
+ repository.pushed_at = "2026-06-04T00:00:00Z";
+ const missingHeadFetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
+ const request = String(init?.body ?? "");
+ if (new URL(input.toString()).pathname === "/graphql" && request.includes("PluginMarketplaceHeads")) {
+ return Response.json({
+ data: { repo0: null },
+ errors: [{ type: "NOT_FOUND", path: ["repo0"], message: "not found" }],
+ });
+ }
+ return baseFetch(input, init);
};
+ const result = await refreshPlugins(env(bucket), {
+ fetch: missingHeadFetch as typeof fetch,
+ logger: { error() {} },
+ });
- const result = await refreshPlugins(env(), {
- fetch,
- now: new Date("2026-06-20T12:00:00.000Z"),
+ expect(result.ok).toBe(false);
+ expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous);
+ });
+
+ test("does not cache a transiently missing manifest response", async () => {
+ const bucket = new MemoryR2();
+ const repository = repo();
+ const fullName = String(repository.full_name);
+ const commits = { [fullName]: HEAD_COMMIT };
+ const baseFetch = repositoryFetch({ repositories: [repository], commits });
+ expect((await refreshPlugins(env(bucket), { fetch: baseFetch, logger: { error() {} } })).ok).toBe(true);
+ const previous = bucket.objects.get("plugins/index.json")?.value;
+
+ repository.pushed_at = "2026-06-04T00:00:00Z";
+ commits[fullName] = SECOND_COMMIT;
+ const missingFetch = async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
+ const request = String(init?.body ?? "");
+ if (new URL(input.toString()).pathname === "/graphql" && request.includes("PluginMarketplaceManifests")) {
+ return Response.json({
+ data: { item0: { manifest: null } },
+ errors: [{ type: "NOT_FOUND", path: ["item0"], message: "not found" }],
+ });
+ }
+ return baseFetch(input, init);
+ };
+ const result = await refreshPlugins(env(bucket), {
+ fetch: missingFetch as typeof fetch,
logger: { error() {} },
});
+ expect(result.ok).toBe(false);
+ expect(bucket.objects.get("plugins/index.json")?.value).toBe(previous);
+ });
+
+ test("retries budget-skipped repositories when capacity becomes available", async () => {
+ const repositories = Array.from({ length: 51 }, (_, index) =>
+ repo({
+ id: index + 1,
+ full_name: `owner/repo-${index}`,
+ owner: { login: "owner" },
+ name: `repo-${index}`,
+ html_url: `https://github.com/owner/repo-${index}`,
+ }),
+ );
+ const trees = Object.fromEntries(
+ repositories.map((repository) => [
+ repository.full_name,
+ Array.from({ length: 100 }, (_, index) => ({
+ path: `plugins/${index}/herdr-plugin.toml`,
+ content: manifest(),
+ })),
+ ]),
+ );
+
+ const bucket = new MemoryR2();
+ const fetch = repositoryFetch({ repositories, trees });
+ const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } });
+
expect(result.ok).toBe(true);
if (!result.ok) return;
- expect(result.snapshot.source.collectedCount).toBe(1000);
- expect(result.snapshot.source.truncated).toBe(true);
- expect(result.snapshot.source.warnings?.[0]).toContain("1200");
+ expect(result.snapshot.source.skippedRepositoryCount).toBe(1);
+ expect(result.snapshot.source.warnings?.[0]).toContain("scan budget was exhausted");
+
+ repositories.shift();
+ const recovered = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } });
+ expect(recovered.ok).toBe(true);
+ if (!recovered.ok) return;
+ expect(recovered.snapshot.source.skippedRepositoryCount).toBe(0);
+ expect(recovered.snapshot.repositoryCount).toBe(50);
});
for (const { name, fetch } of [
{
name: "GitHub failure",
- fetch: async (): Promise<Response> => new Response("rate limited", { status: 429 }),
- },
- {
- name: "no listable repositories",
- fetch: async (): Promise<Response> => Response.json({ total_count: 0, items: [] }),
+ fetch: repositoryFetch({ repositories: [], searchStatus: 429 }),
},
{
name: "incomplete search results",
- fetch: async (): Promise<Response> =>
- Response.json({ total_count: 1, incomplete_results: true, items: [repo()] }),
+ fetch: repositoryFetch({ repositories: [repo()], incompleteResults: true }),
},
]) {
- test(`does not overwrite the R2 snapshot on ${name}`, async () => {
+ test(`preserves the current snapshot on ${name}`, async () => {
const bucket = new MemoryR2();
await bucket.put("plugins/index.json", '{"schemaVersion":1,"plugins":[{"id":1}]}');
-
- const result = await refreshPlugins(env(bucket), {
- fetch,
- logger: { error() {} },
- });
-
+ const result = await refreshPlugins(env(bucket), { fetch, logger: { error() {} } });
expect(result.ok).toBe(false);
expect(bucket.objects.get("plugins/index.json")?.value).toBe(
'{"schemaVersion":1,"plugins":[{"id":1}]}',
@@ -330,7 +678,6 @@ describe("refreshPlugins", () => {
describe("fetch handler", () => {
test("does not expose a public Worker API", async () => {
const response = await worker.fetch(new Request("https://herdr.dev/api/plugins"), env());
-
expect(response.status).toBe(404);
expect(response.headers.get("Cache-Control")).toBe("no-store");
});
diff --git a/workers/plugin-marketplace/src/index.ts b/workers/plugin-marketplace/src/index.ts
index e4523340..a504e1af 100644
--- a/workers/plugin-marketplace/src/index.ts
+++ b/workers/plugin-marketplace/src/index.ts
@@ -1,12 +1,32 @@
+import { parse } from "smol-toml";
+
const SNAPSHOT_KEY = "plugins/index.json";
+const SCAN_CACHE_KEY = "plugins/scan-cache.json";
const SNAPSHOT_CACHE_CONTROL = "public, max-age=300, s-maxage=1800, stale-while-revalidate=3600";
const GITHUB_QUERY = "topic:herdr-plugin is:public";
const GITHUB_API_VERSION = "2022-11-28";
-const GITHUB_SEARCH_URL = "https://api.github.com/search/repositories";
+const GITHUB_API_URL = "https://api.github.com";
+const GITHUB_SEARCH_URL = `${GITHUB_API_URL}/search/repositories`;
+const GITHUB_GRAPHQL_URL = `${GITHUB_API_URL}/graphql`;
const BLACKLIST_REPO_KEY_PREFIX = "repo:";
+const PLUGIN_MANIFEST_FILE = "herdr-plugin.toml";
const PER_PAGE = 100;
const MAX_REPOS = 1000;
+const GRAPHQL_BATCH_SIZE = 50;
+const NETWORK_CONCURRENCY = 5;
+const MANIFEST_MAX_BYTES = 32 * 1024;
+const MAX_TREE_ENTRIES = 20_000;
+const MAX_MANIFESTS_PER_REPOSITORY = 100;
+const MAX_TOTAL_MANIFESTS = 5000;
+const MAX_TOTAL_MANIFEST_BYTES = 16 * 1024 * 1024;
+const PLUGIN_NAME_MAX_CHARS = 120;
+const PLUGIN_VERSION_MAX_CHARS = 64;
+const PLUGIN_DESCRIPTION_MAX_CHARS = 500;
const REQUEST_TIMEOUT_MS = 10_000;
+const SCAN_CACHE_SCHEMA_VERSION = 1;
+
+const PLUGIN_PLATFORMS = new Set(["linux", "macos", "windows"]);
+const REGULAR_BLOB_MODES = new Set(["100644", "100755"]);
type R2Bucket = {
put(
@@ -19,6 +39,7 @@ type R2Bucket = {
};
},
): Promise<unknown>;
+ get(key: string): Promise<{ text(): Promise<string> } | null>;
};
type KVNamespace = {
@@ -50,7 +71,7 @@ type RefreshOptions = {
type GitHubRepository = Record<string, unknown>;
-export type PluginListing = {
+type RepositoryListing = {
id: number;
fullName: string;
owner: string;
@@ -65,22 +86,94 @@ export type PluginListing = {
createdAt: string | null;
updatedAt: string | null;
pushedAt: string | null;
+ defaultBranch: string;
+};
+
+export type PluginManifestListing = {
+ path: string;
+ id: string;
+ name: string;
+ version: string;
+ minHerdrVersion: string;
+ description: string | null;
+ platforms: string[] | null;
+};
+
+export type PluginListing = Omit<RepositoryListing, "defaultBranch"> & {
+ headCommit: string;
+ manifests: PluginManifestListing[];
};
export type PluginSnapshot = {
schemaVersion: 1;
generatedAt: string;
+ pluginCount: number;
+ repositoryCount: number;
source: {
provider: "github";
query: string;
totalCount: number;
collectedCount: number;
+ blacklistedCount: number;
+ missingManifestCount: number;
+ invalidManifestCount: number;
+ duplicateManifestCount: number;
+ skippedRepositoryCount: number;
+ unavailableRepositoryCount: number;
truncated: boolean;
warnings?: string[];
};
plugins: PluginListing[];
};
+type ScanCacheEntry = {
+ repositoryId: number;
+ fullName: string;
+ defaultBranch: string;
+ pushedAt: string | null;
+ headCommit: string;
+ manifestFileCount: number;
+ invalidManifestCount: number;
+ duplicateManifestCount: number;
+ warning: string | null;
+ retryable: boolean;
+ manifests: PluginManifestListing[];
+};
+
+type ScanCache = {
+ schemaVersion: 1;
+ entries: ScanCacheEntry[];
+};
+
+type ResolvedRepository = {
+ repository: RepositoryListing;
+ headCommit: string;
+ cached: ScanCacheEntry | null;
+};
+
+type TreeManifestCandidate = {
+ repository: RepositoryListing;
+ headCommit: string;
+ path: string;
+ size: number;
+};
+
+type PendingRepositoryScan = {
+ resolved: ResolvedRepository;
+ manifestFileCount: number;
+ invalidManifestCount: number;
+ warning: string | null;
+ retryable: boolean;
+ candidates: TreeManifestCandidate[];
+};
+
+type GitHubTreeEntry = {
+ path: string;
+ mode: string;
+ type: string;
+ size: number | null;
+};
+
export type RefreshResult =
| { ok: true; snapshot: PluginSnapshot }
| { ok: false; error: string };
@@ -107,39 +200,112 @@ export async function refreshPlugins(
}
const fetchFn = options.fetch ?? fetch;
- const result = await fetchGitHubRepositories(fetchFn, token);
- const normalizedPlugins = normalizeRepositories(result.repositories);
- if (normalizedPlugins.length === 0) {
- throw new Error("GitHub returned no listable plugin repositories");
+ const search = await fetchGitHubRepositories(fetchFn, token);
+ const repositories = normalizeRepositories(search.repositories);
+ const blockedRepositories = await readBlacklistedRepositories(env);
+ const candidates = repositories.filter(
+ (repository) => !blockedRepositories.has(repository.fullName.toLowerCase()),
+ );
+ const blacklistedCount = repositories.length - candidates.length;
+ const cache = await readScanCache(env.PLUGIN_MARKETPLACE_BUCKET, logger);
+ if (cache.entries.length > 0 && repositories.length * 2 < cache.entries.length) {
+ throw new Error(
+ `GitHub repository inventory collapsed from ${cache.entries.length} to ${repositories.length}; remove ${SCAN_CACHE_KEY} to accept an intentional reset`,
+ );
}
+ const cachedById = new Map(cache.entries.map((entry) => [entry.repositoryId, entry]));
- const blockedRepositories = await readBlacklistedRepositories(env);
- const plugins =
- blockedRepositories.size === 0
- ? normalizedPlugins
- : normalizedPlugins.filter(
- (plugin) => !blockedRepositories.has(plugin.fullName.toLowerCase()),
- );
+ const { resolved, unavailableRepositoryCount } = await resolveRepositoryHeads(
+ fetchFn,
+ token,
+ candidates,
+ cachedById,
+ );
+ const resolvedIds = new Set(resolved.map(({ repository }) => repository.id));
+ const missingCachedRepository = candidates.find(
+ (repository) => cachedById.has(repository.id) && !resolvedIds.has(repository.id),
+ );
+ if (
+ missingCachedRepository ||
+ (candidates.length > 0 && resolved.length * 2 < candidates.length)
+ ) {
+ throw new Error(
+ `GitHub head resolution omitted marketplace repositories${missingCachedRepository ? ` including ${missingCachedRepository.fullName}` : ""}`,
+ );
+ }
+ const changed = resolved.filter(
+ ({ headCommit, cached }) => !cached || cached.headCommit !== headCommit || cached.retryable,
+ );
+ const scannedById = await scanChangedRepositories(fetchFn, token, changed);
+
+ const nextEntries = resolved.map(({ repository, headCommit, cached }) => {
+ const scanned = scannedById.get(repository.id);
+ if (scanned) return scanned;
+ if (!cached || cached.headCommit !== headCommit) {
+ throw new Error(`missing marketplace scan result for ${repository.fullName}`);
+ }
+ return {
+ ...cached,
+ fullName: repository.fullName,
+ defaultBranch: repository.defaultBranch,
+ pushedAt: repository.pushedAt,
+ };
+ });
+ const entriesById = new Map(nextEntries.map((entry) => [entry.repositoryId, entry]));
+ const plugins = resolved
+ .map(({ repository }) => {
+ const entry = entriesById.get(repository.id);
+ if (!entry || entry.manifests.length === 0) return null;
+ const { defaultBranch: _defaultBranch, ...card } = repository;
+ return { ...card, headCommit: entry.headCommit, manifests: entry.manifests };
+ })
+ .filter((plugin): plugin is PluginListing => plugin !== null);
+ const pluginCount = plugins.reduce((total, plugin) => total + plugin.manifests.length, 0);
+ const warnings = nextEntries.flatMap((entry) => (entry.warning ? [entry.warning] : []));
+ if (search.truncated) {
+ warnings.unshift(
+ `GitHub returned ${search.totalCount} results; only the first ${search.repositories.length} were collected.`,
+ );
+ }
+ const generatedAt = (options.now ?? new Date()).toISOString();
const snapshot: PluginSnapshot = {
schemaVersion: 1,
- generatedAt: (options.now ?? new Date()).toISOString(),
+ generatedAt,
+ pluginCount,
+ repositoryCount: plugins.length,
source: {
provider: "github",
query: GITHUB_QUERY,
- totalCount: result.totalCount,
- collectedCount: result.repositories.length,
- truncated: result.truncated,
+ totalCount: search.totalCount,
+ collectedCount: search.repositories.length,
+ blacklistedCount,
+ missingManifestCount: nextEntries.filter(
+ (entry) => entry.manifestFileCount === 0 && !entry.warning,
+ ).length,
+ invalidManifestCount: nextEntries.reduce(
+ (total, entry) => total + entry.invalidManifestCount,
+ 0,
+ ),
+ duplicateManifestCount: nextEntries.reduce(
+ (total, entry) => total + entry.duplicateManifestCount,
+ 0,
+ ),
+ skippedRepositoryCount: nextEntries.filter((entry) => entry.warning).length,
+ unavailableRepositoryCount,
+ truncated: search.truncated,
+ ...(warnings.length > 0 ? { warnings } : {}),
},
plugins,
};
- if (result.truncated) {
- snapshot.source.warnings = [
- `GitHub returned ${result.totalCount} results; only the first ${result.repositories.length} were collected.`,
- ];
- }
-
+ const nextCache: ScanCache = { schemaVersion: SCAN_CACHE_SCHEMA_VERSION, entries: nextEntries };
+ await env.PLUGIN_MARKETPLACE_BUCKET.put(SCAN_CACHE_KEY, JSON.stringify(nextCache), {
+ httpMetadata: {
+ contentType: "application/json; charset=utf-8",
+ cacheControl: "no-store",
+ },
+ });
await env.PLUGIN_MARKETPLACE_BUCKET.put(SNAPSHOT_KEY, JSON.stringify(snapshot), {
httpMetadata: {
contentType: "application/json; charset=utf-8",
@@ -170,24 +336,10 @@ async function fetchGitHubRepositories(
url.searchParams.set("sort", "stars");
url.searchParams.set("order", "desc");
- const response = await fetchWithTimeout(
- fetchFn,
- url,
- {
- headers: {
- Accept: "application/vnd.github+json",
- Authorization: `Bearer ${token}`,
- "User-Agent": "herdr-plugin-marketplace",
- "X-GitHub-Api-Version": GITHUB_API_VERSION,
- },
- },
- timeoutMs,
- );
-
+ const response = await fetchWithTimeout(fetchFn, url, githubRequestInit(token), timeoutMs);
if (!response.ok) {
throw new Error(`GitHub search failed with status ${response.status}`);
}
-
const body = await response.json();
if (!isObject(body) || typeof body.total_count !== "number" || !Array.isArray(body.items)) {
throw new Error("GitHub search returned malformed JSON");
@@ -198,10 +350,7 @@ async function fetchGitHubRepositories(
totalCount = body.total_count;
repositories.push(...body.items.slice(0, MAX_REPOS - repositories.length));
-
- if (repositories.length >= totalCount || body.items.length === 0) {
- break;
- }
+ if (repositories.length >= totalCount || body.items.length === 0) break;
}
return {
@@ -211,51 +360,463 @@ async function fetchGitHubRepositories(
};
}
-async function fetchWithTimeout(
+async function resolveRepositoryHeads(
fetchFn: FetchLike,
- url: URL,
- init: RequestInit,
- timeoutMs: number,
-): Promise<Response> {
- const controller = new AbortController();
- const timeout = setTimeout(() => controller.abort(), timeoutMs);
+ token: string,
+ repositories: RepositoryListing[],
+ cachedById: Map<number, ScanCacheEntry>,
+): Promise<{ resolved: ResolvedRepository[]; unavailableRepositoryCount: number }> {
+ const resolvedById = new Map<number, ResolvedRepository>();
+ const needResolution: RepositoryListing[] = [];
+
+ for (const repository of repositories) {
+ const cached = cachedById.get(repository.id) ?? null;
+ if (
+ cached &&
+ cached.fullName.toLowerCase() === repository.fullName.toLowerCase() &&
+ cached.defaultBranch === repository.defaultBranch &&
+ !cached.retryable &&
+ cached.pushedAt !== null &&
+ cached.pushedAt === repository.pushedAt
+ ) {
+ resolvedById.set(repository.id, { repository, headCommit: cached.headCommit, cached });
+ } else {
+ needResolution.push(repository);
+ }
+ }
+
+ const batches = chunk(needResolution, GRAPHQL_BATCH_SIZE);
+ const results = await mapConcurrent(batches, NETWORK_CONCURRENCY, async (batch) => {
+ const data = await fetchGraphqlData(fetchFn, token, repositoryHeadQuery(batch));
+ return batch.flatMap((repository, index) => {
+ const raw = data[`repo${index}`];
+ if (!isObject(raw)) return [];
+ const branch = isObject(raw.defaultBranchRef) ? readString(raw.defaultBranchRef.name) : null;
+ const target = isObject(raw.defaultBranchRef?.target) ? raw.defaultBranchRef.target : null;
+ const headCommit = isObject(target) ? readString(target.oid) : null;
+ if (
+ branch !== repository.defaultBranch ||
+ !headCommit ||
+ !/^[0-9a-f]{40,64}$/i.test(headCommit)
+ ) {
+ return [];
+ }
+ return [{ repository, headCommit, cached: cachedById.get(repository.id) ?? null }];
+ });
+ });
+ for (const resolved of results.flat()) {
+ resolvedById.set(resolved.repository.id, resolved);
+ }
+
+ return {
+ resolved: repositories.flatMap((repository) => {
+ const value = resolvedById.get(repository.id);
+ return value ? [value] : [];
+ }),
+ unavailableRepositoryCount: repositories.length - resolvedById.size,
+ };
+}
+
+async function scanChangedRepositories(
+ fetchFn: FetchLike,
+ token: string,
+ repositories: ResolvedRepository[],
+): Promise<Map<number, ScanCacheEntry>> {
+ const pending = await mapConcurrent(repositories, NETWORK_CONCURRENCY, (resolved) =>
+ fetchRepositoryTree(fetchFn, token, resolved),
+ );
+ const candidates: TreeManifestCandidate[] = [];
+ let candidateBytes = 0;
+ for (const scan of pending) {
+ const scanBytes = scan.candidates.reduce((total, candidate) => total + candidate.size, 0);
+ if (
+ candidates.length + scan.candidates.length > MAX_TOTAL_MANIFESTS ||
+ candidateBytes + scanBytes > MAX_TOTAL_MANIFEST_BYTES
+ ) {
+ scan.warning = `${scan.resolved.repository.fullName} was skipped because the marketplace scan budget was exhausted.`;
+ scan.retryable = true;
+ scan.candidates = [];
+ continue;
+ }
+ candidates.push(...scan.candidates);
+ candidateBytes += scanBytes;
+ }
+ const contents = await fetchManifestContents(fetchFn, token, candidates);
+ const entries = new Map<number, ScanCacheEntry>();
+
+ for (const scan of pending) {
+ const manifests: PluginManifestListing[] = [];
+ let invalidManifestCount = scan.invalidManifestCount;
+ for (const candidate of scan.candidates) {
+ const content = contents.get(manifestCandidateKey(candidate));
+ const metadata = typeof content === "string" ? parseManifestSummary(content) : null;
+ if (!metadata) {
+ invalidManifestCount += 1;
+ continue;
+ }
+ manifests.push({ path: candidate.path, ...metadata });
+ }
+ const { manifests: uniqueManifests, duplicateCount } = deduplicateManifests(manifests);
+ const { repository, headCommit } = scan.resolved;
+ entries.set(repository.id, {
+ repositoryId: repository.id,
+ fullName: repository.fullName,
+ defaultBranch: repository.defaultBranch,
+ pushedAt: repository.pushedAt,
+ headCommit,
+ manifestFileCount: scan.manifestFileCount,
+ invalidManifestCount,
+ duplicateManifestCount: duplicateCount,
+ warning: scan.warning,
+ retryable: scan.retryable,
+ manifests: uniqueManifests,
+ });
+ }
+ return entries;
+}
+
+async function fetchRepositoryTree(
+ fetchFn: FetchLike,
+ token: string,
+ resolved: ResolvedRepository,
+): Promise<PendingRepositoryScan> {
+ const { repository, headCommit } = resolved;
+ const url = new URL(
+ `${GITHUB_API_URL}/repos/${encodeURIComponent(repository.owner)}/${encodeURIComponent(repository.name)}/git/trees/${headCommit}`,
+ );
+ url.searchParams.set("recursive", "1");
+ const response = await fetchWithTimeout(fetchFn, url, githubRequestInit(token), REQUEST_TIMEOUT_MS);
+ if (!response.ok) {
+ throw new Error(`GitHub tree fetch failed for ${repository.fullName} with status ${response.status}`);
+ }
+ const body = await response.json();
+ if (!isObject(body) || !Array.isArray(body.tree)) {
+ throw new Error(`GitHub tree fetch returned malformed JSON for ${repository.fullName}`);
+ }
+ if (body.truncated === true || body.tree.length > MAX_TREE_ENTRIES) {
+ return skippedRepositoryScan(resolved, `${repository.fullName} was skipped because its Git tree is too large.`);
+ }
+
+ const manifestEntries = body.tree
+ .map(normalizeTreeEntry)
+ .filter((entry): entry is GitHubTreeEntry => entry !== null)
+ .filter(
+ (entry) =>
+ isDiscoverableManifestPath(entry.path) &&
+ entry.type === "blob" &&
+ REGULAR_BLOB_MODES.has(entry.mode),
+ );
+ if (manifestEntries.length > MAX_MANIFESTS_PER_REPOSITORY) {
+ return skippedRepositoryScan(
+ resolved,
+ `${repository.fullName} was skipped because it contains more than ${MAX_MANIFESTS_PER_REPOSITORY} plugin manifests.`,
+ );
+ }
+
+ let invalidManifestCount = 0;
+ const candidates: TreeManifestCandidate[] = [];
+ for (const entry of manifestEntries) {
+ if (
+ entry.size === null ||
+ entry.size < 0 ||
+ entry.size > MANIFEST_MAX_BYTES
+ ) {
+ invalidManifestCount += 1;
+ continue;
+ }
+ candidates.push({ repository, headCommit, path: entry.path, size: entry.size });
+ }
+
+ return {
+ resolved,
+ manifestFileCount: manifestEntries.length,
+ invalidManifestCount,
+ warning: null,
+ retryable: false,
+ candidates,
+ };
+}
+
+function skippedRepositoryScan(
+ resolved: ResolvedRepository,
+ warning: string,
+): PendingRepositoryScan {
+ return {
+ resolved,
+ manifestFileCount: 0,
+ invalidManifestCount: 0,
+ warning,
+ retryable: false,
+ candidates: [],
+ };
+}
+
+async function fetchManifestContents(
+ fetchFn: FetchLike,
+ token: string,
+ candidates: TreeManifestCandidate[],
+): Promise<Map<string, string | null>> {
+ const batches = chunk(candidates, GRAPHQL_BATCH_SIZE);
+ const results = await mapConcurrent(batches, NETWORK_CONCURRENCY, async (batch) => {
+ const data = await fetchGraphqlData(fetchFn, token, manifestContentQuery(batch));
+ return batch.flatMap((candidate, index) => {
+ const raw = data[`item${index}`];
+ const manifest = isObject(raw) && isObject(raw.manifest) ? raw.manifest : null;
+ if (!manifest) {
+ throw new Error(
+ `GitHub manifest fetch omitted ${candidate.repository.fullName}:${candidate.path}`,
+ );
+ }
+ if (manifest.isBinary === true) {
+ return [[manifestCandidateKey(candidate), null] as const];
+ }
+ if (typeof manifest.text !== "string") {
+ throw new Error(
+ `GitHub manifest fetch returned malformed data for ${candidate.repository.fullName}:${candidate.path}`,
+ );
+ }
+ return [[manifestCandidateKey(candidate), manifest.text] as const];
+ });
+ });
+ return new Map(results.flat());
+}
+
+async function fetchGraphqlData(
+ fetchFn: FetchLike,
+ token: string,
+ query: string,
+): Promise<Record<string, unknown>> {
+ const response = await fetchWithTimeout(
+ fetchFn,
+ new URL(GITHUB_GRAPHQL_URL),
+ {
+ ...githubRequestInit(token),
+ method: "POST",
+ body: JSON.stringify({ query }),
+ },
+ REQUEST_TIMEOUT_MS,
+ );
+ if (!response.ok) {
+ throw new Error(`GitHub marketplace query failed with status ${response.status}`);
+ }
+ const body = await response.json();
+ if (!isObject(body) || !isObject(body.data)) {
+ throw new Error("GitHub marketplace query returned malformed JSON");
+ }
+ if (
+ Array.isArray(body.errors) &&
+ body.errors.some((error) => !isMissingAliasGraphqlError(error))
+ ) {
+ throw new Error("GitHub marketplace query returned GraphQL errors");
+ }
+ return body.data;
+}
+
+function isMissingAliasGraphqlError(error: unknown): boolean {
+ if (!isObject(error) || error.type !== "NOT_FOUND" || !Array.isArray(error.path)) {
+ return false;
+ }
+ const alias = error.path[0];
+ return typeof alias === "string" && /^(repo|item)\d+$/.test(alias);
+}
+
+function repositoryHeadQuery(repositories: RepositoryListing[]): string {
+ const selections = repositories.map((repository, index) => {
+ const owner = JSON.stringify(repository.owner);
+ const name = JSON.stringify(repository.name);
+ return `repo${index}: repository(owner: ${owner}, name: ${name}) {
+ defaultBranchRef { name target { ... on Commit { oid } } }
+ }`;
+ });
+ return `query PluginMarketplaceHeads {\n${selections.join("\n")}\n}`;
+}
+
+function manifestContentQuery(candidates: TreeManifestCandidate[]): string {
+ const selections = candidates.map((candidate, index) => {
+ const owner = JSON.stringify(candidate.repository.owner);
+ const name = JSON.stringify(candidate.repository.name);
+ const expression = JSON.stringify(`${candidate.headCommit}:${candidate.path}`);
+ return `item${index}: repository(owner: ${owner}, name: ${name}) {
+ manifest: object(expression: ${expression}) { ... on Blob { text isBinary } }
+ }`;
+ });
+ return `query PluginMarketplaceManifests {\n${selections.join("\n")}\n}`;
+}
+
+export function parseManifestSummary(manifestText: string): Omit<PluginManifestListing, "path"> | null {
try {
- return await fetchFn(url, { ...init, signal: controller.signal });
- } finally {
- clearTimeout(timeout);
+ const manifest = parse(manifestText.replace(/^\uFEFF/, ""));
+ if (!isObject(manifest)) return null;
+ const id = readTrimmedString(manifest.id);
+ const name = readTrimmedString(manifest.name);
+ const version = readTrimmedString(manifest.version);
+ const minHerdrVersion = readTrimmedString(manifest.min_herdr_version);
+ if (
+ !id ||
+ id.length > 120 ||
+ !/^[A-Za-z0-9:._-]+$/.test(id) ||
+ !name ||
+ name.length > PLUGIN_NAME_MAX_CHARS ||
+ !version ||
+ version.length > PLUGIN_VERSION_MAX_CHARS ||
+ !minHerdrVersion ||
+ minHerdrVersion.length > PLUGIN_VERSION_MAX_CHARS ||
+ !isHerdrVersion(minHerdrVersion)
+ ) {
+ return null;
+ }
+
+ let description: string | null = null;
+ if (manifest.description !== undefined) {
+ if (typeof manifest.description !== "string") return null;
+ description = manifest.description.trim() || null;
+ if (description && description.length > PLUGIN_DESCRIPTION_MAX_CHARS) return null;
+ }
+
+ let platforms: string[] | null = null;
+ if (manifest.platforms !== undefined) {
+ if (
+ !Array.isArray(manifest.platforms) ||
+ manifest.platforms.length === 0 ||
+ manifest.platforms.length > PLUGIN_PLATFORMS.size ||
+ !manifest.platforms.every(
+ (platform) => typeof platform === "string" && PLUGIN_PLATFORMS.has(platform),
+ )
+ ) {
+ return null;
+ }
+ platforms = [...manifest.platforms];
+ if (new Set(platforms).size !== platforms.length) return null;
+ }
+
+ return { id, name, version, minHerdrVersion, description, platforms };
+ } catch {
+ return null;
}
}
-export function normalizeRepositories(repositories: GitHubRepository[]): PluginListing[] {
- return repositories
- .map(normalizeRepository)
- .filter((plugin): plugin is PluginListing => plugin !== null)
- .sort(comparePlugins);
+async function readScanCache(
+ bucket: R2Bucket,
+ logger: Pick<Console, "error">,
+): Promise<ScanCache> {
+ const empty = (): ScanCache => ({ schemaVersion: SCAN_CACHE_SCHEMA_VERSION, entries: [] });
+ const object = await bucket.get(SCAN_CACHE_KEY);
+ if (!object) return empty();
+ try {
+ const value: unknown = JSON.parse(await object.text());
+ if (
+ !isObject(value) ||
+ value.schemaVersion !== SCAN_CACHE_SCHEMA_VERSION ||
+ !Array.isArray(value.entries)
+ ) {
+ throw new Error("unsupported cache shape");
+ }
+ const entries = value.entries.map(readScanCacheEntry);
+ if (entries.some((entry) => entry === null)) {
+ throw new Error("invalid cache entry");
+ }
+ return { schemaVersion: SCAN_CACHE_SCHEMA_VERSION, entries: entries as ScanCacheEntry[] };
+ } catch (error) {
+ const message = error instanceof Error ? error.message : "unknown cache error";
+ logger.error(`discarding invalid plugin marketplace scan cache: ${message}`);
+ return empty();
+ }
+}
+
+function readScanCacheEntry(value: unknown): ScanCacheEntry | null {
+ if (!isObject(value) || !Array.isArray(value.manifests)) return null;
+ const repositoryId = readInteger(value.repositoryId);
+ const fullName = readString(value.fullName);
+ const defaultBranch = readString(value.defaultBranch);
+ const pushedAt = value.pushedAt === null ? null : readIsoString(value.pushedAt);
+ const headCommit = readString(value.headCommit);
+ const manifestFileCount = readNonNegativeIntegerOrNull(value.manifestFileCount);
+ const invalidManifestCount = readNonNegativeIntegerOrNull(value.invalidManifestCount);
+ const duplicateManifestCount = readNonNegativeIntegerOrNull(value.duplicateManifestCount);
+ const warning = value.warning === null ? null : readString(value.warning);
+ const retryable = typeof value.retryable === "boolean" ? value.retryable : null;
+ const manifests = value.manifests.map(readCachedManifest);
+ if (
+ repositoryId === null ||
+ !fullName ||
+ !defaultBranch ||
+ (value.pushedAt !== null && pushedAt === null) ||
+ !headCommit ||
+ !/^[0-9a-f]{40,64}$/i.test(headCommit) ||
+ manifestFileCount === null ||
+ invalidManifestCount === null ||
+ duplicateManifestCount === null ||
+ retryable === null ||
+ (value.warning !== null && !warning) ||
+ manifests.some((manifest) => manifest === null)
+ ) {
+ return null;
+ }
+ return {
+ repositoryId,
+ fullName,
+ defaultBranch,
+ pushedAt,
+ headCommit,
+ manifestFileCount,
+ invalidManifestCount,
+ duplicateManifestCount,
+ warning,
+ retryable,
+ manifests: manifests as PluginManifestListing[],
+ };
+}
+
+function readCachedManifest(value: unknown): PluginManifestListing | null {
+ if (!isObject(value)) return null;
+ const path = readString(value.path);
+ const id = readString(value.id);
+ const name = readString(value.name);
+ const version = readString(value.version);
+ const minHerdrVersion = readString(value.minHerdrVersion);
+ const description = value.description === null ? null : readString(value.description);
+ const platforms = value.platforms === null ? null : readStringArrayOrNull(value.platforms);
+ if (
+ !path ||
+ !isManifestPath(path) ||
+ !id ||
+ !name ||
+ !version ||
+ !minHerdrVersion ||
+ (value.description !== null && !description) ||
+ (value.platforms !== null && platforms === null)
+ ) {
+ return null;
+ }
+ return { path, id, name, version, minHerdrVersion, description, platforms };
+}
+
+export function normalizeRepositories(repositories: GitHubRepository[]): RepositoryListing[] {
+ const byId = new Map<number, RepositoryListing>();
+ for (const repository of repositories.map(normalizeRepository)) {
+ if (repository && !byId.has(repository.id)) byId.set(repository.id, repository);
+ }
+ return [...byId.values()].sort(compareRepositories);
}
async function readBlacklistedRepositories(env: Env): Promise<Set<string>> {
const kv = env.PLUGIN_MARKETPLACE_BLACKLIST;
const blockedRepositories = new Set<string>();
- if (!kv) {
- return blockedRepositories;
- }
+ if (!kv) return blockedRepositories;
let cursor: string | undefined;
do {
const page = await kv.list({ prefix: BLACKLIST_REPO_KEY_PREFIX, cursor });
for (const key of page.keys) {
const repository = key.name.slice(BLACKLIST_REPO_KEY_PREFIX.length).trim().toLowerCase();
- if (repository.includes("/")) {
- blockedRepositories.add(repository);
- }
+ if (repository.includes("/")) blockedRepositories.add(repository);
}
cursor = page.cursor;
} while (cursor);
-
return blockedRepositories;
}
-function normalizeRepository(repo: GitHubRepository): PluginListing | null {
+function normalizeRepository(repo: GitHubRepository): RepositoryListing | null {
if (
readBoolean(repo.disabled) ||
readBoolean(repo.archived) ||
@@ -270,18 +831,16 @@ function normalizeRepository(repo: GitHubRepository): PluginListing | null {
const ownerObject = isObject(repo.owner) ? repo.owner : {};
const owner = firstString(readString(ownerObject.login), fullNameParts.owner);
const name = firstString(readString(repo.name), fullNameParts.name);
- if (!owner || !name) {
- return null;
- }
+ const defaultBranch = readString(repo.default_branch);
+ if (!owner || !name || !defaultBranch) return null;
const fullName = readString(repo.full_name) ?? `${owner}/${name}`;
const url = readString(repo.html_url);
- if (!url || !isValidGitHubRepoUrl(url, owner, name)) {
- return null;
- }
+ const id = readInteger(repo.id);
+ if (!url || !isValidGitHubRepoUrl(url, owner, name) || id === null || id <= 0) return null;
return {
- id: readInteger(repo.id) ?? 0,
+ id,
fullName,
owner,
name,
@@ -295,10 +854,67 @@ function normalizeRepository(repo: GitHubRepository): PluginListing | null {
createdAt: readIsoString(repo.created_at),
updatedAt: readIsoString(repo.updated_at),
pushedAt: readIsoString(repo.pushed_at),
+ defaultBranch,
};
}
-function comparePlugins(a: PluginListing, b: PluginListing): number {
+function normalizeTreeEntry(value: unknown): GitHubTreeEntry | null {
+ if (!isObject(value)) return null;
+ const path = readString(value.path);
+ const mode = readString(value.mode);
+ const type = readString(value.type);
+ if (!path || !mode || !type) return null;
+ return { path, mode, type, size: readNonNegativeIntegerOrNull(value.size) };
+}
+
+function isManifestPath(path: string): boolean {
+ return path === PLUGIN_MANIFEST_FILE || path.endsWith(`/${PLUGIN_MANIFEST_FILE}`);
+}
+
+function isDiscoverableManifestPath(path: string): boolean {
+ if (!isManifestPath(path)) return false;
+ const ignoredSegments = new Set([
+ "test",
+ "tests",
+ "fixture",
+ "fixtures",
+ "testdata",
+ "__fixtures__",
+ ]);
+ return !path
+ .split("/")
+ .slice(0, -1)
+ .some((segment) => ignoredSegments.has(segment.toLowerCase()));
+}
+
+function deduplicateManifests(manifests: PluginManifestListing[]): {
+ manifests: PluginManifestListing[];
+ duplicateCount: number;
+} {
+ const preferred = [...manifests].sort((a, b) => {
+ const depth = (path: string) => path.split("/").length;
+ return depth(a.path) - depth(b.path) || a.path.localeCompare(b.path);
+ });
+ const byId = new Map<string, PluginManifestListing>();
+ let duplicateCount = 0;
+ for (const manifest of preferred) {
+ if (byId.has(manifest.id)) {
+ duplicateCount += 1;
+ } else {
+ byId.set(manifest.id, manifest);
+ }
+ }
+ return {
+ manifests: [...byId.values()].sort((a, b) => a.path.localeCompare(b.path)),
+ duplicateCount,
+ };
+}
+
+function manifestCandidateKey(candidate: TreeManifestCandidate): string {
+ return `${candidate.repository.id}:${candidate.path}`;
+}
+
+function compareRepositories(a: RepositoryListing, b: RepositoryListing): number {
return (
b.stars - a.stars ||
dateMs(b.pushedAt) - dateMs(a.pushedAt) ||
@@ -306,11 +922,63 @@ function comparePlugins(a: PluginListing, b: PluginListing): number {
);
}
+function githubRequestInit(token: string): RequestInit {
+ return {
+ headers: {
+ Accept: "application/vnd.github+json",
+ Authorization: `Bearer ${token}`,
+ "User-Agent": "herdr-plugin-marketplace",
+ "X-GitHub-Api-Version": GITHUB_API_VERSION,
+ },
+ };
+}
+
+async function fetchWithTimeout(
+ fetchFn: FetchLike,
+ url: URL,
+ init: RequestInit,
+ timeoutMs: number,
+): Promise<Response> {
+ const controller = new AbortController();
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
+ try {
+ return await fetchFn(url, { ...init, signal: controller.signal });
+ } finally {
+ clearTimeout(timeout);
+ }
+}
+
+function chunk<T>(values: T[], size: number): T[][] {
+ const chunks: T[][] = [];
+ for (let index = 0; index < values.length; index += size) {
+ chunks.push(values.slice(index, index + size));
+ }
+ return chunks;
+}
+
+async function mapConcurrent<T, R>(
+ values: T[],
+ concurrency: number,
+ operation: (value: T) => Promise<R>,
+): Promise<R[]> {
+ const results = new Array<R>(values.length);
+ let nextIndex = 0;
+ const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => {
+ while (nextIndex < values.length) {
+ const index = nextIndex;
+ nextIndex += 1;
+ results[index] = await operation(values[index]);
+ }
+ });
+ await Promise.all(workers);
+ return results;
+}
+
function jsonResponse(body: unknown, status: number, cacheControl: string): Response {
return new Response(JSON.stringify(body), {
status,
headers: {
- "Content-Type": "application/json; charset=utf-8",
+ "Content-Type": "application/json",
"Cache-Control": cacheControl,
},
});
@@ -333,14 +1001,9 @@ function isValidGitHubRepoUrl(url: string, owner: string, name: string): boolean
}
function splitFullName(fullName: string | null): { owner: string | null; name: string | null } {
- if (!fullName) {
- return { owner: null, name: null };
- }
+ if (!fullName) return { owner: null, name: null };
const [owner, name, extra] = fullName.split("/");
- if (!owner || !name || extra) {
- return { owner: null, name: null };
- }
- return { owner, name };
+ return owner && name && !extra ? { owner, name } : { owner: null, name: null };
}
function firstString(...values: Array<string | null>): string | null {
@@ -351,6 +1014,10 @@ function readString(value: unknown): string | null {
return typeof value === "string" && value.length > 0 ? value : null;
}
+function readTrimmedString(value: unknown): string | null {
+ return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
+}
+
function readNullableString(value: unknown): string | null {
return typeof value === "string" ? value : null;
}
@@ -359,13 +1026,23 @@ function readStringArray(value: unknown): string[] {
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : [];
}
+function readStringArrayOrNull(value: unknown): string[] | null {
+ return Array.isArray(value) && value.every((item) => typeof item === "string")
+ ? value
+ : null;
+}
+
function readInteger(value: unknown): number | null {
return typeof value === "number" && Number.isInteger(value) ? value : null;
}
function readNonNegativeInteger(value: unknown): number {
+ return readNonNegativeIntegerOrNull(value) ?? 0;
+}
+
+function readNonNegativeIntegerOrNull(value: unknown): number | null {
const integer = readInteger(value);
- return integer !== null && integer >= 0 ? integer : 0;
+ return integer !== null && integer >= 0 ? integer : null;
}
function readBoolean(value: unknown): boolean {
@@ -373,10 +1050,16 @@ function readBoolean(value: unknown): boolean {
}
function readIsoString(value: unknown): string | null {
- if (typeof value !== "string") {
- return null;
- }
- return Number.isNaN(Date.parse(value)) ? null : value;
+ return typeof value === "string" && !Number.isNaN(Date.parse(value)) ? value : null;
+}
+
+function isHerdrVersion(value: string): boolean {
+ const normalized = value.startsWith("v") ? value.slice(1) : value;
+ const parts = normalized.split(".");
+ return (
+ parts.length === 3 &&
+ parts.every((part) => /^\d+$/.test(part) && Number(part) <= 0xffff_ffff)
+ );
}
function dateMs(value: string | null): number {
generated by cgit v1.2.3 (git 2.39.1) at 2026-11-06 22:56:04 +0000